| File: | src/gnu/usr.bin/clang/libLLVM/../../../llvm/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp |
| Warning: | line 672, column 24 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- InstCombineMulDivRem.cpp -------------------------------------------===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // This file implements the visit functions for mul, fmul, sdiv, udiv, fdiv, | ||||||||
| 10 | // srem, urem, frem. | ||||||||
| 11 | // | ||||||||
| 12 | //===----------------------------------------------------------------------===// | ||||||||
| 13 | |||||||||
| 14 | #include "InstCombineInternal.h" | ||||||||
| 15 | #include "llvm/ADT/APFloat.h" | ||||||||
| 16 | #include "llvm/ADT/APInt.h" | ||||||||
| 17 | #include "llvm/ADT/SmallVector.h" | ||||||||
| 18 | #include "llvm/Analysis/InstructionSimplify.h" | ||||||||
| 19 | #include "llvm/IR/BasicBlock.h" | ||||||||
| 20 | #include "llvm/IR/Constant.h" | ||||||||
| 21 | #include "llvm/IR/Constants.h" | ||||||||
| 22 | #include "llvm/IR/InstrTypes.h" | ||||||||
| 23 | #include "llvm/IR/Instruction.h" | ||||||||
| 24 | #include "llvm/IR/Instructions.h" | ||||||||
| 25 | #include "llvm/IR/IntrinsicInst.h" | ||||||||
| 26 | #include "llvm/IR/Intrinsics.h" | ||||||||
| 27 | #include "llvm/IR/Operator.h" | ||||||||
| 28 | #include "llvm/IR/PatternMatch.h" | ||||||||
| 29 | #include "llvm/IR/Type.h" | ||||||||
| 30 | #include "llvm/IR/Value.h" | ||||||||
| 31 | #include "llvm/Support/Casting.h" | ||||||||
| 32 | #include "llvm/Support/ErrorHandling.h" | ||||||||
| 33 | #include "llvm/Support/KnownBits.h" | ||||||||
| 34 | #include "llvm/Transforms/InstCombine/InstCombineWorklist.h" | ||||||||
| 35 | #include "llvm/Transforms/InstCombine/InstCombiner.h" | ||||||||
| 36 | #include "llvm/Transforms/Utils/BuildLibCalls.h" | ||||||||
| 37 | #include <cassert> | ||||||||
| 38 | #include <cstddef> | ||||||||
| 39 | #include <cstdint> | ||||||||
| 40 | #include <utility> | ||||||||
| 41 | |||||||||
| 42 | using namespace llvm; | ||||||||
| 43 | using namespace PatternMatch; | ||||||||
| 44 | |||||||||
| 45 | #define DEBUG_TYPE"instcombine" "instcombine" | ||||||||
| 46 | |||||||||
| 47 | /// The specific integer value is used in a context where it is known to be | ||||||||
| 48 | /// non-zero. If this allows us to simplify the computation, do so and return | ||||||||
| 49 | /// the new operand, otherwise return null. | ||||||||
| 50 | static Value *simplifyValueKnownNonZero(Value *V, InstCombinerImpl &IC, | ||||||||
| 51 | Instruction &CxtI) { | ||||||||
| 52 | // If V has multiple uses, then we would have to do more analysis to determine | ||||||||
| 53 | // if this is safe. For example, the use could be in dynamically unreached | ||||||||
| 54 | // code. | ||||||||
| 55 | if (!V->hasOneUse()) return nullptr; | ||||||||
| 56 | |||||||||
| 57 | bool MadeChange = false; | ||||||||
| 58 | |||||||||
| 59 | // ((1 << A) >>u B) --> (1 << (A-B)) | ||||||||
| 60 | // Because V cannot be zero, we know that B is less than A. | ||||||||
| 61 | Value *A = nullptr, *B = nullptr, *One = nullptr; | ||||||||
| 62 | if (match(V, m_LShr(m_OneUse(m_Shl(m_Value(One), m_Value(A))), m_Value(B))) && | ||||||||
| 63 | match(One, m_One())) { | ||||||||
| 64 | A = IC.Builder.CreateSub(A, B); | ||||||||
| 65 | return IC.Builder.CreateShl(One, A); | ||||||||
| 66 | } | ||||||||
| 67 | |||||||||
| 68 | // (PowerOfTwo >>u B) --> isExact since shifting out the result would make it | ||||||||
| 69 | // inexact. Similarly for <<. | ||||||||
| 70 | BinaryOperator *I = dyn_cast<BinaryOperator>(V); | ||||||||
| 71 | if (I && I->isLogicalShift() && | ||||||||
| 72 | IC.isKnownToBeAPowerOfTwo(I->getOperand(0), false, 0, &CxtI)) { | ||||||||
| 73 | // We know that this is an exact/nuw shift and that the input is a | ||||||||
| 74 | // non-zero context as well. | ||||||||
| 75 | if (Value *V2 = simplifyValueKnownNonZero(I->getOperand(0), IC, CxtI)) { | ||||||||
| 76 | IC.replaceOperand(*I, 0, V2); | ||||||||
| 77 | MadeChange = true; | ||||||||
| 78 | } | ||||||||
| 79 | |||||||||
| 80 | if (I->getOpcode() == Instruction::LShr && !I->isExact()) { | ||||||||
| 81 | I->setIsExact(); | ||||||||
| 82 | MadeChange = true; | ||||||||
| 83 | } | ||||||||
| 84 | |||||||||
| 85 | if (I->getOpcode() == Instruction::Shl && !I->hasNoUnsignedWrap()) { | ||||||||
| 86 | I->setHasNoUnsignedWrap(); | ||||||||
| 87 | MadeChange = true; | ||||||||
| 88 | } | ||||||||
| 89 | } | ||||||||
| 90 | |||||||||
| 91 | // TODO: Lots more we could do here: | ||||||||
| 92 | // If V is a phi node, we can call this on each of its operands. | ||||||||
| 93 | // "select cond, X, 0" can simplify to "X". | ||||||||
| 94 | |||||||||
| 95 | return MadeChange ? V : nullptr; | ||||||||
| 96 | } | ||||||||
| 97 | |||||||||
| 98 | // TODO: This is a specific form of a much more general pattern. | ||||||||
| 99 | // We could detect a select with any binop identity constant, or we | ||||||||
| 100 | // could use SimplifyBinOp to see if either arm of the select reduces. | ||||||||
| 101 | // But that needs to be done carefully and/or while removing potential | ||||||||
| 102 | // reverse canonicalizations as in InstCombiner::foldSelectIntoOp(). | ||||||||
| 103 | static Value *foldMulSelectToNegate(BinaryOperator &I, | ||||||||
| 104 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 105 | Value *Cond, *OtherOp; | ||||||||
| 106 | |||||||||
| 107 | // mul (select Cond, 1, -1), OtherOp --> select Cond, OtherOp, -OtherOp | ||||||||
| 108 | // mul OtherOp, (select Cond, 1, -1) --> select Cond, OtherOp, -OtherOp | ||||||||
| 109 | if (match(&I, m_c_Mul(m_OneUse(m_Select(m_Value(Cond), m_One(), m_AllOnes())), | ||||||||
| 110 | m_Value(OtherOp)))) | ||||||||
| 111 | return Builder.CreateSelect(Cond, OtherOp, Builder.CreateNeg(OtherOp)); | ||||||||
| 112 | |||||||||
| 113 | // mul (select Cond, -1, 1), OtherOp --> select Cond, -OtherOp, OtherOp | ||||||||
| 114 | // mul OtherOp, (select Cond, -1, 1) --> select Cond, -OtherOp, OtherOp | ||||||||
| 115 | if (match(&I, m_c_Mul(m_OneUse(m_Select(m_Value(Cond), m_AllOnes(), m_One())), | ||||||||
| 116 | m_Value(OtherOp)))) | ||||||||
| 117 | return Builder.CreateSelect(Cond, Builder.CreateNeg(OtherOp), OtherOp); | ||||||||
| 118 | |||||||||
| 119 | // fmul (select Cond, 1.0, -1.0), OtherOp --> select Cond, OtherOp, -OtherOp | ||||||||
| 120 | // fmul OtherOp, (select Cond, 1.0, -1.0) --> select Cond, OtherOp, -OtherOp | ||||||||
| 121 | if (match(&I, m_c_FMul(m_OneUse(m_Select(m_Value(Cond), m_SpecificFP(1.0), | ||||||||
| 122 | m_SpecificFP(-1.0))), | ||||||||
| 123 | m_Value(OtherOp)))) { | ||||||||
| 124 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | ||||||||
| 125 | Builder.setFastMathFlags(I.getFastMathFlags()); | ||||||||
| 126 | return Builder.CreateSelect(Cond, OtherOp, Builder.CreateFNeg(OtherOp)); | ||||||||
| 127 | } | ||||||||
| 128 | |||||||||
| 129 | // fmul (select Cond, -1.0, 1.0), OtherOp --> select Cond, -OtherOp, OtherOp | ||||||||
| 130 | // fmul OtherOp, (select Cond, -1.0, 1.0) --> select Cond, -OtherOp, OtherOp | ||||||||
| 131 | if (match(&I, m_c_FMul(m_OneUse(m_Select(m_Value(Cond), m_SpecificFP(-1.0), | ||||||||
| 132 | m_SpecificFP(1.0))), | ||||||||
| 133 | m_Value(OtherOp)))) { | ||||||||
| 134 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | ||||||||
| 135 | Builder.setFastMathFlags(I.getFastMathFlags()); | ||||||||
| 136 | return Builder.CreateSelect(Cond, Builder.CreateFNeg(OtherOp), OtherOp); | ||||||||
| 137 | } | ||||||||
| 138 | |||||||||
| 139 | return nullptr; | ||||||||
| 140 | } | ||||||||
| 141 | |||||||||
| 142 | Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) { | ||||||||
| 143 | if (Value *V = SimplifyMulInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 144 | SQ.getWithInstruction(&I))) | ||||||||
| 145 | return replaceInstUsesWith(I, V); | ||||||||
| 146 | |||||||||
| 147 | if (SimplifyAssociativeOrCommutative(I)) | ||||||||
| 148 | return &I; | ||||||||
| 149 | |||||||||
| 150 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 151 | return X; | ||||||||
| 152 | |||||||||
| 153 | if (Value *V = SimplifyUsingDistributiveLaws(I)) | ||||||||
| 154 | return replaceInstUsesWith(I, V); | ||||||||
| 155 | |||||||||
| 156 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 157 | unsigned BitWidth = I.getType()->getScalarSizeInBits(); | ||||||||
| 158 | |||||||||
| 159 | // X * -1 == 0 - X | ||||||||
| 160 | if (match(Op1, m_AllOnes())) { | ||||||||
| 161 | BinaryOperator *BO = BinaryOperator::CreateNeg(Op0, I.getName()); | ||||||||
| 162 | if (I.hasNoSignedWrap()) | ||||||||
| 163 | BO->setHasNoSignedWrap(); | ||||||||
| 164 | return BO; | ||||||||
| 165 | } | ||||||||
| 166 | |||||||||
| 167 | // Also allow combining multiply instructions on vectors. | ||||||||
| 168 | { | ||||||||
| 169 | Value *NewOp; | ||||||||
| 170 | Constant *C1, *C2; | ||||||||
| 171 | const APInt *IVal; | ||||||||
| 172 | if (match(&I, m_Mul(m_Shl(m_Value(NewOp), m_Constant(C2)), | ||||||||
| 173 | m_Constant(C1))) && | ||||||||
| 174 | match(C1, m_APInt(IVal))) { | ||||||||
| 175 | // ((X << C2)*C1) == (X * (C1 << C2)) | ||||||||
| 176 | Constant *Shl = ConstantExpr::getShl(C1, C2); | ||||||||
| 177 | BinaryOperator *Mul = cast<BinaryOperator>(I.getOperand(0)); | ||||||||
| 178 | BinaryOperator *BO = BinaryOperator::CreateMul(NewOp, Shl); | ||||||||
| 179 | if (I.hasNoUnsignedWrap() && Mul->hasNoUnsignedWrap()) | ||||||||
| 180 | BO->setHasNoUnsignedWrap(); | ||||||||
| 181 | if (I.hasNoSignedWrap() && Mul->hasNoSignedWrap() && | ||||||||
| 182 | Shl->isNotMinSignedValue()) | ||||||||
| 183 | BO->setHasNoSignedWrap(); | ||||||||
| 184 | return BO; | ||||||||
| 185 | } | ||||||||
| 186 | |||||||||
| 187 | if (match(&I, m_Mul(m_Value(NewOp), m_Constant(C1)))) { | ||||||||
| 188 | // Replace X*(2^C) with X << C, where C is either a scalar or a vector. | ||||||||
| 189 | if (Constant *NewCst = ConstantExpr::getExactLogBase2(C1)) { | ||||||||
| 190 | BinaryOperator *Shl = BinaryOperator::CreateShl(NewOp, NewCst); | ||||||||
| 191 | |||||||||
| 192 | if (I.hasNoUnsignedWrap()) | ||||||||
| 193 | Shl->setHasNoUnsignedWrap(); | ||||||||
| 194 | if (I.hasNoSignedWrap()) { | ||||||||
| 195 | const APInt *V; | ||||||||
| 196 | if (match(NewCst, m_APInt(V)) && *V != V->getBitWidth() - 1) | ||||||||
| 197 | Shl->setHasNoSignedWrap(); | ||||||||
| 198 | } | ||||||||
| 199 | |||||||||
| 200 | return Shl; | ||||||||
| 201 | } | ||||||||
| 202 | } | ||||||||
| 203 | } | ||||||||
| 204 | |||||||||
| 205 | if (Op0->hasOneUse() && match(Op1, m_NegatedPower2())) { | ||||||||
| 206 | // Interpret X * (-1<<C) as (-X) * (1<<C) and try to sink the negation. | ||||||||
| 207 | // The "* (1<<C)" thus becomes a potential shifting opportunity. | ||||||||
| 208 | if (Value *NegOp0 = Negator::Negate(/*IsNegation*/ true, Op0, *this)) | ||||||||
| 209 | return BinaryOperator::CreateMul( | ||||||||
| 210 | NegOp0, ConstantExpr::getNeg(cast<Constant>(Op1)), I.getName()); | ||||||||
| 211 | } | ||||||||
| 212 | |||||||||
| 213 | if (Instruction *FoldedMul = foldBinOpIntoSelectOrPhi(I)) | ||||||||
| 214 | return FoldedMul; | ||||||||
| 215 | |||||||||
| 216 | if (Value *FoldedMul = foldMulSelectToNegate(I, Builder)) | ||||||||
| 217 | return replaceInstUsesWith(I, FoldedMul); | ||||||||
| 218 | |||||||||
| 219 | // Simplify mul instructions with a constant RHS. | ||||||||
| 220 | if (isa<Constant>(Op1)) { | ||||||||
| 221 | // Canonicalize (X+C1)*CI -> X*CI+C1*CI. | ||||||||
| 222 | Value *X; | ||||||||
| 223 | Constant *C1; | ||||||||
| 224 | if (match(Op0, m_OneUse(m_Add(m_Value(X), m_Constant(C1))))) { | ||||||||
| 225 | Value *Mul = Builder.CreateMul(C1, Op1); | ||||||||
| 226 | // Only go forward with the transform if C1*CI simplifies to a tidier | ||||||||
| 227 | // constant. | ||||||||
| 228 | if (!match(Mul, m_Mul(m_Value(), m_Value()))) | ||||||||
| 229 | return BinaryOperator::CreateAdd(Builder.CreateMul(X, Op1), Mul); | ||||||||
| 230 | } | ||||||||
| 231 | } | ||||||||
| 232 | |||||||||
| 233 | // abs(X) * abs(X) -> X * X | ||||||||
| 234 | // nabs(X) * nabs(X) -> X * X | ||||||||
| 235 | if (Op0 == Op1) { | ||||||||
| 236 | Value *X, *Y; | ||||||||
| 237 | SelectPatternFlavor SPF = matchSelectPattern(Op0, X, Y).Flavor; | ||||||||
| 238 | if (SPF == SPF_ABS || SPF == SPF_NABS) | ||||||||
| 239 | return BinaryOperator::CreateMul(X, X); | ||||||||
| 240 | |||||||||
| 241 | if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X)))) | ||||||||
| 242 | return BinaryOperator::CreateMul(X, X); | ||||||||
| 243 | } | ||||||||
| 244 | |||||||||
| 245 | // -X * C --> X * -C | ||||||||
| 246 | Value *X, *Y; | ||||||||
| 247 | Constant *Op1C; | ||||||||
| 248 | if (match(Op0, m_Neg(m_Value(X))) && match(Op1, m_Constant(Op1C))) | ||||||||
| 249 | return BinaryOperator::CreateMul(X, ConstantExpr::getNeg(Op1C)); | ||||||||
| 250 | |||||||||
| 251 | // -X * -Y --> X * Y | ||||||||
| 252 | if (match(Op0, m_Neg(m_Value(X))) && match(Op1, m_Neg(m_Value(Y)))) { | ||||||||
| 253 | auto *NewMul = BinaryOperator::CreateMul(X, Y); | ||||||||
| 254 | if (I.hasNoSignedWrap() && | ||||||||
| 255 | cast<OverflowingBinaryOperator>(Op0)->hasNoSignedWrap() && | ||||||||
| 256 | cast<OverflowingBinaryOperator>(Op1)->hasNoSignedWrap()) | ||||||||
| 257 | NewMul->setHasNoSignedWrap(); | ||||||||
| 258 | return NewMul; | ||||||||
| 259 | } | ||||||||
| 260 | |||||||||
| 261 | // -X * Y --> -(X * Y) | ||||||||
| 262 | // X * -Y --> -(X * Y) | ||||||||
| 263 | if (match(&I, m_c_Mul(m_OneUse(m_Neg(m_Value(X))), m_Value(Y)))) | ||||||||
| 264 | return BinaryOperator::CreateNeg(Builder.CreateMul(X, Y)); | ||||||||
| 265 | |||||||||
| 266 | // (X / Y) * Y = X - (X % Y) | ||||||||
| 267 | // (X / Y) * -Y = (X % Y) - X | ||||||||
| 268 | { | ||||||||
| 269 | Value *Y = Op1; | ||||||||
| 270 | BinaryOperator *Div = dyn_cast<BinaryOperator>(Op0); | ||||||||
| 271 | if (!Div || (Div->getOpcode() != Instruction::UDiv && | ||||||||
| 272 | Div->getOpcode() != Instruction::SDiv)) { | ||||||||
| 273 | Y = Op0; | ||||||||
| 274 | Div = dyn_cast<BinaryOperator>(Op1); | ||||||||
| 275 | } | ||||||||
| 276 | Value *Neg = dyn_castNegVal(Y); | ||||||||
| 277 | if (Div && Div->hasOneUse() && | ||||||||
| 278 | (Div->getOperand(1) == Y || Div->getOperand(1) == Neg) && | ||||||||
| 279 | (Div->getOpcode() == Instruction::UDiv || | ||||||||
| 280 | Div->getOpcode() == Instruction::SDiv)) { | ||||||||
| 281 | Value *X = Div->getOperand(0), *DivOp1 = Div->getOperand(1); | ||||||||
| 282 | |||||||||
| 283 | // If the division is exact, X % Y is zero, so we end up with X or -X. | ||||||||
| 284 | if (Div->isExact()) { | ||||||||
| 285 | if (DivOp1 == Y) | ||||||||
| 286 | return replaceInstUsesWith(I, X); | ||||||||
| 287 | return BinaryOperator::CreateNeg(X); | ||||||||
| 288 | } | ||||||||
| 289 | |||||||||
| 290 | auto RemOpc = Div->getOpcode() == Instruction::UDiv ? Instruction::URem | ||||||||
| 291 | : Instruction::SRem; | ||||||||
| 292 | Value *Rem = Builder.CreateBinOp(RemOpc, X, DivOp1); | ||||||||
| 293 | if (DivOp1 == Y) | ||||||||
| 294 | return BinaryOperator::CreateSub(X, Rem); | ||||||||
| 295 | return BinaryOperator::CreateSub(Rem, X); | ||||||||
| 296 | } | ||||||||
| 297 | } | ||||||||
| 298 | |||||||||
| 299 | /// i1 mul -> i1 and. | ||||||||
| 300 | if (I.getType()->isIntOrIntVectorTy(1)) | ||||||||
| 301 | return BinaryOperator::CreateAnd(Op0, Op1); | ||||||||
| 302 | |||||||||
| 303 | // X*(1 << Y) --> X << Y | ||||||||
| 304 | // (1 << Y)*X --> X << Y | ||||||||
| 305 | { | ||||||||
| 306 | Value *Y; | ||||||||
| 307 | BinaryOperator *BO = nullptr; | ||||||||
| 308 | bool ShlNSW = false; | ||||||||
| 309 | if (match(Op0, m_Shl(m_One(), m_Value(Y)))) { | ||||||||
| 310 | BO = BinaryOperator::CreateShl(Op1, Y); | ||||||||
| 311 | ShlNSW = cast<ShlOperator>(Op0)->hasNoSignedWrap(); | ||||||||
| 312 | } else if (match(Op1, m_Shl(m_One(), m_Value(Y)))) { | ||||||||
| 313 | BO = BinaryOperator::CreateShl(Op0, Y); | ||||||||
| 314 | ShlNSW = cast<ShlOperator>(Op1)->hasNoSignedWrap(); | ||||||||
| 315 | } | ||||||||
| 316 | if (BO) { | ||||||||
| 317 | if (I.hasNoUnsignedWrap()) | ||||||||
| 318 | BO->setHasNoUnsignedWrap(); | ||||||||
| 319 | if (I.hasNoSignedWrap() && ShlNSW) | ||||||||
| 320 | BO->setHasNoSignedWrap(); | ||||||||
| 321 | return BO; | ||||||||
| 322 | } | ||||||||
| 323 | } | ||||||||
| 324 | |||||||||
| 325 | // (zext bool X) * (zext bool Y) --> zext (and X, Y) | ||||||||
| 326 | // (sext bool X) * (sext bool Y) --> zext (and X, Y) | ||||||||
| 327 | // Note: -1 * -1 == 1 * 1 == 1 (if the extends match, the result is the same) | ||||||||
| 328 | if (((match(Op0, m_ZExt(m_Value(X))) && match(Op1, m_ZExt(m_Value(Y)))) || | ||||||||
| 329 | (match(Op0, m_SExt(m_Value(X))) && match(Op1, m_SExt(m_Value(Y))))) && | ||||||||
| 330 | X->getType()->isIntOrIntVectorTy(1) && X->getType() == Y->getType() && | ||||||||
| 331 | (Op0->hasOneUse() || Op1->hasOneUse() || X == Y)) { | ||||||||
| 332 | Value *And = Builder.CreateAnd(X, Y, "mulbool"); | ||||||||
| 333 | return CastInst::Create(Instruction::ZExt, And, I.getType()); | ||||||||
| 334 | } | ||||||||
| 335 | // (sext bool X) * (zext bool Y) --> sext (and X, Y) | ||||||||
| 336 | // (zext bool X) * (sext bool Y) --> sext (and X, Y) | ||||||||
| 337 | // Note: -1 * 1 == 1 * -1 == -1 | ||||||||
| 338 | if (((match(Op0, m_SExt(m_Value(X))) && match(Op1, m_ZExt(m_Value(Y)))) || | ||||||||
| 339 | (match(Op0, m_ZExt(m_Value(X))) && match(Op1, m_SExt(m_Value(Y))))) && | ||||||||
| 340 | X->getType()->isIntOrIntVectorTy(1) && X->getType() == Y->getType() && | ||||||||
| 341 | (Op0->hasOneUse() || Op1->hasOneUse())) { | ||||||||
| 342 | Value *And = Builder.CreateAnd(X, Y, "mulbool"); | ||||||||
| 343 | return CastInst::Create(Instruction::SExt, And, I.getType()); | ||||||||
| 344 | } | ||||||||
| 345 | |||||||||
| 346 | // (bool X) * Y --> X ? Y : 0 | ||||||||
| 347 | // Y * (bool X) --> X ? Y : 0 | ||||||||
| 348 | if (match(Op0, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 349 | return SelectInst::Create(X, Op1, ConstantInt::get(I.getType(), 0)); | ||||||||
| 350 | if (match(Op1, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 351 | return SelectInst::Create(X, Op0, ConstantInt::get(I.getType(), 0)); | ||||||||
| 352 | |||||||||
| 353 | // (lshr X, 31) * Y --> (ashr X, 31) & Y | ||||||||
| 354 | // Y * (lshr X, 31) --> (ashr X, 31) & Y | ||||||||
| 355 | // TODO: We are not checking one-use because the elimination of the multiply | ||||||||
| 356 | // is better for analysis? | ||||||||
| 357 | // TODO: Should we canonicalize to '(X < 0) ? Y : 0' instead? That would be | ||||||||
| 358 | // more similar to what we're doing above. | ||||||||
| 359 | const APInt *C; | ||||||||
| 360 | if (match(Op0, m_LShr(m_Value(X), m_APInt(C))) && *C == C->getBitWidth() - 1) | ||||||||
| 361 | return BinaryOperator::CreateAnd(Builder.CreateAShr(X, *C), Op1); | ||||||||
| 362 | if (match(Op1, m_LShr(m_Value(X), m_APInt(C))) && *C == C->getBitWidth() - 1) | ||||||||
| 363 | return BinaryOperator::CreateAnd(Builder.CreateAShr(X, *C), Op0); | ||||||||
| 364 | |||||||||
| 365 | // ((ashr X, 31) | 1) * X --> abs(X) | ||||||||
| 366 | // X * ((ashr X, 31) | 1) --> abs(X) | ||||||||
| 367 | if (match(&I, m_c_BinOp(m_Or(m_AShr(m_Value(X), | ||||||||
| 368 | m_SpecificIntAllowUndef(BitWidth - 1)), | ||||||||
| 369 | m_One()), | ||||||||
| 370 | m_Deferred(X)))) { | ||||||||
| 371 | Value *Abs = Builder.CreateBinaryIntrinsic( | ||||||||
| 372 | Intrinsic::abs, X, | ||||||||
| 373 | ConstantInt::getBool(I.getContext(), I.hasNoSignedWrap())); | ||||||||
| 374 | Abs->takeName(&I); | ||||||||
| 375 | return replaceInstUsesWith(I, Abs); | ||||||||
| 376 | } | ||||||||
| 377 | |||||||||
| 378 | if (Instruction *Ext = narrowMathIfNoOverflow(I)) | ||||||||
| 379 | return Ext; | ||||||||
| 380 | |||||||||
| 381 | bool Changed = false; | ||||||||
| 382 | if (!I.hasNoSignedWrap() && willNotOverflowSignedMul(Op0, Op1, I)) { | ||||||||
| 383 | Changed = true; | ||||||||
| 384 | I.setHasNoSignedWrap(true); | ||||||||
| 385 | } | ||||||||
| 386 | |||||||||
| 387 | if (!I.hasNoUnsignedWrap() && willNotOverflowUnsignedMul(Op0, Op1, I)) { | ||||||||
| 388 | Changed = true; | ||||||||
| 389 | I.setHasNoUnsignedWrap(true); | ||||||||
| 390 | } | ||||||||
| 391 | |||||||||
| 392 | return Changed ? &I : nullptr; | ||||||||
| 393 | } | ||||||||
| 394 | |||||||||
| 395 | Instruction *InstCombinerImpl::foldFPSignBitOps(BinaryOperator &I) { | ||||||||
| 396 | BinaryOperator::BinaryOps Opcode = I.getOpcode(); | ||||||||
| 397 | assert((Opcode == Instruction::FMul || Opcode == Instruction::FDiv) &&((void)0) | ||||||||
| 398 | "Expected fmul or fdiv")((void)0); | ||||||||
| 399 | |||||||||
| 400 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 401 | Value *X, *Y; | ||||||||
| 402 | |||||||||
| 403 | // -X * -Y --> X * Y | ||||||||
| 404 | // -X / -Y --> X / Y | ||||||||
| 405 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) | ||||||||
| 406 | return BinaryOperator::CreateWithCopiedFlags(Opcode, X, Y, &I); | ||||||||
| 407 | |||||||||
| 408 | // fabs(X) * fabs(X) -> X * X | ||||||||
| 409 | // fabs(X) / fabs(X) -> X / X | ||||||||
| 410 | if (Op0 == Op1 && match(Op0, m_FAbs(m_Value(X)))) | ||||||||
| 411 | return BinaryOperator::CreateWithCopiedFlags(Opcode, X, X, &I); | ||||||||
| 412 | |||||||||
| 413 | // fabs(X) * fabs(Y) --> fabs(X * Y) | ||||||||
| 414 | // fabs(X) / fabs(Y) --> fabs(X / Y) | ||||||||
| 415 | if (match(Op0, m_FAbs(m_Value(X))) && match(Op1, m_FAbs(m_Value(Y))) && | ||||||||
| 416 | (Op0->hasOneUse() || Op1->hasOneUse())) { | ||||||||
| 417 | IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); | ||||||||
| 418 | Builder.setFastMathFlags(I.getFastMathFlags()); | ||||||||
| 419 | Value *XY = Builder.CreateBinOp(Opcode, X, Y); | ||||||||
| 420 | Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, XY); | ||||||||
| 421 | Fabs->takeName(&I); | ||||||||
| 422 | return replaceInstUsesWith(I, Fabs); | ||||||||
| 423 | } | ||||||||
| 424 | |||||||||
| 425 | return nullptr; | ||||||||
| 426 | } | ||||||||
| 427 | |||||||||
| 428 | Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) { | ||||||||
| 429 | if (Value *V = SimplifyFMulInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 430 | I.getFastMathFlags(), | ||||||||
| 431 | SQ.getWithInstruction(&I))) | ||||||||
| 432 | return replaceInstUsesWith(I, V); | ||||||||
| 433 | |||||||||
| 434 | if (SimplifyAssociativeOrCommutative(I)) | ||||||||
| 435 | return &I; | ||||||||
| 436 | |||||||||
| 437 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 438 | return X; | ||||||||
| 439 | |||||||||
| 440 | if (Instruction *FoldedMul = foldBinOpIntoSelectOrPhi(I)) | ||||||||
| 441 | return FoldedMul; | ||||||||
| 442 | |||||||||
| 443 | if (Value *FoldedMul = foldMulSelectToNegate(I, Builder)) | ||||||||
| 444 | return replaceInstUsesWith(I, FoldedMul); | ||||||||
| 445 | |||||||||
| 446 | if (Instruction *R = foldFPSignBitOps(I)) | ||||||||
| 447 | return R; | ||||||||
| 448 | |||||||||
| 449 | // X * -1.0 --> -X | ||||||||
| 450 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 451 | if (match(Op1, m_SpecificFP(-1.0))) | ||||||||
| 452 | return UnaryOperator::CreateFNegFMF(Op0, &I); | ||||||||
| 453 | |||||||||
| 454 | // -X * C --> X * -C | ||||||||
| 455 | Value *X, *Y; | ||||||||
| 456 | Constant *C; | ||||||||
| 457 | if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_Constant(C))) | ||||||||
| 458 | return BinaryOperator::CreateFMulFMF(X, ConstantExpr::getFNeg(C), &I); | ||||||||
| 459 | |||||||||
| 460 | // (select A, B, C) * (select A, D, E) --> select A, (B*D), (C*E) | ||||||||
| 461 | if (Value *V = SimplifySelectsFeedingBinaryOp(I, Op0, Op1)) | ||||||||
| 462 | return replaceInstUsesWith(I, V); | ||||||||
| 463 | |||||||||
| 464 | if (I.hasAllowReassoc()) { | ||||||||
| 465 | // Reassociate constant RHS with another constant to form constant | ||||||||
| 466 | // expression. | ||||||||
| 467 | if (match(Op1, m_Constant(C)) && C->isFiniteNonZeroFP()) { | ||||||||
| 468 | Constant *C1; | ||||||||
| 469 | if (match(Op0, m_OneUse(m_FDiv(m_Constant(C1), m_Value(X))))) { | ||||||||
| 470 | // (C1 / X) * C --> (C * C1) / X | ||||||||
| 471 | Constant *CC1 = ConstantExpr::getFMul(C, C1); | ||||||||
| 472 | if (CC1->isNormalFP()) | ||||||||
| 473 | return BinaryOperator::CreateFDivFMF(CC1, X, &I); | ||||||||
| 474 | } | ||||||||
| 475 | if (match(Op0, m_FDiv(m_Value(X), m_Constant(C1)))) { | ||||||||
| 476 | // (X / C1) * C --> X * (C / C1) | ||||||||
| 477 | Constant *CDivC1 = ConstantExpr::getFDiv(C, C1); | ||||||||
| 478 | if (CDivC1->isNormalFP()) | ||||||||
| 479 | return BinaryOperator::CreateFMulFMF(X, CDivC1, &I); | ||||||||
| 480 | |||||||||
| 481 | // If the constant was a denormal, try reassociating differently. | ||||||||
| 482 | // (X / C1) * C --> X / (C1 / C) | ||||||||
| 483 | Constant *C1DivC = ConstantExpr::getFDiv(C1, C); | ||||||||
| 484 | if (Op0->hasOneUse() && C1DivC->isNormalFP()) | ||||||||
| 485 | return BinaryOperator::CreateFDivFMF(X, C1DivC, &I); | ||||||||
| 486 | } | ||||||||
| 487 | |||||||||
| 488 | // We do not need to match 'fadd C, X' and 'fsub X, C' because they are | ||||||||
| 489 | // canonicalized to 'fadd X, C'. Distributing the multiply may allow | ||||||||
| 490 | // further folds and (X * C) + C2 is 'fma'. | ||||||||
| 491 | if (match(Op0, m_OneUse(m_FAdd(m_Value(X), m_Constant(C1))))) { | ||||||||
| 492 | // (X + C1) * C --> (X * C) + (C * C1) | ||||||||
| 493 | Constant *CC1 = ConstantExpr::getFMul(C, C1); | ||||||||
| 494 | Value *XC = Builder.CreateFMulFMF(X, C, &I); | ||||||||
| 495 | return BinaryOperator::CreateFAddFMF(XC, CC1, &I); | ||||||||
| 496 | } | ||||||||
| 497 | if (match(Op0, m_OneUse(m_FSub(m_Constant(C1), m_Value(X))))) { | ||||||||
| 498 | // (C1 - X) * C --> (C * C1) - (X * C) | ||||||||
| 499 | Constant *CC1 = ConstantExpr::getFMul(C, C1); | ||||||||
| 500 | Value *XC = Builder.CreateFMulFMF(X, C, &I); | ||||||||
| 501 | return BinaryOperator::CreateFSubFMF(CC1, XC, &I); | ||||||||
| 502 | } | ||||||||
| 503 | } | ||||||||
| 504 | |||||||||
| 505 | Value *Z; | ||||||||
| 506 | if (match(&I, m_c_FMul(m_OneUse(m_FDiv(m_Value(X), m_Value(Y))), | ||||||||
| 507 | m_Value(Z)))) { | ||||||||
| 508 | // Sink division: (X / Y) * Z --> (X * Z) / Y | ||||||||
| 509 | Value *NewFMul = Builder.CreateFMulFMF(X, Z, &I); | ||||||||
| 510 | return BinaryOperator::CreateFDivFMF(NewFMul, Y, &I); | ||||||||
| 511 | } | ||||||||
| 512 | |||||||||
| 513 | // sqrt(X) * sqrt(Y) -> sqrt(X * Y) | ||||||||
| 514 | // nnan disallows the possibility of returning a number if both operands are | ||||||||
| 515 | // negative (in that case, we should return NaN). | ||||||||
| 516 | if (I.hasNoNaNs() && | ||||||||
| 517 | match(Op0, m_OneUse(m_Intrinsic<Intrinsic::sqrt>(m_Value(X)))) && | ||||||||
| 518 | match(Op1, m_OneUse(m_Intrinsic<Intrinsic::sqrt>(m_Value(Y))))) { | ||||||||
| 519 | Value *XY = Builder.CreateFMulFMF(X, Y, &I); | ||||||||
| 520 | Value *Sqrt = Builder.CreateUnaryIntrinsic(Intrinsic::sqrt, XY, &I); | ||||||||
| 521 | return replaceInstUsesWith(I, Sqrt); | ||||||||
| 522 | } | ||||||||
| 523 | |||||||||
| 524 | // The following transforms are done irrespective of the number of uses | ||||||||
| 525 | // for the expression "1.0/sqrt(X)". | ||||||||
| 526 | // 1) 1.0/sqrt(X) * X -> X/sqrt(X) | ||||||||
| 527 | // 2) X * 1.0/sqrt(X) -> X/sqrt(X) | ||||||||
| 528 | // We always expect the backend to reduce X/sqrt(X) to sqrt(X), if it | ||||||||
| 529 | // has the necessary (reassoc) fast-math-flags. | ||||||||
| 530 | if (I.hasNoSignedZeros() && | ||||||||
| 531 | match(Op0, (m_FDiv(m_SpecificFP(1.0), m_Value(Y)))) && | ||||||||
| 532 | match(Y, m_Intrinsic<Intrinsic::sqrt>(m_Value(X))) && Op1 == X) | ||||||||
| 533 | return BinaryOperator::CreateFDivFMF(X, Y, &I); | ||||||||
| 534 | if (I.hasNoSignedZeros() && | ||||||||
| 535 | match(Op1, (m_FDiv(m_SpecificFP(1.0), m_Value(Y)))) && | ||||||||
| 536 | match(Y, m_Intrinsic<Intrinsic::sqrt>(m_Value(X))) && Op0 == X) | ||||||||
| 537 | return BinaryOperator::CreateFDivFMF(X, Y, &I); | ||||||||
| 538 | |||||||||
| 539 | // Like the similar transform in instsimplify, this requires 'nsz' because | ||||||||
| 540 | // sqrt(-0.0) = -0.0, and -0.0 * -0.0 does not simplify to -0.0. | ||||||||
| 541 | if (I.hasNoNaNs() && I.hasNoSignedZeros() && Op0 == Op1 && | ||||||||
| 542 | Op0->hasNUses(2)) { | ||||||||
| 543 | // Peek through fdiv to find squaring of square root: | ||||||||
| 544 | // (X / sqrt(Y)) * (X / sqrt(Y)) --> (X * X) / Y | ||||||||
| 545 | if (match(Op0, m_FDiv(m_Value(X), | ||||||||
| 546 | m_Intrinsic<Intrinsic::sqrt>(m_Value(Y))))) { | ||||||||
| 547 | Value *XX = Builder.CreateFMulFMF(X, X, &I); | ||||||||
| 548 | return BinaryOperator::CreateFDivFMF(XX, Y, &I); | ||||||||
| 549 | } | ||||||||
| 550 | // (sqrt(Y) / X) * (sqrt(Y) / X) --> Y / (X * X) | ||||||||
| 551 | if (match(Op0, m_FDiv(m_Intrinsic<Intrinsic::sqrt>(m_Value(Y)), | ||||||||
| 552 | m_Value(X)))) { | ||||||||
| 553 | Value *XX = Builder.CreateFMulFMF(X, X, &I); | ||||||||
| 554 | return BinaryOperator::CreateFDivFMF(Y, XX, &I); | ||||||||
| 555 | } | ||||||||
| 556 | } | ||||||||
| 557 | |||||||||
| 558 | if (I.isOnlyUserOfAnyOperand()) { | ||||||||
| 559 | // pow(x, y) * pow(x, z) -> pow(x, y + z) | ||||||||
| 560 | if (match(Op0, m_Intrinsic<Intrinsic::pow>(m_Value(X), m_Value(Y))) && | ||||||||
| 561 | match(Op1, m_Intrinsic<Intrinsic::pow>(m_Specific(X), m_Value(Z)))) { | ||||||||
| 562 | auto *YZ = Builder.CreateFAddFMF(Y, Z, &I); | ||||||||
| 563 | auto *NewPow = Builder.CreateBinaryIntrinsic(Intrinsic::pow, X, YZ, &I); | ||||||||
| 564 | return replaceInstUsesWith(I, NewPow); | ||||||||
| 565 | } | ||||||||
| 566 | |||||||||
| 567 | // exp(X) * exp(Y) -> exp(X + Y) | ||||||||
| 568 | if (match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X))) && | ||||||||
| 569 | match(Op1, m_Intrinsic<Intrinsic::exp>(m_Value(Y)))) { | ||||||||
| 570 | Value *XY = Builder.CreateFAddFMF(X, Y, &I); | ||||||||
| 571 | Value *Exp = Builder.CreateUnaryIntrinsic(Intrinsic::exp, XY, &I); | ||||||||
| 572 | return replaceInstUsesWith(I, Exp); | ||||||||
| 573 | } | ||||||||
| 574 | |||||||||
| 575 | // exp2(X) * exp2(Y) -> exp2(X + Y) | ||||||||
| 576 | if (match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) && | ||||||||
| 577 | match(Op1, m_Intrinsic<Intrinsic::exp2>(m_Value(Y)))) { | ||||||||
| 578 | Value *XY = Builder.CreateFAddFMF(X, Y, &I); | ||||||||
| 579 | Value *Exp2 = Builder.CreateUnaryIntrinsic(Intrinsic::exp2, XY, &I); | ||||||||
| 580 | return replaceInstUsesWith(I, Exp2); | ||||||||
| 581 | } | ||||||||
| 582 | } | ||||||||
| 583 | |||||||||
| 584 | // (X*Y) * X => (X*X) * Y where Y != X | ||||||||
| 585 | // The purpose is two-fold: | ||||||||
| 586 | // 1) to form a power expression (of X). | ||||||||
| 587 | // 2) potentially shorten the critical path: After transformation, the | ||||||||
| 588 | // latency of the instruction Y is amortized by the expression of X*X, | ||||||||
| 589 | // and therefore Y is in a "less critical" position compared to what it | ||||||||
| 590 | // was before the transformation. | ||||||||
| 591 | if (match(Op0, m_OneUse(m_c_FMul(m_Specific(Op1), m_Value(Y)))) && | ||||||||
| 592 | Op1 != Y) { | ||||||||
| 593 | Value *XX = Builder.CreateFMulFMF(Op1, Op1, &I); | ||||||||
| 594 | return BinaryOperator::CreateFMulFMF(XX, Y, &I); | ||||||||
| 595 | } | ||||||||
| 596 | if (match(Op1, m_OneUse(m_c_FMul(m_Specific(Op0), m_Value(Y)))) && | ||||||||
| 597 | Op0 != Y) { | ||||||||
| 598 | Value *XX = Builder.CreateFMulFMF(Op0, Op0, &I); | ||||||||
| 599 | return BinaryOperator::CreateFMulFMF(XX, Y, &I); | ||||||||
| 600 | } | ||||||||
| 601 | } | ||||||||
| 602 | |||||||||
| 603 | // log2(X * 0.5) * Y = log2(X) * Y - Y | ||||||||
| 604 | if (I.isFast()) { | ||||||||
| 605 | IntrinsicInst *Log2 = nullptr; | ||||||||
| 606 | if (match(Op0, m_OneUse(m_Intrinsic<Intrinsic::log2>( | ||||||||
| 607 | m_OneUse(m_FMul(m_Value(X), m_SpecificFP(0.5))))))) { | ||||||||
| 608 | Log2 = cast<IntrinsicInst>(Op0); | ||||||||
| 609 | Y = Op1; | ||||||||
| 610 | } | ||||||||
| 611 | if (match(Op1, m_OneUse(m_Intrinsic<Intrinsic::log2>( | ||||||||
| 612 | m_OneUse(m_FMul(m_Value(X), m_SpecificFP(0.5))))))) { | ||||||||
| 613 | Log2 = cast<IntrinsicInst>(Op1); | ||||||||
| 614 | Y = Op0; | ||||||||
| 615 | } | ||||||||
| 616 | if (Log2) { | ||||||||
| 617 | Value *Log2 = Builder.CreateUnaryIntrinsic(Intrinsic::log2, X, &I); | ||||||||
| 618 | Value *LogXTimesY = Builder.CreateFMulFMF(Log2, Y, &I); | ||||||||
| 619 | return BinaryOperator::CreateFSubFMF(LogXTimesY, Y, &I); | ||||||||
| 620 | } | ||||||||
| 621 | } | ||||||||
| 622 | |||||||||
| 623 | return nullptr; | ||||||||
| 624 | } | ||||||||
| 625 | |||||||||
| 626 | /// Fold a divide or remainder with a select instruction divisor when one of the | ||||||||
| 627 | /// select operands is zero. In that case, we can use the other select operand | ||||||||
| 628 | /// because div/rem by zero is undefined. | ||||||||
| 629 | bool InstCombinerImpl::simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I) { | ||||||||
| 630 | SelectInst *SI = dyn_cast<SelectInst>(I.getOperand(1)); | ||||||||
| 631 | if (!SI) | ||||||||
| 632 | return false; | ||||||||
| 633 | |||||||||
| 634 | int NonNullOperand; | ||||||||
| 635 | if (match(SI->getTrueValue(), m_Zero())) | ||||||||
| 636 | // div/rem X, (Cond ? 0 : Y) -> div/rem X, Y | ||||||||
| 637 | NonNullOperand = 2; | ||||||||
| 638 | else if (match(SI->getFalseValue(), m_Zero())) | ||||||||
| 639 | // div/rem X, (Cond ? Y : 0) -> div/rem X, Y | ||||||||
| 640 | NonNullOperand = 1; | ||||||||
| 641 | else | ||||||||
| 642 | return false; | ||||||||
| 643 | |||||||||
| 644 | // Change the div/rem to use 'Y' instead of the select. | ||||||||
| 645 | replaceOperand(I, 1, SI->getOperand(NonNullOperand)); | ||||||||
| 646 | |||||||||
| 647 | // Okay, we know we replace the operand of the div/rem with 'Y' with no | ||||||||
| 648 | // problem. However, the select, or the condition of the select may have | ||||||||
| 649 | // multiple uses. Based on our knowledge that the operand must be non-zero, | ||||||||
| 650 | // propagate the known value for the select into other uses of it, and | ||||||||
| 651 | // propagate a known value of the condition into its other users. | ||||||||
| 652 | |||||||||
| 653 | // If the select and condition only have a single use, don't bother with this, | ||||||||
| 654 | // early exit. | ||||||||
| 655 | Value *SelectCond = SI->getCondition(); | ||||||||
| 656 | if (SI->use_empty() && SelectCond->hasOneUse()) | ||||||||
| 657 | return true; | ||||||||
| 658 | |||||||||
| 659 | // Scan the current block backward, looking for other uses of SI. | ||||||||
| 660 | BasicBlock::iterator BBI = I.getIterator(), BBFront = I.getParent()->begin(); | ||||||||
| 661 | Type *CondTy = SelectCond->getType(); | ||||||||
| 662 | while (BBI != BBFront) { | ||||||||
| 663 | --BBI; | ||||||||
| 664 | // If we found an instruction that we can't assume will return, so | ||||||||
| 665 | // information from below it cannot be propagated above it. | ||||||||
| 666 | if (!isGuaranteedToTransferExecutionToSuccessor(&*BBI)) | ||||||||
| 667 | break; | ||||||||
| 668 | |||||||||
| 669 | // Replace uses of the select or its condition with the known values. | ||||||||
| 670 | for (Use &Op : BBI->operands()) { | ||||||||
| 671 | if (Op == SI) { | ||||||||
| 672 | replaceUse(Op, SI->getOperand(NonNullOperand)); | ||||||||
| |||||||||
| 673 | Worklist.push(&*BBI); | ||||||||
| 674 | } else if (Op == SelectCond) { | ||||||||
| 675 | replaceUse(Op, NonNullOperand == 1 ? ConstantInt::getTrue(CondTy) | ||||||||
| 676 | : ConstantInt::getFalse(CondTy)); | ||||||||
| 677 | Worklist.push(&*BBI); | ||||||||
| 678 | } | ||||||||
| 679 | } | ||||||||
| 680 | |||||||||
| 681 | // If we past the instruction, quit looking for it. | ||||||||
| 682 | if (&*BBI == SI) | ||||||||
| 683 | SI = nullptr; | ||||||||
| 684 | if (&*BBI == SelectCond) | ||||||||
| 685 | SelectCond = nullptr; | ||||||||
| 686 | |||||||||
| 687 | // If we ran out of things to eliminate, break out of the loop. | ||||||||
| 688 | if (!SelectCond
| ||||||||
| 689 | break; | ||||||||
| 690 | |||||||||
| 691 | } | ||||||||
| 692 | return true; | ||||||||
| 693 | } | ||||||||
| 694 | |||||||||
| 695 | /// True if the multiply can not be expressed in an int this size. | ||||||||
| 696 | static bool multiplyOverflows(const APInt &C1, const APInt &C2, APInt &Product, | ||||||||
| 697 | bool IsSigned) { | ||||||||
| 698 | bool Overflow; | ||||||||
| 699 | Product = IsSigned ? C1.smul_ov(C2, Overflow) : C1.umul_ov(C2, Overflow); | ||||||||
| 700 | return Overflow; | ||||||||
| 701 | } | ||||||||
| 702 | |||||||||
| 703 | /// True if C1 is a multiple of C2. Quotient contains C1/C2. | ||||||||
| 704 | static bool isMultiple(const APInt &C1, const APInt &C2, APInt &Quotient, | ||||||||
| 705 | bool IsSigned) { | ||||||||
| 706 | assert(C1.getBitWidth() == C2.getBitWidth() && "Constant widths not equal")((void)0); | ||||||||
| 707 | |||||||||
| 708 | // Bail if we will divide by zero. | ||||||||
| 709 | if (C2.isNullValue()) | ||||||||
| 710 | return false; | ||||||||
| 711 | |||||||||
| 712 | // Bail if we would divide INT_MIN by -1. | ||||||||
| 713 | if (IsSigned && C1.isMinSignedValue() && C2.isAllOnesValue()) | ||||||||
| 714 | return false; | ||||||||
| 715 | |||||||||
| 716 | APInt Remainder(C1.getBitWidth(), /*val=*/0ULL, IsSigned); | ||||||||
| 717 | if (IsSigned) | ||||||||
| 718 | APInt::sdivrem(C1, C2, Quotient, Remainder); | ||||||||
| 719 | else | ||||||||
| 720 | APInt::udivrem(C1, C2, Quotient, Remainder); | ||||||||
| 721 | |||||||||
| 722 | return Remainder.isMinValue(); | ||||||||
| 723 | } | ||||||||
| 724 | |||||||||
| 725 | /// This function implements the transforms common to both integer division | ||||||||
| 726 | /// instructions (udiv and sdiv). It is called by the visitors to those integer | ||||||||
| 727 | /// division instructions. | ||||||||
| 728 | /// Common integer divide transforms | ||||||||
| 729 | Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) { | ||||||||
| 730 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 731 | bool IsSigned = I.getOpcode() == Instruction::SDiv; | ||||||||
| 732 | Type *Ty = I.getType(); | ||||||||
| 733 | |||||||||
| 734 | // The RHS is known non-zero. | ||||||||
| 735 | if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I)) | ||||||||
| 736 | return replaceOperand(I, 1, V); | ||||||||
| 737 | |||||||||
| 738 | // Handle cases involving: [su]div X, (select Cond, Y, Z) | ||||||||
| 739 | // This does not apply for fdiv. | ||||||||
| 740 | if (simplifyDivRemOfSelectWithZeroOp(I)) | ||||||||
| 741 | return &I; | ||||||||
| 742 | |||||||||
| 743 | const APInt *C2; | ||||||||
| 744 | if (match(Op1, m_APInt(C2))) { | ||||||||
| 745 | Value *X; | ||||||||
| 746 | const APInt *C1; | ||||||||
| 747 | |||||||||
| 748 | // (X / C1) / C2 -> X / (C1*C2) | ||||||||
| 749 | if ((IsSigned && match(Op0, m_SDiv(m_Value(X), m_APInt(C1)))) || | ||||||||
| 750 | (!IsSigned && match(Op0, m_UDiv(m_Value(X), m_APInt(C1))))) { | ||||||||
| 751 | APInt Product(C1->getBitWidth(), /*val=*/0ULL, IsSigned); | ||||||||
| 752 | if (!multiplyOverflows(*C1, *C2, Product, IsSigned)) | ||||||||
| 753 | return BinaryOperator::Create(I.getOpcode(), X, | ||||||||
| 754 | ConstantInt::get(Ty, Product)); | ||||||||
| 755 | } | ||||||||
| 756 | |||||||||
| 757 | if ((IsSigned && match(Op0, m_NSWMul(m_Value(X), m_APInt(C1)))) || | ||||||||
| 758 | (!IsSigned && match(Op0, m_NUWMul(m_Value(X), m_APInt(C1))))) { | ||||||||
| 759 | APInt Quotient(C1->getBitWidth(), /*val=*/0ULL, IsSigned); | ||||||||
| 760 | |||||||||
| 761 | // (X * C1) / C2 -> X / (C2 / C1) if C2 is a multiple of C1. | ||||||||
| 762 | if (isMultiple(*C2, *C1, Quotient, IsSigned)) { | ||||||||
| 763 | auto *NewDiv = BinaryOperator::Create(I.getOpcode(), X, | ||||||||
| 764 | ConstantInt::get(Ty, Quotient)); | ||||||||
| 765 | NewDiv->setIsExact(I.isExact()); | ||||||||
| 766 | return NewDiv; | ||||||||
| 767 | } | ||||||||
| 768 | |||||||||
| 769 | // (X * C1) / C2 -> X * (C1 / C2) if C1 is a multiple of C2. | ||||||||
| 770 | if (isMultiple(*C1, *C2, Quotient, IsSigned)) { | ||||||||
| 771 | auto *Mul = BinaryOperator::Create(Instruction::Mul, X, | ||||||||
| 772 | ConstantInt::get(Ty, Quotient)); | ||||||||
| 773 | auto *OBO = cast<OverflowingBinaryOperator>(Op0); | ||||||||
| 774 | Mul->setHasNoUnsignedWrap(!IsSigned && OBO->hasNoUnsignedWrap()); | ||||||||
| 775 | Mul->setHasNoSignedWrap(OBO->hasNoSignedWrap()); | ||||||||
| 776 | return Mul; | ||||||||
| 777 | } | ||||||||
| 778 | } | ||||||||
| 779 | |||||||||
| 780 | if ((IsSigned && match(Op0, m_NSWShl(m_Value(X), m_APInt(C1))) && | ||||||||
| 781 | *C1 != C1->getBitWidth() - 1) || | ||||||||
| 782 | (!IsSigned && match(Op0, m_NUWShl(m_Value(X), m_APInt(C1))))) { | ||||||||
| 783 | APInt Quotient(C1->getBitWidth(), /*val=*/0ULL, IsSigned); | ||||||||
| 784 | APInt C1Shifted = APInt::getOneBitSet( | ||||||||
| 785 | C1->getBitWidth(), static_cast<unsigned>(C1->getLimitedValue())); | ||||||||
| 786 | |||||||||
| 787 | // (X << C1) / C2 -> X / (C2 >> C1) if C2 is a multiple of 1 << C1. | ||||||||
| 788 | if (isMultiple(*C2, C1Shifted, Quotient, IsSigned)) { | ||||||||
| 789 | auto *BO = BinaryOperator::Create(I.getOpcode(), X, | ||||||||
| 790 | ConstantInt::get(Ty, Quotient)); | ||||||||
| 791 | BO->setIsExact(I.isExact()); | ||||||||
| 792 | return BO; | ||||||||
| 793 | } | ||||||||
| 794 | |||||||||
| 795 | // (X << C1) / C2 -> X * ((1 << C1) / C2) if 1 << C1 is a multiple of C2. | ||||||||
| 796 | if (isMultiple(C1Shifted, *C2, Quotient, IsSigned)) { | ||||||||
| 797 | auto *Mul = BinaryOperator::Create(Instruction::Mul, X, | ||||||||
| 798 | ConstantInt::get(Ty, Quotient)); | ||||||||
| 799 | auto *OBO = cast<OverflowingBinaryOperator>(Op0); | ||||||||
| 800 | Mul->setHasNoUnsignedWrap(!IsSigned && OBO->hasNoUnsignedWrap()); | ||||||||
| 801 | Mul->setHasNoSignedWrap(OBO->hasNoSignedWrap()); | ||||||||
| 802 | return Mul; | ||||||||
| 803 | } | ||||||||
| 804 | } | ||||||||
| 805 | |||||||||
| 806 | if (!C2->isNullValue()) // avoid X udiv 0 | ||||||||
| 807 | if (Instruction *FoldedDiv = foldBinOpIntoSelectOrPhi(I)) | ||||||||
| 808 | return FoldedDiv; | ||||||||
| 809 | } | ||||||||
| 810 | |||||||||
| 811 | if (match(Op0, m_One())) { | ||||||||
| 812 | assert(!Ty->isIntOrIntVectorTy(1) && "i1 divide not removed?")((void)0); | ||||||||
| 813 | if (IsSigned) { | ||||||||
| 814 | // If Op1 is 0 then it's undefined behaviour, if Op1 is 1 then the | ||||||||
| 815 | // result is one, if Op1 is -1 then the result is minus one, otherwise | ||||||||
| 816 | // it's zero. | ||||||||
| 817 | Value *Inc = Builder.CreateAdd(Op1, Op0); | ||||||||
| 818 | Value *Cmp = Builder.CreateICmpULT(Inc, ConstantInt::get(Ty, 3)); | ||||||||
| 819 | return SelectInst::Create(Cmp, Op1, ConstantInt::get(Ty, 0)); | ||||||||
| 820 | } else { | ||||||||
| 821 | // If Op1 is 0 then it's undefined behaviour. If Op1 is 1 then the | ||||||||
| 822 | // result is one, otherwise it's zero. | ||||||||
| 823 | return new ZExtInst(Builder.CreateICmpEQ(Op1, Op0), Ty); | ||||||||
| 824 | } | ||||||||
| 825 | } | ||||||||
| 826 | |||||||||
| 827 | // See if we can fold away this div instruction. | ||||||||
| 828 | if (SimplifyDemandedInstructionBits(I)) | ||||||||
| 829 | return &I; | ||||||||
| 830 | |||||||||
| 831 | // (X - (X rem Y)) / Y -> X / Y; usually originates as ((X / Y) * Y) / Y | ||||||||
| 832 | Value *X, *Z; | ||||||||
| 833 | if (match(Op0, m_Sub(m_Value(X), m_Value(Z)))) // (X - Z) / Y; Y = Op1 | ||||||||
| 834 | if ((IsSigned && match(Z, m_SRem(m_Specific(X), m_Specific(Op1)))) || | ||||||||
| 835 | (!IsSigned && match(Z, m_URem(m_Specific(X), m_Specific(Op1))))) | ||||||||
| 836 | return BinaryOperator::Create(I.getOpcode(), X, Op1); | ||||||||
| 837 | |||||||||
| 838 | // (X << Y) / X -> 1 << Y | ||||||||
| 839 | Value *Y; | ||||||||
| 840 | if (IsSigned && match(Op0, m_NSWShl(m_Specific(Op1), m_Value(Y)))) | ||||||||
| 841 | return BinaryOperator::CreateNSWShl(ConstantInt::get(Ty, 1), Y); | ||||||||
| 842 | if (!IsSigned && match(Op0, m_NUWShl(m_Specific(Op1), m_Value(Y)))) | ||||||||
| 843 | return BinaryOperator::CreateNUWShl(ConstantInt::get(Ty, 1), Y); | ||||||||
| 844 | |||||||||
| 845 | // X / (X * Y) -> 1 / Y if the multiplication does not overflow. | ||||||||
| 846 | if (match(Op1, m_c_Mul(m_Specific(Op0), m_Value(Y)))) { | ||||||||
| 847 | bool HasNSW = cast<OverflowingBinaryOperator>(Op1)->hasNoSignedWrap(); | ||||||||
| 848 | bool HasNUW = cast<OverflowingBinaryOperator>(Op1)->hasNoUnsignedWrap(); | ||||||||
| 849 | if ((IsSigned && HasNSW) || (!IsSigned && HasNUW)) { | ||||||||
| 850 | replaceOperand(I, 0, ConstantInt::get(Ty, 1)); | ||||||||
| 851 | replaceOperand(I, 1, Y); | ||||||||
| 852 | return &I; | ||||||||
| 853 | } | ||||||||
| 854 | } | ||||||||
| 855 | |||||||||
| 856 | return nullptr; | ||||||||
| 857 | } | ||||||||
| 858 | |||||||||
| 859 | static const unsigned MaxDepth = 6; | ||||||||
| 860 | |||||||||
| 861 | namespace { | ||||||||
| 862 | |||||||||
| 863 | using FoldUDivOperandCb = Instruction *(*)(Value *Op0, Value *Op1, | ||||||||
| 864 | const BinaryOperator &I, | ||||||||
| 865 | InstCombinerImpl &IC); | ||||||||
| 866 | |||||||||
| 867 | /// Used to maintain state for visitUDivOperand(). | ||||||||
| 868 | struct UDivFoldAction { | ||||||||
| 869 | /// Informs visitUDiv() how to fold this operand. This can be zero if this | ||||||||
| 870 | /// action joins two actions together. | ||||||||
| 871 | FoldUDivOperandCb FoldAction; | ||||||||
| 872 | |||||||||
| 873 | /// Which operand to fold. | ||||||||
| 874 | Value *OperandToFold; | ||||||||
| 875 | |||||||||
| 876 | union { | ||||||||
| 877 | /// The instruction returned when FoldAction is invoked. | ||||||||
| 878 | Instruction *FoldResult; | ||||||||
| 879 | |||||||||
| 880 | /// Stores the LHS action index if this action joins two actions together. | ||||||||
| 881 | size_t SelectLHSIdx; | ||||||||
| 882 | }; | ||||||||
| 883 | |||||||||
| 884 | UDivFoldAction(FoldUDivOperandCb FA, Value *InputOperand) | ||||||||
| 885 | : FoldAction(FA), OperandToFold(InputOperand), FoldResult(nullptr) {} | ||||||||
| 886 | UDivFoldAction(FoldUDivOperandCb FA, Value *InputOperand, size_t SLHS) | ||||||||
| 887 | : FoldAction(FA), OperandToFold(InputOperand), SelectLHSIdx(SLHS) {} | ||||||||
| 888 | }; | ||||||||
| 889 | |||||||||
| 890 | } // end anonymous namespace | ||||||||
| 891 | |||||||||
| 892 | // X udiv 2^C -> X >> C | ||||||||
| 893 | static Instruction *foldUDivPow2Cst(Value *Op0, Value *Op1, | ||||||||
| 894 | const BinaryOperator &I, | ||||||||
| 895 | InstCombinerImpl &IC) { | ||||||||
| 896 | Constant *C1 = ConstantExpr::getExactLogBase2(cast<Constant>(Op1)); | ||||||||
| 897 | if (!C1) | ||||||||
| 898 | llvm_unreachable("Failed to constant fold udiv -> logbase2")__builtin_unreachable(); | ||||||||
| 899 | BinaryOperator *LShr = BinaryOperator::CreateLShr(Op0, C1); | ||||||||
| 900 | if (I.isExact()) | ||||||||
| 901 | LShr->setIsExact(); | ||||||||
| 902 | return LShr; | ||||||||
| 903 | } | ||||||||
| 904 | |||||||||
| 905 | // X udiv (C1 << N), where C1 is "1<<C2" --> X >> (N+C2) | ||||||||
| 906 | // X udiv (zext (C1 << N)), where C1 is "1<<C2" --> X >> (N+C2) | ||||||||
| 907 | static Instruction *foldUDivShl(Value *Op0, Value *Op1, const BinaryOperator &I, | ||||||||
| 908 | InstCombinerImpl &IC) { | ||||||||
| 909 | Value *ShiftLeft; | ||||||||
| 910 | if (!match(Op1, m_ZExt(m_Value(ShiftLeft)))) | ||||||||
| 911 | ShiftLeft = Op1; | ||||||||
| 912 | |||||||||
| 913 | Constant *CI; | ||||||||
| 914 | Value *N; | ||||||||
| 915 | if (!match(ShiftLeft, m_Shl(m_Constant(CI), m_Value(N)))) | ||||||||
| 916 | llvm_unreachable("match should never fail here!")__builtin_unreachable(); | ||||||||
| 917 | Constant *Log2Base = ConstantExpr::getExactLogBase2(CI); | ||||||||
| 918 | if (!Log2Base) | ||||||||
| 919 | llvm_unreachable("getLogBase2 should never fail here!")__builtin_unreachable(); | ||||||||
| 920 | N = IC.Builder.CreateAdd(N, Log2Base); | ||||||||
| 921 | if (Op1 != ShiftLeft) | ||||||||
| 922 | N = IC.Builder.CreateZExt(N, Op1->getType()); | ||||||||
| 923 | BinaryOperator *LShr = BinaryOperator::CreateLShr(Op0, N); | ||||||||
| 924 | if (I.isExact()) | ||||||||
| 925 | LShr->setIsExact(); | ||||||||
| 926 | return LShr; | ||||||||
| 927 | } | ||||||||
| 928 | |||||||||
| 929 | // Recursively visits the possible right hand operands of a udiv | ||||||||
| 930 | // instruction, seeing through select instructions, to determine if we can | ||||||||
| 931 | // replace the udiv with something simpler. If we find that an operand is not | ||||||||
| 932 | // able to simplify the udiv, we abort the entire transformation. | ||||||||
| 933 | static size_t visitUDivOperand(Value *Op0, Value *Op1, const BinaryOperator &I, | ||||||||
| 934 | SmallVectorImpl<UDivFoldAction> &Actions, | ||||||||
| 935 | unsigned Depth = 0) { | ||||||||
| 936 | // FIXME: assert that Op1 isn't/doesn't contain undef. | ||||||||
| 937 | |||||||||
| 938 | // Check to see if this is an unsigned division with an exact power of 2, | ||||||||
| 939 | // if so, convert to a right shift. | ||||||||
| 940 | if (match(Op1, m_Power2())) { | ||||||||
| 941 | Actions.push_back(UDivFoldAction(foldUDivPow2Cst, Op1)); | ||||||||
| 942 | return Actions.size(); | ||||||||
| 943 | } | ||||||||
| 944 | |||||||||
| 945 | // X udiv (C1 << N), where C1 is "1<<C2" --> X >> (N+C2) | ||||||||
| 946 | if (match(Op1, m_Shl(m_Power2(), m_Value())) || | ||||||||
| 947 | match(Op1, m_ZExt(m_Shl(m_Power2(), m_Value())))) { | ||||||||
| 948 | Actions.push_back(UDivFoldAction(foldUDivShl, Op1)); | ||||||||
| 949 | return Actions.size(); | ||||||||
| 950 | } | ||||||||
| 951 | |||||||||
| 952 | // The remaining tests are all recursive, so bail out if we hit the limit. | ||||||||
| 953 | if (Depth++ == MaxDepth) | ||||||||
| 954 | return 0; | ||||||||
| 955 | |||||||||
| 956 | if (SelectInst *SI = dyn_cast<SelectInst>(Op1)) | ||||||||
| 957 | // FIXME: missed optimization: if one of the hands of select is/contains | ||||||||
| 958 | // undef, just directly pick the other one. | ||||||||
| 959 | // FIXME: can both hands contain undef? | ||||||||
| 960 | if (size_t LHSIdx = | ||||||||
| 961 | visitUDivOperand(Op0, SI->getOperand(1), I, Actions, Depth)) | ||||||||
| 962 | if (visitUDivOperand(Op0, SI->getOperand(2), I, Actions, Depth)) { | ||||||||
| 963 | Actions.push_back(UDivFoldAction(nullptr, Op1, LHSIdx - 1)); | ||||||||
| 964 | return Actions.size(); | ||||||||
| 965 | } | ||||||||
| 966 | |||||||||
| 967 | return 0; | ||||||||
| 968 | } | ||||||||
| 969 | |||||||||
| 970 | /// If we have zero-extended operands of an unsigned div or rem, we may be able | ||||||||
| 971 | /// to narrow the operation (sink the zext below the math). | ||||||||
| 972 | static Instruction *narrowUDivURem(BinaryOperator &I, | ||||||||
| 973 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 974 | Instruction::BinaryOps Opcode = I.getOpcode(); | ||||||||
| 975 | Value *N = I.getOperand(0); | ||||||||
| 976 | Value *D = I.getOperand(1); | ||||||||
| 977 | Type *Ty = I.getType(); | ||||||||
| 978 | Value *X, *Y; | ||||||||
| 979 | if (match(N, m_ZExt(m_Value(X))) && match(D, m_ZExt(m_Value(Y))) && | ||||||||
| 980 | X->getType() == Y->getType() && (N->hasOneUse() || D->hasOneUse())) { | ||||||||
| 981 | // udiv (zext X), (zext Y) --> zext (udiv X, Y) | ||||||||
| 982 | // urem (zext X), (zext Y) --> zext (urem X, Y) | ||||||||
| 983 | Value *NarrowOp = Builder.CreateBinOp(Opcode, X, Y); | ||||||||
| 984 | return new ZExtInst(NarrowOp, Ty); | ||||||||
| 985 | } | ||||||||
| 986 | |||||||||
| 987 | Constant *C; | ||||||||
| 988 | if ((match(N, m_OneUse(m_ZExt(m_Value(X)))) && match(D, m_Constant(C))) || | ||||||||
| 989 | (match(D, m_OneUse(m_ZExt(m_Value(X)))) && match(N, m_Constant(C)))) { | ||||||||
| 990 | // If the constant is the same in the smaller type, use the narrow version. | ||||||||
| 991 | Constant *TruncC = ConstantExpr::getTrunc(C, X->getType()); | ||||||||
| 992 | if (ConstantExpr::getZExt(TruncC, Ty) != C) | ||||||||
| 993 | return nullptr; | ||||||||
| 994 | |||||||||
| 995 | // udiv (zext X), C --> zext (udiv X, C') | ||||||||
| 996 | // urem (zext X), C --> zext (urem X, C') | ||||||||
| 997 | // udiv C, (zext X) --> zext (udiv C', X) | ||||||||
| 998 | // urem C, (zext X) --> zext (urem C', X) | ||||||||
| 999 | Value *NarrowOp = isa<Constant>(D) ? Builder.CreateBinOp(Opcode, X, TruncC) | ||||||||
| 1000 | : Builder.CreateBinOp(Opcode, TruncC, X); | ||||||||
| 1001 | return new ZExtInst(NarrowOp, Ty); | ||||||||
| 1002 | } | ||||||||
| 1003 | |||||||||
| 1004 | return nullptr; | ||||||||
| 1005 | } | ||||||||
| 1006 | |||||||||
| 1007 | Instruction *InstCombinerImpl::visitUDiv(BinaryOperator &I) { | ||||||||
| 1008 | if (Value *V = SimplifyUDivInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 1009 | SQ.getWithInstruction(&I))) | ||||||||
| 1010 | return replaceInstUsesWith(I, V); | ||||||||
| 1011 | |||||||||
| 1012 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 1013 | return X; | ||||||||
| 1014 | |||||||||
| 1015 | // Handle the integer div common cases | ||||||||
| 1016 | if (Instruction *Common = commonIDivTransforms(I)) | ||||||||
| 1017 | return Common; | ||||||||
| 1018 | |||||||||
| 1019 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1020 | Value *X; | ||||||||
| 1021 | const APInt *C1, *C2; | ||||||||
| 1022 | if (match(Op0, m_LShr(m_Value(X), m_APInt(C1))) && match(Op1, m_APInt(C2))) { | ||||||||
| 1023 | // (X lshr C1) udiv C2 --> X udiv (C2 << C1) | ||||||||
| 1024 | bool Overflow; | ||||||||
| 1025 | APInt C2ShlC1 = C2->ushl_ov(*C1, Overflow); | ||||||||
| 1026 | if (!Overflow) { | ||||||||
| 1027 | bool IsExact = I.isExact() && match(Op0, m_Exact(m_Value())); | ||||||||
| 1028 | BinaryOperator *BO = BinaryOperator::CreateUDiv( | ||||||||
| 1029 | X, ConstantInt::get(X->getType(), C2ShlC1)); | ||||||||
| 1030 | if (IsExact) | ||||||||
| 1031 | BO->setIsExact(); | ||||||||
| 1032 | return BO; | ||||||||
| 1033 | } | ||||||||
| 1034 | } | ||||||||
| 1035 | |||||||||
| 1036 | // Op0 / C where C is large (negative) --> zext (Op0 >= C) | ||||||||
| 1037 | // TODO: Could use isKnownNegative() to handle non-constant values. | ||||||||
| 1038 | Type *Ty = I.getType(); | ||||||||
| 1039 | if (match(Op1, m_Negative())) { | ||||||||
| 1040 | Value *Cmp = Builder.CreateICmpUGE(Op0, Op1); | ||||||||
| 1041 | return CastInst::CreateZExtOrBitCast(Cmp, Ty); | ||||||||
| 1042 | } | ||||||||
| 1043 | // Op0 / (sext i1 X) --> zext (Op0 == -1) (if X is 0, the div is undefined) | ||||||||
| 1044 | if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) { | ||||||||
| 1045 | Value *Cmp = Builder.CreateICmpEQ(Op0, ConstantInt::getAllOnesValue(Ty)); | ||||||||
| 1046 | return CastInst::CreateZExtOrBitCast(Cmp, Ty); | ||||||||
| 1047 | } | ||||||||
| 1048 | |||||||||
| 1049 | if (Instruction *NarrowDiv = narrowUDivURem(I, Builder)) | ||||||||
| 1050 | return NarrowDiv; | ||||||||
| 1051 | |||||||||
| 1052 | // If the udiv operands are non-overflowing multiplies with a common operand, | ||||||||
| 1053 | // then eliminate the common factor: | ||||||||
| 1054 | // (A * B) / (A * X) --> B / X (and commuted variants) | ||||||||
| 1055 | // TODO: The code would be reduced if we had m_c_NUWMul pattern matching. | ||||||||
| 1056 | // TODO: If -reassociation handled this generally, we could remove this. | ||||||||
| 1057 | Value *A, *B; | ||||||||
| 1058 | if (match(Op0, m_NUWMul(m_Value(A), m_Value(B)))) { | ||||||||
| 1059 | if (match(Op1, m_NUWMul(m_Specific(A), m_Value(X))) || | ||||||||
| 1060 | match(Op1, m_NUWMul(m_Value(X), m_Specific(A)))) | ||||||||
| 1061 | return BinaryOperator::CreateUDiv(B, X); | ||||||||
| 1062 | if (match(Op1, m_NUWMul(m_Specific(B), m_Value(X))) || | ||||||||
| 1063 | match(Op1, m_NUWMul(m_Value(X), m_Specific(B)))) | ||||||||
| 1064 | return BinaryOperator::CreateUDiv(A, X); | ||||||||
| 1065 | } | ||||||||
| 1066 | |||||||||
| 1067 | // (LHS udiv (select (select (...)))) -> (LHS >> (select (select (...)))) | ||||||||
| 1068 | SmallVector<UDivFoldAction, 6> UDivActions; | ||||||||
| 1069 | if (visitUDivOperand(Op0, Op1, I, UDivActions)) | ||||||||
| 1070 | for (unsigned i = 0, e = UDivActions.size(); i != e; ++i) { | ||||||||
| 1071 | FoldUDivOperandCb Action = UDivActions[i].FoldAction; | ||||||||
| 1072 | Value *ActionOp1 = UDivActions[i].OperandToFold; | ||||||||
| 1073 | Instruction *Inst; | ||||||||
| 1074 | if (Action) | ||||||||
| 1075 | Inst = Action(Op0, ActionOp1, I, *this); | ||||||||
| 1076 | else { | ||||||||
| 1077 | // This action joins two actions together. The RHS of this action is | ||||||||
| 1078 | // simply the last action we processed, we saved the LHS action index in | ||||||||
| 1079 | // the joining action. | ||||||||
| 1080 | size_t SelectRHSIdx = i - 1; | ||||||||
| 1081 | Value *SelectRHS = UDivActions[SelectRHSIdx].FoldResult; | ||||||||
| 1082 | size_t SelectLHSIdx = UDivActions[i].SelectLHSIdx; | ||||||||
| 1083 | Value *SelectLHS = UDivActions[SelectLHSIdx].FoldResult; | ||||||||
| 1084 | Inst = SelectInst::Create(cast<SelectInst>(ActionOp1)->getCondition(), | ||||||||
| 1085 | SelectLHS, SelectRHS); | ||||||||
| 1086 | } | ||||||||
| 1087 | |||||||||
| 1088 | // If this is the last action to process, return it to the InstCombiner. | ||||||||
| 1089 | // Otherwise, we insert it before the UDiv and record it so that we may | ||||||||
| 1090 | // use it as part of a joining action (i.e., a SelectInst). | ||||||||
| 1091 | if (e - i != 1) { | ||||||||
| 1092 | Inst->insertBefore(&I); | ||||||||
| 1093 | UDivActions[i].FoldResult = Inst; | ||||||||
| 1094 | } else | ||||||||
| 1095 | return Inst; | ||||||||
| 1096 | } | ||||||||
| 1097 | |||||||||
| 1098 | return nullptr; | ||||||||
| 1099 | } | ||||||||
| 1100 | |||||||||
| 1101 | Instruction *InstCombinerImpl::visitSDiv(BinaryOperator &I) { | ||||||||
| 1102 | if (Value *V = SimplifySDivInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 1103 | SQ.getWithInstruction(&I))) | ||||||||
| 1104 | return replaceInstUsesWith(I, V); | ||||||||
| 1105 | |||||||||
| 1106 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 1107 | return X; | ||||||||
| 1108 | |||||||||
| 1109 | // Handle the integer div common cases | ||||||||
| 1110 | if (Instruction *Common = commonIDivTransforms(I)) | ||||||||
| 1111 | return Common; | ||||||||
| 1112 | |||||||||
| 1113 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1114 | Type *Ty = I.getType(); | ||||||||
| 1115 | Value *X; | ||||||||
| 1116 | // sdiv Op0, -1 --> -Op0 | ||||||||
| 1117 | // sdiv Op0, (sext i1 X) --> -Op0 (because if X is 0, the op is undefined) | ||||||||
| 1118 | if (match(Op1, m_AllOnes()) || | ||||||||
| 1119 | (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))) | ||||||||
| 1120 | return BinaryOperator::CreateNeg(Op0); | ||||||||
| 1121 | |||||||||
| 1122 | // X / INT_MIN --> X == INT_MIN | ||||||||
| 1123 | if (match(Op1, m_SignMask())) | ||||||||
| 1124 | return new ZExtInst(Builder.CreateICmpEQ(Op0, Op1), Ty); | ||||||||
| 1125 | |||||||||
| 1126 | // sdiv exact X, 1<<C --> ashr exact X, C iff 1<<C is non-negative | ||||||||
| 1127 | // sdiv exact X, -1<<C --> -(ashr exact X, C) | ||||||||
| 1128 | if (I.isExact() && ((match(Op1, m_Power2()) && match(Op1, m_NonNegative())) || | ||||||||
| 1129 | match(Op1, m_NegatedPower2()))) { | ||||||||
| 1130 | bool DivisorWasNegative = match(Op1, m_NegatedPower2()); | ||||||||
| 1131 | if (DivisorWasNegative) | ||||||||
| 1132 | Op1 = ConstantExpr::getNeg(cast<Constant>(Op1)); | ||||||||
| 1133 | auto *AShr = BinaryOperator::CreateExactAShr( | ||||||||
| 1134 | Op0, ConstantExpr::getExactLogBase2(cast<Constant>(Op1)), I.getName()); | ||||||||
| 1135 | if (!DivisorWasNegative) | ||||||||
| 1136 | return AShr; | ||||||||
| 1137 | Builder.Insert(AShr); | ||||||||
| 1138 | AShr->setName(I.getName() + ".neg"); | ||||||||
| 1139 | return BinaryOperator::CreateNeg(AShr, I.getName()); | ||||||||
| 1140 | } | ||||||||
| 1141 | |||||||||
| 1142 | const APInt *Op1C; | ||||||||
| 1143 | if (match(Op1, m_APInt(Op1C))) { | ||||||||
| 1144 | // If the dividend is sign-extended and the constant divisor is small enough | ||||||||
| 1145 | // to fit in the source type, shrink the division to the narrower type: | ||||||||
| 1146 | // (sext X) sdiv C --> sext (X sdiv C) | ||||||||
| 1147 | Value *Op0Src; | ||||||||
| 1148 | if (match(Op0, m_OneUse(m_SExt(m_Value(Op0Src)))) && | ||||||||
| 1149 | Op0Src->getType()->getScalarSizeInBits() >= Op1C->getMinSignedBits()) { | ||||||||
| 1150 | |||||||||
| 1151 | // In the general case, we need to make sure that the dividend is not the | ||||||||
| 1152 | // minimum signed value because dividing that by -1 is UB. But here, we | ||||||||
| 1153 | // know that the -1 divisor case is already handled above. | ||||||||
| 1154 | |||||||||
| 1155 | Constant *NarrowDivisor = | ||||||||
| 1156 | ConstantExpr::getTrunc(cast<Constant>(Op1), Op0Src->getType()); | ||||||||
| 1157 | Value *NarrowOp = Builder.CreateSDiv(Op0Src, NarrowDivisor); | ||||||||
| 1158 | return new SExtInst(NarrowOp, Ty); | ||||||||
| 1159 | } | ||||||||
| 1160 | |||||||||
| 1161 | // -X / C --> X / -C (if the negation doesn't overflow). | ||||||||
| 1162 | // TODO: This could be enhanced to handle arbitrary vector constants by | ||||||||
| 1163 | // checking if all elements are not the min-signed-val. | ||||||||
| 1164 | if (!Op1C->isMinSignedValue() && | ||||||||
| 1165 | match(Op0, m_NSWSub(m_Zero(), m_Value(X)))) { | ||||||||
| 1166 | Constant *NegC = ConstantInt::get(Ty, -(*Op1C)); | ||||||||
| 1167 | Instruction *BO = BinaryOperator::CreateSDiv(X, NegC); | ||||||||
| 1168 | BO->setIsExact(I.isExact()); | ||||||||
| 1169 | return BO; | ||||||||
| 1170 | } | ||||||||
| 1171 | } | ||||||||
| 1172 | |||||||||
| 1173 | // -X / Y --> -(X / Y) | ||||||||
| 1174 | Value *Y; | ||||||||
| 1175 | if (match(&I, m_SDiv(m_OneUse(m_NSWSub(m_Zero(), m_Value(X))), m_Value(Y)))) | ||||||||
| 1176 | return BinaryOperator::CreateNSWNeg( | ||||||||
| 1177 | Builder.CreateSDiv(X, Y, I.getName(), I.isExact())); | ||||||||
| 1178 | |||||||||
| 1179 | // abs(X) / X --> X > -1 ? 1 : -1 | ||||||||
| 1180 | // X / abs(X) --> X > -1 ? 1 : -1 | ||||||||
| 1181 | if (match(&I, m_c_BinOp( | ||||||||
| 1182 | m_OneUse(m_Intrinsic<Intrinsic::abs>(m_Value(X), m_One())), | ||||||||
| 1183 | m_Deferred(X)))) { | ||||||||
| 1184 | Constant *NegOne = ConstantInt::getAllOnesValue(Ty); | ||||||||
| 1185 | Value *Cond = Builder.CreateICmpSGT(X, NegOne); | ||||||||
| 1186 | return SelectInst::Create(Cond, ConstantInt::get(Ty, 1), NegOne); | ||||||||
| 1187 | } | ||||||||
| 1188 | |||||||||
| 1189 | // If the sign bits of both operands are zero (i.e. we can prove they are | ||||||||
| 1190 | // unsigned inputs), turn this into a udiv. | ||||||||
| 1191 | APInt Mask(APInt::getSignMask(Ty->getScalarSizeInBits())); | ||||||||
| 1192 | if (MaskedValueIsZero(Op0, Mask, 0, &I)) { | ||||||||
| 1193 | if (MaskedValueIsZero(Op1, Mask, 0, &I)) { | ||||||||
| 1194 | // X sdiv Y -> X udiv Y, iff X and Y don't have sign bit set | ||||||||
| 1195 | auto *BO = BinaryOperator::CreateUDiv(Op0, Op1, I.getName()); | ||||||||
| 1196 | BO->setIsExact(I.isExact()); | ||||||||
| 1197 | return BO; | ||||||||
| 1198 | } | ||||||||
| 1199 | |||||||||
| 1200 | if (match(Op1, m_NegatedPower2())) { | ||||||||
| 1201 | // X sdiv (-(1 << C)) -> -(X sdiv (1 << C)) -> | ||||||||
| 1202 | // -> -(X udiv (1 << C)) -> -(X u>> C) | ||||||||
| 1203 | return BinaryOperator::CreateNeg(Builder.Insert(foldUDivPow2Cst( | ||||||||
| 1204 | Op0, ConstantExpr::getNeg(cast<Constant>(Op1)), I, *this))); | ||||||||
| 1205 | } | ||||||||
| 1206 | |||||||||
| 1207 | if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, &I)) { | ||||||||
| 1208 | // X sdiv (1 << Y) -> X udiv (1 << Y) ( -> X u>> Y) | ||||||||
| 1209 | // Safe because the only negative value (1 << Y) can take on is | ||||||||
| 1210 | // INT_MIN, and X sdiv INT_MIN == X udiv INT_MIN == 0 if X doesn't have | ||||||||
| 1211 | // the sign bit set. | ||||||||
| 1212 | auto *BO = BinaryOperator::CreateUDiv(Op0, Op1, I.getName()); | ||||||||
| 1213 | BO->setIsExact(I.isExact()); | ||||||||
| 1214 | return BO; | ||||||||
| 1215 | } | ||||||||
| 1216 | } | ||||||||
| 1217 | |||||||||
| 1218 | return nullptr; | ||||||||
| 1219 | } | ||||||||
| 1220 | |||||||||
| 1221 | /// Remove negation and try to convert division into multiplication. | ||||||||
| 1222 | static Instruction *foldFDivConstantDivisor(BinaryOperator &I) { | ||||||||
| 1223 | Constant *C; | ||||||||
| 1224 | if (!match(I.getOperand(1), m_Constant(C))) | ||||||||
| 1225 | return nullptr; | ||||||||
| 1226 | |||||||||
| 1227 | // -X / C --> X / -C | ||||||||
| 1228 | Value *X; | ||||||||
| 1229 | if (match(I.getOperand(0), m_FNeg(m_Value(X)))) | ||||||||
| 1230 | return BinaryOperator::CreateFDivFMF(X, ConstantExpr::getFNeg(C), &I); | ||||||||
| 1231 | |||||||||
| 1232 | // If the constant divisor has an exact inverse, this is always safe. If not, | ||||||||
| 1233 | // then we can still create a reciprocal if fast-math-flags allow it and the | ||||||||
| 1234 | // constant is a regular number (not zero, infinite, or denormal). | ||||||||
| 1235 | if (!(C->hasExactInverseFP() || (I.hasAllowReciprocal() && C->isNormalFP()))) | ||||||||
| 1236 | return nullptr; | ||||||||
| 1237 | |||||||||
| 1238 | // Disallow denormal constants because we don't know what would happen | ||||||||
| 1239 | // on all targets. | ||||||||
| 1240 | // TODO: Use Intrinsic::canonicalize or let function attributes tell us that | ||||||||
| 1241 | // denorms are flushed? | ||||||||
| 1242 | auto *RecipC = ConstantExpr::getFDiv(ConstantFP::get(I.getType(), 1.0), C); | ||||||||
| 1243 | if (!RecipC->isNormalFP()) | ||||||||
| 1244 | return nullptr; | ||||||||
| 1245 | |||||||||
| 1246 | // X / C --> X * (1 / C) | ||||||||
| 1247 | return BinaryOperator::CreateFMulFMF(I.getOperand(0), RecipC, &I); | ||||||||
| 1248 | } | ||||||||
| 1249 | |||||||||
| 1250 | /// Remove negation and try to reassociate constant math. | ||||||||
| 1251 | static Instruction *foldFDivConstantDividend(BinaryOperator &I) { | ||||||||
| 1252 | Constant *C; | ||||||||
| 1253 | if (!match(I.getOperand(0), m_Constant(C))) | ||||||||
| 1254 | return nullptr; | ||||||||
| 1255 | |||||||||
| 1256 | // C / -X --> -C / X | ||||||||
| 1257 | Value *X; | ||||||||
| 1258 | if (match(I.getOperand(1), m_FNeg(m_Value(X)))) | ||||||||
| 1259 | return BinaryOperator::CreateFDivFMF(ConstantExpr::getFNeg(C), X, &I); | ||||||||
| 1260 | |||||||||
| 1261 | if (!I.hasAllowReassoc() || !I.hasAllowReciprocal()) | ||||||||
| 1262 | return nullptr; | ||||||||
| 1263 | |||||||||
| 1264 | // Try to reassociate C / X expressions where X includes another constant. | ||||||||
| 1265 | Constant *C2, *NewC = nullptr; | ||||||||
| 1266 | if (match(I.getOperand(1), m_FMul(m_Value(X), m_Constant(C2)))) { | ||||||||
| 1267 | // C / (X * C2) --> (C / C2) / X | ||||||||
| 1268 | NewC = ConstantExpr::getFDiv(C, C2); | ||||||||
| 1269 | } else if (match(I.getOperand(1), m_FDiv(m_Value(X), m_Constant(C2)))) { | ||||||||
| 1270 | // C / (X / C2) --> (C * C2) / X | ||||||||
| 1271 | NewC = ConstantExpr::getFMul(C, C2); | ||||||||
| 1272 | } | ||||||||
| 1273 | // Disallow denormal constants because we don't know what would happen | ||||||||
| 1274 | // on all targets. | ||||||||
| 1275 | // TODO: Use Intrinsic::canonicalize or let function attributes tell us that | ||||||||
| 1276 | // denorms are flushed? | ||||||||
| 1277 | if (!NewC || !NewC->isNormalFP()) | ||||||||
| 1278 | return nullptr; | ||||||||
| 1279 | |||||||||
| 1280 | return BinaryOperator::CreateFDivFMF(NewC, X, &I); | ||||||||
| 1281 | } | ||||||||
| 1282 | |||||||||
| 1283 | /// Negate the exponent of pow/exp to fold division-by-pow() into multiply. | ||||||||
| 1284 | static Instruction *foldFDivPowDivisor(BinaryOperator &I, | ||||||||
| 1285 | InstCombiner::BuilderTy &Builder) { | ||||||||
| 1286 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1287 | auto *II = dyn_cast<IntrinsicInst>(Op1); | ||||||||
| 1288 | if (!II || !II->hasOneUse() || !I.hasAllowReassoc() || | ||||||||
| 1289 | !I.hasAllowReciprocal()) | ||||||||
| 1290 | return nullptr; | ||||||||
| 1291 | |||||||||
| 1292 | // Z / pow(X, Y) --> Z * pow(X, -Y) | ||||||||
| 1293 | // Z / exp{2}(Y) --> Z * exp{2}(-Y) | ||||||||
| 1294 | // In the general case, this creates an extra instruction, but fmul allows | ||||||||
| 1295 | // for better canonicalization and optimization than fdiv. | ||||||||
| 1296 | Intrinsic::ID IID = II->getIntrinsicID(); | ||||||||
| 1297 | SmallVector<Value *> Args; | ||||||||
| 1298 | switch (IID) { | ||||||||
| 1299 | case Intrinsic::pow: | ||||||||
| 1300 | Args.push_back(II->getArgOperand(0)); | ||||||||
| 1301 | Args.push_back(Builder.CreateFNegFMF(II->getArgOperand(1), &I)); | ||||||||
| 1302 | break; | ||||||||
| 1303 | case Intrinsic::powi: { | ||||||||
| 1304 | // Require 'ninf' assuming that makes powi(X, -INT_MIN) acceptable. | ||||||||
| 1305 | // That is, X ** (huge negative number) is 0.0, ~1.0, or INF and so | ||||||||
| 1306 | // dividing by that is INF, ~1.0, or 0.0. Code that uses powi allows | ||||||||
| 1307 | // non-standard results, so this corner case should be acceptable if the | ||||||||
| 1308 | // code rules out INF values. | ||||||||
| 1309 | if (!I.hasNoInfs()) | ||||||||
| 1310 | return nullptr; | ||||||||
| 1311 | Args.push_back(II->getArgOperand(0)); | ||||||||
| 1312 | Args.push_back(Builder.CreateNeg(II->getArgOperand(1))); | ||||||||
| 1313 | Type *Tys[] = {I.getType(), II->getArgOperand(1)->getType()}; | ||||||||
| 1314 | Value *Pow = Builder.CreateIntrinsic(IID, Tys, Args, &I); | ||||||||
| 1315 | return BinaryOperator::CreateFMulFMF(Op0, Pow, &I); | ||||||||
| 1316 | } | ||||||||
| 1317 | case Intrinsic::exp: | ||||||||
| 1318 | case Intrinsic::exp2: | ||||||||
| 1319 | Args.push_back(Builder.CreateFNegFMF(II->getArgOperand(0), &I)); | ||||||||
| 1320 | break; | ||||||||
| 1321 | default: | ||||||||
| 1322 | return nullptr; | ||||||||
| 1323 | } | ||||||||
| 1324 | Value *Pow = Builder.CreateIntrinsic(IID, I.getType(), Args, &I); | ||||||||
| 1325 | return BinaryOperator::CreateFMulFMF(Op0, Pow, &I); | ||||||||
| 1326 | } | ||||||||
| 1327 | |||||||||
| 1328 | Instruction *InstCombinerImpl::visitFDiv(BinaryOperator &I) { | ||||||||
| 1329 | if (Value *V = SimplifyFDivInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 1330 | I.getFastMathFlags(), | ||||||||
| 1331 | SQ.getWithInstruction(&I))) | ||||||||
| 1332 | return replaceInstUsesWith(I, V); | ||||||||
| 1333 | |||||||||
| 1334 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 1335 | return X; | ||||||||
| 1336 | |||||||||
| 1337 | if (Instruction *R = foldFDivConstantDivisor(I)) | ||||||||
| 1338 | return R; | ||||||||
| 1339 | |||||||||
| 1340 | if (Instruction *R = foldFDivConstantDividend(I)) | ||||||||
| 1341 | return R; | ||||||||
| 1342 | |||||||||
| 1343 | if (Instruction *R = foldFPSignBitOps(I)) | ||||||||
| 1344 | return R; | ||||||||
| 1345 | |||||||||
| 1346 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1347 | if (isa<Constant>(Op0)) | ||||||||
| 1348 | if (SelectInst *SI = dyn_cast<SelectInst>(Op1)) | ||||||||
| 1349 | if (Instruction *R = FoldOpIntoSelect(I, SI)) | ||||||||
| 1350 | return R; | ||||||||
| 1351 | |||||||||
| 1352 | if (isa<Constant>(Op1)) | ||||||||
| 1353 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0)) | ||||||||
| 1354 | if (Instruction *R = FoldOpIntoSelect(I, SI)) | ||||||||
| 1355 | return R; | ||||||||
| 1356 | |||||||||
| 1357 | if (I.hasAllowReassoc() && I.hasAllowReciprocal()) { | ||||||||
| 1358 | Value *X, *Y; | ||||||||
| 1359 | if (match(Op0, m_OneUse(m_FDiv(m_Value(X), m_Value(Y)))) && | ||||||||
| 1360 | (!isa<Constant>(Y) || !isa<Constant>(Op1))) { | ||||||||
| 1361 | // (X / Y) / Z => X / (Y * Z) | ||||||||
| 1362 | Value *YZ = Builder.CreateFMulFMF(Y, Op1, &I); | ||||||||
| 1363 | return BinaryOperator::CreateFDivFMF(X, YZ, &I); | ||||||||
| 1364 | } | ||||||||
| 1365 | if (match(Op1, m_OneUse(m_FDiv(m_Value(X), m_Value(Y)))) && | ||||||||
| 1366 | (!isa<Constant>(Y) || !isa<Constant>(Op0))) { | ||||||||
| 1367 | // Z / (X / Y) => (Y * Z) / X | ||||||||
| 1368 | Value *YZ = Builder.CreateFMulFMF(Y, Op0, &I); | ||||||||
| 1369 | return BinaryOperator::CreateFDivFMF(YZ, X, &I); | ||||||||
| 1370 | } | ||||||||
| 1371 | // Z / (1.0 / Y) => (Y * Z) | ||||||||
| 1372 | // | ||||||||
| 1373 | // This is a special case of Z / (X / Y) => (Y * Z) / X, with X = 1.0. The | ||||||||
| 1374 | // m_OneUse check is avoided because even in the case of the multiple uses | ||||||||
| 1375 | // for 1.0/Y, the number of instructions remain the same and a division is | ||||||||
| 1376 | // replaced by a multiplication. | ||||||||
| 1377 | if (match(Op1, m_FDiv(m_SpecificFP(1.0), m_Value(Y)))) | ||||||||
| 1378 | return BinaryOperator::CreateFMulFMF(Y, Op0, &I); | ||||||||
| 1379 | } | ||||||||
| 1380 | |||||||||
| 1381 | if (I.hasAllowReassoc() && Op0->hasOneUse() && Op1->hasOneUse()) { | ||||||||
| 1382 | // sin(X) / cos(X) -> tan(X) | ||||||||
| 1383 | // cos(X) / sin(X) -> 1/tan(X) (cotangent) | ||||||||
| 1384 | Value *X; | ||||||||
| 1385 | bool IsTan = match(Op0, m_Intrinsic<Intrinsic::sin>(m_Value(X))) && | ||||||||
| 1386 | match(Op1, m_Intrinsic<Intrinsic::cos>(m_Specific(X))); | ||||||||
| 1387 | bool IsCot = | ||||||||
| 1388 | !IsTan && match(Op0, m_Intrinsic<Intrinsic::cos>(m_Value(X))) && | ||||||||
| 1389 | match(Op1, m_Intrinsic<Intrinsic::sin>(m_Specific(X))); | ||||||||
| 1390 | |||||||||
| 1391 | if ((IsTan || IsCot) && | ||||||||
| 1392 | hasFloatFn(&TLI, I.getType(), LibFunc_tan, LibFunc_tanf, LibFunc_tanl)) { | ||||||||
| 1393 | IRBuilder<> B(&I); | ||||||||
| 1394 | IRBuilder<>::FastMathFlagGuard FMFGuard(B); | ||||||||
| 1395 | B.setFastMathFlags(I.getFastMathFlags()); | ||||||||
| 1396 | AttributeList Attrs = | ||||||||
| 1397 | cast<CallBase>(Op0)->getCalledFunction()->getAttributes(); | ||||||||
| 1398 | Value *Res = emitUnaryFloatFnCall(X, &TLI, LibFunc_tan, LibFunc_tanf, | ||||||||
| 1399 | LibFunc_tanl, B, Attrs); | ||||||||
| 1400 | if (IsCot) | ||||||||
| 1401 | Res = B.CreateFDiv(ConstantFP::get(I.getType(), 1.0), Res); | ||||||||
| 1402 | return replaceInstUsesWith(I, Res); | ||||||||
| 1403 | } | ||||||||
| 1404 | } | ||||||||
| 1405 | |||||||||
| 1406 | // X / (X * Y) --> 1.0 / Y | ||||||||
| 1407 | // Reassociate to (X / X -> 1.0) is legal when NaNs are not allowed. | ||||||||
| 1408 | // We can ignore the possibility that X is infinity because INF/INF is NaN. | ||||||||
| 1409 | Value *X, *Y; | ||||||||
| 1410 | if (I.hasNoNaNs() && I.hasAllowReassoc() && | ||||||||
| 1411 | match(Op1, m_c_FMul(m_Specific(Op0), m_Value(Y)))) { | ||||||||
| 1412 | replaceOperand(I, 0, ConstantFP::get(I.getType(), 1.0)); | ||||||||
| 1413 | replaceOperand(I, 1, Y); | ||||||||
| 1414 | return &I; | ||||||||
| 1415 | } | ||||||||
| 1416 | |||||||||
| 1417 | // X / fabs(X) -> copysign(1.0, X) | ||||||||
| 1418 | // fabs(X) / X -> copysign(1.0, X) | ||||||||
| 1419 | if (I.hasNoNaNs() && I.hasNoInfs() && | ||||||||
| 1420 | (match(&I, m_FDiv(m_Value(X), m_FAbs(m_Deferred(X)))) || | ||||||||
| 1421 | match(&I, m_FDiv(m_FAbs(m_Value(X)), m_Deferred(X))))) { | ||||||||
| 1422 | Value *V = Builder.CreateBinaryIntrinsic( | ||||||||
| 1423 | Intrinsic::copysign, ConstantFP::get(I.getType(), 1.0), X, &I); | ||||||||
| 1424 | return replaceInstUsesWith(I, V); | ||||||||
| 1425 | } | ||||||||
| 1426 | |||||||||
| 1427 | if (Instruction *Mul = foldFDivPowDivisor(I, Builder)) | ||||||||
| 1428 | return Mul; | ||||||||
| 1429 | |||||||||
| 1430 | return nullptr; | ||||||||
| 1431 | } | ||||||||
| 1432 | |||||||||
| 1433 | /// This function implements the transforms common to both integer remainder | ||||||||
| 1434 | /// instructions (urem and srem). It is called by the visitors to those integer | ||||||||
| 1435 | /// remainder instructions. | ||||||||
| 1436 | /// Common integer remainder transforms | ||||||||
| 1437 | Instruction *InstCombinerImpl::commonIRemTransforms(BinaryOperator &I) { | ||||||||
| 1438 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1439 | |||||||||
| 1440 | // The RHS is known non-zero. | ||||||||
| 1441 | if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I)) | ||||||||
| 1442 | return replaceOperand(I, 1, V); | ||||||||
| 1443 | |||||||||
| 1444 | // Handle cases involving: rem X, (select Cond, Y, Z) | ||||||||
| 1445 | if (simplifyDivRemOfSelectWithZeroOp(I)) | ||||||||
| 1446 | return &I; | ||||||||
| 1447 | |||||||||
| 1448 | if (isa<Constant>(Op1)) { | ||||||||
| 1449 | if (Instruction *Op0I = dyn_cast<Instruction>(Op0)) { | ||||||||
| 1450 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0I)) { | ||||||||
| 1451 | if (Instruction *R = FoldOpIntoSelect(I, SI)) | ||||||||
| 1452 | return R; | ||||||||
| 1453 | } else if (auto *PN = dyn_cast<PHINode>(Op0I)) { | ||||||||
| 1454 | const APInt *Op1Int; | ||||||||
| 1455 | if (match(Op1, m_APInt(Op1Int)) && !Op1Int->isMinValue() && | ||||||||
| 1456 | (I.getOpcode() == Instruction::URem || | ||||||||
| 1457 | !Op1Int->isMinSignedValue())) { | ||||||||
| 1458 | // foldOpIntoPhi will speculate instructions to the end of the PHI's | ||||||||
| 1459 | // predecessor blocks, so do this only if we know the srem or urem | ||||||||
| 1460 | // will not fault. | ||||||||
| 1461 | if (Instruction *NV = foldOpIntoPhi(I, PN)) | ||||||||
| 1462 | return NV; | ||||||||
| 1463 | } | ||||||||
| 1464 | } | ||||||||
| 1465 | |||||||||
| 1466 | // See if we can fold away this rem instruction. | ||||||||
| 1467 | if (SimplifyDemandedInstructionBits(I)) | ||||||||
| 1468 | return &I; | ||||||||
| 1469 | } | ||||||||
| 1470 | } | ||||||||
| 1471 | |||||||||
| 1472 | return nullptr; | ||||||||
| 1473 | } | ||||||||
| 1474 | |||||||||
| 1475 | Instruction *InstCombinerImpl::visitURem(BinaryOperator &I) { | ||||||||
| 1476 | if (Value *V = SimplifyURemInst(I.getOperand(0), I.getOperand(1), | ||||||||
| |||||||||
| 1477 | SQ.getWithInstruction(&I))) | ||||||||
| 1478 | return replaceInstUsesWith(I, V); | ||||||||
| 1479 | |||||||||
| 1480 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 1481 | return X; | ||||||||
| 1482 | |||||||||
| 1483 | if (Instruction *common = commonIRemTransforms(I)) | ||||||||
| 1484 | return common; | ||||||||
| 1485 | |||||||||
| 1486 | if (Instruction *NarrowRem = narrowUDivURem(I, Builder)) | ||||||||
| 1487 | return NarrowRem; | ||||||||
| 1488 | |||||||||
| 1489 | // X urem Y -> X and Y-1, where Y is a power of 2, | ||||||||
| 1490 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1491 | Type *Ty = I.getType(); | ||||||||
| 1492 | if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/ true, 0, &I)) { | ||||||||
| 1493 | // This may increase instruction count, we don't enforce that Y is a | ||||||||
| 1494 | // constant. | ||||||||
| 1495 | Constant *N1 = Constant::getAllOnesValue(Ty); | ||||||||
| 1496 | Value *Add = Builder.CreateAdd(Op1, N1); | ||||||||
| 1497 | return BinaryOperator::CreateAnd(Op0, Add); | ||||||||
| 1498 | } | ||||||||
| 1499 | |||||||||
| 1500 | // 1 urem X -> zext(X != 1) | ||||||||
| 1501 | if (match(Op0, m_One())) { | ||||||||
| 1502 | Value *Cmp = Builder.CreateICmpNE(Op1, ConstantInt::get(Ty, 1)); | ||||||||
| 1503 | return CastInst::CreateZExtOrBitCast(Cmp, Ty); | ||||||||
| 1504 | } | ||||||||
| 1505 | |||||||||
| 1506 | // X urem C -> X < C ? X : X - C, where C >= signbit. | ||||||||
| 1507 | if (match(Op1, m_Negative())) { | ||||||||
| 1508 | Value *Cmp = Builder.CreateICmpULT(Op0, Op1); | ||||||||
| 1509 | Value *Sub = Builder.CreateSub(Op0, Op1); | ||||||||
| 1510 | return SelectInst::Create(Cmp, Op0, Sub); | ||||||||
| 1511 | } | ||||||||
| 1512 | |||||||||
| 1513 | // If the divisor is a sext of a boolean, then the divisor must be max | ||||||||
| 1514 | // unsigned value (-1). Therefore, the remainder is Op0 unless Op0 is also | ||||||||
| 1515 | // max unsigned value. In that case, the remainder is 0: | ||||||||
| 1516 | // urem Op0, (sext i1 X) --> (Op0 == -1) ? 0 : Op0 | ||||||||
| 1517 | Value *X; | ||||||||
| 1518 | if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) { | ||||||||
| 1519 | Value *Cmp = Builder.CreateICmpEQ(Op0, ConstantInt::getAllOnesValue(Ty)); | ||||||||
| 1520 | return SelectInst::Create(Cmp, ConstantInt::getNullValue(Ty), Op0); | ||||||||
| 1521 | } | ||||||||
| 1522 | |||||||||
| 1523 | return nullptr; | ||||||||
| 1524 | } | ||||||||
| 1525 | |||||||||
| 1526 | Instruction *InstCombinerImpl::visitSRem(BinaryOperator &I) { | ||||||||
| 1527 | if (Value *V = SimplifySRemInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 1528 | SQ.getWithInstruction(&I))) | ||||||||
| 1529 | return replaceInstUsesWith(I, V); | ||||||||
| 1530 | |||||||||
| 1531 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 1532 | return X; | ||||||||
| 1533 | |||||||||
| 1534 | // Handle the integer rem common cases | ||||||||
| 1535 | if (Instruction *Common = commonIRemTransforms(I)) | ||||||||
| 1536 | return Common; | ||||||||
| 1537 | |||||||||
| 1538 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); | ||||||||
| 1539 | { | ||||||||
| 1540 | const APInt *Y; | ||||||||
| 1541 | // X % -Y -> X % Y | ||||||||
| 1542 | if (match(Op1, m_Negative(Y)) && !Y->isMinSignedValue()) | ||||||||
| 1543 | return replaceOperand(I, 1, ConstantInt::get(I.getType(), -*Y)); | ||||||||
| 1544 | } | ||||||||
| 1545 | |||||||||
| 1546 | // -X srem Y --> -(X srem Y) | ||||||||
| 1547 | Value *X, *Y; | ||||||||
| 1548 | if (match(&I, m_SRem(m_OneUse(m_NSWSub(m_Zero(), m_Value(X))), m_Value(Y)))) | ||||||||
| 1549 | return BinaryOperator::CreateNSWNeg(Builder.CreateSRem(X, Y)); | ||||||||
| 1550 | |||||||||
| 1551 | // If the sign bits of both operands are zero (i.e. we can prove they are | ||||||||
| 1552 | // unsigned inputs), turn this into a urem. | ||||||||
| 1553 | APInt Mask(APInt::getSignMask(I.getType()->getScalarSizeInBits())); | ||||||||
| 1554 | if (MaskedValueIsZero(Op1, Mask, 0, &I) && | ||||||||
| 1555 | MaskedValueIsZero(Op0, Mask, 0, &I)) { | ||||||||
| 1556 | // X srem Y -> X urem Y, iff X and Y don't have sign bit set | ||||||||
| 1557 | return BinaryOperator::CreateURem(Op0, Op1, I.getName()); | ||||||||
| 1558 | } | ||||||||
| 1559 | |||||||||
| 1560 | // If it's a constant vector, flip any negative values positive. | ||||||||
| 1561 | if (isa<ConstantVector>(Op1) || isa<ConstantDataVector>(Op1)) { | ||||||||
| 1562 | Constant *C = cast<Constant>(Op1); | ||||||||
| 1563 | unsigned VWidth = cast<FixedVectorType>(C->getType())->getNumElements(); | ||||||||
| 1564 | |||||||||
| 1565 | bool hasNegative = false; | ||||||||
| 1566 | bool hasMissing = false; | ||||||||
| 1567 | for (unsigned i = 0; i != VWidth; ++i) { | ||||||||
| 1568 | Constant *Elt = C->getAggregateElement(i); | ||||||||
| 1569 | if (!Elt) { | ||||||||
| 1570 | hasMissing = true; | ||||||||
| 1571 | break; | ||||||||
| 1572 | } | ||||||||
| 1573 | |||||||||
| 1574 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Elt)) | ||||||||
| 1575 | if (RHS->isNegative()) | ||||||||
| 1576 | hasNegative = true; | ||||||||
| 1577 | } | ||||||||
| 1578 | |||||||||
| 1579 | if (hasNegative && !hasMissing) { | ||||||||
| 1580 | SmallVector<Constant *, 16> Elts(VWidth); | ||||||||
| 1581 | for (unsigned i = 0; i != VWidth; ++i) { | ||||||||
| 1582 | Elts[i] = C->getAggregateElement(i); // Handle undef, etc. | ||||||||
| 1583 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Elts[i])) { | ||||||||
| 1584 | if (RHS->isNegative()) | ||||||||
| 1585 | Elts[i] = cast<ConstantInt>(ConstantExpr::getNeg(RHS)); | ||||||||
| 1586 | } | ||||||||
| 1587 | } | ||||||||
| 1588 | |||||||||
| 1589 | Constant *NewRHSV = ConstantVector::get(Elts); | ||||||||
| 1590 | if (NewRHSV != C) // Don't loop on -MININT | ||||||||
| 1591 | return replaceOperand(I, 1, NewRHSV); | ||||||||
| 1592 | } | ||||||||
| 1593 | } | ||||||||
| 1594 | |||||||||
| 1595 | return nullptr; | ||||||||
| 1596 | } | ||||||||
| 1597 | |||||||||
| 1598 | Instruction *InstCombinerImpl::visitFRem(BinaryOperator &I) { | ||||||||
| 1599 | if (Value *V = SimplifyFRemInst(I.getOperand(0), I.getOperand(1), | ||||||||
| 1600 | I.getFastMathFlags(), | ||||||||
| 1601 | SQ.getWithInstruction(&I))) | ||||||||
| 1602 | return replaceInstUsesWith(I, V); | ||||||||
| 1603 | |||||||||
| 1604 | if (Instruction *X = foldVectorBinop(I)) | ||||||||
| 1605 | return X; | ||||||||
| 1606 | |||||||||
| 1607 | return nullptr; | ||||||||
| 1608 | } |
| 1 | //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // This file provides a simple and efficient mechanism for performing general | ||||||||
| 10 | // tree-based pattern matches on the LLVM IR. The power of these routines is | ||||||||
| 11 | // that it allows you to write concise patterns that are expressive and easy to | ||||||||
| 12 | // understand. The other major advantage of this is that it allows you to | ||||||||
| 13 | // trivially capture/bind elements in the pattern to variables. For example, | ||||||||
| 14 | // you can do something like this: | ||||||||
| 15 | // | ||||||||
| 16 | // Value *Exp = ... | ||||||||
| 17 | // Value *X, *Y; ConstantInt *C1, *C2; // (X & C1) | (Y & C2) | ||||||||
| 18 | // if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)), | ||||||||
| 19 | // m_And(m_Value(Y), m_ConstantInt(C2))))) { | ||||||||
| 20 | // ... Pattern is matched and variables are bound ... | ||||||||
| 21 | // } | ||||||||
| 22 | // | ||||||||
| 23 | // This is primarily useful to things like the instruction combiner, but can | ||||||||
| 24 | // also be useful for static analysis tools or code generators. | ||||||||
| 25 | // | ||||||||
| 26 | //===----------------------------------------------------------------------===// | ||||||||
| 27 | |||||||||
| 28 | #ifndef LLVM_IR_PATTERNMATCH_H | ||||||||
| 29 | #define LLVM_IR_PATTERNMATCH_H | ||||||||
| 30 | |||||||||
| 31 | #include "llvm/ADT/APFloat.h" | ||||||||
| 32 | #include "llvm/ADT/APInt.h" | ||||||||
| 33 | #include "llvm/IR/Constant.h" | ||||||||
| 34 | #include "llvm/IR/Constants.h" | ||||||||
| 35 | #include "llvm/IR/DataLayout.h" | ||||||||
| 36 | #include "llvm/IR/InstrTypes.h" | ||||||||
| 37 | #include "llvm/IR/Instruction.h" | ||||||||
| 38 | #include "llvm/IR/Instructions.h" | ||||||||
| 39 | #include "llvm/IR/IntrinsicInst.h" | ||||||||
| 40 | #include "llvm/IR/Intrinsics.h" | ||||||||
| 41 | #include "llvm/IR/Operator.h" | ||||||||
| 42 | #include "llvm/IR/Value.h" | ||||||||
| 43 | #include "llvm/Support/Casting.h" | ||||||||
| 44 | #include <cstdint> | ||||||||
| 45 | |||||||||
| 46 | namespace llvm { | ||||||||
| 47 | namespace PatternMatch { | ||||||||
| 48 | |||||||||
| 49 | template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) { | ||||||||
| 50 | return const_cast<Pattern &>(P).match(V); | ||||||||
| 51 | } | ||||||||
| 52 | |||||||||
| 53 | template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) { | ||||||||
| 54 | return const_cast<Pattern &>(P).match(Mask); | ||||||||
| 55 | } | ||||||||
| 56 | |||||||||
| 57 | template <typename SubPattern_t> struct OneUse_match { | ||||||||
| 58 | SubPattern_t SubPattern; | ||||||||
| 59 | |||||||||
| 60 | OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} | ||||||||
| 61 | |||||||||
| 62 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 63 | return V->hasOneUse() && SubPattern.match(V); | ||||||||
| 64 | } | ||||||||
| 65 | }; | ||||||||
| 66 | |||||||||
| 67 | template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) { | ||||||||
| 68 | return SubPattern; | ||||||||
| 69 | } | ||||||||
| 70 | |||||||||
| 71 | template <typename Class> struct class_match { | ||||||||
| 72 | template <typename ITy> bool match(ITy *V) { return isa<Class>(V); } | ||||||||
| 73 | }; | ||||||||
| 74 | |||||||||
| 75 | /// Match an arbitrary value and ignore it. | ||||||||
| 76 | inline class_match<Value> m_Value() { return class_match<Value>(); } | ||||||||
| 77 | |||||||||
| 78 | /// Match an arbitrary unary operation and ignore it. | ||||||||
| 79 | inline class_match<UnaryOperator> m_UnOp() { | ||||||||
| 80 | return class_match<UnaryOperator>(); | ||||||||
| 81 | } | ||||||||
| 82 | |||||||||
| 83 | /// Match an arbitrary binary operation and ignore it. | ||||||||
| 84 | inline class_match<BinaryOperator> m_BinOp() { | ||||||||
| 85 | return class_match<BinaryOperator>(); | ||||||||
| 86 | } | ||||||||
| 87 | |||||||||
| 88 | /// Matches any compare instruction and ignore it. | ||||||||
| 89 | inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); } | ||||||||
| 90 | |||||||||
| 91 | struct undef_match { | ||||||||
| 92 | static bool check(const Value *V) { | ||||||||
| 93 | if (isa<UndefValue>(V)) | ||||||||
| 94 | return true; | ||||||||
| 95 | |||||||||
| 96 | const auto *CA = dyn_cast<ConstantAggregate>(V); | ||||||||
| 97 | if (!CA) | ||||||||
| 98 | return false; | ||||||||
| 99 | |||||||||
| 100 | SmallPtrSet<const ConstantAggregate *, 8> Seen; | ||||||||
| 101 | SmallVector<const ConstantAggregate *, 8> Worklist; | ||||||||
| 102 | |||||||||
| 103 | // Either UndefValue, PoisonValue, or an aggregate that only contains | ||||||||
| 104 | // these is accepted by matcher. | ||||||||
| 105 | // CheckValue returns false if CA cannot satisfy this constraint. | ||||||||
| 106 | auto CheckValue = [&](const ConstantAggregate *CA) { | ||||||||
| 107 | for (const Value *Op : CA->operand_values()) { | ||||||||
| 108 | if (isa<UndefValue>(Op)) | ||||||||
| 109 | continue; | ||||||||
| 110 | |||||||||
| 111 | const auto *CA = dyn_cast<ConstantAggregate>(Op); | ||||||||
| 112 | if (!CA) | ||||||||
| 113 | return false; | ||||||||
| 114 | if (Seen.insert(CA).second) | ||||||||
| 115 | Worklist.emplace_back(CA); | ||||||||
| 116 | } | ||||||||
| 117 | |||||||||
| 118 | return true; | ||||||||
| 119 | }; | ||||||||
| 120 | |||||||||
| 121 | if (!CheckValue(CA)) | ||||||||
| 122 | return false; | ||||||||
| 123 | |||||||||
| 124 | while (!Worklist.empty()) { | ||||||||
| 125 | if (!CheckValue(Worklist.pop_back_val())) | ||||||||
| 126 | return false; | ||||||||
| 127 | } | ||||||||
| 128 | return true; | ||||||||
| 129 | } | ||||||||
| 130 | template <typename ITy> bool match(ITy *V) { return check(V); } | ||||||||
| 131 | }; | ||||||||
| 132 | |||||||||
| 133 | /// Match an arbitrary undef constant. This matches poison as well. | ||||||||
| 134 | /// If this is an aggregate and contains a non-aggregate element that is | ||||||||
| 135 | /// neither undef nor poison, the aggregate is not matched. | ||||||||
| 136 | inline auto m_Undef() { return undef_match(); } | ||||||||
| 137 | |||||||||
| 138 | /// Match an arbitrary poison constant. | ||||||||
| 139 | inline class_match<PoisonValue> m_Poison() { return class_match<PoisonValue>(); } | ||||||||
| 140 | |||||||||
| 141 | /// Match an arbitrary Constant and ignore it. | ||||||||
| 142 | inline class_match<Constant> m_Constant() { return class_match<Constant>(); } | ||||||||
| 143 | |||||||||
| 144 | /// Match an arbitrary ConstantInt and ignore it. | ||||||||
| 145 | inline class_match<ConstantInt> m_ConstantInt() { | ||||||||
| 146 | return class_match<ConstantInt>(); | ||||||||
| 147 | } | ||||||||
| 148 | |||||||||
| 149 | /// Match an arbitrary ConstantFP and ignore it. | ||||||||
| 150 | inline class_match<ConstantFP> m_ConstantFP() { | ||||||||
| 151 | return class_match<ConstantFP>(); | ||||||||
| 152 | } | ||||||||
| 153 | |||||||||
| 154 | /// Match an arbitrary ConstantExpr and ignore it. | ||||||||
| 155 | inline class_match<ConstantExpr> m_ConstantExpr() { | ||||||||
| 156 | return class_match<ConstantExpr>(); | ||||||||
| 157 | } | ||||||||
| 158 | |||||||||
| 159 | /// Match an arbitrary basic block value and ignore it. | ||||||||
| 160 | inline class_match<BasicBlock> m_BasicBlock() { | ||||||||
| 161 | return class_match<BasicBlock>(); | ||||||||
| 162 | } | ||||||||
| 163 | |||||||||
| 164 | /// Inverting matcher | ||||||||
| 165 | template <typename Ty> struct match_unless { | ||||||||
| 166 | Ty M; | ||||||||
| 167 | |||||||||
| 168 | match_unless(const Ty &Matcher) : M(Matcher) {} | ||||||||
| 169 | |||||||||
| 170 | template <typename ITy> bool match(ITy *V) { return !M.match(V); } | ||||||||
| 171 | }; | ||||||||
| 172 | |||||||||
| 173 | /// Match if the inner matcher does *NOT* match. | ||||||||
| 174 | template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) { | ||||||||
| 175 | return match_unless<Ty>(M); | ||||||||
| 176 | } | ||||||||
| 177 | |||||||||
| 178 | /// Matching combinators | ||||||||
| 179 | template <typename LTy, typename RTy> struct match_combine_or { | ||||||||
| 180 | LTy L; | ||||||||
| 181 | RTy R; | ||||||||
| 182 | |||||||||
| 183 | match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | ||||||||
| 184 | |||||||||
| 185 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 186 | if (L.match(V)) | ||||||||
| 187 | return true; | ||||||||
| 188 | if (R.match(V)) | ||||||||
| 189 | return true; | ||||||||
| 190 | return false; | ||||||||
| 191 | } | ||||||||
| 192 | }; | ||||||||
| 193 | |||||||||
| 194 | template <typename LTy, typename RTy> struct match_combine_and { | ||||||||
| 195 | LTy L; | ||||||||
| 196 | RTy R; | ||||||||
| 197 | |||||||||
| 198 | match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | ||||||||
| 199 | |||||||||
| 200 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 201 | if (L.match(V)) | ||||||||
| 202 | if (R.match(V)) | ||||||||
| 203 | return true; | ||||||||
| 204 | return false; | ||||||||
| 205 | } | ||||||||
| 206 | }; | ||||||||
| 207 | |||||||||
| 208 | /// Combine two pattern matchers matching L || R | ||||||||
| 209 | template <typename LTy, typename RTy> | ||||||||
| 210 | inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) { | ||||||||
| 211 | return match_combine_or<LTy, RTy>(L, R); | ||||||||
| 212 | } | ||||||||
| 213 | |||||||||
| 214 | /// Combine two pattern matchers matching L && R | ||||||||
| 215 | template <typename LTy, typename RTy> | ||||||||
| 216 | inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) { | ||||||||
| 217 | return match_combine_and<LTy, RTy>(L, R); | ||||||||
| 218 | } | ||||||||
| 219 | |||||||||
| 220 | struct apint_match { | ||||||||
| 221 | const APInt *&Res; | ||||||||
| 222 | bool AllowUndef; | ||||||||
| 223 | |||||||||
| 224 | apint_match(const APInt *&Res, bool AllowUndef) | ||||||||
| 225 | : Res(Res), AllowUndef(AllowUndef) {} | ||||||||
| 226 | |||||||||
| 227 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 228 | if (auto *CI = dyn_cast<ConstantInt>(V)) { | ||||||||
| 229 | Res = &CI->getValue(); | ||||||||
| 230 | return true; | ||||||||
| 231 | } | ||||||||
| 232 | if (V->getType()->isVectorTy()) | ||||||||
| 233 | if (const auto *C = dyn_cast<Constant>(V)) | ||||||||
| 234 | if (auto *CI = dyn_cast_or_null<ConstantInt>( | ||||||||
| 235 | C->getSplatValue(AllowUndef))) { | ||||||||
| 236 | Res = &CI->getValue(); | ||||||||
| 237 | return true; | ||||||||
| 238 | } | ||||||||
| 239 | return false; | ||||||||
| 240 | } | ||||||||
| 241 | }; | ||||||||
| 242 | // Either constexpr if or renaming ConstantFP::getValueAPF to | ||||||||
| 243 | // ConstantFP::getValue is needed to do it via single template | ||||||||
| 244 | // function for both apint/apfloat. | ||||||||
| 245 | struct apfloat_match { | ||||||||
| 246 | const APFloat *&Res; | ||||||||
| 247 | bool AllowUndef; | ||||||||
| 248 | |||||||||
| 249 | apfloat_match(const APFloat *&Res, bool AllowUndef) | ||||||||
| 250 | : Res(Res), AllowUndef(AllowUndef) {} | ||||||||
| 251 | |||||||||
| 252 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 253 | if (auto *CI = dyn_cast<ConstantFP>(V)) { | ||||||||
| 254 | Res = &CI->getValueAPF(); | ||||||||
| 255 | return true; | ||||||||
| 256 | } | ||||||||
| 257 | if (V->getType()->isVectorTy()) | ||||||||
| 258 | if (const auto *C = dyn_cast<Constant>(V)) | ||||||||
| 259 | if (auto *CI = dyn_cast_or_null<ConstantFP>( | ||||||||
| 260 | C->getSplatValue(AllowUndef))) { | ||||||||
| 261 | Res = &CI->getValueAPF(); | ||||||||
| 262 | return true; | ||||||||
| 263 | } | ||||||||
| 264 | return false; | ||||||||
| 265 | } | ||||||||
| 266 | }; | ||||||||
| 267 | |||||||||
| 268 | /// Match a ConstantInt or splatted ConstantVector, binding the | ||||||||
| 269 | /// specified pointer to the contained APInt. | ||||||||
| 270 | inline apint_match m_APInt(const APInt *&Res) { | ||||||||
| 271 | // Forbid undefs by default to maintain previous behavior. | ||||||||
| 272 | return apint_match(Res, /* AllowUndef */ false); | ||||||||
| 273 | } | ||||||||
| 274 | |||||||||
| 275 | /// Match APInt while allowing undefs in splat vector constants. | ||||||||
| 276 | inline apint_match m_APIntAllowUndef(const APInt *&Res) { | ||||||||
| 277 | return apint_match(Res, /* AllowUndef */ true); | ||||||||
| 278 | } | ||||||||
| 279 | |||||||||
| 280 | /// Match APInt while forbidding undefs in splat vector constants. | ||||||||
| 281 | inline apint_match m_APIntForbidUndef(const APInt *&Res) { | ||||||||
| 282 | return apint_match(Res, /* AllowUndef */ false); | ||||||||
| 283 | } | ||||||||
| 284 | |||||||||
| 285 | /// Match a ConstantFP or splatted ConstantVector, binding the | ||||||||
| 286 | /// specified pointer to the contained APFloat. | ||||||||
| 287 | inline apfloat_match m_APFloat(const APFloat *&Res) { | ||||||||
| 288 | // Forbid undefs by default to maintain previous behavior. | ||||||||
| 289 | return apfloat_match(Res, /* AllowUndef */ false); | ||||||||
| 290 | } | ||||||||
| 291 | |||||||||
| 292 | /// Match APFloat while allowing undefs in splat vector constants. | ||||||||
| 293 | inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) { | ||||||||
| 294 | return apfloat_match(Res, /* AllowUndef */ true); | ||||||||
| 295 | } | ||||||||
| 296 | |||||||||
| 297 | /// Match APFloat while forbidding undefs in splat vector constants. | ||||||||
| 298 | inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) { | ||||||||
| 299 | return apfloat_match(Res, /* AllowUndef */ false); | ||||||||
| 300 | } | ||||||||
| 301 | |||||||||
| 302 | template <int64_t Val> struct constantint_match { | ||||||||
| 303 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 304 | if (const auto *CI = dyn_cast<ConstantInt>(V)) { | ||||||||
| 305 | const APInt &CIV = CI->getValue(); | ||||||||
| 306 | if (Val >= 0) | ||||||||
| 307 | return CIV == static_cast<uint64_t>(Val); | ||||||||
| 308 | // If Val is negative, and CI is shorter than it, truncate to the right | ||||||||
| 309 | // number of bits. If it is larger, then we have to sign extend. Just | ||||||||
| 310 | // compare their negated values. | ||||||||
| 311 | return -CIV == -Val; | ||||||||
| 312 | } | ||||||||
| 313 | return false; | ||||||||
| 314 | } | ||||||||
| 315 | }; | ||||||||
| 316 | |||||||||
| 317 | /// Match a ConstantInt with a specific value. | ||||||||
| 318 | template <int64_t Val> inline constantint_match<Val> m_ConstantInt() { | ||||||||
| 319 | return constantint_match<Val>(); | ||||||||
| 320 | } | ||||||||
| 321 | |||||||||
| 322 | /// This helper class is used to match constant scalars, vector splats, | ||||||||
| 323 | /// and fixed width vectors that satisfy a specified predicate. | ||||||||
| 324 | /// For fixed width vector constants, undefined elements are ignored. | ||||||||
| 325 | template <typename Predicate, typename ConstantVal> | ||||||||
| 326 | struct cstval_pred_ty : public Predicate { | ||||||||
| 327 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 328 | if (const auto *CV = dyn_cast<ConstantVal>(V)) | ||||||||
| 329 | return this->isValue(CV->getValue()); | ||||||||
| 330 | if (const auto *VTy = dyn_cast<VectorType>(V->getType())) { | ||||||||
| 331 | if (const auto *C = dyn_cast<Constant>(V)) { | ||||||||
| 332 | if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue())) | ||||||||
| 333 | return this->isValue(CV->getValue()); | ||||||||
| 334 | |||||||||
| 335 | // Number of elements of a scalable vector unknown at compile time | ||||||||
| 336 | auto *FVTy = dyn_cast<FixedVectorType>(VTy); | ||||||||
| 337 | if (!FVTy) | ||||||||
| 338 | return false; | ||||||||
| 339 | |||||||||
| 340 | // Non-splat vector constant: check each element for a match. | ||||||||
| 341 | unsigned NumElts = FVTy->getNumElements(); | ||||||||
| 342 | assert(NumElts != 0 && "Constant vector with no elements?")((void)0); | ||||||||
| 343 | bool HasNonUndefElements = false; | ||||||||
| 344 | for (unsigned i = 0; i != NumElts; ++i) { | ||||||||
| 345 | Constant *Elt = C->getAggregateElement(i); | ||||||||
| 346 | if (!Elt) | ||||||||
| 347 | return false; | ||||||||
| 348 | if (isa<UndefValue>(Elt)) | ||||||||
| 349 | continue; | ||||||||
| 350 | auto *CV = dyn_cast<ConstantVal>(Elt); | ||||||||
| 351 | if (!CV || !this->isValue(CV->getValue())) | ||||||||
| 352 | return false; | ||||||||
| 353 | HasNonUndefElements = true; | ||||||||
| 354 | } | ||||||||
| 355 | return HasNonUndefElements; | ||||||||
| 356 | } | ||||||||
| 357 | } | ||||||||
| 358 | return false; | ||||||||
| 359 | } | ||||||||
| 360 | }; | ||||||||
| 361 | |||||||||
| 362 | /// specialization of cstval_pred_ty for ConstantInt | ||||||||
| 363 | template <typename Predicate> | ||||||||
| 364 | using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>; | ||||||||
| 365 | |||||||||
| 366 | /// specialization of cstval_pred_ty for ConstantFP | ||||||||
| 367 | template <typename Predicate> | ||||||||
| 368 | using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>; | ||||||||
| 369 | |||||||||
| 370 | /// This helper class is used to match scalar and vector constants that | ||||||||
| 371 | /// satisfy a specified predicate, and bind them to an APInt. | ||||||||
| 372 | template <typename Predicate> struct api_pred_ty : public Predicate { | ||||||||
| 373 | const APInt *&Res; | ||||||||
| 374 | |||||||||
| 375 | api_pred_ty(const APInt *&R) : Res(R) {} | ||||||||
| 376 | |||||||||
| 377 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 378 | if (const auto *CI = dyn_cast<ConstantInt>(V)) | ||||||||
| 379 | if (this->isValue(CI->getValue())) { | ||||||||
| 380 | Res = &CI->getValue(); | ||||||||
| 381 | return true; | ||||||||
| 382 | } | ||||||||
| 383 | if (V->getType()->isVectorTy()) | ||||||||
| 384 | if (const auto *C = dyn_cast<Constant>(V)) | ||||||||
| 385 | if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue())) | ||||||||
| 386 | if (this->isValue(CI->getValue())) { | ||||||||
| 387 | Res = &CI->getValue(); | ||||||||
| 388 | return true; | ||||||||
| 389 | } | ||||||||
| 390 | |||||||||
| 391 | return false; | ||||||||
| 392 | } | ||||||||
| 393 | }; | ||||||||
| 394 | |||||||||
| 395 | /// This helper class is used to match scalar and vector constants that | ||||||||
| 396 | /// satisfy a specified predicate, and bind them to an APFloat. | ||||||||
| 397 | /// Undefs are allowed in splat vector constants. | ||||||||
| 398 | template <typename Predicate> struct apf_pred_ty : public Predicate { | ||||||||
| 399 | const APFloat *&Res; | ||||||||
| 400 | |||||||||
| 401 | apf_pred_ty(const APFloat *&R) : Res(R) {} | ||||||||
| 402 | |||||||||
| 403 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 404 | if (const auto *CI = dyn_cast<ConstantFP>(V)) | ||||||||
| 405 | if (this->isValue(CI->getValue())) { | ||||||||
| 406 | Res = &CI->getValue(); | ||||||||
| 407 | return true; | ||||||||
| 408 | } | ||||||||
| 409 | if (V->getType()->isVectorTy()) | ||||||||
| 410 | if (const auto *C = dyn_cast<Constant>(V)) | ||||||||
| 411 | if (auto *CI = dyn_cast_or_null<ConstantFP>( | ||||||||
| 412 | C->getSplatValue(/* AllowUndef */ true))) | ||||||||
| 413 | if (this->isValue(CI->getValue())) { | ||||||||
| 414 | Res = &CI->getValue(); | ||||||||
| 415 | return true; | ||||||||
| 416 | } | ||||||||
| 417 | |||||||||
| 418 | return false; | ||||||||
| 419 | } | ||||||||
| 420 | }; | ||||||||
| 421 | |||||||||
| 422 | /////////////////////////////////////////////////////////////////////////////// | ||||||||
| 423 | // | ||||||||
| 424 | // Encapsulate constant value queries for use in templated predicate matchers. | ||||||||
| 425 | // This allows checking if constants match using compound predicates and works | ||||||||
| 426 | // with vector constants, possibly with relaxed constraints. For example, ignore | ||||||||
| 427 | // undef values. | ||||||||
| 428 | // | ||||||||
| 429 | /////////////////////////////////////////////////////////////////////////////// | ||||||||
| 430 | |||||||||
| 431 | struct is_any_apint { | ||||||||
| 432 | bool isValue(const APInt &C) { return true; } | ||||||||
| 433 | }; | ||||||||
| 434 | /// Match an integer or vector with any integral constant. | ||||||||
| 435 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 436 | inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() { | ||||||||
| 437 | return cst_pred_ty<is_any_apint>(); | ||||||||
| 438 | } | ||||||||
| 439 | |||||||||
| 440 | struct is_all_ones { | ||||||||
| 441 | bool isValue(const APInt &C) { return C.isAllOnesValue(); } | ||||||||
| 442 | }; | ||||||||
| 443 | /// Match an integer or vector with all bits set. | ||||||||
| 444 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 445 | inline cst_pred_ty<is_all_ones> m_AllOnes() { | ||||||||
| 446 | return cst_pred_ty<is_all_ones>(); | ||||||||
| 447 | } | ||||||||
| 448 | |||||||||
| 449 | struct is_maxsignedvalue { | ||||||||
| 450 | bool isValue(const APInt &C) { return C.isMaxSignedValue(); } | ||||||||
| 451 | }; | ||||||||
| 452 | /// Match an integer or vector with values having all bits except for the high | ||||||||
| 453 | /// bit set (0x7f...). | ||||||||
| 454 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 455 | inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() { | ||||||||
| 456 | return cst_pred_ty<is_maxsignedvalue>(); | ||||||||
| 457 | } | ||||||||
| 458 | inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) { | ||||||||
| 459 | return V; | ||||||||
| 460 | } | ||||||||
| 461 | |||||||||
| 462 | struct is_negative { | ||||||||
| 463 | bool isValue(const APInt &C) { return C.isNegative(); } | ||||||||
| 464 | }; | ||||||||
| 465 | /// Match an integer or vector of negative values. | ||||||||
| 466 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 467 | inline cst_pred_ty<is_negative> m_Negative() { | ||||||||
| 468 | return cst_pred_ty<is_negative>(); | ||||||||
| 469 | } | ||||||||
| 470 | inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { | ||||||||
| 471 | return V; | ||||||||
| 472 | } | ||||||||
| 473 | |||||||||
| 474 | struct is_nonnegative { | ||||||||
| 475 | bool isValue(const APInt &C) { return C.isNonNegative(); } | ||||||||
| 476 | }; | ||||||||
| 477 | /// Match an integer or vector of non-negative values. | ||||||||
| 478 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 479 | inline cst_pred_ty<is_nonnegative> m_NonNegative() { | ||||||||
| 480 | return cst_pred_ty<is_nonnegative>(); | ||||||||
| 481 | } | ||||||||
| 482 | inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { | ||||||||
| 483 | return V; | ||||||||
| 484 | } | ||||||||
| 485 | |||||||||
| 486 | struct is_strictlypositive { | ||||||||
| 487 | bool isValue(const APInt &C) { return C.isStrictlyPositive(); } | ||||||||
| 488 | }; | ||||||||
| 489 | /// Match an integer or vector of strictly positive values. | ||||||||
| 490 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 491 | inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() { | ||||||||
| 492 | return cst_pred_ty<is_strictlypositive>(); | ||||||||
| 493 | } | ||||||||
| 494 | inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) { | ||||||||
| 495 | return V; | ||||||||
| 496 | } | ||||||||
| 497 | |||||||||
| 498 | struct is_nonpositive { | ||||||||
| 499 | bool isValue(const APInt &C) { return C.isNonPositive(); } | ||||||||
| 500 | }; | ||||||||
| 501 | /// Match an integer or vector of non-positive values. | ||||||||
| 502 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 503 | inline cst_pred_ty<is_nonpositive> m_NonPositive() { | ||||||||
| 504 | return cst_pred_ty<is_nonpositive>(); | ||||||||
| 505 | } | ||||||||
| 506 | inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } | ||||||||
| 507 | |||||||||
| 508 | struct is_one { | ||||||||
| 509 | bool isValue(const APInt &C) { return C.isOneValue(); } | ||||||||
| 510 | }; | ||||||||
| 511 | /// Match an integer 1 or a vector with all elements equal to 1. | ||||||||
| 512 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 513 | inline cst_pred_ty<is_one> m_One() { | ||||||||
| 514 | return cst_pred_ty<is_one>(); | ||||||||
| 515 | } | ||||||||
| 516 | |||||||||
| 517 | struct is_zero_int { | ||||||||
| 518 | bool isValue(const APInt &C) { return C.isNullValue(); } | ||||||||
| 519 | }; | ||||||||
| 520 | /// Match an integer 0 or a vector with all elements equal to 0. | ||||||||
| 521 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 522 | inline cst_pred_ty<is_zero_int> m_ZeroInt() { | ||||||||
| 523 | return cst_pred_ty<is_zero_int>(); | ||||||||
| 524 | } | ||||||||
| 525 | |||||||||
| 526 | struct is_zero { | ||||||||
| 527 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 528 | auto *C = dyn_cast<Constant>(V); | ||||||||
| 529 | // FIXME: this should be able to do something for scalable vectors | ||||||||
| 530 | return C
| ||||||||
| 531 | } | ||||||||
| 532 | }; | ||||||||
| 533 | /// Match any null constant or a vector with all elements equal to 0. | ||||||||
| 534 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 535 | inline is_zero m_Zero() { | ||||||||
| 536 | return is_zero(); | ||||||||
| 537 | } | ||||||||
| 538 | |||||||||
| 539 | struct is_power2 { | ||||||||
| 540 | bool isValue(const APInt &C) { return C.isPowerOf2(); } | ||||||||
| 541 | }; | ||||||||
| 542 | /// Match an integer or vector power-of-2. | ||||||||
| 543 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 544 | inline cst_pred_ty<is_power2> m_Power2() { | ||||||||
| 545 | return cst_pred_ty<is_power2>(); | ||||||||
| 546 | } | ||||||||
| 547 | inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { | ||||||||
| 548 | return V; | ||||||||
| 549 | } | ||||||||
| 550 | |||||||||
| 551 | struct is_negated_power2 { | ||||||||
| 552 | bool isValue(const APInt &C) { return (-C).isPowerOf2(); } | ||||||||
| 553 | }; | ||||||||
| 554 | /// Match a integer or vector negated power-of-2. | ||||||||
| 555 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 556 | inline cst_pred_ty<is_negated_power2> m_NegatedPower2() { | ||||||||
| 557 | return cst_pred_ty<is_negated_power2>(); | ||||||||
| 558 | } | ||||||||
| 559 | inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) { | ||||||||
| 560 | return V; | ||||||||
| 561 | } | ||||||||
| 562 | |||||||||
| 563 | struct is_power2_or_zero { | ||||||||
| 564 | bool isValue(const APInt &C) { return !C || C.isPowerOf2(); } | ||||||||
| 565 | }; | ||||||||
| 566 | /// Match an integer or vector of 0 or power-of-2 values. | ||||||||
| 567 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 568 | inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() { | ||||||||
| 569 | return cst_pred_ty<is_power2_or_zero>(); | ||||||||
| 570 | } | ||||||||
| 571 | inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) { | ||||||||
| 572 | return V; | ||||||||
| 573 | } | ||||||||
| 574 | |||||||||
| 575 | struct is_sign_mask { | ||||||||
| 576 | bool isValue(const APInt &C) { return C.isSignMask(); } | ||||||||
| 577 | }; | ||||||||
| 578 | /// Match an integer or vector with only the sign bit(s) set. | ||||||||
| 579 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 580 | inline cst_pred_ty<is_sign_mask> m_SignMask() { | ||||||||
| 581 | return cst_pred_ty<is_sign_mask>(); | ||||||||
| 582 | } | ||||||||
| 583 | |||||||||
| 584 | struct is_lowbit_mask { | ||||||||
| 585 | bool isValue(const APInt &C) { return C.isMask(); } | ||||||||
| 586 | }; | ||||||||
| 587 | /// Match an integer or vector with only the low bit(s) set. | ||||||||
| 588 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 589 | inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() { | ||||||||
| 590 | return cst_pred_ty<is_lowbit_mask>(); | ||||||||
| 591 | } | ||||||||
| 592 | |||||||||
| 593 | struct icmp_pred_with_threshold { | ||||||||
| 594 | ICmpInst::Predicate Pred; | ||||||||
| 595 | const APInt *Thr; | ||||||||
| 596 | bool isValue(const APInt &C) { | ||||||||
| 597 | switch (Pred) { | ||||||||
| 598 | case ICmpInst::Predicate::ICMP_EQ: | ||||||||
| 599 | return C.eq(*Thr); | ||||||||
| 600 | case ICmpInst::Predicate::ICMP_NE: | ||||||||
| 601 | return C.ne(*Thr); | ||||||||
| 602 | case ICmpInst::Predicate::ICMP_UGT: | ||||||||
| 603 | return C.ugt(*Thr); | ||||||||
| 604 | case ICmpInst::Predicate::ICMP_UGE: | ||||||||
| 605 | return C.uge(*Thr); | ||||||||
| 606 | case ICmpInst::Predicate::ICMP_ULT: | ||||||||
| 607 | return C.ult(*Thr); | ||||||||
| 608 | case ICmpInst::Predicate::ICMP_ULE: | ||||||||
| 609 | return C.ule(*Thr); | ||||||||
| 610 | case ICmpInst::Predicate::ICMP_SGT: | ||||||||
| 611 | return C.sgt(*Thr); | ||||||||
| 612 | case ICmpInst::Predicate::ICMP_SGE: | ||||||||
| 613 | return C.sge(*Thr); | ||||||||
| 614 | case ICmpInst::Predicate::ICMP_SLT: | ||||||||
| 615 | return C.slt(*Thr); | ||||||||
| 616 | case ICmpInst::Predicate::ICMP_SLE: | ||||||||
| 617 | return C.sle(*Thr); | ||||||||
| 618 | default: | ||||||||
| 619 | llvm_unreachable("Unhandled ICmp predicate")__builtin_unreachable(); | ||||||||
| 620 | } | ||||||||
| 621 | } | ||||||||
| 622 | }; | ||||||||
| 623 | /// Match an integer or vector with every element comparing 'pred' (eg/ne/...) | ||||||||
| 624 | /// to Threshold. For vectors, this includes constants with undefined elements. | ||||||||
| 625 | inline cst_pred_ty<icmp_pred_with_threshold> | ||||||||
| 626 | m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) { | ||||||||
| 627 | cst_pred_ty<icmp_pred_with_threshold> P; | ||||||||
| 628 | P.Pred = Predicate; | ||||||||
| 629 | P.Thr = &Threshold; | ||||||||
| 630 | return P; | ||||||||
| 631 | } | ||||||||
| 632 | |||||||||
| 633 | struct is_nan { | ||||||||
| 634 | bool isValue(const APFloat &C) { return C.isNaN(); } | ||||||||
| 635 | }; | ||||||||
| 636 | /// Match an arbitrary NaN constant. This includes quiet and signalling nans. | ||||||||
| 637 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 638 | inline cstfp_pred_ty<is_nan> m_NaN() { | ||||||||
| 639 | return cstfp_pred_ty<is_nan>(); | ||||||||
| 640 | } | ||||||||
| 641 | |||||||||
| 642 | struct is_nonnan { | ||||||||
| 643 | bool isValue(const APFloat &C) { return !C.isNaN(); } | ||||||||
| 644 | }; | ||||||||
| 645 | /// Match a non-NaN FP constant. | ||||||||
| 646 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 647 | inline cstfp_pred_ty<is_nonnan> m_NonNaN() { | ||||||||
| 648 | return cstfp_pred_ty<is_nonnan>(); | ||||||||
| 649 | } | ||||||||
| 650 | |||||||||
| 651 | struct is_inf { | ||||||||
| 652 | bool isValue(const APFloat &C) { return C.isInfinity(); } | ||||||||
| 653 | }; | ||||||||
| 654 | /// Match a positive or negative infinity FP constant. | ||||||||
| 655 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 656 | inline cstfp_pred_ty<is_inf> m_Inf() { | ||||||||
| 657 | return cstfp_pred_ty<is_inf>(); | ||||||||
| 658 | } | ||||||||
| 659 | |||||||||
| 660 | struct is_noninf { | ||||||||
| 661 | bool isValue(const APFloat &C) { return !C.isInfinity(); } | ||||||||
| 662 | }; | ||||||||
| 663 | /// Match a non-infinity FP constant, i.e. finite or NaN. | ||||||||
| 664 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 665 | inline cstfp_pred_ty<is_noninf> m_NonInf() { | ||||||||
| 666 | return cstfp_pred_ty<is_noninf>(); | ||||||||
| 667 | } | ||||||||
| 668 | |||||||||
| 669 | struct is_finite { | ||||||||
| 670 | bool isValue(const APFloat &C) { return C.isFinite(); } | ||||||||
| 671 | }; | ||||||||
| 672 | /// Match a finite FP constant, i.e. not infinity or NaN. | ||||||||
| 673 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 674 | inline cstfp_pred_ty<is_finite> m_Finite() { | ||||||||
| 675 | return cstfp_pred_ty<is_finite>(); | ||||||||
| 676 | } | ||||||||
| 677 | inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; } | ||||||||
| 678 | |||||||||
| 679 | struct is_finitenonzero { | ||||||||
| 680 | bool isValue(const APFloat &C) { return C.isFiniteNonZero(); } | ||||||||
| 681 | }; | ||||||||
| 682 | /// Match a finite non-zero FP constant. | ||||||||
| 683 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 684 | inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() { | ||||||||
| 685 | return cstfp_pred_ty<is_finitenonzero>(); | ||||||||
| 686 | } | ||||||||
| 687 | inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) { | ||||||||
| 688 | return V; | ||||||||
| 689 | } | ||||||||
| 690 | |||||||||
| 691 | struct is_any_zero_fp { | ||||||||
| 692 | bool isValue(const APFloat &C) { return C.isZero(); } | ||||||||
| 693 | }; | ||||||||
| 694 | /// Match a floating-point negative zero or positive zero. | ||||||||
| 695 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 696 | inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() { | ||||||||
| 697 | return cstfp_pred_ty<is_any_zero_fp>(); | ||||||||
| 698 | } | ||||||||
| 699 | |||||||||
| 700 | struct is_pos_zero_fp { | ||||||||
| 701 | bool isValue(const APFloat &C) { return C.isPosZero(); } | ||||||||
| 702 | }; | ||||||||
| 703 | /// Match a floating-point positive zero. | ||||||||
| 704 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 705 | inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() { | ||||||||
| 706 | return cstfp_pred_ty<is_pos_zero_fp>(); | ||||||||
| 707 | } | ||||||||
| 708 | |||||||||
| 709 | struct is_neg_zero_fp { | ||||||||
| 710 | bool isValue(const APFloat &C) { return C.isNegZero(); } | ||||||||
| 711 | }; | ||||||||
| 712 | /// Match a floating-point negative zero. | ||||||||
| 713 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 714 | inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() { | ||||||||
| 715 | return cstfp_pred_ty<is_neg_zero_fp>(); | ||||||||
| 716 | } | ||||||||
| 717 | |||||||||
| 718 | struct is_non_zero_fp { | ||||||||
| 719 | bool isValue(const APFloat &C) { return C.isNonZero(); } | ||||||||
| 720 | }; | ||||||||
| 721 | /// Match a floating-point non-zero. | ||||||||
| 722 | /// For vectors, this includes constants with undefined elements. | ||||||||
| 723 | inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() { | ||||||||
| 724 | return cstfp_pred_ty<is_non_zero_fp>(); | ||||||||
| 725 | } | ||||||||
| 726 | |||||||||
| 727 | /////////////////////////////////////////////////////////////////////////////// | ||||||||
| 728 | |||||||||
| 729 | template <typename Class> struct bind_ty { | ||||||||
| 730 | Class *&VR; | ||||||||
| 731 | |||||||||
| 732 | bind_ty(Class *&V) : VR(V) {} | ||||||||
| 733 | |||||||||
| 734 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 735 | if (auto *CV = dyn_cast<Class>(V)) { | ||||||||
| 736 | VR = CV; | ||||||||
| 737 | return true; | ||||||||
| 738 | } | ||||||||
| 739 | return false; | ||||||||
| 740 | } | ||||||||
| 741 | }; | ||||||||
| 742 | |||||||||
| 743 | /// Match a value, capturing it if we match. | ||||||||
| 744 | inline bind_ty<Value> m_Value(Value *&V) { return V; } | ||||||||
| 745 | inline bind_ty<const Value> m_Value(const Value *&V) { return V; } | ||||||||
| 746 | |||||||||
| 747 | /// Match an instruction, capturing it if we match. | ||||||||
| 748 | inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } | ||||||||
| 749 | /// Match a unary operator, capturing it if we match. | ||||||||
| 750 | inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; } | ||||||||
| 751 | /// Match a binary operator, capturing it if we match. | ||||||||
| 752 | inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } | ||||||||
| 753 | /// Match a with overflow intrinsic, capturing it if we match. | ||||||||
| 754 | inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { return I; } | ||||||||
| 755 | inline bind_ty<const WithOverflowInst> | ||||||||
| 756 | m_WithOverflowInst(const WithOverflowInst *&I) { | ||||||||
| 757 | return I; | ||||||||
| 758 | } | ||||||||
| 759 | |||||||||
| 760 | /// Match a Constant, capturing the value if we match. | ||||||||
| 761 | inline bind_ty<Constant> m_Constant(Constant *&C) { return C; } | ||||||||
| 762 | |||||||||
| 763 | /// Match a ConstantInt, capturing the value if we match. | ||||||||
| 764 | inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; } | ||||||||
| 765 | |||||||||
| 766 | /// Match a ConstantFP, capturing the value if we match. | ||||||||
| 767 | inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; } | ||||||||
| 768 | |||||||||
| 769 | /// Match a ConstantExpr, capturing the value if we match. | ||||||||
| 770 | inline bind_ty<ConstantExpr> m_ConstantExpr(ConstantExpr *&C) { return C; } | ||||||||
| 771 | |||||||||
| 772 | /// Match a basic block value, capturing it if we match. | ||||||||
| 773 | inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; } | ||||||||
| 774 | inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) { | ||||||||
| 775 | return V; | ||||||||
| 776 | } | ||||||||
| 777 | |||||||||
| 778 | /// Match an arbitrary immediate Constant and ignore it. | ||||||||
| 779 | inline match_combine_and<class_match<Constant>, | ||||||||
| 780 | match_unless<class_match<ConstantExpr>>> | ||||||||
| 781 | m_ImmConstant() { | ||||||||
| 782 | return m_CombineAnd(m_Constant(), m_Unless(m_ConstantExpr())); | ||||||||
| 783 | } | ||||||||
| 784 | |||||||||
| 785 | /// Match an immediate Constant, capturing the value if we match. | ||||||||
| 786 | inline match_combine_and<bind_ty<Constant>, | ||||||||
| 787 | match_unless<class_match<ConstantExpr>>> | ||||||||
| 788 | m_ImmConstant(Constant *&C) { | ||||||||
| 789 | return m_CombineAnd(m_Constant(C), m_Unless(m_ConstantExpr())); | ||||||||
| 790 | } | ||||||||
| 791 | |||||||||
| 792 | /// Match a specified Value*. | ||||||||
| 793 | struct specificval_ty { | ||||||||
| 794 | const Value *Val; | ||||||||
| 795 | |||||||||
| 796 | specificval_ty(const Value *V) : Val(V) {} | ||||||||
| 797 | |||||||||
| 798 | template <typename ITy> bool match(ITy *V) { return V == Val; } | ||||||||
| 799 | }; | ||||||||
| 800 | |||||||||
| 801 | /// Match if we have a specific specified value. | ||||||||
| 802 | inline specificval_ty m_Specific(const Value *V) { return V; } | ||||||||
| 803 | |||||||||
| 804 | /// Stores a reference to the Value *, not the Value * itself, | ||||||||
| 805 | /// thus can be used in commutative matchers. | ||||||||
| 806 | template <typename Class> struct deferredval_ty { | ||||||||
| 807 | Class *const &Val; | ||||||||
| 808 | |||||||||
| 809 | deferredval_ty(Class *const &V) : Val(V) {} | ||||||||
| 810 | |||||||||
| 811 | template <typename ITy> bool match(ITy *const V) { return V == Val; } | ||||||||
| 812 | }; | ||||||||
| 813 | |||||||||
| 814 | /// Like m_Specific(), but works if the specific value to match is determined | ||||||||
| 815 | /// as part of the same match() expression. For example: | ||||||||
| 816 | /// m_Add(m_Value(X), m_Specific(X)) is incorrect, because m_Specific() will | ||||||||
| 817 | /// bind X before the pattern match starts. | ||||||||
| 818 | /// m_Add(m_Value(X), m_Deferred(X)) is correct, and will check against | ||||||||
| 819 | /// whichever value m_Value(X) populated. | ||||||||
| 820 | inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; } | ||||||||
| 821 | inline deferredval_ty<const Value> m_Deferred(const Value *const &V) { | ||||||||
| 822 | return V; | ||||||||
| 823 | } | ||||||||
| 824 | |||||||||
| 825 | /// Match a specified floating point value or vector of all elements of | ||||||||
| 826 | /// that value. | ||||||||
| 827 | struct specific_fpval { | ||||||||
| 828 | double Val; | ||||||||
| 829 | |||||||||
| 830 | specific_fpval(double V) : Val(V) {} | ||||||||
| 831 | |||||||||
| 832 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 833 | if (const auto *CFP = dyn_cast<ConstantFP>(V)) | ||||||||
| 834 | return CFP->isExactlyValue(Val); | ||||||||
| 835 | if (V->getType()->isVectorTy()) | ||||||||
| 836 | if (const auto *C = dyn_cast<Constant>(V)) | ||||||||
| 837 | if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue())) | ||||||||
| 838 | return CFP->isExactlyValue(Val); | ||||||||
| 839 | return false; | ||||||||
| 840 | } | ||||||||
| 841 | }; | ||||||||
| 842 | |||||||||
| 843 | /// Match a specific floating point value or vector with all elements | ||||||||
| 844 | /// equal to the value. | ||||||||
| 845 | inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); } | ||||||||
| 846 | |||||||||
| 847 | /// Match a float 1.0 or vector with all elements equal to 1.0. | ||||||||
| 848 | inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); } | ||||||||
| 849 | |||||||||
| 850 | struct bind_const_intval_ty { | ||||||||
| 851 | uint64_t &VR; | ||||||||
| 852 | |||||||||
| 853 | bind_const_intval_ty(uint64_t &V) : VR(V) {} | ||||||||
| 854 | |||||||||
| 855 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 856 | if (const auto *CV = dyn_cast<ConstantInt>(V)) | ||||||||
| 857 | if (CV->getValue().ule(UINT64_MAX0xffffffffffffffffULL)) { | ||||||||
| 858 | VR = CV->getZExtValue(); | ||||||||
| 859 | return true; | ||||||||
| 860 | } | ||||||||
| 861 | return false; | ||||||||
| 862 | } | ||||||||
| 863 | }; | ||||||||
| 864 | |||||||||
| 865 | /// Match a specified integer value or vector of all elements of that | ||||||||
| 866 | /// value. | ||||||||
| 867 | template <bool AllowUndefs> | ||||||||
| 868 | struct specific_intval { | ||||||||
| 869 | APInt Val; | ||||||||
| 870 | |||||||||
| 871 | specific_intval(APInt V) : Val(std::move(V)) {} | ||||||||
| 872 | |||||||||
| 873 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 874 | const auto *CI = dyn_cast<ConstantInt>(V); | ||||||||
| 875 | if (!CI && V->getType()->isVectorTy()) | ||||||||
| 876 | if (const auto *C = dyn_cast<Constant>(V)) | ||||||||
| 877 | CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs)); | ||||||||
| 878 | |||||||||
| 879 | return CI && APInt::isSameValue(CI->getValue(), Val); | ||||||||
| 880 | } | ||||||||
| 881 | }; | ||||||||
| 882 | |||||||||
| 883 | /// Match a specific integer value or vector with all elements equal to | ||||||||
| 884 | /// the value. | ||||||||
| 885 | inline specific_intval<false> m_SpecificInt(APInt V) { | ||||||||
| 886 | return specific_intval<false>(std::move(V)); | ||||||||
| 887 | } | ||||||||
| 888 | |||||||||
| 889 | inline specific_intval<false> m_SpecificInt(uint64_t V) { | ||||||||
| 890 | return m_SpecificInt(APInt(64, V)); | ||||||||
| 891 | } | ||||||||
| 892 | |||||||||
| 893 | inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) { | ||||||||
| 894 | return specific_intval<true>(std::move(V)); | ||||||||
| 895 | } | ||||||||
| 896 | |||||||||
| 897 | inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) { | ||||||||
| 898 | return m_SpecificIntAllowUndef(APInt(64, V)); | ||||||||
| 899 | } | ||||||||
| 900 | |||||||||
| 901 | /// Match a ConstantInt and bind to its value. This does not match | ||||||||
| 902 | /// ConstantInts wider than 64-bits. | ||||||||
| 903 | inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; } | ||||||||
| 904 | |||||||||
| 905 | /// Match a specified basic block value. | ||||||||
| 906 | struct specific_bbval { | ||||||||
| 907 | BasicBlock *Val; | ||||||||
| 908 | |||||||||
| 909 | specific_bbval(BasicBlock *Val) : Val(Val) {} | ||||||||
| 910 | |||||||||
| 911 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 912 | const auto *BB = dyn_cast<BasicBlock>(V); | ||||||||
| 913 | return BB && BB == Val; | ||||||||
| 914 | } | ||||||||
| 915 | }; | ||||||||
| 916 | |||||||||
| 917 | /// Match a specific basic block value. | ||||||||
| 918 | inline specific_bbval m_SpecificBB(BasicBlock *BB) { | ||||||||
| 919 | return specific_bbval(BB); | ||||||||
| 920 | } | ||||||||
| 921 | |||||||||
| 922 | /// A commutative-friendly version of m_Specific(). | ||||||||
| 923 | inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) { | ||||||||
| 924 | return BB; | ||||||||
| 925 | } | ||||||||
| 926 | inline deferredval_ty<const BasicBlock> | ||||||||
| 927 | m_Deferred(const BasicBlock *const &BB) { | ||||||||
| 928 | return BB; | ||||||||
| 929 | } | ||||||||
| 930 | |||||||||
| 931 | //===----------------------------------------------------------------------===// | ||||||||
| 932 | // Matcher for any binary operator. | ||||||||
| 933 | // | ||||||||
| 934 | template <typename LHS_t, typename RHS_t, bool Commutable = false> | ||||||||
| 935 | struct AnyBinaryOp_match { | ||||||||
| 936 | LHS_t L; | ||||||||
| 937 | RHS_t R; | ||||||||
| 938 | |||||||||
| 939 | // The evaluation order is always stable, regardless of Commutability. | ||||||||
| 940 | // The LHS is always matched first. | ||||||||
| 941 | AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||||||
| 942 | |||||||||
| 943 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 944 | if (auto *I = dyn_cast<BinaryOperator>(V)) | ||||||||
| 945 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || | ||||||||
| 946 | (Commutable && L.match(I->getOperand(1)) && | ||||||||
| 947 | R.match(I->getOperand(0))); | ||||||||
| 948 | return false; | ||||||||
| 949 | } | ||||||||
| 950 | }; | ||||||||
| 951 | |||||||||
| 952 | template <typename LHS, typename RHS> | ||||||||
| 953 | inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) { | ||||||||
| 954 | return AnyBinaryOp_match<LHS, RHS>(L, R); | ||||||||
| 955 | } | ||||||||
| 956 | |||||||||
| 957 | //===----------------------------------------------------------------------===// | ||||||||
| 958 | // Matcher for any unary operator. | ||||||||
| 959 | // TODO fuse unary, binary matcher into n-ary matcher | ||||||||
| 960 | // | ||||||||
| 961 | template <typename OP_t> struct AnyUnaryOp_match { | ||||||||
| 962 | OP_t X; | ||||||||
| 963 | |||||||||
| 964 | AnyUnaryOp_match(const OP_t &X) : X(X) {} | ||||||||
| 965 | |||||||||
| 966 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 967 | if (auto *I = dyn_cast<UnaryOperator>(V)) | ||||||||
| 968 | return X.match(I->getOperand(0)); | ||||||||
| 969 | return false; | ||||||||
| 970 | } | ||||||||
| 971 | }; | ||||||||
| 972 | |||||||||
| 973 | template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) { | ||||||||
| 974 | return AnyUnaryOp_match<OP_t>(X); | ||||||||
| 975 | } | ||||||||
| 976 | |||||||||
| 977 | //===----------------------------------------------------------------------===// | ||||||||
| 978 | // Matchers for specific binary operators. | ||||||||
| 979 | // | ||||||||
| 980 | |||||||||
| 981 | template <typename LHS_t, typename RHS_t, unsigned Opcode, | ||||||||
| 982 | bool Commutable = false> | ||||||||
| 983 | struct BinaryOp_match { | ||||||||
| 984 | LHS_t L; | ||||||||
| 985 | RHS_t R; | ||||||||
| 986 | |||||||||
| 987 | // The evaluation order is always stable, regardless of Commutability. | ||||||||
| 988 | // The LHS is always matched first. | ||||||||
| 989 | BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||||||
| 990 | |||||||||
| 991 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 992 | if (V->getValueID() == Value::InstructionVal + Opcode) { | ||||||||
| 993 | auto *I = cast<BinaryOperator>(V); | ||||||||
| 994 | return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) || | ||||||||
| 995 | (Commutable && L.match(I->getOperand(1)) && | ||||||||
| 996 | R.match(I->getOperand(0))); | ||||||||
| 997 | } | ||||||||
| 998 | if (auto *CE = dyn_cast<ConstantExpr>(V)) | ||||||||
| 999 | return CE->getOpcode() == Opcode && | ||||||||
| 1000 | ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) || | ||||||||
| 1001 | (Commutable && L.match(CE->getOperand(1)) && | ||||||||
| 1002 | R.match(CE->getOperand(0)))); | ||||||||
| 1003 | return false; | ||||||||
| 1004 | } | ||||||||
| 1005 | }; | ||||||||
| 1006 | |||||||||
| 1007 | template <typename LHS, typename RHS> | ||||||||
| 1008 | inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L, | ||||||||
| 1009 | const RHS &R) { | ||||||||
| 1010 | return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R); | ||||||||
| 1011 | } | ||||||||
| 1012 | |||||||||
| 1013 | template <typename LHS, typename RHS> | ||||||||
| 1014 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L, | ||||||||
| 1015 | const RHS &R) { | ||||||||
| 1016 | return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R); | ||||||||
| 1017 | } | ||||||||
| 1018 | |||||||||
| 1019 | template <typename LHS, typename RHS> | ||||||||
| 1020 | inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L, | ||||||||
| 1021 | const RHS &R) { | ||||||||
| 1022 | return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R); | ||||||||
| 1023 | } | ||||||||
| 1024 | |||||||||
| 1025 | template <typename LHS, typename RHS> | ||||||||
| 1026 | inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L, | ||||||||
| 1027 | const RHS &R) { | ||||||||
| 1028 | return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R); | ||||||||
| 1029 | } | ||||||||
| 1030 | |||||||||
| 1031 | template <typename Op_t> struct FNeg_match { | ||||||||
| 1032 | Op_t X; | ||||||||
| 1033 | |||||||||
| 1034 | FNeg_match(const Op_t &Op) : X(Op) {} | ||||||||
| 1035 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1036 | auto *FPMO = dyn_cast<FPMathOperator>(V); | ||||||||
| 1037 | if (!FPMO) return false; | ||||||||
| 1038 | |||||||||
| 1039 | if (FPMO->getOpcode() == Instruction::FNeg) | ||||||||
| 1040 | return X.match(FPMO->getOperand(0)); | ||||||||
| 1041 | |||||||||
| 1042 | if (FPMO->getOpcode() == Instruction::FSub) { | ||||||||
| 1043 | if (FPMO->hasNoSignedZeros()) { | ||||||||
| 1044 | // With 'nsz', any zero goes. | ||||||||
| 1045 | if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0))) | ||||||||
| 1046 | return false; | ||||||||
| 1047 | } else { | ||||||||
| 1048 | // Without 'nsz', we need fsub -0.0, X exactly. | ||||||||
| 1049 | if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0))) | ||||||||
| 1050 | return false; | ||||||||
| 1051 | } | ||||||||
| 1052 | |||||||||
| 1053 | return X.match(FPMO->getOperand(1)); | ||||||||
| 1054 | } | ||||||||
| 1055 | |||||||||
| 1056 | return false; | ||||||||
| 1057 | } | ||||||||
| 1058 | }; | ||||||||
| 1059 | |||||||||
| 1060 | /// Match 'fneg X' as 'fsub -0.0, X'. | ||||||||
| 1061 | template <typename OpTy> | ||||||||
| 1062 | inline FNeg_match<OpTy> | ||||||||
| 1063 | m_FNeg(const OpTy &X) { | ||||||||
| 1064 | return FNeg_match<OpTy>(X); | ||||||||
| 1065 | } | ||||||||
| 1066 | |||||||||
| 1067 | /// Match 'fneg X' as 'fsub +-0.0, X'. | ||||||||
| 1068 | template <typename RHS> | ||||||||
| 1069 | inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub> | ||||||||
| 1070 | m_FNegNSZ(const RHS &X) { | ||||||||
| 1071 | return m_FSub(m_AnyZeroFP(), X); | ||||||||
| 1072 | } | ||||||||
| 1073 | |||||||||
| 1074 | template <typename LHS, typename RHS> | ||||||||
| 1075 | inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L, | ||||||||
| 1076 | const RHS &R) { | ||||||||
| 1077 | return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R); | ||||||||
| 1078 | } | ||||||||
| 1079 | |||||||||
| 1080 | template <typename LHS, typename RHS> | ||||||||
| 1081 | inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L, | ||||||||
| 1082 | const RHS &R) { | ||||||||
| 1083 | return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R); | ||||||||
| 1084 | } | ||||||||
| 1085 | |||||||||
| 1086 | template <typename LHS, typename RHS> | ||||||||
| 1087 | inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L, | ||||||||
| 1088 | const RHS &R) { | ||||||||
| 1089 | return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R); | ||||||||
| 1090 | } | ||||||||
| 1091 | |||||||||
| 1092 | template <typename LHS, typename RHS> | ||||||||
| 1093 | inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L, | ||||||||
| 1094 | const RHS &R) { | ||||||||
| 1095 | return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R); | ||||||||
| 1096 | } | ||||||||
| 1097 | |||||||||
| 1098 | template <typename LHS, typename RHS> | ||||||||
| 1099 | inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L, | ||||||||
| 1100 | const RHS &R) { | ||||||||
| 1101 | return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R); | ||||||||
| 1102 | } | ||||||||
| 1103 | |||||||||
| 1104 | template <typename LHS, typename RHS> | ||||||||
| 1105 | inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L, | ||||||||
| 1106 | const RHS &R) { | ||||||||
| 1107 | return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R); | ||||||||
| 1108 | } | ||||||||
| 1109 | |||||||||
| 1110 | template <typename LHS, typename RHS> | ||||||||
| 1111 | inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L, | ||||||||
| 1112 | const RHS &R) { | ||||||||
| 1113 | return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R); | ||||||||
| 1114 | } | ||||||||
| 1115 | |||||||||
| 1116 | template <typename LHS, typename RHS> | ||||||||
| 1117 | inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L, | ||||||||
| 1118 | const RHS &R) { | ||||||||
| 1119 | return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R); | ||||||||
| 1120 | } | ||||||||
| 1121 | |||||||||
| 1122 | template <typename LHS, typename RHS> | ||||||||
| 1123 | inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L, | ||||||||
| 1124 | const RHS &R) { | ||||||||
| 1125 | return BinaryOp_match<LHS, RHS, Instruction::And>(L, R); | ||||||||
| 1126 | } | ||||||||
| 1127 | |||||||||
| 1128 | template <typename LHS, typename RHS> | ||||||||
| 1129 | inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L, | ||||||||
| 1130 | const RHS &R) { | ||||||||
| 1131 | return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R); | ||||||||
| 1132 | } | ||||||||
| 1133 | |||||||||
| 1134 | template <typename LHS, typename RHS> | ||||||||
| 1135 | inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L, | ||||||||
| 1136 | const RHS &R) { | ||||||||
| 1137 | return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R); | ||||||||
| 1138 | } | ||||||||
| 1139 | |||||||||
| 1140 | template <typename LHS, typename RHS> | ||||||||
| 1141 | inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L, | ||||||||
| 1142 | const RHS &R) { | ||||||||
| 1143 | return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R); | ||||||||
| 1144 | } | ||||||||
| 1145 | |||||||||
| 1146 | template <typename LHS, typename RHS> | ||||||||
| 1147 | inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L, | ||||||||
| 1148 | const RHS &R) { | ||||||||
| 1149 | return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R); | ||||||||
| 1150 | } | ||||||||
| 1151 | |||||||||
| 1152 | template <typename LHS, typename RHS> | ||||||||
| 1153 | inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L, | ||||||||
| 1154 | const RHS &R) { | ||||||||
| 1155 | return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R); | ||||||||
| 1156 | } | ||||||||
| 1157 | |||||||||
| 1158 | template <typename LHS_t, typename RHS_t, unsigned Opcode, | ||||||||
| 1159 | unsigned WrapFlags = 0> | ||||||||
| 1160 | struct OverflowingBinaryOp_match { | ||||||||
| 1161 | LHS_t L; | ||||||||
| 1162 | RHS_t R; | ||||||||
| 1163 | |||||||||
| 1164 | OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) | ||||||||
| 1165 | : L(LHS), R(RHS) {} | ||||||||
| 1166 | |||||||||
| 1167 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1168 | if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) { | ||||||||
| 1169 | if (Op->getOpcode() != Opcode) | ||||||||
| 1170 | return false; | ||||||||
| 1171 | if ((WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap) && | ||||||||
| 1172 | !Op->hasNoUnsignedWrap()) | ||||||||
| 1173 | return false; | ||||||||
| 1174 | if ((WrapFlags & OverflowingBinaryOperator::NoSignedWrap) && | ||||||||
| 1175 | !Op->hasNoSignedWrap()) | ||||||||
| 1176 | return false; | ||||||||
| 1177 | return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1)); | ||||||||
| 1178 | } | ||||||||
| 1179 | return false; | ||||||||
| 1180 | } | ||||||||
| 1181 | }; | ||||||||
| 1182 | |||||||||
| 1183 | template <typename LHS, typename RHS> | ||||||||
| 1184 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||||||
| 1185 | OverflowingBinaryOperator::NoSignedWrap> | ||||||||
| 1186 | m_NSWAdd(const LHS &L, const RHS &R) { | ||||||||
| 1187 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||||||
| 1188 | OverflowingBinaryOperator::NoSignedWrap>( | ||||||||
| 1189 | L, R); | ||||||||
| 1190 | } | ||||||||
| 1191 | template <typename LHS, typename RHS> | ||||||||
| 1192 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||||||
| 1193 | OverflowingBinaryOperator::NoSignedWrap> | ||||||||
| 1194 | m_NSWSub(const LHS &L, const RHS &R) { | ||||||||
| 1195 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||||||
| 1196 | OverflowingBinaryOperator::NoSignedWrap>( | ||||||||
| 1197 | L, R); | ||||||||
| 1198 | } | ||||||||
| 1199 | template <typename LHS, typename RHS> | ||||||||
| 1200 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||||||
| 1201 | OverflowingBinaryOperator::NoSignedWrap> | ||||||||
| 1202 | m_NSWMul(const LHS &L, const RHS &R) { | ||||||||
| 1203 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||||||
| 1204 | OverflowingBinaryOperator::NoSignedWrap>( | ||||||||
| 1205 | L, R); | ||||||||
| 1206 | } | ||||||||
| 1207 | template <typename LHS, typename RHS> | ||||||||
| 1208 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||||||
| 1209 | OverflowingBinaryOperator::NoSignedWrap> | ||||||||
| 1210 | m_NSWShl(const LHS &L, const RHS &R) { | ||||||||
| 1211 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||||||
| 1212 | OverflowingBinaryOperator::NoSignedWrap>( | ||||||||
| 1213 | L, R); | ||||||||
| 1214 | } | ||||||||
| 1215 | |||||||||
| 1216 | template <typename LHS, typename RHS> | ||||||||
| 1217 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||||||
| 1218 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||||||
| 1219 | m_NUWAdd(const LHS &L, const RHS &R) { | ||||||||
| 1220 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add, | ||||||||
| 1221 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||||||
| 1222 | L, R); | ||||||||
| 1223 | } | ||||||||
| 1224 | template <typename LHS, typename RHS> | ||||||||
| 1225 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||||||
| 1226 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||||||
| 1227 | m_NUWSub(const LHS &L, const RHS &R) { | ||||||||
| 1228 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub, | ||||||||
| 1229 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||||||
| 1230 | L, R); | ||||||||
| 1231 | } | ||||||||
| 1232 | template <typename LHS, typename RHS> | ||||||||
| 1233 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||||||
| 1234 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||||||
| 1235 | m_NUWMul(const LHS &L, const RHS &R) { | ||||||||
| 1236 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul, | ||||||||
| 1237 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||||||
| 1238 | L, R); | ||||||||
| 1239 | } | ||||||||
| 1240 | template <typename LHS, typename RHS> | ||||||||
| 1241 | inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||||||
| 1242 | OverflowingBinaryOperator::NoUnsignedWrap> | ||||||||
| 1243 | m_NUWShl(const LHS &L, const RHS &R) { | ||||||||
| 1244 | return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl, | ||||||||
| 1245 | OverflowingBinaryOperator::NoUnsignedWrap>( | ||||||||
| 1246 | L, R); | ||||||||
| 1247 | } | ||||||||
| 1248 | |||||||||
| 1249 | //===----------------------------------------------------------------------===// | ||||||||
| 1250 | // Class that matches a group of binary opcodes. | ||||||||
| 1251 | // | ||||||||
| 1252 | template <typename LHS_t, typename RHS_t, typename Predicate> | ||||||||
| 1253 | struct BinOpPred_match : Predicate { | ||||||||
| 1254 | LHS_t L; | ||||||||
| 1255 | RHS_t R; | ||||||||
| 1256 | |||||||||
| 1257 | BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||||||
| 1258 | |||||||||
| 1259 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1260 | if (auto *I = dyn_cast<Instruction>(V)) | ||||||||
| 1261 | return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) && | ||||||||
| 1262 | R.match(I->getOperand(1)); | ||||||||
| 1263 | if (auto *CE = dyn_cast<ConstantExpr>(V)) | ||||||||
| 1264 | return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) && | ||||||||
| 1265 | R.match(CE->getOperand(1)); | ||||||||
| 1266 | return false; | ||||||||
| 1267 | } | ||||||||
| 1268 | }; | ||||||||
| 1269 | |||||||||
| 1270 | struct is_shift_op { | ||||||||
| 1271 | bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); } | ||||||||
| 1272 | }; | ||||||||
| 1273 | |||||||||
| 1274 | struct is_right_shift_op { | ||||||||
| 1275 | bool isOpType(unsigned Opcode) { | ||||||||
| 1276 | return Opcode == Instruction::LShr || Opcode == Instruction::AShr; | ||||||||
| 1277 | } | ||||||||
| 1278 | }; | ||||||||
| 1279 | |||||||||
| 1280 | struct is_logical_shift_op { | ||||||||
| 1281 | bool isOpType(unsigned Opcode) { | ||||||||
| 1282 | return Opcode == Instruction::LShr || Opcode == Instruction::Shl; | ||||||||
| 1283 | } | ||||||||
| 1284 | }; | ||||||||
| 1285 | |||||||||
| 1286 | struct is_bitwiselogic_op { | ||||||||
| 1287 | bool isOpType(unsigned Opcode) { | ||||||||
| 1288 | return Instruction::isBitwiseLogicOp(Opcode); | ||||||||
| 1289 | } | ||||||||
| 1290 | }; | ||||||||
| 1291 | |||||||||
| 1292 | struct is_idiv_op { | ||||||||
| 1293 | bool isOpType(unsigned Opcode) { | ||||||||
| 1294 | return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv; | ||||||||
| 1295 | } | ||||||||
| 1296 | }; | ||||||||
| 1297 | |||||||||
| 1298 | struct is_irem_op { | ||||||||
| 1299 | bool isOpType(unsigned Opcode) { | ||||||||
| 1300 | return Opcode == Instruction::SRem || Opcode == Instruction::URem; | ||||||||
| 1301 | } | ||||||||
| 1302 | }; | ||||||||
| 1303 | |||||||||
| 1304 | /// Matches shift operations. | ||||||||
| 1305 | template <typename LHS, typename RHS> | ||||||||
| 1306 | inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L, | ||||||||
| 1307 | const RHS &R) { | ||||||||
| 1308 | return BinOpPred_match<LHS, RHS, is_shift_op>(L, R); | ||||||||
| 1309 | } | ||||||||
| 1310 | |||||||||
| 1311 | /// Matches logical shift operations. | ||||||||
| 1312 | template <typename LHS, typename RHS> | ||||||||
| 1313 | inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L, | ||||||||
| 1314 | const RHS &R) { | ||||||||
| 1315 | return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R); | ||||||||
| 1316 | } | ||||||||
| 1317 | |||||||||
| 1318 | /// Matches logical shift operations. | ||||||||
| 1319 | template <typename LHS, typename RHS> | ||||||||
| 1320 | inline BinOpPred_match<LHS, RHS, is_logical_shift_op> | ||||||||
| 1321 | m_LogicalShift(const LHS &L, const RHS &R) { | ||||||||
| 1322 | return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R); | ||||||||
| 1323 | } | ||||||||
| 1324 | |||||||||
| 1325 | /// Matches bitwise logic operations. | ||||||||
| 1326 | template <typename LHS, typename RHS> | ||||||||
| 1327 | inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op> | ||||||||
| 1328 | m_BitwiseLogic(const LHS &L, const RHS &R) { | ||||||||
| 1329 | return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R); | ||||||||
| 1330 | } | ||||||||
| 1331 | |||||||||
| 1332 | /// Matches integer division operations. | ||||||||
| 1333 | template <typename LHS, typename RHS> | ||||||||
| 1334 | inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L, | ||||||||
| 1335 | const RHS &R) { | ||||||||
| 1336 | return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R); | ||||||||
| 1337 | } | ||||||||
| 1338 | |||||||||
| 1339 | /// Matches integer remainder operations. | ||||||||
| 1340 | template <typename LHS, typename RHS> | ||||||||
| 1341 | inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L, | ||||||||
| 1342 | const RHS &R) { | ||||||||
| 1343 | return BinOpPred_match<LHS, RHS, is_irem_op>(L, R); | ||||||||
| 1344 | } | ||||||||
| 1345 | |||||||||
| 1346 | //===----------------------------------------------------------------------===// | ||||||||
| 1347 | // Class that matches exact binary ops. | ||||||||
| 1348 | // | ||||||||
| 1349 | template <typename SubPattern_t> struct Exact_match { | ||||||||
| 1350 | SubPattern_t SubPattern; | ||||||||
| 1351 | |||||||||
| 1352 | Exact_match(const SubPattern_t &SP) : SubPattern(SP) {} | ||||||||
| 1353 | |||||||||
| 1354 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1355 | if (auto *PEO = dyn_cast<PossiblyExactOperator>(V)) | ||||||||
| 1356 | return PEO->isExact() && SubPattern.match(V); | ||||||||
| 1357 | return false; | ||||||||
| 1358 | } | ||||||||
| 1359 | }; | ||||||||
| 1360 | |||||||||
| 1361 | template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) { | ||||||||
| 1362 | return SubPattern; | ||||||||
| 1363 | } | ||||||||
| 1364 | |||||||||
| 1365 | //===----------------------------------------------------------------------===// | ||||||||
| 1366 | // Matchers for CmpInst classes | ||||||||
| 1367 | // | ||||||||
| 1368 | |||||||||
| 1369 | template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy, | ||||||||
| 1370 | bool Commutable = false> | ||||||||
| 1371 | struct CmpClass_match { | ||||||||
| 1372 | PredicateTy &Predicate; | ||||||||
| 1373 | LHS_t L; | ||||||||
| 1374 | RHS_t R; | ||||||||
| 1375 | |||||||||
| 1376 | // The evaluation order is always stable, regardless of Commutability. | ||||||||
| 1377 | // The LHS is always matched first. | ||||||||
| 1378 | CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS) | ||||||||
| 1379 | : Predicate(Pred), L(LHS), R(RHS) {} | ||||||||
| 1380 | |||||||||
| 1381 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1382 | if (auto *I = dyn_cast<Class>(V)) { | ||||||||
| 1383 | if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) { | ||||||||
| 1384 | Predicate = I->getPredicate(); | ||||||||
| 1385 | return true; | ||||||||
| 1386 | } else if (Commutable && L.match(I->getOperand(1)) && | ||||||||
| 1387 | R.match(I->getOperand(0))) { | ||||||||
| 1388 | Predicate = I->getSwappedPredicate(); | ||||||||
| 1389 | return true; | ||||||||
| 1390 | } | ||||||||
| 1391 | } | ||||||||
| 1392 | return false; | ||||||||
| 1393 | } | ||||||||
| 1394 | }; | ||||||||
| 1395 | |||||||||
| 1396 | template <typename LHS, typename RHS> | ||||||||
| 1397 | inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate> | ||||||||
| 1398 | m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||||||
| 1399 | return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R); | ||||||||
| 1400 | } | ||||||||
| 1401 | |||||||||
| 1402 | template <typename LHS, typename RHS> | ||||||||
| 1403 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate> | ||||||||
| 1404 | m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||||||
| 1405 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R); | ||||||||
| 1406 | } | ||||||||
| 1407 | |||||||||
| 1408 | template <typename LHS, typename RHS> | ||||||||
| 1409 | inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate> | ||||||||
| 1410 | m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||||||
| 1411 | return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R); | ||||||||
| 1412 | } | ||||||||
| 1413 | |||||||||
| 1414 | //===----------------------------------------------------------------------===// | ||||||||
| 1415 | // Matchers for instructions with a given opcode and number of operands. | ||||||||
| 1416 | // | ||||||||
| 1417 | |||||||||
| 1418 | /// Matches instructions with Opcode and three operands. | ||||||||
| 1419 | template <typename T0, unsigned Opcode> struct OneOps_match { | ||||||||
| 1420 | T0 Op1; | ||||||||
| 1421 | |||||||||
| 1422 | OneOps_match(const T0 &Op1) : Op1(Op1) {} | ||||||||
| 1423 | |||||||||
| 1424 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1425 | if (V->getValueID() == Value::InstructionVal + Opcode) { | ||||||||
| 1426 | auto *I = cast<Instruction>(V); | ||||||||
| 1427 | return Op1.match(I->getOperand(0)); | ||||||||
| 1428 | } | ||||||||
| 1429 | return false; | ||||||||
| 1430 | } | ||||||||
| 1431 | }; | ||||||||
| 1432 | |||||||||
| 1433 | /// Matches instructions with Opcode and three operands. | ||||||||
| 1434 | template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match { | ||||||||
| 1435 | T0 Op1; | ||||||||
| 1436 | T1 Op2; | ||||||||
| 1437 | |||||||||
| 1438 | TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {} | ||||||||
| 1439 | |||||||||
| 1440 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1441 | if (V->getValueID() == Value::InstructionVal + Opcode) { | ||||||||
| 1442 | auto *I = cast<Instruction>(V); | ||||||||
| 1443 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)); | ||||||||
| 1444 | } | ||||||||
| 1445 | return false; | ||||||||
| 1446 | } | ||||||||
| 1447 | }; | ||||||||
| 1448 | |||||||||
| 1449 | /// Matches instructions with Opcode and three operands. | ||||||||
| 1450 | template <typename T0, typename T1, typename T2, unsigned Opcode> | ||||||||
| 1451 | struct ThreeOps_match { | ||||||||
| 1452 | T0 Op1; | ||||||||
| 1453 | T1 Op2; | ||||||||
| 1454 | T2 Op3; | ||||||||
| 1455 | |||||||||
| 1456 | ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3) | ||||||||
| 1457 | : Op1(Op1), Op2(Op2), Op3(Op3) {} | ||||||||
| 1458 | |||||||||
| 1459 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1460 | if (V->getValueID() == Value::InstructionVal + Opcode) { | ||||||||
| 1461 | auto *I = cast<Instruction>(V); | ||||||||
| 1462 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && | ||||||||
| 1463 | Op3.match(I->getOperand(2)); | ||||||||
| 1464 | } | ||||||||
| 1465 | return false; | ||||||||
| 1466 | } | ||||||||
| 1467 | }; | ||||||||
| 1468 | |||||||||
| 1469 | /// Matches SelectInst. | ||||||||
| 1470 | template <typename Cond, typename LHS, typename RHS> | ||||||||
| 1471 | inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select> | ||||||||
| 1472 | m_Select(const Cond &C, const LHS &L, const RHS &R) { | ||||||||
| 1473 | return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R); | ||||||||
| 1474 | } | ||||||||
| 1475 | |||||||||
| 1476 | /// This matches a select of two constants, e.g.: | ||||||||
| 1477 | /// m_SelectCst<-1, 0>(m_Value(V)) | ||||||||
| 1478 | template <int64_t L, int64_t R, typename Cond> | ||||||||
| 1479 | inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>, | ||||||||
| 1480 | Instruction::Select> | ||||||||
| 1481 | m_SelectCst(const Cond &C) { | ||||||||
| 1482 | return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>()); | ||||||||
| 1483 | } | ||||||||
| 1484 | |||||||||
| 1485 | /// Matches FreezeInst. | ||||||||
| 1486 | template <typename OpTy> | ||||||||
| 1487 | inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) { | ||||||||
| 1488 | return OneOps_match<OpTy, Instruction::Freeze>(Op); | ||||||||
| 1489 | } | ||||||||
| 1490 | |||||||||
| 1491 | /// Matches InsertElementInst. | ||||||||
| 1492 | template <typename Val_t, typename Elt_t, typename Idx_t> | ||||||||
| 1493 | inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement> | ||||||||
| 1494 | m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) { | ||||||||
| 1495 | return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>( | ||||||||
| 1496 | Val, Elt, Idx); | ||||||||
| 1497 | } | ||||||||
| 1498 | |||||||||
| 1499 | /// Matches ExtractElementInst. | ||||||||
| 1500 | template <typename Val_t, typename Idx_t> | ||||||||
| 1501 | inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement> | ||||||||
| 1502 | m_ExtractElt(const Val_t &Val, const Idx_t &Idx) { | ||||||||
| 1503 | return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx); | ||||||||
| 1504 | } | ||||||||
| 1505 | |||||||||
| 1506 | /// Matches shuffle. | ||||||||
| 1507 | template <typename T0, typename T1, typename T2> struct Shuffle_match { | ||||||||
| 1508 | T0 Op1; | ||||||||
| 1509 | T1 Op2; | ||||||||
| 1510 | T2 Mask; | ||||||||
| 1511 | |||||||||
| 1512 | Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask) | ||||||||
| 1513 | : Op1(Op1), Op2(Op2), Mask(Mask) {} | ||||||||
| 1514 | |||||||||
| 1515 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1516 | if (auto *I = dyn_cast<ShuffleVectorInst>(V)) { | ||||||||
| 1517 | return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) && | ||||||||
| 1518 | Mask.match(I->getShuffleMask()); | ||||||||
| 1519 | } | ||||||||
| 1520 | return false; | ||||||||
| 1521 | } | ||||||||
| 1522 | }; | ||||||||
| 1523 | |||||||||
| 1524 | struct m_Mask { | ||||||||
| 1525 | ArrayRef<int> &MaskRef; | ||||||||
| 1526 | m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} | ||||||||
| 1527 | bool match(ArrayRef<int> Mask) { | ||||||||
| 1528 | MaskRef = Mask; | ||||||||
| 1529 | return true; | ||||||||
| 1530 | } | ||||||||
| 1531 | }; | ||||||||
| 1532 | |||||||||
| 1533 | struct m_ZeroMask { | ||||||||
| 1534 | bool match(ArrayRef<int> Mask) { | ||||||||
| 1535 | return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; }); | ||||||||
| 1536 | } | ||||||||
| 1537 | }; | ||||||||
| 1538 | |||||||||
| 1539 | struct m_SpecificMask { | ||||||||
| 1540 | ArrayRef<int> &MaskRef; | ||||||||
| 1541 | m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {} | ||||||||
| 1542 | bool match(ArrayRef<int> Mask) { return MaskRef == Mask; } | ||||||||
| 1543 | }; | ||||||||
| 1544 | |||||||||
| 1545 | struct m_SplatOrUndefMask { | ||||||||
| 1546 | int &SplatIndex; | ||||||||
| 1547 | m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {} | ||||||||
| 1548 | bool match(ArrayRef<int> Mask) { | ||||||||
| 1549 | auto First = find_if(Mask, [](int Elem) { return Elem != -1; }); | ||||||||
| 1550 | if (First == Mask.end()) | ||||||||
| 1551 | return false; | ||||||||
| 1552 | SplatIndex = *First; | ||||||||
| 1553 | return all_of(Mask, | ||||||||
| 1554 | [First](int Elem) { return Elem == *First || Elem == -1; }); | ||||||||
| 1555 | } | ||||||||
| 1556 | }; | ||||||||
| 1557 | |||||||||
| 1558 | /// Matches ShuffleVectorInst independently of mask value. | ||||||||
| 1559 | template <typename V1_t, typename V2_t> | ||||||||
| 1560 | inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector> | ||||||||
| 1561 | m_Shuffle(const V1_t &v1, const V2_t &v2) { | ||||||||
| 1562 | return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2); | ||||||||
| 1563 | } | ||||||||
| 1564 | |||||||||
| 1565 | template <typename V1_t, typename V2_t, typename Mask_t> | ||||||||
| 1566 | inline Shuffle_match<V1_t, V2_t, Mask_t> | ||||||||
| 1567 | m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) { | ||||||||
| 1568 | return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask); | ||||||||
| 1569 | } | ||||||||
| 1570 | |||||||||
| 1571 | /// Matches LoadInst. | ||||||||
| 1572 | template <typename OpTy> | ||||||||
| 1573 | inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) { | ||||||||
| 1574 | return OneOps_match<OpTy, Instruction::Load>(Op); | ||||||||
| 1575 | } | ||||||||
| 1576 | |||||||||
| 1577 | /// Matches StoreInst. | ||||||||
| 1578 | template <typename ValueOpTy, typename PointerOpTy> | ||||||||
| 1579 | inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store> | ||||||||
| 1580 | m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) { | ||||||||
| 1581 | return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp, | ||||||||
| 1582 | PointerOp); | ||||||||
| 1583 | } | ||||||||
| 1584 | |||||||||
| 1585 | //===----------------------------------------------------------------------===// | ||||||||
| 1586 | // Matchers for CastInst classes | ||||||||
| 1587 | // | ||||||||
| 1588 | |||||||||
| 1589 | template <typename Op_t, unsigned Opcode> struct CastClass_match { | ||||||||
| 1590 | Op_t Op; | ||||||||
| 1591 | |||||||||
| 1592 | CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {} | ||||||||
| 1593 | |||||||||
| 1594 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1595 | if (auto *O = dyn_cast<Operator>(V)) | ||||||||
| 1596 | return O->getOpcode() == Opcode && Op.match(O->getOperand(0)); | ||||||||
| 1597 | return false; | ||||||||
| 1598 | } | ||||||||
| 1599 | }; | ||||||||
| 1600 | |||||||||
| 1601 | /// Matches BitCast. | ||||||||
| 1602 | template <typename OpTy> | ||||||||
| 1603 | inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) { | ||||||||
| 1604 | return CastClass_match<OpTy, Instruction::BitCast>(Op); | ||||||||
| 1605 | } | ||||||||
| 1606 | |||||||||
| 1607 | /// Matches PtrToInt. | ||||||||
| 1608 | template <typename OpTy> | ||||||||
| 1609 | inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) { | ||||||||
| 1610 | return CastClass_match<OpTy, Instruction::PtrToInt>(Op); | ||||||||
| 1611 | } | ||||||||
| 1612 | |||||||||
| 1613 | /// Matches IntToPtr. | ||||||||
| 1614 | template <typename OpTy> | ||||||||
| 1615 | inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) { | ||||||||
| 1616 | return CastClass_match<OpTy, Instruction::IntToPtr>(Op); | ||||||||
| 1617 | } | ||||||||
| 1618 | |||||||||
| 1619 | /// Matches Trunc. | ||||||||
| 1620 | template <typename OpTy> | ||||||||
| 1621 | inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) { | ||||||||
| 1622 | return CastClass_match<OpTy, Instruction::Trunc>(Op); | ||||||||
| 1623 | } | ||||||||
| 1624 | |||||||||
| 1625 | template <typename OpTy> | ||||||||
| 1626 | inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy> | ||||||||
| 1627 | m_TruncOrSelf(const OpTy &Op) { | ||||||||
| 1628 | return m_CombineOr(m_Trunc(Op), Op); | ||||||||
| 1629 | } | ||||||||
| 1630 | |||||||||
| 1631 | /// Matches SExt. | ||||||||
| 1632 | template <typename OpTy> | ||||||||
| 1633 | inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) { | ||||||||
| 1634 | return CastClass_match<OpTy, Instruction::SExt>(Op); | ||||||||
| 1635 | } | ||||||||
| 1636 | |||||||||
| 1637 | /// Matches ZExt. | ||||||||
| 1638 | template <typename OpTy> | ||||||||
| 1639 | inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) { | ||||||||
| 1640 | return CastClass_match<OpTy, Instruction::ZExt>(Op); | ||||||||
| 1641 | } | ||||||||
| 1642 | |||||||||
| 1643 | template <typename OpTy> | ||||||||
| 1644 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy> | ||||||||
| 1645 | m_ZExtOrSelf(const OpTy &Op) { | ||||||||
| 1646 | return m_CombineOr(m_ZExt(Op), Op); | ||||||||
| 1647 | } | ||||||||
| 1648 | |||||||||
| 1649 | template <typename OpTy> | ||||||||
| 1650 | inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy> | ||||||||
| 1651 | m_SExtOrSelf(const OpTy &Op) { | ||||||||
| 1652 | return m_CombineOr(m_SExt(Op), Op); | ||||||||
| 1653 | } | ||||||||
| 1654 | |||||||||
| 1655 | template <typename OpTy> | ||||||||
| 1656 | inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, | ||||||||
| 1657 | CastClass_match<OpTy, Instruction::SExt>> | ||||||||
| 1658 | m_ZExtOrSExt(const OpTy &Op) { | ||||||||
| 1659 | return m_CombineOr(m_ZExt(Op), m_SExt(Op)); | ||||||||
| 1660 | } | ||||||||
| 1661 | |||||||||
| 1662 | template <typename OpTy> | ||||||||
| 1663 | inline match_combine_or< | ||||||||
| 1664 | match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, | ||||||||
| 1665 | CastClass_match<OpTy, Instruction::SExt>>, | ||||||||
| 1666 | OpTy> | ||||||||
| 1667 | m_ZExtOrSExtOrSelf(const OpTy &Op) { | ||||||||
| 1668 | return m_CombineOr(m_ZExtOrSExt(Op), Op); | ||||||||
| 1669 | } | ||||||||
| 1670 | |||||||||
| 1671 | template <typename OpTy> | ||||||||
| 1672 | inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) { | ||||||||
| 1673 | return CastClass_match<OpTy, Instruction::UIToFP>(Op); | ||||||||
| 1674 | } | ||||||||
| 1675 | |||||||||
| 1676 | template <typename OpTy> | ||||||||
| 1677 | inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) { | ||||||||
| 1678 | return CastClass_match<OpTy, Instruction::SIToFP>(Op); | ||||||||
| 1679 | } | ||||||||
| 1680 | |||||||||
| 1681 | template <typename OpTy> | ||||||||
| 1682 | inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) { | ||||||||
| 1683 | return CastClass_match<OpTy, Instruction::FPToUI>(Op); | ||||||||
| 1684 | } | ||||||||
| 1685 | |||||||||
| 1686 | template <typename OpTy> | ||||||||
| 1687 | inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) { | ||||||||
| 1688 | return CastClass_match<OpTy, Instruction::FPToSI>(Op); | ||||||||
| 1689 | } | ||||||||
| 1690 | |||||||||
| 1691 | template <typename OpTy> | ||||||||
| 1692 | inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) { | ||||||||
| 1693 | return CastClass_match<OpTy, Instruction::FPTrunc>(Op); | ||||||||
| 1694 | } | ||||||||
| 1695 | |||||||||
| 1696 | template <typename OpTy> | ||||||||
| 1697 | inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) { | ||||||||
| 1698 | return CastClass_match<OpTy, Instruction::FPExt>(Op); | ||||||||
| 1699 | } | ||||||||
| 1700 | |||||||||
| 1701 | //===----------------------------------------------------------------------===// | ||||||||
| 1702 | // Matchers for control flow. | ||||||||
| 1703 | // | ||||||||
| 1704 | |||||||||
| 1705 | struct br_match { | ||||||||
| 1706 | BasicBlock *&Succ; | ||||||||
| 1707 | |||||||||
| 1708 | br_match(BasicBlock *&Succ) : Succ(Succ) {} | ||||||||
| 1709 | |||||||||
| 1710 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1711 | if (auto *BI = dyn_cast<BranchInst>(V)) | ||||||||
| 1712 | if (BI->isUnconditional()) { | ||||||||
| 1713 | Succ = BI->getSuccessor(0); | ||||||||
| 1714 | return true; | ||||||||
| 1715 | } | ||||||||
| 1716 | return false; | ||||||||
| 1717 | } | ||||||||
| 1718 | }; | ||||||||
| 1719 | |||||||||
| 1720 | inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); } | ||||||||
| 1721 | |||||||||
| 1722 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> | ||||||||
| 1723 | struct brc_match { | ||||||||
| 1724 | Cond_t Cond; | ||||||||
| 1725 | TrueBlock_t T; | ||||||||
| 1726 | FalseBlock_t F; | ||||||||
| 1727 | |||||||||
| 1728 | brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f) | ||||||||
| 1729 | : Cond(C), T(t), F(f) {} | ||||||||
| 1730 | |||||||||
| 1731 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1732 | if (auto *BI = dyn_cast<BranchInst>(V)) | ||||||||
| 1733 | if (BI->isConditional() && Cond.match(BI->getCondition())) | ||||||||
| 1734 | return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1)); | ||||||||
| 1735 | return false; | ||||||||
| 1736 | } | ||||||||
| 1737 | }; | ||||||||
| 1738 | |||||||||
| 1739 | template <typename Cond_t> | ||||||||
| 1740 | inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>> | ||||||||
| 1741 | m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) { | ||||||||
| 1742 | return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>( | ||||||||
| 1743 | C, m_BasicBlock(T), m_BasicBlock(F)); | ||||||||
| 1744 | } | ||||||||
| 1745 | |||||||||
| 1746 | template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t> | ||||||||
| 1747 | inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t> | ||||||||
| 1748 | m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) { | ||||||||
| 1749 | return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F); | ||||||||
| 1750 | } | ||||||||
| 1751 | |||||||||
| 1752 | //===----------------------------------------------------------------------===// | ||||||||
| 1753 | // Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y). | ||||||||
| 1754 | // | ||||||||
| 1755 | |||||||||
| 1756 | template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t, | ||||||||
| 1757 | bool Commutable = false> | ||||||||
| 1758 | struct MaxMin_match { | ||||||||
| 1759 | using PredType = Pred_t; | ||||||||
| 1760 | LHS_t L; | ||||||||
| 1761 | RHS_t R; | ||||||||
| 1762 | |||||||||
| 1763 | // The evaluation order is always stable, regardless of Commutability. | ||||||||
| 1764 | // The LHS is always matched first. | ||||||||
| 1765 | MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {} | ||||||||
| 1766 | |||||||||
| 1767 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1768 | if (auto *II = dyn_cast<IntrinsicInst>(V)) { | ||||||||
| 1769 | Intrinsic::ID IID = II->getIntrinsicID(); | ||||||||
| 1770 | if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) || | ||||||||
| 1771 | (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) || | ||||||||
| 1772 | (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) || | ||||||||
| 1773 | (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) { | ||||||||
| 1774 | Value *LHS = II->getOperand(0), *RHS = II->getOperand(1); | ||||||||
| 1775 | return (L.match(LHS) && R.match(RHS)) || | ||||||||
| 1776 | (Commutable && L.match(RHS) && R.match(LHS)); | ||||||||
| 1777 | } | ||||||||
| 1778 | } | ||||||||
| 1779 | // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x". | ||||||||
| 1780 | auto *SI = dyn_cast<SelectInst>(V); | ||||||||
| 1781 | if (!SI) | ||||||||
| 1782 | return false; | ||||||||
| 1783 | auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition()); | ||||||||
| 1784 | if (!Cmp) | ||||||||
| 1785 | return false; | ||||||||
| 1786 | // At this point we have a select conditioned on a comparison. Check that | ||||||||
| 1787 | // it is the values returned by the select that are being compared. | ||||||||
| 1788 | auto *TrueVal = SI->getTrueValue(); | ||||||||
| 1789 | auto *FalseVal = SI->getFalseValue(); | ||||||||
| 1790 | auto *LHS = Cmp->getOperand(0); | ||||||||
| 1791 | auto *RHS = Cmp->getOperand(1); | ||||||||
| 1792 | if ((TrueVal != LHS || FalseVal != RHS) && | ||||||||
| 1793 | (TrueVal != RHS || FalseVal != LHS)) | ||||||||
| 1794 | return false; | ||||||||
| 1795 | typename CmpInst_t::Predicate Pred = | ||||||||
| 1796 | LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate(); | ||||||||
| 1797 | // Does "(x pred y) ? x : y" represent the desired max/min operation? | ||||||||
| 1798 | if (!Pred_t::match(Pred)) | ||||||||
| 1799 | return false; | ||||||||
| 1800 | // It does! Bind the operands. | ||||||||
| 1801 | return (L.match(LHS) && R.match(RHS)) || | ||||||||
| 1802 | (Commutable && L.match(RHS) && R.match(LHS)); | ||||||||
| 1803 | } | ||||||||
| 1804 | }; | ||||||||
| 1805 | |||||||||
| 1806 | /// Helper class for identifying signed max predicates. | ||||||||
| 1807 | struct smax_pred_ty { | ||||||||
| 1808 | static bool match(ICmpInst::Predicate Pred) { | ||||||||
| 1809 | return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE; | ||||||||
| 1810 | } | ||||||||
| 1811 | }; | ||||||||
| 1812 | |||||||||
| 1813 | /// Helper class for identifying signed min predicates. | ||||||||
| 1814 | struct smin_pred_ty { | ||||||||
| 1815 | static bool match(ICmpInst::Predicate Pred) { | ||||||||
| 1816 | return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE; | ||||||||
| 1817 | } | ||||||||
| 1818 | }; | ||||||||
| 1819 | |||||||||
| 1820 | /// Helper class for identifying unsigned max predicates. | ||||||||
| 1821 | struct umax_pred_ty { | ||||||||
| 1822 | static bool match(ICmpInst::Predicate Pred) { | ||||||||
| 1823 | return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE; | ||||||||
| 1824 | } | ||||||||
| 1825 | }; | ||||||||
| 1826 | |||||||||
| 1827 | /// Helper class for identifying unsigned min predicates. | ||||||||
| 1828 | struct umin_pred_ty { | ||||||||
| 1829 | static bool match(ICmpInst::Predicate Pred) { | ||||||||
| 1830 | return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE; | ||||||||
| 1831 | } | ||||||||
| 1832 | }; | ||||||||
| 1833 | |||||||||
| 1834 | /// Helper class for identifying ordered max predicates. | ||||||||
| 1835 | struct ofmax_pred_ty { | ||||||||
| 1836 | static bool match(FCmpInst::Predicate Pred) { | ||||||||
| 1837 | return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE; | ||||||||
| 1838 | } | ||||||||
| 1839 | }; | ||||||||
| 1840 | |||||||||
| 1841 | /// Helper class for identifying ordered min predicates. | ||||||||
| 1842 | struct ofmin_pred_ty { | ||||||||
| 1843 | static bool match(FCmpInst::Predicate Pred) { | ||||||||
| 1844 | return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE; | ||||||||
| 1845 | } | ||||||||
| 1846 | }; | ||||||||
| 1847 | |||||||||
| 1848 | /// Helper class for identifying unordered max predicates. | ||||||||
| 1849 | struct ufmax_pred_ty { | ||||||||
| 1850 | static bool match(FCmpInst::Predicate Pred) { | ||||||||
| 1851 | return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE; | ||||||||
| 1852 | } | ||||||||
| 1853 | }; | ||||||||
| 1854 | |||||||||
| 1855 | /// Helper class for identifying unordered min predicates. | ||||||||
| 1856 | struct ufmin_pred_ty { | ||||||||
| 1857 | static bool match(FCmpInst::Predicate Pred) { | ||||||||
| 1858 | return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE; | ||||||||
| 1859 | } | ||||||||
| 1860 | }; | ||||||||
| 1861 | |||||||||
| 1862 | template <typename LHS, typename RHS> | ||||||||
| 1863 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L, | ||||||||
| 1864 | const RHS &R) { | ||||||||
| 1865 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R); | ||||||||
| 1866 | } | ||||||||
| 1867 | |||||||||
| 1868 | template <typename LHS, typename RHS> | ||||||||
| 1869 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L, | ||||||||
| 1870 | const RHS &R) { | ||||||||
| 1871 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R); | ||||||||
| 1872 | } | ||||||||
| 1873 | |||||||||
| 1874 | template <typename LHS, typename RHS> | ||||||||
| 1875 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L, | ||||||||
| 1876 | const RHS &R) { | ||||||||
| 1877 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R); | ||||||||
| 1878 | } | ||||||||
| 1879 | |||||||||
| 1880 | template <typename LHS, typename RHS> | ||||||||
| 1881 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L, | ||||||||
| 1882 | const RHS &R) { | ||||||||
| 1883 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R); | ||||||||
| 1884 | } | ||||||||
| 1885 | |||||||||
| 1886 | template <typename LHS, typename RHS> | ||||||||
| 1887 | inline match_combine_or< | ||||||||
| 1888 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>, | ||||||||
| 1889 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>, | ||||||||
| 1890 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>, | ||||||||
| 1891 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>> | ||||||||
| 1892 | m_MaxOrMin(const LHS &L, const RHS &R) { | ||||||||
| 1893 | return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)), | ||||||||
| 1894 | m_CombineOr(m_UMax(L, R), m_UMin(L, R))); | ||||||||
| 1895 | } | ||||||||
| 1896 | |||||||||
| 1897 | /// Match an 'ordered' floating point maximum function. | ||||||||
| 1898 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||||||
| 1899 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||||||
| 1900 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' | ||||||||
| 1901 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||||||
| 1902 | /// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate. | ||||||||
| 1903 | /// | ||||||||
| 1904 | /// max(L, R) iff L and R are not NaN | ||||||||
| 1905 | /// m_OrdFMax(L, R) = R iff L or R are NaN | ||||||||
| 1906 | template <typename LHS, typename RHS> | ||||||||
| 1907 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L, | ||||||||
| 1908 | const RHS &R) { | ||||||||
| 1909 | return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R); | ||||||||
| 1910 | } | ||||||||
| 1911 | |||||||||
| 1912 | /// Match an 'ordered' floating point minimum function. | ||||||||
| 1913 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||||||
| 1914 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||||||
| 1915 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' | ||||||||
| 1916 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||||||
| 1917 | /// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate. | ||||||||
| 1918 | /// | ||||||||
| 1919 | /// min(L, R) iff L and R are not NaN | ||||||||
| 1920 | /// m_OrdFMin(L, R) = R iff L or R are NaN | ||||||||
| 1921 | template <typename LHS, typename RHS> | ||||||||
| 1922 | inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L, | ||||||||
| 1923 | const RHS &R) { | ||||||||
| 1924 | return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R); | ||||||||
| 1925 | } | ||||||||
| 1926 | |||||||||
| 1927 | /// Match an 'unordered' floating point maximum function. | ||||||||
| 1928 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||||||
| 1929 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||||||
| 1930 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum' | ||||||||
| 1931 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||||||
| 1932 | /// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate. | ||||||||
| 1933 | /// | ||||||||
| 1934 | /// max(L, R) iff L and R are not NaN | ||||||||
| 1935 | /// m_UnordFMax(L, R) = L iff L or R are NaN | ||||||||
| 1936 | template <typename LHS, typename RHS> | ||||||||
| 1937 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty> | ||||||||
| 1938 | m_UnordFMax(const LHS &L, const RHS &R) { | ||||||||
| 1939 | return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R); | ||||||||
| 1940 | } | ||||||||
| 1941 | |||||||||
| 1942 | /// Match an 'unordered' floating point minimum function. | ||||||||
| 1943 | /// Floating point has one special value 'NaN'. Therefore, there is no total | ||||||||
| 1944 | /// order. However, if we can ignore the 'NaN' value (for example, because of a | ||||||||
| 1945 | /// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum' | ||||||||
| 1946 | /// semantics. In the presence of 'NaN' we have to preserve the original | ||||||||
| 1947 | /// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate. | ||||||||
| 1948 | /// | ||||||||
| 1949 | /// min(L, R) iff L and R are not NaN | ||||||||
| 1950 | /// m_UnordFMin(L, R) = L iff L or R are NaN | ||||||||
| 1951 | template <typename LHS, typename RHS> | ||||||||
| 1952 | inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty> | ||||||||
| 1953 | m_UnordFMin(const LHS &L, const RHS &R) { | ||||||||
| 1954 | return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R); | ||||||||
| 1955 | } | ||||||||
| 1956 | |||||||||
| 1957 | //===----------------------------------------------------------------------===// | ||||||||
| 1958 | // Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b | ||||||||
| 1959 | // Note that S might be matched to other instructions than AddInst. | ||||||||
| 1960 | // | ||||||||
| 1961 | |||||||||
| 1962 | template <typename LHS_t, typename RHS_t, typename Sum_t> | ||||||||
| 1963 | struct UAddWithOverflow_match { | ||||||||
| 1964 | LHS_t L; | ||||||||
| 1965 | RHS_t R; | ||||||||
| 1966 | Sum_t S; | ||||||||
| 1967 | |||||||||
| 1968 | UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S) | ||||||||
| 1969 | : L(L), R(R), S(S) {} | ||||||||
| 1970 | |||||||||
| 1971 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 1972 | Value *ICmpLHS, *ICmpRHS; | ||||||||
| 1973 | ICmpInst::Predicate Pred; | ||||||||
| 1974 | if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V)) | ||||||||
| 1975 | return false; | ||||||||
| 1976 | |||||||||
| 1977 | Value *AddLHS, *AddRHS; | ||||||||
| 1978 | auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS)); | ||||||||
| 1979 | |||||||||
| 1980 | // (a + b) u< a, (a + b) u< b | ||||||||
| 1981 | if (Pred == ICmpInst::ICMP_ULT) | ||||||||
| 1982 | if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS)) | ||||||||
| 1983 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); | ||||||||
| 1984 | |||||||||
| 1985 | // a >u (a + b), b >u (a + b) | ||||||||
| 1986 | if (Pred == ICmpInst::ICMP_UGT) | ||||||||
| 1987 | if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS)) | ||||||||
| 1988 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); | ||||||||
| 1989 | |||||||||
| 1990 | Value *Op1; | ||||||||
| 1991 | auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes())); | ||||||||
| 1992 | // (a ^ -1) <u b | ||||||||
| 1993 | if (Pred == ICmpInst::ICMP_ULT) { | ||||||||
| 1994 | if (XorExpr.match(ICmpLHS)) | ||||||||
| 1995 | return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS); | ||||||||
| 1996 | } | ||||||||
| 1997 | // b > u (a ^ -1) | ||||||||
| 1998 | if (Pred == ICmpInst::ICMP_UGT) { | ||||||||
| 1999 | if (XorExpr.match(ICmpRHS)) | ||||||||
| 2000 | return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS); | ||||||||
| 2001 | } | ||||||||
| 2002 | |||||||||
| 2003 | // Match special-case for increment-by-1. | ||||||||
| 2004 | if (Pred == ICmpInst::ICMP_EQ) { | ||||||||
| 2005 | // (a + 1) == 0 | ||||||||
| 2006 | // (1 + a) == 0 | ||||||||
| 2007 | if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) && | ||||||||
| 2008 | (m_One().match(AddLHS) || m_One().match(AddRHS))) | ||||||||
| 2009 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); | ||||||||
| 2010 | // 0 == (a + 1) | ||||||||
| 2011 | // 0 == (1 + a) | ||||||||
| 2012 | if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) && | ||||||||
| 2013 | (m_One().match(AddLHS) || m_One().match(AddRHS))) | ||||||||
| 2014 | return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); | ||||||||
| 2015 | } | ||||||||
| 2016 | |||||||||
| 2017 | return false; | ||||||||
| 2018 | } | ||||||||
| 2019 | }; | ||||||||
| 2020 | |||||||||
| 2021 | /// Match an icmp instruction checking for unsigned overflow on addition. | ||||||||
| 2022 | /// | ||||||||
| 2023 | /// S is matched to the addition whose result is being checked for overflow, and | ||||||||
| 2024 | /// L and R are matched to the LHS and RHS of S. | ||||||||
| 2025 | template <typename LHS_t, typename RHS_t, typename Sum_t> | ||||||||
| 2026 | UAddWithOverflow_match<LHS_t, RHS_t, Sum_t> | ||||||||
| 2027 | m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) { | ||||||||
| 2028 | return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S); | ||||||||
| 2029 | } | ||||||||
| 2030 | |||||||||
| 2031 | template <typename Opnd_t> struct Argument_match { | ||||||||
| 2032 | unsigned OpI; | ||||||||
| 2033 | Opnd_t Val; | ||||||||
| 2034 | |||||||||
| 2035 | Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} | ||||||||
| 2036 | |||||||||
| 2037 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 2038 | // FIXME: Should likely be switched to use `CallBase`. | ||||||||
| 2039 | if (const auto *CI = dyn_cast<CallInst>(V)) | ||||||||
| 2040 | return Val.match(CI->getArgOperand(OpI)); | ||||||||
| 2041 | return false; | ||||||||
| 2042 | } | ||||||||
| 2043 | }; | ||||||||
| 2044 | |||||||||
| 2045 | /// Match an argument. | ||||||||
| 2046 | template <unsigned OpI, typename Opnd_t> | ||||||||
| 2047 | inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) { | ||||||||
| 2048 | return Argument_match<Opnd_t>(OpI, Op); | ||||||||
| 2049 | } | ||||||||
| 2050 | |||||||||
| 2051 | /// Intrinsic matchers. | ||||||||
| 2052 | struct IntrinsicID_match { | ||||||||
| 2053 | unsigned ID; | ||||||||
| 2054 | |||||||||
| 2055 | IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} | ||||||||
| 2056 | |||||||||
| 2057 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 2058 | if (const auto *CI = dyn_cast<CallInst>(V)) | ||||||||
| 2059 | if (const auto *F = CI->getCalledFunction()) | ||||||||
| 2060 | return F->getIntrinsicID() == ID; | ||||||||
| 2061 | return false; | ||||||||
| 2062 | } | ||||||||
| 2063 | }; | ||||||||
| 2064 | |||||||||
| 2065 | /// Intrinsic matches are combinations of ID matchers, and argument | ||||||||
| 2066 | /// matchers. Higher arity matcher are defined recursively in terms of and-ing | ||||||||
| 2067 | /// them with lower arity matchers. Here's some convenient typedefs for up to | ||||||||
| 2068 | /// several arguments, and more can be added as needed | ||||||||
| 2069 | template <typename T0 = void, typename T1 = void, typename T2 = void, | ||||||||
| 2070 | typename T3 = void, typename T4 = void, typename T5 = void, | ||||||||
| 2071 | typename T6 = void, typename T7 = void, typename T8 = void, | ||||||||
| 2072 | typename T9 = void, typename T10 = void> | ||||||||
| 2073 | struct m_Intrinsic_Ty; | ||||||||
| 2074 | template <typename T0> struct m_Intrinsic_Ty<T0> { | ||||||||
| 2075 | using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>; | ||||||||
| 2076 | }; | ||||||||
| 2077 | template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { | ||||||||
| 2078 | using Ty = | ||||||||
| 2079 | match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>; | ||||||||
| 2080 | }; | ||||||||
| 2081 | template <typename T0, typename T1, typename T2> | ||||||||
| 2082 | struct m_Intrinsic_Ty<T0, T1, T2> { | ||||||||
| 2083 | using Ty = | ||||||||
| 2084 | match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, | ||||||||
| 2085 | Argument_match<T2>>; | ||||||||
| 2086 | }; | ||||||||
| 2087 | template <typename T0, typename T1, typename T2, typename T3> | ||||||||
| 2088 | struct m_Intrinsic_Ty<T0, T1, T2, T3> { | ||||||||
| 2089 | using Ty = | ||||||||
| 2090 | match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, | ||||||||
| 2091 | Argument_match<T3>>; | ||||||||
| 2092 | }; | ||||||||
| 2093 | |||||||||
| 2094 | template <typename T0, typename T1, typename T2, typename T3, typename T4> | ||||||||
| 2095 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { | ||||||||
| 2096 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty, | ||||||||
| 2097 | Argument_match<T4>>; | ||||||||
| 2098 | }; | ||||||||
| 2099 | |||||||||
| 2100 | template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5> | ||||||||
| 2101 | struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> { | ||||||||
| 2102 | using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty, | ||||||||
| 2103 | Argument_match<T5>>; | ||||||||
| 2104 | }; | ||||||||
| 2105 | |||||||||
| 2106 | /// Match intrinsic calls like this: | ||||||||
| 2107 | /// m_Intrinsic<Intrinsic::fabs>(m_Value(X)) | ||||||||
| 2108 | template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { | ||||||||
| 2109 | return IntrinsicID_match(IntrID); | ||||||||
| 2110 | } | ||||||||
| 2111 | |||||||||
| 2112 | /// Matches MaskedLoad Intrinsic. | ||||||||
| 2113 | template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3> | ||||||||
| 2114 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty | ||||||||
| 2115 | m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2, | ||||||||
| 2116 | const Opnd3 &Op3) { | ||||||||
| 2117 | return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3); | ||||||||
| 2118 | } | ||||||||
| 2119 | |||||||||
| 2120 | template <Intrinsic::ID IntrID, typename T0> | ||||||||
| 2121 | inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { | ||||||||
| 2122 | return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); | ||||||||
| 2123 | } | ||||||||
| 2124 | |||||||||
| 2125 | template <Intrinsic::ID IntrID, typename T0, typename T1> | ||||||||
| 2126 | inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0, | ||||||||
| 2127 | const T1 &Op1) { | ||||||||
| 2128 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1)); | ||||||||
| 2129 | } | ||||||||
| 2130 | |||||||||
| 2131 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2> | ||||||||
| 2132 | inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty | ||||||||
| 2133 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { | ||||||||
| 2134 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2)); | ||||||||
| 2135 | } | ||||||||
| 2136 | |||||||||
| 2137 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | ||||||||
| 2138 | typename T3> | ||||||||
| 2139 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty | ||||||||
| 2140 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { | ||||||||
| 2141 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); | ||||||||
| 2142 | } | ||||||||
| 2143 | |||||||||
| 2144 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | ||||||||
| 2145 | typename T3, typename T4> | ||||||||
| 2146 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty | ||||||||
| 2147 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, | ||||||||
| 2148 | const T4 &Op4) { | ||||||||
| 2149 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3), | ||||||||
| 2150 | m_Argument<4>(Op4)); | ||||||||
| 2151 | } | ||||||||
| 2152 | |||||||||
| 2153 | template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | ||||||||
| 2154 | typename T3, typename T4, typename T5> | ||||||||
| 2155 | inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty | ||||||||
| 2156 | m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, | ||||||||
| 2157 | const T4 &Op4, const T5 &Op5) { | ||||||||
| 2158 | return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4), | ||||||||
| 2159 | m_Argument<5>(Op5)); | ||||||||
| 2160 | } | ||||||||
| 2161 | |||||||||
| 2162 | // Helper intrinsic matching specializations. | ||||||||
| 2163 | template <typename Opnd0> | ||||||||
| 2164 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) { | ||||||||
| 2165 | return m_Intrinsic<Intrinsic::bitreverse>(Op0); | ||||||||
| 2166 | } | ||||||||
| 2167 | |||||||||
| 2168 | template <typename Opnd0> | ||||||||
| 2169 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) { | ||||||||
| 2170 | return m_Intrinsic<Intrinsic::bswap>(Op0); | ||||||||
| 2171 | } | ||||||||
| 2172 | |||||||||
| 2173 | template <typename Opnd0> | ||||||||
| 2174 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) { | ||||||||
| 2175 | return m_Intrinsic<Intrinsic::fabs>(Op0); | ||||||||
| 2176 | } | ||||||||
| 2177 | |||||||||
| 2178 | template <typename Opnd0> | ||||||||
| 2179 | inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) { | ||||||||
| 2180 | return m_Intrinsic<Intrinsic::canonicalize>(Op0); | ||||||||
| 2181 | } | ||||||||
| 2182 | |||||||||
| 2183 | template <typename Opnd0, typename Opnd1> | ||||||||
| 2184 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0, | ||||||||
| 2185 | const Opnd1 &Op1) { | ||||||||
| 2186 | return m_Intrinsic<Intrinsic::minnum>(Op0, Op1); | ||||||||
| 2187 | } | ||||||||
| 2188 | |||||||||
| 2189 | template <typename Opnd0, typename Opnd1> | ||||||||
| 2190 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0, | ||||||||
| 2191 | const Opnd1 &Op1) { | ||||||||
| 2192 | return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1); | ||||||||
| 2193 | } | ||||||||
| 2194 | |||||||||
| 2195 | template <typename Opnd0, typename Opnd1, typename Opnd2> | ||||||||
| 2196 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty | ||||||||
| 2197 | m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { | ||||||||
| 2198 | return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2); | ||||||||
| 2199 | } | ||||||||
| 2200 | |||||||||
| 2201 | template <typename Opnd0, typename Opnd1, typename Opnd2> | ||||||||
| 2202 | inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty | ||||||||
| 2203 | m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) { | ||||||||
| 2204 | return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2); | ||||||||
| 2205 | } | ||||||||
| 2206 | |||||||||
| 2207 | //===----------------------------------------------------------------------===// | ||||||||
| 2208 | // Matchers for two-operands operators with the operators in either order | ||||||||
| 2209 | // | ||||||||
| 2210 | |||||||||
| 2211 | /// Matches a BinaryOperator with LHS and RHS in either order. | ||||||||
| 2212 | template <typename LHS, typename RHS> | ||||||||
| 2213 | inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) { | ||||||||
| 2214 | return AnyBinaryOp_match<LHS, RHS, true>(L, R); | ||||||||
| 2215 | } | ||||||||
| 2216 | |||||||||
| 2217 | /// Matches an ICmp with a predicate over LHS and RHS in either order. | ||||||||
| 2218 | /// Swaps the predicate if operands are commuted. | ||||||||
| 2219 | template <typename LHS, typename RHS> | ||||||||
| 2220 | inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true> | ||||||||
| 2221 | m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { | ||||||||
| 2222 | return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L, | ||||||||
| 2223 | R); | ||||||||
| 2224 | } | ||||||||
| 2225 | |||||||||
| 2226 | /// Matches a Add with LHS and RHS in either order. | ||||||||
| 2227 | template <typename LHS, typename RHS> | ||||||||
| 2228 | inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L, | ||||||||
| 2229 | const RHS &R) { | ||||||||
| 2230 | return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R); | ||||||||
| 2231 | } | ||||||||
| 2232 | |||||||||
| 2233 | /// Matches a Mul with LHS and RHS in either order. | ||||||||
| 2234 | template <typename LHS, typename RHS> | ||||||||
| 2235 | inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L, | ||||||||
| 2236 | const RHS &R) { | ||||||||
| 2237 | return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R); | ||||||||
| 2238 | } | ||||||||
| 2239 | |||||||||
| 2240 | /// Matches an And with LHS and RHS in either order. | ||||||||
| 2241 | template <typename LHS, typename RHS> | ||||||||
| 2242 | inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L, | ||||||||
| 2243 | const RHS &R) { | ||||||||
| 2244 | return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R); | ||||||||
| 2245 | } | ||||||||
| 2246 | |||||||||
| 2247 | /// Matches an Or with LHS and RHS in either order. | ||||||||
| 2248 | template <typename LHS, typename RHS> | ||||||||
| 2249 | inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L, | ||||||||
| 2250 | const RHS &R) { | ||||||||
| 2251 | return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R); | ||||||||
| 2252 | } | ||||||||
| 2253 | |||||||||
| 2254 | /// Matches an Xor with LHS and RHS in either order. | ||||||||
| 2255 | template <typename LHS, typename RHS> | ||||||||
| 2256 | inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L, | ||||||||
| 2257 | const RHS &R) { | ||||||||
| 2258 | return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R); | ||||||||
| 2259 | } | ||||||||
| 2260 | |||||||||
| 2261 | /// Matches a 'Neg' as 'sub 0, V'. | ||||||||
| 2262 | template <typename ValTy> | ||||||||
| 2263 | inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub> | ||||||||
| 2264 | m_Neg(const ValTy &V) { | ||||||||
| 2265 | return m_Sub(m_ZeroInt(), V); | ||||||||
| 2266 | } | ||||||||
| 2267 | |||||||||
| 2268 | /// Matches a 'Neg' as 'sub nsw 0, V'. | ||||||||
| 2269 | template <typename ValTy> | ||||||||
| 2270 | inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, | ||||||||
| 2271 | Instruction::Sub, | ||||||||
| 2272 | OverflowingBinaryOperator::NoSignedWrap> | ||||||||
| 2273 | m_NSWNeg(const ValTy &V) { | ||||||||
| 2274 | return m_NSWSub(m_ZeroInt(), V); | ||||||||
| 2275 | } | ||||||||
| 2276 | |||||||||
| 2277 | /// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'. | ||||||||
| 2278 | template <typename ValTy> | ||||||||
| 2279 | inline BinaryOp_match<ValTy, cst_pred_ty<is_all_ones>, Instruction::Xor, true> | ||||||||
| 2280 | m_Not(const ValTy &V) { | ||||||||
| 2281 | return m_c_Xor(V, m_AllOnes()); | ||||||||
| 2282 | } | ||||||||
| 2283 | |||||||||
| 2284 | /// Matches an SMin with LHS and RHS in either order. | ||||||||
| 2285 | template <typename LHS, typename RHS> | ||||||||
| 2286 | inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true> | ||||||||
| 2287 | m_c_SMin(const LHS &L, const RHS &R) { | ||||||||
| 2288 | return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R); | ||||||||
| 2289 | } | ||||||||
| 2290 | /// Matches an SMax with LHS and RHS in either order. | ||||||||
| 2291 | template <typename LHS, typename RHS> | ||||||||
| 2292 | inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true> | ||||||||
| 2293 | m_c_SMax(const LHS &L, const RHS &R) { | ||||||||
| 2294 | return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R); | ||||||||
| 2295 | } | ||||||||
| 2296 | /// Matches a UMin with LHS and RHS in either order. | ||||||||
| 2297 | template <typename LHS, typename RHS> | ||||||||
| 2298 | inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true> | ||||||||
| 2299 | m_c_UMin(const LHS &L, const RHS &R) { | ||||||||
| 2300 | return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R); | ||||||||
| 2301 | } | ||||||||
| 2302 | /// Matches a UMax with LHS and RHS in either order. | ||||||||
| 2303 | template <typename LHS, typename RHS> | ||||||||
| 2304 | inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true> | ||||||||
| 2305 | m_c_UMax(const LHS &L, const RHS &R) { | ||||||||
| 2306 | return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R); | ||||||||
| 2307 | } | ||||||||
| 2308 | |||||||||
| 2309 | template <typename LHS, typename RHS> | ||||||||
| 2310 | inline match_combine_or< | ||||||||
| 2311 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>, | ||||||||
| 2312 | MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>, | ||||||||
| 2313 | match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>, | ||||||||
| 2314 | MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>> | ||||||||
| 2315 | m_c_MaxOrMin(const LHS &L, const RHS &R) { | ||||||||
| 2316 | return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)), | ||||||||
| 2317 | m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R))); | ||||||||
| 2318 | } | ||||||||
| 2319 | |||||||||
| 2320 | /// Matches FAdd with LHS and RHS in either order. | ||||||||
| 2321 | template <typename LHS, typename RHS> | ||||||||
| 2322 | inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true> | ||||||||
| 2323 | m_c_FAdd(const LHS &L, const RHS &R) { | ||||||||
| 2324 | return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R); | ||||||||
| 2325 | } | ||||||||
| 2326 | |||||||||
| 2327 | /// Matches FMul with LHS and RHS in either order. | ||||||||
| 2328 | template <typename LHS, typename RHS> | ||||||||
| 2329 | inline BinaryOp_match<LHS, RHS, Instruction::FMul, true> | ||||||||
| 2330 | m_c_FMul(const LHS &L, const RHS &R) { | ||||||||
| 2331 | return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R); | ||||||||
| 2332 | } | ||||||||
| 2333 | |||||||||
| 2334 | template <typename Opnd_t> struct Signum_match { | ||||||||
| 2335 | Opnd_t Val; | ||||||||
| 2336 | Signum_match(const Opnd_t &V) : Val(V) {} | ||||||||
| 2337 | |||||||||
| 2338 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 2339 | unsigned TypeSize = V->getType()->getScalarSizeInBits(); | ||||||||
| 2340 | if (TypeSize == 0) | ||||||||
| 2341 | return false; | ||||||||
| 2342 | |||||||||
| 2343 | unsigned ShiftWidth = TypeSize - 1; | ||||||||
| 2344 | Value *OpL = nullptr, *OpR = nullptr; | ||||||||
| 2345 | |||||||||
| 2346 | // This is the representation of signum we match: | ||||||||
| 2347 | // | ||||||||
| 2348 | // signum(x) == (x >> 63) | (-x >>u 63) | ||||||||
| 2349 | // | ||||||||
| 2350 | // An i1 value is its own signum, so it's correct to match | ||||||||
| 2351 | // | ||||||||
| 2352 | // signum(x) == (x >> 0) | (-x >>u 0) | ||||||||
| 2353 | // | ||||||||
| 2354 | // for i1 values. | ||||||||
| 2355 | |||||||||
| 2356 | auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth)); | ||||||||
| 2357 | auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth)); | ||||||||
| 2358 | auto Signum = m_Or(LHS, RHS); | ||||||||
| 2359 | |||||||||
| 2360 | return Signum.match(V) && OpL == OpR && Val.match(OpL); | ||||||||
| 2361 | } | ||||||||
| 2362 | }; | ||||||||
| 2363 | |||||||||
| 2364 | /// Matches a signum pattern. | ||||||||
| 2365 | /// | ||||||||
| 2366 | /// signum(x) = | ||||||||
| 2367 | /// x > 0 -> 1 | ||||||||
| 2368 | /// x == 0 -> 0 | ||||||||
| 2369 | /// x < 0 -> -1 | ||||||||
| 2370 | template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { | ||||||||
| 2371 | return Signum_match<Val_t>(V); | ||||||||
| 2372 | } | ||||||||
| 2373 | |||||||||
| 2374 | template <int Ind, typename Opnd_t> struct ExtractValue_match { | ||||||||
| 2375 | Opnd_t Val; | ||||||||
| 2376 | ExtractValue_match(const Opnd_t &V) : Val(V) {} | ||||||||
| 2377 | |||||||||
| 2378 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 2379 | if (auto *I = dyn_cast<ExtractValueInst>(V)) { | ||||||||
| 2380 | // If Ind is -1, don't inspect indices | ||||||||
| 2381 | if (Ind != -1 && | ||||||||
| 2382 | !(I->getNumIndices() == 1 && I->getIndices()[0] == (unsigned)Ind)) | ||||||||
| 2383 | return false; | ||||||||
| 2384 | return Val.match(I->getAggregateOperand()); | ||||||||
| 2385 | } | ||||||||
| 2386 | return false; | ||||||||
| 2387 | } | ||||||||
| 2388 | }; | ||||||||
| 2389 | |||||||||
| 2390 | /// Match a single index ExtractValue instruction. | ||||||||
| 2391 | /// For example m_ExtractValue<1>(...) | ||||||||
| 2392 | template <int Ind, typename Val_t> | ||||||||
| 2393 | inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) { | ||||||||
| 2394 | return ExtractValue_match<Ind, Val_t>(V); | ||||||||
| 2395 | } | ||||||||
| 2396 | |||||||||
| 2397 | /// Match an ExtractValue instruction with any index. | ||||||||
| 2398 | /// For example m_ExtractValue(...) | ||||||||
| 2399 | template <typename Val_t> | ||||||||
| 2400 | inline ExtractValue_match<-1, Val_t> m_ExtractValue(const Val_t &V) { | ||||||||
| 2401 | return ExtractValue_match<-1, Val_t>(V); | ||||||||
| 2402 | } | ||||||||
| 2403 | |||||||||
| 2404 | /// Matcher for a single index InsertValue instruction. | ||||||||
| 2405 | template <int Ind, typename T0, typename T1> struct InsertValue_match { | ||||||||
| 2406 | T0 Op0; | ||||||||
| 2407 | T1 Op1; | ||||||||
| 2408 | |||||||||
| 2409 | InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {} | ||||||||
| 2410 | |||||||||
| 2411 | template <typename OpTy> bool match(OpTy *V) { | ||||||||
| 2412 | if (auto *I = dyn_cast<InsertValueInst>(V)) { | ||||||||
| 2413 | return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) && | ||||||||
| 2414 | I->getNumIndices() == 1 && Ind == I->getIndices()[0]; | ||||||||
| 2415 | } | ||||||||
| 2416 | return false; | ||||||||
| 2417 | } | ||||||||
| 2418 | }; | ||||||||
| 2419 | |||||||||
| 2420 | /// Matches a single index InsertValue instruction. | ||||||||
| 2421 | template <int Ind, typename Val_t, typename Elt_t> | ||||||||
| 2422 | inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val, | ||||||||
| 2423 | const Elt_t &Elt) { | ||||||||
| 2424 | return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt); | ||||||||
| 2425 | } | ||||||||
| 2426 | |||||||||
| 2427 | /// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or | ||||||||
| 2428 | /// the constant expression | ||||||||
| 2429 | /// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>` | ||||||||
| 2430 | /// under the right conditions determined by DataLayout. | ||||||||
| 2431 | struct VScaleVal_match { | ||||||||
| 2432 | const DataLayout &DL; | ||||||||
| 2433 | VScaleVal_match(const DataLayout &DL) : DL(DL) {} | ||||||||
| 2434 | |||||||||
| 2435 | template <typename ITy> bool match(ITy *V) { | ||||||||
| 2436 | if (m_Intrinsic<Intrinsic::vscale>().match(V)) | ||||||||
| 2437 | return true; | ||||||||
| 2438 | |||||||||
| 2439 | Value *Ptr; | ||||||||
| 2440 | if (m_PtrToInt(m_Value(Ptr)).match(V)) { | ||||||||
| 2441 | if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) { | ||||||||
| 2442 | auto *DerefTy = GEP->getSourceElementType(); | ||||||||
| 2443 | if (GEP->getNumIndices() == 1 && isa<ScalableVectorType>(DerefTy) && | ||||||||
| 2444 | m_Zero().match(GEP->getPointerOperand()) && | ||||||||
| 2445 | m_SpecificInt(1).match(GEP->idx_begin()->get()) && | ||||||||
| 2446 | DL.getTypeAllocSizeInBits(DerefTy).getKnownMinSize() == 8) | ||||||||
| 2447 | return true; | ||||||||
| 2448 | } | ||||||||
| 2449 | } | ||||||||
| 2450 | |||||||||
| 2451 | return false; | ||||||||
| 2452 | } | ||||||||
| 2453 | }; | ||||||||
| 2454 | |||||||||
| 2455 | inline VScaleVal_match m_VScale(const DataLayout &DL) { | ||||||||
| 2456 | return VScaleVal_match(DL); | ||||||||
| 2457 | } | ||||||||
| 2458 | |||||||||
| 2459 | template <typename LHS, typename RHS, unsigned Opcode> | ||||||||
| 2460 | struct LogicalOp_match { | ||||||||
| 2461 | LHS L; | ||||||||
| 2462 | RHS R; | ||||||||
| 2463 | |||||||||
| 2464 | LogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {} | ||||||||
| 2465 | |||||||||
| 2466 | template <typename T> bool match(T *V) { | ||||||||
| 2467 | if (auto *I = dyn_cast<Instruction>(V)) { | ||||||||
| 2468 | if (!I->getType()->isIntOrIntVectorTy(1)) | ||||||||
| 2469 | return false; | ||||||||
| 2470 | |||||||||
| 2471 | if (I->getOpcode() == Opcode && L.match(I->getOperand(0)) && | ||||||||
| 2472 | R.match(I->getOperand(1))) | ||||||||
| 2473 | return true; | ||||||||
| 2474 | |||||||||
| 2475 | if (auto *SI = dyn_cast<SelectInst>(I)) { | ||||||||
| 2476 | if (Opcode == Instruction::And) { | ||||||||
| 2477 | if (const auto *C = dyn_cast<Constant>(SI->getFalseValue())) | ||||||||
| 2478 | if (C->isNullValue() && L.match(SI->getCondition()) && | ||||||||
| 2479 | R.match(SI->getTrueValue())) | ||||||||
| 2480 | return true; | ||||||||
| 2481 | } else { | ||||||||
| 2482 | assert(Opcode == Instruction::Or)((void)0); | ||||||||
| 2483 | if (const auto *C = dyn_cast<Constant>(SI->getTrueValue())) | ||||||||
| 2484 | if (C->isOneValue() && L.match(SI->getCondition()) && | ||||||||
| 2485 | R.match(SI->getFalseValue())) | ||||||||
| 2486 | return true; | ||||||||
| 2487 | } | ||||||||
| 2488 | } | ||||||||
| 2489 | } | ||||||||
| 2490 | |||||||||
| 2491 | return false; | ||||||||
| 2492 | } | ||||||||
| 2493 | }; | ||||||||
| 2494 | |||||||||
| 2495 | /// Matches L && R either in the form of L & R or L ? R : false. | ||||||||
| 2496 | /// Note that the latter form is poison-blocking. | ||||||||
| 2497 | template <typename LHS, typename RHS> | ||||||||
| 2498 | inline LogicalOp_match<LHS, RHS, Instruction::And> | ||||||||
| 2499 | m_LogicalAnd(const LHS &L, const RHS &R) { | ||||||||
| 2500 | return LogicalOp_match<LHS, RHS, Instruction::And>(L, R); | ||||||||
| 2501 | } | ||||||||
| 2502 | |||||||||
| 2503 | /// Matches L && R where L and R are arbitrary values. | ||||||||
| 2504 | inline auto m_LogicalAnd() { return m_LogicalAnd(m_Value(), m_Value()); } | ||||||||
| 2505 | |||||||||
| 2506 | /// Matches L || R either in the form of L | R or L ? true : R. | ||||||||
| 2507 | /// Note that the latter form is poison-blocking. | ||||||||
| 2508 | template <typename LHS, typename RHS> | ||||||||
| 2509 | inline LogicalOp_match<LHS, RHS, Instruction::Or> | ||||||||
| 2510 | m_LogicalOr(const LHS &L, const RHS &R) { | ||||||||
| 2511 | return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R); | ||||||||
| 2512 | } | ||||||||
| 2513 | |||||||||
| 2514 | /// Matches L || R where L and R are arbitrary values. | ||||||||
| 2515 | inline auto m_LogicalOr() { | ||||||||
| 2516 | return m_LogicalOr(m_Value(), m_Value()); | ||||||||
| 2517 | } | ||||||||
| 2518 | |||||||||
| 2519 | } // end namespace PatternMatch | ||||||||
| 2520 | } // end namespace llvm | ||||||||
| 2521 | |||||||||
| 2522 | #endif // LLVM_IR_PATTERNMATCH_H |
| 1 | //===- llvm/Value.h - Definition of the Value class -------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file declares the Value class. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef LLVM_IR_VALUE_H |
| 14 | #define LLVM_IR_VALUE_H |
| 15 | |
| 16 | #include "llvm-c/Types.h" |
| 17 | #include "llvm/ADT/STLExtras.h" |
| 18 | #include "llvm/ADT/StringRef.h" |
| 19 | #include "llvm/ADT/iterator_range.h" |
| 20 | #include "llvm/IR/Use.h" |
| 21 | #include "llvm/Support/Alignment.h" |
| 22 | #include "llvm/Support/CBindingWrapping.h" |
| 23 | #include "llvm/Support/Casting.h" |
| 24 | #include <cassert> |
| 25 | #include <iterator> |
| 26 | #include <memory> |
| 27 | |
| 28 | namespace llvm { |
| 29 | |
| 30 | class APInt; |
| 31 | class Argument; |
| 32 | class BasicBlock; |
| 33 | class Constant; |
| 34 | class ConstantData; |
| 35 | class ConstantAggregate; |
| 36 | class DataLayout; |
| 37 | class Function; |
| 38 | class GlobalAlias; |
| 39 | class GlobalIFunc; |
| 40 | class GlobalIndirectSymbol; |
| 41 | class GlobalObject; |
| 42 | class GlobalValue; |
| 43 | class GlobalVariable; |
| 44 | class InlineAsm; |
| 45 | class Instruction; |
| 46 | class LLVMContext; |
| 47 | class MDNode; |
| 48 | class Module; |
| 49 | class ModuleSlotTracker; |
| 50 | class raw_ostream; |
| 51 | template<typename ValueTy> class StringMapEntry; |
| 52 | class Twine; |
| 53 | class Type; |
| 54 | class User; |
| 55 | |
| 56 | using ValueName = StringMapEntry<Value *>; |
| 57 | |
| 58 | //===----------------------------------------------------------------------===// |
| 59 | // Value Class |
| 60 | //===----------------------------------------------------------------------===// |
| 61 | |
| 62 | /// LLVM Value Representation |
| 63 | /// |
| 64 | /// This is a very important LLVM class. It is the base class of all values |
| 65 | /// computed by a program that may be used as operands to other values. Value is |
| 66 | /// the super class of other important classes such as Instruction and Function. |
| 67 | /// All Values have a Type. Type is not a subclass of Value. Some values can |
| 68 | /// have a name and they belong to some Module. Setting the name on the Value |
| 69 | /// automatically updates the module's symbol table. |
| 70 | /// |
| 71 | /// Every value has a "use list" that keeps track of which other Values are |
| 72 | /// using this Value. A Value can also have an arbitrary number of ValueHandle |
| 73 | /// objects that watch it and listen to RAUW and Destroy events. See |
| 74 | /// llvm/IR/ValueHandle.h for details. |
| 75 | class Value { |
| 76 | Type *VTy; |
| 77 | Use *UseList; |
| 78 | |
| 79 | friend class ValueAsMetadata; // Allow access to IsUsedByMD. |
| 80 | friend class ValueHandleBase; |
| 81 | |
| 82 | const unsigned char SubclassID; // Subclass identifier (for isa/dyn_cast) |
| 83 | unsigned char HasValueHandle : 1; // Has a ValueHandle pointing to this? |
| 84 | |
| 85 | protected: |
| 86 | /// Hold subclass data that can be dropped. |
| 87 | /// |
| 88 | /// This member is similar to SubclassData, however it is for holding |
| 89 | /// information which may be used to aid optimization, but which may be |
| 90 | /// cleared to zero without affecting conservative interpretation. |
| 91 | unsigned char SubclassOptionalData : 7; |
| 92 | |
| 93 | private: |
| 94 | /// Hold arbitrary subclass data. |
| 95 | /// |
| 96 | /// This member is defined by this class, but is not used for anything. |
| 97 | /// Subclasses can use it to hold whatever state they find useful. This |
| 98 | /// field is initialized to zero by the ctor. |
| 99 | unsigned short SubclassData; |
| 100 | |
| 101 | protected: |
| 102 | /// The number of operands in the subclass. |
| 103 | /// |
| 104 | /// This member is defined by this class, but not used for anything. |
| 105 | /// Subclasses can use it to store their number of operands, if they have |
| 106 | /// any. |
| 107 | /// |
| 108 | /// This is stored here to save space in User on 64-bit hosts. Since most |
| 109 | /// instances of Value have operands, 32-bit hosts aren't significantly |
| 110 | /// affected. |
| 111 | /// |
| 112 | /// Note, this should *NOT* be used directly by any class other than User. |
| 113 | /// User uses this value to find the Use list. |
| 114 | enum : unsigned { NumUserOperandsBits = 27 }; |
| 115 | unsigned NumUserOperands : NumUserOperandsBits; |
| 116 | |
| 117 | // Use the same type as the bitfield above so that MSVC will pack them. |
| 118 | unsigned IsUsedByMD : 1; |
| 119 | unsigned HasName : 1; |
| 120 | unsigned HasMetadata : 1; // Has metadata attached to this? |
| 121 | unsigned HasHungOffUses : 1; |
| 122 | unsigned HasDescriptor : 1; |
| 123 | |
| 124 | private: |
| 125 | template <typename UseT> // UseT == 'Use' or 'const Use' |
| 126 | class use_iterator_impl { |
| 127 | friend class Value; |
| 128 | |
| 129 | UseT *U; |
| 130 | |
| 131 | explicit use_iterator_impl(UseT *u) : U(u) {} |
| 132 | |
| 133 | public: |
| 134 | using iterator_category = std::forward_iterator_tag; |
| 135 | using value_type = UseT *; |
| 136 | using difference_type = std::ptrdiff_t; |
| 137 | using pointer = value_type *; |
| 138 | using reference = value_type &; |
| 139 | |
| 140 | use_iterator_impl() : U() {} |
| 141 | |
| 142 | bool operator==(const use_iterator_impl &x) const { return U == x.U; } |
| 143 | bool operator!=(const use_iterator_impl &x) const { return !operator==(x); } |
| 144 | |
| 145 | use_iterator_impl &operator++() { // Preincrement |
| 146 | assert(U && "Cannot increment end iterator!")((void)0); |
| 147 | U = U->getNext(); |
| 148 | return *this; |
| 149 | } |
| 150 | |
| 151 | use_iterator_impl operator++(int) { // Postincrement |
| 152 | auto tmp = *this; |
| 153 | ++*this; |
| 154 | return tmp; |
| 155 | } |
| 156 | |
| 157 | UseT &operator*() const { |
| 158 | assert(U && "Cannot dereference end iterator!")((void)0); |
| 159 | return *U; |
| 160 | } |
| 161 | |
| 162 | UseT *operator->() const { return &operator*(); } |
| 163 | |
| 164 | operator use_iterator_impl<const UseT>() const { |
| 165 | return use_iterator_impl<const UseT>(U); |
| 166 | } |
| 167 | }; |
| 168 | |
| 169 | template <typename UserTy> // UserTy == 'User' or 'const User' |
| 170 | class user_iterator_impl { |
| 171 | use_iterator_impl<Use> UI; |
| 172 | explicit user_iterator_impl(Use *U) : UI(U) {} |
| 173 | friend class Value; |
| 174 | |
| 175 | public: |
| 176 | using iterator_category = std::forward_iterator_tag; |
| 177 | using value_type = UserTy *; |
| 178 | using difference_type = std::ptrdiff_t; |
| 179 | using pointer = value_type *; |
| 180 | using reference = value_type &; |
| 181 | |
| 182 | user_iterator_impl() = default; |
| 183 | |
| 184 | bool operator==(const user_iterator_impl &x) const { return UI == x.UI; } |
| 185 | bool operator!=(const user_iterator_impl &x) const { return !operator==(x); } |
| 186 | |
| 187 | /// Returns true if this iterator is equal to user_end() on the value. |
| 188 | bool atEnd() const { return *this == user_iterator_impl(); } |
| 189 | |
| 190 | user_iterator_impl &operator++() { // Preincrement |
| 191 | ++UI; |
| 192 | return *this; |
| 193 | } |
| 194 | |
| 195 | user_iterator_impl operator++(int) { // Postincrement |
| 196 | auto tmp = *this; |
| 197 | ++*this; |
| 198 | return tmp; |
| 199 | } |
| 200 | |
| 201 | // Retrieve a pointer to the current User. |
| 202 | UserTy *operator*() const { |
| 203 | return UI->getUser(); |
| 204 | } |
| 205 | |
| 206 | UserTy *operator->() const { return operator*(); } |
| 207 | |
| 208 | operator user_iterator_impl<const UserTy>() const { |
| 209 | return user_iterator_impl<const UserTy>(*UI); |
| 210 | } |
| 211 | |
| 212 | Use &getUse() const { return *UI; } |
| 213 | }; |
| 214 | |
| 215 | protected: |
| 216 | Value(Type *Ty, unsigned scid); |
| 217 | |
| 218 | /// Value's destructor should be virtual by design, but that would require |
| 219 | /// that Value and all of its subclasses have a vtable that effectively |
| 220 | /// duplicates the information in the value ID. As a size optimization, the |
| 221 | /// destructor has been protected, and the caller should manually call |
| 222 | /// deleteValue. |
| 223 | ~Value(); // Use deleteValue() to delete a generic Value. |
| 224 | |
| 225 | public: |
| 226 | Value(const Value &) = delete; |
| 227 | Value &operator=(const Value &) = delete; |
| 228 | |
| 229 | /// Delete a pointer to a generic Value. |
| 230 | void deleteValue(); |
| 231 | |
| 232 | /// Support for debugging, callable in GDB: V->dump() |
| 233 | void dump() const; |
| 234 | |
| 235 | /// Implement operator<< on Value. |
| 236 | /// @{ |
| 237 | void print(raw_ostream &O, bool IsForDebug = false) const; |
| 238 | void print(raw_ostream &O, ModuleSlotTracker &MST, |
| 239 | bool IsForDebug = false) const; |
| 240 | /// @} |
| 241 | |
| 242 | /// Print the name of this Value out to the specified raw_ostream. |
| 243 | /// |
| 244 | /// This is useful when you just want to print 'int %reg126', not the |
| 245 | /// instruction that generated it. If you specify a Module for context, then |
| 246 | /// even constanst get pretty-printed; for example, the type of a null |
| 247 | /// pointer is printed symbolically. |
| 248 | /// @{ |
| 249 | void printAsOperand(raw_ostream &O, bool PrintType = true, |
| 250 | const Module *M = nullptr) const; |
| 251 | void printAsOperand(raw_ostream &O, bool PrintType, |
| 252 | ModuleSlotTracker &MST) const; |
| 253 | /// @} |
| 254 | |
| 255 | /// All values are typed, get the type of this value. |
| 256 | Type *getType() const { return VTy; } |
| 257 | |
| 258 | /// All values hold a context through their type. |
| 259 | LLVMContext &getContext() const; |
| 260 | |
| 261 | // All values can potentially be named. |
| 262 | bool hasName() const { return HasName; } |
| 263 | ValueName *getValueName() const; |
| 264 | void setValueName(ValueName *VN); |
| 265 | |
| 266 | private: |
| 267 | void destroyValueName(); |
| 268 | enum class ReplaceMetadataUses { No, Yes }; |
| 269 | void doRAUW(Value *New, ReplaceMetadataUses); |
| 270 | void setNameImpl(const Twine &Name); |
| 271 | |
| 272 | public: |
| 273 | /// Return a constant reference to the value's name. |
| 274 | /// |
| 275 | /// This guaranteed to return the same reference as long as the value is not |
| 276 | /// modified. If the value has a name, this does a hashtable lookup, so it's |
| 277 | /// not free. |
| 278 | StringRef getName() const; |
| 279 | |
| 280 | /// Change the name of the value. |
| 281 | /// |
| 282 | /// Choose a new unique name if the provided name is taken. |
| 283 | /// |
| 284 | /// \param Name The new name; or "" if the value's name should be removed. |
| 285 | void setName(const Twine &Name); |
| 286 | |
| 287 | /// Transfer the name from V to this value. |
| 288 | /// |
| 289 | /// After taking V's name, sets V's name to empty. |
| 290 | /// |
| 291 | /// \note It is an error to call V->takeName(V). |
| 292 | void takeName(Value *V); |
| 293 | |
| 294 | #ifndef NDEBUG1 |
| 295 | std::string getNameOrAsOperand() const; |
| 296 | #endif |
| 297 | |
| 298 | /// Change all uses of this to point to a new Value. |
| 299 | /// |
| 300 | /// Go through the uses list for this definition and make each use point to |
| 301 | /// "V" instead of "this". After this completes, 'this's use list is |
| 302 | /// guaranteed to be empty. |
| 303 | void replaceAllUsesWith(Value *V); |
| 304 | |
| 305 | /// Change non-metadata uses of this to point to a new Value. |
| 306 | /// |
| 307 | /// Go through the uses list for this definition and make each use point to |
| 308 | /// "V" instead of "this". This function skips metadata entries in the list. |
| 309 | void replaceNonMetadataUsesWith(Value *V); |
| 310 | |
| 311 | /// Go through the uses list for this definition and make each use point |
| 312 | /// to "V" if the callback ShouldReplace returns true for the given Use. |
| 313 | /// Unlike replaceAllUsesWith() this function does not support basic block |
| 314 | /// values. |
| 315 | void replaceUsesWithIf(Value *New, |
| 316 | llvm::function_ref<bool(Use &U)> ShouldReplace); |
| 317 | |
| 318 | /// replaceUsesOutsideBlock - Go through the uses list for this definition and |
| 319 | /// make each use point to "V" instead of "this" when the use is outside the |
| 320 | /// block. 'This's use list is expected to have at least one element. |
| 321 | /// Unlike replaceAllUsesWith() this function does not support basic block |
| 322 | /// values. |
| 323 | void replaceUsesOutsideBlock(Value *V, BasicBlock *BB); |
| 324 | |
| 325 | //---------------------------------------------------------------------- |
| 326 | // Methods for handling the chain of uses of this Value. |
| 327 | // |
| 328 | // Materializing a function can introduce new uses, so these methods come in |
| 329 | // two variants: |
| 330 | // The methods that start with materialized_ check the uses that are |
| 331 | // currently known given which functions are materialized. Be very careful |
| 332 | // when using them since you might not get all uses. |
| 333 | // The methods that don't start with materialized_ assert that modules is |
| 334 | // fully materialized. |
| 335 | void assertModuleIsMaterializedImpl() const; |
| 336 | // This indirection exists so we can keep assertModuleIsMaterializedImpl() |
| 337 | // around in release builds of Value.cpp to be linked with other code built |
| 338 | // in debug mode. But this avoids calling it in any of the release built code. |
| 339 | void assertModuleIsMaterialized() const { |
| 340 | #ifndef NDEBUG1 |
| 341 | assertModuleIsMaterializedImpl(); |
| 342 | #endif |
| 343 | } |
| 344 | |
| 345 | bool use_empty() const { |
| 346 | assertModuleIsMaterialized(); |
| 347 | return UseList == nullptr; |
| 348 | } |
| 349 | |
| 350 | bool materialized_use_empty() const { |
| 351 | return UseList == nullptr; |
| 352 | } |
| 353 | |
| 354 | using use_iterator = use_iterator_impl<Use>; |
| 355 | using const_use_iterator = use_iterator_impl<const Use>; |
| 356 | |
| 357 | use_iterator materialized_use_begin() { return use_iterator(UseList); } |
| 358 | const_use_iterator materialized_use_begin() const { |
| 359 | return const_use_iterator(UseList); |
| 360 | } |
| 361 | use_iterator use_begin() { |
| 362 | assertModuleIsMaterialized(); |
| 363 | return materialized_use_begin(); |
| 364 | } |
| 365 | const_use_iterator use_begin() const { |
| 366 | assertModuleIsMaterialized(); |
| 367 | return materialized_use_begin(); |
| 368 | } |
| 369 | use_iterator use_end() { return use_iterator(); } |
| 370 | const_use_iterator use_end() const { return const_use_iterator(); } |
| 371 | iterator_range<use_iterator> materialized_uses() { |
| 372 | return make_range(materialized_use_begin(), use_end()); |
| 373 | } |
| 374 | iterator_range<const_use_iterator> materialized_uses() const { |
| 375 | return make_range(materialized_use_begin(), use_end()); |
| 376 | } |
| 377 | iterator_range<use_iterator> uses() { |
| 378 | assertModuleIsMaterialized(); |
| 379 | return materialized_uses(); |
| 380 | } |
| 381 | iterator_range<const_use_iterator> uses() const { |
| 382 | assertModuleIsMaterialized(); |
| 383 | return materialized_uses(); |
| 384 | } |
| 385 | |
| 386 | bool user_empty() const { |
| 387 | assertModuleIsMaterialized(); |
| 388 | return UseList == nullptr; |
| 389 | } |
| 390 | |
| 391 | using user_iterator = user_iterator_impl<User>; |
| 392 | using const_user_iterator = user_iterator_impl<const User>; |
| 393 | |
| 394 | user_iterator materialized_user_begin() { return user_iterator(UseList); } |
| 395 | const_user_iterator materialized_user_begin() const { |
| 396 | return const_user_iterator(UseList); |
| 397 | } |
| 398 | user_iterator user_begin() { |
| 399 | assertModuleIsMaterialized(); |
| 400 | return materialized_user_begin(); |
| 401 | } |
| 402 | const_user_iterator user_begin() const { |
| 403 | assertModuleIsMaterialized(); |
| 404 | return materialized_user_begin(); |
| 405 | } |
| 406 | user_iterator user_end() { return user_iterator(); } |
| 407 | const_user_iterator user_end() const { return const_user_iterator(); } |
| 408 | User *user_back() { |
| 409 | assertModuleIsMaterialized(); |
| 410 | return *materialized_user_begin(); |
| 411 | } |
| 412 | const User *user_back() const { |
| 413 | assertModuleIsMaterialized(); |
| 414 | return *materialized_user_begin(); |
| 415 | } |
| 416 | iterator_range<user_iterator> materialized_users() { |
| 417 | return make_range(materialized_user_begin(), user_end()); |
| 418 | } |
| 419 | iterator_range<const_user_iterator> materialized_users() const { |
| 420 | return make_range(materialized_user_begin(), user_end()); |
| 421 | } |
| 422 | iterator_range<user_iterator> users() { |
| 423 | assertModuleIsMaterialized(); |
| 424 | return materialized_users(); |
| 425 | } |
| 426 | iterator_range<const_user_iterator> users() const { |
| 427 | assertModuleIsMaterialized(); |
| 428 | return materialized_users(); |
| 429 | } |
| 430 | |
| 431 | /// Return true if there is exactly one use of this value. |
| 432 | /// |
| 433 | /// This is specialized because it is a common request and does not require |
| 434 | /// traversing the whole use list. |
| 435 | bool hasOneUse() const { return hasSingleElement(uses()); } |
| 436 | |
| 437 | /// Return true if this Value has exactly N uses. |
| 438 | bool hasNUses(unsigned N) const; |
| 439 | |
| 440 | /// Return true if this value has N uses or more. |
| 441 | /// |
| 442 | /// This is logically equivalent to getNumUses() >= N. |
| 443 | bool hasNUsesOrMore(unsigned N) const; |
| 444 | |
| 445 | /// Return true if there is exactly one user of this value. |
| 446 | /// |
| 447 | /// Note that this is not the same as "has one use". If a value has one use, |
| 448 | /// then there certainly is a single user. But if value has several uses, |
| 449 | /// it is possible that all uses are in a single user, or not. |
| 450 | /// |
| 451 | /// This check is potentially costly, since it requires traversing, |
| 452 | /// in the worst case, the whole use list of a value. |
| 453 | bool hasOneUser() const; |
| 454 | |
| 455 | /// Return true if there is exactly one use of this value that cannot be |
| 456 | /// dropped. |
| 457 | /// |
| 458 | /// This is specialized because it is a common request and does not require |
| 459 | /// traversing the whole use list. |
| 460 | Use *getSingleUndroppableUse(); |
| 461 | const Use *getSingleUndroppableUse() const { |
| 462 | return const_cast<Value *>(this)->getSingleUndroppableUse(); |
| 463 | } |
| 464 | |
| 465 | /// Return true if there this value. |
| 466 | /// |
| 467 | /// This is specialized because it is a common request and does not require |
| 468 | /// traversing the whole use list. |
| 469 | bool hasNUndroppableUses(unsigned N) const; |
| 470 | |
| 471 | /// Return true if this value has N uses or more. |
| 472 | /// |
| 473 | /// This is logically equivalent to getNumUses() >= N. |
| 474 | bool hasNUndroppableUsesOrMore(unsigned N) const; |
| 475 | |
| 476 | /// Remove every uses that can safely be removed. |
| 477 | /// |
| 478 | /// This will remove for example uses in llvm.assume. |
| 479 | /// This should be used when performing want to perform a tranformation but |
| 480 | /// some Droppable uses pervent it. |
| 481 | /// This function optionally takes a filter to only remove some droppable |
| 482 | /// uses. |
| 483 | void dropDroppableUses(llvm::function_ref<bool(const Use *)> ShouldDrop = |
| 484 | [](const Use *) { return true; }); |
| 485 | |
| 486 | /// Remove every use of this value in \p User that can safely be removed. |
| 487 | void dropDroppableUsesIn(User &Usr); |
| 488 | |
| 489 | /// Remove the droppable use \p U. |
| 490 | static void dropDroppableUse(Use &U); |
| 491 | |
| 492 | /// Check if this value is used in the specified basic block. |
| 493 | bool isUsedInBasicBlock(const BasicBlock *BB) const; |
| 494 | |
| 495 | /// This method computes the number of uses of this Value. |
| 496 | /// |
| 497 | /// This is a linear time operation. Use hasOneUse, hasNUses, or |
| 498 | /// hasNUsesOrMore to check for specific values. |
| 499 | unsigned getNumUses() const; |
| 500 | |
| 501 | /// This method should only be used by the Use class. |
| 502 | void addUse(Use &U) { U.addToList(&UseList); } |
| 503 | |
| 504 | /// Concrete subclass of this. |
| 505 | /// |
| 506 | /// An enumeration for keeping track of the concrete subclass of Value that |
| 507 | /// is actually instantiated. Values of this enumeration are kept in the |
| 508 | /// Value classes SubclassID field. They are used for concrete type |
| 509 | /// identification. |
| 510 | enum ValueTy { |
| 511 | #define HANDLE_VALUE(Name) Name##Val, |
| 512 | #include "llvm/IR/Value.def" |
| 513 | |
| 514 | // Markers: |
| 515 | #define HANDLE_CONSTANT_MARKER(Marker, Constant) Marker = Constant##Val, |
| 516 | #include "llvm/IR/Value.def" |
| 517 | }; |
| 518 | |
| 519 | /// Return an ID for the concrete type of this object. |
| 520 | /// |
| 521 | /// This is used to implement the classof checks. This should not be used |
| 522 | /// for any other purpose, as the values may change as LLVM evolves. Also, |
| 523 | /// note that for instructions, the Instruction's opcode is added to |
| 524 | /// InstructionVal. So this means three things: |
| 525 | /// # there is no value with code InstructionVal (no opcode==0). |
| 526 | /// # there are more possible values for the value type than in ValueTy enum. |
| 527 | /// # the InstructionVal enumerator must be the highest valued enumerator in |
| 528 | /// the ValueTy enum. |
| 529 | unsigned getValueID() const { |
| 530 | return SubclassID; |
| 531 | } |
| 532 | |
| 533 | /// Return the raw optional flags value contained in this value. |
| 534 | /// |
| 535 | /// This should only be used when testing two Values for equivalence. |
| 536 | unsigned getRawSubclassOptionalData() const { |
| 537 | return SubclassOptionalData; |
| 538 | } |
| 539 | |
| 540 | /// Clear the optional flags contained in this value. |
| 541 | void clearSubclassOptionalData() { |
| 542 | SubclassOptionalData = 0; |
| 543 | } |
| 544 | |
| 545 | /// Check the optional flags for equality. |
| 546 | bool hasSameSubclassOptionalData(const Value *V) const { |
| 547 | return SubclassOptionalData == V->SubclassOptionalData; |
| 548 | } |
| 549 | |
| 550 | /// Return true if there is a value handle associated with this value. |
| 551 | bool hasValueHandle() const { return HasValueHandle; } |
| 552 | |
| 553 | /// Return true if there is metadata referencing this value. |
| 554 | bool isUsedByMetadata() const { return IsUsedByMD; } |
| 555 | |
| 556 | // Return true if this value is only transitively referenced by metadata. |
| 557 | bool isTransitiveUsedByMetadataOnly() const; |
| 558 | |
| 559 | protected: |
| 560 | /// Get the current metadata attachments for the given kind, if any. |
| 561 | /// |
| 562 | /// These functions require that the value have at most a single attachment |
| 563 | /// of the given kind, and return \c nullptr if such an attachment is missing. |
| 564 | /// @{ |
| 565 | MDNode *getMetadata(unsigned KindID) const; |
| 566 | MDNode *getMetadata(StringRef Kind) const; |
| 567 | /// @} |
| 568 | |
| 569 | /// Appends all attachments with the given ID to \c MDs in insertion order. |
| 570 | /// If the Value has no attachments with the given ID, or if ID is invalid, |
| 571 | /// leaves MDs unchanged. |
| 572 | /// @{ |
| 573 | void getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const; |
| 574 | void getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const; |
| 575 | /// @} |
| 576 | |
| 577 | /// Appends all metadata attached to this value to \c MDs, sorting by |
| 578 | /// KindID. The first element of each pair returned is the KindID, the second |
| 579 | /// element is the metadata value. Attachments with the same ID appear in |
| 580 | /// insertion order. |
| 581 | void |
| 582 | getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const; |
| 583 | |
| 584 | /// Return true if this value has any metadata attached to it. |
| 585 | bool hasMetadata() const { return (bool)HasMetadata; } |
| 586 | |
| 587 | /// Return true if this value has the given type of metadata attached. |
| 588 | /// @{ |
| 589 | bool hasMetadata(unsigned KindID) const { |
| 590 | return getMetadata(KindID) != nullptr; |
| 591 | } |
| 592 | bool hasMetadata(StringRef Kind) const { |
| 593 | return getMetadata(Kind) != nullptr; |
| 594 | } |
| 595 | /// @} |
| 596 | |
| 597 | /// Set a particular kind of metadata attachment. |
| 598 | /// |
| 599 | /// Sets the given attachment to \c MD, erasing it if \c MD is \c nullptr or |
| 600 | /// replacing it if it already exists. |
| 601 | /// @{ |
| 602 | void setMetadata(unsigned KindID, MDNode *Node); |
| 603 | void setMetadata(StringRef Kind, MDNode *Node); |
| 604 | /// @} |
| 605 | |
| 606 | /// Add a metadata attachment. |
| 607 | /// @{ |
| 608 | void addMetadata(unsigned KindID, MDNode &MD); |
| 609 | void addMetadata(StringRef Kind, MDNode &MD); |
| 610 | /// @} |
| 611 | |
| 612 | /// Erase all metadata attachments with the given kind. |
| 613 | /// |
| 614 | /// \returns true if any metadata was removed. |
| 615 | bool eraseMetadata(unsigned KindID); |
| 616 | |
| 617 | /// Erase all metadata attached to this Value. |
| 618 | void clearMetadata(); |
| 619 | |
| 620 | public: |
| 621 | /// Return true if this value is a swifterror value. |
| 622 | /// |
| 623 | /// swifterror values can be either a function argument or an alloca with a |
| 624 | /// swifterror attribute. |
| 625 | bool isSwiftError() const; |
| 626 | |
| 627 | /// Strip off pointer casts, all-zero GEPs and address space casts. |
| 628 | /// |
| 629 | /// Returns the original uncasted value. If this is called on a non-pointer |
| 630 | /// value, it returns 'this'. |
| 631 | const Value *stripPointerCasts() const; |
| 632 | Value *stripPointerCasts() { |
| 633 | return const_cast<Value *>( |
| 634 | static_cast<const Value *>(this)->stripPointerCasts()); |
| 635 | } |
| 636 | |
| 637 | /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases. |
| 638 | /// |
| 639 | /// Returns the original uncasted value. If this is called on a non-pointer |
| 640 | /// value, it returns 'this'. |
| 641 | const Value *stripPointerCastsAndAliases() const; |
| 642 | Value *stripPointerCastsAndAliases() { |
| 643 | return const_cast<Value *>( |
| 644 | static_cast<const Value *>(this)->stripPointerCastsAndAliases()); |
| 645 | } |
| 646 | |
| 647 | /// Strip off pointer casts, all-zero GEPs and address space casts |
| 648 | /// but ensures the representation of the result stays the same. |
| 649 | /// |
| 650 | /// Returns the original uncasted value with the same representation. If this |
| 651 | /// is called on a non-pointer value, it returns 'this'. |
| 652 | const Value *stripPointerCastsSameRepresentation() const; |
| 653 | Value *stripPointerCastsSameRepresentation() { |
| 654 | return const_cast<Value *>(static_cast<const Value *>(this) |
| 655 | ->stripPointerCastsSameRepresentation()); |
| 656 | } |
| 657 | |
| 658 | /// Strip off pointer casts, all-zero GEPs, single-argument phi nodes and |
| 659 | /// invariant group info. |
| 660 | /// |
| 661 | /// Returns the original uncasted value. If this is called on a non-pointer |
| 662 | /// value, it returns 'this'. This function should be used only in |
| 663 | /// Alias analysis. |
| 664 | const Value *stripPointerCastsForAliasAnalysis() const; |
| 665 | Value *stripPointerCastsForAliasAnalysis() { |
| 666 | return const_cast<Value *>(static_cast<const Value *>(this) |
| 667 | ->stripPointerCastsForAliasAnalysis()); |
| 668 | } |
| 669 | |
| 670 | /// Strip off pointer casts and all-constant inbounds GEPs. |
| 671 | /// |
| 672 | /// Returns the original pointer value. If this is called on a non-pointer |
| 673 | /// value, it returns 'this'. |
| 674 | const Value *stripInBoundsConstantOffsets() const; |
| 675 | Value *stripInBoundsConstantOffsets() { |
| 676 | return const_cast<Value *>( |
| 677 | static_cast<const Value *>(this)->stripInBoundsConstantOffsets()); |
| 678 | } |
| 679 | |
| 680 | /// Accumulate the constant offset this value has compared to a base pointer. |
| 681 | /// Only 'getelementptr' instructions (GEPs) are accumulated but other |
| 682 | /// instructions, e.g., casts, are stripped away as well. |
| 683 | /// The accumulated constant offset is added to \p Offset and the base |
| 684 | /// pointer is returned. |
| 685 | /// |
| 686 | /// The APInt \p Offset has to have a bit-width equal to the IntPtr type for |
| 687 | /// the address space of 'this' pointer value, e.g., use |
| 688 | /// DataLayout::getIndexTypeSizeInBits(Ty). |
| 689 | /// |
| 690 | /// If \p AllowNonInbounds is true, offsets in GEPs are stripped and |
| 691 | /// accumulated even if the GEP is not "inbounds". |
| 692 | /// |
| 693 | /// If \p ExternalAnalysis is provided it will be used to calculate a offset |
| 694 | /// when a operand of GEP is not constant. |
| 695 | /// For example, for a value \p ExternalAnalysis might try to calculate a |
| 696 | /// lower bound. If \p ExternalAnalysis is successful, it should return true. |
| 697 | /// |
| 698 | /// If this is called on a non-pointer value, it returns 'this' and the |
| 699 | /// \p Offset is not modified. |
| 700 | /// |
| 701 | /// Note that this function will never return a nullptr. It will also never |
| 702 | /// manipulate the \p Offset in a way that would not match the difference |
| 703 | /// between the underlying value and the returned one. Thus, if no constant |
| 704 | /// offset was found, the returned value is the underlying one and \p Offset |
| 705 | /// is unchanged. |
| 706 | const Value *stripAndAccumulateConstantOffsets( |
| 707 | const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, |
| 708 | function_ref<bool(Value &Value, APInt &Offset)> ExternalAnalysis = |
| 709 | nullptr) const; |
| 710 | Value *stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, |
| 711 | bool AllowNonInbounds) { |
| 712 | return const_cast<Value *>( |
| 713 | static_cast<const Value *>(this)->stripAndAccumulateConstantOffsets( |
| 714 | DL, Offset, AllowNonInbounds)); |
| 715 | } |
| 716 | |
| 717 | /// This is a wrapper around stripAndAccumulateConstantOffsets with the |
| 718 | /// in-bounds requirement set to false. |
| 719 | const Value *stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, |
| 720 | APInt &Offset) const { |
| 721 | return stripAndAccumulateConstantOffsets(DL, Offset, |
| 722 | /* AllowNonInbounds */ false); |
| 723 | } |
| 724 | Value *stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, |
| 725 | APInt &Offset) { |
| 726 | return stripAndAccumulateConstantOffsets(DL, Offset, |
| 727 | /* AllowNonInbounds */ false); |
| 728 | } |
| 729 | |
| 730 | /// Strip off pointer casts and inbounds GEPs. |
| 731 | /// |
| 732 | /// Returns the original pointer value. If this is called on a non-pointer |
| 733 | /// value, it returns 'this'. |
| 734 | const Value *stripInBoundsOffsets(function_ref<void(const Value *)> Func = |
| 735 | [](const Value *) {}) const; |
| 736 | inline Value *stripInBoundsOffsets(function_ref<void(const Value *)> Func = |
| 737 | [](const Value *) {}) { |
| 738 | return const_cast<Value *>( |
| 739 | static_cast<const Value *>(this)->stripInBoundsOffsets(Func)); |
| 740 | } |
| 741 | |
| 742 | /// Return true if the memory object referred to by V can by freed in the |
| 743 | /// scope for which the SSA value defining the allocation is statically |
| 744 | /// defined. E.g. deallocation after the static scope of a value does not |
| 745 | /// count, but a deallocation before that does. |
| 746 | bool canBeFreed() const; |
| 747 | |
| 748 | /// Returns the number of bytes known to be dereferenceable for the |
| 749 | /// pointer value. |
| 750 | /// |
| 751 | /// If CanBeNull is set by this function the pointer can either be null or be |
| 752 | /// dereferenceable up to the returned number of bytes. |
| 753 | /// |
| 754 | /// IF CanBeFreed is true, the pointer is known to be dereferenceable at |
| 755 | /// point of definition only. Caller must prove that allocation is not |
| 756 | /// deallocated between point of definition and use. |
| 757 | uint64_t getPointerDereferenceableBytes(const DataLayout &DL, |
| 758 | bool &CanBeNull, |
| 759 | bool &CanBeFreed) const; |
| 760 | |
| 761 | /// Returns an alignment of the pointer value. |
| 762 | /// |
| 763 | /// Returns an alignment which is either specified explicitly, e.g. via |
| 764 | /// align attribute of a function argument, or guaranteed by DataLayout. |
| 765 | Align getPointerAlignment(const DataLayout &DL) const; |
| 766 | |
| 767 | /// Translate PHI node to its predecessor from the given basic block. |
| 768 | /// |
| 769 | /// If this value is a PHI node with CurBB as its parent, return the value in |
| 770 | /// the PHI node corresponding to PredBB. If not, return ourself. This is |
| 771 | /// useful if you want to know the value something has in a predecessor |
| 772 | /// block. |
| 773 | const Value *DoPHITranslation(const BasicBlock *CurBB, |
| 774 | const BasicBlock *PredBB) const; |
| 775 | Value *DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) { |
| 776 | return const_cast<Value *>( |
| 777 | static_cast<const Value *>(this)->DoPHITranslation(CurBB, PredBB)); |
| 778 | } |
| 779 | |
| 780 | /// The maximum alignment for instructions. |
| 781 | /// |
| 782 | /// This is the greatest alignment value supported by load, store, and alloca |
| 783 | /// instructions, and global values. |
| 784 | static const unsigned MaxAlignmentExponent = 29; |
| 785 | static const unsigned MaximumAlignment = 1u << MaxAlignmentExponent; |
| 786 | |
| 787 | /// Mutate the type of this Value to be of the specified type. |
| 788 | /// |
| 789 | /// Note that this is an extremely dangerous operation which can create |
| 790 | /// completely invalid IR very easily. It is strongly recommended that you |
| 791 | /// recreate IR objects with the right types instead of mutating them in |
| 792 | /// place. |
| 793 | void mutateType(Type *Ty) { |
| 794 | VTy = Ty; |
| 795 | } |
| 796 | |
| 797 | /// Sort the use-list. |
| 798 | /// |
| 799 | /// Sorts the Value's use-list by Cmp using a stable mergesort. Cmp is |
| 800 | /// expected to compare two \a Use references. |
| 801 | template <class Compare> void sortUseList(Compare Cmp); |
| 802 | |
| 803 | /// Reverse the use-list. |
| 804 | void reverseUseList(); |
| 805 | |
| 806 | private: |
| 807 | /// Merge two lists together. |
| 808 | /// |
| 809 | /// Merges \c L and \c R using \c Cmp. To enable stable sorts, always pushes |
| 810 | /// "equal" items from L before items from R. |
| 811 | /// |
| 812 | /// \return the first element in the list. |
| 813 | /// |
| 814 | /// \note Completely ignores \a Use::Prev (doesn't read, doesn't update). |
| 815 | template <class Compare> |
| 816 | static Use *mergeUseLists(Use *L, Use *R, Compare Cmp) { |
| 817 | Use *Merged; |
| 818 | Use **Next = &Merged; |
| 819 | |
| 820 | while (true) { |
| 821 | if (!L) { |
| 822 | *Next = R; |
| 823 | break; |
| 824 | } |
| 825 | if (!R) { |
| 826 | *Next = L; |
| 827 | break; |
| 828 | } |
| 829 | if (Cmp(*R, *L)) { |
| 830 | *Next = R; |
| 831 | Next = &R->Next; |
| 832 | R = R->Next; |
| 833 | } else { |
| 834 | *Next = L; |
| 835 | Next = &L->Next; |
| 836 | L = L->Next; |
| 837 | } |
| 838 | } |
| 839 | |
| 840 | return Merged; |
| 841 | } |
| 842 | |
| 843 | protected: |
| 844 | unsigned short getSubclassDataFromValue() const { return SubclassData; } |
| 845 | void setValueSubclassData(unsigned short D) { SubclassData = D; } |
| 846 | }; |
| 847 | |
| 848 | struct ValueDeleter { void operator()(Value *V) { V->deleteValue(); } }; |
| 849 | |
| 850 | /// Use this instead of std::unique_ptr<Value> or std::unique_ptr<Instruction>. |
| 851 | /// Those don't work because Value and Instruction's destructors are protected, |
| 852 | /// aren't virtual, and won't destroy the complete object. |
| 853 | using unique_value = std::unique_ptr<Value, ValueDeleter>; |
| 854 | |
| 855 | inline raw_ostream &operator<<(raw_ostream &OS, const Value &V) { |
| 856 | V.print(OS); |
| 857 | return OS; |
| 858 | } |
| 859 | |
| 860 | void Use::set(Value *V) { |
| 861 | if (Val) removeFromList(); |
| 862 | Val = V; |
| 863 | if (V) V->addUse(*this); |
| 864 | } |
| 865 | |
| 866 | Value *Use::operator=(Value *RHS) { |
| 867 | set(RHS); |
| 868 | return RHS; |
| 869 | } |
| 870 | |
| 871 | const Use &Use::operator=(const Use &RHS) { |
| 872 | set(RHS.Val); |
| 873 | return *this; |
| 874 | } |
| 875 | |
| 876 | template <class Compare> void Value::sortUseList(Compare Cmp) { |
| 877 | if (!UseList || !UseList->Next) |
| 878 | // No need to sort 0 or 1 uses. |
| 879 | return; |
| 880 | |
| 881 | // Note: this function completely ignores Prev pointers until the end when |
| 882 | // they're fixed en masse. |
| 883 | |
| 884 | // Create a binomial vector of sorted lists, visiting uses one at a time and |
| 885 | // merging lists as necessary. |
| 886 | const unsigned MaxSlots = 32; |
| 887 | Use *Slots[MaxSlots]; |
| 888 | |
| 889 | // Collect the first use, turning it into a single-item list. |
| 890 | Use *Next = UseList->Next; |
| 891 | UseList->Next = nullptr; |
| 892 | unsigned NumSlots = 1; |
| 893 | Slots[0] = UseList; |
| 894 | |
| 895 | // Collect all but the last use. |
| 896 | while (Next->Next) { |
| 897 | Use *Current = Next; |
| 898 | Next = Current->Next; |
| 899 | |
| 900 | // Turn Current into a single-item list. |
| 901 | Current->Next = nullptr; |
| 902 | |
| 903 | // Save Current in the first available slot, merging on collisions. |
| 904 | unsigned I; |
| 905 | for (I = 0; I < NumSlots; ++I) { |
| 906 | if (!Slots[I]) |
| 907 | break; |
| 908 | |
| 909 | // Merge two lists, doubling the size of Current and emptying slot I. |
| 910 | // |
| 911 | // Since the uses in Slots[I] originally preceded those in Current, send |
| 912 | // Slots[I] in as the left parameter to maintain a stable sort. |
| 913 | Current = mergeUseLists(Slots[I], Current, Cmp); |
| 914 | Slots[I] = nullptr; |
| 915 | } |
| 916 | // Check if this is a new slot. |
| 917 | if (I == NumSlots) { |
| 918 | ++NumSlots; |
| 919 | assert(NumSlots <= MaxSlots && "Use list bigger than 2^32")((void)0); |
| 920 | } |
| 921 | |
| 922 | // Found an open slot. |
| 923 | Slots[I] = Current; |
| 924 | } |
| 925 | |
| 926 | // Merge all the lists together. |
| 927 | assert(Next && "Expected one more Use")((void)0); |
| 928 | assert(!Next->Next && "Expected only one Use")((void)0); |
| 929 | UseList = Next; |
| 930 | for (unsigned I = 0; I < NumSlots; ++I) |
| 931 | if (Slots[I]) |
| 932 | // Since the uses in Slots[I] originally preceded those in UseList, send |
| 933 | // Slots[I] in as the left parameter to maintain a stable sort. |
| 934 | UseList = mergeUseLists(Slots[I], UseList, Cmp); |
| 935 | |
| 936 | // Fix the Prev pointers. |
| 937 | for (Use *I = UseList, **Prev = &UseList; I; I = I->Next) { |
| 938 | I->Prev = Prev; |
| 939 | Prev = &I->Next; |
| 940 | } |
| 941 | } |
| 942 | |
| 943 | // isa - Provide some specializations of isa so that we don't have to include |
| 944 | // the subtype header files to test to see if the value is a subclass... |
| 945 | // |
| 946 | template <> struct isa_impl<Constant, Value> { |
| 947 | static inline bool doit(const Value &Val) { |
| 948 | static_assert(Value::ConstantFirstVal == 0, "Val.getValueID() >= Value::ConstantFirstVal"); |
| 949 | return Val.getValueID() <= Value::ConstantLastVal; |
| 950 | } |
| 951 | }; |
| 952 | |
| 953 | template <> struct isa_impl<ConstantData, Value> { |
| 954 | static inline bool doit(const Value &Val) { |
| 955 | return Val.getValueID() >= Value::ConstantDataFirstVal && |
| 956 | Val.getValueID() <= Value::ConstantDataLastVal; |
| 957 | } |
| 958 | }; |
| 959 | |
| 960 | template <> struct isa_impl<ConstantAggregate, Value> { |
| 961 | static inline bool doit(const Value &Val) { |
| 962 | return Val.getValueID() >= Value::ConstantAggregateFirstVal && |
| 963 | Val.getValueID() <= Value::ConstantAggregateLastVal; |
| 964 | } |
| 965 | }; |
| 966 | |
| 967 | template <> struct isa_impl<Argument, Value> { |
| 968 | static inline bool doit (const Value &Val) { |
| 969 | return Val.getValueID() == Value::ArgumentVal; |
| 970 | } |
| 971 | }; |
| 972 | |
| 973 | template <> struct isa_impl<InlineAsm, Value> { |
| 974 | static inline bool doit(const Value &Val) { |
| 975 | return Val.getValueID() == Value::InlineAsmVal; |
| 976 | } |
| 977 | }; |
| 978 | |
| 979 | template <> struct isa_impl<Instruction, Value> { |
| 980 | static inline bool doit(const Value &Val) { |
| 981 | return Val.getValueID() >= Value::InstructionVal; |
| 982 | } |
| 983 | }; |
| 984 | |
| 985 | template <> struct isa_impl<BasicBlock, Value> { |
| 986 | static inline bool doit(const Value &Val) { |
| 987 | return Val.getValueID() == Value::BasicBlockVal; |
| 988 | } |
| 989 | }; |
| 990 | |
| 991 | template <> struct isa_impl<Function, Value> { |
| 992 | static inline bool doit(const Value &Val) { |
| 993 | return Val.getValueID() == Value::FunctionVal; |
| 994 | } |
| 995 | }; |
| 996 | |
| 997 | template <> struct isa_impl<GlobalVariable, Value> { |
| 998 | static inline bool doit(const Value &Val) { |
| 999 | return Val.getValueID() == Value::GlobalVariableVal; |
| 1000 | } |
| 1001 | }; |
| 1002 | |
| 1003 | template <> struct isa_impl<GlobalAlias, Value> { |
| 1004 | static inline bool doit(const Value &Val) { |
| 1005 | return Val.getValueID() == Value::GlobalAliasVal; |
| 1006 | } |
| 1007 | }; |
| 1008 | |
| 1009 | template <> struct isa_impl<GlobalIFunc, Value> { |
| 1010 | static inline bool doit(const Value &Val) { |
| 1011 | return Val.getValueID() == Value::GlobalIFuncVal; |
| 1012 | } |
| 1013 | }; |
| 1014 | |
| 1015 | template <> struct isa_impl<GlobalIndirectSymbol, Value> { |
| 1016 | static inline bool doit(const Value &Val) { |
| 1017 | return isa<GlobalAlias>(Val) || isa<GlobalIFunc>(Val); |
| 1018 | } |
| 1019 | }; |
| 1020 | |
| 1021 | template <> struct isa_impl<GlobalValue, Value> { |
| 1022 | static inline bool doit(const Value &Val) { |
| 1023 | return isa<GlobalObject>(Val) || isa<GlobalIndirectSymbol>(Val); |
| 1024 | } |
| 1025 | }; |
| 1026 | |
| 1027 | template <> struct isa_impl<GlobalObject, Value> { |
| 1028 | static inline bool doit(const Value &Val) { |
| 1029 | return isa<GlobalVariable>(Val) || isa<Function>(Val); |
| 1030 | } |
| 1031 | }; |
| 1032 | |
| 1033 | // Create wrappers for C Binding types (see CBindingWrapping.h). |
| 1034 | DEFINE_ISA_CONVERSION_FUNCTIONS(Value, LLVMValueRef)inline Value *unwrap(LLVMValueRef P) { return reinterpret_cast <Value*>(P); } inline LLVMValueRef wrap(const Value *P) { return reinterpret_cast<LLVMValueRef>(const_cast< Value*>(P)); } template<typename T> inline T *unwrap (LLVMValueRef P) { return cast<T>(unwrap(P)); } |
| 1035 | |
| 1036 | // Specialized opaque value conversions. |
| 1037 | inline Value **unwrap(LLVMValueRef *Vals) { |
| 1038 | return reinterpret_cast<Value**>(Vals); |
| 1039 | } |
| 1040 | |
| 1041 | template<typename T> |
| 1042 | inline T **unwrap(LLVMValueRef *Vals, unsigned Length) { |
| 1043 | #ifndef NDEBUG1 |
| 1044 | for (LLVMValueRef *I = Vals, *E = Vals + Length; I != E; ++I) |
| 1045 | unwrap<T>(*I); // For side effect of calling assert on invalid usage. |
| 1046 | #endif |
| 1047 | (void)Length; |
| 1048 | return reinterpret_cast<T**>(Vals); |
| 1049 | } |
| 1050 | |
| 1051 | inline LLVMValueRef *wrap(const Value **Vals) { |
| 1052 | return reinterpret_cast<LLVMValueRef*>(const_cast<Value**>(Vals)); |
| 1053 | } |
| 1054 | |
| 1055 | } // end namespace llvm |
| 1056 | |
| 1057 | #endif // LLVM_IR_VALUE_H |
| 1 | //===- llvm/ADT/ilist_iterator.h - Intrusive List Iterator ------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef LLVM_ADT_ILIST_ITERATOR_H |
| 10 | #define LLVM_ADT_ILIST_ITERATOR_H |
| 11 | |
| 12 | #include "llvm/ADT/ilist_node.h" |
| 13 | #include <cassert> |
| 14 | #include <cstddef> |
| 15 | #include <iterator> |
| 16 | #include <type_traits> |
| 17 | |
| 18 | namespace llvm { |
| 19 | |
| 20 | namespace ilist_detail { |
| 21 | |
| 22 | /// Find const-correct node types. |
| 23 | template <class OptionsT, bool IsConst> struct IteratorTraits; |
| 24 | template <class OptionsT> struct IteratorTraits<OptionsT, false> { |
| 25 | using value_type = typename OptionsT::value_type; |
| 26 | using pointer = typename OptionsT::pointer; |
| 27 | using reference = typename OptionsT::reference; |
| 28 | using node_pointer = ilist_node_impl<OptionsT> *; |
| 29 | using node_reference = ilist_node_impl<OptionsT> &; |
| 30 | }; |
| 31 | template <class OptionsT> struct IteratorTraits<OptionsT, true> { |
| 32 | using value_type = const typename OptionsT::value_type; |
| 33 | using pointer = typename OptionsT::const_pointer; |
| 34 | using reference = typename OptionsT::const_reference; |
| 35 | using node_pointer = const ilist_node_impl<OptionsT> *; |
| 36 | using node_reference = const ilist_node_impl<OptionsT> &; |
| 37 | }; |
| 38 | |
| 39 | template <bool IsReverse> struct IteratorHelper; |
| 40 | template <> struct IteratorHelper<false> : ilist_detail::NodeAccess { |
| 41 | using Access = ilist_detail::NodeAccess; |
| 42 | |
| 43 | template <class T> static void increment(T *&I) { I = Access::getNext(*I); } |
| 44 | template <class T> static void decrement(T *&I) { I = Access::getPrev(*I); } |
| 45 | }; |
| 46 | template <> struct IteratorHelper<true> : ilist_detail::NodeAccess { |
| 47 | using Access = ilist_detail::NodeAccess; |
| 48 | |
| 49 | template <class T> static void increment(T *&I) { I = Access::getPrev(*I); } |
| 50 | template <class T> static void decrement(T *&I) { I = Access::getNext(*I); } |
| 51 | }; |
| 52 | |
| 53 | } // end namespace ilist_detail |
| 54 | |
| 55 | /// Iterator for intrusive lists based on ilist_node. |
| 56 | template <class OptionsT, bool IsReverse, bool IsConst> |
| 57 | class ilist_iterator : ilist_detail::SpecificNodeAccess<OptionsT> { |
| 58 | friend ilist_iterator<OptionsT, IsReverse, !IsConst>; |
| 59 | friend ilist_iterator<OptionsT, !IsReverse, IsConst>; |
| 60 | friend ilist_iterator<OptionsT, !IsReverse, !IsConst>; |
| 61 | |
| 62 | using Traits = ilist_detail::IteratorTraits<OptionsT, IsConst>; |
| 63 | using Access = ilist_detail::SpecificNodeAccess<OptionsT>; |
| 64 | |
| 65 | public: |
| 66 | using value_type = typename Traits::value_type; |
| 67 | using pointer = typename Traits::pointer; |
| 68 | using reference = typename Traits::reference; |
| 69 | using difference_type = ptrdiff_t; |
| 70 | using iterator_category = std::bidirectional_iterator_tag; |
| 71 | using const_pointer = typename OptionsT::const_pointer; |
| 72 | using const_reference = typename OptionsT::const_reference; |
| 73 | |
| 74 | private: |
| 75 | using node_pointer = typename Traits::node_pointer; |
| 76 | using node_reference = typename Traits::node_reference; |
| 77 | |
| 78 | node_pointer NodePtr = nullptr; |
| 79 | |
| 80 | public: |
| 81 | /// Create from an ilist_node. |
| 82 | explicit ilist_iterator(node_reference N) : NodePtr(&N) {} |
| 83 | |
| 84 | explicit ilist_iterator(pointer NP) : NodePtr(Access::getNodePtr(NP)) {} |
| 85 | explicit ilist_iterator(reference NR) : NodePtr(Access::getNodePtr(&NR)) {} |
| 86 | ilist_iterator() = default; |
| 87 | |
| 88 | // This is templated so that we can allow constructing a const iterator from |
| 89 | // a nonconst iterator... |
| 90 | template <bool RHSIsConst> |
| 91 | ilist_iterator(const ilist_iterator<OptionsT, IsReverse, RHSIsConst> &RHS, |
| 92 | std::enable_if_t<IsConst || !RHSIsConst, void *> = nullptr) |
| 93 | : NodePtr(RHS.NodePtr) {} |
| 94 | |
| 95 | // This is templated so that we can allow assigning to a const iterator from |
| 96 | // a nonconst iterator... |
| 97 | template <bool RHSIsConst> |
| 98 | std::enable_if_t<IsConst || !RHSIsConst, ilist_iterator &> |
| 99 | operator=(const ilist_iterator<OptionsT, IsReverse, RHSIsConst> &RHS) { |
| 100 | NodePtr = RHS.NodePtr; |
| 101 | return *this; |
| 102 | } |
| 103 | |
| 104 | /// Explicit conversion between forward/reverse iterators. |
| 105 | /// |
| 106 | /// Translate between forward and reverse iterators without changing range |
| 107 | /// boundaries. The resulting iterator will dereference (and have a handle) |
| 108 | /// to the previous node, which is somewhat unexpected; but converting the |
| 109 | /// two endpoints in a range will give the same range in reverse. |
| 110 | /// |
| 111 | /// This matches std::reverse_iterator conversions. |
| 112 | explicit ilist_iterator( |
| 113 | const ilist_iterator<OptionsT, !IsReverse, IsConst> &RHS) |
| 114 | : ilist_iterator(++RHS.getReverse()) {} |
| 115 | |
| 116 | /// Get a reverse iterator to the same node. |
| 117 | /// |
| 118 | /// Gives a reverse iterator that will dereference (and have a handle) to the |
| 119 | /// same node. Converting the endpoint iterators in a range will give a |
| 120 | /// different range; for range operations, use the explicit conversions. |
| 121 | ilist_iterator<OptionsT, !IsReverse, IsConst> getReverse() const { |
| 122 | if (NodePtr) |
| 123 | return ilist_iterator<OptionsT, !IsReverse, IsConst>(*NodePtr); |
| 124 | return ilist_iterator<OptionsT, !IsReverse, IsConst>(); |
| 125 | } |
| 126 | |
| 127 | /// Const-cast. |
| 128 | ilist_iterator<OptionsT, IsReverse, false> getNonConst() const { |
| 129 | if (NodePtr) |
| 130 | return ilist_iterator<OptionsT, IsReverse, false>( |
| 131 | const_cast<typename ilist_iterator<OptionsT, IsReverse, |
| 132 | false>::node_reference>(*NodePtr)); |
| 133 | return ilist_iterator<OptionsT, IsReverse, false>(); |
| 134 | } |
| 135 | |
| 136 | // Accessors... |
| 137 | reference operator*() const { |
| 138 | assert(!NodePtr->isKnownSentinel())((void)0); |
| 139 | return *Access::getValuePtr(NodePtr); |
| 140 | } |
| 141 | pointer operator->() const { return &operator*(); } |
| 142 | |
| 143 | // Comparison operators |
| 144 | friend bool operator==(const ilist_iterator &LHS, const ilist_iterator &RHS) { |
| 145 | return LHS.NodePtr == RHS.NodePtr; |
| 146 | } |
| 147 | friend bool operator!=(const ilist_iterator &LHS, const ilist_iterator &RHS) { |
| 148 | return LHS.NodePtr != RHS.NodePtr; |
| 149 | } |
| 150 | |
| 151 | // Increment and decrement operators... |
| 152 | ilist_iterator &operator--() { |
| 153 | NodePtr = IsReverse ? NodePtr->getNext() : NodePtr->getPrev(); |
| 154 | return *this; |
| 155 | } |
| 156 | ilist_iterator &operator++() { |
| 157 | NodePtr = IsReverse ? NodePtr->getPrev() : NodePtr->getNext(); |
| 158 | return *this; |
| 159 | } |
| 160 | ilist_iterator operator--(int) { |
| 161 | ilist_iterator tmp = *this; |
| 162 | --*this; |
| 163 | return tmp; |
| 164 | } |
| 165 | ilist_iterator operator++(int) { |
| 166 | ilist_iterator tmp = *this; |
| 167 | ++*this; |
| 168 | return tmp; |
| 169 | } |
| 170 | |
| 171 | /// Get the underlying ilist_node. |
| 172 | node_pointer getNodePtr() const { return static_cast<node_pointer>(NodePtr); } |
| 173 | |
| 174 | /// Check for end. Only valid if ilist_sentinel_tracking<true>. |
| 175 | bool isEnd() const { return NodePtr ? NodePtr->isSentinel() : false; } |
| 176 | }; |
| 177 | |
| 178 | template <typename From> struct simplify_type; |
| 179 | |
| 180 | /// Allow ilist_iterators to convert into pointers to a node automatically when |
| 181 | /// used by the dyn_cast, cast, isa mechanisms... |
| 182 | /// |
| 183 | /// FIXME: remove this, since there is no implicit conversion to NodeTy. |
| 184 | template <class OptionsT, bool IsConst> |
| 185 | struct simplify_type<ilist_iterator<OptionsT, false, IsConst>> { |
| 186 | using iterator = ilist_iterator<OptionsT, false, IsConst>; |
| 187 | using SimpleType = typename iterator::pointer; |
| 188 | |
| 189 | static SimpleType getSimplifiedValue(const iterator &Node) { return &*Node; } |
| 190 | }; |
| 191 | template <class OptionsT, bool IsConst> |
| 192 | struct simplify_type<const ilist_iterator<OptionsT, false, IsConst>> |
| 193 | : simplify_type<ilist_iterator<OptionsT, false, IsConst>> {}; |
| 194 | |
| 195 | } // end namespace llvm |
| 196 | |
| 197 | #endif // LLVM_ADT_ILIST_ITERATOR_H |