1193323Sed//===- ScalarEvolution.cpp - Scalar Evolution Analysis ----------*- C++ -*-===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file contains the implementation of the scalar evolution analysis 11193323Sed// engine, which is used primarily to analyze expressions involving induction 12193323Sed// variables in loops. 13193323Sed// 14193323Sed// There are several aspects to this library. First is the representation of 15193323Sed// scalar expressions, which are represented as subclasses of the SCEV class. 16193323Sed// These classes are used to represent certain types of subexpressions that we 17198090Srdivacky// can handle. We only create one SCEV of a particular shape, so 18198090Srdivacky// pointer-comparisons for equality are legal. 19193323Sed// 20193323Sed// One important aspect of the SCEV objects is that they are never cyclic, even 21193323Sed// if there is a cycle in the dataflow for an expression (ie, a PHI node). If 22193323Sed// the PHI node is one of the idioms that we can represent (e.g., a polynomial 23193323Sed// recurrence) then we represent it directly as a recurrence node, otherwise we 24193323Sed// represent it as a SCEVUnknown node. 25193323Sed// 26193323Sed// In addition to being able to represent expressions of various types, we also 27193323Sed// have folders that are used to build the *canonical* representation for a 28193323Sed// particular expression. These folders are capable of using a variety of 29193323Sed// rewrite rules to simplify the expressions. 30193323Sed// 31193323Sed// Once the folders are defined, we can implement the more interesting 32193323Sed// higher-level code, such as the code that recognizes PHI nodes of various 33193323Sed// types, computes the execution count of a loop, etc. 34193323Sed// 35193323Sed// TODO: We should use these routines and value representations to implement 36193323Sed// dependence analysis! 37193323Sed// 38193323Sed//===----------------------------------------------------------------------===// 39193323Sed// 40193323Sed// There are several good references for the techniques used in this analysis. 41193323Sed// 42193323Sed// Chains of recurrences -- a method to expedite the evaluation 43193323Sed// of closed-form functions 44193323Sed// Olaf Bachmann, Paul S. Wang, Eugene V. Zima 45193323Sed// 46193323Sed// On computational properties of chains of recurrences 47193323Sed// Eugene V. Zima 48193323Sed// 49193323Sed// Symbolic Evaluation of Chains of Recurrences for Loop Optimization 50193323Sed// Robert A. van Engelen 51193323Sed// 52193323Sed// Efficient Symbolic Analysis for Optimizing Compilers 53193323Sed// Robert A. van Engelen 54193323Sed// 55193323Sed// Using the chains of recurrences algebra for data dependence testing and 56193323Sed// induction variable substitution 57193323Sed// MS Thesis, Johnie Birch 58193323Sed// 59193323Sed//===----------------------------------------------------------------------===// 60193323Sed 61193323Sed#define DEBUG_TYPE "scalar-evolution" 62249423Sdim#include "llvm/Analysis/ScalarEvolution.h" 63249423Sdim#include "llvm/ADT/STLExtras.h" 64249423Sdim#include "llvm/ADT/SmallPtrSet.h" 65249423Sdim#include "llvm/ADT/Statistic.h" 66193323Sed#include "llvm/Analysis/ConstantFolding.h" 67193323Sed#include "llvm/Analysis/Dominators.h" 68218893Sdim#include "llvm/Analysis/InstructionSimplify.h" 69193323Sed#include "llvm/Analysis/LoopInfo.h" 70249423Sdim#include "llvm/Analysis/ScalarEvolutionExpressions.h" 71194612Sed#include "llvm/Analysis/ValueTracking.h" 72193323Sed#include "llvm/Assembly/Writer.h" 73249423Sdim#include "llvm/IR/Constants.h" 74249423Sdim#include "llvm/IR/DataLayout.h" 75249423Sdim#include "llvm/IR/DerivedTypes.h" 76249423Sdim#include "llvm/IR/GlobalAlias.h" 77249423Sdim#include "llvm/IR/GlobalVariable.h" 78249423Sdim#include "llvm/IR/Instructions.h" 79249423Sdim#include "llvm/IR/LLVMContext.h" 80249423Sdim#include "llvm/IR/Operator.h" 81193323Sed#include "llvm/Support/CommandLine.h" 82193323Sed#include "llvm/Support/ConstantRange.h" 83201360Srdivacky#include "llvm/Support/Debug.h" 84198090Srdivacky#include "llvm/Support/ErrorHandling.h" 85193323Sed#include "llvm/Support/GetElementPtrTypeIterator.h" 86193323Sed#include "llvm/Support/InstIterator.h" 87193323Sed#include "llvm/Support/MathExtras.h" 88193323Sed#include "llvm/Support/raw_ostream.h" 89249423Sdim#include "llvm/Target/TargetLibraryInfo.h" 90193323Sed#include <algorithm> 91193323Sedusing namespace llvm; 92193323Sed 93193323SedSTATISTIC(NumArrayLenItCounts, 94193323Sed "Number of trip counts computed with array length"); 95193323SedSTATISTIC(NumTripCountsComputed, 96193323Sed "Number of loops with predictable loop counts"); 97193323SedSTATISTIC(NumTripCountsNotComputed, 98193323Sed "Number of loops without predictable loop counts"); 99193323SedSTATISTIC(NumBruteForceTripCountsComputed, 100193323Sed "Number of loops with trip counts computed by force"); 101193323Sed 102193323Sedstatic cl::opt<unsigned> 103193323SedMaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, 104193323Sed cl::desc("Maximum number of iterations SCEV will " 105195098Sed "symbolically execute a constant " 106195098Sed "derived loop"), 107193323Sed cl::init(100)); 108193323Sed 109243830Sdim// FIXME: Enable this with XDEBUG when the test suite is clean. 110243830Sdimstatic cl::opt<bool> 111243830SdimVerifySCEV("verify-scev", 112243830Sdim cl::desc("Verify ScalarEvolution's backedge taken counts (slow)")); 113243830Sdim 114218893SdimINITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution", 115218893Sdim "Scalar Evolution Analysis", false, true) 116218893SdimINITIALIZE_PASS_DEPENDENCY(LoopInfo) 117218893SdimINITIALIZE_PASS_DEPENDENCY(DominatorTree) 118234353SdimINITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo) 119218893SdimINITIALIZE_PASS_END(ScalarEvolution, "scalar-evolution", 120218893Sdim "Scalar Evolution Analysis", false, true) 121193323Sedchar ScalarEvolution::ID = 0; 122193323Sed 123193323Sed//===----------------------------------------------------------------------===// 124193323Sed// SCEV class definitions 125193323Sed//===----------------------------------------------------------------------===// 126193323Sed 127193323Sed//===----------------------------------------------------------------------===// 128193323Sed// Implementation of the SCEV class. 129193323Sed// 130195340Sed 131243830Sdim#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 132193323Sedvoid SCEV::dump() const { 133201360Srdivacky print(dbgs()); 134201360Srdivacky dbgs() << '\n'; 135193323Sed} 136243830Sdim#endif 137193323Sed 138218893Sdimvoid SCEV::print(raw_ostream &OS) const { 139218893Sdim switch (getSCEVType()) { 140218893Sdim case scConstant: 141218893Sdim WriteAsOperand(OS, cast<SCEVConstant>(this)->getValue(), false); 142218893Sdim return; 143218893Sdim case scTruncate: { 144218893Sdim const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this); 145218893Sdim const SCEV *Op = Trunc->getOperand(); 146218893Sdim OS << "(trunc " << *Op->getType() << " " << *Op << " to " 147218893Sdim << *Trunc->getType() << ")"; 148218893Sdim return; 149218893Sdim } 150218893Sdim case scZeroExtend: { 151218893Sdim const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this); 152218893Sdim const SCEV *Op = ZExt->getOperand(); 153218893Sdim OS << "(zext " << *Op->getType() << " " << *Op << " to " 154218893Sdim << *ZExt->getType() << ")"; 155218893Sdim return; 156218893Sdim } 157218893Sdim case scSignExtend: { 158218893Sdim const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this); 159218893Sdim const SCEV *Op = SExt->getOperand(); 160218893Sdim OS << "(sext " << *Op->getType() << " " << *Op << " to " 161218893Sdim << *SExt->getType() << ")"; 162218893Sdim return; 163218893Sdim } 164218893Sdim case scAddRecExpr: { 165218893Sdim const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this); 166218893Sdim OS << "{" << *AR->getOperand(0); 167218893Sdim for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i) 168218893Sdim OS << ",+," << *AR->getOperand(i); 169218893Sdim OS << "}<"; 170221345Sdim if (AR->getNoWrapFlags(FlagNUW)) 171218893Sdim OS << "nuw><"; 172221345Sdim if (AR->getNoWrapFlags(FlagNSW)) 173218893Sdim OS << "nsw><"; 174221345Sdim if (AR->getNoWrapFlags(FlagNW) && 175221345Sdim !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW))) 176221345Sdim OS << "nw><"; 177218893Sdim WriteAsOperand(OS, AR->getLoop()->getHeader(), /*PrintType=*/false); 178218893Sdim OS << ">"; 179218893Sdim return; 180218893Sdim } 181218893Sdim case scAddExpr: 182218893Sdim case scMulExpr: 183218893Sdim case scUMaxExpr: 184218893Sdim case scSMaxExpr: { 185218893Sdim const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this); 186218893Sdim const char *OpStr = 0; 187218893Sdim switch (NAry->getSCEVType()) { 188218893Sdim case scAddExpr: OpStr = " + "; break; 189218893Sdim case scMulExpr: OpStr = " * "; break; 190218893Sdim case scUMaxExpr: OpStr = " umax "; break; 191218893Sdim case scSMaxExpr: OpStr = " smax "; break; 192218893Sdim } 193218893Sdim OS << "("; 194218893Sdim for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 195218893Sdim I != E; ++I) { 196218893Sdim OS << **I; 197218893Sdim if (llvm::next(I) != E) 198218893Sdim OS << OpStr; 199218893Sdim } 200218893Sdim OS << ")"; 201234353Sdim switch (NAry->getSCEVType()) { 202234353Sdim case scAddExpr: 203234353Sdim case scMulExpr: 204234353Sdim if (NAry->getNoWrapFlags(FlagNUW)) 205234353Sdim OS << "<nuw>"; 206234353Sdim if (NAry->getNoWrapFlags(FlagNSW)) 207234353Sdim OS << "<nsw>"; 208234353Sdim } 209218893Sdim return; 210218893Sdim } 211218893Sdim case scUDivExpr: { 212218893Sdim const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this); 213218893Sdim OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")"; 214218893Sdim return; 215218893Sdim } 216218893Sdim case scUnknown: { 217218893Sdim const SCEVUnknown *U = cast<SCEVUnknown>(this); 218226633Sdim Type *AllocTy; 219218893Sdim if (U->isSizeOf(AllocTy)) { 220218893Sdim OS << "sizeof(" << *AllocTy << ")"; 221218893Sdim return; 222218893Sdim } 223218893Sdim if (U->isAlignOf(AllocTy)) { 224218893Sdim OS << "alignof(" << *AllocTy << ")"; 225218893Sdim return; 226218893Sdim } 227221345Sdim 228226633Sdim Type *CTy; 229218893Sdim Constant *FieldNo; 230218893Sdim if (U->isOffsetOf(CTy, FieldNo)) { 231218893Sdim OS << "offsetof(" << *CTy << ", "; 232218893Sdim WriteAsOperand(OS, FieldNo, false); 233218893Sdim OS << ")"; 234218893Sdim return; 235218893Sdim } 236221345Sdim 237218893Sdim // Otherwise just print it normally. 238218893Sdim WriteAsOperand(OS, U->getValue(), false); 239218893Sdim return; 240218893Sdim } 241218893Sdim case scCouldNotCompute: 242218893Sdim OS << "***COULDNOTCOMPUTE***"; 243218893Sdim return; 244218893Sdim default: break; 245218893Sdim } 246218893Sdim llvm_unreachable("Unknown SCEV kind!"); 247218893Sdim} 248218893Sdim 249226633SdimType *SCEV::getType() const { 250218893Sdim switch (getSCEVType()) { 251218893Sdim case scConstant: 252218893Sdim return cast<SCEVConstant>(this)->getType(); 253218893Sdim case scTruncate: 254218893Sdim case scZeroExtend: 255218893Sdim case scSignExtend: 256218893Sdim return cast<SCEVCastExpr>(this)->getType(); 257218893Sdim case scAddRecExpr: 258218893Sdim case scMulExpr: 259218893Sdim case scUMaxExpr: 260218893Sdim case scSMaxExpr: 261218893Sdim return cast<SCEVNAryExpr>(this)->getType(); 262218893Sdim case scAddExpr: 263218893Sdim return cast<SCEVAddExpr>(this)->getType(); 264218893Sdim case scUDivExpr: 265218893Sdim return cast<SCEVUDivExpr>(this)->getType(); 266218893Sdim case scUnknown: 267218893Sdim return cast<SCEVUnknown>(this)->getType(); 268218893Sdim case scCouldNotCompute: 269218893Sdim llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 270234353Sdim default: 271234353Sdim llvm_unreachable("Unknown SCEV kind!"); 272218893Sdim } 273218893Sdim} 274218893Sdim 275193323Sedbool SCEV::isZero() const { 276193323Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 277193323Sed return SC->getValue()->isZero(); 278193323Sed return false; 279193323Sed} 280193323Sed 281193323Sedbool SCEV::isOne() const { 282193323Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 283193323Sed return SC->getValue()->isOne(); 284193323Sed return false; 285193323Sed} 286193323Sed 287195098Sedbool SCEV::isAllOnesValue() const { 288195098Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this)) 289195098Sed return SC->getValue()->isAllOnesValue(); 290195098Sed return false; 291195098Sed} 292195098Sed 293234353Sdim/// isNonConstantNegative - Return true if the specified scev is negated, but 294234353Sdim/// not a constant. 295234353Sdimbool SCEV::isNonConstantNegative() const { 296234353Sdim const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this); 297234353Sdim if (!Mul) return false; 298234353Sdim 299234353Sdim // If there is a constant factor, it will be first. 300234353Sdim const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); 301234353Sdim if (!SC) return false; 302234353Sdim 303234353Sdim // Return true if the value is negative, this matches things like (-42 * V). 304234353Sdim return SC->getValue()->getValue().isNegative(); 305234353Sdim} 306234353Sdim 307194710SedSCEVCouldNotCompute::SCEVCouldNotCompute() : 308205407Srdivacky SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {} 309193323Sed 310193323Sedbool SCEVCouldNotCompute::classof(const SCEV *S) { 311193323Sed return S->getSCEVType() == scCouldNotCompute; 312193323Sed} 313193323Sed 314198090Srdivackyconst SCEV *ScalarEvolution::getConstant(ConstantInt *V) { 315195340Sed FoldingSetNodeID ID; 316195340Sed ID.AddInteger(scConstant); 317195340Sed ID.AddPointer(V); 318195340Sed void *IP = 0; 319195340Sed if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 320205407Srdivacky SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V); 321195340Sed UniqueSCEVs.InsertNode(S, IP); 322195340Sed return S; 323193323Sed} 324193323Sed 325198090Srdivackyconst SCEV *ScalarEvolution::getConstant(const APInt& Val) { 326198090Srdivacky return getConstant(ConstantInt::get(getContext(), Val)); 327193323Sed} 328193323Sed 329198090Srdivackyconst SCEV * 330226633SdimScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) { 331226633Sdim IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty)); 332207618Srdivacky return getConstant(ConstantInt::get(ITy, V, isSigned)); 333194612Sed} 334194612Sed 335205407SrdivackySCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, 336226633Sdim unsigned SCEVTy, const SCEV *op, Type *ty) 337198090Srdivacky : SCEV(ID, SCEVTy), Op(op), Ty(ty) {} 338193323Sed 339205407SrdivackySCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, 340226633Sdim const SCEV *op, Type *ty) 341198090Srdivacky : SCEVCastExpr(ID, scTruncate, op, ty) { 342204642Srdivacky assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 343204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 344193323Sed "Cannot truncate non-integer value!"); 345193323Sed} 346193323Sed 347205407SrdivackySCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, 348226633Sdim const SCEV *op, Type *ty) 349198090Srdivacky : SCEVCastExpr(ID, scZeroExtend, op, ty) { 350204642Srdivacky assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 351204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 352193323Sed "Cannot zero extend non-integer value!"); 353193323Sed} 354193323Sed 355205407SrdivackySCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, 356226633Sdim const SCEV *op, Type *ty) 357198090Srdivacky : SCEVCastExpr(ID, scSignExtend, op, ty) { 358204642Srdivacky assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) && 359204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 360193323Sed "Cannot sign extend non-integer value!"); 361193323Sed} 362193323Sed 363212904Sdimvoid SCEVUnknown::deleted() { 364218893Sdim // Clear this SCEVUnknown from various maps. 365218893Sdim SE->forgetMemoizedResults(this); 366212904Sdim 367212904Sdim // Remove this SCEVUnknown from the uniquing map. 368212904Sdim SE->UniqueSCEVs.RemoveNode(this); 369212904Sdim 370212904Sdim // Release the value. 371212904Sdim setValPtr(0); 372212904Sdim} 373212904Sdim 374212904Sdimvoid SCEVUnknown::allUsesReplacedWith(Value *New) { 375218893Sdim // Clear this SCEVUnknown from various maps. 376218893Sdim SE->forgetMemoizedResults(this); 377212904Sdim 378212904Sdim // Remove this SCEVUnknown from the uniquing map. 379212904Sdim SE->UniqueSCEVs.RemoveNode(this); 380212904Sdim 381212904Sdim // Update this SCEVUnknown to point to the new value. This is needed 382212904Sdim // because there may still be outstanding SCEVs which still point to 383212904Sdim // this SCEVUnknown. 384212904Sdim setValPtr(New); 385212904Sdim} 386212904Sdim 387226633Sdimbool SCEVUnknown::isSizeOf(Type *&AllocTy) const { 388212904Sdim if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 389203954Srdivacky if (VCE->getOpcode() == Instruction::PtrToInt) 390203954Srdivacky if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 391203954Srdivacky if (CE->getOpcode() == Instruction::GetElementPtr && 392203954Srdivacky CE->getOperand(0)->isNullValue() && 393203954Srdivacky CE->getNumOperands() == 2) 394203954Srdivacky if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1))) 395203954Srdivacky if (CI->isOne()) { 396203954Srdivacky AllocTy = cast<PointerType>(CE->getOperand(0)->getType()) 397203954Srdivacky ->getElementType(); 398203954Srdivacky return true; 399203954Srdivacky } 400203954Srdivacky 401203954Srdivacky return false; 402203954Srdivacky} 403203954Srdivacky 404226633Sdimbool SCEVUnknown::isAlignOf(Type *&AllocTy) const { 405212904Sdim if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 406203954Srdivacky if (VCE->getOpcode() == Instruction::PtrToInt) 407203954Srdivacky if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 408203954Srdivacky if (CE->getOpcode() == Instruction::GetElementPtr && 409203954Srdivacky CE->getOperand(0)->isNullValue()) { 410226633Sdim Type *Ty = 411203954Srdivacky cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 412226633Sdim if (StructType *STy = dyn_cast<StructType>(Ty)) 413203954Srdivacky if (!STy->isPacked() && 414203954Srdivacky CE->getNumOperands() == 3 && 415203954Srdivacky CE->getOperand(1)->isNullValue()) { 416203954Srdivacky if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2))) 417203954Srdivacky if (CI->isOne() && 418203954Srdivacky STy->getNumElements() == 2 && 419203954Srdivacky STy->getElementType(0)->isIntegerTy(1)) { 420203954Srdivacky AllocTy = STy->getElementType(1); 421203954Srdivacky return true; 422203954Srdivacky } 423203954Srdivacky } 424203954Srdivacky } 425203954Srdivacky 426203954Srdivacky return false; 427203954Srdivacky} 428203954Srdivacky 429226633Sdimbool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const { 430212904Sdim if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue())) 431203954Srdivacky if (VCE->getOpcode() == Instruction::PtrToInt) 432203954Srdivacky if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0))) 433203954Srdivacky if (CE->getOpcode() == Instruction::GetElementPtr && 434203954Srdivacky CE->getNumOperands() == 3 && 435203954Srdivacky CE->getOperand(0)->isNullValue() && 436203954Srdivacky CE->getOperand(1)->isNullValue()) { 437226633Sdim Type *Ty = 438203954Srdivacky cast<PointerType>(CE->getOperand(0)->getType())->getElementType(); 439203954Srdivacky // Ignore vector types here so that ScalarEvolutionExpander doesn't 440203954Srdivacky // emit getelementptrs that index into vectors. 441204642Srdivacky if (Ty->isStructTy() || Ty->isArrayTy()) { 442203954Srdivacky CTy = Ty; 443203954Srdivacky FieldNo = CE->getOperand(2); 444203954Srdivacky return true; 445203954Srdivacky } 446203954Srdivacky } 447203954Srdivacky 448203954Srdivacky return false; 449203954Srdivacky} 450203954Srdivacky 451193323Sed//===----------------------------------------------------------------------===// 452193323Sed// SCEV Utilities 453193323Sed//===----------------------------------------------------------------------===// 454193323Sed 455193323Sednamespace { 456193323Sed /// SCEVComplexityCompare - Return true if the complexity of the LHS is less 457193323Sed /// than the complexity of the RHS. This comparator is used to canonicalize 458193323Sed /// expressions. 459198892Srdivacky class SCEVComplexityCompare { 460212904Sdim const LoopInfo *const LI; 461193323Sed public: 462212904Sdim explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {} 463193323Sed 464212904Sdim // Return true or false if LHS is less than, or at least RHS, respectively. 465193323Sed bool operator()(const SCEV *LHS, const SCEV *RHS) const { 466212904Sdim return compare(LHS, RHS) < 0; 467212904Sdim } 468212904Sdim 469212904Sdim // Return negative, zero, or positive, if LHS is less than, equal to, or 470212904Sdim // greater than RHS, respectively. A three-way result allows recursive 471212904Sdim // comparisons to be more efficient. 472212904Sdim int compare(const SCEV *LHS, const SCEV *RHS) const { 473198090Srdivacky // Fast-path: SCEVs are uniqued so we can do a quick equality check. 474198090Srdivacky if (LHS == RHS) 475212904Sdim return 0; 476198090Srdivacky 477193323Sed // Primarily, sort the SCEVs by their getSCEVType(). 478212904Sdim unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType(); 479212904Sdim if (LType != RType) 480212904Sdim return (int)LType - (int)RType; 481193323Sed 482193323Sed // Aside from the getSCEVType() ordering, the particular ordering 483193323Sed // isn't very important except that it's beneficial to be consistent, 484193323Sed // so that (a + b) and (b + a) don't end up as different expressions. 485212904Sdim switch (LType) { 486212904Sdim case scUnknown: { 487212904Sdim const SCEVUnknown *LU = cast<SCEVUnknown>(LHS); 488193323Sed const SCEVUnknown *RU = cast<SCEVUnknown>(RHS); 489193323Sed 490212904Sdim // Sort SCEVUnknown values with some loose heuristics. TODO: This is 491212904Sdim // not as complete as it could be. 492212904Sdim const Value *LV = LU->getValue(), *RV = RU->getValue(); 493212904Sdim 494193323Sed // Order pointer values after integer values. This helps SCEVExpander 495193323Sed // form GEPs. 496212904Sdim bool LIsPointer = LV->getType()->isPointerTy(), 497212904Sdim RIsPointer = RV->getType()->isPointerTy(); 498212904Sdim if (LIsPointer != RIsPointer) 499212904Sdim return (int)LIsPointer - (int)RIsPointer; 500193323Sed 501193323Sed // Compare getValueID values. 502212904Sdim unsigned LID = LV->getValueID(), 503212904Sdim RID = RV->getValueID(); 504212904Sdim if (LID != RID) 505212904Sdim return (int)LID - (int)RID; 506193323Sed 507193323Sed // Sort arguments by their position. 508212904Sdim if (const Argument *LA = dyn_cast<Argument>(LV)) { 509212904Sdim const Argument *RA = cast<Argument>(RV); 510212904Sdim unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo(); 511212904Sdim return (int)LArgNo - (int)RArgNo; 512193323Sed } 513193323Sed 514212904Sdim // For instructions, compare their loop depth, and their operand 515212904Sdim // count. This is pretty loose. 516212904Sdim if (const Instruction *LInst = dyn_cast<Instruction>(LV)) { 517212904Sdim const Instruction *RInst = cast<Instruction>(RV); 518193323Sed 519193323Sed // Compare loop depths. 520212904Sdim const BasicBlock *LParent = LInst->getParent(), 521212904Sdim *RParent = RInst->getParent(); 522212904Sdim if (LParent != RParent) { 523212904Sdim unsigned LDepth = LI->getLoopDepth(LParent), 524212904Sdim RDepth = LI->getLoopDepth(RParent); 525212904Sdim if (LDepth != RDepth) 526212904Sdim return (int)LDepth - (int)RDepth; 527212904Sdim } 528193323Sed 529193323Sed // Compare the number of operands. 530212904Sdim unsigned LNumOps = LInst->getNumOperands(), 531212904Sdim RNumOps = RInst->getNumOperands(); 532212904Sdim return (int)LNumOps - (int)RNumOps; 533193323Sed } 534193323Sed 535212904Sdim return 0; 536193323Sed } 537193323Sed 538212904Sdim case scConstant: { 539212904Sdim const SCEVConstant *LC = cast<SCEVConstant>(LHS); 540194612Sed const SCEVConstant *RC = cast<SCEVConstant>(RHS); 541212904Sdim 542212904Sdim // Compare constant values. 543212904Sdim const APInt &LA = LC->getValue()->getValue(); 544212904Sdim const APInt &RA = RC->getValue()->getValue(); 545212904Sdim unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth(); 546212904Sdim if (LBitWidth != RBitWidth) 547212904Sdim return (int)LBitWidth - (int)RBitWidth; 548212904Sdim return LA.ult(RA) ? -1 : 1; 549194612Sed } 550193323Sed 551212904Sdim case scAddRecExpr: { 552212904Sdim const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS); 553194612Sed const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS); 554212904Sdim 555212904Sdim // Compare addrec loop depths. 556212904Sdim const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop(); 557212904Sdim if (LLoop != RLoop) { 558212904Sdim unsigned LDepth = LLoop->getLoopDepth(), 559212904Sdim RDepth = RLoop->getLoopDepth(); 560212904Sdim if (LDepth != RDepth) 561212904Sdim return (int)LDepth - (int)RDepth; 562212904Sdim } 563212904Sdim 564212904Sdim // Addrec complexity grows with operand count. 565212904Sdim unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands(); 566212904Sdim if (LNumOps != RNumOps) 567212904Sdim return (int)LNumOps - (int)RNumOps; 568212904Sdim 569212904Sdim // Lexicographically compare. 570212904Sdim for (unsigned i = 0; i != LNumOps; ++i) { 571212904Sdim long X = compare(LA->getOperand(i), RA->getOperand(i)); 572212904Sdim if (X != 0) 573212904Sdim return X; 574212904Sdim } 575212904Sdim 576212904Sdim return 0; 577194612Sed } 578194612Sed 579212904Sdim case scAddExpr: 580212904Sdim case scMulExpr: 581212904Sdim case scSMaxExpr: 582212904Sdim case scUMaxExpr: { 583212904Sdim const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS); 584193323Sed const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS); 585212904Sdim 586212904Sdim // Lexicographically compare n-ary expressions. 587212904Sdim unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands(); 588212904Sdim for (unsigned i = 0; i != LNumOps; ++i) { 589212904Sdim if (i >= RNumOps) 590212904Sdim return 1; 591212904Sdim long X = compare(LC->getOperand(i), RC->getOperand(i)); 592212904Sdim if (X != 0) 593212904Sdim return X; 594193323Sed } 595212904Sdim return (int)LNumOps - (int)RNumOps; 596193323Sed } 597193323Sed 598212904Sdim case scUDivExpr: { 599212904Sdim const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS); 600193323Sed const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS); 601212904Sdim 602212904Sdim // Lexicographically compare udiv expressions. 603212904Sdim long X = compare(LC->getLHS(), RC->getLHS()); 604212904Sdim if (X != 0) 605212904Sdim return X; 606212904Sdim return compare(LC->getRHS(), RC->getRHS()); 607193323Sed } 608193323Sed 609212904Sdim case scTruncate: 610212904Sdim case scZeroExtend: 611212904Sdim case scSignExtend: { 612212904Sdim const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS); 613193323Sed const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS); 614212904Sdim 615212904Sdim // Compare cast expressions by operand. 616212904Sdim return compare(LC->getOperand(), RC->getOperand()); 617193323Sed } 618193323Sed 619212904Sdim default: 620234353Sdim llvm_unreachable("Unknown SCEV kind!"); 621212904Sdim } 622193323Sed } 623193323Sed }; 624193323Sed} 625193323Sed 626193323Sed/// GroupByComplexity - Given a list of SCEV objects, order them by their 627193323Sed/// complexity, and group objects of the same complexity together by value. 628193323Sed/// When this routine is finished, we know that any duplicates in the vector are 629193323Sed/// consecutive and that complexity is monotonically increasing. 630193323Sed/// 631204642Srdivacky/// Note that we go take special precautions to ensure that we get deterministic 632193323Sed/// results from this routine. In other words, we don't want the results of 633193323Sed/// this to depend on where the addresses of various SCEV objects happened to 634193323Sed/// land in memory. 635193323Sed/// 636198090Srdivackystatic void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops, 637193323Sed LoopInfo *LI) { 638193323Sed if (Ops.size() < 2) return; // Noop 639193323Sed if (Ops.size() == 2) { 640193323Sed // This is the common case, which also happens to be trivially simple. 641193323Sed // Special case it. 642212904Sdim const SCEV *&LHS = Ops[0], *&RHS = Ops[1]; 643212904Sdim if (SCEVComplexityCompare(LI)(RHS, LHS)) 644212904Sdim std::swap(LHS, RHS); 645193323Sed return; 646193323Sed } 647193323Sed 648193323Sed // Do the rough sort by complexity. 649193323Sed std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI)); 650193323Sed 651193323Sed // Now that we are sorted by complexity, group elements of the same 652193323Sed // complexity. Note that this is, at worst, N^2, but the vector is likely to 653193323Sed // be extremely short in practice. Note that we take this approach because we 654193323Sed // do not want to depend on the addresses of the objects we are grouping. 655193323Sed for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) { 656193323Sed const SCEV *S = Ops[i]; 657193323Sed unsigned Complexity = S->getSCEVType(); 658193323Sed 659193323Sed // If there are any objects of the same complexity and same value as this 660193323Sed // one, group them. 661193323Sed for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) { 662193323Sed if (Ops[j] == S) { // Found a duplicate. 663193323Sed // Move it to immediately after i'th element. 664193323Sed std::swap(Ops[i+1], Ops[j]); 665193323Sed ++i; // no need to rescan it. 666193323Sed if (i == e-2) return; // Done! 667193323Sed } 668193323Sed } 669193323Sed } 670193323Sed} 671193323Sed 672193323Sed 673193323Sed 674193323Sed//===----------------------------------------------------------------------===// 675193323Sed// Simple SCEV method implementations 676193323Sed//===----------------------------------------------------------------------===// 677193323Sed 678193323Sed/// BinomialCoefficient - Compute BC(It, K). The result has width W. 679193323Sed/// Assume, K > 0. 680198090Srdivackystatic const SCEV *BinomialCoefficient(const SCEV *It, unsigned K, 681198090Srdivacky ScalarEvolution &SE, 682226633Sdim Type *ResultTy) { 683193323Sed // Handle the simplest case efficiently. 684193323Sed if (K == 1) 685193323Sed return SE.getTruncateOrZeroExtend(It, ResultTy); 686193323Sed 687193323Sed // We are using the following formula for BC(It, K): 688193323Sed // 689193323Sed // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K! 690193323Sed // 691193323Sed // Suppose, W is the bitwidth of the return value. We must be prepared for 692193323Sed // overflow. Hence, we must assure that the result of our computation is 693193323Sed // equal to the accurate one modulo 2^W. Unfortunately, division isn't 694193323Sed // safe in modular arithmetic. 695193323Sed // 696193323Sed // However, this code doesn't use exactly that formula; the formula it uses 697195098Sed // is something like the following, where T is the number of factors of 2 in 698193323Sed // K! (i.e. trailing zeros in the binary representation of K!), and ^ is 699193323Sed // exponentiation: 700193323Sed // 701193323Sed // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T) 702193323Sed // 703193323Sed // This formula is trivially equivalent to the previous formula. However, 704193323Sed // this formula can be implemented much more efficiently. The trick is that 705193323Sed // K! / 2^T is odd, and exact division by an odd number *is* safe in modular 706193323Sed // arithmetic. To do exact division in modular arithmetic, all we have 707193323Sed // to do is multiply by the inverse. Therefore, this step can be done at 708193323Sed // width W. 709195098Sed // 710193323Sed // The next issue is how to safely do the division by 2^T. The way this 711193323Sed // is done is by doing the multiplication step at a width of at least W + T 712193323Sed // bits. This way, the bottom W+T bits of the product are accurate. Then, 713193323Sed // when we perform the division by 2^T (which is equivalent to a right shift 714193323Sed // by T), the bottom W bits are accurate. Extra bits are okay; they'll get 715193323Sed // truncated out after the division by 2^T. 716193323Sed // 717193323Sed // In comparison to just directly using the first formula, this technique 718193323Sed // is much more efficient; using the first formula requires W * K bits, 719193323Sed // but this formula less than W + K bits. Also, the first formula requires 720193323Sed // a division step, whereas this formula only requires multiplies and shifts. 721193323Sed // 722193323Sed // It doesn't matter whether the subtraction step is done in the calculation 723193323Sed // width or the input iteration count's width; if the subtraction overflows, 724193323Sed // the result must be zero anyway. We prefer here to do it in the width of 725193323Sed // the induction variable because it helps a lot for certain cases; CodeGen 726193323Sed // isn't smart enough to ignore the overflow, which leads to much less 727193323Sed // efficient code if the width of the subtraction is wider than the native 728193323Sed // register width. 729193323Sed // 730193323Sed // (It's possible to not widen at all by pulling out factors of 2 before 731193323Sed // the multiplication; for example, K=2 can be calculated as 732193323Sed // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires 733193323Sed // extra arithmetic, so it's not an obvious win, and it gets 734193323Sed // much more complicated for K > 3.) 735193323Sed 736193323Sed // Protection from insane SCEVs; this bound is conservative, 737193323Sed // but it probably doesn't matter. 738193323Sed if (K > 1000) 739193323Sed return SE.getCouldNotCompute(); 740193323Sed 741193323Sed unsigned W = SE.getTypeSizeInBits(ResultTy); 742193323Sed 743193323Sed // Calculate K! / 2^T and T; we divide out the factors of two before 744193323Sed // multiplying for calculating K! / 2^T to avoid overflow. 745193323Sed // Other overflow doesn't matter because we only care about the bottom 746193323Sed // W bits of the result. 747193323Sed APInt OddFactorial(W, 1); 748193323Sed unsigned T = 1; 749193323Sed for (unsigned i = 3; i <= K; ++i) { 750193323Sed APInt Mult(W, i); 751193323Sed unsigned TwoFactors = Mult.countTrailingZeros(); 752193323Sed T += TwoFactors; 753193323Sed Mult = Mult.lshr(TwoFactors); 754193323Sed OddFactorial *= Mult; 755193323Sed } 756193323Sed 757193323Sed // We need at least W + T bits for the multiplication step 758193323Sed unsigned CalculationBits = W + T; 759193323Sed 760204642Srdivacky // Calculate 2^T, at width T+W. 761193323Sed APInt DivFactor = APInt(CalculationBits, 1).shl(T); 762193323Sed 763193323Sed // Calculate the multiplicative inverse of K! / 2^T; 764193323Sed // this multiplication factor will perform the exact division by 765193323Sed // K! / 2^T. 766193323Sed APInt Mod = APInt::getSignedMinValue(W+1); 767193323Sed APInt MultiplyFactor = OddFactorial.zext(W+1); 768193323Sed MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod); 769193323Sed MultiplyFactor = MultiplyFactor.trunc(W); 770193323Sed 771193323Sed // Calculate the product, at width T+W 772226633Sdim IntegerType *CalculationTy = IntegerType::get(SE.getContext(), 773198090Srdivacky CalculationBits); 774198090Srdivacky const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy); 775193323Sed for (unsigned i = 1; i != K; ++i) { 776207618Srdivacky const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i)); 777193323Sed Dividend = SE.getMulExpr(Dividend, 778193323Sed SE.getTruncateOrZeroExtend(S, CalculationTy)); 779193323Sed } 780193323Sed 781193323Sed // Divide by 2^T 782198090Srdivacky const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor)); 783193323Sed 784193323Sed // Truncate the result, and divide by K! / 2^T. 785193323Sed 786193323Sed return SE.getMulExpr(SE.getConstant(MultiplyFactor), 787193323Sed SE.getTruncateOrZeroExtend(DivResult, ResultTy)); 788193323Sed} 789193323Sed 790193323Sed/// evaluateAtIteration - Return the value of this chain of recurrences at 791193323Sed/// the specified iteration number. We can evaluate this recurrence by 792193323Sed/// multiplying each element in the chain by the binomial coefficient 793193323Sed/// corresponding to it. In other words, we can evaluate {A,+,B,+,C,+,D} as: 794193323Sed/// 795193323Sed/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3) 796193323Sed/// 797193323Sed/// where BC(It, k) stands for binomial coefficient. 798193323Sed/// 799198090Srdivackyconst SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It, 800198090Srdivacky ScalarEvolution &SE) const { 801198090Srdivacky const SCEV *Result = getStart(); 802193323Sed for (unsigned i = 1, e = getNumOperands(); i != e; ++i) { 803193323Sed // The computation is correct in the face of overflow provided that the 804193323Sed // multiplication is performed _after_ the evaluation of the binomial 805193323Sed // coefficient. 806198090Srdivacky const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType()); 807193323Sed if (isa<SCEVCouldNotCompute>(Coeff)) 808193323Sed return Coeff; 809193323Sed 810193323Sed Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff)); 811193323Sed } 812193323Sed return Result; 813193323Sed} 814193323Sed 815193323Sed//===----------------------------------------------------------------------===// 816193323Sed// SCEV Expression folder implementations 817193323Sed//===----------------------------------------------------------------------===// 818193323Sed 819198090Srdivackyconst SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, 820226633Sdim Type *Ty) { 821193323Sed assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && 822193323Sed "This is not a truncating conversion!"); 823193323Sed assert(isSCEVable(Ty) && 824193323Sed "This is not a conversion to a SCEVable type!"); 825193323Sed Ty = getEffectiveSCEVType(Ty); 826193323Sed 827198090Srdivacky FoldingSetNodeID ID; 828198090Srdivacky ID.AddInteger(scTruncate); 829198090Srdivacky ID.AddPointer(Op); 830198090Srdivacky ID.AddPointer(Ty); 831198090Srdivacky void *IP = 0; 832198090Srdivacky if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 833198090Srdivacky 834195340Sed // Fold if the operand is constant. 835193323Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 836195098Sed return getConstant( 837239462Sdim cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty))); 838193323Sed 839193323Sed // trunc(trunc(x)) --> trunc(x) 840193323Sed if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) 841193323Sed return getTruncateExpr(ST->getOperand(), Ty); 842193323Sed 843193323Sed // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing 844193323Sed if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 845193323Sed return getTruncateOrSignExtend(SS->getOperand(), Ty); 846193323Sed 847193323Sed // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing 848193323Sed if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 849193323Sed return getTruncateOrZeroExtend(SZ->getOperand(), Ty); 850193323Sed 851218893Sdim // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can 852218893Sdim // eliminate all the truncates. 853218893Sdim if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) { 854218893Sdim SmallVector<const SCEV *, 4> Operands; 855218893Sdim bool hasTrunc = false; 856218893Sdim for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) { 857218893Sdim const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty); 858218893Sdim hasTrunc = isa<SCEVTruncateExpr>(S); 859218893Sdim Operands.push_back(S); 860218893Sdim } 861218893Sdim if (!hasTrunc) 862221345Sdim return getAddExpr(Operands); 863218893Sdim UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 864218893Sdim } 865218893Sdim 866218893Sdim // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can 867218893Sdim // eliminate all the truncates. 868218893Sdim if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) { 869218893Sdim SmallVector<const SCEV *, 4> Operands; 870218893Sdim bool hasTrunc = false; 871218893Sdim for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) { 872218893Sdim const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty); 873218893Sdim hasTrunc = isa<SCEVTruncateExpr>(S); 874218893Sdim Operands.push_back(S); 875218893Sdim } 876218893Sdim if (!hasTrunc) 877221345Sdim return getMulExpr(Operands); 878218893Sdim UniqueSCEVs.FindNodeOrInsertPos(ID, IP); // Mutates IP, returns NULL. 879218893Sdim } 880218893Sdim 881194612Sed // If the input value is a chrec scev, truncate the chrec's operands. 882193323Sed if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) { 883198090Srdivacky SmallVector<const SCEV *, 4> Operands; 884193323Sed for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 885193323Sed Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty)); 886221345Sdim return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap); 887193323Sed } 888193323Sed 889210299Sed // The cast wasn't folded; create an explicit cast node. We can reuse 890210299Sed // the existing insert position since if we get here, we won't have 891210299Sed // made any changes which would invalidate it. 892205407Srdivacky SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), 893205407Srdivacky Op, Ty); 894195340Sed UniqueSCEVs.InsertNode(S, IP); 895195340Sed return S; 896193323Sed} 897193323Sed 898198090Srdivackyconst SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op, 899226633Sdim Type *Ty) { 900193323Sed assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 901193323Sed "This is not an extending conversion!"); 902193323Sed assert(isSCEVable(Ty) && 903193323Sed "This is not a conversion to a SCEVable type!"); 904193323Sed Ty = getEffectiveSCEVType(Ty); 905193323Sed 906195340Sed // Fold if the operand is constant. 907210299Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 908210299Sed return getConstant( 909239462Sdim cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty))); 910193323Sed 911193323Sed // zext(zext(x)) --> zext(x) 912193323Sed if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 913193323Sed return getZeroExtendExpr(SZ->getOperand(), Ty); 914193323Sed 915198090Srdivacky // Before doing any expensive analysis, check to see if we've already 916198090Srdivacky // computed a SCEV for this Op and Ty. 917198090Srdivacky FoldingSetNodeID ID; 918198090Srdivacky ID.AddInteger(scZeroExtend); 919198090Srdivacky ID.AddPointer(Op); 920198090Srdivacky ID.AddPointer(Ty); 921198090Srdivacky void *IP = 0; 922198090Srdivacky if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 923198090Srdivacky 924218893Sdim // zext(trunc(x)) --> zext(x) or x or trunc(x) 925218893Sdim if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 926218893Sdim // It's possible the bits taken off by the truncate were all zero bits. If 927218893Sdim // so, we should be able to simplify this further. 928218893Sdim const SCEV *X = ST->getOperand(); 929218893Sdim ConstantRange CR = getUnsignedRange(X); 930218893Sdim unsigned TruncBits = getTypeSizeInBits(ST->getType()); 931218893Sdim unsigned NewBits = getTypeSizeInBits(Ty); 932218893Sdim if (CR.truncate(TruncBits).zeroExtend(NewBits).contains( 933218893Sdim CR.zextOrTrunc(NewBits))) 934218893Sdim return getTruncateOrZeroExtend(X, Ty); 935218893Sdim } 936218893Sdim 937193323Sed // If the input value is a chrec scev, and we can prove that the value 938193323Sed // did not overflow the old, smaller, value, we can zero extend all of the 939193323Sed // operands (often constants). This allows analysis of something like 940193323Sed // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; } 941193323Sed if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 942193323Sed if (AR->isAffine()) { 943198090Srdivacky const SCEV *Start = AR->getStart(); 944198090Srdivacky const SCEV *Step = AR->getStepRecurrence(*this); 945198090Srdivacky unsigned BitWidth = getTypeSizeInBits(AR->getType()); 946198090Srdivacky const Loop *L = AR->getLoop(); 947198090Srdivacky 948198090Srdivacky // If we have special knowledge that this addrec won't overflow, 949198090Srdivacky // we don't need to do any further analysis. 950221345Sdim if (AR->getNoWrapFlags(SCEV::FlagNUW)) 951198090Srdivacky return getAddRecExpr(getZeroExtendExpr(Start, Ty), 952198090Srdivacky getZeroExtendExpr(Step, Ty), 953221345Sdim L, AR->getNoWrapFlags()); 954198090Srdivacky 955193323Sed // Check whether the backedge-taken count is SCEVCouldNotCompute. 956193323Sed // Note that this serves two purposes: It filters out loops that are 957193323Sed // simply not analyzable, and it covers the case where this code is 958193323Sed // being called from within backedge-taken count analysis, such that 959193323Sed // attempting to ask for the backedge-taken count would likely result 960193323Sed // in infinite recursion. In the later case, the analysis code will 961193323Sed // cope with a conservative value, and it will take care to purge 962193323Sed // that value once it has finished. 963198090Srdivacky const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 964193323Sed if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 965193323Sed // Manually compute the final value for AR, checking for 966193323Sed // overflow. 967193323Sed 968193323Sed // Check whether the backedge-taken count can be losslessly casted to 969193323Sed // the addrec's type. The count is always unsigned. 970198090Srdivacky const SCEV *CastedMaxBECount = 971193323Sed getTruncateOrZeroExtend(MaxBECount, Start->getType()); 972198090Srdivacky const SCEV *RecastedMaxBECount = 973193323Sed getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 974193323Sed if (MaxBECount == RecastedMaxBECount) { 975226633Sdim Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 976193323Sed // Check whether Start+Step*MaxBECount has no unsigned overflow. 977204642Srdivacky const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step); 978239462Sdim const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy); 979239462Sdim const SCEV *WideStart = getZeroExtendExpr(Start, WideTy); 980239462Sdim const SCEV *WideMaxBECount = 981239462Sdim getZeroExtendExpr(CastedMaxBECount, WideTy); 982198090Srdivacky const SCEV *OperandExtendedAdd = 983239462Sdim getAddExpr(WideStart, 984239462Sdim getMulExpr(WideMaxBECount, 985193323Sed getZeroExtendExpr(Step, WideTy))); 986239462Sdim if (ZAdd == OperandExtendedAdd) { 987221345Sdim // Cache knowledge of AR NUW, which is propagated to this AddRec. 988221345Sdim const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 989193323Sed // Return the expression with the addrec on the outside. 990193323Sed return getAddRecExpr(getZeroExtendExpr(Start, Ty), 991193323Sed getZeroExtendExpr(Step, Ty), 992221345Sdim L, AR->getNoWrapFlags()); 993221345Sdim } 994193323Sed // Similar to above, only this time treat the step value as signed. 995193323Sed // This covers loops that count down. 996193323Sed OperandExtendedAdd = 997239462Sdim getAddExpr(WideStart, 998239462Sdim getMulExpr(WideMaxBECount, 999193323Sed getSignExtendExpr(Step, WideTy))); 1000239462Sdim if (ZAdd == OperandExtendedAdd) { 1001221345Sdim // Cache knowledge of AR NW, which is propagated to this AddRec. 1002221345Sdim // Negative step causes unsigned wrap, but it still can't self-wrap. 1003221345Sdim const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1004193323Sed // Return the expression with the addrec on the outside. 1005193323Sed return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1006193323Sed getSignExtendExpr(Step, Ty), 1007221345Sdim L, AR->getNoWrapFlags()); 1008221345Sdim } 1009193323Sed } 1010198090Srdivacky 1011198090Srdivacky // If the backedge is guarded by a comparison with the pre-inc value 1012198090Srdivacky // the addrec is safe. Also, if the entry is guarded by a comparison 1013198090Srdivacky // with the start value and the backedge is guarded by a comparison 1014198090Srdivacky // with the post-inc value, the addrec is safe. 1015198090Srdivacky if (isKnownPositive(Step)) { 1016198090Srdivacky const SCEV *N = getConstant(APInt::getMinValue(BitWidth) - 1017198090Srdivacky getUnsignedRange(Step).getUnsignedMax()); 1018198090Srdivacky if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) || 1019207618Srdivacky (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) && 1020198090Srdivacky isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, 1021221345Sdim AR->getPostIncExpr(*this), N))) { 1022221345Sdim // Cache knowledge of AR NUW, which is propagated to this AddRec. 1023221345Sdim const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW); 1024198090Srdivacky // Return the expression with the addrec on the outside. 1025198090Srdivacky return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1026198090Srdivacky getZeroExtendExpr(Step, Ty), 1027221345Sdim L, AR->getNoWrapFlags()); 1028221345Sdim } 1029198090Srdivacky } else if (isKnownNegative(Step)) { 1030198090Srdivacky const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) - 1031198090Srdivacky getSignedRange(Step).getSignedMin()); 1032207618Srdivacky if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) || 1033207618Srdivacky (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) && 1034198090Srdivacky isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, 1035221345Sdim AR->getPostIncExpr(*this), N))) { 1036221345Sdim // Cache knowledge of AR NW, which is propagated to this AddRec. 1037221345Sdim // Negative step causes unsigned wrap, but it still can't self-wrap. 1038221345Sdim const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW); 1039198090Srdivacky // Return the expression with the addrec on the outside. 1040198090Srdivacky return getAddRecExpr(getZeroExtendExpr(Start, Ty), 1041198090Srdivacky getSignExtendExpr(Step, Ty), 1042221345Sdim L, AR->getNoWrapFlags()); 1043221345Sdim } 1044198090Srdivacky } 1045193323Sed } 1046193323Sed } 1047193323Sed 1048198090Srdivacky // The cast wasn't folded; create an explicit cast node. 1049198090Srdivacky // Recompute the insert position, as it may have been invalidated. 1050195340Sed if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1051205407Srdivacky SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), 1052205407Srdivacky Op, Ty); 1053195340Sed UniqueSCEVs.InsertNode(S, IP); 1054195340Sed return S; 1055193323Sed} 1056193323Sed 1057223017Sdim// Get the limit of a recurrence such that incrementing by Step cannot cause 1058223017Sdim// signed overflow as long as the value of the recurrence within the loop does 1059223017Sdim// not exceed this limit before incrementing. 1060223017Sdimstatic const SCEV *getOverflowLimitForStep(const SCEV *Step, 1061223017Sdim ICmpInst::Predicate *Pred, 1062223017Sdim ScalarEvolution *SE) { 1063223017Sdim unsigned BitWidth = SE->getTypeSizeInBits(Step->getType()); 1064223017Sdim if (SE->isKnownPositive(Step)) { 1065223017Sdim *Pred = ICmpInst::ICMP_SLT; 1066223017Sdim return SE->getConstant(APInt::getSignedMinValue(BitWidth) - 1067223017Sdim SE->getSignedRange(Step).getSignedMax()); 1068223017Sdim } 1069223017Sdim if (SE->isKnownNegative(Step)) { 1070223017Sdim *Pred = ICmpInst::ICMP_SGT; 1071223017Sdim return SE->getConstant(APInt::getSignedMaxValue(BitWidth) - 1072223017Sdim SE->getSignedRange(Step).getSignedMin()); 1073223017Sdim } 1074223017Sdim return 0; 1075223017Sdim} 1076223017Sdim 1077223017Sdim// The recurrence AR has been shown to have no signed wrap. Typically, if we can 1078223017Sdim// prove NSW for AR, then we can just as easily prove NSW for its preincrement 1079223017Sdim// or postincrement sibling. This allows normalizing a sign extended AddRec as 1080223017Sdim// such: {sext(Step + Start),+,Step} => {(Step + sext(Start),+,Step} As a 1081223017Sdim// result, the expression "Step + sext(PreIncAR)" is congruent with 1082223017Sdim// "sext(PostIncAR)" 1083223017Sdimstatic const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR, 1084226633Sdim Type *Ty, 1085223017Sdim ScalarEvolution *SE) { 1086223017Sdim const Loop *L = AR->getLoop(); 1087223017Sdim const SCEV *Start = AR->getStart(); 1088223017Sdim const SCEV *Step = AR->getStepRecurrence(*SE); 1089223017Sdim 1090223017Sdim // Check for a simple looking step prior to loop entry. 1091223017Sdim const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start); 1092226633Sdim if (!SA) 1093223017Sdim return 0; 1094223017Sdim 1095226633Sdim // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV 1096226633Sdim // subtraction is expensive. For this purpose, perform a quick and dirty 1097226633Sdim // difference, by checking for Step in the operand list. 1098226633Sdim SmallVector<const SCEV *, 4> DiffOps; 1099226633Sdim for (SCEVAddExpr::op_iterator I = SA->op_begin(), E = SA->op_end(); 1100226633Sdim I != E; ++I) { 1101226633Sdim if (*I != Step) 1102226633Sdim DiffOps.push_back(*I); 1103226633Sdim } 1104226633Sdim if (DiffOps.size() == SA->getNumOperands()) 1105226633Sdim return 0; 1106226633Sdim 1107223017Sdim // This is a postinc AR. Check for overflow on the preinc recurrence using the 1108223017Sdim // same three conditions that getSignExtendedExpr checks. 1109223017Sdim 1110223017Sdim // 1. NSW flags on the step increment. 1111226633Sdim const SCEV *PreStart = SE->getAddExpr(DiffOps, SA->getNoWrapFlags()); 1112223017Sdim const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>( 1113223017Sdim SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap)); 1114223017Sdim 1115223017Sdim if (PreAR && PreAR->getNoWrapFlags(SCEV::FlagNSW)) 1116223017Sdim return PreStart; 1117223017Sdim 1118223017Sdim // 2. Direct overflow check on the step operation's expression. 1119223017Sdim unsigned BitWidth = SE->getTypeSizeInBits(AR->getType()); 1120226633Sdim Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2); 1121223017Sdim const SCEV *OperandExtendedStart = 1122223017Sdim SE->getAddExpr(SE->getSignExtendExpr(PreStart, WideTy), 1123223017Sdim SE->getSignExtendExpr(Step, WideTy)); 1124223017Sdim if (SE->getSignExtendExpr(Start, WideTy) == OperandExtendedStart) { 1125223017Sdim // Cache knowledge of PreAR NSW. 1126223017Sdim if (PreAR) 1127223017Sdim const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(SCEV::FlagNSW); 1128223017Sdim // FIXME: this optimization needs a unit test 1129223017Sdim DEBUG(dbgs() << "SCEV: untested prestart overflow check\n"); 1130223017Sdim return PreStart; 1131223017Sdim } 1132223017Sdim 1133223017Sdim // 3. Loop precondition. 1134223017Sdim ICmpInst::Predicate Pred; 1135223017Sdim const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, SE); 1136223017Sdim 1137223017Sdim if (OverflowLimit && 1138223017Sdim SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) { 1139223017Sdim return PreStart; 1140223017Sdim } 1141223017Sdim return 0; 1142223017Sdim} 1143223017Sdim 1144223017Sdim// Get the normalized sign-extended expression for this AddRec's Start. 1145223017Sdimstatic const SCEV *getSignExtendAddRecStart(const SCEVAddRecExpr *AR, 1146226633Sdim Type *Ty, 1147223017Sdim ScalarEvolution *SE) { 1148223017Sdim const SCEV *PreStart = getPreStartForSignExtend(AR, Ty, SE); 1149223017Sdim if (!PreStart) 1150223017Sdim return SE->getSignExtendExpr(AR->getStart(), Ty); 1151223017Sdim 1152223017Sdim return SE->getAddExpr(SE->getSignExtendExpr(AR->getStepRecurrence(*SE), Ty), 1153223017Sdim SE->getSignExtendExpr(PreStart, Ty)); 1154223017Sdim} 1155223017Sdim 1156198090Srdivackyconst SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op, 1157226633Sdim Type *Ty) { 1158193323Sed assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1159193323Sed "This is not an extending conversion!"); 1160193323Sed assert(isSCEVable(Ty) && 1161193323Sed "This is not a conversion to a SCEVable type!"); 1162193323Sed Ty = getEffectiveSCEVType(Ty); 1163193323Sed 1164195340Sed // Fold if the operand is constant. 1165210299Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1166210299Sed return getConstant( 1167239462Sdim cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty))); 1168193323Sed 1169193323Sed // sext(sext(x)) --> sext(x) 1170193323Sed if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op)) 1171193323Sed return getSignExtendExpr(SS->getOperand(), Ty); 1172193323Sed 1173218893Sdim // sext(zext(x)) --> zext(x) 1174218893Sdim if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op)) 1175218893Sdim return getZeroExtendExpr(SZ->getOperand(), Ty); 1176218893Sdim 1177198090Srdivacky // Before doing any expensive analysis, check to see if we've already 1178198090Srdivacky // computed a SCEV for this Op and Ty. 1179198090Srdivacky FoldingSetNodeID ID; 1180198090Srdivacky ID.AddInteger(scSignExtend); 1181198090Srdivacky ID.AddPointer(Op); 1182198090Srdivacky ID.AddPointer(Ty); 1183198090Srdivacky void *IP = 0; 1184198090Srdivacky if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1185198090Srdivacky 1186218893Sdim // If the input value is provably positive, build a zext instead. 1187218893Sdim if (isKnownNonNegative(Op)) 1188218893Sdim return getZeroExtendExpr(Op, Ty); 1189218893Sdim 1190218893Sdim // sext(trunc(x)) --> sext(x) or x or trunc(x) 1191218893Sdim if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) { 1192218893Sdim // It's possible the bits taken off by the truncate were all sign bits. If 1193218893Sdim // so, we should be able to simplify this further. 1194218893Sdim const SCEV *X = ST->getOperand(); 1195218893Sdim ConstantRange CR = getSignedRange(X); 1196218893Sdim unsigned TruncBits = getTypeSizeInBits(ST->getType()); 1197218893Sdim unsigned NewBits = getTypeSizeInBits(Ty); 1198218893Sdim if (CR.truncate(TruncBits).signExtend(NewBits).contains( 1199218893Sdim CR.sextOrTrunc(NewBits))) 1200218893Sdim return getTruncateOrSignExtend(X, Ty); 1201218893Sdim } 1202218893Sdim 1203193323Sed // If the input value is a chrec scev, and we can prove that the value 1204193323Sed // did not overflow the old, smaller, value, we can sign extend all of the 1205193323Sed // operands (often constants). This allows analysis of something like 1206193323Sed // this: for (signed char X = 0; X < 100; ++X) { int Y = X; } 1207193323Sed if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) 1208193323Sed if (AR->isAffine()) { 1209198090Srdivacky const SCEV *Start = AR->getStart(); 1210198090Srdivacky const SCEV *Step = AR->getStepRecurrence(*this); 1211198090Srdivacky unsigned BitWidth = getTypeSizeInBits(AR->getType()); 1212198090Srdivacky const Loop *L = AR->getLoop(); 1213198090Srdivacky 1214198090Srdivacky // If we have special knowledge that this addrec won't overflow, 1215198090Srdivacky // we don't need to do any further analysis. 1216221345Sdim if (AR->getNoWrapFlags(SCEV::FlagNSW)) 1217223017Sdim return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1218198090Srdivacky getSignExtendExpr(Step, Ty), 1219221345Sdim L, SCEV::FlagNSW); 1220198090Srdivacky 1221193323Sed // Check whether the backedge-taken count is SCEVCouldNotCompute. 1222193323Sed // Note that this serves two purposes: It filters out loops that are 1223193323Sed // simply not analyzable, and it covers the case where this code is 1224193323Sed // being called from within backedge-taken count analysis, such that 1225193323Sed // attempting to ask for the backedge-taken count would likely result 1226193323Sed // in infinite recursion. In the later case, the analysis code will 1227193323Sed // cope with a conservative value, and it will take care to purge 1228193323Sed // that value once it has finished. 1229198090Srdivacky const SCEV *MaxBECount = getMaxBackedgeTakenCount(L); 1230193323Sed if (!isa<SCEVCouldNotCompute>(MaxBECount)) { 1231193323Sed // Manually compute the final value for AR, checking for 1232193323Sed // overflow. 1233193323Sed 1234193323Sed // Check whether the backedge-taken count can be losslessly casted to 1235193323Sed // the addrec's type. The count is always unsigned. 1236198090Srdivacky const SCEV *CastedMaxBECount = 1237193323Sed getTruncateOrZeroExtend(MaxBECount, Start->getType()); 1238198090Srdivacky const SCEV *RecastedMaxBECount = 1239193323Sed getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); 1240193323Sed if (MaxBECount == RecastedMaxBECount) { 1241226633Sdim Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); 1242193323Sed // Check whether Start+Step*MaxBECount has no signed overflow. 1243204642Srdivacky const SCEV *SMul = getMulExpr(CastedMaxBECount, Step); 1244239462Sdim const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy); 1245239462Sdim const SCEV *WideStart = getSignExtendExpr(Start, WideTy); 1246239462Sdim const SCEV *WideMaxBECount = 1247239462Sdim getZeroExtendExpr(CastedMaxBECount, WideTy); 1248198090Srdivacky const SCEV *OperandExtendedAdd = 1249239462Sdim getAddExpr(WideStart, 1250239462Sdim getMulExpr(WideMaxBECount, 1251193323Sed getSignExtendExpr(Step, WideTy))); 1252239462Sdim if (SAdd == OperandExtendedAdd) { 1253221345Sdim // Cache knowledge of AR NSW, which is propagated to this AddRec. 1254221345Sdim const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1255193323Sed // Return the expression with the addrec on the outside. 1256223017Sdim return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1257193323Sed getSignExtendExpr(Step, Ty), 1258221345Sdim L, AR->getNoWrapFlags()); 1259221345Sdim } 1260198090Srdivacky // Similar to above, only this time treat the step value as unsigned. 1261198090Srdivacky // This covers loops that count up with an unsigned step. 1262198090Srdivacky OperandExtendedAdd = 1263239462Sdim getAddExpr(WideStart, 1264239462Sdim getMulExpr(WideMaxBECount, 1265198090Srdivacky getZeroExtendExpr(Step, WideTy))); 1266239462Sdim if (SAdd == OperandExtendedAdd) { 1267221345Sdim // Cache knowledge of AR NSW, which is propagated to this AddRec. 1268221345Sdim const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1269198090Srdivacky // Return the expression with the addrec on the outside. 1270223017Sdim return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1271198090Srdivacky getZeroExtendExpr(Step, Ty), 1272221345Sdim L, AR->getNoWrapFlags()); 1273221345Sdim } 1274193323Sed } 1275198090Srdivacky 1276198090Srdivacky // If the backedge is guarded by a comparison with the pre-inc value 1277198090Srdivacky // the addrec is safe. Also, if the entry is guarded by a comparison 1278198090Srdivacky // with the start value and the backedge is guarded by a comparison 1279198090Srdivacky // with the post-inc value, the addrec is safe. 1280223017Sdim ICmpInst::Predicate Pred; 1281223017Sdim const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, this); 1282223017Sdim if (OverflowLimit && 1283223017Sdim (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) || 1284223017Sdim (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) && 1285223017Sdim isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this), 1286223017Sdim OverflowLimit)))) { 1287223017Sdim // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec. 1288223017Sdim const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW); 1289223017Sdim return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this), 1290223017Sdim getSignExtendExpr(Step, Ty), 1291223017Sdim L, AR->getNoWrapFlags()); 1292198090Srdivacky } 1293193323Sed } 1294193323Sed } 1295193323Sed 1296198090Srdivacky // The cast wasn't folded; create an explicit cast node. 1297198090Srdivacky // Recompute the insert position, as it may have been invalidated. 1298195340Sed if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 1299205407Srdivacky SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), 1300205407Srdivacky Op, Ty); 1301195340Sed UniqueSCEVs.InsertNode(S, IP); 1302195340Sed return S; 1303193323Sed} 1304193323Sed 1305194178Sed/// getAnyExtendExpr - Return a SCEV for the given operand extended with 1306194178Sed/// unspecified bits out to the given type. 1307194178Sed/// 1308198090Srdivackyconst SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op, 1309226633Sdim Type *Ty) { 1310194178Sed assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) && 1311194178Sed "This is not an extending conversion!"); 1312194178Sed assert(isSCEVable(Ty) && 1313194178Sed "This is not a conversion to a SCEVable type!"); 1314194178Sed Ty = getEffectiveSCEVType(Ty); 1315194178Sed 1316194178Sed // Sign-extend negative constants. 1317194178Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op)) 1318194178Sed if (SC->getValue()->getValue().isNegative()) 1319194178Sed return getSignExtendExpr(Op, Ty); 1320194178Sed 1321194178Sed // Peel off a truncate cast. 1322194178Sed if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) { 1323198090Srdivacky const SCEV *NewOp = T->getOperand(); 1324194178Sed if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty)) 1325194178Sed return getAnyExtendExpr(NewOp, Ty); 1326194178Sed return getTruncateOrNoop(NewOp, Ty); 1327194178Sed } 1328194178Sed 1329194178Sed // Next try a zext cast. If the cast is folded, use it. 1330198090Srdivacky const SCEV *ZExt = getZeroExtendExpr(Op, Ty); 1331194178Sed if (!isa<SCEVZeroExtendExpr>(ZExt)) 1332194178Sed return ZExt; 1333194178Sed 1334194178Sed // Next try a sext cast. If the cast is folded, use it. 1335198090Srdivacky const SCEV *SExt = getSignExtendExpr(Op, Ty); 1336194178Sed if (!isa<SCEVSignExtendExpr>(SExt)) 1337194178Sed return SExt; 1338194178Sed 1339202878Srdivacky // Force the cast to be folded into the operands of an addrec. 1340202878Srdivacky if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) { 1341202878Srdivacky SmallVector<const SCEV *, 4> Ops; 1342202878Srdivacky for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end(); 1343202878Srdivacky I != E; ++I) 1344202878Srdivacky Ops.push_back(getAnyExtendExpr(*I, Ty)); 1345221345Sdim return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW); 1346202878Srdivacky } 1347202878Srdivacky 1348194178Sed // If the expression is obviously signed, use the sext cast value. 1349194178Sed if (isa<SCEVSMaxExpr>(Op)) 1350194178Sed return SExt; 1351194178Sed 1352194178Sed // Absent any other information, use the zext cast value. 1353194178Sed return ZExt; 1354194178Sed} 1355194178Sed 1356194612Sed/// CollectAddOperandsWithScales - Process the given Ops list, which is 1357194612Sed/// a list of operands to be added under the given scale, update the given 1358194612Sed/// map. This is a helper function for getAddRecExpr. As an example of 1359194612Sed/// what it does, given a sequence of operands that would form an add 1360194612Sed/// expression like this: 1361194612Sed/// 1362194612Sed/// m + n + 13 + (A * (o + p + (B * q + m + 29))) + r + (-1 * r) 1363194612Sed/// 1364194612Sed/// where A and B are constants, update the map with these values: 1365194612Sed/// 1366194612Sed/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0) 1367194612Sed/// 1368194612Sed/// and add 13 + A*B*29 to AccumulatedConstant. 1369194612Sed/// This will allow getAddRecExpr to produce this: 1370194612Sed/// 1371194612Sed/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B) 1372194612Sed/// 1373194612Sed/// This form often exposes folding opportunities that are hidden in 1374194612Sed/// the original operand list. 1375194612Sed/// 1376194612Sed/// Return true iff it appears that any interesting folding opportunities 1377194612Sed/// may be exposed. This helps getAddRecExpr short-circuit extra work in 1378194612Sed/// the common case where no interesting opportunities are present, and 1379194612Sed/// is also used as a check to avoid infinite recursion. 1380194612Sed/// 1381194612Sedstatic bool 1382198090SrdivackyCollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M, 1383198090Srdivacky SmallVector<const SCEV *, 8> &NewOps, 1384194612Sed APInt &AccumulatedConstant, 1385205407Srdivacky const SCEV *const *Ops, size_t NumOperands, 1386194612Sed const APInt &Scale, 1387194612Sed ScalarEvolution &SE) { 1388194612Sed bool Interesting = false; 1389194612Sed 1390210299Sed // Iterate over the add operands. They are sorted, with constants first. 1391210299Sed unsigned i = 0; 1392210299Sed while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1393210299Sed ++i; 1394210299Sed // Pull a buried constant out to the outside. 1395210299Sed if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero()) 1396210299Sed Interesting = true; 1397210299Sed AccumulatedConstant += Scale * C->getValue()->getValue(); 1398210299Sed } 1399210299Sed 1400210299Sed // Next comes everything else. We're especially interested in multiplies 1401210299Sed // here, but they're in the middle, so just visit the rest with one loop. 1402210299Sed for (; i != NumOperands; ++i) { 1403194612Sed const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]); 1404194612Sed if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) { 1405194612Sed APInt NewScale = 1406194612Sed Scale * cast<SCEVConstant>(Mul->getOperand(0))->getValue()->getValue(); 1407194612Sed if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) { 1408194612Sed // A multiplication of a constant with another add; recurse. 1409205407Srdivacky const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1)); 1410194612Sed Interesting |= 1411194612Sed CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1412205407Srdivacky Add->op_begin(), Add->getNumOperands(), 1413194612Sed NewScale, SE); 1414194612Sed } else { 1415194612Sed // A multiplication of a constant with some other value. Update 1416194612Sed // the map. 1417198090Srdivacky SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end()); 1418198090Srdivacky const SCEV *Key = SE.getMulExpr(MulOps); 1419198090Srdivacky std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1420195340Sed M.insert(std::make_pair(Key, NewScale)); 1421194612Sed if (Pair.second) { 1422194612Sed NewOps.push_back(Pair.first->first); 1423194612Sed } else { 1424194612Sed Pair.first->second += NewScale; 1425194612Sed // The map already had an entry for this value, which may indicate 1426194612Sed // a folding opportunity. 1427194612Sed Interesting = true; 1428194612Sed } 1429194612Sed } 1430194612Sed } else { 1431194612Sed // An ordinary operand. Update the map. 1432198090Srdivacky std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair = 1433195340Sed M.insert(std::make_pair(Ops[i], Scale)); 1434194612Sed if (Pair.second) { 1435194612Sed NewOps.push_back(Pair.first->first); 1436194612Sed } else { 1437194612Sed Pair.first->second += Scale; 1438194612Sed // The map already had an entry for this value, which may indicate 1439194612Sed // a folding opportunity. 1440194612Sed Interesting = true; 1441194612Sed } 1442194612Sed } 1443194612Sed } 1444194612Sed 1445194612Sed return Interesting; 1446194612Sed} 1447194612Sed 1448194612Sednamespace { 1449194612Sed struct APIntCompare { 1450194612Sed bool operator()(const APInt &LHS, const APInt &RHS) const { 1451194612Sed return LHS.ult(RHS); 1452194612Sed } 1453194612Sed }; 1454194612Sed} 1455194612Sed 1456193323Sed/// getAddExpr - Get a canonical add expression, or something simpler if 1457193323Sed/// possible. 1458198090Srdivackyconst SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops, 1459221345Sdim SCEV::NoWrapFlags Flags) { 1460221345Sdim assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) && 1461221345Sdim "only nuw or nsw allowed"); 1462193323Sed assert(!Ops.empty() && "Cannot get empty add!"); 1463193323Sed if (Ops.size() == 1) return Ops[0]; 1464193323Sed#ifndef NDEBUG 1465226633Sdim Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 1466193323Sed for (unsigned i = 1, e = Ops.size(); i != e; ++i) 1467210299Sed assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 1468193323Sed "SCEVAddExpr operand types don't match!"); 1469193323Sed#endif 1470193323Sed 1471221345Sdim // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 1472221345Sdim // And vice-versa. 1473221345Sdim int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 1474221345Sdim SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 1475221345Sdim if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 1476202878Srdivacky bool All = true; 1477212904Sdim for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(), 1478212904Sdim E = Ops.end(); I != E; ++I) 1479212904Sdim if (!isKnownNonNegative(*I)) { 1480202878Srdivacky All = false; 1481202878Srdivacky break; 1482202878Srdivacky } 1483221345Sdim if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 1484202878Srdivacky } 1485202878Srdivacky 1486193323Sed // Sort by complexity, this groups all similar expression types together. 1487193323Sed GroupByComplexity(Ops, LI); 1488193323Sed 1489193323Sed // If there are any constants, fold them together. 1490193323Sed unsigned Idx = 0; 1491193323Sed if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 1492193323Sed ++Idx; 1493193323Sed assert(Idx < Ops.size()); 1494193323Sed while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 1495193323Sed // We found two constants, fold them together! 1496194612Sed Ops[0] = getConstant(LHSC->getValue()->getValue() + 1497194612Sed RHSC->getValue()->getValue()); 1498194612Sed if (Ops.size() == 2) return Ops[0]; 1499193323Sed Ops.erase(Ops.begin()+1); // Erase the folded element 1500193323Sed LHSC = cast<SCEVConstant>(Ops[0]); 1501193323Sed } 1502193323Sed 1503193323Sed // If we are left with a constant zero being added, strip it off. 1504207618Srdivacky if (LHSC->getValue()->isZero()) { 1505193323Sed Ops.erase(Ops.begin()); 1506193323Sed --Idx; 1507193323Sed } 1508207618Srdivacky 1509207618Srdivacky if (Ops.size() == 1) return Ops[0]; 1510193323Sed } 1511193323Sed 1512212904Sdim // Okay, check to see if the same value occurs in the operand list more than 1513212904Sdim // once. If so, merge them together into an multiply expression. Since we 1514212904Sdim // sorted the list, these values are required to be adjacent. 1515226633Sdim Type *Ty = Ops[0]->getType(); 1516212904Sdim bool FoundMatch = false; 1517212904Sdim for (unsigned i = 0, e = Ops.size(); i != e-1; ++i) 1518193323Sed if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2 1519212904Sdim // Scan ahead to count how many equal operands there are. 1520212904Sdim unsigned Count = 2; 1521212904Sdim while (i+Count != e && Ops[i+Count] == Ops[i]) 1522212904Sdim ++Count; 1523212904Sdim // Merge the values into a multiply. 1524212904Sdim const SCEV *Scale = getConstant(Ty, Count); 1525212904Sdim const SCEV *Mul = getMulExpr(Scale, Ops[i]); 1526212904Sdim if (Ops.size() == Count) 1527193323Sed return Mul; 1528212904Sdim Ops[i] = Mul; 1529212904Sdim Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count); 1530212904Sdim --i; e -= Count - 1; 1531212904Sdim FoundMatch = true; 1532193323Sed } 1533212904Sdim if (FoundMatch) 1534221345Sdim return getAddExpr(Ops, Flags); 1535193323Sed 1536193323Sed // Check for truncates. If all the operands are truncated from the same 1537193323Sed // type, see if factoring out the truncate would permit the result to be 1538193323Sed // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n) 1539193323Sed // if the contents of the resulting outer trunc fold to something simple. 1540193323Sed for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) { 1541193323Sed const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]); 1542226633Sdim Type *DstType = Trunc->getType(); 1543226633Sdim Type *SrcType = Trunc->getOperand()->getType(); 1544198090Srdivacky SmallVector<const SCEV *, 8> LargeOps; 1545193323Sed bool Ok = true; 1546193323Sed // Check all the operands to see if they can be represented in the 1547193323Sed // source type of the truncate. 1548193323Sed for (unsigned i = 0, e = Ops.size(); i != e; ++i) { 1549193323Sed if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) { 1550193323Sed if (T->getOperand()->getType() != SrcType) { 1551193323Sed Ok = false; 1552193323Sed break; 1553193323Sed } 1554193323Sed LargeOps.push_back(T->getOperand()); 1555193323Sed } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) { 1556207618Srdivacky LargeOps.push_back(getAnyExtendExpr(C, SrcType)); 1557193323Sed } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) { 1558198090Srdivacky SmallVector<const SCEV *, 8> LargeMulOps; 1559193323Sed for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) { 1560193323Sed if (const SCEVTruncateExpr *T = 1561193323Sed dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) { 1562193323Sed if (T->getOperand()->getType() != SrcType) { 1563193323Sed Ok = false; 1564193323Sed break; 1565193323Sed } 1566193323Sed LargeMulOps.push_back(T->getOperand()); 1567193323Sed } else if (const SCEVConstant *C = 1568193323Sed dyn_cast<SCEVConstant>(M->getOperand(j))) { 1569207618Srdivacky LargeMulOps.push_back(getAnyExtendExpr(C, SrcType)); 1570193323Sed } else { 1571193323Sed Ok = false; 1572193323Sed break; 1573193323Sed } 1574193323Sed } 1575193323Sed if (Ok) 1576193323Sed LargeOps.push_back(getMulExpr(LargeMulOps)); 1577193323Sed } else { 1578193323Sed Ok = false; 1579193323Sed break; 1580193323Sed } 1581193323Sed } 1582193323Sed if (Ok) { 1583193323Sed // Evaluate the expression in the larger type. 1584221345Sdim const SCEV *Fold = getAddExpr(LargeOps, Flags); 1585193323Sed // If it folds to something simple, use it. Otherwise, don't. 1586193323Sed if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold)) 1587193323Sed return getTruncateExpr(Fold, DstType); 1588193323Sed } 1589193323Sed } 1590193323Sed 1591193323Sed // Skip past any other cast SCEVs. 1592193323Sed while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr) 1593193323Sed ++Idx; 1594193323Sed 1595193323Sed // If there are add operands they would be next. 1596193323Sed if (Idx < Ops.size()) { 1597193323Sed bool DeletedAdd = false; 1598193323Sed while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) { 1599193323Sed // If we have an add, expand the add operands onto the end of the operands 1600193323Sed // list. 1601193323Sed Ops.erase(Ops.begin()+Idx); 1602210299Sed Ops.append(Add->op_begin(), Add->op_end()); 1603193323Sed DeletedAdd = true; 1604193323Sed } 1605193323Sed 1606193323Sed // If we deleted at least one add, we added operands to the end of the list, 1607193323Sed // and they are not necessarily sorted. Recurse to resort and resimplify 1608204642Srdivacky // any operands we just acquired. 1609193323Sed if (DeletedAdd) 1610193323Sed return getAddExpr(Ops); 1611193323Sed } 1612193323Sed 1613193323Sed // Skip over the add expression until we get to a multiply. 1614193323Sed while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 1615193323Sed ++Idx; 1616193323Sed 1617194612Sed // Check to see if there are any folding opportunities present with 1618194612Sed // operands multiplied by constant values. 1619194612Sed if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) { 1620194612Sed uint64_t BitWidth = getTypeSizeInBits(Ty); 1621198090Srdivacky DenseMap<const SCEV *, APInt> M; 1622198090Srdivacky SmallVector<const SCEV *, 8> NewOps; 1623194612Sed APInt AccumulatedConstant(BitWidth, 0); 1624194612Sed if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant, 1625205407Srdivacky Ops.data(), Ops.size(), 1626205407Srdivacky APInt(BitWidth, 1), *this)) { 1627194612Sed // Some interesting folding opportunity is present, so its worthwhile to 1628194612Sed // re-generate the operands list. Group the operands by constant scale, 1629194612Sed // to avoid multiplying by the same constant scale multiple times. 1630198090Srdivacky std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists; 1631212904Sdim for (SmallVector<const SCEV *, 8>::const_iterator I = NewOps.begin(), 1632194612Sed E = NewOps.end(); I != E; ++I) 1633194612Sed MulOpLists[M.find(*I)->second].push_back(*I); 1634194612Sed // Re-generate the operands list. 1635194612Sed Ops.clear(); 1636194612Sed if (AccumulatedConstant != 0) 1637194612Sed Ops.push_back(getConstant(AccumulatedConstant)); 1638195098Sed for (std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare>::iterator 1639195098Sed I = MulOpLists.begin(), E = MulOpLists.end(); I != E; ++I) 1640194612Sed if (I->first != 0) 1641195098Sed Ops.push_back(getMulExpr(getConstant(I->first), 1642195098Sed getAddExpr(I->second))); 1643194612Sed if (Ops.empty()) 1644207618Srdivacky return getConstant(Ty, 0); 1645194612Sed if (Ops.size() == 1) 1646194612Sed return Ops[0]; 1647194612Sed return getAddExpr(Ops); 1648194612Sed } 1649194612Sed } 1650194612Sed 1651193323Sed // If we are adding something to a multiply expression, make sure the 1652193323Sed // something is not already an operand of the multiply. If so, merge it into 1653193323Sed // the multiply. 1654193323Sed for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) { 1655193323Sed const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]); 1656193323Sed for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) { 1657193323Sed const SCEV *MulOpSCEV = Mul->getOperand(MulOp); 1658212904Sdim if (isa<SCEVConstant>(MulOpSCEV)) 1659212904Sdim continue; 1660193323Sed for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) 1661212904Sdim if (MulOpSCEV == Ops[AddOp]) { 1662193323Sed // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1)) 1663198090Srdivacky const SCEV *InnerMul = Mul->getOperand(MulOp == 0); 1664193323Sed if (Mul->getNumOperands() != 2) { 1665193323Sed // If the multiply has more than two operands, we must get the 1666193323Sed // Y*Z term. 1667212904Sdim SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 1668212904Sdim Mul->op_begin()+MulOp); 1669212904Sdim MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 1670193323Sed InnerMul = getMulExpr(MulOps); 1671193323Sed } 1672207618Srdivacky const SCEV *One = getConstant(Ty, 1); 1673212904Sdim const SCEV *AddOne = getAddExpr(One, InnerMul); 1674212904Sdim const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV); 1675193323Sed if (Ops.size() == 2) return OuterMul; 1676193323Sed if (AddOp < Idx) { 1677193323Sed Ops.erase(Ops.begin()+AddOp); 1678193323Sed Ops.erase(Ops.begin()+Idx-1); 1679193323Sed } else { 1680193323Sed Ops.erase(Ops.begin()+Idx); 1681193323Sed Ops.erase(Ops.begin()+AddOp-1); 1682193323Sed } 1683193323Sed Ops.push_back(OuterMul); 1684193323Sed return getAddExpr(Ops); 1685193323Sed } 1686193323Sed 1687193323Sed // Check this multiply against other multiplies being added together. 1688193323Sed for (unsigned OtherMulIdx = Idx+1; 1689193323Sed OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]); 1690193323Sed ++OtherMulIdx) { 1691193323Sed const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]); 1692193323Sed // If MulOp occurs in OtherMul, we can fold the two multiplies 1693193323Sed // together. 1694193323Sed for (unsigned OMulOp = 0, e = OtherMul->getNumOperands(); 1695193323Sed OMulOp != e; ++OMulOp) 1696193323Sed if (OtherMul->getOperand(OMulOp) == MulOpSCEV) { 1697193323Sed // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E)) 1698198090Srdivacky const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0); 1699193323Sed if (Mul->getNumOperands() != 2) { 1700195098Sed SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), 1701212904Sdim Mul->op_begin()+MulOp); 1702212904Sdim MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end()); 1703193323Sed InnerMul1 = getMulExpr(MulOps); 1704193323Sed } 1705198090Srdivacky const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0); 1706193323Sed if (OtherMul->getNumOperands() != 2) { 1707195098Sed SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(), 1708212904Sdim OtherMul->op_begin()+OMulOp); 1709212904Sdim MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end()); 1710193323Sed InnerMul2 = getMulExpr(MulOps); 1711193323Sed } 1712198090Srdivacky const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2); 1713198090Srdivacky const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum); 1714193323Sed if (Ops.size() == 2) return OuterMul; 1715193323Sed Ops.erase(Ops.begin()+Idx); 1716193323Sed Ops.erase(Ops.begin()+OtherMulIdx-1); 1717193323Sed Ops.push_back(OuterMul); 1718193323Sed return getAddExpr(Ops); 1719193323Sed } 1720193323Sed } 1721193323Sed } 1722193323Sed } 1723193323Sed 1724193323Sed // If there are any add recurrences in the operands list, see if any other 1725193323Sed // added values are loop invariant. If so, we can fold them into the 1726193323Sed // recurrence. 1727193323Sed while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 1728193323Sed ++Idx; 1729193323Sed 1730193323Sed // Scan over all recurrences, trying to fold loop invariants into them. 1731193323Sed for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 1732193323Sed // Scan all of the other operands to this add and add them to the vector if 1733193323Sed // they are loop invariant w.r.t. the recurrence. 1734198090Srdivacky SmallVector<const SCEV *, 8> LIOps; 1735193323Sed const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 1736207618Srdivacky const Loop *AddRecLoop = AddRec->getLoop(); 1737193323Sed for (unsigned i = 0, e = Ops.size(); i != e; ++i) 1738218893Sdim if (isLoopInvariant(Ops[i], AddRecLoop)) { 1739193323Sed LIOps.push_back(Ops[i]); 1740193323Sed Ops.erase(Ops.begin()+i); 1741193323Sed --i; --e; 1742193323Sed } 1743193323Sed 1744193323Sed // If we found some loop invariants, fold them into the recurrence. 1745193323Sed if (!LIOps.empty()) { 1746193323Sed // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step} 1747193323Sed LIOps.push_back(AddRec->getStart()); 1748193323Sed 1749198090Srdivacky SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 1750201360Srdivacky AddRec->op_end()); 1751193323Sed AddRecOps[0] = getAddExpr(LIOps); 1752193323Sed 1753210299Sed // Build the new addrec. Propagate the NUW and NSW flags if both the 1754210299Sed // outer add and the inner addrec are guaranteed to have no overflow. 1755221345Sdim // Always propagate NW. 1756221345Sdim Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW)); 1757221345Sdim const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags); 1758201360Srdivacky 1759193323Sed // If all of the other operands were loop invariant, we are done. 1760193323Sed if (Ops.size() == 1) return NewRec; 1761193323Sed 1762226633Sdim // Otherwise, add the folded AddRec by the non-invariant parts. 1763193323Sed for (unsigned i = 0;; ++i) 1764193323Sed if (Ops[i] == AddRec) { 1765193323Sed Ops[i] = NewRec; 1766193323Sed break; 1767193323Sed } 1768193323Sed return getAddExpr(Ops); 1769193323Sed } 1770193323Sed 1771193323Sed // Okay, if there weren't any loop invariants to be folded, check to see if 1772193323Sed // there are multiple AddRec's with the same loop induction variable being 1773193323Sed // added together. If so, we can fold them. 1774193323Sed for (unsigned OtherIdx = Idx+1; 1775212904Sdim OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 1776212904Sdim ++OtherIdx) 1777212904Sdim if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) { 1778212904Sdim // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L> 1779212904Sdim SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(), 1780212904Sdim AddRec->op_end()); 1781212904Sdim for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 1782212904Sdim ++OtherIdx) 1783212904Sdim if (const SCEVAddRecExpr *OtherAddRec = 1784212904Sdim dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx])) 1785212904Sdim if (OtherAddRec->getLoop() == AddRecLoop) { 1786212904Sdim for (unsigned i = 0, e = OtherAddRec->getNumOperands(); 1787212904Sdim i != e; ++i) { 1788212904Sdim if (i >= AddRecOps.size()) { 1789212904Sdim AddRecOps.append(OtherAddRec->op_begin()+i, 1790212904Sdim OtherAddRec->op_end()); 1791212904Sdim break; 1792212904Sdim } 1793212904Sdim AddRecOps[i] = getAddExpr(AddRecOps[i], 1794212904Sdim OtherAddRec->getOperand(i)); 1795212904Sdim } 1796212904Sdim Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 1797193323Sed } 1798221345Sdim // Step size has changed, so we cannot guarantee no self-wraparound. 1799221345Sdim Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap); 1800212904Sdim return getAddExpr(Ops); 1801193323Sed } 1802193323Sed 1803193323Sed // Otherwise couldn't fold anything into this recurrence. Move onto the 1804193323Sed // next one. 1805193323Sed } 1806193323Sed 1807193323Sed // Okay, it looks like we really DO need an add expr. Check to see if we 1808193323Sed // already have one, otherwise create a new one. 1809195340Sed FoldingSetNodeID ID; 1810195340Sed ID.AddInteger(scAddExpr); 1811195340Sed for (unsigned i = 0, e = Ops.size(); i != e; ++i) 1812195340Sed ID.AddPointer(Ops[i]); 1813195340Sed void *IP = 0; 1814202878Srdivacky SCEVAddExpr *S = 1815202878Srdivacky static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 1816202878Srdivacky if (!S) { 1817205407Srdivacky const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 1818205407Srdivacky std::uninitialized_copy(Ops.begin(), Ops.end(), O); 1819205407Srdivacky S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator), 1820205407Srdivacky O, Ops.size()); 1821202878Srdivacky UniqueSCEVs.InsertNode(S, IP); 1822202878Srdivacky } 1823221345Sdim S->setNoWrapFlags(Flags); 1824195340Sed return S; 1825193323Sed} 1826193323Sed 1827226633Sdimstatic uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) { 1828226633Sdim uint64_t k = i*j; 1829226633Sdim if (j > 1 && k / j != i) Overflow = true; 1830226633Sdim return k; 1831226633Sdim} 1832226633Sdim 1833226633Sdim/// Compute the result of "n choose k", the binomial coefficient. If an 1834226633Sdim/// intermediate computation overflows, Overflow will be set and the return will 1835239462Sdim/// be garbage. Overflow is not cleared on absence of overflow. 1836226633Sdimstatic uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) { 1837226633Sdim // We use the multiplicative formula: 1838226633Sdim // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 . 1839226633Sdim // At each iteration, we take the n-th term of the numeral and divide by the 1840226633Sdim // (k-n)th term of the denominator. This division will always produce an 1841226633Sdim // integral result, and helps reduce the chance of overflow in the 1842226633Sdim // intermediate computations. However, we can still overflow even when the 1843226633Sdim // final result would fit. 1844226633Sdim 1845226633Sdim if (n == 0 || n == k) return 1; 1846226633Sdim if (k > n) return 0; 1847226633Sdim 1848226633Sdim if (k > n/2) 1849226633Sdim k = n-k; 1850226633Sdim 1851226633Sdim uint64_t r = 1; 1852226633Sdim for (uint64_t i = 1; i <= k; ++i) { 1853226633Sdim r = umul_ov(r, n-(i-1), Overflow); 1854226633Sdim r /= i; 1855226633Sdim } 1856226633Sdim return r; 1857226633Sdim} 1858226633Sdim 1859193323Sed/// getMulExpr - Get a canonical multiply expression, or something simpler if 1860193323Sed/// possible. 1861198090Srdivackyconst SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops, 1862221345Sdim SCEV::NoWrapFlags Flags) { 1863221345Sdim assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) && 1864221345Sdim "only nuw or nsw allowed"); 1865193323Sed assert(!Ops.empty() && "Cannot get empty mul!"); 1866202878Srdivacky if (Ops.size() == 1) return Ops[0]; 1867193323Sed#ifndef NDEBUG 1868226633Sdim Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 1869193323Sed for (unsigned i = 1, e = Ops.size(); i != e; ++i) 1870212904Sdim assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 1871193323Sed "SCEVMulExpr operand types don't match!"); 1872193323Sed#endif 1873193323Sed 1874221345Sdim // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 1875221345Sdim // And vice-versa. 1876221345Sdim int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 1877221345Sdim SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 1878221345Sdim if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 1879202878Srdivacky bool All = true; 1880212904Sdim for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(), 1881212904Sdim E = Ops.end(); I != E; ++I) 1882212904Sdim if (!isKnownNonNegative(*I)) { 1883202878Srdivacky All = false; 1884202878Srdivacky break; 1885202878Srdivacky } 1886221345Sdim if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 1887202878Srdivacky } 1888202878Srdivacky 1889193323Sed // Sort by complexity, this groups all similar expression types together. 1890193323Sed GroupByComplexity(Ops, LI); 1891193323Sed 1892193323Sed // If there are any constants, fold them together. 1893193323Sed unsigned Idx = 0; 1894193323Sed if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 1895193323Sed 1896193323Sed // C1*(C2+V) -> C1*C2 + C1*V 1897193323Sed if (Ops.size() == 2) 1898193323Sed if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) 1899193323Sed if (Add->getNumOperands() == 2 && 1900193323Sed isa<SCEVConstant>(Add->getOperand(0))) 1901193323Sed return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)), 1902193323Sed getMulExpr(LHSC, Add->getOperand(1))); 1903193323Sed 1904193323Sed ++Idx; 1905193323Sed while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 1906193323Sed // We found two constants, fold them together! 1907198090Srdivacky ConstantInt *Fold = ConstantInt::get(getContext(), 1908198090Srdivacky LHSC->getValue()->getValue() * 1909193323Sed RHSC->getValue()->getValue()); 1910193323Sed Ops[0] = getConstant(Fold); 1911193323Sed Ops.erase(Ops.begin()+1); // Erase the folded element 1912193323Sed if (Ops.size() == 1) return Ops[0]; 1913193323Sed LHSC = cast<SCEVConstant>(Ops[0]); 1914193323Sed } 1915193323Sed 1916193323Sed // If we are left with a constant one being multiplied, strip it off. 1917193323Sed if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) { 1918193323Sed Ops.erase(Ops.begin()); 1919193323Sed --Idx; 1920193323Sed } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) { 1921193323Sed // If we have a multiply of zero, it will always be zero. 1922193323Sed return Ops[0]; 1923202878Srdivacky } else if (Ops[0]->isAllOnesValue()) { 1924202878Srdivacky // If we have a mul by -1 of an add, try distributing the -1 among the 1925202878Srdivacky // add operands. 1926221345Sdim if (Ops.size() == 2) { 1927202878Srdivacky if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) { 1928202878Srdivacky SmallVector<const SCEV *, 4> NewOps; 1929202878Srdivacky bool AnyFolded = false; 1930221345Sdim for (SCEVAddRecExpr::op_iterator I = Add->op_begin(), 1931221345Sdim E = Add->op_end(); I != E; ++I) { 1932202878Srdivacky const SCEV *Mul = getMulExpr(Ops[0], *I); 1933202878Srdivacky if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true; 1934202878Srdivacky NewOps.push_back(Mul); 1935202878Srdivacky } 1936202878Srdivacky if (AnyFolded) 1937202878Srdivacky return getAddExpr(NewOps); 1938202878Srdivacky } 1939221345Sdim else if (const SCEVAddRecExpr * 1940221345Sdim AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) { 1941221345Sdim // Negation preserves a recurrence's no self-wrap property. 1942221345Sdim SmallVector<const SCEV *, 4> Operands; 1943221345Sdim for (SCEVAddRecExpr::op_iterator I = AddRec->op_begin(), 1944221345Sdim E = AddRec->op_end(); I != E; ++I) { 1945221345Sdim Operands.push_back(getMulExpr(Ops[0], *I)); 1946221345Sdim } 1947221345Sdim return getAddRecExpr(Operands, AddRec->getLoop(), 1948221345Sdim AddRec->getNoWrapFlags(SCEV::FlagNW)); 1949221345Sdim } 1950221345Sdim } 1951193323Sed } 1952207618Srdivacky 1953207618Srdivacky if (Ops.size() == 1) 1954207618Srdivacky return Ops[0]; 1955193323Sed } 1956193323Sed 1957193323Sed // Skip over the add expression until we get to a multiply. 1958193323Sed while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr) 1959193323Sed ++Idx; 1960193323Sed 1961193323Sed // If there are mul operands inline them all into this expression. 1962193323Sed if (Idx < Ops.size()) { 1963193323Sed bool DeletedMul = false; 1964193323Sed while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) { 1965193323Sed // If we have an mul, expand the mul operands onto the end of the operands 1966193323Sed // list. 1967193323Sed Ops.erase(Ops.begin()+Idx); 1968210299Sed Ops.append(Mul->op_begin(), Mul->op_end()); 1969193323Sed DeletedMul = true; 1970193323Sed } 1971193323Sed 1972193323Sed // If we deleted at least one mul, we added operands to the end of the list, 1973193323Sed // and they are not necessarily sorted. Recurse to resort and resimplify 1974204642Srdivacky // any operands we just acquired. 1975193323Sed if (DeletedMul) 1976193323Sed return getMulExpr(Ops); 1977193323Sed } 1978193323Sed 1979193323Sed // If there are any add recurrences in the operands list, see if any other 1980193323Sed // added values are loop invariant. If so, we can fold them into the 1981193323Sed // recurrence. 1982193323Sed while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr) 1983193323Sed ++Idx; 1984193323Sed 1985193323Sed // Scan over all recurrences, trying to fold loop invariants into them. 1986193323Sed for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) { 1987193323Sed // Scan all of the other operands to this mul and add them to the vector if 1988193323Sed // they are loop invariant w.r.t. the recurrence. 1989198090Srdivacky SmallVector<const SCEV *, 8> LIOps; 1990193323Sed const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]); 1991212904Sdim const Loop *AddRecLoop = AddRec->getLoop(); 1992193323Sed for (unsigned i = 0, e = Ops.size(); i != e; ++i) 1993218893Sdim if (isLoopInvariant(Ops[i], AddRecLoop)) { 1994193323Sed LIOps.push_back(Ops[i]); 1995193323Sed Ops.erase(Ops.begin()+i); 1996193323Sed --i; --e; 1997193323Sed } 1998193323Sed 1999193323Sed // If we found some loop invariants, fold them into the recurrence. 2000193323Sed if (!LIOps.empty()) { 2001193323Sed // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step} 2002198090Srdivacky SmallVector<const SCEV *, 4> NewOps; 2003193323Sed NewOps.reserve(AddRec->getNumOperands()); 2004210299Sed const SCEV *Scale = getMulExpr(LIOps); 2005210299Sed for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) 2006210299Sed NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i))); 2007193323Sed 2008210299Sed // Build the new addrec. Propagate the NUW and NSW flags if both the 2009210299Sed // outer mul and the inner addrec are guaranteed to have no overflow. 2010221345Sdim // 2011221345Sdim // No self-wrap cannot be guaranteed after changing the step size, but 2012221345Sdim // will be inferred if either NUW or NSW is true. 2013221345Sdim Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW)); 2014221345Sdim const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags); 2015193323Sed 2016193323Sed // If all of the other operands were loop invariant, we are done. 2017193323Sed if (Ops.size() == 1) return NewRec; 2018193323Sed 2019226633Sdim // Otherwise, multiply the folded AddRec by the non-invariant parts. 2020193323Sed for (unsigned i = 0;; ++i) 2021193323Sed if (Ops[i] == AddRec) { 2022193323Sed Ops[i] = NewRec; 2023193323Sed break; 2024193323Sed } 2025193323Sed return getMulExpr(Ops); 2026193323Sed } 2027193323Sed 2028193323Sed // Okay, if there weren't any loop invariants to be folded, check to see if 2029193323Sed // there are multiple AddRec's with the same loop induction variable being 2030193323Sed // multiplied together. If so, we can fold them. 2031193323Sed for (unsigned OtherIdx = Idx+1; 2032212904Sdim OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2033226633Sdim ++OtherIdx) { 2034239462Sdim if (AddRecLoop != cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) 2035239462Sdim continue; 2036239462Sdim 2037239462Sdim // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L> 2038239462Sdim // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [ 2039239462Sdim // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z 2040239462Sdim // ]]],+,...up to x=2n}. 2041239462Sdim // Note that the arguments to choose() are always integers with values 2042239462Sdim // known at compile time, never SCEV objects. 2043239462Sdim // 2044239462Sdim // The implementation avoids pointless extra computations when the two 2045239462Sdim // addrec's are of different length (mathematically, it's equivalent to 2046239462Sdim // an infinite stream of zeros on the right). 2047239462Sdim bool OpsModified = false; 2048239462Sdim for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]); 2049239462Sdim ++OtherIdx) { 2050239462Sdim const SCEVAddRecExpr *OtherAddRec = 2051239462Sdim dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]); 2052239462Sdim if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop) 2053239462Sdim continue; 2054239462Sdim 2055239462Sdim bool Overflow = false; 2056239462Sdim Type *Ty = AddRec->getType(); 2057239462Sdim bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64; 2058239462Sdim SmallVector<const SCEV*, 7> AddRecOps; 2059239462Sdim for (int x = 0, xe = AddRec->getNumOperands() + 2060239462Sdim OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) { 2061239462Sdim const SCEV *Term = getConstant(Ty, 0); 2062239462Sdim for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) { 2063239462Sdim uint64_t Coeff1 = Choose(x, 2*x - y, Overflow); 2064239462Sdim for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1), 2065239462Sdim ze = std::min(x+1, (int)OtherAddRec->getNumOperands()); 2066239462Sdim z < ze && !Overflow; ++z) { 2067239462Sdim uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow); 2068239462Sdim uint64_t Coeff; 2069239462Sdim if (LargerThan64Bits) 2070239462Sdim Coeff = umul_ov(Coeff1, Coeff2, Overflow); 2071239462Sdim else 2072239462Sdim Coeff = Coeff1*Coeff2; 2073239462Sdim const SCEV *CoeffTerm = getConstant(Ty, Coeff); 2074239462Sdim const SCEV *Term1 = AddRec->getOperand(y-z); 2075239462Sdim const SCEV *Term2 = OtherAddRec->getOperand(z); 2076239462Sdim Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2)); 2077212904Sdim } 2078239462Sdim } 2079239462Sdim AddRecOps.push_back(Term); 2080239462Sdim } 2081239462Sdim if (!Overflow) { 2082239462Sdim const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(), 2083239462Sdim SCEV::FlagAnyWrap); 2084239462Sdim if (Ops.size() == 2) return NewAddRec; 2085239462Sdim Ops[Idx] = NewAddRec; 2086239462Sdim Ops.erase(Ops.begin() + OtherIdx); --OtherIdx; 2087239462Sdim OpsModified = true; 2088239462Sdim AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec); 2089239462Sdim if (!AddRec) 2090239462Sdim break; 2091239462Sdim } 2092193323Sed } 2093239462Sdim if (OpsModified) 2094239462Sdim return getMulExpr(Ops); 2095226633Sdim } 2096193323Sed 2097193323Sed // Otherwise couldn't fold anything into this recurrence. Move onto the 2098193323Sed // next one. 2099193323Sed } 2100193323Sed 2101193323Sed // Okay, it looks like we really DO need an mul expr. Check to see if we 2102193323Sed // already have one, otherwise create a new one. 2103195340Sed FoldingSetNodeID ID; 2104195340Sed ID.AddInteger(scMulExpr); 2105195340Sed for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2106195340Sed ID.AddPointer(Ops[i]); 2107195340Sed void *IP = 0; 2108202878Srdivacky SCEVMulExpr *S = 2109202878Srdivacky static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2110202878Srdivacky if (!S) { 2111205407Srdivacky const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2112205407Srdivacky std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2113205407Srdivacky S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator), 2114205407Srdivacky O, Ops.size()); 2115202878Srdivacky UniqueSCEVs.InsertNode(S, IP); 2116202878Srdivacky } 2117221345Sdim S->setNoWrapFlags(Flags); 2118195340Sed return S; 2119193323Sed} 2120193323Sed 2121198090Srdivacky/// getUDivExpr - Get a canonical unsigned division expression, or something 2122198090Srdivacky/// simpler if possible. 2123195098Sedconst SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS, 2124195098Sed const SCEV *RHS) { 2125193323Sed assert(getEffectiveSCEVType(LHS->getType()) == 2126193323Sed getEffectiveSCEVType(RHS->getType()) && 2127193323Sed "SCEVUDivExpr operand types don't match!"); 2128193323Sed 2129193323Sed if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 2130193323Sed if (RHSC->getValue()->equalsInt(1)) 2131198090Srdivacky return LHS; // X udiv 1 --> x 2132207618Srdivacky // If the denominator is zero, the result of the udiv is undefined. Don't 2133207618Srdivacky // try to analyze it, because the resolution chosen here may differ from 2134207618Srdivacky // the resolution chosen in other parts of the compiler. 2135207618Srdivacky if (!RHSC->getValue()->isZero()) { 2136207618Srdivacky // Determine if the division can be folded into the operands of 2137207618Srdivacky // its operands. 2138207618Srdivacky // TODO: Generalize this to non-constants by using known-bits information. 2139226633Sdim Type *Ty = LHS->getType(); 2140207618Srdivacky unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros(); 2141212904Sdim unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1; 2142207618Srdivacky // For non-power-of-two values, effectively round the value up to the 2143207618Srdivacky // nearest power of two. 2144207618Srdivacky if (!RHSC->getValue()->getValue().isPowerOf2()) 2145207618Srdivacky ++MaxShiftAmt; 2146226633Sdim IntegerType *ExtTy = 2147207618Srdivacky IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt); 2148207618Srdivacky if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) 2149207618Srdivacky if (const SCEVConstant *Step = 2150226633Sdim dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) { 2151226633Sdim // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded. 2152226633Sdim const APInt &StepInt = Step->getValue()->getValue(); 2153226633Sdim const APInt &DivInt = RHSC->getValue()->getValue(); 2154226633Sdim if (!StepInt.urem(DivInt) && 2155207618Srdivacky getZeroExtendExpr(AR, ExtTy) == 2156207618Srdivacky getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2157207618Srdivacky getZeroExtendExpr(Step, ExtTy), 2158221345Sdim AR->getLoop(), SCEV::FlagAnyWrap)) { 2159207618Srdivacky SmallVector<const SCEV *, 4> Operands; 2160207618Srdivacky for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i) 2161207618Srdivacky Operands.push_back(getUDivExpr(AR->getOperand(i), RHS)); 2162221345Sdim return getAddRecExpr(Operands, AR->getLoop(), 2163221345Sdim SCEV::FlagNW); 2164193323Sed } 2165226633Sdim /// Get a canonical UDivExpr for a recurrence. 2166226633Sdim /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0. 2167226633Sdim // We can currently only fold X%N if X is constant. 2168226633Sdim const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart()); 2169226633Sdim if (StartC && !DivInt.urem(StepInt) && 2170226633Sdim getZeroExtendExpr(AR, ExtTy) == 2171226633Sdim getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy), 2172226633Sdim getZeroExtendExpr(Step, ExtTy), 2173226633Sdim AR->getLoop(), SCEV::FlagAnyWrap)) { 2174226633Sdim const APInt &StartInt = StartC->getValue()->getValue(); 2175226633Sdim const APInt &StartRem = StartInt.urem(StepInt); 2176226633Sdim if (StartRem != 0) 2177226633Sdim LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step, 2178226633Sdim AR->getLoop(), SCEV::FlagNW); 2179226633Sdim } 2180226633Sdim } 2181207618Srdivacky // (A*B)/C --> A*(B/C) if safe and B/C can be folded. 2182207618Srdivacky if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) { 2183207618Srdivacky SmallVector<const SCEV *, 4> Operands; 2184207618Srdivacky for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) 2185207618Srdivacky Operands.push_back(getZeroExtendExpr(M->getOperand(i), ExtTy)); 2186207618Srdivacky if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands)) 2187207618Srdivacky // Find an operand that's safely divisible. 2188207618Srdivacky for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 2189207618Srdivacky const SCEV *Op = M->getOperand(i); 2190207618Srdivacky const SCEV *Div = getUDivExpr(Op, RHSC); 2191207618Srdivacky if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) { 2192207618Srdivacky Operands = SmallVector<const SCEV *, 4>(M->op_begin(), 2193207618Srdivacky M->op_end()); 2194207618Srdivacky Operands[i] = Div; 2195207618Srdivacky return getMulExpr(Operands); 2196207618Srdivacky } 2197207618Srdivacky } 2198207618Srdivacky } 2199207618Srdivacky // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded. 2200221345Sdim if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) { 2201207618Srdivacky SmallVector<const SCEV *, 4> Operands; 2202207618Srdivacky for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) 2203207618Srdivacky Operands.push_back(getZeroExtendExpr(A->getOperand(i), ExtTy)); 2204207618Srdivacky if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) { 2205207618Srdivacky Operands.clear(); 2206207618Srdivacky for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) { 2207207618Srdivacky const SCEV *Op = getUDivExpr(A->getOperand(i), RHS); 2208207618Srdivacky if (isa<SCEVUDivExpr>(Op) || 2209207618Srdivacky getMulExpr(Op, RHS) != A->getOperand(i)) 2210207618Srdivacky break; 2211207618Srdivacky Operands.push_back(Op); 2212207618Srdivacky } 2213207618Srdivacky if (Operands.size() == A->getNumOperands()) 2214207618Srdivacky return getAddExpr(Operands); 2215193323Sed } 2216193323Sed } 2217193323Sed 2218207618Srdivacky // Fold if both operands are constant. 2219207618Srdivacky if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 2220207618Srdivacky Constant *LHSCV = LHSC->getValue(); 2221207618Srdivacky Constant *RHSCV = RHSC->getValue(); 2222207618Srdivacky return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV, 2223207618Srdivacky RHSCV))); 2224207618Srdivacky } 2225193323Sed } 2226193323Sed } 2227193323Sed 2228195340Sed FoldingSetNodeID ID; 2229195340Sed ID.AddInteger(scUDivExpr); 2230195340Sed ID.AddPointer(LHS); 2231195340Sed ID.AddPointer(RHS); 2232195340Sed void *IP = 0; 2233195340Sed if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2234205407Srdivacky SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), 2235205407Srdivacky LHS, RHS); 2236195340Sed UniqueSCEVs.InsertNode(S, IP); 2237195340Sed return S; 2238193323Sed} 2239193323Sed 2240193323Sed 2241193323Sed/// getAddRecExpr - Get an add recurrence expression for the specified loop. 2242193323Sed/// Simplify the expression as much as possible. 2243221345Sdimconst SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step, 2244221345Sdim const Loop *L, 2245221345Sdim SCEV::NoWrapFlags Flags) { 2246198090Srdivacky SmallVector<const SCEV *, 4> Operands; 2247193323Sed Operands.push_back(Start); 2248193323Sed if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step)) 2249193323Sed if (StepChrec->getLoop() == L) { 2250210299Sed Operands.append(StepChrec->op_begin(), StepChrec->op_end()); 2251221345Sdim return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW)); 2252193323Sed } 2253193323Sed 2254193323Sed Operands.push_back(Step); 2255221345Sdim return getAddRecExpr(Operands, L, Flags); 2256193323Sed} 2257193323Sed 2258193323Sed/// getAddRecExpr - Get an add recurrence expression for the specified loop. 2259193323Sed/// Simplify the expression as much as possible. 2260195098Sedconst SCEV * 2261198090SrdivackyScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands, 2262221345Sdim const Loop *L, SCEV::NoWrapFlags Flags) { 2263193323Sed if (Operands.size() == 1) return Operands[0]; 2264193323Sed#ifndef NDEBUG 2265226633Sdim Type *ETy = getEffectiveSCEVType(Operands[0]->getType()); 2266193323Sed for (unsigned i = 1, e = Operands.size(); i != e; ++i) 2267212904Sdim assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy && 2268193323Sed "SCEVAddRecExpr operand types don't match!"); 2269218893Sdim for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2270218893Sdim assert(isLoopInvariant(Operands[i], L) && 2271218893Sdim "SCEVAddRecExpr operand is not loop-invariant!"); 2272193323Sed#endif 2273193323Sed 2274193323Sed if (Operands.back()->isZero()) { 2275193323Sed Operands.pop_back(); 2276221345Sdim return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0} --> X 2277193323Sed } 2278193323Sed 2279204642Srdivacky // It's tempting to want to call getMaxBackedgeTakenCount count here and 2280204642Srdivacky // use that information to infer NUW and NSW flags. However, computing a 2281204642Srdivacky // BE count requires calling getAddRecExpr, so we may not yet have a 2282204642Srdivacky // meaningful BE count at this point (and if we don't, we'd be stuck 2283204642Srdivacky // with a SCEVCouldNotCompute as the cached BE count). 2284204642Srdivacky 2285221345Sdim // If FlagNSW is true and all the operands are non-negative, infer FlagNUW. 2286221345Sdim // And vice-versa. 2287221345Sdim int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW; 2288221345Sdim SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask); 2289221345Sdim if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) { 2290202878Srdivacky bool All = true; 2291212904Sdim for (SmallVectorImpl<const SCEV *>::const_iterator I = Operands.begin(), 2292212904Sdim E = Operands.end(); I != E; ++I) 2293212904Sdim if (!isKnownNonNegative(*I)) { 2294202878Srdivacky All = false; 2295202878Srdivacky break; 2296202878Srdivacky } 2297221345Sdim if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask); 2298202878Srdivacky } 2299202878Srdivacky 2300193323Sed // Canonicalize nested AddRecs in by nesting them in order of loop depth. 2301193323Sed if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) { 2302201360Srdivacky const Loop *NestedLoop = NestedAR->getLoop(); 2303212904Sdim if (L->contains(NestedLoop) ? 2304202878Srdivacky (L->getLoopDepth() < NestedLoop->getLoopDepth()) : 2305212904Sdim (!NestedLoop->contains(L) && 2306202878Srdivacky DT->dominates(L->getHeader(), NestedLoop->getHeader()))) { 2307198090Srdivacky SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(), 2308201360Srdivacky NestedAR->op_end()); 2309193323Sed Operands[0] = NestedAR->getStart(); 2310195098Sed // AddRecs require their operands be loop-invariant with respect to their 2311195098Sed // loops. Don't perform this transformation if it would break this 2312195098Sed // requirement. 2313195098Sed bool AllInvariant = true; 2314195098Sed for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2315218893Sdim if (!isLoopInvariant(Operands[i], L)) { 2316195098Sed AllInvariant = false; 2317195098Sed break; 2318195098Sed } 2319195098Sed if (AllInvariant) { 2320221345Sdim // Create a recurrence for the outer loop with the same step size. 2321221345Sdim // 2322221345Sdim // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the 2323221345Sdim // inner recurrence has the same property. 2324221345Sdim SCEV::NoWrapFlags OuterFlags = 2325221345Sdim maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags()); 2326221345Sdim 2327221345Sdim NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags); 2328195098Sed AllInvariant = true; 2329195098Sed for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i) 2330218893Sdim if (!isLoopInvariant(NestedOperands[i], NestedLoop)) { 2331195098Sed AllInvariant = false; 2332195098Sed break; 2333195098Sed } 2334221345Sdim if (AllInvariant) { 2335195098Sed // Ok, both add recurrences are valid after the transformation. 2336221345Sdim // 2337221345Sdim // The inner recurrence keeps its NW flag but only keeps NUW/NSW if 2338221345Sdim // the outer recurrence has the same property. 2339221345Sdim SCEV::NoWrapFlags InnerFlags = 2340221345Sdim maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags); 2341221345Sdim return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags); 2342221345Sdim } 2343195098Sed } 2344195098Sed // Reset Operands to its original state. 2345195098Sed Operands[0] = NestedAR; 2346193323Sed } 2347193323Sed } 2348193323Sed 2349202878Srdivacky // Okay, it looks like we really DO need an addrec expr. Check to see if we 2350202878Srdivacky // already have one, otherwise create a new one. 2351195340Sed FoldingSetNodeID ID; 2352195340Sed ID.AddInteger(scAddRecExpr); 2353195340Sed for (unsigned i = 0, e = Operands.size(); i != e; ++i) 2354195340Sed ID.AddPointer(Operands[i]); 2355195340Sed ID.AddPointer(L); 2356195340Sed void *IP = 0; 2357202878Srdivacky SCEVAddRecExpr *S = 2358202878Srdivacky static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); 2359202878Srdivacky if (!S) { 2360205407Srdivacky const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size()); 2361205407Srdivacky std::uninitialized_copy(Operands.begin(), Operands.end(), O); 2362205407Srdivacky S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator), 2363205407Srdivacky O, Operands.size(), L); 2364202878Srdivacky UniqueSCEVs.InsertNode(S, IP); 2365202878Srdivacky } 2366221345Sdim S->setNoWrapFlags(Flags); 2367195340Sed return S; 2368193323Sed} 2369193323Sed 2370195098Sedconst SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, 2371195098Sed const SCEV *RHS) { 2372198090Srdivacky SmallVector<const SCEV *, 2> Ops; 2373193323Sed Ops.push_back(LHS); 2374193323Sed Ops.push_back(RHS); 2375193323Sed return getSMaxExpr(Ops); 2376193323Sed} 2377193323Sed 2378198090Srdivackyconst SCEV * 2379198090SrdivackyScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 2380193323Sed assert(!Ops.empty() && "Cannot get empty smax!"); 2381193323Sed if (Ops.size() == 1) return Ops[0]; 2382193323Sed#ifndef NDEBUG 2383226633Sdim Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2384193323Sed for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2385212904Sdim assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2386193323Sed "SCEVSMaxExpr operand types don't match!"); 2387193323Sed#endif 2388193323Sed 2389193323Sed // Sort by complexity, this groups all similar expression types together. 2390193323Sed GroupByComplexity(Ops, LI); 2391193323Sed 2392193323Sed // If there are any constants, fold them together. 2393193323Sed unsigned Idx = 0; 2394193323Sed if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2395193323Sed ++Idx; 2396193323Sed assert(Idx < Ops.size()); 2397193323Sed while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2398193323Sed // We found two constants, fold them together! 2399198090Srdivacky ConstantInt *Fold = ConstantInt::get(getContext(), 2400193323Sed APIntOps::smax(LHSC->getValue()->getValue(), 2401193323Sed RHSC->getValue()->getValue())); 2402193323Sed Ops[0] = getConstant(Fold); 2403193323Sed Ops.erase(Ops.begin()+1); // Erase the folded element 2404193323Sed if (Ops.size() == 1) return Ops[0]; 2405193323Sed LHSC = cast<SCEVConstant>(Ops[0]); 2406193323Sed } 2407193323Sed 2408195098Sed // If we are left with a constant minimum-int, strip it off. 2409193323Sed if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) { 2410193323Sed Ops.erase(Ops.begin()); 2411193323Sed --Idx; 2412195098Sed } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) { 2413195098Sed // If we have an smax with a constant maximum-int, it will always be 2414195098Sed // maximum-int. 2415195098Sed return Ops[0]; 2416193323Sed } 2417207618Srdivacky 2418207618Srdivacky if (Ops.size() == 1) return Ops[0]; 2419193323Sed } 2420193323Sed 2421193323Sed // Find the first SMax 2422193323Sed while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr) 2423193323Sed ++Idx; 2424193323Sed 2425193323Sed // Check to see if one of the operands is an SMax. If so, expand its operands 2426193323Sed // onto our operand list, and recurse to simplify. 2427193323Sed if (Idx < Ops.size()) { 2428193323Sed bool DeletedSMax = false; 2429193323Sed while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) { 2430193323Sed Ops.erase(Ops.begin()+Idx); 2431210299Sed Ops.append(SMax->op_begin(), SMax->op_end()); 2432193323Sed DeletedSMax = true; 2433193323Sed } 2434193323Sed 2435193323Sed if (DeletedSMax) 2436193323Sed return getSMaxExpr(Ops); 2437193323Sed } 2438193323Sed 2439193323Sed // Okay, check to see if the same value occurs in the operand list twice. If 2440193323Sed // so, delete one. Since we sorted the list, these values are required to 2441193323Sed // be adjacent. 2442193323Sed for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 2443207618Srdivacky // X smax Y smax Y --> X smax Y 2444207618Srdivacky // X smax Y --> X, if X is always greater than Y 2445207618Srdivacky if (Ops[i] == Ops[i+1] || 2446207618Srdivacky isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) { 2447207618Srdivacky Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 2448207618Srdivacky --i; --e; 2449207618Srdivacky } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) { 2450193323Sed Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 2451193323Sed --i; --e; 2452193323Sed } 2453193323Sed 2454193323Sed if (Ops.size() == 1) return Ops[0]; 2455193323Sed 2456193323Sed assert(!Ops.empty() && "Reduced smax down to nothing!"); 2457193323Sed 2458193323Sed // Okay, it looks like we really DO need an smax expr. Check to see if we 2459193323Sed // already have one, otherwise create a new one. 2460195340Sed FoldingSetNodeID ID; 2461195340Sed ID.AddInteger(scSMaxExpr); 2462195340Sed for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2463195340Sed ID.AddPointer(Ops[i]); 2464195340Sed void *IP = 0; 2465195340Sed if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2466205407Srdivacky const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2467205407Srdivacky std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2468205407Srdivacky SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator), 2469205407Srdivacky O, Ops.size()); 2470195340Sed UniqueSCEVs.InsertNode(S, IP); 2471195340Sed return S; 2472193323Sed} 2473193323Sed 2474195098Sedconst SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, 2475195098Sed const SCEV *RHS) { 2476198090Srdivacky SmallVector<const SCEV *, 2> Ops; 2477193323Sed Ops.push_back(LHS); 2478193323Sed Ops.push_back(RHS); 2479193323Sed return getUMaxExpr(Ops); 2480193323Sed} 2481193323Sed 2482198090Srdivackyconst SCEV * 2483198090SrdivackyScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) { 2484193323Sed assert(!Ops.empty() && "Cannot get empty umax!"); 2485193323Sed if (Ops.size() == 1) return Ops[0]; 2486193323Sed#ifndef NDEBUG 2487226633Sdim Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); 2488193323Sed for (unsigned i = 1, e = Ops.size(); i != e; ++i) 2489212904Sdim assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && 2490193323Sed "SCEVUMaxExpr operand types don't match!"); 2491193323Sed#endif 2492193323Sed 2493193323Sed // Sort by complexity, this groups all similar expression types together. 2494193323Sed GroupByComplexity(Ops, LI); 2495193323Sed 2496193323Sed // If there are any constants, fold them together. 2497193323Sed unsigned Idx = 0; 2498193323Sed if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) { 2499193323Sed ++Idx; 2500193323Sed assert(Idx < Ops.size()); 2501193323Sed while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) { 2502193323Sed // We found two constants, fold them together! 2503198090Srdivacky ConstantInt *Fold = ConstantInt::get(getContext(), 2504193323Sed APIntOps::umax(LHSC->getValue()->getValue(), 2505193323Sed RHSC->getValue()->getValue())); 2506193323Sed Ops[0] = getConstant(Fold); 2507193323Sed Ops.erase(Ops.begin()+1); // Erase the folded element 2508193323Sed if (Ops.size() == 1) return Ops[0]; 2509193323Sed LHSC = cast<SCEVConstant>(Ops[0]); 2510193323Sed } 2511193323Sed 2512195098Sed // If we are left with a constant minimum-int, strip it off. 2513193323Sed if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) { 2514193323Sed Ops.erase(Ops.begin()); 2515193323Sed --Idx; 2516195098Sed } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) { 2517195098Sed // If we have an umax with a constant maximum-int, it will always be 2518195098Sed // maximum-int. 2519195098Sed return Ops[0]; 2520193323Sed } 2521207618Srdivacky 2522207618Srdivacky if (Ops.size() == 1) return Ops[0]; 2523193323Sed } 2524193323Sed 2525193323Sed // Find the first UMax 2526193323Sed while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr) 2527193323Sed ++Idx; 2528193323Sed 2529193323Sed // Check to see if one of the operands is a UMax. If so, expand its operands 2530193323Sed // onto our operand list, and recurse to simplify. 2531193323Sed if (Idx < Ops.size()) { 2532193323Sed bool DeletedUMax = false; 2533193323Sed while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) { 2534193323Sed Ops.erase(Ops.begin()+Idx); 2535210299Sed Ops.append(UMax->op_begin(), UMax->op_end()); 2536193323Sed DeletedUMax = true; 2537193323Sed } 2538193323Sed 2539193323Sed if (DeletedUMax) 2540193323Sed return getUMaxExpr(Ops); 2541193323Sed } 2542193323Sed 2543193323Sed // Okay, check to see if the same value occurs in the operand list twice. If 2544193323Sed // so, delete one. Since we sorted the list, these values are required to 2545193323Sed // be adjacent. 2546193323Sed for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) 2547207618Srdivacky // X umax Y umax Y --> X umax Y 2548207618Srdivacky // X umax Y --> X, if X is always greater than Y 2549207618Srdivacky if (Ops[i] == Ops[i+1] || 2550207618Srdivacky isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) { 2551207618Srdivacky Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); 2552207618Srdivacky --i; --e; 2553207618Srdivacky } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) { 2554193323Sed Ops.erase(Ops.begin()+i, Ops.begin()+i+1); 2555193323Sed --i; --e; 2556193323Sed } 2557193323Sed 2558193323Sed if (Ops.size() == 1) return Ops[0]; 2559193323Sed 2560193323Sed assert(!Ops.empty() && "Reduced umax down to nothing!"); 2561193323Sed 2562193323Sed // Okay, it looks like we really DO need a umax expr. Check to see if we 2563193323Sed // already have one, otherwise create a new one. 2564195340Sed FoldingSetNodeID ID; 2565195340Sed ID.AddInteger(scUMaxExpr); 2566195340Sed for (unsigned i = 0, e = Ops.size(); i != e; ++i) 2567195340Sed ID.AddPointer(Ops[i]); 2568195340Sed void *IP = 0; 2569195340Sed if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; 2570205407Srdivacky const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size()); 2571205407Srdivacky std::uninitialized_copy(Ops.begin(), Ops.end(), O); 2572205407Srdivacky SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator), 2573205407Srdivacky O, Ops.size()); 2574195340Sed UniqueSCEVs.InsertNode(S, IP); 2575195340Sed return S; 2576193323Sed} 2577193323Sed 2578195098Sedconst SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS, 2579195098Sed const SCEV *RHS) { 2580194612Sed // ~smax(~x, ~y) == smin(x, y). 2581194612Sed return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 2582194612Sed} 2583194612Sed 2584195098Sedconst SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, 2585195098Sed const SCEV *RHS) { 2586194612Sed // ~umax(~x, ~y) == umin(x, y) 2587194612Sed return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS))); 2588194612Sed} 2589194612Sed 2590226633Sdimconst SCEV *ScalarEvolution::getSizeOfExpr(Type *AllocTy) { 2591243830Sdim // If we have DataLayout, we can bypass creating a target-independent 2592207618Srdivacky // constant expression and then folding it back into a ConstantInt. 2593207618Srdivacky // This is just a compile-time optimization. 2594207618Srdivacky if (TD) 2595207618Srdivacky return getConstant(TD->getIntPtrType(getContext()), 2596207618Srdivacky TD->getTypeAllocSize(AllocTy)); 2597207618Srdivacky 2598203954Srdivacky Constant *C = ConstantExpr::getSizeOf(AllocTy); 2599203954Srdivacky if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2600234353Sdim if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2601210299Sed C = Folded; 2602226633Sdim Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy)); 2603203954Srdivacky return getTruncateOrZeroExtend(getSCEV(C), Ty); 2604203954Srdivacky} 2605198090Srdivacky 2606226633Sdimconst SCEV *ScalarEvolution::getAlignOfExpr(Type *AllocTy) { 2607203954Srdivacky Constant *C = ConstantExpr::getAlignOf(AllocTy); 2608203954Srdivacky if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2609234353Sdim if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2610210299Sed C = Folded; 2611226633Sdim Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy)); 2612203954Srdivacky return getTruncateOrZeroExtend(getSCEV(C), Ty); 2613203954Srdivacky} 2614198090Srdivacky 2615226633Sdimconst SCEV *ScalarEvolution::getOffsetOfExpr(StructType *STy, 2616203954Srdivacky unsigned FieldNo) { 2617243830Sdim // If we have DataLayout, we can bypass creating a target-independent 2618207618Srdivacky // constant expression and then folding it back into a ConstantInt. 2619207618Srdivacky // This is just a compile-time optimization. 2620207618Srdivacky if (TD) 2621207618Srdivacky return getConstant(TD->getIntPtrType(getContext()), 2622207618Srdivacky TD->getStructLayout(STy)->getElementOffset(FieldNo)); 2623207618Srdivacky 2624203954Srdivacky Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo); 2625203954Srdivacky if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2626234353Sdim if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2627210299Sed C = Folded; 2628226633Sdim Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy)); 2629203954Srdivacky return getTruncateOrZeroExtend(getSCEV(C), Ty); 2630198090Srdivacky} 2631198090Srdivacky 2632226633Sdimconst SCEV *ScalarEvolution::getOffsetOfExpr(Type *CTy, 2633203954Srdivacky Constant *FieldNo) { 2634203954Srdivacky Constant *C = ConstantExpr::getOffsetOf(CTy, FieldNo); 2635203954Srdivacky if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) 2636234353Sdim if (Constant *Folded = ConstantFoldConstantExpression(CE, TD, TLI)) 2637210299Sed C = Folded; 2638226633Sdim Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(CTy)); 2639203954Srdivacky return getTruncateOrZeroExtend(getSCEV(C), Ty); 2640198090Srdivacky} 2641198090Srdivacky 2642198090Srdivackyconst SCEV *ScalarEvolution::getUnknown(Value *V) { 2643195098Sed // Don't attempt to do anything other than create a SCEVUnknown object 2644195098Sed // here. createSCEV only calls getUnknown after checking for all other 2645195098Sed // interesting possibilities, and any other code that calls getUnknown 2646195098Sed // is doing so in order to hide a value from SCEV canonicalization. 2647195098Sed 2648195340Sed FoldingSetNodeID ID; 2649195340Sed ID.AddInteger(scUnknown); 2650195340Sed ID.AddPointer(V); 2651195340Sed void *IP = 0; 2652212904Sdim if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) { 2653212904Sdim assert(cast<SCEVUnknown>(S)->getValue() == V && 2654212904Sdim "Stale SCEVUnknown in uniquing map!"); 2655212904Sdim return S; 2656212904Sdim } 2657212904Sdim SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this, 2658212904Sdim FirstUnknown); 2659212904Sdim FirstUnknown = cast<SCEVUnknown>(S); 2660195340Sed UniqueSCEVs.InsertNode(S, IP); 2661195340Sed return S; 2662193323Sed} 2663193323Sed 2664193323Sed//===----------------------------------------------------------------------===// 2665193323Sed// Basic SCEV Analysis and PHI Idiom Recognition Code 2666193323Sed// 2667193323Sed 2668193323Sed/// isSCEVable - Test if values of the given type are analyzable within 2669193323Sed/// the SCEV framework. This primarily includes integer types, and it 2670193323Sed/// can optionally include pointer types if the ScalarEvolution class 2671193323Sed/// has access to target-specific information. 2672226633Sdimbool ScalarEvolution::isSCEVable(Type *Ty) const { 2673198090Srdivacky // Integers and pointers are always SCEVable. 2674204642Srdivacky return Ty->isIntegerTy() || Ty->isPointerTy(); 2675193323Sed} 2676193323Sed 2677193323Sed/// getTypeSizeInBits - Return the size in bits of the specified type, 2678193323Sed/// for which isSCEVable must return true. 2679226633Sdimuint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const { 2680193323Sed assert(isSCEVable(Ty) && "Type is not SCEVable!"); 2681193323Sed 2682243830Sdim // If we have a DataLayout, use it! 2683193323Sed if (TD) 2684193323Sed return TD->getTypeSizeInBits(Ty); 2685193323Sed 2686198090Srdivacky // Integer types have fixed sizes. 2687203954Srdivacky if (Ty->isIntegerTy()) 2688198090Srdivacky return Ty->getPrimitiveSizeInBits(); 2689198090Srdivacky 2690243830Sdim // The only other support type is pointer. Without DataLayout, conservatively 2691198090Srdivacky // assume pointers are 64-bit. 2692204642Srdivacky assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!"); 2693198090Srdivacky return 64; 2694193323Sed} 2695193323Sed 2696193323Sed/// getEffectiveSCEVType - Return a type with the same bitwidth as 2697193323Sed/// the given type and which represents how SCEV will treat the given 2698193323Sed/// type, for which isSCEVable must return true. For pointer types, 2699193323Sed/// this is the pointer-sized integer type. 2700226633SdimType *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const { 2701193323Sed assert(isSCEVable(Ty) && "Type is not SCEVable!"); 2702193323Sed 2703203954Srdivacky if (Ty->isIntegerTy()) 2704193323Sed return Ty; 2705193323Sed 2706198090Srdivacky // The only other support type is pointer. 2707204642Srdivacky assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!"); 2708198090Srdivacky if (TD) return TD->getIntPtrType(getContext()); 2709198090Srdivacky 2710243830Sdim // Without DataLayout, conservatively assume pointers are 64-bit. 2711198090Srdivacky return Type::getInt64Ty(getContext()); 2712193323Sed} 2713193323Sed 2714198090Srdivackyconst SCEV *ScalarEvolution::getCouldNotCompute() { 2715195340Sed return &CouldNotCompute; 2716193323Sed} 2717193323Sed 2718193323Sed/// getSCEV - Return an existing SCEV if it exists, otherwise analyze the 2719193323Sed/// expression and create a new one. 2720198090Srdivackyconst SCEV *ScalarEvolution::getSCEV(Value *V) { 2721193323Sed assert(isSCEVable(V->getType()) && "Value is not SCEVable!"); 2722193323Sed 2723239462Sdim ValueExprMapType::const_iterator I = ValueExprMap.find_as(V); 2724212904Sdim if (I != ValueExprMap.end()) return I->second; 2725198090Srdivacky const SCEV *S = createSCEV(V); 2726212904Sdim 2727212904Sdim // The process of creating a SCEV for V may have caused other SCEVs 2728212904Sdim // to have been created, so it's necessary to insert the new entry 2729212904Sdim // from scratch, rather than trying to remember the insert position 2730212904Sdim // above. 2731212904Sdim ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S)); 2732193323Sed return S; 2733193323Sed} 2734193323Sed 2735193323Sed/// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V 2736193323Sed/// 2737198090Srdivackyconst SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) { 2738193323Sed if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 2739198090Srdivacky return getConstant( 2740198090Srdivacky cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue()))); 2741193323Sed 2742226633Sdim Type *Ty = V->getType(); 2743193323Sed Ty = getEffectiveSCEVType(Ty); 2744198090Srdivacky return getMulExpr(V, 2745198090Srdivacky getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)))); 2746193323Sed} 2747193323Sed 2748193323Sed/// getNotSCEV - Return a SCEV corresponding to ~V = -1-V 2749198090Srdivackyconst SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { 2750193323Sed if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V)) 2751198090Srdivacky return getConstant( 2752198090Srdivacky cast<ConstantInt>(ConstantExpr::getNot(VC->getValue()))); 2753193323Sed 2754226633Sdim Type *Ty = V->getType(); 2755193323Sed Ty = getEffectiveSCEVType(Ty); 2756198090Srdivacky const SCEV *AllOnes = 2757198090Srdivacky getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))); 2758193323Sed return getMinusSCEV(AllOnes, V); 2759193323Sed} 2760193323Sed 2761221345Sdim/// getMinusSCEV - Return LHS-RHS. Minus is represented in SCEV as A+B*-1. 2762218893Sdimconst SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, 2763221345Sdim SCEV::NoWrapFlags Flags) { 2764221345Sdim assert(!maskFlags(Flags, SCEV::FlagNUW) && "subtraction does not have NUW"); 2765221345Sdim 2766212904Sdim // Fast path: X - X --> 0. 2767212904Sdim if (LHS == RHS) 2768212904Sdim return getConstant(LHS->getType(), 0); 2769212904Sdim 2770193323Sed // X - Y --> X + -Y 2771221345Sdim return getAddExpr(LHS, getNegativeSCEV(RHS), Flags); 2772193323Sed} 2773193323Sed 2774193323Sed/// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the 2775193323Sed/// input value to the specified type. If the type must be extended, it is zero 2776193323Sed/// extended. 2777198090Srdivackyconst SCEV * 2778226633SdimScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) { 2779226633Sdim Type *SrcTy = V->getType(); 2780204642Srdivacky assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2781204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 2782193323Sed "Cannot truncate or zero extend with non-integer arguments!"); 2783193323Sed if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2784193323Sed return V; // No conversion 2785193323Sed if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 2786193323Sed return getTruncateExpr(V, Ty); 2787193323Sed return getZeroExtendExpr(V, Ty); 2788193323Sed} 2789193323Sed 2790193323Sed/// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the 2791193323Sed/// input value to the specified type. If the type must be extended, it is sign 2792193323Sed/// extended. 2793198090Srdivackyconst SCEV * 2794198090SrdivackyScalarEvolution::getTruncateOrSignExtend(const SCEV *V, 2795226633Sdim Type *Ty) { 2796226633Sdim Type *SrcTy = V->getType(); 2797204642Srdivacky assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2798204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 2799193323Sed "Cannot truncate or zero extend with non-integer arguments!"); 2800193323Sed if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2801193323Sed return V; // No conversion 2802193323Sed if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) 2803193323Sed return getTruncateExpr(V, Ty); 2804193323Sed return getSignExtendExpr(V, Ty); 2805193323Sed} 2806193323Sed 2807193323Sed/// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the 2808193323Sed/// input value to the specified type. If the type must be extended, it is zero 2809193323Sed/// extended. The conversion must not be narrowing. 2810198090Srdivackyconst SCEV * 2811226633SdimScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) { 2812226633Sdim Type *SrcTy = V->getType(); 2813204642Srdivacky assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2814204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 2815193323Sed "Cannot noop or zero extend with non-integer arguments!"); 2816193323Sed assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 2817193323Sed "getNoopOrZeroExtend cannot truncate!"); 2818193323Sed if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2819193323Sed return V; // No conversion 2820193323Sed return getZeroExtendExpr(V, Ty); 2821193323Sed} 2822193323Sed 2823193323Sed/// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the 2824193323Sed/// input value to the specified type. If the type must be extended, it is sign 2825193323Sed/// extended. The conversion must not be narrowing. 2826198090Srdivackyconst SCEV * 2827226633SdimScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) { 2828226633Sdim Type *SrcTy = V->getType(); 2829204642Srdivacky assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2830204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 2831193323Sed "Cannot noop or sign extend with non-integer arguments!"); 2832193323Sed assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 2833193323Sed "getNoopOrSignExtend cannot truncate!"); 2834193323Sed if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2835193323Sed return V; // No conversion 2836193323Sed return getSignExtendExpr(V, Ty); 2837193323Sed} 2838193323Sed 2839194178Sed/// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of 2840194178Sed/// the input value to the specified type. If the type must be extended, 2841194178Sed/// it is extended with unspecified bits. The conversion must not be 2842194178Sed/// narrowing. 2843198090Srdivackyconst SCEV * 2844226633SdimScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) { 2845226633Sdim Type *SrcTy = V->getType(); 2846204642Srdivacky assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2847204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 2848194178Sed "Cannot noop or any extend with non-integer arguments!"); 2849194178Sed assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) && 2850194178Sed "getNoopOrAnyExtend cannot truncate!"); 2851194178Sed if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2852194178Sed return V; // No conversion 2853194178Sed return getAnyExtendExpr(V, Ty); 2854194178Sed} 2855194178Sed 2856193323Sed/// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the 2857193323Sed/// input value to the specified type. The conversion must not be widening. 2858198090Srdivackyconst SCEV * 2859226633SdimScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) { 2860226633Sdim Type *SrcTy = V->getType(); 2861204642Srdivacky assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) && 2862204642Srdivacky (Ty->isIntegerTy() || Ty->isPointerTy()) && 2863193323Sed "Cannot truncate or noop with non-integer arguments!"); 2864193323Sed assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) && 2865193323Sed "getTruncateOrNoop cannot extend!"); 2866193323Sed if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) 2867193323Sed return V; // No conversion 2868193323Sed return getTruncateExpr(V, Ty); 2869193323Sed} 2870193323Sed 2871194612Sed/// getUMaxFromMismatchedTypes - Promote the operands to the wider of 2872194612Sed/// the types using zero-extension, and then perform a umax operation 2873194612Sed/// with them. 2874195098Sedconst SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS, 2875195098Sed const SCEV *RHS) { 2876198090Srdivacky const SCEV *PromotedLHS = LHS; 2877198090Srdivacky const SCEV *PromotedRHS = RHS; 2878194612Sed 2879194612Sed if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 2880194612Sed PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 2881194612Sed else 2882194612Sed PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 2883194612Sed 2884194612Sed return getUMaxExpr(PromotedLHS, PromotedRHS); 2885194612Sed} 2886194612Sed 2887194710Sed/// getUMinFromMismatchedTypes - Promote the operands to the wider of 2888194710Sed/// the types using zero-extension, and then perform a umin operation 2889194710Sed/// with them. 2890195098Sedconst SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS, 2891195098Sed const SCEV *RHS) { 2892198090Srdivacky const SCEV *PromotedLHS = LHS; 2893198090Srdivacky const SCEV *PromotedRHS = RHS; 2894194710Sed 2895194710Sed if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType())) 2896194710Sed PromotedRHS = getZeroExtendExpr(RHS, LHS->getType()); 2897194710Sed else 2898194710Sed PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType()); 2899194710Sed 2900194710Sed return getUMinExpr(PromotedLHS, PromotedRHS); 2901194710Sed} 2902194710Sed 2903221345Sdim/// getPointerBase - Transitively follow the chain of pointer-type operands 2904221345Sdim/// until reaching a SCEV that does not have a single pointer operand. This 2905221345Sdim/// returns a SCEVUnknown pointer for well-formed pointer-type expressions, 2906221345Sdim/// but corner cases do exist. 2907221345Sdimconst SCEV *ScalarEvolution::getPointerBase(const SCEV *V) { 2908221345Sdim // A pointer operand may evaluate to a nonpointer expression, such as null. 2909221345Sdim if (!V->getType()->isPointerTy()) 2910221345Sdim return V; 2911221345Sdim 2912221345Sdim if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) { 2913221345Sdim return getPointerBase(Cast->getOperand()); 2914221345Sdim } 2915221345Sdim else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) { 2916221345Sdim const SCEV *PtrOp = 0; 2917221345Sdim for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 2918221345Sdim I != E; ++I) { 2919221345Sdim if ((*I)->getType()->isPointerTy()) { 2920221345Sdim // Cannot find the base of an expression with multiple pointer operands. 2921221345Sdim if (PtrOp) 2922221345Sdim return V; 2923221345Sdim PtrOp = *I; 2924221345Sdim } 2925221345Sdim } 2926221345Sdim if (!PtrOp) 2927221345Sdim return V; 2928221345Sdim return getPointerBase(PtrOp); 2929221345Sdim } 2930221345Sdim return V; 2931221345Sdim} 2932221345Sdim 2933198090Srdivacky/// PushDefUseChildren - Push users of the given Instruction 2934198090Srdivacky/// onto the given Worklist. 2935198090Srdivackystatic void 2936198090SrdivackyPushDefUseChildren(Instruction *I, 2937198090Srdivacky SmallVectorImpl<Instruction *> &Worklist) { 2938198090Srdivacky // Push the def-use children onto the Worklist stack. 2939198090Srdivacky for (Value::use_iterator UI = I->use_begin(), UE = I->use_end(); 2940198090Srdivacky UI != UE; ++UI) 2941212904Sdim Worklist.push_back(cast<Instruction>(*UI)); 2942198090Srdivacky} 2943198090Srdivacky 2944198090Srdivacky/// ForgetSymbolicValue - This looks up computed SCEV values for all 2945198090Srdivacky/// instructions that depend on the given instruction and removes them from 2946212904Sdim/// the ValueExprMapType map if they reference SymName. This is used during PHI 2947198090Srdivacky/// resolution. 2948195098Sedvoid 2949204642SrdivackyScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) { 2950198090Srdivacky SmallVector<Instruction *, 16> Worklist; 2951204642Srdivacky PushDefUseChildren(PN, Worklist); 2952193323Sed 2953198090Srdivacky SmallPtrSet<Instruction *, 8> Visited; 2954204642Srdivacky Visited.insert(PN); 2955198090Srdivacky while (!Worklist.empty()) { 2956198090Srdivacky Instruction *I = Worklist.pop_back_val(); 2957198090Srdivacky if (!Visited.insert(I)) continue; 2958193323Sed 2959212904Sdim ValueExprMapType::iterator It = 2960239462Sdim ValueExprMap.find_as(static_cast<Value *>(I)); 2961212904Sdim if (It != ValueExprMap.end()) { 2962218893Sdim const SCEV *Old = It->second; 2963218893Sdim 2964198090Srdivacky // Short-circuit the def-use traversal if the symbolic name 2965198090Srdivacky // ceases to appear in expressions. 2966218893Sdim if (Old != SymName && !hasOperand(Old, SymName)) 2967198090Srdivacky continue; 2968193323Sed 2969198090Srdivacky // SCEVUnknown for a PHI either means that it has an unrecognized 2970204642Srdivacky // structure, it's a PHI that's in the progress of being computed 2971204642Srdivacky // by createNodeForPHI, or it's a single-value PHI. In the first case, 2972204642Srdivacky // additional loop trip count information isn't going to change anything. 2973204642Srdivacky // In the second case, createNodeForPHI will perform the necessary 2974204642Srdivacky // updates on its own when it gets to that point. In the third, we do 2975204642Srdivacky // want to forget the SCEVUnknown. 2976204642Srdivacky if (!isa<PHINode>(I) || 2977218893Sdim !isa<SCEVUnknown>(Old) || 2978218893Sdim (I != PN && Old == SymName)) { 2979218893Sdim forgetMemoizedResults(Old); 2980212904Sdim ValueExprMap.erase(It); 2981198090Srdivacky } 2982198090Srdivacky } 2983198090Srdivacky 2984198090Srdivacky PushDefUseChildren(I, Worklist); 2985198090Srdivacky } 2986193323Sed} 2987193323Sed 2988193323Sed/// createNodeForPHI - PHI nodes have two cases. Either the PHI node exists in 2989193323Sed/// a loop header, making it a potential recurrence, or it doesn't. 2990193323Sed/// 2991198090Srdivackyconst SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) { 2992207618Srdivacky if (const Loop *L = LI->getLoopFor(PN->getParent())) 2993207618Srdivacky if (L->getHeader() == PN->getParent()) { 2994207618Srdivacky // The loop may have multiple entrances or multiple exits; we can analyze 2995207618Srdivacky // this phi as an addrec if it has a unique entry value and a unique 2996207618Srdivacky // backedge value. 2997207618Srdivacky Value *BEValueV = 0, *StartValueV = 0; 2998207618Srdivacky for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { 2999207618Srdivacky Value *V = PN->getIncomingValue(i); 3000207618Srdivacky if (L->contains(PN->getIncomingBlock(i))) { 3001207618Srdivacky if (!BEValueV) { 3002207618Srdivacky BEValueV = V; 3003207618Srdivacky } else if (BEValueV != V) { 3004207618Srdivacky BEValueV = 0; 3005207618Srdivacky break; 3006207618Srdivacky } 3007207618Srdivacky } else if (!StartValueV) { 3008207618Srdivacky StartValueV = V; 3009207618Srdivacky } else if (StartValueV != V) { 3010207618Srdivacky StartValueV = 0; 3011207618Srdivacky break; 3012207618Srdivacky } 3013207618Srdivacky } 3014207618Srdivacky if (BEValueV && StartValueV) { 3015193323Sed // While we are analyzing this PHI node, handle its value symbolically. 3016198090Srdivacky const SCEV *SymbolicName = getUnknown(PN); 3017239462Sdim assert(ValueExprMap.find_as(PN) == ValueExprMap.end() && 3018193323Sed "PHI node already processed?"); 3019212904Sdim ValueExprMap.insert(std::make_pair(SCEVCallbackVH(PN, this), SymbolicName)); 3020193323Sed 3021193323Sed // Using this symbolic name for the PHI, analyze the value coming around 3022193323Sed // the back-edge. 3023198090Srdivacky const SCEV *BEValue = getSCEV(BEValueV); 3024193323Sed 3025193323Sed // NOTE: If BEValue is loop invariant, we know that the PHI node just 3026193323Sed // has a special value for the first iteration of the loop. 3027193323Sed 3028193323Sed // If the value coming around the backedge is an add with the symbolic 3029193323Sed // value we just inserted, then we found a simple induction variable! 3030193323Sed if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) { 3031193323Sed // If there is a single occurrence of the symbolic value, replace it 3032193323Sed // with a recurrence. 3033193323Sed unsigned FoundIndex = Add->getNumOperands(); 3034193323Sed for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3035193323Sed if (Add->getOperand(i) == SymbolicName) 3036193323Sed if (FoundIndex == e) { 3037193323Sed FoundIndex = i; 3038193323Sed break; 3039193323Sed } 3040193323Sed 3041193323Sed if (FoundIndex != Add->getNumOperands()) { 3042193323Sed // Create an add with everything but the specified operand. 3043198090Srdivacky SmallVector<const SCEV *, 8> Ops; 3044193323Sed for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i) 3045193323Sed if (i != FoundIndex) 3046193323Sed Ops.push_back(Add->getOperand(i)); 3047198090Srdivacky const SCEV *Accum = getAddExpr(Ops); 3048193323Sed 3049193323Sed // This is not a valid addrec if the step amount is varying each 3050193323Sed // loop iteration, but is not itself an addrec in this loop. 3051218893Sdim if (isLoopInvariant(Accum, L) || 3052193323Sed (isa<SCEVAddRecExpr>(Accum) && 3053193323Sed cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) { 3054221345Sdim SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; 3055202878Srdivacky 3056202878Srdivacky // If the increment doesn't overflow, then neither the addrec nor 3057202878Srdivacky // the post-increment will overflow. 3058202878Srdivacky if (const AddOperator *OBO = dyn_cast<AddOperator>(BEValueV)) { 3059202878Srdivacky if (OBO->hasNoUnsignedWrap()) 3060221345Sdim Flags = setFlags(Flags, SCEV::FlagNUW); 3061202878Srdivacky if (OBO->hasNoSignedWrap()) 3062221345Sdim Flags = setFlags(Flags, SCEV::FlagNSW); 3063221345Sdim } else if (const GEPOperator *GEP = 3064221345Sdim dyn_cast<GEPOperator>(BEValueV)) { 3065221345Sdim // If the increment is an inbounds GEP, then we know the address 3066221345Sdim // space cannot be wrapped around. We cannot make any guarantee 3067221345Sdim // about signed or unsigned overflow because pointers are 3068221345Sdim // unsigned but we may have a negative index from the base 3069221345Sdim // pointer. 3070221345Sdim if (GEP->isInBounds()) 3071221345Sdim Flags = setFlags(Flags, SCEV::FlagNW); 3072202878Srdivacky } 3073202878Srdivacky 3074207618Srdivacky const SCEV *StartVal = getSCEV(StartValueV); 3075221345Sdim const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); 3076193323Sed 3077202878Srdivacky // Since the no-wrap flags are on the increment, they apply to the 3078202878Srdivacky // post-incremented value as well. 3079218893Sdim if (isLoopInvariant(Accum, L)) 3080202878Srdivacky (void)getAddRecExpr(getAddExpr(StartVal, Accum), 3081221345Sdim Accum, L, Flags); 3082198090Srdivacky 3083193323Sed // Okay, for the entire analysis of this edge we assumed the PHI 3084198090Srdivacky // to be symbolic. We now need to go back and purge all of the 3085198090Srdivacky // entries for the scalars that use the symbolic expression. 3086198090Srdivacky ForgetSymbolicName(PN, SymbolicName); 3087212904Sdim ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 3088193323Sed return PHISCEV; 3089193323Sed } 3090193323Sed } 3091193323Sed } else if (const SCEVAddRecExpr *AddRec = 3092193323Sed dyn_cast<SCEVAddRecExpr>(BEValue)) { 3093193323Sed // Otherwise, this could be a loop like this: 3094193323Sed // i = 0; for (j = 1; ..; ++j) { .... i = j; } 3095193323Sed // In this case, j = {1,+,1} and BEValue is j. 3096193323Sed // Because the other in-value of i (0) fits the evolution of BEValue 3097193323Sed // i really is an addrec evolution. 3098193323Sed if (AddRec->getLoop() == L && AddRec->isAffine()) { 3099207618Srdivacky const SCEV *StartVal = getSCEV(StartValueV); 3100193323Sed 3101193323Sed // If StartVal = j.start - j.stride, we can use StartVal as the 3102193323Sed // initial step of the addrec evolution. 3103193323Sed if (StartVal == getMinusSCEV(AddRec->getOperand(0), 3104207618Srdivacky AddRec->getOperand(1))) { 3105221345Sdim // FIXME: For constant StartVal, we should be able to infer 3106221345Sdim // no-wrap flags. 3107198090Srdivacky const SCEV *PHISCEV = 3108221345Sdim getAddRecExpr(StartVal, AddRec->getOperand(1), L, 3109221345Sdim SCEV::FlagAnyWrap); 3110193323Sed 3111193323Sed // Okay, for the entire analysis of this edge we assumed the PHI 3112198090Srdivacky // to be symbolic. We now need to go back and purge all of the 3113198090Srdivacky // entries for the scalars that use the symbolic expression. 3114198090Srdivacky ForgetSymbolicName(PN, SymbolicName); 3115212904Sdim ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV; 3116193323Sed return PHISCEV; 3117193323Sed } 3118193323Sed } 3119193323Sed } 3120193323Sed } 3121207618Srdivacky } 3122193323Sed 3123204642Srdivacky // If the PHI has a single incoming value, follow that value, unless the 3124204642Srdivacky // PHI's incoming blocks are in a different loop, in which case doing so 3125204642Srdivacky // risks breaking LCSSA form. Instcombine would normally zap these, but 3126204642Srdivacky // it doesn't have DominatorTree information, so it may miss cases. 3127234353Sdim if (Value *V = SimplifyInstruction(PN, TD, TLI, DT)) 3128218893Sdim if (LI->replacementPreservesLCSSAForm(PN, V)) 3129204642Srdivacky return getSCEV(V); 3130198090Srdivacky 3131193323Sed // If it's not a loop phi, we can't handle it yet. 3132193323Sed return getUnknown(PN); 3133193323Sed} 3134193323Sed 3135193323Sed/// createNodeForGEP - Expand GEP instructions into add and multiply 3136193323Sed/// operations. This allows them to be analyzed by regular SCEV code. 3137193323Sed/// 3138201360Srdivackyconst SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { 3139193323Sed 3140210299Sed // Don't blindly transfer the inbounds flag from the GEP instruction to the 3141210299Sed // Add expression, because the Instruction may be guarded by control flow 3142210299Sed // and the no-overflow bits may not be valid for the expression in any 3143210299Sed // context. 3144218893Sdim bool isInBounds = GEP->isInBounds(); 3145210299Sed 3146226633Sdim Type *IntPtrTy = getEffectiveSCEVType(GEP->getType()); 3147193323Sed Value *Base = GEP->getOperand(0); 3148193323Sed // Don't attempt to analyze GEPs over unsized objects. 3149193323Sed if (!cast<PointerType>(Base->getType())->getElementType()->isSized()) 3150193323Sed return getUnknown(GEP); 3151207618Srdivacky const SCEV *TotalOffset = getConstant(IntPtrTy, 0); 3152193323Sed gep_type_iterator GTI = gep_type_begin(GEP); 3153212904Sdim for (GetElementPtrInst::op_iterator I = llvm::next(GEP->op_begin()), 3154193323Sed E = GEP->op_end(); 3155193323Sed I != E; ++I) { 3156193323Sed Value *Index = *I; 3157193323Sed // Compute the (potentially symbolic) offset in bytes for this index. 3158226633Sdim if (StructType *STy = dyn_cast<StructType>(*GTI++)) { 3159193323Sed // For a struct, add the member offset. 3160193323Sed unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue(); 3161210299Sed const SCEV *FieldOffset = getOffsetOfExpr(STy, FieldNo); 3162210299Sed 3163210299Sed // Add the field offset to the running total offset. 3164210299Sed TotalOffset = getAddExpr(TotalOffset, FieldOffset); 3165193323Sed } else { 3166193323Sed // For an array, add the element offset, explicitly scaled. 3167210299Sed const SCEV *ElementSize = getSizeOfExpr(*GTI); 3168210299Sed const SCEV *IndexS = getSCEV(Index); 3169204642Srdivacky // Getelementptr indices are signed. 3170210299Sed IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy); 3171210299Sed 3172210299Sed // Multiply the index by the element size to compute the element offset. 3173221345Sdim const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, 3174221345Sdim isInBounds ? SCEV::FlagNSW : 3175221345Sdim SCEV::FlagAnyWrap); 3176210299Sed 3177210299Sed // Add the element offset to the running total offset. 3178210299Sed TotalOffset = getAddExpr(TotalOffset, LocalOffset); 3179193323Sed } 3180193323Sed } 3181210299Sed 3182210299Sed // Get the SCEV for the GEP base. 3183210299Sed const SCEV *BaseS = getSCEV(Base); 3184210299Sed 3185210299Sed // Add the total offset from all the GEP indices to the base. 3186221345Sdim return getAddExpr(BaseS, TotalOffset, 3187234982Sdim isInBounds ? SCEV::FlagNSW : SCEV::FlagAnyWrap); 3188193323Sed} 3189193323Sed 3190193323Sed/// GetMinTrailingZeros - Determine the minimum number of zero bits that S is 3191193323Sed/// guaranteed to end in (at every loop iteration). It is, at the same time, 3192193323Sed/// the minimum number of times S is divisible by 2. For example, given {4,+,8} 3193193323Sed/// it returns 2. If S is guaranteed to be 0, it returns the bitwidth of S. 3194194612Seduint32_t 3195198090SrdivackyScalarEvolution::GetMinTrailingZeros(const SCEV *S) { 3196193323Sed if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3197193323Sed return C->getValue()->getValue().countTrailingZeros(); 3198193323Sed 3199193323Sed if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S)) 3200194612Sed return std::min(GetMinTrailingZeros(T->getOperand()), 3201194612Sed (uint32_t)getTypeSizeInBits(T->getType())); 3202193323Sed 3203193323Sed if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) { 3204194612Sed uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 3205194612Sed return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 3206194612Sed getTypeSizeInBits(E->getType()) : OpRes; 3207193323Sed } 3208193323Sed 3209193323Sed if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) { 3210194612Sed uint32_t OpRes = GetMinTrailingZeros(E->getOperand()); 3211194612Sed return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ? 3212194612Sed getTypeSizeInBits(E->getType()) : OpRes; 3213193323Sed } 3214193323Sed 3215193323Sed if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) { 3216193323Sed // The result is the min of all operands results. 3217194612Sed uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 3218193323Sed for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 3219194612Sed MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 3220193323Sed return MinOpRes; 3221193323Sed } 3222193323Sed 3223193323Sed if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) { 3224193323Sed // The result is the sum of all operands results. 3225194612Sed uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0)); 3226194612Sed uint32_t BitWidth = getTypeSizeInBits(M->getType()); 3227193323Sed for (unsigned i = 1, e = M->getNumOperands(); 3228193323Sed SumOpRes != BitWidth && i != e; ++i) 3229194612Sed SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)), 3230193323Sed BitWidth); 3231193323Sed return SumOpRes; 3232193323Sed } 3233193323Sed 3234193323Sed if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) { 3235193323Sed // The result is the min of all operands results. 3236194612Sed uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0)); 3237193323Sed for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i) 3238194612Sed MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i))); 3239193323Sed return MinOpRes; 3240193323Sed } 3241193323Sed 3242193323Sed if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) { 3243193323Sed // The result is the min of all operands results. 3244194612Sed uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 3245193323Sed for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 3246194612Sed MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 3247193323Sed return MinOpRes; 3248193323Sed } 3249193323Sed 3250193323Sed if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) { 3251193323Sed // The result is the min of all operands results. 3252194612Sed uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0)); 3253193323Sed for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i) 3254194612Sed MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i))); 3255193323Sed return MinOpRes; 3256193323Sed } 3257193323Sed 3258194612Sed if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3259194612Sed // For a SCEVUnknown, ask ValueTracking. 3260194612Sed unsigned BitWidth = getTypeSizeInBits(U->getType()); 3261194612Sed APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 3262234353Sdim ComputeMaskedBits(U->getValue(), Zeros, Ones); 3263194612Sed return Zeros.countTrailingOnes(); 3264194612Sed } 3265194612Sed 3266194612Sed // SCEVUDivExpr 3267193323Sed return 0; 3268193323Sed} 3269193323Sed 3270198090Srdivacky/// getUnsignedRange - Determine the unsigned range for a particular SCEV. 3271198090Srdivacky/// 3272198090SrdivackyConstantRange 3273198090SrdivackyScalarEvolution::getUnsignedRange(const SCEV *S) { 3274218893Sdim // See if we've computed this range already. 3275218893Sdim DenseMap<const SCEV *, ConstantRange>::iterator I = UnsignedRanges.find(S); 3276218893Sdim if (I != UnsignedRanges.end()) 3277218893Sdim return I->second; 3278194612Sed 3279194612Sed if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3280218893Sdim return setUnsignedRange(C, ConstantRange(C->getValue()->getValue())); 3281194612Sed 3282203954Srdivacky unsigned BitWidth = getTypeSizeInBits(S->getType()); 3283203954Srdivacky ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 3284203954Srdivacky 3285203954Srdivacky // If the value has known zeros, the maximum unsigned value will have those 3286203954Srdivacky // known zeros as well. 3287203954Srdivacky uint32_t TZ = GetMinTrailingZeros(S); 3288203954Srdivacky if (TZ != 0) 3289203954Srdivacky ConservativeResult = 3290203954Srdivacky ConstantRange(APInt::getMinValue(BitWidth), 3291203954Srdivacky APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1); 3292203954Srdivacky 3293198090Srdivacky if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 3294198090Srdivacky ConstantRange X = getUnsignedRange(Add->getOperand(0)); 3295198090Srdivacky for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 3296198090Srdivacky X = X.add(getUnsignedRange(Add->getOperand(i))); 3297218893Sdim return setUnsignedRange(Add, ConservativeResult.intersectWith(X)); 3298194612Sed } 3299194612Sed 3300198090Srdivacky if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3301198090Srdivacky ConstantRange X = getUnsignedRange(Mul->getOperand(0)); 3302198090Srdivacky for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 3303198090Srdivacky X = X.multiply(getUnsignedRange(Mul->getOperand(i))); 3304218893Sdim return setUnsignedRange(Mul, ConservativeResult.intersectWith(X)); 3305198090Srdivacky } 3306198090Srdivacky 3307198090Srdivacky if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 3308198090Srdivacky ConstantRange X = getUnsignedRange(SMax->getOperand(0)); 3309198090Srdivacky for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 3310198090Srdivacky X = X.smax(getUnsignedRange(SMax->getOperand(i))); 3311218893Sdim return setUnsignedRange(SMax, ConservativeResult.intersectWith(X)); 3312198090Srdivacky } 3313198090Srdivacky 3314198090Srdivacky if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 3315198090Srdivacky ConstantRange X = getUnsignedRange(UMax->getOperand(0)); 3316198090Srdivacky for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 3317198090Srdivacky X = X.umax(getUnsignedRange(UMax->getOperand(i))); 3318218893Sdim return setUnsignedRange(UMax, ConservativeResult.intersectWith(X)); 3319198090Srdivacky } 3320198090Srdivacky 3321198090Srdivacky if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 3322198090Srdivacky ConstantRange X = getUnsignedRange(UDiv->getLHS()); 3323198090Srdivacky ConstantRange Y = getUnsignedRange(UDiv->getRHS()); 3324218893Sdim return setUnsignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); 3325198090Srdivacky } 3326198090Srdivacky 3327198090Srdivacky if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 3328198090Srdivacky ConstantRange X = getUnsignedRange(ZExt->getOperand()); 3329218893Sdim return setUnsignedRange(ZExt, 3330218893Sdim ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 3331198090Srdivacky } 3332198090Srdivacky 3333198090Srdivacky if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 3334198090Srdivacky ConstantRange X = getUnsignedRange(SExt->getOperand()); 3335218893Sdim return setUnsignedRange(SExt, 3336218893Sdim ConservativeResult.intersectWith(X.signExtend(BitWidth))); 3337198090Srdivacky } 3338198090Srdivacky 3339198090Srdivacky if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 3340198090Srdivacky ConstantRange X = getUnsignedRange(Trunc->getOperand()); 3341218893Sdim return setUnsignedRange(Trunc, 3342218893Sdim ConservativeResult.intersectWith(X.truncate(BitWidth))); 3343198090Srdivacky } 3344198090Srdivacky 3345198090Srdivacky if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 3346202878Srdivacky // If there's no unsigned wrap, the value will never be less than its 3347202878Srdivacky // initial value. 3348221345Sdim if (AddRec->getNoWrapFlags(SCEV::FlagNUW)) 3349202878Srdivacky if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart())) 3350207618Srdivacky if (!C->getValue()->isZero()) 3351207618Srdivacky ConservativeResult = 3352210299Sed ConservativeResult.intersectWith( 3353210299Sed ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0))); 3354202878Srdivacky 3355198090Srdivacky // TODO: non-affine addrec 3356203954Srdivacky if (AddRec->isAffine()) { 3357226633Sdim Type *Ty = AddRec->getType(); 3358198090Srdivacky const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 3359203954Srdivacky if (!isa<SCEVCouldNotCompute>(MaxBECount) && 3360203954Srdivacky getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 3361198090Srdivacky MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); 3362198090Srdivacky 3363198090Srdivacky const SCEV *Start = AddRec->getStart(); 3364207618Srdivacky const SCEV *Step = AddRec->getStepRecurrence(*this); 3365198090Srdivacky 3366207618Srdivacky ConstantRange StartRange = getUnsignedRange(Start); 3367207618Srdivacky ConstantRange StepRange = getSignedRange(Step); 3368207618Srdivacky ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 3369207618Srdivacky ConstantRange EndRange = 3370207618Srdivacky StartRange.add(MaxBECountRange.multiply(StepRange)); 3371207618Srdivacky 3372207618Srdivacky // Check for overflow. This must be done with ConstantRange arithmetic 3373207618Srdivacky // because we could be called from within the ScalarEvolution overflow 3374207618Srdivacky // checking code. 3375207618Srdivacky ConstantRange ExtStartRange = StartRange.zextOrTrunc(BitWidth*2+1); 3376207618Srdivacky ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); 3377207618Srdivacky ConstantRange ExtMaxBECountRange = 3378207618Srdivacky MaxBECountRange.zextOrTrunc(BitWidth*2+1); 3379207618Srdivacky ConstantRange ExtEndRange = EndRange.zextOrTrunc(BitWidth*2+1); 3380207618Srdivacky if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != 3381207618Srdivacky ExtEndRange) 3382218893Sdim return setUnsignedRange(AddRec, ConservativeResult); 3383198090Srdivacky 3384198090Srdivacky APInt Min = APIntOps::umin(StartRange.getUnsignedMin(), 3385198090Srdivacky EndRange.getUnsignedMin()); 3386198090Srdivacky APInt Max = APIntOps::umax(StartRange.getUnsignedMax(), 3387198090Srdivacky EndRange.getUnsignedMax()); 3388198090Srdivacky if (Min.isMinValue() && Max.isMaxValue()) 3389218893Sdim return setUnsignedRange(AddRec, ConservativeResult); 3390218893Sdim return setUnsignedRange(AddRec, 3391218893Sdim ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); 3392198090Srdivacky } 3393198090Srdivacky } 3394202878Srdivacky 3395218893Sdim return setUnsignedRange(AddRec, ConservativeResult); 3396198090Srdivacky } 3397198090Srdivacky 3398194612Sed if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3399194612Sed // For a SCEVUnknown, ask ValueTracking. 3400194612Sed APInt Zeros(BitWidth, 0), Ones(BitWidth, 0); 3401234353Sdim ComputeMaskedBits(U->getValue(), Zeros, Ones, TD); 3402198090Srdivacky if (Ones == ~Zeros + 1) 3403218893Sdim return setUnsignedRange(U, ConservativeResult); 3404218893Sdim return setUnsignedRange(U, 3405218893Sdim ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1))); 3406194612Sed } 3407194612Sed 3408218893Sdim return setUnsignedRange(S, ConservativeResult); 3409194612Sed} 3410194612Sed 3411198090Srdivacky/// getSignedRange - Determine the signed range for a particular SCEV. 3412198090Srdivacky/// 3413198090SrdivackyConstantRange 3414198090SrdivackyScalarEvolution::getSignedRange(const SCEV *S) { 3415218893Sdim // See if we've computed this range already. 3416218893Sdim DenseMap<const SCEV *, ConstantRange>::iterator I = SignedRanges.find(S); 3417218893Sdim if (I != SignedRanges.end()) 3418218893Sdim return I->second; 3419194612Sed 3420198090Srdivacky if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) 3421218893Sdim return setSignedRange(C, ConstantRange(C->getValue()->getValue())); 3422198090Srdivacky 3423203954Srdivacky unsigned BitWidth = getTypeSizeInBits(S->getType()); 3424203954Srdivacky ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true); 3425203954Srdivacky 3426203954Srdivacky // If the value has known zeros, the maximum signed value will have those 3427203954Srdivacky // known zeros as well. 3428203954Srdivacky uint32_t TZ = GetMinTrailingZeros(S); 3429203954Srdivacky if (TZ != 0) 3430203954Srdivacky ConservativeResult = 3431203954Srdivacky ConstantRange(APInt::getSignedMinValue(BitWidth), 3432203954Srdivacky APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1); 3433203954Srdivacky 3434198090Srdivacky if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { 3435198090Srdivacky ConstantRange X = getSignedRange(Add->getOperand(0)); 3436198090Srdivacky for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) 3437198090Srdivacky X = X.add(getSignedRange(Add->getOperand(i))); 3438218893Sdim return setSignedRange(Add, ConservativeResult.intersectWith(X)); 3439194612Sed } 3440194612Sed 3441198090Srdivacky if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { 3442198090Srdivacky ConstantRange X = getSignedRange(Mul->getOperand(0)); 3443198090Srdivacky for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) 3444198090Srdivacky X = X.multiply(getSignedRange(Mul->getOperand(i))); 3445218893Sdim return setSignedRange(Mul, ConservativeResult.intersectWith(X)); 3446194612Sed } 3447194612Sed 3448198090Srdivacky if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) { 3449198090Srdivacky ConstantRange X = getSignedRange(SMax->getOperand(0)); 3450198090Srdivacky for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) 3451198090Srdivacky X = X.smax(getSignedRange(SMax->getOperand(i))); 3452218893Sdim return setSignedRange(SMax, ConservativeResult.intersectWith(X)); 3453198090Srdivacky } 3454195098Sed 3455198090Srdivacky if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) { 3456198090Srdivacky ConstantRange X = getSignedRange(UMax->getOperand(0)); 3457198090Srdivacky for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) 3458198090Srdivacky X = X.umax(getSignedRange(UMax->getOperand(i))); 3459218893Sdim return setSignedRange(UMax, ConservativeResult.intersectWith(X)); 3460198090Srdivacky } 3461195098Sed 3462198090Srdivacky if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { 3463198090Srdivacky ConstantRange X = getSignedRange(UDiv->getLHS()); 3464198090Srdivacky ConstantRange Y = getSignedRange(UDiv->getRHS()); 3465218893Sdim return setSignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y))); 3466198090Srdivacky } 3467195098Sed 3468198090Srdivacky if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) { 3469198090Srdivacky ConstantRange X = getSignedRange(ZExt->getOperand()); 3470218893Sdim return setSignedRange(ZExt, 3471218893Sdim ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); 3472198090Srdivacky } 3473198090Srdivacky 3474198090Srdivacky if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) { 3475198090Srdivacky ConstantRange X = getSignedRange(SExt->getOperand()); 3476218893Sdim return setSignedRange(SExt, 3477218893Sdim ConservativeResult.intersectWith(X.signExtend(BitWidth))); 3478198090Srdivacky } 3479198090Srdivacky 3480198090Srdivacky if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) { 3481198090Srdivacky ConstantRange X = getSignedRange(Trunc->getOperand()); 3482218893Sdim return setSignedRange(Trunc, 3483218893Sdim ConservativeResult.intersectWith(X.truncate(BitWidth))); 3484198090Srdivacky } 3485198090Srdivacky 3486198090Srdivacky if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) { 3487202878Srdivacky // If there's no signed wrap, and all the operands have the same sign or 3488202878Srdivacky // zero, the value won't ever change sign. 3489221345Sdim if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) { 3490202878Srdivacky bool AllNonNeg = true; 3491202878Srdivacky bool AllNonPos = true; 3492202878Srdivacky for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 3493202878Srdivacky if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false; 3494202878Srdivacky if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false; 3495202878Srdivacky } 3496202878Srdivacky if (AllNonNeg) 3497203954Srdivacky ConservativeResult = ConservativeResult.intersectWith( 3498203954Srdivacky ConstantRange(APInt(BitWidth, 0), 3499203954Srdivacky APInt::getSignedMinValue(BitWidth))); 3500202878Srdivacky else if (AllNonPos) 3501203954Srdivacky ConservativeResult = ConservativeResult.intersectWith( 3502203954Srdivacky ConstantRange(APInt::getSignedMinValue(BitWidth), 3503203954Srdivacky APInt(BitWidth, 1))); 3504202878Srdivacky } 3505202878Srdivacky 3506198090Srdivacky // TODO: non-affine addrec 3507203954Srdivacky if (AddRec->isAffine()) { 3508226633Sdim Type *Ty = AddRec->getType(); 3509198090Srdivacky const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop()); 3510203954Srdivacky if (!isa<SCEVCouldNotCompute>(MaxBECount) && 3511203954Srdivacky getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) { 3512198090Srdivacky MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty); 3513198090Srdivacky 3514198090Srdivacky const SCEV *Start = AddRec->getStart(); 3515207618Srdivacky const SCEV *Step = AddRec->getStepRecurrence(*this); 3516198090Srdivacky 3517207618Srdivacky ConstantRange StartRange = getSignedRange(Start); 3518207618Srdivacky ConstantRange StepRange = getSignedRange(Step); 3519207618Srdivacky ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount); 3520207618Srdivacky ConstantRange EndRange = 3521207618Srdivacky StartRange.add(MaxBECountRange.multiply(StepRange)); 3522207618Srdivacky 3523207618Srdivacky // Check for overflow. This must be done with ConstantRange arithmetic 3524207618Srdivacky // because we could be called from within the ScalarEvolution overflow 3525207618Srdivacky // checking code. 3526207618Srdivacky ConstantRange ExtStartRange = StartRange.sextOrTrunc(BitWidth*2+1); 3527207618Srdivacky ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1); 3528207618Srdivacky ConstantRange ExtMaxBECountRange = 3529207618Srdivacky MaxBECountRange.zextOrTrunc(BitWidth*2+1); 3530207618Srdivacky ConstantRange ExtEndRange = EndRange.sextOrTrunc(BitWidth*2+1); 3531207618Srdivacky if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) != 3532207618Srdivacky ExtEndRange) 3533218893Sdim return setSignedRange(AddRec, ConservativeResult); 3534198090Srdivacky 3535198090Srdivacky APInt Min = APIntOps::smin(StartRange.getSignedMin(), 3536198090Srdivacky EndRange.getSignedMin()); 3537198090Srdivacky APInt Max = APIntOps::smax(StartRange.getSignedMax(), 3538198090Srdivacky EndRange.getSignedMax()); 3539198090Srdivacky if (Min.isMinSignedValue() && Max.isMaxSignedValue()) 3540218893Sdim return setSignedRange(AddRec, ConservativeResult); 3541218893Sdim return setSignedRange(AddRec, 3542218893Sdim ConservativeResult.intersectWith(ConstantRange(Min, Max+1))); 3543195098Sed } 3544195098Sed } 3545202878Srdivacky 3546218893Sdim return setSignedRange(AddRec, ConservativeResult); 3547195098Sed } 3548195098Sed 3549194612Sed if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { 3550194612Sed // For a SCEVUnknown, ask ValueTracking. 3551203954Srdivacky if (!U->getValue()->getType()->isIntegerTy() && !TD) 3552218893Sdim return setSignedRange(U, ConservativeResult); 3553198090Srdivacky unsigned NS = ComputeNumSignBits(U->getValue(), TD); 3554198090Srdivacky if (NS == 1) 3555218893Sdim return setSignedRange(U, ConservativeResult); 3556218893Sdim return setSignedRange(U, ConservativeResult.intersectWith( 3557198090Srdivacky ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), 3558218893Sdim APInt::getSignedMaxValue(BitWidth).ashr(NS - 1)+1))); 3559194612Sed } 3560194612Sed 3561218893Sdim return setSignedRange(S, ConservativeResult); 3562194612Sed} 3563194612Sed 3564193323Sed/// createSCEV - We know that there is no SCEV for the specified value. 3565193323Sed/// Analyze the expression. 3566193323Sed/// 3567198090Srdivackyconst SCEV *ScalarEvolution::createSCEV(Value *V) { 3568193323Sed if (!isSCEVable(V->getType())) 3569193323Sed return getUnknown(V); 3570193323Sed 3571193323Sed unsigned Opcode = Instruction::UserOp1; 3572204961Srdivacky if (Instruction *I = dyn_cast<Instruction>(V)) { 3573193323Sed Opcode = I->getOpcode(); 3574204961Srdivacky 3575204961Srdivacky // Don't attempt to analyze instructions in blocks that aren't 3576204961Srdivacky // reachable. Such instructions don't matter, and they aren't required 3577204961Srdivacky // to obey basic rules for definitions dominating uses which this 3578204961Srdivacky // analysis depends on. 3579204961Srdivacky if (!DT->isReachableFromEntry(I->getParent())) 3580204961Srdivacky return getUnknown(V); 3581204961Srdivacky } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) 3582193323Sed Opcode = CE->getOpcode(); 3583195098Sed else if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) 3584195098Sed return getConstant(CI); 3585195098Sed else if (isa<ConstantPointerNull>(V)) 3586207618Srdivacky return getConstant(V->getType(), 0); 3587198090Srdivacky else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 3588198090Srdivacky return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee()); 3589193323Sed else 3590193323Sed return getUnknown(V); 3591193323Sed 3592198090Srdivacky Operator *U = cast<Operator>(V); 3593193323Sed switch (Opcode) { 3594212904Sdim case Instruction::Add: { 3595212904Sdim // The simple thing to do would be to just call getSCEV on both operands 3596212904Sdim // and call getAddExpr with the result. However if we're looking at a 3597212904Sdim // bunch of things all added together, this can be quite inefficient, 3598212904Sdim // because it leads to N-1 getAddExpr calls for N ultimate operands. 3599212904Sdim // Instead, gather up all the operands and make a single getAddExpr call. 3600212904Sdim // LLVM IR canonical form means we need only traverse the left operands. 3601234353Sdim // 3602234353Sdim // Don't apply this instruction's NSW or NUW flags to the new 3603234353Sdim // expression. The instruction may be guarded by control flow that the 3604234353Sdim // no-wrap behavior depends on. Non-control-equivalent instructions can be 3605234353Sdim // mapped to the same SCEV expression, and it would be incorrect to transfer 3606234353Sdim // NSW/NUW semantics to those operations. 3607212904Sdim SmallVector<const SCEV *, 4> AddOps; 3608212904Sdim AddOps.push_back(getSCEV(U->getOperand(1))); 3609212904Sdim for (Value *Op = U->getOperand(0); ; Op = U->getOperand(0)) { 3610212904Sdim unsigned Opcode = Op->getValueID() - Value::InstructionVal; 3611212904Sdim if (Opcode != Instruction::Add && Opcode != Instruction::Sub) 3612212904Sdim break; 3613212904Sdim U = cast<Operator>(Op); 3614212904Sdim const SCEV *Op1 = getSCEV(U->getOperand(1)); 3615212904Sdim if (Opcode == Instruction::Sub) 3616212904Sdim AddOps.push_back(getNegativeSCEV(Op1)); 3617212904Sdim else 3618212904Sdim AddOps.push_back(Op1); 3619212904Sdim } 3620212904Sdim AddOps.push_back(getSCEV(U->getOperand(0))); 3621234353Sdim return getAddExpr(AddOps); 3622212904Sdim } 3623212904Sdim case Instruction::Mul: { 3624234353Sdim // Don't transfer NSW/NUW for the same reason as AddExpr. 3625212904Sdim SmallVector<const SCEV *, 4> MulOps; 3626212904Sdim MulOps.push_back(getSCEV(U->getOperand(1))); 3627212904Sdim for (Value *Op = U->getOperand(0); 3628221345Sdim Op->getValueID() == Instruction::Mul + Value::InstructionVal; 3629212904Sdim Op = U->getOperand(0)) { 3630212904Sdim U = cast<Operator>(Op); 3631212904Sdim MulOps.push_back(getSCEV(U->getOperand(1))); 3632212904Sdim } 3633212904Sdim MulOps.push_back(getSCEV(U->getOperand(0))); 3634212904Sdim return getMulExpr(MulOps); 3635212904Sdim } 3636193323Sed case Instruction::UDiv: 3637193323Sed return getUDivExpr(getSCEV(U->getOperand(0)), 3638193323Sed getSCEV(U->getOperand(1))); 3639193323Sed case Instruction::Sub: 3640193323Sed return getMinusSCEV(getSCEV(U->getOperand(0)), 3641193323Sed getSCEV(U->getOperand(1))); 3642193323Sed case Instruction::And: 3643193323Sed // For an expression like x&255 that merely masks off the high bits, 3644193323Sed // use zext(trunc(x)) as the SCEV expression. 3645193323Sed if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 3646193323Sed if (CI->isNullValue()) 3647193323Sed return getSCEV(U->getOperand(1)); 3648193323Sed if (CI->isAllOnesValue()) 3649193323Sed return getSCEV(U->getOperand(0)); 3650193323Sed const APInt &A = CI->getValue(); 3651194612Sed 3652194612Sed // Instcombine's ShrinkDemandedConstant may strip bits out of 3653194612Sed // constants, obscuring what would otherwise be a low-bits mask. 3654194612Sed // Use ComputeMaskedBits to compute what ShrinkDemandedConstant 3655194612Sed // knew about to reconstruct a low-bits mask value. 3656194612Sed unsigned LZ = A.countLeadingZeros(); 3657194612Sed unsigned BitWidth = A.getBitWidth(); 3658194612Sed APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0); 3659234353Sdim ComputeMaskedBits(U->getOperand(0), KnownZero, KnownOne, TD); 3660194612Sed 3661194612Sed APInt EffectiveMask = APInt::getLowBitsSet(BitWidth, BitWidth - LZ); 3662194612Sed 3663194612Sed if (LZ != 0 && !((~A & ~KnownZero) & EffectiveMask)) 3664193323Sed return 3665193323Sed getZeroExtendExpr(getTruncateExpr(getSCEV(U->getOperand(0)), 3666198090Srdivacky IntegerType::get(getContext(), BitWidth - LZ)), 3667193323Sed U->getType()); 3668193323Sed } 3669193323Sed break; 3670194612Sed 3671193323Sed case Instruction::Or: 3672193323Sed // If the RHS of the Or is a constant, we may have something like: 3673193323Sed // X*4+1 which got turned into X*4|1. Handle this as an Add so loop 3674193323Sed // optimizations will transparently handle this case. 3675193323Sed // 3676193323Sed // In order for this transformation to be safe, the LHS must be of the 3677193323Sed // form X*(2^n) and the Or constant must be less than 2^n. 3678193323Sed if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 3679198090Srdivacky const SCEV *LHS = getSCEV(U->getOperand(0)); 3680193323Sed const APInt &CIVal = CI->getValue(); 3681194612Sed if (GetMinTrailingZeros(LHS) >= 3682198090Srdivacky (CIVal.getBitWidth() - CIVal.countLeadingZeros())) { 3683198090Srdivacky // Build a plain add SCEV. 3684198090Srdivacky const SCEV *S = getAddExpr(LHS, getSCEV(CI)); 3685198090Srdivacky // If the LHS of the add was an addrec and it has no-wrap flags, 3686198090Srdivacky // transfer the no-wrap flags, since an or won't introduce a wrap. 3687198090Srdivacky if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) { 3688198090Srdivacky const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS); 3689221345Sdim const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags( 3690221345Sdim OldAR->getNoWrapFlags()); 3691198090Srdivacky } 3692198090Srdivacky return S; 3693198090Srdivacky } 3694193323Sed } 3695193323Sed break; 3696193323Sed case Instruction::Xor: 3697193323Sed if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) { 3698193323Sed // If the RHS of the xor is a signbit, then this is just an add. 3699193323Sed // Instcombine turns add of signbit into xor as a strength reduction step. 3700193323Sed if (CI->getValue().isSignBit()) 3701193323Sed return getAddExpr(getSCEV(U->getOperand(0)), 3702193323Sed getSCEV(U->getOperand(1))); 3703193323Sed 3704193323Sed // If the RHS of xor is -1, then this is a not operation. 3705193323Sed if (CI->isAllOnesValue()) 3706193323Sed return getNotSCEV(getSCEV(U->getOperand(0))); 3707193323Sed 3708193323Sed // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask. 3709193323Sed // This is a variant of the check for xor with -1, and it handles 3710193323Sed // the case where instcombine has trimmed non-demanded bits out 3711193323Sed // of an xor with -1. 3712193323Sed if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getOperand(0))) 3713193323Sed if (ConstantInt *LCI = dyn_cast<ConstantInt>(BO->getOperand(1))) 3714193323Sed if (BO->getOpcode() == Instruction::And && 3715193323Sed LCI->getValue() == CI->getValue()) 3716193323Sed if (const SCEVZeroExtendExpr *Z = 3717194612Sed dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) { 3718226633Sdim Type *UTy = U->getType(); 3719198090Srdivacky const SCEV *Z0 = Z->getOperand(); 3720226633Sdim Type *Z0Ty = Z0->getType(); 3721194612Sed unsigned Z0TySize = getTypeSizeInBits(Z0Ty); 3722194612Sed 3723204642Srdivacky // If C is a low-bits mask, the zero extend is serving to 3724194612Sed // mask off the high bits. Complement the operand and 3725194612Sed // re-apply the zext. 3726194612Sed if (APIntOps::isMask(Z0TySize, CI->getValue())) 3727194612Sed return getZeroExtendExpr(getNotSCEV(Z0), UTy); 3728194612Sed 3729194612Sed // If C is a single bit, it may be in the sign-bit position 3730194612Sed // before the zero-extend. In this case, represent the xor 3731194612Sed // using an add, which is equivalent, and re-apply the zext. 3732218893Sdim APInt Trunc = CI->getValue().trunc(Z0TySize); 3733218893Sdim if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() && 3734194612Sed Trunc.isSignBit()) 3735194612Sed return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)), 3736194612Sed UTy); 3737194612Sed } 3738193323Sed } 3739193323Sed break; 3740193323Sed 3741193323Sed case Instruction::Shl: 3742193323Sed // Turn shift left of a constant amount into a multiply. 3743193323Sed if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) { 3744203954Srdivacky uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth(); 3745207618Srdivacky 3746207618Srdivacky // If the shift count is not less than the bitwidth, the result of 3747207618Srdivacky // the shift is undefined. Don't try to analyze it, because the 3748207618Srdivacky // resolution chosen here may differ from the resolution chosen in 3749207618Srdivacky // other parts of the compiler. 3750207618Srdivacky if (SA->getValue().uge(BitWidth)) 3751207618Srdivacky break; 3752207618Srdivacky 3753198090Srdivacky Constant *X = ConstantInt::get(getContext(), 3754207618Srdivacky APInt(BitWidth, 1).shl(SA->getZExtValue())); 3755193323Sed return getMulExpr(getSCEV(U->getOperand(0)), getSCEV(X)); 3756193323Sed } 3757193323Sed break; 3758193323Sed 3759193323Sed case Instruction::LShr: 3760193323Sed // Turn logical shift right of a constant into a unsigned divide. 3761193323Sed if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) { 3762203954Srdivacky uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth(); 3763207618Srdivacky 3764207618Srdivacky // If the shift count is not less than the bitwidth, the result of 3765207618Srdivacky // the shift is undefined. Don't try to analyze it, because the 3766207618Srdivacky // resolution chosen here may differ from the resolution chosen in 3767207618Srdivacky // other parts of the compiler. 3768207618Srdivacky if (SA->getValue().uge(BitWidth)) 3769207618Srdivacky break; 3770207618Srdivacky 3771198090Srdivacky Constant *X = ConstantInt::get(getContext(), 3772207618Srdivacky APInt(BitWidth, 1).shl(SA->getZExtValue())); 3773193323Sed return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(X)); 3774193323Sed } 3775193323Sed break; 3776193323Sed 3777193323Sed case Instruction::AShr: 3778193323Sed // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression. 3779193323Sed if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) 3780207618Srdivacky if (Operator *L = dyn_cast<Operator>(U->getOperand(0))) 3781193323Sed if (L->getOpcode() == Instruction::Shl && 3782193323Sed L->getOperand(1) == U->getOperand(1)) { 3783207618Srdivacky uint64_t BitWidth = getTypeSizeInBits(U->getType()); 3784207618Srdivacky 3785207618Srdivacky // If the shift count is not less than the bitwidth, the result of 3786207618Srdivacky // the shift is undefined. Don't try to analyze it, because the 3787207618Srdivacky // resolution chosen here may differ from the resolution chosen in 3788207618Srdivacky // other parts of the compiler. 3789207618Srdivacky if (CI->getValue().uge(BitWidth)) 3790207618Srdivacky break; 3791207618Srdivacky 3792193323Sed uint64_t Amt = BitWidth - CI->getZExtValue(); 3793193323Sed if (Amt == BitWidth) 3794193323Sed return getSCEV(L->getOperand(0)); // shift by zero --> noop 3795193323Sed return 3796193323Sed getSignExtendExpr(getTruncateExpr(getSCEV(L->getOperand(0)), 3797207618Srdivacky IntegerType::get(getContext(), 3798207618Srdivacky Amt)), 3799207618Srdivacky U->getType()); 3800193323Sed } 3801193323Sed break; 3802193323Sed 3803193323Sed case Instruction::Trunc: 3804193323Sed return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType()); 3805193323Sed 3806193323Sed case Instruction::ZExt: 3807193323Sed return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 3808193323Sed 3809193323Sed case Instruction::SExt: 3810193323Sed return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType()); 3811193323Sed 3812193323Sed case Instruction::BitCast: 3813193323Sed // BitCasts are no-op casts so we just eliminate the cast. 3814193323Sed if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) 3815193323Sed return getSCEV(U->getOperand(0)); 3816193323Sed break; 3817193323Sed 3818203954Srdivacky // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can 3819203954Srdivacky // lead to pointer expressions which cannot safely be expanded to GEPs, 3820203954Srdivacky // because ScalarEvolution doesn't respect the GEP aliasing rules when 3821203954Srdivacky // simplifying integer expressions. 3822193323Sed 3823193323Sed case Instruction::GetElementPtr: 3824201360Srdivacky return createNodeForGEP(cast<GEPOperator>(U)); 3825193323Sed 3826193323Sed case Instruction::PHI: 3827193323Sed return createNodeForPHI(cast<PHINode>(U)); 3828193323Sed 3829193323Sed case Instruction::Select: 3830193323Sed // This could be a smax or umax that was lowered earlier. 3831193323Sed // Try to recover it. 3832193323Sed if (ICmpInst *ICI = dyn_cast<ICmpInst>(U->getOperand(0))) { 3833193323Sed Value *LHS = ICI->getOperand(0); 3834193323Sed Value *RHS = ICI->getOperand(1); 3835193323Sed switch (ICI->getPredicate()) { 3836193323Sed case ICmpInst::ICMP_SLT: 3837193323Sed case ICmpInst::ICMP_SLE: 3838193323Sed std::swap(LHS, RHS); 3839193323Sed // fall through 3840193323Sed case ICmpInst::ICMP_SGT: 3841193323Sed case ICmpInst::ICMP_SGE: 3842207618Srdivacky // a >s b ? a+x : b+x -> smax(a, b)+x 3843207618Srdivacky // a >s b ? b+x : a+x -> smin(a, b)+x 3844207618Srdivacky if (LHS->getType() == U->getType()) { 3845207618Srdivacky const SCEV *LS = getSCEV(LHS); 3846207618Srdivacky const SCEV *RS = getSCEV(RHS); 3847207618Srdivacky const SCEV *LA = getSCEV(U->getOperand(1)); 3848207618Srdivacky const SCEV *RA = getSCEV(U->getOperand(2)); 3849207618Srdivacky const SCEV *LDiff = getMinusSCEV(LA, LS); 3850207618Srdivacky const SCEV *RDiff = getMinusSCEV(RA, RS); 3851207618Srdivacky if (LDiff == RDiff) 3852207618Srdivacky return getAddExpr(getSMaxExpr(LS, RS), LDiff); 3853207618Srdivacky LDiff = getMinusSCEV(LA, RS); 3854207618Srdivacky RDiff = getMinusSCEV(RA, LS); 3855207618Srdivacky if (LDiff == RDiff) 3856207618Srdivacky return getAddExpr(getSMinExpr(LS, RS), LDiff); 3857207618Srdivacky } 3858193323Sed break; 3859193323Sed case ICmpInst::ICMP_ULT: 3860193323Sed case ICmpInst::ICMP_ULE: 3861193323Sed std::swap(LHS, RHS); 3862193323Sed // fall through 3863193323Sed case ICmpInst::ICMP_UGT: 3864193323Sed case ICmpInst::ICMP_UGE: 3865207618Srdivacky // a >u b ? a+x : b+x -> umax(a, b)+x 3866207618Srdivacky // a >u b ? b+x : a+x -> umin(a, b)+x 3867207618Srdivacky if (LHS->getType() == U->getType()) { 3868207618Srdivacky const SCEV *LS = getSCEV(LHS); 3869207618Srdivacky const SCEV *RS = getSCEV(RHS); 3870207618Srdivacky const SCEV *LA = getSCEV(U->getOperand(1)); 3871207618Srdivacky const SCEV *RA = getSCEV(U->getOperand(2)); 3872207618Srdivacky const SCEV *LDiff = getMinusSCEV(LA, LS); 3873207618Srdivacky const SCEV *RDiff = getMinusSCEV(RA, RS); 3874207618Srdivacky if (LDiff == RDiff) 3875207618Srdivacky return getAddExpr(getUMaxExpr(LS, RS), LDiff); 3876207618Srdivacky LDiff = getMinusSCEV(LA, RS); 3877207618Srdivacky RDiff = getMinusSCEV(RA, LS); 3878207618Srdivacky if (LDiff == RDiff) 3879207618Srdivacky return getAddExpr(getUMinExpr(LS, RS), LDiff); 3880207618Srdivacky } 3881193323Sed break; 3882194612Sed case ICmpInst::ICMP_NE: 3883207618Srdivacky // n != 0 ? n+x : 1+x -> umax(n, 1)+x 3884207618Srdivacky if (LHS->getType() == U->getType() && 3885194612Sed isa<ConstantInt>(RHS) && 3886207618Srdivacky cast<ConstantInt>(RHS)->isZero()) { 3887207618Srdivacky const SCEV *One = getConstant(LHS->getType(), 1); 3888207618Srdivacky const SCEV *LS = getSCEV(LHS); 3889207618Srdivacky const SCEV *LA = getSCEV(U->getOperand(1)); 3890207618Srdivacky const SCEV *RA = getSCEV(U->getOperand(2)); 3891207618Srdivacky const SCEV *LDiff = getMinusSCEV(LA, LS); 3892207618Srdivacky const SCEV *RDiff = getMinusSCEV(RA, One); 3893207618Srdivacky if (LDiff == RDiff) 3894212904Sdim return getAddExpr(getUMaxExpr(One, LS), LDiff); 3895207618Srdivacky } 3896194612Sed break; 3897194612Sed case ICmpInst::ICMP_EQ: 3898207618Srdivacky // n == 0 ? 1+x : n+x -> umax(n, 1)+x 3899207618Srdivacky if (LHS->getType() == U->getType() && 3900194612Sed isa<ConstantInt>(RHS) && 3901207618Srdivacky cast<ConstantInt>(RHS)->isZero()) { 3902207618Srdivacky const SCEV *One = getConstant(LHS->getType(), 1); 3903207618Srdivacky const SCEV *LS = getSCEV(LHS); 3904207618Srdivacky const SCEV *LA = getSCEV(U->getOperand(1)); 3905207618Srdivacky const SCEV *RA = getSCEV(U->getOperand(2)); 3906207618Srdivacky const SCEV *LDiff = getMinusSCEV(LA, One); 3907207618Srdivacky const SCEV *RDiff = getMinusSCEV(RA, LS); 3908207618Srdivacky if (LDiff == RDiff) 3909212904Sdim return getAddExpr(getUMaxExpr(One, LS), LDiff); 3910207618Srdivacky } 3911194612Sed break; 3912193323Sed default: 3913193323Sed break; 3914193323Sed } 3915193323Sed } 3916193323Sed 3917193323Sed default: // We cannot analyze this expression. 3918193323Sed break; 3919193323Sed } 3920193323Sed 3921193323Sed return getUnknown(V); 3922193323Sed} 3923193323Sed 3924193323Sed 3925193323Sed 3926193323Sed//===----------------------------------------------------------------------===// 3927193323Sed// Iteration Count Computation Code 3928193323Sed// 3929193323Sed 3930226633Sdim/// getSmallConstantTripCount - Returns the maximum trip count of this loop as a 3931234353Sdim/// normal unsigned value. Returns 0 if the trip count is unknown or not 3932234353Sdim/// constant. Will also return 0 if the maximum trip count is very large (>= 3933234353Sdim/// 2^32). 3934234353Sdim/// 3935234353Sdim/// This "trip count" assumes that control exits via ExitingBlock. More 3936234353Sdim/// precisely, it is the number of times that control may reach ExitingBlock 3937234353Sdim/// before taking the branch. For loops with multiple exits, it may not be the 3938234353Sdim/// number times that the loop header executes because the loop may exit 3939234353Sdim/// prematurely via another branch. 3940251662Sdim/// 3941251662Sdim/// FIXME: We conservatively call getBackedgeTakenCount(L) instead of 3942251662Sdim/// getExitCount(L, ExitingBlock) to compute a safe trip count considering all 3943251662Sdim/// loop exits. getExitCount() may return an exact count for this branch 3944251662Sdim/// assuming no-signed-wrap. The number of well-defined iterations may actually 3945251662Sdim/// be higher than this trip count if this exit test is skipped and the loop 3946251662Sdim/// exits via a different branch. Ideally, getExitCount() would know whether it 3947251662Sdim/// depends on a NSW assumption, and we would only fall back to a conservative 3948251662Sdim/// trip count in that case. 3949234353Sdimunsigned ScalarEvolution:: 3950251662SdimgetSmallConstantTripCount(Loop *L, BasicBlock */*ExitingBlock*/) { 3951226633Sdim const SCEVConstant *ExitCount = 3952251662Sdim dyn_cast<SCEVConstant>(getBackedgeTakenCount(L)); 3953226633Sdim if (!ExitCount) 3954226633Sdim return 0; 3955226633Sdim 3956226633Sdim ConstantInt *ExitConst = ExitCount->getValue(); 3957226633Sdim 3958226633Sdim // Guard against huge trip counts. 3959226633Sdim if (ExitConst->getValue().getActiveBits() > 32) 3960226633Sdim return 0; 3961226633Sdim 3962226633Sdim // In case of integer overflow, this returns 0, which is correct. 3963226633Sdim return ((unsigned)ExitConst->getZExtValue()) + 1; 3964226633Sdim} 3965226633Sdim 3966226633Sdim/// getSmallConstantTripMultiple - Returns the largest constant divisor of the 3967226633Sdim/// trip count of this loop as a normal unsigned value, if possible. This 3968226633Sdim/// means that the actual trip count is always a multiple of the returned 3969226633Sdim/// value (don't forget the trip count could very well be zero as well!). 3970226633Sdim/// 3971226633Sdim/// Returns 1 if the trip count is unknown or not guaranteed to be the 3972226633Sdim/// multiple of a constant (which is also the case if the trip count is simply 3973226633Sdim/// constant, use getSmallConstantTripCount for that case), Will also return 1 3974226633Sdim/// if the trip count is very large (>= 2^32). 3975234353Sdim/// 3976234353Sdim/// As explained in the comments for getSmallConstantTripCount, this assumes 3977234353Sdim/// that control exits the loop via ExitingBlock. 3978234353Sdimunsigned ScalarEvolution:: 3979251662SdimgetSmallConstantTripMultiple(Loop *L, BasicBlock */*ExitingBlock*/) { 3980251662Sdim const SCEV *ExitCount = getBackedgeTakenCount(L); 3981226633Sdim if (ExitCount == getCouldNotCompute()) 3982226633Sdim return 1; 3983226633Sdim 3984226633Sdim // Get the trip count from the BE count by adding 1. 3985226633Sdim const SCEV *TCMul = getAddExpr(ExitCount, 3986226633Sdim getConstant(ExitCount->getType(), 1)); 3987226633Sdim // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt 3988226633Sdim // to factor simple cases. 3989226633Sdim if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul)) 3990226633Sdim TCMul = Mul->getOperand(0); 3991226633Sdim 3992226633Sdim const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul); 3993226633Sdim if (!MulC) 3994226633Sdim return 1; 3995226633Sdim 3996226633Sdim ConstantInt *Result = MulC->getValue(); 3997226633Sdim 3998243830Sdim // Guard against huge trip counts (this requires checking 3999243830Sdim // for zero to handle the case where the trip count == -1 and the 4000243830Sdim // addition wraps). 4001243830Sdim if (!Result || Result->getValue().getActiveBits() > 32 || 4002243830Sdim Result->getValue().getActiveBits() == 0) 4003226633Sdim return 1; 4004226633Sdim 4005226633Sdim return (unsigned)Result->getZExtValue(); 4006226633Sdim} 4007226633Sdim 4008226633Sdim// getExitCount - Get the expression for the number of loop iterations for which 4009251662Sdim// this loop is guaranteed not to exit via ExitingBlock. Otherwise return 4010226633Sdim// SCEVCouldNotCompute. 4011226633Sdimconst SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) { 4012226633Sdim return getBackedgeTakenInfo(L).getExact(ExitingBlock, this); 4013226633Sdim} 4014226633Sdim 4015193323Sed/// getBackedgeTakenCount - If the specified loop has a predictable 4016193323Sed/// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute 4017193323Sed/// object. The backedge-taken count is the number of times the loop header 4018193323Sed/// will be branched to from within the loop. This is one less than the 4019193323Sed/// trip count of the loop, since it doesn't count the first iteration, 4020193323Sed/// when the header is branched to from outside the loop. 4021193323Sed/// 4022193323Sed/// Note that it is not valid to call this method on a loop without a 4023193323Sed/// loop-invariant backedge-taken count (see 4024193323Sed/// hasLoopInvariantBackedgeTakenCount). 4025193323Sed/// 4026198090Srdivackyconst SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) { 4027226633Sdim return getBackedgeTakenInfo(L).getExact(this); 4028193323Sed} 4029193323Sed 4030193323Sed/// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except 4031193323Sed/// return the least SCEV value that is known never to be less than the 4032193323Sed/// actual backedge taken count. 4033198090Srdivackyconst SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) { 4034226633Sdim return getBackedgeTakenInfo(L).getMax(this); 4035193323Sed} 4036193323Sed 4037198090Srdivacky/// PushLoopPHIs - Push PHI nodes in the header of the given loop 4038198090Srdivacky/// onto the given Worklist. 4039198090Srdivackystatic void 4040198090SrdivackyPushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) { 4041198090Srdivacky BasicBlock *Header = L->getHeader(); 4042198090Srdivacky 4043198090Srdivacky // Push all Loop-header PHIs onto the Worklist stack. 4044198090Srdivacky for (BasicBlock::iterator I = Header->begin(); 4045198090Srdivacky PHINode *PN = dyn_cast<PHINode>(I); ++I) 4046198090Srdivacky Worklist.push_back(PN); 4047198090Srdivacky} 4048198090Srdivacky 4049193323Sedconst ScalarEvolution::BackedgeTakenInfo & 4050193323SedScalarEvolution::getBackedgeTakenInfo(const Loop *L) { 4051226633Sdim // Initially insert an invalid entry for this loop. If the insertion 4052204642Srdivacky // succeeds, proceed to actually compute a backedge-taken count and 4053193323Sed // update the value. The temporary CouldNotCompute value tells SCEV 4054193323Sed // code elsewhere that it shouldn't attempt to request a new 4055193323Sed // backedge-taken count, which could result in infinite recursion. 4056223017Sdim std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair = 4057226633Sdim BackedgeTakenCounts.insert(std::make_pair(L, BackedgeTakenInfo())); 4058218893Sdim if (!Pair.second) 4059218893Sdim return Pair.first->second; 4060193323Sed 4061226633Sdim // ComputeBackedgeTakenCount may allocate memory for its result. Inserting it 4062226633Sdim // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result 4063226633Sdim // must be cleared in this scope. 4064226633Sdim BackedgeTakenInfo Result = ComputeBackedgeTakenCount(L); 4065226633Sdim 4066226633Sdim if (Result.getExact(this) != getCouldNotCompute()) { 4067226633Sdim assert(isLoopInvariant(Result.getExact(this), L) && 4068226633Sdim isLoopInvariant(Result.getMax(this), L) && 4069218893Sdim "Computed backedge-taken count isn't loop invariant for loop!"); 4070218893Sdim ++NumTripCountsComputed; 4071218893Sdim } 4072226633Sdim else if (Result.getMax(this) == getCouldNotCompute() && 4073226633Sdim isa<PHINode>(L->getHeader()->begin())) { 4074226633Sdim // Only count loops that have phi nodes as not being computable. 4075226633Sdim ++NumTripCountsNotComputed; 4076226633Sdim } 4077193323Sed 4078218893Sdim // Now that we know more about the trip count for this loop, forget any 4079218893Sdim // existing SCEV values for PHI nodes in this loop since they are only 4080218893Sdim // conservative estimates made without the benefit of trip count 4081218893Sdim // information. This is similar to the code in forgetLoop, except that 4082218893Sdim // it handles SCEVUnknown PHI nodes specially. 4083226633Sdim if (Result.hasAnyInfo()) { 4084218893Sdim SmallVector<Instruction *, 16> Worklist; 4085218893Sdim PushLoopPHIs(L, Worklist); 4086198090Srdivacky 4087218893Sdim SmallPtrSet<Instruction *, 8> Visited; 4088218893Sdim while (!Worklist.empty()) { 4089218893Sdim Instruction *I = Worklist.pop_back_val(); 4090218893Sdim if (!Visited.insert(I)) continue; 4091198090Srdivacky 4092218893Sdim ValueExprMapType::iterator It = 4093239462Sdim ValueExprMap.find_as(static_cast<Value *>(I)); 4094218893Sdim if (It != ValueExprMap.end()) { 4095218893Sdim const SCEV *Old = It->second; 4096218893Sdim 4097218893Sdim // SCEVUnknown for a PHI either means that it has an unrecognized 4098218893Sdim // structure, or it's a PHI that's in the progress of being computed 4099218893Sdim // by createNodeForPHI. In the former case, additional loop trip 4100218893Sdim // count information isn't going to change anything. In the later 4101218893Sdim // case, createNodeForPHI will perform the necessary updates on its 4102218893Sdim // own when it gets to that point. 4103218893Sdim if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) { 4104218893Sdim forgetMemoizedResults(Old); 4105218893Sdim ValueExprMap.erase(It); 4106198090Srdivacky } 4107218893Sdim if (PHINode *PN = dyn_cast<PHINode>(I)) 4108218893Sdim ConstantEvolutionLoopExitValue.erase(PN); 4109218893Sdim } 4110198090Srdivacky 4111218893Sdim PushDefUseChildren(I, Worklist); 4112198090Srdivacky } 4113193323Sed } 4114221345Sdim 4115221345Sdim // Re-lookup the insert position, since the call to 4116221345Sdim // ComputeBackedgeTakenCount above could result in a 4117221345Sdim // recusive call to getBackedgeTakenInfo (on a different 4118221345Sdim // loop), which would invalidate the iterator computed 4119221345Sdim // earlier. 4120221345Sdim return BackedgeTakenCounts.find(L)->second = Result; 4121193323Sed} 4122193323Sed 4123198892Srdivacky/// forgetLoop - This method should be called by the client when it has 4124198892Srdivacky/// changed a loop in a way that may effect ScalarEvolution's ability to 4125198892Srdivacky/// compute a trip count, or if the loop is deleted. 4126198892Srdivackyvoid ScalarEvolution::forgetLoop(const Loop *L) { 4127198892Srdivacky // Drop any stored trip count value. 4128226633Sdim DenseMap<const Loop*, BackedgeTakenInfo>::iterator BTCPos = 4129226633Sdim BackedgeTakenCounts.find(L); 4130226633Sdim if (BTCPos != BackedgeTakenCounts.end()) { 4131226633Sdim BTCPos->second.clear(); 4132226633Sdim BackedgeTakenCounts.erase(BTCPos); 4133226633Sdim } 4134193323Sed 4135198892Srdivacky // Drop information about expressions based on loop-header PHIs. 4136193323Sed SmallVector<Instruction *, 16> Worklist; 4137198090Srdivacky PushLoopPHIs(L, Worklist); 4138193323Sed 4139198090Srdivacky SmallPtrSet<Instruction *, 8> Visited; 4140193323Sed while (!Worklist.empty()) { 4141193323Sed Instruction *I = Worklist.pop_back_val(); 4142198090Srdivacky if (!Visited.insert(I)) continue; 4143198090Srdivacky 4144239462Sdim ValueExprMapType::iterator It = 4145239462Sdim ValueExprMap.find_as(static_cast<Value *>(I)); 4146212904Sdim if (It != ValueExprMap.end()) { 4147218893Sdim forgetMemoizedResults(It->second); 4148212904Sdim ValueExprMap.erase(It); 4149198090Srdivacky if (PHINode *PN = dyn_cast<PHINode>(I)) 4150198090Srdivacky ConstantEvolutionLoopExitValue.erase(PN); 4151198090Srdivacky } 4152198090Srdivacky 4153198090Srdivacky PushDefUseChildren(I, Worklist); 4154193323Sed } 4155218893Sdim 4156218893Sdim // Forget all contained loops too, to avoid dangling entries in the 4157218893Sdim // ValuesAtScopes map. 4158218893Sdim for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 4159218893Sdim forgetLoop(*I); 4160193323Sed} 4161193323Sed 4162204642Srdivacky/// forgetValue - This method should be called by the client when it has 4163204642Srdivacky/// changed a value in a way that may effect its value, or which may 4164204642Srdivacky/// disconnect it from a def-use chain linking it to a loop. 4165204642Srdivackyvoid ScalarEvolution::forgetValue(Value *V) { 4166204642Srdivacky Instruction *I = dyn_cast<Instruction>(V); 4167204642Srdivacky if (!I) return; 4168204642Srdivacky 4169204642Srdivacky // Drop information about expressions based on loop-header PHIs. 4170204642Srdivacky SmallVector<Instruction *, 16> Worklist; 4171204642Srdivacky Worklist.push_back(I); 4172204642Srdivacky 4173204642Srdivacky SmallPtrSet<Instruction *, 8> Visited; 4174204642Srdivacky while (!Worklist.empty()) { 4175204642Srdivacky I = Worklist.pop_back_val(); 4176204642Srdivacky if (!Visited.insert(I)) continue; 4177204642Srdivacky 4178239462Sdim ValueExprMapType::iterator It = 4179239462Sdim ValueExprMap.find_as(static_cast<Value *>(I)); 4180212904Sdim if (It != ValueExprMap.end()) { 4181218893Sdim forgetMemoizedResults(It->second); 4182212904Sdim ValueExprMap.erase(It); 4183204642Srdivacky if (PHINode *PN = dyn_cast<PHINode>(I)) 4184204642Srdivacky ConstantEvolutionLoopExitValue.erase(PN); 4185204642Srdivacky } 4186204642Srdivacky 4187204642Srdivacky PushDefUseChildren(I, Worklist); 4188204642Srdivacky } 4189204642Srdivacky} 4190204642Srdivacky 4191226633Sdim/// getExact - Get the exact loop backedge taken count considering all loop 4192234353Sdim/// exits. A computable result can only be return for loops with a single exit. 4193234353Sdim/// Returning the minimum taken count among all exits is incorrect because one 4194234353Sdim/// of the loop's exit limit's may have been skipped. HowFarToZero assumes that 4195234353Sdim/// the limit of each loop test is never skipped. This is a valid assumption as 4196234353Sdim/// long as the loop exits via that test. For precise results, it is the 4197234353Sdim/// caller's responsibility to specify the relevant loop exit using 4198234353Sdim/// getExact(ExitingBlock, SE). 4199226633Sdimconst SCEV * 4200226633SdimScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const { 4201226633Sdim // If any exits were not computable, the loop is not computable. 4202226633Sdim if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute(); 4203226633Sdim 4204234353Sdim // We need exactly one computable exit. 4205226633Sdim if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute(); 4206226633Sdim assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info"); 4207226633Sdim 4208226633Sdim const SCEV *BECount = 0; 4209226633Sdim for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4210226633Sdim ENT != 0; ENT = ENT->getNextExit()) { 4211226633Sdim 4212226633Sdim assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV"); 4213226633Sdim 4214226633Sdim if (!BECount) 4215226633Sdim BECount = ENT->ExactNotTaken; 4216234353Sdim else if (BECount != ENT->ExactNotTaken) 4217234353Sdim return SE->getCouldNotCompute(); 4218226633Sdim } 4219226633Sdim assert(BECount && "Invalid not taken count for loop exit"); 4220226633Sdim return BECount; 4221226633Sdim} 4222226633Sdim 4223226633Sdim/// getExact - Get the exact not taken count for this loop exit. 4224226633Sdimconst SCEV * 4225226633SdimScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock, 4226226633Sdim ScalarEvolution *SE) const { 4227226633Sdim for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4228226633Sdim ENT != 0; ENT = ENT->getNextExit()) { 4229226633Sdim 4230226633Sdim if (ENT->ExitingBlock == ExitingBlock) 4231226633Sdim return ENT->ExactNotTaken; 4232226633Sdim } 4233226633Sdim return SE->getCouldNotCompute(); 4234226633Sdim} 4235226633Sdim 4236226633Sdim/// getMax - Get the max backedge taken count for the loop. 4237226633Sdimconst SCEV * 4238226633SdimScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const { 4239226633Sdim return Max ? Max : SE->getCouldNotCompute(); 4240226633Sdim} 4241226633Sdim 4242249423Sdimbool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S, 4243249423Sdim ScalarEvolution *SE) const { 4244249423Sdim if (Max && Max != SE->getCouldNotCompute() && SE->hasOperand(Max, S)) 4245249423Sdim return true; 4246249423Sdim 4247249423Sdim if (!ExitNotTaken.ExitingBlock) 4248249423Sdim return false; 4249249423Sdim 4250249423Sdim for (const ExitNotTakenInfo *ENT = &ExitNotTaken; 4251249423Sdim ENT != 0; ENT = ENT->getNextExit()) { 4252249423Sdim 4253249423Sdim if (ENT->ExactNotTaken != SE->getCouldNotCompute() 4254249423Sdim && SE->hasOperand(ENT->ExactNotTaken, S)) { 4255249423Sdim return true; 4256249423Sdim } 4257249423Sdim } 4258249423Sdim return false; 4259249423Sdim} 4260249423Sdim 4261226633Sdim/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each 4262226633Sdim/// computable exit into a persistent ExitNotTakenInfo array. 4263226633SdimScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( 4264226633Sdim SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts, 4265226633Sdim bool Complete, const SCEV *MaxCount) : Max(MaxCount) { 4266226633Sdim 4267226633Sdim if (!Complete) 4268226633Sdim ExitNotTaken.setIncomplete(); 4269226633Sdim 4270226633Sdim unsigned NumExits = ExitCounts.size(); 4271226633Sdim if (NumExits == 0) return; 4272226633Sdim 4273226633Sdim ExitNotTaken.ExitingBlock = ExitCounts[0].first; 4274226633Sdim ExitNotTaken.ExactNotTaken = ExitCounts[0].second; 4275226633Sdim if (NumExits == 1) return; 4276226633Sdim 4277226633Sdim // Handle the rare case of multiple computable exits. 4278226633Sdim ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1]; 4279226633Sdim 4280226633Sdim ExitNotTakenInfo *PrevENT = &ExitNotTaken; 4281226633Sdim for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) { 4282226633Sdim PrevENT->setNextExit(ENT); 4283226633Sdim ENT->ExitingBlock = ExitCounts[i].first; 4284226633Sdim ENT->ExactNotTaken = ExitCounts[i].second; 4285226633Sdim } 4286226633Sdim} 4287226633Sdim 4288226633Sdim/// clear - Invalidate this result and free the ExitNotTakenInfo array. 4289226633Sdimvoid ScalarEvolution::BackedgeTakenInfo::clear() { 4290226633Sdim ExitNotTaken.ExitingBlock = 0; 4291226633Sdim ExitNotTaken.ExactNotTaken = 0; 4292226633Sdim delete[] ExitNotTaken.getNextExit(); 4293226633Sdim} 4294226633Sdim 4295193323Sed/// ComputeBackedgeTakenCount - Compute the number of times the backedge 4296193323Sed/// of the specified loop will execute. 4297193323SedScalarEvolution::BackedgeTakenInfo 4298193323SedScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) { 4299201360Srdivacky SmallVector<BasicBlock *, 8> ExitingBlocks; 4300194612Sed L->getExitingBlocks(ExitingBlocks); 4301193323Sed 4302194612Sed // Examine all exits and pick the most conservative values. 4303198090Srdivacky const SCEV *MaxBECount = getCouldNotCompute(); 4304226633Sdim bool CouldComputeBECount = true; 4305226633Sdim SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts; 4306194612Sed for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { 4307226633Sdim ExitLimit EL = ComputeExitLimit(L, ExitingBlocks[i]); 4308226633Sdim if (EL.Exact == getCouldNotCompute()) 4309194612Sed // We couldn't compute an exact value for this exit, so 4310194710Sed // we won't be able to compute an exact value for the loop. 4311226633Sdim CouldComputeBECount = false; 4312226633Sdim else 4313226633Sdim ExitCounts.push_back(std::make_pair(ExitingBlocks[i], EL.Exact)); 4314226633Sdim 4315195340Sed if (MaxBECount == getCouldNotCompute()) 4316226633Sdim MaxBECount = EL.Max; 4317234353Sdim else if (EL.Max != getCouldNotCompute()) { 4318234353Sdim // We cannot take the "min" MaxBECount, because non-unit stride loops may 4319234353Sdim // skip some loop tests. Taking the max over the exits is sufficiently 4320234353Sdim // conservative. TODO: We could do better taking into consideration 4321234353Sdim // that (1) the loop has unit stride (2) the last loop test is 4322234353Sdim // less-than/greater-than (3) any loop test is less-than/greater-than AND 4323234353Sdim // falls-through some constant times less then the other tests. 4324234353Sdim MaxBECount = getUMaxFromMismatchedTypes(MaxBECount, EL.Max); 4325234353Sdim } 4326194612Sed } 4327194612Sed 4328226633Sdim return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount); 4329194612Sed} 4330194612Sed 4331226633Sdim/// ComputeExitLimit - Compute the number of times the backedge of the specified 4332226633Sdim/// loop will execute if it exits via the specified block. 4333226633SdimScalarEvolution::ExitLimit 4334226633SdimScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) { 4335194612Sed 4336194612Sed // Okay, we've chosen an exiting block. See what condition causes us to 4337194612Sed // exit at this block. 4338193323Sed // 4339193323Sed // FIXME: we should be able to handle switch instructions (with a single exit) 4340193323Sed BranchInst *ExitBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); 4341195340Sed if (ExitBr == 0) return getCouldNotCompute(); 4342193323Sed assert(ExitBr->isConditional() && "If unconditional, it can't be in loop!"); 4343195098Sed 4344193323Sed // At this point, we know we have a conditional branch that determines whether 4345193323Sed // the loop is exited. However, we don't know if the branch is executed each 4346193323Sed // time through the loop. If not, then the execution count of the branch will 4347193323Sed // not be equal to the trip count of the loop. 4348193323Sed // 4349193323Sed // Currently we check for this by checking to see if the Exit branch goes to 4350193323Sed // the loop header. If so, we know it will always execute the same number of 4351193323Sed // times as the loop. We also handle the case where the exit block *is* the 4352194612Sed // loop header. This is common for un-rotated loops. 4353194612Sed // 4354194612Sed // If both of those tests fail, walk up the unique predecessor chain to the 4355194612Sed // header, stopping if there is an edge that doesn't exit the loop. If the 4356194612Sed // header is reached, the execution count of the branch will be equal to the 4357194612Sed // trip count of the loop. 4358194612Sed // 4359194612Sed // More extensive analysis could be done to handle more cases here. 4360194612Sed // 4361193323Sed if (ExitBr->getSuccessor(0) != L->getHeader() && 4362193323Sed ExitBr->getSuccessor(1) != L->getHeader() && 4363194612Sed ExitBr->getParent() != L->getHeader()) { 4364194612Sed // The simple checks failed, try climbing the unique predecessor chain 4365194612Sed // up to the header. 4366194612Sed bool Ok = false; 4367194612Sed for (BasicBlock *BB = ExitBr->getParent(); BB; ) { 4368194612Sed BasicBlock *Pred = BB->getUniquePredecessor(); 4369194612Sed if (!Pred) 4370195340Sed return getCouldNotCompute(); 4371194612Sed TerminatorInst *PredTerm = Pred->getTerminator(); 4372194612Sed for (unsigned i = 0, e = PredTerm->getNumSuccessors(); i != e; ++i) { 4373194612Sed BasicBlock *PredSucc = PredTerm->getSuccessor(i); 4374194612Sed if (PredSucc == BB) 4375194612Sed continue; 4376194612Sed // If the predecessor has a successor that isn't BB and isn't 4377194612Sed // outside the loop, assume the worst. 4378194612Sed if (L->contains(PredSucc)) 4379195340Sed return getCouldNotCompute(); 4380194612Sed } 4381194612Sed if (Pred == L->getHeader()) { 4382194612Sed Ok = true; 4383194612Sed break; 4384194612Sed } 4385194612Sed BB = Pred; 4386194612Sed } 4387194612Sed if (!Ok) 4388195340Sed return getCouldNotCompute(); 4389194612Sed } 4390193323Sed 4391204642Srdivacky // Proceed to the next level to examine the exit condition expression. 4392226633Sdim return ComputeExitLimitFromCond(L, ExitBr->getCondition(), 4393226633Sdim ExitBr->getSuccessor(0), 4394251662Sdim ExitBr->getSuccessor(1), 4395251662Sdim /*IsSubExpr=*/false); 4396194612Sed} 4397194612Sed 4398226633Sdim/// ComputeExitLimitFromCond - Compute the number of times the 4399194612Sed/// backedge of the specified loop will execute if its exit condition 4400194612Sed/// were a conditional branch of ExitCond, TBB, and FBB. 4401251662Sdim/// 4402251662Sdim/// @param IsSubExpr is true if ExitCond does not directly control the exit 4403251662Sdim/// branch. In this case, we cannot assume that the loop only exits when the 4404251662Sdim/// condition is true and cannot infer that failing to meet the condition prior 4405251662Sdim/// to integer wraparound results in undefined behavior. 4406226633SdimScalarEvolution::ExitLimit 4407226633SdimScalarEvolution::ComputeExitLimitFromCond(const Loop *L, 4408226633Sdim Value *ExitCond, 4409226633Sdim BasicBlock *TBB, 4410251662Sdim BasicBlock *FBB, 4411251662Sdim bool IsSubExpr) { 4412195098Sed // Check if the controlling expression for this loop is an And or Or. 4413194612Sed if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) { 4414194612Sed if (BO->getOpcode() == Instruction::And) { 4415194612Sed // Recurse on the operands of the and. 4416251662Sdim bool EitherMayExit = L->contains(TBB); 4417251662Sdim ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 4418251662Sdim IsSubExpr || EitherMayExit); 4419251662Sdim ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 4420251662Sdim IsSubExpr || EitherMayExit); 4421198090Srdivacky const SCEV *BECount = getCouldNotCompute(); 4422198090Srdivacky const SCEV *MaxBECount = getCouldNotCompute(); 4423251662Sdim if (EitherMayExit) { 4424194612Sed // Both conditions must be true for the loop to continue executing. 4425194612Sed // Choose the less conservative count. 4426226633Sdim if (EL0.Exact == getCouldNotCompute() || 4427226633Sdim EL1.Exact == getCouldNotCompute()) 4428195340Sed BECount = getCouldNotCompute(); 4429194710Sed else 4430226633Sdim BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 4431226633Sdim if (EL0.Max == getCouldNotCompute()) 4432226633Sdim MaxBECount = EL1.Max; 4433226633Sdim else if (EL1.Max == getCouldNotCompute()) 4434226633Sdim MaxBECount = EL0.Max; 4435194710Sed else 4436226633Sdim MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 4437194612Sed } else { 4438212904Sdim // Both conditions must be true at the same time for the loop to exit. 4439212904Sdim // For now, be conservative. 4440194612Sed assert(L->contains(FBB) && "Loop block has no successor in loop!"); 4441226633Sdim if (EL0.Max == EL1.Max) 4442226633Sdim MaxBECount = EL0.Max; 4443226633Sdim if (EL0.Exact == EL1.Exact) 4444226633Sdim BECount = EL0.Exact; 4445194612Sed } 4446194612Sed 4447226633Sdim return ExitLimit(BECount, MaxBECount); 4448194612Sed } 4449194612Sed if (BO->getOpcode() == Instruction::Or) { 4450194612Sed // Recurse on the operands of the or. 4451251662Sdim bool EitherMayExit = L->contains(FBB); 4452251662Sdim ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, 4453251662Sdim IsSubExpr || EitherMayExit); 4454251662Sdim ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, 4455251662Sdim IsSubExpr || EitherMayExit); 4456198090Srdivacky const SCEV *BECount = getCouldNotCompute(); 4457198090Srdivacky const SCEV *MaxBECount = getCouldNotCompute(); 4458251662Sdim if (EitherMayExit) { 4459194612Sed // Both conditions must be false for the loop to continue executing. 4460194612Sed // Choose the less conservative count. 4461226633Sdim if (EL0.Exact == getCouldNotCompute() || 4462226633Sdim EL1.Exact == getCouldNotCompute()) 4463195340Sed BECount = getCouldNotCompute(); 4464194710Sed else 4465226633Sdim BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact); 4466226633Sdim if (EL0.Max == getCouldNotCompute()) 4467226633Sdim MaxBECount = EL1.Max; 4468226633Sdim else if (EL1.Max == getCouldNotCompute()) 4469226633Sdim MaxBECount = EL0.Max; 4470194710Sed else 4471226633Sdim MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max); 4472194612Sed } else { 4473212904Sdim // Both conditions must be false at the same time for the loop to exit. 4474212904Sdim // For now, be conservative. 4475194612Sed assert(L->contains(TBB) && "Loop block has no successor in loop!"); 4476226633Sdim if (EL0.Max == EL1.Max) 4477226633Sdim MaxBECount = EL0.Max; 4478226633Sdim if (EL0.Exact == EL1.Exact) 4479226633Sdim BECount = EL0.Exact; 4480194612Sed } 4481194612Sed 4482226633Sdim return ExitLimit(BECount, MaxBECount); 4483194612Sed } 4484194612Sed } 4485194612Sed 4486194612Sed // With an icmp, it may be feasible to compute an exact backedge-taken count. 4487204642Srdivacky // Proceed to the next level to examine the icmp. 4488194612Sed if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) 4489251662Sdim return ComputeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, IsSubExpr); 4490194612Sed 4491204642Srdivacky // Check for a constant condition. These are normally stripped out by 4492204642Srdivacky // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to 4493204642Srdivacky // preserve the CFG and is temporarily leaving constant conditions 4494204642Srdivacky // in place. 4495204642Srdivacky if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) { 4496204642Srdivacky if (L->contains(FBB) == !CI->getZExtValue()) 4497204642Srdivacky // The backedge is always taken. 4498204642Srdivacky return getCouldNotCompute(); 4499204642Srdivacky else 4500204642Srdivacky // The backedge is never taken. 4501207618Srdivacky return getConstant(CI->getType(), 0); 4502204642Srdivacky } 4503204642Srdivacky 4504193323Sed // If it's not an integer or pointer comparison then compute it the hard way. 4505226633Sdim return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 4506194612Sed} 4507193323Sed 4508226633Sdim/// ComputeExitLimitFromICmp - Compute the number of times the 4509194612Sed/// backedge of the specified loop will execute if its exit condition 4510194612Sed/// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB. 4511226633SdimScalarEvolution::ExitLimit 4512226633SdimScalarEvolution::ComputeExitLimitFromICmp(const Loop *L, 4513226633Sdim ICmpInst *ExitCond, 4514226633Sdim BasicBlock *TBB, 4515251662Sdim BasicBlock *FBB, 4516251662Sdim bool IsSubExpr) { 4517194612Sed 4518193323Sed // If the condition was exit on true, convert the condition to exit on false 4519193323Sed ICmpInst::Predicate Cond; 4520194612Sed if (!L->contains(FBB)) 4521193323Sed Cond = ExitCond->getPredicate(); 4522193323Sed else 4523193323Sed Cond = ExitCond->getInversePredicate(); 4524193323Sed 4525193323Sed // Handle common loops like: for (X = "string"; *X; ++X) 4526193323Sed if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0))) 4527193323Sed if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) { 4528226633Sdim ExitLimit ItCnt = 4529226633Sdim ComputeLoadConstantCompareExitLimit(LI, RHS, L, Cond); 4530204642Srdivacky if (ItCnt.hasAnyInfo()) 4531204642Srdivacky return ItCnt; 4532193323Sed } 4533193323Sed 4534198090Srdivacky const SCEV *LHS = getSCEV(ExitCond->getOperand(0)); 4535198090Srdivacky const SCEV *RHS = getSCEV(ExitCond->getOperand(1)); 4536193323Sed 4537193323Sed // Try to evaluate any dependencies out of the loop. 4538193323Sed LHS = getSCEVAtScope(LHS, L); 4539193323Sed RHS = getSCEVAtScope(RHS, L); 4540193323Sed 4541195098Sed // At this point, we would like to compute how many iterations of the 4542193323Sed // loop the predicate will return true for these inputs. 4543218893Sdim if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) { 4544193323Sed // If there is a loop-invariant, force it into the RHS. 4545193323Sed std::swap(LHS, RHS); 4546193323Sed Cond = ICmpInst::getSwappedPredicate(Cond); 4547193323Sed } 4548193323Sed 4549207618Srdivacky // Simplify the operands before analyzing them. 4550207618Srdivacky (void)SimplifyICmpOperands(Cond, LHS, RHS); 4551207618Srdivacky 4552193323Sed // If we have a comparison of a chrec against a constant, try to use value 4553193323Sed // ranges to answer this query. 4554193323Sed if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) 4555193323Sed if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS)) 4556193323Sed if (AddRec->getLoop() == L) { 4557193323Sed // Form the constant range. 4558193323Sed ConstantRange CompRange( 4559193323Sed ICmpInst::makeConstantRange(Cond, RHSC->getValue()->getValue())); 4560193323Sed 4561198090Srdivacky const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this); 4562193323Sed if (!isa<SCEVCouldNotCompute>(Ret)) return Ret; 4563193323Sed } 4564193323Sed 4565193323Sed switch (Cond) { 4566193323Sed case ICmpInst::ICMP_NE: { // while (X != Y) 4567193323Sed // Convert to: while (X-Y != 0) 4568251662Sdim ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, IsSubExpr); 4569226633Sdim if (EL.hasAnyInfo()) return EL; 4570193323Sed break; 4571193323Sed } 4572198090Srdivacky case ICmpInst::ICMP_EQ: { // while (X == Y) 4573198090Srdivacky // Convert to: while (X-Y == 0) 4574226633Sdim ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L); 4575226633Sdim if (EL.hasAnyInfo()) return EL; 4576193323Sed break; 4577193323Sed } 4578193323Sed case ICmpInst::ICMP_SLT: { 4579251662Sdim ExitLimit EL = HowManyLessThans(LHS, RHS, L, true, IsSubExpr); 4580226633Sdim if (EL.hasAnyInfo()) return EL; 4581193323Sed break; 4582193323Sed } 4583193323Sed case ICmpInst::ICMP_SGT: { 4584226633Sdim ExitLimit EL = HowManyLessThans(getNotSCEV(LHS), 4585251662Sdim getNotSCEV(RHS), L, true, IsSubExpr); 4586226633Sdim if (EL.hasAnyInfo()) return EL; 4587193323Sed break; 4588193323Sed } 4589193323Sed case ICmpInst::ICMP_ULT: { 4590251662Sdim ExitLimit EL = HowManyLessThans(LHS, RHS, L, false, IsSubExpr); 4591226633Sdim if (EL.hasAnyInfo()) return EL; 4592193323Sed break; 4593193323Sed } 4594193323Sed case ICmpInst::ICMP_UGT: { 4595226633Sdim ExitLimit EL = HowManyLessThans(getNotSCEV(LHS), 4596251662Sdim getNotSCEV(RHS), L, false, IsSubExpr); 4597226633Sdim if (EL.hasAnyInfo()) return EL; 4598193323Sed break; 4599193323Sed } 4600193323Sed default: 4601193323Sed#if 0 4602201360Srdivacky dbgs() << "ComputeBackedgeTakenCount "; 4603193323Sed if (ExitCond->getOperand(0)->getType()->isUnsigned()) 4604201360Srdivacky dbgs() << "[unsigned] "; 4605201360Srdivacky dbgs() << *LHS << " " 4606195098Sed << Instruction::getOpcodeName(Instruction::ICmp) 4607193323Sed << " " << *RHS << "\n"; 4608193323Sed#endif 4609193323Sed break; 4610193323Sed } 4611226633Sdim return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB)); 4612193323Sed} 4613193323Sed 4614193323Sedstatic ConstantInt * 4615193323SedEvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, 4616193323Sed ScalarEvolution &SE) { 4617198090Srdivacky const SCEV *InVal = SE.getConstant(C); 4618198090Srdivacky const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE); 4619193323Sed assert(isa<SCEVConstant>(Val) && 4620193323Sed "Evaluation of SCEV at constant didn't fold correctly?"); 4621193323Sed return cast<SCEVConstant>(Val)->getValue(); 4622193323Sed} 4623193323Sed 4624226633Sdim/// ComputeLoadConstantCompareExitLimit - Given an exit condition of 4625193323Sed/// 'icmp op load X, cst', try to see if we can compute the backedge 4626193323Sed/// execution count. 4627226633SdimScalarEvolution::ExitLimit 4628226633SdimScalarEvolution::ComputeLoadConstantCompareExitLimit( 4629226633Sdim LoadInst *LI, 4630226633Sdim Constant *RHS, 4631226633Sdim const Loop *L, 4632226633Sdim ICmpInst::Predicate predicate) { 4633226633Sdim 4634195340Sed if (LI->isVolatile()) return getCouldNotCompute(); 4635193323Sed 4636193323Sed // Check to see if the loaded pointer is a getelementptr of a global. 4637204642Srdivacky // TODO: Use SCEV instead of manually grubbing with GEPs. 4638193323Sed GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0)); 4639195340Sed if (!GEP) return getCouldNotCompute(); 4640193323Sed 4641193323Sed // Make sure that it is really a constant global we are gepping, with an 4642193323Sed // initializer, and make sure the first IDX is really 0. 4643193323Sed GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0)); 4644198090Srdivacky if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() || 4645193323Sed GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) || 4646193323Sed !cast<Constant>(GEP->getOperand(1))->isNullValue()) 4647195340Sed return getCouldNotCompute(); 4648193323Sed 4649193323Sed // Okay, we allow one non-constant index into the GEP instruction. 4650193323Sed Value *VarIdx = 0; 4651234353Sdim std::vector<Constant*> Indexes; 4652193323Sed unsigned VarIdxNum = 0; 4653193323Sed for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i) 4654193323Sed if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) { 4655193323Sed Indexes.push_back(CI); 4656193323Sed } else if (!isa<ConstantInt>(GEP->getOperand(i))) { 4657195340Sed if (VarIdx) return getCouldNotCompute(); // Multiple non-constant idx's. 4658193323Sed VarIdx = GEP->getOperand(i); 4659193323Sed VarIdxNum = i-2; 4660193323Sed Indexes.push_back(0); 4661193323Sed } 4662193323Sed 4663234353Sdim // Loop-invariant loads may be a byproduct of loop optimization. Skip them. 4664234353Sdim if (!VarIdx) 4665234353Sdim return getCouldNotCompute(); 4666234353Sdim 4667193323Sed // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant. 4668193323Sed // Check to see if X is a loop variant variable value now. 4669198090Srdivacky const SCEV *Idx = getSCEV(VarIdx); 4670193323Sed Idx = getSCEVAtScope(Idx, L); 4671193323Sed 4672193323Sed // We can only recognize very limited forms of loop index expressions, in 4673193323Sed // particular, only affine AddRec's like {C1,+,C2}. 4674193323Sed const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx); 4675218893Sdim if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) || 4676193323Sed !isa<SCEVConstant>(IdxExpr->getOperand(0)) || 4677193323Sed !isa<SCEVConstant>(IdxExpr->getOperand(1))) 4678195340Sed return getCouldNotCompute(); 4679193323Sed 4680193323Sed unsigned MaxSteps = MaxBruteForceIterations; 4681193323Sed for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) { 4682198090Srdivacky ConstantInt *ItCst = ConstantInt::get( 4683198090Srdivacky cast<IntegerType>(IdxExpr->getType()), IterationNum); 4684193323Sed ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this); 4685193323Sed 4686193323Sed // Form the GEP offset. 4687193323Sed Indexes[VarIdxNum] = Val; 4688193323Sed 4689234353Sdim Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(), 4690234353Sdim Indexes); 4691193323Sed if (Result == 0) break; // Cannot compute! 4692193323Sed 4693193323Sed // Evaluate the condition for this iteration. 4694193323Sed Result = ConstantExpr::getICmp(predicate, Result, RHS); 4695193323Sed if (!isa<ConstantInt>(Result)) break; // Couldn't decide for sure 4696193323Sed if (cast<ConstantInt>(Result)->getValue().isMinValue()) { 4697193323Sed#if 0 4698201360Srdivacky dbgs() << "\n***\n*** Computed loop count " << *ItCst 4699193323Sed << "\n*** From global " << *GV << "*** BB: " << *L->getHeader() 4700193323Sed << "***\n"; 4701193323Sed#endif 4702193323Sed ++NumArrayLenItCounts; 4703193323Sed return getConstant(ItCst); // Found terminating iteration! 4704193323Sed } 4705193323Sed } 4706195340Sed return getCouldNotCompute(); 4707193323Sed} 4708193323Sed 4709193323Sed 4710193323Sed/// CanConstantFold - Return true if we can constant fold an instruction of the 4711193323Sed/// specified type, assuming that all operands were constants. 4712193323Sedstatic bool CanConstantFold(const Instruction *I) { 4713193323Sed if (isa<BinaryOperator>(I) || isa<CmpInst>(I) || 4714234353Sdim isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) || 4715234353Sdim isa<LoadInst>(I)) 4716193323Sed return true; 4717193323Sed 4718193323Sed if (const CallInst *CI = dyn_cast<CallInst>(I)) 4719193323Sed if (const Function *F = CI->getCalledFunction()) 4720193323Sed return canConstantFoldCallTo(F); 4721193323Sed return false; 4722193323Sed} 4723193323Sed 4724226633Sdim/// Determine whether this instruction can constant evolve within this loop 4725226633Sdim/// assuming its operands can all constant evolve. 4726226633Sdimstatic bool canConstantEvolve(Instruction *I, const Loop *L) { 4727226633Sdim // An instruction outside of the loop can't be derived from a loop PHI. 4728226633Sdim if (!L->contains(I)) return false; 4729193323Sed 4730226633Sdim if (isa<PHINode>(I)) { 4731193323Sed if (L->getHeader() == I->getParent()) 4732226633Sdim return true; 4733193323Sed else 4734193323Sed // We don't currently keep track of the control flow needed to evaluate 4735193323Sed // PHIs, so we cannot handle PHIs inside of loops. 4736226633Sdim return false; 4737193323Sed } 4738193323Sed 4739193323Sed // If we won't be able to constant fold this expression even if the operands 4740226633Sdim // are constants, bail early. 4741226633Sdim return CanConstantFold(I); 4742226633Sdim} 4743193323Sed 4744226633Sdim/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by 4745226633Sdim/// recursing through each instruction operand until reaching a loop header phi. 4746226633Sdimstatic PHINode * 4747226633SdimgetConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, 4748226633Sdim DenseMap<Instruction *, PHINode *> &PHIMap) { 4749226633Sdim 4750193323Sed // Otherwise, we can evaluate this instruction if all of its operands are 4751193323Sed // constant or derived from a PHI node themselves. 4752193323Sed PHINode *PHI = 0; 4753226633Sdim for (Instruction::op_iterator OpI = UseInst->op_begin(), 4754226633Sdim OpE = UseInst->op_end(); OpI != OpE; ++OpI) { 4755226633Sdim 4756226633Sdim if (isa<Constant>(*OpI)) continue; 4757226633Sdim 4758226633Sdim Instruction *OpInst = dyn_cast<Instruction>(*OpI); 4759226633Sdim if (!OpInst || !canConstantEvolve(OpInst, L)) return 0; 4760226633Sdim 4761226633Sdim PHINode *P = dyn_cast<PHINode>(OpInst); 4762226633Sdim if (!P) 4763226633Sdim // If this operand is already visited, reuse the prior result. 4764226633Sdim // We may have P != PHI if this is the deepest point at which the 4765226633Sdim // inconsistent paths meet. 4766226633Sdim P = PHIMap.lookup(OpInst); 4767226633Sdim if (!P) { 4768226633Sdim // Recurse and memoize the results, whether a phi is found or not. 4769226633Sdim // This recursive call invalidates pointers into PHIMap. 4770226633Sdim P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap); 4771226633Sdim PHIMap[OpInst] = P; 4772193323Sed } 4773226633Sdim if (P == 0) return 0; // Not evolving from PHI 4774226633Sdim if (PHI && PHI != P) return 0; // Evolving from multiple different PHIs. 4775226633Sdim PHI = P; 4776226633Sdim } 4777193323Sed // This is a expression evolving from a constant PHI! 4778193323Sed return PHI; 4779193323Sed} 4780193323Sed 4781226633Sdim/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node 4782226633Sdim/// in the loop that V is derived from. We allow arbitrary operations along the 4783226633Sdim/// way, but the operands of an operation must either be constants or a value 4784226633Sdim/// derived from a constant PHI. If this expression does not fit with these 4785226633Sdim/// constraints, return null. 4786226633Sdimstatic PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) { 4787226633Sdim Instruction *I = dyn_cast<Instruction>(V); 4788226633Sdim if (I == 0 || !canConstantEvolve(I, L)) return 0; 4789226633Sdim 4790226633Sdim if (PHINode *PN = dyn_cast<PHINode>(I)) { 4791226633Sdim return PN; 4792226633Sdim } 4793226633Sdim 4794226633Sdim // Record non-constant instructions contained by the loop. 4795226633Sdim DenseMap<Instruction *, PHINode *> PHIMap; 4796226633Sdim return getConstantEvolvingPHIOperands(I, L, PHIMap); 4797226633Sdim} 4798226633Sdim 4799193323Sed/// EvaluateExpression - Given an expression that passes the 4800193323Sed/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node 4801193323Sed/// in the loop has the value PHIVal. If we can't fold this expression for some 4802193323Sed/// reason, return null. 4803226633Sdimstatic Constant *EvaluateExpression(Value *V, const Loop *L, 4804226633Sdim DenseMap<Instruction *, Constant *> &Vals, 4805243830Sdim const DataLayout *TD, 4806234353Sdim const TargetLibraryInfo *TLI) { 4807226633Sdim // Convenient constant check, but redundant for recursive calls. 4808193323Sed if (Constant *C = dyn_cast<Constant>(V)) return C; 4809234353Sdim Instruction *I = dyn_cast<Instruction>(V); 4810234353Sdim if (!I) return 0; 4811226633Sdim 4812226633Sdim if (Constant *C = Vals.lookup(I)) return C; 4813193323Sed 4814234353Sdim // An instruction inside the loop depends on a value outside the loop that we 4815234353Sdim // weren't given a mapping for, or a value such as a call inside the loop. 4816234353Sdim if (!canConstantEvolve(I, L)) return 0; 4817226633Sdim 4818234353Sdim // An unmapped PHI can be due to a branch or another loop inside this loop, 4819234353Sdim // or due to this not being the initial iteration through a loop where we 4820234353Sdim // couldn't compute the evolution of this particular PHI last time. 4821234353Sdim if (isa<PHINode>(I)) return 0; 4822234353Sdim 4823210299Sed std::vector<Constant*> Operands(I->getNumOperands()); 4824193323Sed 4825193323Sed for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 4826226633Sdim Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i)); 4827226633Sdim if (!Operand) { 4828226633Sdim Operands[i] = dyn_cast<Constant>(I->getOperand(i)); 4829226633Sdim if (!Operands[i]) return 0; 4830226633Sdim continue; 4831226633Sdim } 4832234353Sdim Constant *C = EvaluateExpression(Operand, L, Vals, TD, TLI); 4833226633Sdim Vals[Operand] = C; 4834226633Sdim if (!C) return 0; 4835226633Sdim Operands[i] = C; 4836193323Sed } 4837193323Sed 4838234353Sdim if (CmpInst *CI = dyn_cast<CmpInst>(I)) 4839199481Srdivacky return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0], 4840234353Sdim Operands[1], TD, TLI); 4841234353Sdim if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 4842234353Sdim if (!LI->isVolatile()) 4843234353Sdim return ConstantFoldLoadFromConstPtr(Operands[0], TD); 4844234353Sdim } 4845234353Sdim return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, TD, 4846234353Sdim TLI); 4847193323Sed} 4848193323Sed 4849193323Sed/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is 4850193323Sed/// in the header of its containing loop, we know the loop executes a 4851193323Sed/// constant number of times, and the PHI node is just a recurrence 4852193323Sed/// involving constants, fold it. 4853195098SedConstant * 4854195098SedScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN, 4855201360Srdivacky const APInt &BEs, 4856195098Sed const Loop *L) { 4857223017Sdim DenseMap<PHINode*, Constant*>::const_iterator I = 4858193323Sed ConstantEvolutionLoopExitValue.find(PN); 4859193323Sed if (I != ConstantEvolutionLoopExitValue.end()) 4860193323Sed return I->second; 4861193323Sed 4862207618Srdivacky if (BEs.ugt(MaxBruteForceIterations)) 4863193323Sed return ConstantEvolutionLoopExitValue[PN] = 0; // Not going to evaluate it. 4864193323Sed 4865193323Sed Constant *&RetVal = ConstantEvolutionLoopExitValue[PN]; 4866193323Sed 4867226633Sdim DenseMap<Instruction *, Constant *> CurrentIterVals; 4868234353Sdim BasicBlock *Header = L->getHeader(); 4869234353Sdim assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 4870226633Sdim 4871193323Sed // Since the loop is canonicalized, the PHI node must have two entries. One 4872193323Sed // entry must be a constant (coming in from outside of the loop), and the 4873193323Sed // second must be derived from the same PHI. 4874193323Sed bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1)); 4875234353Sdim PHINode *PHI = 0; 4876234353Sdim for (BasicBlock::iterator I = Header->begin(); 4877234353Sdim (PHI = dyn_cast<PHINode>(I)); ++I) { 4878234353Sdim Constant *StartCST = 4879234353Sdim dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge)); 4880234353Sdim if (StartCST == 0) continue; 4881234353Sdim CurrentIterVals[PHI] = StartCST; 4882234353Sdim } 4883234353Sdim if (!CurrentIterVals.count(PN)) 4884234353Sdim return RetVal = 0; 4885193323Sed 4886193323Sed Value *BEValue = PN->getIncomingValue(SecondIsBackedge); 4887193323Sed 4888193323Sed // Execute the loop symbolically to determine the exit value. 4889193323Sed if (BEs.getActiveBits() >= 32) 4890193323Sed return RetVal = 0; // More than 2^32-1 iterations?? Not doing it! 4891193323Sed 4892193323Sed unsigned NumIterations = BEs.getZExtValue(); // must be in range 4893193323Sed unsigned IterationNum = 0; 4894226633Sdim for (; ; ++IterationNum) { 4895193323Sed if (IterationNum == NumIterations) 4896226633Sdim return RetVal = CurrentIterVals[PN]; // Got exit value! 4897193323Sed 4898234353Sdim // Compute the value of the PHIs for the next iteration. 4899226633Sdim // EvaluateExpression adds non-phi values to the CurrentIterVals map. 4900234353Sdim DenseMap<Instruction *, Constant *> NextIterVals; 4901234353Sdim Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, 4902234353Sdim TLI); 4903193323Sed if (NextPHI == 0) 4904193323Sed return 0; // Couldn't evaluate! 4905226633Sdim NextIterVals[PN] = NextPHI; 4906234353Sdim 4907234353Sdim bool StoppedEvolving = NextPHI == CurrentIterVals[PN]; 4908234353Sdim 4909234353Sdim // Also evaluate the other PHI nodes. However, we don't get to stop if we 4910234353Sdim // cease to be able to evaluate one of them or if they stop evolving, 4911234353Sdim // because that doesn't necessarily prevent us from computing PN. 4912234353Sdim SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute; 4913234353Sdim for (DenseMap<Instruction *, Constant *>::const_iterator 4914234353Sdim I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){ 4915234353Sdim PHINode *PHI = dyn_cast<PHINode>(I->first); 4916234353Sdim if (!PHI || PHI == PN || PHI->getParent() != Header) continue; 4917234353Sdim PHIsToCompute.push_back(std::make_pair(PHI, I->second)); 4918234353Sdim } 4919234353Sdim // We use two distinct loops because EvaluateExpression may invalidate any 4920234353Sdim // iterators into CurrentIterVals. 4921234353Sdim for (SmallVectorImpl<std::pair<PHINode *, Constant*> >::const_iterator 4922234353Sdim I = PHIsToCompute.begin(), E = PHIsToCompute.end(); I != E; ++I) { 4923234353Sdim PHINode *PHI = I->first; 4924234353Sdim Constant *&NextPHI = NextIterVals[PHI]; 4925234353Sdim if (!NextPHI) { // Not already computed. 4926234353Sdim Value *BEValue = PHI->getIncomingValue(SecondIsBackedge); 4927234353Sdim NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, TLI); 4928234353Sdim } 4929234353Sdim if (NextPHI != I->second) 4930234353Sdim StoppedEvolving = false; 4931234353Sdim } 4932234353Sdim 4933234353Sdim // If all entries in CurrentIterVals == NextIterVals then we can stop 4934234353Sdim // iterating, the loop can't continue to change. 4935234353Sdim if (StoppedEvolving) 4936234353Sdim return RetVal = CurrentIterVals[PN]; 4937234353Sdim 4938226633Sdim CurrentIterVals.swap(NextIterVals); 4939193323Sed } 4940193323Sed} 4941193323Sed 4942226633Sdim/// ComputeExitCountExhaustively - If the loop is known to execute a 4943193323Sed/// constant number of times (the condition evolves only from constants), 4944193323Sed/// try to evaluate a few iterations of the loop until we get the exit 4945193323Sed/// condition gets a value of ExitWhen (true or false). If we cannot 4946195340Sed/// evaluate the trip count of the loop, return getCouldNotCompute(). 4947234353Sdimconst SCEV *ScalarEvolution::ComputeExitCountExhaustively(const Loop *L, 4948234353Sdim Value *Cond, 4949234353Sdim bool ExitWhen) { 4950193323Sed PHINode *PN = getConstantEvolvingPHI(Cond, L); 4951195340Sed if (PN == 0) return getCouldNotCompute(); 4952193323Sed 4953210299Sed // If the loop is canonicalized, the PHI will have exactly two entries. 4954210299Sed // That's the only form we support here. 4955210299Sed if (PN->getNumIncomingValues() != 2) return getCouldNotCompute(); 4956210299Sed 4957234353Sdim DenseMap<Instruction *, Constant *> CurrentIterVals; 4958234353Sdim BasicBlock *Header = L->getHeader(); 4959234353Sdim assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!"); 4960234353Sdim 4961210299Sed // One entry must be a constant (coming in from outside of the loop), and the 4962193323Sed // second must be derived from the same PHI. 4963193323Sed bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1)); 4964234353Sdim PHINode *PHI = 0; 4965234353Sdim for (BasicBlock::iterator I = Header->begin(); 4966234353Sdim (PHI = dyn_cast<PHINode>(I)); ++I) { 4967234353Sdim Constant *StartCST = 4968234353Sdim dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge)); 4969234353Sdim if (StartCST == 0) continue; 4970234353Sdim CurrentIterVals[PHI] = StartCST; 4971234353Sdim } 4972234353Sdim if (!CurrentIterVals.count(PN)) 4973234353Sdim return getCouldNotCompute(); 4974193323Sed 4975193323Sed // Okay, we find a PHI node that defines the trip count of this loop. Execute 4976193323Sed // the loop symbolically to determine when the condition gets a value of 4977193323Sed // "ExitWhen". 4978234353Sdim 4979193323Sed unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis. 4980234353Sdim for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){ 4981193323Sed ConstantInt *CondVal = 4982234353Sdim dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, L, CurrentIterVals, 4983234353Sdim TD, TLI)); 4984193323Sed 4985193323Sed // Couldn't symbolically evaluate. 4986195340Sed if (!CondVal) return getCouldNotCompute(); 4987193323Sed 4988193323Sed if (CondVal->getValue() == uint64_t(ExitWhen)) { 4989193323Sed ++NumBruteForceTripCountsComputed; 4990198090Srdivacky return getConstant(Type::getInt32Ty(getContext()), IterationNum); 4991193323Sed } 4992193323Sed 4993234353Sdim // Update all the PHI nodes for the next iteration. 4994234353Sdim DenseMap<Instruction *, Constant *> NextIterVals; 4995234353Sdim 4996234353Sdim // Create a list of which PHIs we need to compute. We want to do this before 4997234353Sdim // calling EvaluateExpression on them because that may invalidate iterators 4998234353Sdim // into CurrentIterVals. 4999234353Sdim SmallVector<PHINode *, 8> PHIsToCompute; 5000234353Sdim for (DenseMap<Instruction *, Constant *>::const_iterator 5001234353Sdim I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){ 5002234353Sdim PHINode *PHI = dyn_cast<PHINode>(I->first); 5003234353Sdim if (!PHI || PHI->getParent() != Header) continue; 5004234353Sdim PHIsToCompute.push_back(PHI); 5005234353Sdim } 5006234353Sdim for (SmallVectorImpl<PHINode *>::const_iterator I = PHIsToCompute.begin(), 5007234353Sdim E = PHIsToCompute.end(); I != E; ++I) { 5008234353Sdim PHINode *PHI = *I; 5009234353Sdim Constant *&NextPHI = NextIterVals[PHI]; 5010234353Sdim if (NextPHI) continue; // Already computed! 5011234353Sdim 5012234353Sdim Value *BEValue = PHI->getIncomingValue(SecondIsBackedge); 5013234353Sdim NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, TD, TLI); 5014234353Sdim } 5015234353Sdim CurrentIterVals.swap(NextIterVals); 5016193323Sed } 5017193323Sed 5018193323Sed // Too many iterations were needed to evaluate. 5019195340Sed return getCouldNotCompute(); 5020193323Sed} 5021193323Sed 5022198090Srdivacky/// getSCEVAtScope - Return a SCEV expression for the specified value 5023193323Sed/// at the specified scope in the program. The L value specifies a loop 5024193323Sed/// nest to evaluate the expression at, where null is the top-level or a 5025193323Sed/// specified loop is immediately inside of the loop. 5026193323Sed/// 5027193323Sed/// This method can be used to compute the exit value for a variable defined 5028193323Sed/// in a loop by querying what the value will hold in the parent loop. 5029193323Sed/// 5030193323Sed/// In the case that a relevant loop exit value cannot be computed, the 5031193323Sed/// original value V is returned. 5032198090Srdivackyconst SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) { 5033198090Srdivacky // Check to see if we've folded this expression at this loop before. 5034198090Srdivacky std::map<const Loop *, const SCEV *> &Values = ValuesAtScopes[V]; 5035198090Srdivacky std::pair<std::map<const Loop *, const SCEV *>::iterator, bool> Pair = 5036198090Srdivacky Values.insert(std::make_pair(L, static_cast<const SCEV *>(0))); 5037198090Srdivacky if (!Pair.second) 5038198090Srdivacky return Pair.first->second ? Pair.first->second : V; 5039193323Sed 5040198090Srdivacky // Otherwise compute it. 5041198090Srdivacky const SCEV *C = computeSCEVAtScope(V, L); 5042198090Srdivacky ValuesAtScopes[V][L] = C; 5043198090Srdivacky return C; 5044198090Srdivacky} 5045198090Srdivacky 5046234353Sdim/// This builds up a Constant using the ConstantExpr interface. That way, we 5047234353Sdim/// will return Constants for objects which aren't represented by a 5048234353Sdim/// SCEVConstant, because SCEVConstant is restricted to ConstantInt. 5049234353Sdim/// Returns NULL if the SCEV isn't representable as a Constant. 5050234353Sdimstatic Constant *BuildConstantFromSCEV(const SCEV *V) { 5051234353Sdim switch (V->getSCEVType()) { 5052234353Sdim default: // TODO: smax, umax. 5053234353Sdim case scCouldNotCompute: 5054234353Sdim case scAddRecExpr: 5055234353Sdim break; 5056234353Sdim case scConstant: 5057234353Sdim return cast<SCEVConstant>(V)->getValue(); 5058234353Sdim case scUnknown: 5059234353Sdim return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue()); 5060234353Sdim case scSignExtend: { 5061234353Sdim const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V); 5062234353Sdim if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand())) 5063234353Sdim return ConstantExpr::getSExt(CastOp, SS->getType()); 5064234353Sdim break; 5065234353Sdim } 5066234353Sdim case scZeroExtend: { 5067234353Sdim const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V); 5068234353Sdim if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand())) 5069234353Sdim return ConstantExpr::getZExt(CastOp, SZ->getType()); 5070234353Sdim break; 5071234353Sdim } 5072234353Sdim case scTruncate: { 5073234353Sdim const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V); 5074234353Sdim if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand())) 5075234353Sdim return ConstantExpr::getTrunc(CastOp, ST->getType()); 5076234353Sdim break; 5077234353Sdim } 5078234353Sdim case scAddExpr: { 5079234353Sdim const SCEVAddExpr *SA = cast<SCEVAddExpr>(V); 5080234353Sdim if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) { 5081234353Sdim if (C->getType()->isPointerTy()) 5082234353Sdim C = ConstantExpr::getBitCast(C, Type::getInt8PtrTy(C->getContext())); 5083234353Sdim for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) { 5084234353Sdim Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i)); 5085234353Sdim if (!C2) return 0; 5086234353Sdim 5087234353Sdim // First pointer! 5088234353Sdim if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) { 5089234353Sdim std::swap(C, C2); 5090234353Sdim // The offsets have been converted to bytes. We can add bytes to an 5091234353Sdim // i8* by GEP with the byte count in the first index. 5092234353Sdim C = ConstantExpr::getBitCast(C,Type::getInt8PtrTy(C->getContext())); 5093234353Sdim } 5094234353Sdim 5095234353Sdim // Don't bother trying to sum two pointers. We probably can't 5096234353Sdim // statically compute a load that results from it anyway. 5097234353Sdim if (C2->getType()->isPointerTy()) 5098234353Sdim return 0; 5099234353Sdim 5100234353Sdim if (C->getType()->isPointerTy()) { 5101234353Sdim if (cast<PointerType>(C->getType())->getElementType()->isStructTy()) 5102234353Sdim C2 = ConstantExpr::getIntegerCast( 5103234353Sdim C2, Type::getInt32Ty(C->getContext()), true); 5104234353Sdim C = ConstantExpr::getGetElementPtr(C, C2); 5105234353Sdim } else 5106234353Sdim C = ConstantExpr::getAdd(C, C2); 5107234353Sdim } 5108234353Sdim return C; 5109234353Sdim } 5110234353Sdim break; 5111234353Sdim } 5112234353Sdim case scMulExpr: { 5113234353Sdim const SCEVMulExpr *SM = cast<SCEVMulExpr>(V); 5114234353Sdim if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) { 5115234353Sdim // Don't bother with pointers at all. 5116234353Sdim if (C->getType()->isPointerTy()) return 0; 5117234353Sdim for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) { 5118234353Sdim Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i)); 5119234353Sdim if (!C2 || C2->getType()->isPointerTy()) return 0; 5120234353Sdim C = ConstantExpr::getMul(C, C2); 5121234353Sdim } 5122234353Sdim return C; 5123234353Sdim } 5124234353Sdim break; 5125234353Sdim } 5126234353Sdim case scUDivExpr: { 5127234353Sdim const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V); 5128234353Sdim if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS())) 5129234353Sdim if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS())) 5130234353Sdim if (LHS->getType() == RHS->getType()) 5131234353Sdim return ConstantExpr::getUDiv(LHS, RHS); 5132234353Sdim break; 5133234353Sdim } 5134234353Sdim } 5135234353Sdim return 0; 5136234353Sdim} 5137234353Sdim 5138198090Srdivackyconst SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { 5139193323Sed if (isa<SCEVConstant>(V)) return V; 5140193323Sed 5141193323Sed // If this instruction is evolved from a constant-evolving PHI, compute the 5142193323Sed // exit value from the loop without using SCEVs. 5143193323Sed if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) { 5144193323Sed if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) { 5145193323Sed const Loop *LI = (*this->LI)[I->getParent()]; 5146193323Sed if (LI && LI->getParentLoop() == L) // Looking for loop exit value. 5147193323Sed if (PHINode *PN = dyn_cast<PHINode>(I)) 5148193323Sed if (PN->getParent() == LI->getHeader()) { 5149193323Sed // Okay, there is no closed form solution for the PHI node. Check 5150193323Sed // to see if the loop that contains it has a known backedge-taken 5151193323Sed // count. If so, we may be able to force computation of the exit 5152193323Sed // value. 5153198090Srdivacky const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI); 5154193323Sed if (const SCEVConstant *BTCC = 5155193323Sed dyn_cast<SCEVConstant>(BackedgeTakenCount)) { 5156193323Sed // Okay, we know how many times the containing loop executes. If 5157193323Sed // this is a constant evolving PHI node, get the final value at 5158193323Sed // the specified iteration number. 5159193323Sed Constant *RV = getConstantEvolutionLoopExitValue(PN, 5160193323Sed BTCC->getValue()->getValue(), 5161193323Sed LI); 5162195340Sed if (RV) return getSCEV(RV); 5163193323Sed } 5164193323Sed } 5165193323Sed 5166193323Sed // Okay, this is an expression that we cannot symbolically evaluate 5167193323Sed // into a SCEV. Check to see if it's possible to symbolically evaluate 5168193323Sed // the arguments into constants, and if so, try to constant propagate the 5169193323Sed // result. This is particularly useful for computing loop exit values. 5170193323Sed if (CanConstantFold(I)) { 5171210299Sed SmallVector<Constant *, 4> Operands; 5172210299Sed bool MadeImprovement = false; 5173193323Sed for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { 5174193323Sed Value *Op = I->getOperand(i); 5175193323Sed if (Constant *C = dyn_cast<Constant>(Op)) { 5176193323Sed Operands.push_back(C); 5177210299Sed continue; 5178210299Sed } 5179193323Sed 5180210299Sed // If any of the operands is non-constant and if they are 5181210299Sed // non-integer and non-pointer, don't even try to analyze them 5182210299Sed // with scev techniques. 5183210299Sed if (!isSCEVable(Op->getType())) 5184210299Sed return V; 5185210299Sed 5186210299Sed const SCEV *OrigV = getSCEV(Op); 5187210299Sed const SCEV *OpV = getSCEVAtScope(OrigV, L); 5188210299Sed MadeImprovement |= OrigV != OpV; 5189210299Sed 5190234353Sdim Constant *C = BuildConstantFromSCEV(OpV); 5191210299Sed if (!C) return V; 5192210299Sed if (C->getType() != Op->getType()) 5193210299Sed C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, 5194210299Sed Op->getType(), 5195210299Sed false), 5196210299Sed C, Op->getType()); 5197210299Sed Operands.push_back(C); 5198193323Sed } 5199195098Sed 5200210299Sed // Check to see if getSCEVAtScope actually made an improvement. 5201210299Sed if (MadeImprovement) { 5202210299Sed Constant *C = 0; 5203210299Sed if (const CmpInst *CI = dyn_cast<CmpInst>(I)) 5204210299Sed C = ConstantFoldCompareInstOperands(CI->getPredicate(), 5205234353Sdim Operands[0], Operands[1], TD, 5206234353Sdim TLI); 5207234353Sdim else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) { 5208234353Sdim if (!LI->isVolatile()) 5209234353Sdim C = ConstantFoldLoadFromConstPtr(Operands[0], TD); 5210234353Sdim } else 5211210299Sed C = ConstantFoldInstOperands(I->getOpcode(), I->getType(), 5212234353Sdim Operands, TD, TLI); 5213210299Sed if (!C) return V; 5214204642Srdivacky return getSCEV(C); 5215210299Sed } 5216193323Sed } 5217193323Sed } 5218193323Sed 5219193323Sed // This is some other type of SCEVUnknown, just return it. 5220193323Sed return V; 5221193323Sed } 5222193323Sed 5223193323Sed if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) { 5224193323Sed // Avoid performing the look-up in the common case where the specified 5225193323Sed // expression has no loop-variant portions. 5226193323Sed for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) { 5227198090Srdivacky const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 5228193323Sed if (OpAtScope != Comm->getOperand(i)) { 5229193323Sed // Okay, at least one of these operands is loop variant but might be 5230193323Sed // foldable. Build a new instance of the folded commutative expression. 5231195098Sed SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(), 5232195098Sed Comm->op_begin()+i); 5233193323Sed NewOps.push_back(OpAtScope); 5234193323Sed 5235193323Sed for (++i; i != e; ++i) { 5236193323Sed OpAtScope = getSCEVAtScope(Comm->getOperand(i), L); 5237193323Sed NewOps.push_back(OpAtScope); 5238193323Sed } 5239193323Sed if (isa<SCEVAddExpr>(Comm)) 5240193323Sed return getAddExpr(NewOps); 5241193323Sed if (isa<SCEVMulExpr>(Comm)) 5242193323Sed return getMulExpr(NewOps); 5243193323Sed if (isa<SCEVSMaxExpr>(Comm)) 5244193323Sed return getSMaxExpr(NewOps); 5245193323Sed if (isa<SCEVUMaxExpr>(Comm)) 5246193323Sed return getUMaxExpr(NewOps); 5247198090Srdivacky llvm_unreachable("Unknown commutative SCEV type!"); 5248193323Sed } 5249193323Sed } 5250193323Sed // If we got here, all operands are loop invariant. 5251193323Sed return Comm; 5252193323Sed } 5253193323Sed 5254193323Sed if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) { 5255198090Srdivacky const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L); 5256198090Srdivacky const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L); 5257193323Sed if (LHS == Div->getLHS() && RHS == Div->getRHS()) 5258193323Sed return Div; // must be loop invariant 5259193323Sed return getUDivExpr(LHS, RHS); 5260193323Sed } 5261193323Sed 5262193323Sed // If this is a loop recurrence for a loop that does not contain L, then we 5263193323Sed // are dealing with the final value computed by the loop. 5264193323Sed if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) { 5265210299Sed // First, attempt to evaluate each operand. 5266210299Sed // Avoid performing the look-up in the common case where the specified 5267210299Sed // expression has no loop-variant portions. 5268210299Sed for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) { 5269210299Sed const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L); 5270210299Sed if (OpAtScope == AddRec->getOperand(i)) 5271210299Sed continue; 5272210299Sed 5273210299Sed // Okay, at least one of these operands is loop variant but might be 5274210299Sed // foldable. Build a new instance of the folded commutative expression. 5275210299Sed SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(), 5276210299Sed AddRec->op_begin()+i); 5277210299Sed NewOps.push_back(OpAtScope); 5278210299Sed for (++i; i != e; ++i) 5279210299Sed NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L)); 5280210299Sed 5281221345Sdim const SCEV *FoldedRec = 5282221345Sdim getAddRecExpr(NewOps, AddRec->getLoop(), 5283221345Sdim AddRec->getNoWrapFlags(SCEV::FlagNW)); 5284221345Sdim AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec); 5285221345Sdim // The addrec may be folded to a nonrecurrence, for example, if the 5286221345Sdim // induction variable is multiplied by zero after constant folding. Go 5287221345Sdim // ahead and return the folded value. 5288221345Sdim if (!AddRec) 5289221345Sdim return FoldedRec; 5290210299Sed break; 5291210299Sed } 5292210299Sed 5293210299Sed // If the scope is outside the addrec's loop, evaluate it by using the 5294210299Sed // loop exit value of the addrec. 5295210299Sed if (!AddRec->getLoop()->contains(L)) { 5296193323Sed // To evaluate this recurrence, we need to know how many times the AddRec 5297193323Sed // loop iterates. Compute this now. 5298198090Srdivacky const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop()); 5299195340Sed if (BackedgeTakenCount == getCouldNotCompute()) return AddRec; 5300193323Sed 5301193323Sed // Then, evaluate the AddRec. 5302193323Sed return AddRec->evaluateAtIteration(BackedgeTakenCount, *this); 5303193323Sed } 5304210299Sed 5305193323Sed return AddRec; 5306193323Sed } 5307193323Sed 5308193323Sed if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) { 5309198090Srdivacky const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5310193323Sed if (Op == Cast->getOperand()) 5311193323Sed return Cast; // must be loop invariant 5312193323Sed return getZeroExtendExpr(Op, Cast->getType()); 5313193323Sed } 5314193323Sed 5315193323Sed if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) { 5316198090Srdivacky const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5317193323Sed if (Op == Cast->getOperand()) 5318193323Sed return Cast; // must be loop invariant 5319193323Sed return getSignExtendExpr(Op, Cast->getType()); 5320193323Sed } 5321193323Sed 5322193323Sed if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) { 5323198090Srdivacky const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L); 5324193323Sed if (Op == Cast->getOperand()) 5325193323Sed return Cast; // must be loop invariant 5326193323Sed return getTruncateExpr(Op, Cast->getType()); 5327193323Sed } 5328193323Sed 5329198090Srdivacky llvm_unreachable("Unknown SCEV type!"); 5330193323Sed} 5331193323Sed 5332193323Sed/// getSCEVAtScope - This is a convenience function which does 5333193323Sed/// getSCEVAtScope(getSCEV(V), L). 5334198090Srdivackyconst SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) { 5335193323Sed return getSCEVAtScope(getSCEV(V), L); 5336193323Sed} 5337193323Sed 5338193323Sed/// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the 5339193323Sed/// following equation: 5340193323Sed/// 5341193323Sed/// A * X = B (mod N) 5342193323Sed/// 5343193323Sed/// where N = 2^BW and BW is the common bit width of A and B. The signedness of 5344193323Sed/// A and B isn't important. 5345193323Sed/// 5346193323Sed/// If the equation does not have a solution, SCEVCouldNotCompute is returned. 5347198090Srdivackystatic const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B, 5348193323Sed ScalarEvolution &SE) { 5349193323Sed uint32_t BW = A.getBitWidth(); 5350193323Sed assert(BW == B.getBitWidth() && "Bit widths must be the same."); 5351193323Sed assert(A != 0 && "A must be non-zero."); 5352193323Sed 5353193323Sed // 1. D = gcd(A, N) 5354193323Sed // 5355193323Sed // The gcd of A and N may have only one prime factor: 2. The number of 5356193323Sed // trailing zeros in A is its multiplicity 5357193323Sed uint32_t Mult2 = A.countTrailingZeros(); 5358193323Sed // D = 2^Mult2 5359193323Sed 5360193323Sed // 2. Check if B is divisible by D. 5361193323Sed // 5362193323Sed // B is divisible by D if and only if the multiplicity of prime factor 2 for B 5363193323Sed // is not less than multiplicity of this prime factor for D. 5364193323Sed if (B.countTrailingZeros() < Mult2) 5365193323Sed return SE.getCouldNotCompute(); 5366193323Sed 5367193323Sed // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic 5368193323Sed // modulo (N / D). 5369193323Sed // 5370193323Sed // (N / D) may need BW+1 bits in its representation. Hence, we'll use this 5371193323Sed // bit width during computations. 5372193323Sed APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D 5373193323Sed APInt Mod(BW + 1, 0); 5374218893Sdim Mod.setBit(BW - Mult2); // Mod = N / D 5375193323Sed APInt I = AD.multiplicativeInverse(Mod); 5376193323Sed 5377193323Sed // 4. Compute the minimum unsigned root of the equation: 5378193323Sed // I * (B / D) mod (N / D) 5379193323Sed APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod); 5380193323Sed 5381193323Sed // The result is guaranteed to be less than 2^BW so we may truncate it to BW 5382193323Sed // bits. 5383193323Sed return SE.getConstant(Result.trunc(BW)); 5384193323Sed} 5385193323Sed 5386193323Sed/// SolveQuadraticEquation - Find the roots of the quadratic equation for the 5387193323Sed/// given quadratic chrec {L,+,M,+,N}. This returns either the two roots (which 5388193323Sed/// might be the same) or two SCEVCouldNotCompute objects. 5389193323Sed/// 5390198090Srdivackystatic std::pair<const SCEV *,const SCEV *> 5391193323SedSolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) { 5392193323Sed assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!"); 5393193323Sed const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0)); 5394193323Sed const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1)); 5395193323Sed const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2)); 5396193323Sed 5397193323Sed // We currently can only solve this if the coefficients are constants. 5398193323Sed if (!LC || !MC || !NC) { 5399193323Sed const SCEV *CNC = SE.getCouldNotCompute(); 5400193323Sed return std::make_pair(CNC, CNC); 5401193323Sed } 5402193323Sed 5403193323Sed uint32_t BitWidth = LC->getValue()->getValue().getBitWidth(); 5404193323Sed const APInt &L = LC->getValue()->getValue(); 5405193323Sed const APInt &M = MC->getValue()->getValue(); 5406193323Sed const APInt &N = NC->getValue()->getValue(); 5407193323Sed APInt Two(BitWidth, 2); 5408193323Sed APInt Four(BitWidth, 4); 5409193323Sed 5410195098Sed { 5411193323Sed using namespace APIntOps; 5412193323Sed const APInt& C = L; 5413193323Sed // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C 5414193323Sed // The B coefficient is M-N/2 5415193323Sed APInt B(M); 5416193323Sed B -= sdiv(N,Two); 5417193323Sed 5418193323Sed // The A coefficient is N/2 5419193323Sed APInt A(N.sdiv(Two)); 5420193323Sed 5421193323Sed // Compute the B^2-4ac term. 5422193323Sed APInt SqrtTerm(B); 5423193323Sed SqrtTerm *= B; 5424193323Sed SqrtTerm -= Four * (A * C); 5425193323Sed 5426239462Sdim if (SqrtTerm.isNegative()) { 5427239462Sdim // The loop is provably infinite. 5428239462Sdim const SCEV *CNC = SE.getCouldNotCompute(); 5429239462Sdim return std::make_pair(CNC, CNC); 5430239462Sdim } 5431239462Sdim 5432193323Sed // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest 5433193323Sed // integer value or else APInt::sqrt() will assert. 5434193323Sed APInt SqrtVal(SqrtTerm.sqrt()); 5435193323Sed 5436195098Sed // Compute the two solutions for the quadratic formula. 5437193323Sed // The divisions must be performed as signed divisions. 5438193323Sed APInt NegB(-B); 5439226633Sdim APInt TwoA(A << 1); 5440193323Sed if (TwoA.isMinValue()) { 5441193323Sed const SCEV *CNC = SE.getCouldNotCompute(); 5442193323Sed return std::make_pair(CNC, CNC); 5443193323Sed } 5444193323Sed 5445198090Srdivacky LLVMContext &Context = SE.getContext(); 5446193323Sed 5447198090Srdivacky ConstantInt *Solution1 = 5448198090Srdivacky ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA)); 5449198090Srdivacky ConstantInt *Solution2 = 5450198090Srdivacky ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA)); 5451198090Srdivacky 5452195098Sed return std::make_pair(SE.getConstant(Solution1), 5453193323Sed SE.getConstant(Solution2)); 5454226633Sdim } // end APIntOps namespace 5455193323Sed} 5456193323Sed 5457193323Sed/// HowFarToZero - Return the number of times a backedge comparing the specified 5458193630Sed/// value to zero will execute. If not computable, return CouldNotCompute. 5459221345Sdim/// 5460221345Sdim/// This is only used for loops with a "x != y" exit test. The exit condition is 5461221345Sdim/// now expressed as a single expression, V = x-y. So the exit test is 5462221345Sdim/// effectively V != 0. We know and take advantage of the fact that this 5463221345Sdim/// expression only being used in a comparison by zero context. 5464226633SdimScalarEvolution::ExitLimit 5465251662SdimScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr) { 5466193323Sed // If the value is a constant 5467193323Sed if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 5468193323Sed // If the value is already zero, the branch will execute zero times. 5469193323Sed if (C->getValue()->isZero()) return C; 5470195340Sed return getCouldNotCompute(); // Otherwise it will loop infinitely. 5471193323Sed } 5472193323Sed 5473193323Sed const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V); 5474193323Sed if (!AddRec || AddRec->getLoop() != L) 5475195340Sed return getCouldNotCompute(); 5476193323Sed 5477218893Sdim // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of 5478218893Sdim // the quadratic equation to solve it. 5479218893Sdim if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) { 5480218893Sdim std::pair<const SCEV *,const SCEV *> Roots = 5481218893Sdim SolveQuadraticEquation(AddRec, *this); 5482193323Sed const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 5483193323Sed const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 5484218893Sdim if (R1 && R2) { 5485193323Sed#if 0 5486201360Srdivacky dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1 5487193323Sed << " sol#2: " << *R2 << "\n"; 5488193323Sed#endif 5489193323Sed // Pick the smallest positive root value. 5490193323Sed if (ConstantInt *CB = 5491218893Sdim dyn_cast<ConstantInt>(ConstantExpr::getICmp(CmpInst::ICMP_ULT, 5492218893Sdim R1->getValue(), 5493218893Sdim R2->getValue()))) { 5494193323Sed if (CB->getZExtValue() == false) 5495193323Sed std::swap(R1, R2); // R1 is the minimum root now. 5496221345Sdim 5497193323Sed // We can only use this value if the chrec ends up with an exact zero 5498193323Sed // value at this index. When solving for "X*X != 5", for example, we 5499193323Sed // should not accept a root of 2. 5500198090Srdivacky const SCEV *Val = AddRec->evaluateAtIteration(R1, *this); 5501193323Sed if (Val->isZero()) 5502193323Sed return R1; // We found a quadratic root! 5503193323Sed } 5504193323Sed } 5505218893Sdim return getCouldNotCompute(); 5506193323Sed } 5507193323Sed 5508218893Sdim // Otherwise we can only handle this if it is affine. 5509218893Sdim if (!AddRec->isAffine()) 5510218893Sdim return getCouldNotCompute(); 5511218893Sdim 5512218893Sdim // If this is an affine expression, the execution count of this branch is 5513218893Sdim // the minimum unsigned root of the following equation: 5514218893Sdim // 5515218893Sdim // Start + Step*N = 0 (mod 2^BW) 5516218893Sdim // 5517218893Sdim // equivalent to: 5518218893Sdim // 5519218893Sdim // Step*N = -Start (mod 2^BW) 5520218893Sdim // 5521218893Sdim // where BW is the common bit width of Start and Step. 5522218893Sdim 5523218893Sdim // Get the initial value for the loop. 5524218893Sdim const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop()); 5525218893Sdim const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop()); 5526218893Sdim 5527218893Sdim // For now we handle only constant steps. 5528221345Sdim // 5529221345Sdim // TODO: Handle a nonconstant Step given AddRec<NUW>. If the 5530221345Sdim // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap 5531221345Sdim // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step. 5532221345Sdim // We have not yet seen any such cases. 5533218893Sdim const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step); 5534239462Sdim if (StepC == 0 || StepC->getValue()->equalsInt(0)) 5535218893Sdim return getCouldNotCompute(); 5536218893Sdim 5537221345Sdim // For positive steps (counting up until unsigned overflow): 5538221345Sdim // N = -Start/Step (as unsigned) 5539221345Sdim // For negative steps (counting down to zero): 5540221345Sdim // N = Start/-Step 5541221345Sdim // First compute the unsigned distance from zero in the direction of Step. 5542221345Sdim bool CountDown = StepC->getValue()->getValue().isNegative(); 5543221345Sdim const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start); 5544218893Sdim 5545221345Sdim // Handle unitary steps, which cannot wraparound. 5546221345Sdim // 1*N = -Start; -1*N = Start (mod 2^BW), so: 5547221345Sdim // N = Distance (as unsigned) 5548226633Sdim if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) { 5549226633Sdim ConstantRange CR = getUnsignedRange(Start); 5550226633Sdim const SCEV *MaxBECount; 5551226633Sdim if (!CountDown && CR.getUnsignedMin().isMinValue()) 5552226633Sdim // When counting up, the worst starting value is 1, not 0. 5553226633Sdim MaxBECount = CR.getUnsignedMax().isMinValue() 5554226633Sdim ? getConstant(APInt::getMinValue(CR.getBitWidth())) 5555226633Sdim : getConstant(APInt::getMaxValue(CR.getBitWidth())); 5556226633Sdim else 5557226633Sdim MaxBECount = getConstant(CountDown ? CR.getUnsignedMax() 5558226633Sdim : -CR.getUnsignedMin()); 5559226633Sdim return ExitLimit(Distance, MaxBECount); 5560226633Sdim } 5561221345Sdim 5562221345Sdim // If the recurrence is known not to wraparound, unsigned divide computes the 5563251662Sdim // back edge count. (Ideally we would have an "isexact" bit for udiv). We know 5564251662Sdim // that the value will either become zero (and thus the loop terminates), that 5565251662Sdim // the loop will terminate through some other exit condition first, or that 5566251662Sdim // the loop has undefined behavior. This means we can't "miss" the exit 5567251662Sdim // value, even with nonunit stride. 5568221345Sdim // 5569251662Sdim // This is only valid for expressions that directly compute the loop exit. It 5570251662Sdim // is invalid for subexpressions in which the loop may exit through this 5571251662Sdim // branch even if this subexpression is false. In that case, the trip count 5572251662Sdim // computed by this udiv could be smaller than the number of well-defined 5573251662Sdim // iterations. 5574251662Sdim if (!IsSubExpr && AddRec->getNoWrapFlags(SCEV::FlagNW)) 5575221345Sdim return getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step); 5576251662Sdim 5577218893Sdim // Then, try to solve the above equation provided that Start is constant. 5578218893Sdim if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start)) 5579218893Sdim return SolveLinEquationWithOverflow(StepC->getValue()->getValue(), 5580218893Sdim -StartC->getValue()->getValue(), 5581218893Sdim *this); 5582195340Sed return getCouldNotCompute(); 5583193323Sed} 5584193323Sed 5585193323Sed/// HowFarToNonZero - Return the number of times a backedge checking the 5586193323Sed/// specified value for nonzero will execute. If not computable, return 5587193630Sed/// CouldNotCompute 5588226633SdimScalarEvolution::ExitLimit 5589204642SrdivackyScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) { 5590193323Sed // Loops that look like: while (X == 0) are very strange indeed. We don't 5591193323Sed // handle them yet except for the trivial case. This could be expanded in the 5592193323Sed // future as needed. 5593193323Sed 5594193323Sed // If the value is a constant, check to see if it is known to be non-zero 5595193323Sed // already. If so, the backedge will execute zero times. 5596193323Sed if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) { 5597193323Sed if (!C->getValue()->isNullValue()) 5598207618Srdivacky return getConstant(C->getType(), 0); 5599195340Sed return getCouldNotCompute(); // Otherwise it will loop infinitely. 5600193323Sed } 5601193323Sed 5602193323Sed // We could implement others, but I really doubt anyone writes loops like 5603193323Sed // this, and if they did, they would already be constant folded. 5604195340Sed return getCouldNotCompute(); 5605193323Sed} 5606193323Sed 5607193323Sed/// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB 5608193323Sed/// (which may not be an immediate predecessor) which has exactly one 5609193323Sed/// successor from which BB is reachable, or null if no such block is 5610193323Sed/// found. 5611193323Sed/// 5612207618Srdivackystd::pair<BasicBlock *, BasicBlock *> 5613193323SedScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) { 5614193323Sed // If the block has a unique predecessor, then there is no path from the 5615193323Sed // predecessor to the block that does not go through the direct edge 5616193323Sed // from the predecessor to the block. 5617193323Sed if (BasicBlock *Pred = BB->getSinglePredecessor()) 5618207618Srdivacky return std::make_pair(Pred, BB); 5619193323Sed 5620193323Sed // A loop's header is defined to be a block that dominates the loop. 5621193323Sed // If the header has a unique predecessor outside the loop, it must be 5622193323Sed // a block that has exactly one successor that can reach the loop. 5623193323Sed if (Loop *L = LI->getLoopFor(BB)) 5624210299Sed return std::make_pair(L->getLoopPredecessor(), L->getHeader()); 5625193323Sed 5626207618Srdivacky return std::pair<BasicBlock *, BasicBlock *>(); 5627193323Sed} 5628193323Sed 5629194612Sed/// HasSameValue - SCEV structural equivalence is usually sufficient for 5630194612Sed/// testing whether two expressions are equal, however for the purposes of 5631194612Sed/// looking for a condition guarding a loop, it can be useful to be a little 5632194612Sed/// more general, since a front-end may have replicated the controlling 5633194612Sed/// expression. 5634194612Sed/// 5635198090Srdivackystatic bool HasSameValue(const SCEV *A, const SCEV *B) { 5636194612Sed // Quick check to see if they are the same SCEV. 5637194612Sed if (A == B) return true; 5638194612Sed 5639194612Sed // Otherwise, if they're both SCEVUnknown, it's possible that they hold 5640194612Sed // two different instructions with the same value. Check for this case. 5641194612Sed if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A)) 5642194612Sed if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B)) 5643194612Sed if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue())) 5644194612Sed if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue())) 5645198090Srdivacky if (AI->isIdenticalTo(BI) && !AI->mayReadFromMemory()) 5646194612Sed return true; 5647194612Sed 5648194612Sed // Otherwise assume they may have a different value. 5649194612Sed return false; 5650194612Sed} 5651194612Sed 5652207618Srdivacky/// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with 5653207618Srdivacky/// predicate Pred. Return true iff any changes were made. 5654207618Srdivacky/// 5655207618Srdivackybool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, 5656239462Sdim const SCEV *&LHS, const SCEV *&RHS, 5657239462Sdim unsigned Depth) { 5658207618Srdivacky bool Changed = false; 5659207618Srdivacky 5660239462Sdim // If we hit the max recursion limit bail out. 5661239462Sdim if (Depth >= 3) 5662239462Sdim return false; 5663239462Sdim 5664207618Srdivacky // Canonicalize a constant to the right side. 5665207618Srdivacky if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) { 5666207618Srdivacky // Check for both operands constant. 5667207618Srdivacky if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) { 5668207618Srdivacky if (ConstantExpr::getICmp(Pred, 5669207618Srdivacky LHSC->getValue(), 5670207618Srdivacky RHSC->getValue())->isNullValue()) 5671207618Srdivacky goto trivially_false; 5672207618Srdivacky else 5673207618Srdivacky goto trivially_true; 5674207618Srdivacky } 5675207618Srdivacky // Otherwise swap the operands to put the constant on the right. 5676207618Srdivacky std::swap(LHS, RHS); 5677207618Srdivacky Pred = ICmpInst::getSwappedPredicate(Pred); 5678207618Srdivacky Changed = true; 5679207618Srdivacky } 5680207618Srdivacky 5681207618Srdivacky // If we're comparing an addrec with a value which is loop-invariant in the 5682207618Srdivacky // addrec's loop, put the addrec on the left. Also make a dominance check, 5683207618Srdivacky // as both operands could be addrecs loop-invariant in each other's loop. 5684207618Srdivacky if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) { 5685207618Srdivacky const Loop *L = AR->getLoop(); 5686218893Sdim if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) { 5687207618Srdivacky std::swap(LHS, RHS); 5688207618Srdivacky Pred = ICmpInst::getSwappedPredicate(Pred); 5689207618Srdivacky Changed = true; 5690207618Srdivacky } 5691207618Srdivacky } 5692207618Srdivacky 5693207618Srdivacky // If there's a constant operand, canonicalize comparisons with boundary 5694207618Srdivacky // cases, and canonicalize *-or-equal comparisons to regular comparisons. 5695207618Srdivacky if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) { 5696207618Srdivacky const APInt &RA = RC->getValue()->getValue(); 5697207618Srdivacky switch (Pred) { 5698207618Srdivacky default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 5699207618Srdivacky case ICmpInst::ICMP_EQ: 5700207618Srdivacky case ICmpInst::ICMP_NE: 5701239462Sdim // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b. 5702239462Sdim if (!RA) 5703239462Sdim if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS)) 5704239462Sdim if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0))) 5705239462Sdim if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 && 5706239462Sdim ME->getOperand(0)->isAllOnesValue()) { 5707239462Sdim RHS = AE->getOperand(1); 5708239462Sdim LHS = ME->getOperand(1); 5709239462Sdim Changed = true; 5710239462Sdim } 5711207618Srdivacky break; 5712207618Srdivacky case ICmpInst::ICMP_UGE: 5713207618Srdivacky if ((RA - 1).isMinValue()) { 5714207618Srdivacky Pred = ICmpInst::ICMP_NE; 5715207618Srdivacky RHS = getConstant(RA - 1); 5716207618Srdivacky Changed = true; 5717207618Srdivacky break; 5718207618Srdivacky } 5719207618Srdivacky if (RA.isMaxValue()) { 5720207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5721207618Srdivacky Changed = true; 5722207618Srdivacky break; 5723207618Srdivacky } 5724207618Srdivacky if (RA.isMinValue()) goto trivially_true; 5725207618Srdivacky 5726207618Srdivacky Pred = ICmpInst::ICMP_UGT; 5727207618Srdivacky RHS = getConstant(RA - 1); 5728207618Srdivacky Changed = true; 5729207618Srdivacky break; 5730207618Srdivacky case ICmpInst::ICMP_ULE: 5731207618Srdivacky if ((RA + 1).isMaxValue()) { 5732207618Srdivacky Pred = ICmpInst::ICMP_NE; 5733207618Srdivacky RHS = getConstant(RA + 1); 5734207618Srdivacky Changed = true; 5735207618Srdivacky break; 5736207618Srdivacky } 5737207618Srdivacky if (RA.isMinValue()) { 5738207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5739207618Srdivacky Changed = true; 5740207618Srdivacky break; 5741207618Srdivacky } 5742207618Srdivacky if (RA.isMaxValue()) goto trivially_true; 5743207618Srdivacky 5744207618Srdivacky Pred = ICmpInst::ICMP_ULT; 5745207618Srdivacky RHS = getConstant(RA + 1); 5746207618Srdivacky Changed = true; 5747207618Srdivacky break; 5748207618Srdivacky case ICmpInst::ICMP_SGE: 5749207618Srdivacky if ((RA - 1).isMinSignedValue()) { 5750207618Srdivacky Pred = ICmpInst::ICMP_NE; 5751207618Srdivacky RHS = getConstant(RA - 1); 5752207618Srdivacky Changed = true; 5753207618Srdivacky break; 5754207618Srdivacky } 5755207618Srdivacky if (RA.isMaxSignedValue()) { 5756207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5757207618Srdivacky Changed = true; 5758207618Srdivacky break; 5759207618Srdivacky } 5760207618Srdivacky if (RA.isMinSignedValue()) goto trivially_true; 5761207618Srdivacky 5762207618Srdivacky Pred = ICmpInst::ICMP_SGT; 5763207618Srdivacky RHS = getConstant(RA - 1); 5764207618Srdivacky Changed = true; 5765207618Srdivacky break; 5766207618Srdivacky case ICmpInst::ICMP_SLE: 5767207618Srdivacky if ((RA + 1).isMaxSignedValue()) { 5768207618Srdivacky Pred = ICmpInst::ICMP_NE; 5769207618Srdivacky RHS = getConstant(RA + 1); 5770207618Srdivacky Changed = true; 5771207618Srdivacky break; 5772207618Srdivacky } 5773207618Srdivacky if (RA.isMinSignedValue()) { 5774207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5775207618Srdivacky Changed = true; 5776207618Srdivacky break; 5777207618Srdivacky } 5778207618Srdivacky if (RA.isMaxSignedValue()) goto trivially_true; 5779207618Srdivacky 5780207618Srdivacky Pred = ICmpInst::ICMP_SLT; 5781207618Srdivacky RHS = getConstant(RA + 1); 5782207618Srdivacky Changed = true; 5783207618Srdivacky break; 5784207618Srdivacky case ICmpInst::ICMP_UGT: 5785207618Srdivacky if (RA.isMinValue()) { 5786207618Srdivacky Pred = ICmpInst::ICMP_NE; 5787207618Srdivacky Changed = true; 5788207618Srdivacky break; 5789207618Srdivacky } 5790207618Srdivacky if ((RA + 1).isMaxValue()) { 5791207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5792207618Srdivacky RHS = getConstant(RA + 1); 5793207618Srdivacky Changed = true; 5794207618Srdivacky break; 5795207618Srdivacky } 5796207618Srdivacky if (RA.isMaxValue()) goto trivially_false; 5797207618Srdivacky break; 5798207618Srdivacky case ICmpInst::ICMP_ULT: 5799207618Srdivacky if (RA.isMaxValue()) { 5800207618Srdivacky Pred = ICmpInst::ICMP_NE; 5801207618Srdivacky Changed = true; 5802207618Srdivacky break; 5803207618Srdivacky } 5804207618Srdivacky if ((RA - 1).isMinValue()) { 5805207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5806207618Srdivacky RHS = getConstant(RA - 1); 5807207618Srdivacky Changed = true; 5808207618Srdivacky break; 5809207618Srdivacky } 5810207618Srdivacky if (RA.isMinValue()) goto trivially_false; 5811207618Srdivacky break; 5812207618Srdivacky case ICmpInst::ICMP_SGT: 5813207618Srdivacky if (RA.isMinSignedValue()) { 5814207618Srdivacky Pred = ICmpInst::ICMP_NE; 5815207618Srdivacky Changed = true; 5816207618Srdivacky break; 5817207618Srdivacky } 5818207618Srdivacky if ((RA + 1).isMaxSignedValue()) { 5819207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5820207618Srdivacky RHS = getConstant(RA + 1); 5821207618Srdivacky Changed = true; 5822207618Srdivacky break; 5823207618Srdivacky } 5824207618Srdivacky if (RA.isMaxSignedValue()) goto trivially_false; 5825207618Srdivacky break; 5826207618Srdivacky case ICmpInst::ICMP_SLT: 5827207618Srdivacky if (RA.isMaxSignedValue()) { 5828207618Srdivacky Pred = ICmpInst::ICMP_NE; 5829207618Srdivacky Changed = true; 5830207618Srdivacky break; 5831207618Srdivacky } 5832207618Srdivacky if ((RA - 1).isMinSignedValue()) { 5833207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5834207618Srdivacky RHS = getConstant(RA - 1); 5835207618Srdivacky Changed = true; 5836207618Srdivacky break; 5837207618Srdivacky } 5838207618Srdivacky if (RA.isMinSignedValue()) goto trivially_false; 5839207618Srdivacky break; 5840207618Srdivacky } 5841207618Srdivacky } 5842207618Srdivacky 5843207618Srdivacky // Check for obvious equality. 5844207618Srdivacky if (HasSameValue(LHS, RHS)) { 5845207618Srdivacky if (ICmpInst::isTrueWhenEqual(Pred)) 5846207618Srdivacky goto trivially_true; 5847207618Srdivacky if (ICmpInst::isFalseWhenEqual(Pred)) 5848207618Srdivacky goto trivially_false; 5849207618Srdivacky } 5850207618Srdivacky 5851207618Srdivacky // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by 5852207618Srdivacky // adding or subtracting 1 from one of the operands. 5853207618Srdivacky switch (Pred) { 5854207618Srdivacky case ICmpInst::ICMP_SLE: 5855207618Srdivacky if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) { 5856207618Srdivacky RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 5857221345Sdim SCEV::FlagNSW); 5858207618Srdivacky Pred = ICmpInst::ICMP_SLT; 5859207618Srdivacky Changed = true; 5860207618Srdivacky } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) { 5861207618Srdivacky LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 5862221345Sdim SCEV::FlagNSW); 5863207618Srdivacky Pred = ICmpInst::ICMP_SLT; 5864207618Srdivacky Changed = true; 5865207618Srdivacky } 5866207618Srdivacky break; 5867207618Srdivacky case ICmpInst::ICMP_SGE: 5868207618Srdivacky if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) { 5869207618Srdivacky RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 5870221345Sdim SCEV::FlagNSW); 5871207618Srdivacky Pred = ICmpInst::ICMP_SGT; 5872207618Srdivacky Changed = true; 5873207618Srdivacky } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) { 5874207618Srdivacky LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 5875221345Sdim SCEV::FlagNSW); 5876207618Srdivacky Pred = ICmpInst::ICMP_SGT; 5877207618Srdivacky Changed = true; 5878207618Srdivacky } 5879207618Srdivacky break; 5880207618Srdivacky case ICmpInst::ICMP_ULE: 5881207618Srdivacky if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) { 5882207618Srdivacky RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS, 5883221345Sdim SCEV::FlagNUW); 5884207618Srdivacky Pred = ICmpInst::ICMP_ULT; 5885207618Srdivacky Changed = true; 5886207618Srdivacky } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) { 5887207618Srdivacky LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS, 5888221345Sdim SCEV::FlagNUW); 5889207618Srdivacky Pred = ICmpInst::ICMP_ULT; 5890207618Srdivacky Changed = true; 5891207618Srdivacky } 5892207618Srdivacky break; 5893207618Srdivacky case ICmpInst::ICMP_UGE: 5894207618Srdivacky if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) { 5895207618Srdivacky RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS, 5896221345Sdim SCEV::FlagNUW); 5897207618Srdivacky Pred = ICmpInst::ICMP_UGT; 5898207618Srdivacky Changed = true; 5899207618Srdivacky } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) { 5900207618Srdivacky LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS, 5901221345Sdim SCEV::FlagNUW); 5902207618Srdivacky Pred = ICmpInst::ICMP_UGT; 5903207618Srdivacky Changed = true; 5904207618Srdivacky } 5905207618Srdivacky break; 5906207618Srdivacky default: 5907207618Srdivacky break; 5908207618Srdivacky } 5909207618Srdivacky 5910207618Srdivacky // TODO: More simplifications are possible here. 5911207618Srdivacky 5912239462Sdim // Recursively simplify until we either hit a recursion limit or nothing 5913239462Sdim // changes. 5914239462Sdim if (Changed) 5915239462Sdim return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1); 5916239462Sdim 5917207618Srdivacky return Changed; 5918207618Srdivacky 5919207618Srdivackytrivially_true: 5920207618Srdivacky // Return 0 == 0. 5921218893Sdim LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 5922207618Srdivacky Pred = ICmpInst::ICMP_EQ; 5923207618Srdivacky return true; 5924207618Srdivacky 5925207618Srdivackytrivially_false: 5926207618Srdivacky // Return 0 != 0. 5927218893Sdim LHS = RHS = getConstant(ConstantInt::getFalse(getContext())); 5928207618Srdivacky Pred = ICmpInst::ICMP_NE; 5929207618Srdivacky return true; 5930207618Srdivacky} 5931207618Srdivacky 5932198090Srdivackybool ScalarEvolution::isKnownNegative(const SCEV *S) { 5933198090Srdivacky return getSignedRange(S).getSignedMax().isNegative(); 5934198090Srdivacky} 5935198090Srdivacky 5936198090Srdivackybool ScalarEvolution::isKnownPositive(const SCEV *S) { 5937198090Srdivacky return getSignedRange(S).getSignedMin().isStrictlyPositive(); 5938198090Srdivacky} 5939198090Srdivacky 5940198090Srdivackybool ScalarEvolution::isKnownNonNegative(const SCEV *S) { 5941198090Srdivacky return !getSignedRange(S).getSignedMin().isNegative(); 5942198090Srdivacky} 5943198090Srdivacky 5944198090Srdivackybool ScalarEvolution::isKnownNonPositive(const SCEV *S) { 5945198090Srdivacky return !getSignedRange(S).getSignedMax().isStrictlyPositive(); 5946198090Srdivacky} 5947198090Srdivacky 5948198090Srdivackybool ScalarEvolution::isKnownNonZero(const SCEV *S) { 5949198090Srdivacky return isKnownNegative(S) || isKnownPositive(S); 5950198090Srdivacky} 5951198090Srdivacky 5952198090Srdivackybool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred, 5953198090Srdivacky const SCEV *LHS, const SCEV *RHS) { 5954207618Srdivacky // Canonicalize the inputs first. 5955207618Srdivacky (void)SimplifyICmpOperands(Pred, LHS, RHS); 5956198090Srdivacky 5957207618Srdivacky // If LHS or RHS is an addrec, check to see if the condition is true in 5958207618Srdivacky // every iteration of the loop. 5959207618Srdivacky if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) 5960207618Srdivacky if (isLoopEntryGuardedByCond( 5961207618Srdivacky AR->getLoop(), Pred, AR->getStart(), RHS) && 5962207618Srdivacky isLoopBackedgeGuardedByCond( 5963207618Srdivacky AR->getLoop(), Pred, AR->getPostIncExpr(*this), RHS)) 5964207618Srdivacky return true; 5965207618Srdivacky if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) 5966207618Srdivacky if (isLoopEntryGuardedByCond( 5967207618Srdivacky AR->getLoop(), Pred, LHS, AR->getStart()) && 5968207618Srdivacky isLoopBackedgeGuardedByCond( 5969207618Srdivacky AR->getLoop(), Pred, LHS, AR->getPostIncExpr(*this))) 5970207618Srdivacky return true; 5971207618Srdivacky 5972207618Srdivacky // Otherwise see what can be done with known constant ranges. 5973207618Srdivacky return isKnownPredicateWithRanges(Pred, LHS, RHS); 5974207618Srdivacky} 5975207618Srdivacky 5976207618Srdivackybool 5977207618SrdivackyScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred, 5978207618Srdivacky const SCEV *LHS, const SCEV *RHS) { 5979198090Srdivacky if (HasSameValue(LHS, RHS)) 5980198090Srdivacky return ICmpInst::isTrueWhenEqual(Pred); 5981198090Srdivacky 5982207618Srdivacky // This code is split out from isKnownPredicate because it is called from 5983207618Srdivacky // within isLoopEntryGuardedByCond. 5984198090Srdivacky switch (Pred) { 5985198090Srdivacky default: 5986198090Srdivacky llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 5987198090Srdivacky case ICmpInst::ICMP_SGT: 5988198090Srdivacky Pred = ICmpInst::ICMP_SLT; 5989198090Srdivacky std::swap(LHS, RHS); 5990198090Srdivacky case ICmpInst::ICMP_SLT: { 5991198090Srdivacky ConstantRange LHSRange = getSignedRange(LHS); 5992198090Srdivacky ConstantRange RHSRange = getSignedRange(RHS); 5993198090Srdivacky if (LHSRange.getSignedMax().slt(RHSRange.getSignedMin())) 5994198090Srdivacky return true; 5995198090Srdivacky if (LHSRange.getSignedMin().sge(RHSRange.getSignedMax())) 5996198090Srdivacky return false; 5997198090Srdivacky break; 5998198090Srdivacky } 5999198090Srdivacky case ICmpInst::ICMP_SGE: 6000198090Srdivacky Pred = ICmpInst::ICMP_SLE; 6001198090Srdivacky std::swap(LHS, RHS); 6002198090Srdivacky case ICmpInst::ICMP_SLE: { 6003198090Srdivacky ConstantRange LHSRange = getSignedRange(LHS); 6004198090Srdivacky ConstantRange RHSRange = getSignedRange(RHS); 6005198090Srdivacky if (LHSRange.getSignedMax().sle(RHSRange.getSignedMin())) 6006198090Srdivacky return true; 6007198090Srdivacky if (LHSRange.getSignedMin().sgt(RHSRange.getSignedMax())) 6008198090Srdivacky return false; 6009198090Srdivacky break; 6010198090Srdivacky } 6011198090Srdivacky case ICmpInst::ICMP_UGT: 6012198090Srdivacky Pred = ICmpInst::ICMP_ULT; 6013198090Srdivacky std::swap(LHS, RHS); 6014198090Srdivacky case ICmpInst::ICMP_ULT: { 6015198090Srdivacky ConstantRange LHSRange = getUnsignedRange(LHS); 6016198090Srdivacky ConstantRange RHSRange = getUnsignedRange(RHS); 6017198090Srdivacky if (LHSRange.getUnsignedMax().ult(RHSRange.getUnsignedMin())) 6018198090Srdivacky return true; 6019198090Srdivacky if (LHSRange.getUnsignedMin().uge(RHSRange.getUnsignedMax())) 6020198090Srdivacky return false; 6021198090Srdivacky break; 6022198090Srdivacky } 6023198090Srdivacky case ICmpInst::ICMP_UGE: 6024198090Srdivacky Pred = ICmpInst::ICMP_ULE; 6025198090Srdivacky std::swap(LHS, RHS); 6026198090Srdivacky case ICmpInst::ICMP_ULE: { 6027198090Srdivacky ConstantRange LHSRange = getUnsignedRange(LHS); 6028198090Srdivacky ConstantRange RHSRange = getUnsignedRange(RHS); 6029198090Srdivacky if (LHSRange.getUnsignedMax().ule(RHSRange.getUnsignedMin())) 6030198090Srdivacky return true; 6031198090Srdivacky if (LHSRange.getUnsignedMin().ugt(RHSRange.getUnsignedMax())) 6032198090Srdivacky return false; 6033198090Srdivacky break; 6034198090Srdivacky } 6035198090Srdivacky case ICmpInst::ICMP_NE: { 6036198090Srdivacky if (getUnsignedRange(LHS).intersectWith(getUnsignedRange(RHS)).isEmptySet()) 6037198090Srdivacky return true; 6038198090Srdivacky if (getSignedRange(LHS).intersectWith(getSignedRange(RHS)).isEmptySet()) 6039198090Srdivacky return true; 6040198090Srdivacky 6041198090Srdivacky const SCEV *Diff = getMinusSCEV(LHS, RHS); 6042198090Srdivacky if (isKnownNonZero(Diff)) 6043198090Srdivacky return true; 6044198090Srdivacky break; 6045198090Srdivacky } 6046198090Srdivacky case ICmpInst::ICMP_EQ: 6047198090Srdivacky // The check at the top of the function catches the case where 6048198090Srdivacky // the values are known to be equal. 6049198090Srdivacky break; 6050198090Srdivacky } 6051198090Srdivacky return false; 6052198090Srdivacky} 6053198090Srdivacky 6054198090Srdivacky/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is 6055198090Srdivacky/// protected by a conditional between LHS and RHS. This is used to 6056198090Srdivacky/// to eliminate casts. 6057198090Srdivackybool 6058198090SrdivackyScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L, 6059198090Srdivacky ICmpInst::Predicate Pred, 6060198090Srdivacky const SCEV *LHS, const SCEV *RHS) { 6061193323Sed // Interpret a null as meaning no loop, where there is obviously no guard 6062193323Sed // (interprocedural conditions notwithstanding). 6063198090Srdivacky if (!L) return true; 6064198090Srdivacky 6065198090Srdivacky BasicBlock *Latch = L->getLoopLatch(); 6066198090Srdivacky if (!Latch) 6067198090Srdivacky return false; 6068198090Srdivacky 6069198090Srdivacky BranchInst *LoopContinuePredicate = 6070198090Srdivacky dyn_cast<BranchInst>(Latch->getTerminator()); 6071198090Srdivacky if (!LoopContinuePredicate || 6072198090Srdivacky LoopContinuePredicate->isUnconditional()) 6073198090Srdivacky return false; 6074198090Srdivacky 6075212904Sdim return isImpliedCond(Pred, LHS, RHS, 6076212904Sdim LoopContinuePredicate->getCondition(), 6077198090Srdivacky LoopContinuePredicate->getSuccessor(0) != L->getHeader()); 6078198090Srdivacky} 6079198090Srdivacky 6080207618Srdivacky/// isLoopEntryGuardedByCond - Test whether entry to the loop is protected 6081198090Srdivacky/// by a conditional between LHS and RHS. This is used to help avoid max 6082198090Srdivacky/// expressions in loop trip counts, and to eliminate casts. 6083198090Srdivackybool 6084207618SrdivackyScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, 6085207618Srdivacky ICmpInst::Predicate Pred, 6086207618Srdivacky const SCEV *LHS, const SCEV *RHS) { 6087198090Srdivacky // Interpret a null as meaning no loop, where there is obviously no guard 6088198090Srdivacky // (interprocedural conditions notwithstanding). 6089193323Sed if (!L) return false; 6090193323Sed 6091193323Sed // Starting at the loop predecessor, climb up the predecessor chain, as long 6092193323Sed // as there are predecessors that can be found that have unique successors 6093193323Sed // leading to the original header. 6094207618Srdivacky for (std::pair<BasicBlock *, BasicBlock *> 6095210299Sed Pair(L->getLoopPredecessor(), L->getHeader()); 6096207618Srdivacky Pair.first; 6097207618Srdivacky Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) { 6098193323Sed 6099193323Sed BranchInst *LoopEntryPredicate = 6100207618Srdivacky dyn_cast<BranchInst>(Pair.first->getTerminator()); 6101193323Sed if (!LoopEntryPredicate || 6102193323Sed LoopEntryPredicate->isUnconditional()) 6103193323Sed continue; 6104193323Sed 6105212904Sdim if (isImpliedCond(Pred, LHS, RHS, 6106212904Sdim LoopEntryPredicate->getCondition(), 6107207618Srdivacky LoopEntryPredicate->getSuccessor(0) != Pair.second)) 6108195098Sed return true; 6109195098Sed } 6110193323Sed 6111195098Sed return false; 6112195098Sed} 6113193323Sed 6114239462Sdim/// RAII wrapper to prevent recursive application of isImpliedCond. 6115239462Sdim/// ScalarEvolution's PendingLoopPredicates set must be empty unless we are 6116239462Sdim/// currently evaluating isImpliedCond. 6117239462Sdimstruct MarkPendingLoopPredicate { 6118239462Sdim Value *Cond; 6119239462Sdim DenseSet<Value*> &LoopPreds; 6120239462Sdim bool Pending; 6121239462Sdim 6122239462Sdim MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP) 6123239462Sdim : Cond(C), LoopPreds(LP) { 6124239462Sdim Pending = !LoopPreds.insert(Cond).second; 6125239462Sdim } 6126239462Sdim ~MarkPendingLoopPredicate() { 6127239462Sdim if (!Pending) 6128239462Sdim LoopPreds.erase(Cond); 6129239462Sdim } 6130239462Sdim}; 6131239462Sdim 6132198090Srdivacky/// isImpliedCond - Test whether the condition described by Pred, LHS, 6133198090Srdivacky/// and RHS is true whenever the given Cond value evaluates to true. 6134212904Sdimbool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred, 6135198090Srdivacky const SCEV *LHS, const SCEV *RHS, 6136212904Sdim Value *FoundCondValue, 6137198090Srdivacky bool Inverse) { 6138239462Sdim MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates); 6139239462Sdim if (Mark.Pending) 6140239462Sdim return false; 6141239462Sdim 6142204642Srdivacky // Recursively handle And and Or conditions. 6143212904Sdim if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) { 6144195098Sed if (BO->getOpcode() == Instruction::And) { 6145195098Sed if (!Inverse) 6146212904Sdim return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 6147212904Sdim isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 6148195098Sed } else if (BO->getOpcode() == Instruction::Or) { 6149195098Sed if (Inverse) 6150212904Sdim return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) || 6151212904Sdim isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse); 6152195098Sed } 6153195098Sed } 6154195098Sed 6155212904Sdim ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue); 6156195098Sed if (!ICI) return false; 6157195098Sed 6158198090Srdivacky // Bail if the ICmp's operands' types are wider than the needed type 6159198090Srdivacky // before attempting to call getSCEV on them. This avoids infinite 6160198090Srdivacky // recursion, since the analysis of widening casts can require loop 6161198090Srdivacky // exit condition information for overflow checking, which would 6162198090Srdivacky // lead back here. 6163198090Srdivacky if (getTypeSizeInBits(LHS->getType()) < 6164198090Srdivacky getTypeSizeInBits(ICI->getOperand(0)->getType())) 6165198090Srdivacky return false; 6166198090Srdivacky 6167249423Sdim // Now that we found a conditional branch that dominates the loop or controls 6168249423Sdim // the loop latch. Check to see if it is the comparison we are looking for. 6169198090Srdivacky ICmpInst::Predicate FoundPred; 6170195098Sed if (Inverse) 6171198090Srdivacky FoundPred = ICI->getInversePredicate(); 6172195098Sed else 6173198090Srdivacky FoundPred = ICI->getPredicate(); 6174195098Sed 6175198090Srdivacky const SCEV *FoundLHS = getSCEV(ICI->getOperand(0)); 6176198090Srdivacky const SCEV *FoundRHS = getSCEV(ICI->getOperand(1)); 6177198090Srdivacky 6178198090Srdivacky // Balance the types. The case where FoundLHS' type is wider than 6179198090Srdivacky // LHS' type is checked for above. 6180198090Srdivacky if (getTypeSizeInBits(LHS->getType()) > 6181198090Srdivacky getTypeSizeInBits(FoundLHS->getType())) { 6182198090Srdivacky if (CmpInst::isSigned(Pred)) { 6183198090Srdivacky FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType()); 6184198090Srdivacky FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType()); 6185198090Srdivacky } else { 6186198090Srdivacky FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType()); 6187198090Srdivacky FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType()); 6188198090Srdivacky } 6189198090Srdivacky } 6190198090Srdivacky 6191198090Srdivacky // Canonicalize the query to match the way instcombine will have 6192198090Srdivacky // canonicalized the comparison. 6193207618Srdivacky if (SimplifyICmpOperands(Pred, LHS, RHS)) 6194207618Srdivacky if (LHS == RHS) 6195207618Srdivacky return CmpInst::isTrueWhenEqual(Pred); 6196207618Srdivacky if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS)) 6197207618Srdivacky if (FoundLHS == FoundRHS) 6198243830Sdim return CmpInst::isFalseWhenEqual(FoundPred); 6199193323Sed 6200198090Srdivacky // Check to see if we can make the LHS or RHS match. 6201198090Srdivacky if (LHS == FoundRHS || RHS == FoundLHS) { 6202198090Srdivacky if (isa<SCEVConstant>(RHS)) { 6203198090Srdivacky std::swap(FoundLHS, FoundRHS); 6204198090Srdivacky FoundPred = ICmpInst::getSwappedPredicate(FoundPred); 6205198090Srdivacky } else { 6206198090Srdivacky std::swap(LHS, RHS); 6207198090Srdivacky Pred = ICmpInst::getSwappedPredicate(Pred); 6208198090Srdivacky } 6209198090Srdivacky } 6210193323Sed 6211198090Srdivacky // Check whether the found predicate is the same as the desired predicate. 6212198090Srdivacky if (FoundPred == Pred) 6213198090Srdivacky return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS); 6214198090Srdivacky 6215198090Srdivacky // Check whether swapping the found predicate makes it the same as the 6216198090Srdivacky // desired predicate. 6217198090Srdivacky if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) { 6218198090Srdivacky if (isa<SCEVConstant>(RHS)) 6219198090Srdivacky return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS); 6220198090Srdivacky else 6221198090Srdivacky return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred), 6222198090Srdivacky RHS, LHS, FoundLHS, FoundRHS); 6223198090Srdivacky } 6224198090Srdivacky 6225198090Srdivacky // Check whether the actual condition is beyond sufficient. 6226198090Srdivacky if (FoundPred == ICmpInst::ICMP_EQ) 6227198090Srdivacky if (ICmpInst::isTrueWhenEqual(Pred)) 6228198090Srdivacky if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS)) 6229198090Srdivacky return true; 6230198090Srdivacky if (Pred == ICmpInst::ICMP_NE) 6231198090Srdivacky if (!ICmpInst::isTrueWhenEqual(FoundPred)) 6232198090Srdivacky if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS)) 6233198090Srdivacky return true; 6234198090Srdivacky 6235198090Srdivacky // Otherwise assume the worst. 6236198090Srdivacky return false; 6237193323Sed} 6238193323Sed 6239198090Srdivacky/// isImpliedCondOperands - Test whether the condition described by Pred, 6240204642Srdivacky/// LHS, and RHS is true whenever the condition described by Pred, FoundLHS, 6241198090Srdivacky/// and FoundRHS is true. 6242198090Srdivackybool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, 6243198090Srdivacky const SCEV *LHS, const SCEV *RHS, 6244198090Srdivacky const SCEV *FoundLHS, 6245198090Srdivacky const SCEV *FoundRHS) { 6246198090Srdivacky return isImpliedCondOperandsHelper(Pred, LHS, RHS, 6247198090Srdivacky FoundLHS, FoundRHS) || 6248198090Srdivacky // ~x < ~y --> x > y 6249198090Srdivacky isImpliedCondOperandsHelper(Pred, LHS, RHS, 6250198090Srdivacky getNotSCEV(FoundRHS), 6251198090Srdivacky getNotSCEV(FoundLHS)); 6252198090Srdivacky} 6253198090Srdivacky 6254198090Srdivacky/// isImpliedCondOperandsHelper - Test whether the condition described by 6255204642Srdivacky/// Pred, LHS, and RHS is true whenever the condition described by Pred, 6256198090Srdivacky/// FoundLHS, and FoundRHS is true. 6257198090Srdivackybool 6258198090SrdivackyScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred, 6259198090Srdivacky const SCEV *LHS, const SCEV *RHS, 6260198090Srdivacky const SCEV *FoundLHS, 6261198090Srdivacky const SCEV *FoundRHS) { 6262198090Srdivacky switch (Pred) { 6263198090Srdivacky default: llvm_unreachable("Unexpected ICmpInst::Predicate value!"); 6264198090Srdivacky case ICmpInst::ICMP_EQ: 6265198090Srdivacky case ICmpInst::ICMP_NE: 6266198090Srdivacky if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS)) 6267198090Srdivacky return true; 6268198090Srdivacky break; 6269198090Srdivacky case ICmpInst::ICMP_SLT: 6270198090Srdivacky case ICmpInst::ICMP_SLE: 6271207618Srdivacky if (isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, LHS, FoundLHS) && 6272207618Srdivacky isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, RHS, FoundRHS)) 6273198090Srdivacky return true; 6274198090Srdivacky break; 6275198090Srdivacky case ICmpInst::ICMP_SGT: 6276198090Srdivacky case ICmpInst::ICMP_SGE: 6277207618Srdivacky if (isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, LHS, FoundLHS) && 6278207618Srdivacky isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, RHS, FoundRHS)) 6279198090Srdivacky return true; 6280198090Srdivacky break; 6281198090Srdivacky case ICmpInst::ICMP_ULT: 6282198090Srdivacky case ICmpInst::ICMP_ULE: 6283207618Srdivacky if (isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, LHS, FoundLHS) && 6284207618Srdivacky isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, RHS, FoundRHS)) 6285198090Srdivacky return true; 6286198090Srdivacky break; 6287198090Srdivacky case ICmpInst::ICMP_UGT: 6288198090Srdivacky case ICmpInst::ICMP_UGE: 6289207618Srdivacky if (isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, LHS, FoundLHS) && 6290207618Srdivacky isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, RHS, FoundRHS)) 6291198090Srdivacky return true; 6292198090Srdivacky break; 6293198090Srdivacky } 6294198090Srdivacky 6295198090Srdivacky return false; 6296198090Srdivacky} 6297198090Srdivacky 6298194612Sed/// getBECount - Subtract the end and start values and divide by the step, 6299194612Sed/// rounding up, to get the number of times the backedge is executed. Return 6300194612Sed/// CouldNotCompute if an intermediate computation overflows. 6301198090Srdivackyconst SCEV *ScalarEvolution::getBECount(const SCEV *Start, 6302198090Srdivacky const SCEV *End, 6303198090Srdivacky const SCEV *Step, 6304198090Srdivacky bool NoWrap) { 6305203954Srdivacky assert(!isKnownNegative(Step) && 6306203954Srdivacky "This code doesn't handle negative strides yet!"); 6307203954Srdivacky 6308226633Sdim Type *Ty = Start->getType(); 6309221345Sdim 6310221345Sdim // When Start == End, we have an exact BECount == 0. Short-circuit this case 6311221345Sdim // here because SCEV may not be able to determine that the unsigned division 6312221345Sdim // after rounding is zero. 6313221345Sdim if (Start == End) 6314221345Sdim return getConstant(Ty, 0); 6315221345Sdim 6316207618Srdivacky const SCEV *NegOne = getConstant(Ty, (uint64_t)-1); 6317198090Srdivacky const SCEV *Diff = getMinusSCEV(End, Start); 6318198090Srdivacky const SCEV *RoundUp = getAddExpr(Step, NegOne); 6319194612Sed 6320194612Sed // Add an adjustment to the difference between End and Start so that 6321194612Sed // the division will effectively round up. 6322198090Srdivacky const SCEV *Add = getAddExpr(Diff, RoundUp); 6323194612Sed 6324198090Srdivacky if (!NoWrap) { 6325198090Srdivacky // Check Add for unsigned overflow. 6326198090Srdivacky // TODO: More sophisticated things could be done here. 6327226633Sdim Type *WideTy = IntegerType::get(getContext(), 6328198090Srdivacky getTypeSizeInBits(Ty) + 1); 6329198090Srdivacky const SCEV *EDiff = getZeroExtendExpr(Diff, WideTy); 6330198090Srdivacky const SCEV *ERoundUp = getZeroExtendExpr(RoundUp, WideTy); 6331198090Srdivacky const SCEV *OperandExtendedAdd = getAddExpr(EDiff, ERoundUp); 6332198090Srdivacky if (getZeroExtendExpr(Add, WideTy) != OperandExtendedAdd) 6333198090Srdivacky return getCouldNotCompute(); 6334198090Srdivacky } 6335194612Sed 6336194612Sed return getUDivExpr(Add, Step); 6337194612Sed} 6338194612Sed 6339193323Sed/// HowManyLessThans - Return the number of times a backedge containing the 6340193323Sed/// specified less-than comparison will execute. If not computable, return 6341193630Sed/// CouldNotCompute. 6342251662Sdim/// 6343251662Sdim/// @param IsSubExpr is true when the LHS < RHS condition does not directly 6344251662Sdim/// control the branch. In this case, we can only compute an iteration count for 6345251662Sdim/// a subexpression that cannot overflow before evaluating true. 6346226633SdimScalarEvolution::ExitLimit 6347195098SedScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, 6348251662Sdim const Loop *L, bool isSigned, 6349251662Sdim bool IsSubExpr) { 6350193323Sed // Only handle: "ADDREC < LoopInvariant". 6351218893Sdim if (!isLoopInvariant(RHS, L)) return getCouldNotCompute(); 6352193323Sed 6353193323Sed const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS); 6354193323Sed if (!AddRec || AddRec->getLoop() != L) 6355195340Sed return getCouldNotCompute(); 6356193323Sed 6357198090Srdivacky // Check to see if we have a flag which makes analysis easy. 6358251662Sdim bool NoWrap = false; 6359251662Sdim if (!IsSubExpr) { 6360251662Sdim NoWrap = AddRec->getNoWrapFlags( 6361251662Sdim (SCEV::NoWrapFlags)(((isSigned ? SCEV::FlagNSW : SCEV::FlagNUW)) 6362251662Sdim | SCEV::FlagNW)); 6363251662Sdim } 6364193323Sed if (AddRec->isAffine()) { 6365193323Sed unsigned BitWidth = getTypeSizeInBits(AddRec->getType()); 6366198090Srdivacky const SCEV *Step = AddRec->getStepRecurrence(*this); 6367193323Sed 6368203954Srdivacky if (Step->isZero()) 6369195340Sed return getCouldNotCompute(); 6370203954Srdivacky if (Step->isOne()) { 6371193323Sed // With unit stride, the iteration never steps past the limit value. 6372203954Srdivacky } else if (isKnownPositive(Step)) { 6373203954Srdivacky // Test whether a positive iteration can step past the limit 6374203954Srdivacky // value and past the maximum value for its type in a single step. 6375203954Srdivacky // Note that it's not sufficient to check NoWrap here, because even 6376203954Srdivacky // though the value after a wrap is undefined, it's not undefined 6377203954Srdivacky // behavior, so if wrap does occur, the loop could either terminate or 6378203954Srdivacky // loop infinitely, but in either case, the loop is guaranteed to 6379203954Srdivacky // iterate at least until the iteration where the wrapping occurs. 6380207618Srdivacky const SCEV *One = getConstant(Step->getType(), 1); 6381203954Srdivacky if (isSigned) { 6382203954Srdivacky APInt Max = APInt::getSignedMaxValue(BitWidth); 6383203954Srdivacky if ((Max - getSignedRange(getMinusSCEV(Step, One)).getSignedMax()) 6384203954Srdivacky .slt(getSignedRange(RHS).getSignedMax())) 6385203954Srdivacky return getCouldNotCompute(); 6386203954Srdivacky } else { 6387203954Srdivacky APInt Max = APInt::getMaxValue(BitWidth); 6388203954Srdivacky if ((Max - getUnsignedRange(getMinusSCEV(Step, One)).getUnsignedMax()) 6389203954Srdivacky .ult(getUnsignedRange(RHS).getUnsignedMax())) 6390203954Srdivacky return getCouldNotCompute(); 6391203954Srdivacky } 6392193323Sed } else 6393203954Srdivacky // TODO: Handle negative strides here and below. 6394195340Sed return getCouldNotCompute(); 6395193323Sed 6396193323Sed // We know the LHS is of the form {n,+,s} and the RHS is some loop-invariant 6397193323Sed // m. So, we count the number of iterations in which {n,+,s} < m is true. 6398193323Sed // Note that we cannot simply return max(m-n,0)/s because it's not safe to 6399193323Sed // treat m-n as signed nor unsigned due to overflow possibility. 6400193323Sed 6401193323Sed // First, we get the value of the LHS in the first iteration: n 6402198090Srdivacky const SCEV *Start = AddRec->getOperand(0); 6403193323Sed 6404193323Sed // Determine the minimum constant start value. 6405198090Srdivacky const SCEV *MinStart = getConstant(isSigned ? 6406198090Srdivacky getSignedRange(Start).getSignedMin() : 6407198090Srdivacky getUnsignedRange(Start).getUnsignedMin()); 6408193323Sed 6409193323Sed // If we know that the condition is true in order to enter the loop, 6410193323Sed // then we know that it will run exactly (m-n)/s times. Otherwise, we 6411193323Sed // only know that it will execute (max(m,n)-n)/s times. In both cases, 6412193323Sed // the division must round up. 6413198090Srdivacky const SCEV *End = RHS; 6414207618Srdivacky if (!isLoopEntryGuardedByCond(L, 6415207618Srdivacky isSigned ? ICmpInst::ICMP_SLT : 6416207618Srdivacky ICmpInst::ICMP_ULT, 6417207618Srdivacky getMinusSCEV(Start, Step), RHS)) 6418193323Sed End = isSigned ? getSMaxExpr(RHS, Start) 6419193323Sed : getUMaxExpr(RHS, Start); 6420193323Sed 6421193323Sed // Determine the maximum constant end value. 6422198090Srdivacky const SCEV *MaxEnd = getConstant(isSigned ? 6423198090Srdivacky getSignedRange(End).getSignedMax() : 6424198090Srdivacky getUnsignedRange(End).getUnsignedMax()); 6425193323Sed 6426203954Srdivacky // If MaxEnd is within a step of the maximum integer value in its type, 6427203954Srdivacky // adjust it down to the minimum value which would produce the same effect. 6428204642Srdivacky // This allows the subsequent ceiling division of (N+(step-1))/step to 6429203954Srdivacky // compute the correct value. 6430203954Srdivacky const SCEV *StepMinusOne = getMinusSCEV(Step, 6431207618Srdivacky getConstant(Step->getType(), 1)); 6432203954Srdivacky MaxEnd = isSigned ? 6433203954Srdivacky getSMinExpr(MaxEnd, 6434203954Srdivacky getMinusSCEV(getConstant(APInt::getSignedMaxValue(BitWidth)), 6435203954Srdivacky StepMinusOne)) : 6436203954Srdivacky getUMinExpr(MaxEnd, 6437203954Srdivacky getMinusSCEV(getConstant(APInt::getMaxValue(BitWidth)), 6438203954Srdivacky StepMinusOne)); 6439203954Srdivacky 6440193323Sed // Finally, we subtract these two values and divide, rounding up, to get 6441193323Sed // the number of times the backedge is executed. 6442198090Srdivacky const SCEV *BECount = getBECount(Start, End, Step, NoWrap); 6443193323Sed 6444193323Sed // The maximum backedge count is similar, except using the minimum start 6445193323Sed // value and the maximum end value. 6446221345Sdim // If we already have an exact constant BECount, use it instead. 6447221345Sdim const SCEV *MaxBECount = isa<SCEVConstant>(BECount) ? BECount 6448221345Sdim : getBECount(MinStart, MaxEnd, Step, NoWrap); 6449193323Sed 6450221345Sdim // If the stride is nonconstant, and NoWrap == true, then 6451221345Sdim // getBECount(MinStart, MaxEnd) may not compute. This would result in an 6452221345Sdim // exact BECount and invalid MaxBECount, which should be avoided to catch 6453221345Sdim // more optimization opportunities. 6454221345Sdim if (isa<SCEVCouldNotCompute>(MaxBECount)) 6455221345Sdim MaxBECount = BECount; 6456221345Sdim 6457226633Sdim return ExitLimit(BECount, MaxBECount); 6458193323Sed } 6459193323Sed 6460195340Sed return getCouldNotCompute(); 6461193323Sed} 6462193323Sed 6463193323Sed/// getNumIterationsInRange - Return the number of iterations of this loop that 6464193323Sed/// produce values in the specified constant range. Another way of looking at 6465193323Sed/// this is that it returns the first iteration number where the value is not in 6466193323Sed/// the condition, thus computing the exit count. If the iteration count can't 6467193323Sed/// be computed, an instance of SCEVCouldNotCompute is returned. 6468198090Srdivackyconst SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range, 6469195098Sed ScalarEvolution &SE) const { 6470193323Sed if (Range.isFullSet()) // Infinite loop. 6471193323Sed return SE.getCouldNotCompute(); 6472193323Sed 6473193323Sed // If the start is a non-zero constant, shift the range to simplify things. 6474193323Sed if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart())) 6475193323Sed if (!SC->getValue()->isZero()) { 6476198090Srdivacky SmallVector<const SCEV *, 4> Operands(op_begin(), op_end()); 6477207618Srdivacky Operands[0] = SE.getConstant(SC->getType(), 0); 6478221345Sdim const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(), 6479221345Sdim getNoWrapFlags(FlagNW)); 6480193323Sed if (const SCEVAddRecExpr *ShiftedAddRec = 6481193323Sed dyn_cast<SCEVAddRecExpr>(Shifted)) 6482193323Sed return ShiftedAddRec->getNumIterationsInRange( 6483193323Sed Range.subtract(SC->getValue()->getValue()), SE); 6484193323Sed // This is strange and shouldn't happen. 6485193323Sed return SE.getCouldNotCompute(); 6486193323Sed } 6487193323Sed 6488193323Sed // The only time we can solve this is when we have all constant indices. 6489193323Sed // Otherwise, we cannot determine the overflow conditions. 6490193323Sed for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6491193323Sed if (!isa<SCEVConstant>(getOperand(i))) 6492193323Sed return SE.getCouldNotCompute(); 6493193323Sed 6494193323Sed 6495193323Sed // Okay at this point we know that all elements of the chrec are constants and 6496193323Sed // that the start element is zero. 6497193323Sed 6498193323Sed // First check to see if the range contains zero. If not, the first 6499193323Sed // iteration exits. 6500193323Sed unsigned BitWidth = SE.getTypeSizeInBits(getType()); 6501193323Sed if (!Range.contains(APInt(BitWidth, 0))) 6502207618Srdivacky return SE.getConstant(getType(), 0); 6503193323Sed 6504193323Sed if (isAffine()) { 6505193323Sed // If this is an affine expression then we have this situation: 6506193323Sed // Solve {0,+,A} in Range === Ax in Range 6507193323Sed 6508193323Sed // We know that zero is in the range. If A is positive then we know that 6509193323Sed // the upper value of the range must be the first possible exit value. 6510193323Sed // If A is negative then the lower of the range is the last possible loop 6511193323Sed // value. Also note that we already checked for a full range. 6512193323Sed APInt One(BitWidth,1); 6513193323Sed APInt A = cast<SCEVConstant>(getOperand(1))->getValue()->getValue(); 6514193323Sed APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower(); 6515193323Sed 6516193323Sed // The exit value should be (End+A)/A. 6517193323Sed APInt ExitVal = (End + A).udiv(A); 6518198090Srdivacky ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal); 6519193323Sed 6520193323Sed // Evaluate at the exit value. If we really did fall out of the valid 6521193323Sed // range, then we computed our trip count, otherwise wrap around or other 6522193323Sed // things must have happened. 6523193323Sed ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE); 6524193323Sed if (Range.contains(Val->getValue())) 6525193323Sed return SE.getCouldNotCompute(); // Something strange happened 6526193323Sed 6527193323Sed // Ensure that the previous value is in the range. This is a sanity check. 6528193323Sed assert(Range.contains( 6529195098Sed EvaluateConstantChrecAtConstant(this, 6530198090Srdivacky ConstantInt::get(SE.getContext(), ExitVal - One), SE)->getValue()) && 6531193323Sed "Linear scev computation is off in a bad way!"); 6532193323Sed return SE.getConstant(ExitValue); 6533193323Sed } else if (isQuadratic()) { 6534193323Sed // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of the 6535193323Sed // quadratic equation to solve it. To do this, we must frame our problem in 6536193323Sed // terms of figuring out when zero is crossed, instead of when 6537193323Sed // Range.getUpper() is crossed. 6538198090Srdivacky SmallVector<const SCEV *, 4> NewOps(op_begin(), op_end()); 6539193323Sed NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper())); 6540221345Sdim const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop(), 6541221345Sdim // getNoWrapFlags(FlagNW) 6542221345Sdim FlagAnyWrap); 6543193323Sed 6544193323Sed // Next, solve the constructed addrec 6545198090Srdivacky std::pair<const SCEV *,const SCEV *> Roots = 6546193323Sed SolveQuadraticEquation(cast<SCEVAddRecExpr>(NewAddRec), SE); 6547193323Sed const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first); 6548193323Sed const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second); 6549193323Sed if (R1) { 6550193323Sed // Pick the smallest positive root value. 6551193323Sed if (ConstantInt *CB = 6552195098Sed dyn_cast<ConstantInt>(ConstantExpr::getICmp(ICmpInst::ICMP_ULT, 6553198090Srdivacky R1->getValue(), R2->getValue()))) { 6554193323Sed if (CB->getZExtValue() == false) 6555193323Sed std::swap(R1, R2); // R1 is the minimum root now. 6556193323Sed 6557193323Sed // Make sure the root is not off by one. The returned iteration should 6558193323Sed // not be in the range, but the previous one should be. When solving 6559193323Sed // for "X*X < 5", for example, we should not return a root of 2. 6560193323Sed ConstantInt *R1Val = EvaluateConstantChrecAtConstant(this, 6561193323Sed R1->getValue(), 6562193323Sed SE); 6563193323Sed if (Range.contains(R1Val->getValue())) { 6564193323Sed // The next iteration must be out of the range... 6565198090Srdivacky ConstantInt *NextVal = 6566198090Srdivacky ConstantInt::get(SE.getContext(), R1->getValue()->getValue()+1); 6567193323Sed 6568193323Sed R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 6569193323Sed if (!Range.contains(R1Val->getValue())) 6570193323Sed return SE.getConstant(NextVal); 6571193323Sed return SE.getCouldNotCompute(); // Something strange happened 6572193323Sed } 6573193323Sed 6574193323Sed // If R1 was not in the range, then it is a good return value. Make 6575193323Sed // sure that R1-1 WAS in the range though, just in case. 6576198090Srdivacky ConstantInt *NextVal = 6577198090Srdivacky ConstantInt::get(SE.getContext(), R1->getValue()->getValue()-1); 6578193323Sed R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE); 6579193323Sed if (Range.contains(R1Val->getValue())) 6580193323Sed return R1; 6581193323Sed return SE.getCouldNotCompute(); // Something strange happened 6582193323Sed } 6583193323Sed } 6584193323Sed } 6585193323Sed 6586193323Sed return SE.getCouldNotCompute(); 6587193323Sed} 6588193323Sed 6589193323Sed 6590193323Sed 6591193323Sed//===----------------------------------------------------------------------===// 6592193323Sed// SCEVCallbackVH Class Implementation 6593193323Sed//===----------------------------------------------------------------------===// 6594193323Sed 6595193323Sedvoid ScalarEvolution::SCEVCallbackVH::deleted() { 6596198090Srdivacky assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 6597193323Sed if (PHINode *PN = dyn_cast<PHINode>(getValPtr())) 6598193323Sed SE->ConstantEvolutionLoopExitValue.erase(PN); 6599212904Sdim SE->ValueExprMap.erase(getValPtr()); 6600193323Sed // this now dangles! 6601193323Sed} 6602193323Sed 6603212904Sdimvoid ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) { 6604198090Srdivacky assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!"); 6605193323Sed 6606193323Sed // Forget all the expressions associated with users of the old value, 6607193323Sed // so that future queries will recompute the expressions using the new 6608193323Sed // value. 6609212904Sdim Value *Old = getValPtr(); 6610193323Sed SmallVector<User *, 16> Worklist; 6611198090Srdivacky SmallPtrSet<User *, 8> Visited; 6612193323Sed for (Value::use_iterator UI = Old->use_begin(), UE = Old->use_end(); 6613193323Sed UI != UE; ++UI) 6614193323Sed Worklist.push_back(*UI); 6615193323Sed while (!Worklist.empty()) { 6616193323Sed User *U = Worklist.pop_back_val(); 6617193323Sed // Deleting the Old value will cause this to dangle. Postpone 6618193323Sed // that until everything else is done. 6619212904Sdim if (U == Old) 6620193323Sed continue; 6621198090Srdivacky if (!Visited.insert(U)) 6622198090Srdivacky continue; 6623193323Sed if (PHINode *PN = dyn_cast<PHINode>(U)) 6624193323Sed SE->ConstantEvolutionLoopExitValue.erase(PN); 6625212904Sdim SE->ValueExprMap.erase(U); 6626198090Srdivacky for (Value::use_iterator UI = U->use_begin(), UE = U->use_end(); 6627198090Srdivacky UI != UE; ++UI) 6628198090Srdivacky Worklist.push_back(*UI); 6629193323Sed } 6630212904Sdim // Delete the Old value. 6631212904Sdim if (PHINode *PN = dyn_cast<PHINode>(Old)) 6632212904Sdim SE->ConstantEvolutionLoopExitValue.erase(PN); 6633212904Sdim SE->ValueExprMap.erase(Old); 6634212904Sdim // this now dangles! 6635193323Sed} 6636193323Sed 6637193323SedScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se) 6638193323Sed : CallbackVH(V), SE(se) {} 6639193323Sed 6640193323Sed//===----------------------------------------------------------------------===// 6641193323Sed// ScalarEvolution Class Implementation 6642193323Sed//===----------------------------------------------------------------------===// 6643193323Sed 6644193323SedScalarEvolution::ScalarEvolution() 6645212904Sdim : FunctionPass(ID), FirstUnknown(0) { 6646218893Sdim initializeScalarEvolutionPass(*PassRegistry::getPassRegistry()); 6647193323Sed} 6648193323Sed 6649193323Sedbool ScalarEvolution::runOnFunction(Function &F) { 6650193323Sed this->F = &F; 6651193323Sed LI = &getAnalysis<LoopInfo>(); 6652243830Sdim TD = getAnalysisIfAvailable<DataLayout>(); 6653234353Sdim TLI = &getAnalysis<TargetLibraryInfo>(); 6654202878Srdivacky DT = &getAnalysis<DominatorTree>(); 6655193323Sed return false; 6656193323Sed} 6657193323Sed 6658193323Sedvoid ScalarEvolution::releaseMemory() { 6659212904Sdim // Iterate through all the SCEVUnknown instances and call their 6660212904Sdim // destructors, so that they release their references to their values. 6661212904Sdim for (SCEVUnknown *U = FirstUnknown; U; U = U->Next) 6662212904Sdim U->~SCEVUnknown(); 6663212904Sdim FirstUnknown = 0; 6664212904Sdim 6665212904Sdim ValueExprMap.clear(); 6666226633Sdim 6667226633Sdim // Free any extra memory created for ExitNotTakenInfo in the unlikely event 6668226633Sdim // that a loop had multiple computable exits. 6669226633Sdim for (DenseMap<const Loop*, BackedgeTakenInfo>::iterator I = 6670226633Sdim BackedgeTakenCounts.begin(), E = BackedgeTakenCounts.end(); 6671226633Sdim I != E; ++I) { 6672226633Sdim I->second.clear(); 6673226633Sdim } 6674226633Sdim 6675239462Sdim assert(PendingLoopPredicates.empty() && "isImpliedCond garbage"); 6676239462Sdim 6677193323Sed BackedgeTakenCounts.clear(); 6678193323Sed ConstantEvolutionLoopExitValue.clear(); 6679193323Sed ValuesAtScopes.clear(); 6680218893Sdim LoopDispositions.clear(); 6681218893Sdim BlockDispositions.clear(); 6682218893Sdim UnsignedRanges.clear(); 6683218893Sdim SignedRanges.clear(); 6684195340Sed UniqueSCEVs.clear(); 6685195340Sed SCEVAllocator.Reset(); 6686193323Sed} 6687193323Sed 6688193323Sedvoid ScalarEvolution::getAnalysisUsage(AnalysisUsage &AU) const { 6689193323Sed AU.setPreservesAll(); 6690193323Sed AU.addRequiredTransitive<LoopInfo>(); 6691202878Srdivacky AU.addRequiredTransitive<DominatorTree>(); 6692234353Sdim AU.addRequired<TargetLibraryInfo>(); 6693193323Sed} 6694193323Sed 6695193323Sedbool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) { 6696193323Sed return !isa<SCEVCouldNotCompute>(getBackedgeTakenCount(L)); 6697193323Sed} 6698193323Sed 6699193323Sedstatic void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, 6700193323Sed const Loop *L) { 6701193323Sed // Print all inner loops first 6702193323Sed for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 6703193323Sed PrintLoopInfo(OS, SE, *I); 6704193323Sed 6705202375Srdivacky OS << "Loop "; 6706202375Srdivacky WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false); 6707202375Srdivacky OS << ": "; 6708193323Sed 6709201360Srdivacky SmallVector<BasicBlock *, 8> ExitBlocks; 6710193323Sed L->getExitBlocks(ExitBlocks); 6711193323Sed if (ExitBlocks.size() != 1) 6712193323Sed OS << "<multiple exits> "; 6713193323Sed 6714193323Sed if (SE->hasLoopInvariantBackedgeTakenCount(L)) { 6715193323Sed OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L); 6716193323Sed } else { 6717193323Sed OS << "Unpredictable backedge-taken count. "; 6718193323Sed } 6719193323Sed 6720202375Srdivacky OS << "\n" 6721202375Srdivacky "Loop "; 6722202375Srdivacky WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false); 6723202375Srdivacky OS << ": "; 6724195098Sed 6725195098Sed if (!isa<SCEVCouldNotCompute>(SE->getMaxBackedgeTakenCount(L))) { 6726195098Sed OS << "max backedge-taken count is " << *SE->getMaxBackedgeTakenCount(L); 6727195098Sed } else { 6728195098Sed OS << "Unpredictable max backedge-taken count. "; 6729195098Sed } 6730195098Sed 6731195098Sed OS << "\n"; 6732193323Sed} 6733193323Sed 6734201360Srdivackyvoid ScalarEvolution::print(raw_ostream &OS, const Module *) const { 6735204642Srdivacky // ScalarEvolution's implementation of the print method is to print 6736193323Sed // out SCEV values of all instructions that are interesting. Doing 6737193323Sed // this potentially causes it to create new SCEV objects though, 6738193323Sed // which technically conflicts with the const qualifier. This isn't 6739198090Srdivacky // observable from outside the class though, so casting away the 6740198090Srdivacky // const isn't dangerous. 6741201360Srdivacky ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 6742193323Sed 6743202375Srdivacky OS << "Classifying expressions for: "; 6744202375Srdivacky WriteAsOperand(OS, F, /*PrintType=*/false); 6745202375Srdivacky OS << "\n"; 6746193323Sed for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) 6747207618Srdivacky if (isSCEVable(I->getType()) && !isa<CmpInst>(*I)) { 6748198090Srdivacky OS << *I << '\n'; 6749193323Sed OS << " --> "; 6750198090Srdivacky const SCEV *SV = SE.getSCEV(&*I); 6751193323Sed SV->print(OS); 6752193323Sed 6753194612Sed const Loop *L = LI->getLoopFor((*I).getParent()); 6754194612Sed 6755198090Srdivacky const SCEV *AtUse = SE.getSCEVAtScope(SV, L); 6756194612Sed if (AtUse != SV) { 6757194612Sed OS << " --> "; 6758194612Sed AtUse->print(OS); 6759194612Sed } 6760194612Sed 6761194612Sed if (L) { 6762194612Sed OS << "\t\t" "Exits: "; 6763198090Srdivacky const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop()); 6764218893Sdim if (!SE.isLoopInvariant(ExitValue, L)) { 6765193323Sed OS << "<<Unknown>>"; 6766193323Sed } else { 6767193323Sed OS << *ExitValue; 6768193323Sed } 6769193323Sed } 6770193323Sed 6771193323Sed OS << "\n"; 6772193323Sed } 6773193323Sed 6774202375Srdivacky OS << "Determining loop execution counts for: "; 6775202375Srdivacky WriteAsOperand(OS, F, /*PrintType=*/false); 6776202375Srdivacky OS << "\n"; 6777193323Sed for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I) 6778193323Sed PrintLoopInfo(OS, &SE, *I); 6779193323Sed} 6780193323Sed 6781218893SdimScalarEvolution::LoopDisposition 6782218893SdimScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) { 6783218893Sdim std::map<const Loop *, LoopDisposition> &Values = LoopDispositions[S]; 6784218893Sdim std::pair<std::map<const Loop *, LoopDisposition>::iterator, bool> Pair = 6785218893Sdim Values.insert(std::make_pair(L, LoopVariant)); 6786218893Sdim if (!Pair.second) 6787218893Sdim return Pair.first->second; 6788218893Sdim 6789218893Sdim LoopDisposition D = computeLoopDisposition(S, L); 6790218893Sdim return LoopDispositions[S][L] = D; 6791218893Sdim} 6792218893Sdim 6793218893SdimScalarEvolution::LoopDisposition 6794218893SdimScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) { 6795218893Sdim switch (S->getSCEVType()) { 6796218893Sdim case scConstant: 6797218893Sdim return LoopInvariant; 6798218893Sdim case scTruncate: 6799218893Sdim case scZeroExtend: 6800218893Sdim case scSignExtend: 6801218893Sdim return getLoopDisposition(cast<SCEVCastExpr>(S)->getOperand(), L); 6802218893Sdim case scAddRecExpr: { 6803218893Sdim const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 6804218893Sdim 6805218893Sdim // If L is the addrec's loop, it's computable. 6806218893Sdim if (AR->getLoop() == L) 6807218893Sdim return LoopComputable; 6808218893Sdim 6809218893Sdim // Add recurrences are never invariant in the function-body (null loop). 6810218893Sdim if (!L) 6811218893Sdim return LoopVariant; 6812218893Sdim 6813218893Sdim // This recurrence is variant w.r.t. L if L contains AR's loop. 6814218893Sdim if (L->contains(AR->getLoop())) 6815218893Sdim return LoopVariant; 6816218893Sdim 6817218893Sdim // This recurrence is invariant w.r.t. L if AR's loop contains L. 6818218893Sdim if (AR->getLoop()->contains(L)) 6819218893Sdim return LoopInvariant; 6820218893Sdim 6821218893Sdim // This recurrence is variant w.r.t. L if any of its operands 6822218893Sdim // are variant. 6823218893Sdim for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end(); 6824218893Sdim I != E; ++I) 6825218893Sdim if (!isLoopInvariant(*I, L)) 6826218893Sdim return LoopVariant; 6827218893Sdim 6828218893Sdim // Otherwise it's loop-invariant. 6829218893Sdim return LoopInvariant; 6830218893Sdim } 6831218893Sdim case scAddExpr: 6832218893Sdim case scMulExpr: 6833218893Sdim case scUMaxExpr: 6834218893Sdim case scSMaxExpr: { 6835218893Sdim const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 6836218893Sdim bool HasVarying = false; 6837218893Sdim for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 6838218893Sdim I != E; ++I) { 6839218893Sdim LoopDisposition D = getLoopDisposition(*I, L); 6840218893Sdim if (D == LoopVariant) 6841218893Sdim return LoopVariant; 6842218893Sdim if (D == LoopComputable) 6843218893Sdim HasVarying = true; 6844218893Sdim } 6845218893Sdim return HasVarying ? LoopComputable : LoopInvariant; 6846218893Sdim } 6847218893Sdim case scUDivExpr: { 6848218893Sdim const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 6849218893Sdim LoopDisposition LD = getLoopDisposition(UDiv->getLHS(), L); 6850218893Sdim if (LD == LoopVariant) 6851218893Sdim return LoopVariant; 6852218893Sdim LoopDisposition RD = getLoopDisposition(UDiv->getRHS(), L); 6853218893Sdim if (RD == LoopVariant) 6854218893Sdim return LoopVariant; 6855218893Sdim return (LD == LoopInvariant && RD == LoopInvariant) ? 6856218893Sdim LoopInvariant : LoopComputable; 6857218893Sdim } 6858218893Sdim case scUnknown: 6859218893Sdim // All non-instruction values are loop invariant. All instructions are loop 6860218893Sdim // invariant if they are not contained in the specified loop. 6861218893Sdim // Instructions are never considered invariant in the function body 6862218893Sdim // (null loop) because they are defined within the "loop". 6863218893Sdim if (Instruction *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) 6864218893Sdim return (L && !L->contains(I)) ? LoopInvariant : LoopVariant; 6865218893Sdim return LoopInvariant; 6866218893Sdim case scCouldNotCompute: 6867218893Sdim llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 6868234353Sdim default: llvm_unreachable("Unknown SCEV kind!"); 6869218893Sdim } 6870218893Sdim} 6871218893Sdim 6872218893Sdimbool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) { 6873218893Sdim return getLoopDisposition(S, L) == LoopInvariant; 6874218893Sdim} 6875218893Sdim 6876218893Sdimbool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) { 6877218893Sdim return getLoopDisposition(S, L) == LoopComputable; 6878218893Sdim} 6879218893Sdim 6880218893SdimScalarEvolution::BlockDisposition 6881218893SdimScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) { 6882218893Sdim std::map<const BasicBlock *, BlockDisposition> &Values = BlockDispositions[S]; 6883218893Sdim std::pair<std::map<const BasicBlock *, BlockDisposition>::iterator, bool> 6884218893Sdim Pair = Values.insert(std::make_pair(BB, DoesNotDominateBlock)); 6885218893Sdim if (!Pair.second) 6886218893Sdim return Pair.first->second; 6887218893Sdim 6888218893Sdim BlockDisposition D = computeBlockDisposition(S, BB); 6889218893Sdim return BlockDispositions[S][BB] = D; 6890218893Sdim} 6891218893Sdim 6892218893SdimScalarEvolution::BlockDisposition 6893218893SdimScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { 6894218893Sdim switch (S->getSCEVType()) { 6895218893Sdim case scConstant: 6896218893Sdim return ProperlyDominatesBlock; 6897218893Sdim case scTruncate: 6898218893Sdim case scZeroExtend: 6899218893Sdim case scSignExtend: 6900218893Sdim return getBlockDisposition(cast<SCEVCastExpr>(S)->getOperand(), BB); 6901218893Sdim case scAddRecExpr: { 6902218893Sdim // This uses a "dominates" query instead of "properly dominates" query 6903218893Sdim // to test for proper dominance too, because the instruction which 6904218893Sdim // produces the addrec's value is a PHI, and a PHI effectively properly 6905218893Sdim // dominates its entire containing block. 6906218893Sdim const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S); 6907218893Sdim if (!DT->dominates(AR->getLoop()->getHeader(), BB)) 6908218893Sdim return DoesNotDominateBlock; 6909218893Sdim } 6910218893Sdim // FALL THROUGH into SCEVNAryExpr handling. 6911218893Sdim case scAddExpr: 6912218893Sdim case scMulExpr: 6913218893Sdim case scUMaxExpr: 6914218893Sdim case scSMaxExpr: { 6915218893Sdim const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S); 6916218893Sdim bool Proper = true; 6917218893Sdim for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); 6918218893Sdim I != E; ++I) { 6919218893Sdim BlockDisposition D = getBlockDisposition(*I, BB); 6920218893Sdim if (D == DoesNotDominateBlock) 6921218893Sdim return DoesNotDominateBlock; 6922218893Sdim if (D == DominatesBlock) 6923218893Sdim Proper = false; 6924218893Sdim } 6925218893Sdim return Proper ? ProperlyDominatesBlock : DominatesBlock; 6926218893Sdim } 6927218893Sdim case scUDivExpr: { 6928218893Sdim const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S); 6929218893Sdim const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS(); 6930218893Sdim BlockDisposition LD = getBlockDisposition(LHS, BB); 6931218893Sdim if (LD == DoesNotDominateBlock) 6932218893Sdim return DoesNotDominateBlock; 6933218893Sdim BlockDisposition RD = getBlockDisposition(RHS, BB); 6934218893Sdim if (RD == DoesNotDominateBlock) 6935218893Sdim return DoesNotDominateBlock; 6936218893Sdim return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ? 6937218893Sdim ProperlyDominatesBlock : DominatesBlock; 6938218893Sdim } 6939218893Sdim case scUnknown: 6940218893Sdim if (Instruction *I = 6941218893Sdim dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) { 6942218893Sdim if (I->getParent() == BB) 6943218893Sdim return DominatesBlock; 6944218893Sdim if (DT->properlyDominates(I->getParent(), BB)) 6945218893Sdim return ProperlyDominatesBlock; 6946218893Sdim return DoesNotDominateBlock; 6947218893Sdim } 6948218893Sdim return ProperlyDominatesBlock; 6949218893Sdim case scCouldNotCompute: 6950218893Sdim llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!"); 6951234353Sdim default: 6952234353Sdim llvm_unreachable("Unknown SCEV kind!"); 6953218893Sdim } 6954218893Sdim} 6955218893Sdim 6956218893Sdimbool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) { 6957218893Sdim return getBlockDisposition(S, BB) >= DominatesBlock; 6958218893Sdim} 6959218893Sdim 6960218893Sdimbool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) { 6961218893Sdim return getBlockDisposition(S, BB) == ProperlyDominatesBlock; 6962218893Sdim} 6963218893Sdim 6964239462Sdimnamespace { 6965239462Sdim// Search for a SCEV expression node within an expression tree. 6966239462Sdim// Implements SCEVTraversal::Visitor. 6967239462Sdimstruct SCEVSearch { 6968239462Sdim const SCEV *Node; 6969239462Sdim bool IsFound; 6970239462Sdim 6971239462Sdim SCEVSearch(const SCEV *N): Node(N), IsFound(false) {} 6972239462Sdim 6973239462Sdim bool follow(const SCEV *S) { 6974239462Sdim IsFound |= (S == Node); 6975239462Sdim return !IsFound; 6976218893Sdim } 6977239462Sdim bool isDone() const { return IsFound; } 6978239462Sdim}; 6979218893Sdim} 6980218893Sdim 6981239462Sdimbool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const { 6982239462Sdim SCEVSearch Search(Op); 6983239462Sdim visitAll(S, Search); 6984239462Sdim return Search.IsFound; 6985239462Sdim} 6986239462Sdim 6987218893Sdimvoid ScalarEvolution::forgetMemoizedResults(const SCEV *S) { 6988218893Sdim ValuesAtScopes.erase(S); 6989218893Sdim LoopDispositions.erase(S); 6990218893Sdim BlockDispositions.erase(S); 6991218893Sdim UnsignedRanges.erase(S); 6992218893Sdim SignedRanges.erase(S); 6993249423Sdim 6994249423Sdim for (DenseMap<const Loop*, BackedgeTakenInfo>::iterator I = 6995249423Sdim BackedgeTakenCounts.begin(), E = BackedgeTakenCounts.end(); I != E; ) { 6996249423Sdim BackedgeTakenInfo &BEInfo = I->second; 6997249423Sdim if (BEInfo.hasOperand(S, this)) { 6998249423Sdim BEInfo.clear(); 6999249423Sdim BackedgeTakenCounts.erase(I++); 7000249423Sdim } 7001249423Sdim else 7002249423Sdim ++I; 7003249423Sdim } 7004218893Sdim} 7005243830Sdim 7006243830Sdimtypedef DenseMap<const Loop *, std::string> VerifyMap; 7007243830Sdim 7008243830Sdim/// replaceSubString - Replaces all occurences of From in Str with To. 7009243830Sdimstatic void replaceSubString(std::string &Str, StringRef From, StringRef To) { 7010243830Sdim size_t Pos = 0; 7011243830Sdim while ((Pos = Str.find(From, Pos)) != std::string::npos) { 7012243830Sdim Str.replace(Pos, From.size(), To.data(), To.size()); 7013243830Sdim Pos += To.size(); 7014243830Sdim } 7015243830Sdim} 7016243830Sdim 7017243830Sdim/// getLoopBackedgeTakenCounts - Helper method for verifyAnalysis. 7018243830Sdimstatic void 7019243830SdimgetLoopBackedgeTakenCounts(Loop *L, VerifyMap &Map, ScalarEvolution &SE) { 7020243830Sdim for (Loop::reverse_iterator I = L->rbegin(), E = L->rend(); I != E; ++I) { 7021243830Sdim getLoopBackedgeTakenCounts(*I, Map, SE); // recurse. 7022243830Sdim 7023243830Sdim std::string &S = Map[L]; 7024243830Sdim if (S.empty()) { 7025243830Sdim raw_string_ostream OS(S); 7026243830Sdim SE.getBackedgeTakenCount(L)->print(OS); 7027243830Sdim 7028243830Sdim // false and 0 are semantically equivalent. This can happen in dead loops. 7029243830Sdim replaceSubString(OS.str(), "false", "0"); 7030243830Sdim // Remove wrap flags, their use in SCEV is highly fragile. 7031243830Sdim // FIXME: Remove this when SCEV gets smarter about them. 7032243830Sdim replaceSubString(OS.str(), "<nw>", ""); 7033243830Sdim replaceSubString(OS.str(), "<nsw>", ""); 7034243830Sdim replaceSubString(OS.str(), "<nuw>", ""); 7035243830Sdim } 7036243830Sdim } 7037243830Sdim} 7038243830Sdim 7039243830Sdimvoid ScalarEvolution::verifyAnalysis() const { 7040243830Sdim if (!VerifySCEV) 7041243830Sdim return; 7042243830Sdim 7043243830Sdim ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this); 7044243830Sdim 7045243830Sdim // Gather stringified backedge taken counts for all loops using SCEV's caches. 7046243830Sdim // FIXME: It would be much better to store actual values instead of strings, 7047243830Sdim // but SCEV pointers will change if we drop the caches. 7048243830Sdim VerifyMap BackedgeDumpsOld, BackedgeDumpsNew; 7049243830Sdim for (LoopInfo::reverse_iterator I = LI->rbegin(), E = LI->rend(); I != E; ++I) 7050243830Sdim getLoopBackedgeTakenCounts(*I, BackedgeDumpsOld, SE); 7051243830Sdim 7052243830Sdim // Gather stringified backedge taken counts for all loops without using 7053243830Sdim // SCEV's caches. 7054243830Sdim SE.releaseMemory(); 7055243830Sdim for (LoopInfo::reverse_iterator I = LI->rbegin(), E = LI->rend(); I != E; ++I) 7056243830Sdim getLoopBackedgeTakenCounts(*I, BackedgeDumpsNew, SE); 7057243830Sdim 7058243830Sdim // Now compare whether they're the same with and without caches. This allows 7059243830Sdim // verifying that no pass changed the cache. 7060243830Sdim assert(BackedgeDumpsOld.size() == BackedgeDumpsNew.size() && 7061243830Sdim "New loops suddenly appeared!"); 7062243830Sdim 7063243830Sdim for (VerifyMap::iterator OldI = BackedgeDumpsOld.begin(), 7064243830Sdim OldE = BackedgeDumpsOld.end(), 7065243830Sdim NewI = BackedgeDumpsNew.begin(); 7066243830Sdim OldI != OldE; ++OldI, ++NewI) { 7067243830Sdim assert(OldI->first == NewI->first && "Loop order changed!"); 7068243830Sdim 7069243830Sdim // Compare the stringified SCEVs. We don't care if undef backedgetaken count 7070243830Sdim // changes. 7071243830Sdim // FIXME: We currently ignore SCEV changes from/to CouldNotCompute. This 7072243830Sdim // means that a pass is buggy or SCEV has to learn a new pattern but is 7073243830Sdim // usually not harmful. 7074243830Sdim if (OldI->second != NewI->second && 7075243830Sdim OldI->second.find("undef") == std::string::npos && 7076243830Sdim NewI->second.find("undef") == std::string::npos && 7077243830Sdim OldI->second != "***COULDNOTCOMPUTE***" && 7078243830Sdim NewI->second != "***COULDNOTCOMPUTE***") { 7079243830Sdim dbgs() << "SCEVValidator: SCEV for loop '" 7080243830Sdim << OldI->first->getHeader()->getName() 7081243830Sdim << "' changed from '" << OldI->second 7082243830Sdim << "' to '" << NewI->second << "'!\n"; 7083243830Sdim std::abort(); 7084243830Sdim } 7085243830Sdim } 7086243830Sdim 7087243830Sdim // TODO: Verify more things. 7088243830Sdim} 7089