1193323Sed//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This implements the SelectionDAG class. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13201360Srdivacky 14193323Sed#include "llvm/CodeGen/SelectionDAG.h" 15249423Sdim#include "SDNodeDbgValue.h" 16249423Sdim#include "llvm/ADT/SetVector.h" 17249423Sdim#include "llvm/ADT/SmallPtrSet.h" 18249423Sdim#include "llvm/ADT/SmallSet.h" 19249423Sdim#include "llvm/ADT/SmallVector.h" 20249423Sdim#include "llvm/ADT/StringExtras.h" 21249423Sdim#include "llvm/Analysis/TargetTransformInfo.h" 22239462Sdim#include "llvm/Analysis/ValueTracking.h" 23193323Sed#include "llvm/Assembly/Writer.h" 24193323Sed#include "llvm/CodeGen/MachineBasicBlock.h" 25193323Sed#include "llvm/CodeGen/MachineConstantPool.h" 26193323Sed#include "llvm/CodeGen/MachineFrameInfo.h" 27193323Sed#include "llvm/CodeGen/MachineModuleInfo.h" 28249423Sdim#include "llvm/DebugInfo.h" 29249423Sdim#include "llvm/IR/CallingConv.h" 30249423Sdim#include "llvm/IR/Constants.h" 31249423Sdim#include "llvm/IR/DataLayout.h" 32249423Sdim#include "llvm/IR/DerivedTypes.h" 33249423Sdim#include "llvm/IR/Function.h" 34249423Sdim#include "llvm/IR/GlobalAlias.h" 35249423Sdim#include "llvm/IR/GlobalVariable.h" 36249423Sdim#include "llvm/IR/Intrinsics.h" 37193323Sed#include "llvm/Support/CommandLine.h" 38202375Srdivacky#include "llvm/Support/Debug.h" 39198090Srdivacky#include "llvm/Support/ErrorHandling.h" 40195098Sed#include "llvm/Support/ManagedStatic.h" 41193323Sed#include "llvm/Support/MathExtras.h" 42249423Sdim#include "llvm/Support/Mutex.h" 43193323Sed#include "llvm/Support/raw_ostream.h" 44249423Sdim#include "llvm/Target/TargetInstrInfo.h" 45249423Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 46249423Sdim#include "llvm/Target/TargetLowering.h" 47249423Sdim#include "llvm/Target/TargetMachine.h" 48249423Sdim#include "llvm/Target/TargetOptions.h" 49249423Sdim#include "llvm/Target/TargetRegisterInfo.h" 50249423Sdim#include "llvm/Target/TargetSelectionDAGInfo.h" 51193323Sed#include <algorithm> 52193323Sed#include <cmath> 53193323Sedusing namespace llvm; 54193323Sed 55193323Sed/// makeVTList - Return an instance of the SDVTList struct initialized with the 56193323Sed/// specified members. 57198090Srdivackystatic SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 58193323Sed SDVTList Res = {VTs, NumVTs}; 59193323Sed return Res; 60193323Sed} 61193323Sed 62239462Sdim// Default null implementations of the callbacks. 63239462Sdimvoid SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 64239462Sdimvoid SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 65193323Sed 66193323Sed//===----------------------------------------------------------------------===// 67193323Sed// ConstantFPSDNode Class 68193323Sed//===----------------------------------------------------------------------===// 69193323Sed 70193323Sed/// isExactlyValue - We don't rely on operator== working on double values, as 71193323Sed/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 72193323Sed/// As such, this method can be used to do an exact bit-for-bit comparison of 73193323Sed/// two floating point values. 74193323Sedbool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 75193323Sed return getValueAPF().bitwiseIsEqual(V); 76193323Sed} 77193323Sed 78198090Srdivackybool ConstantFPSDNode::isValueValidForType(EVT VT, 79193323Sed const APFloat& Val) { 80193323Sed assert(VT.isFloatingPoint() && "Can only convert between FP types"); 81193323Sed 82193323Sed // convert modifies in place, so make a copy. 83193323Sed APFloat Val2 = APFloat(Val); 84193323Sed bool losesInfo; 85249423Sdim (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 86249423Sdim APFloat::rmNearestTiesToEven, 87193323Sed &losesInfo); 88193323Sed return !losesInfo; 89193323Sed} 90193323Sed 91193323Sed//===----------------------------------------------------------------------===// 92193323Sed// ISD Namespace 93193323Sed//===----------------------------------------------------------------------===// 94193323Sed 95193323Sed/// isBuildVectorAllOnes - Return true if the specified node is a 96193323Sed/// BUILD_VECTOR where all of the elements are ~0 or undef. 97193323Sedbool ISD::isBuildVectorAllOnes(const SDNode *N) { 98193323Sed // Look through a bit convert. 99218893Sdim if (N->getOpcode() == ISD::BITCAST) 100193323Sed N = N->getOperand(0).getNode(); 101193323Sed 102193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 103193323Sed 104193323Sed unsigned i = 0, e = N->getNumOperands(); 105193323Sed 106193323Sed // Skip over all of the undef values. 107193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 108193323Sed ++i; 109193323Sed 110193323Sed // Do not accept an all-undef vector. 111193323Sed if (i == e) return false; 112193323Sed 113193323Sed // Do not accept build_vectors that aren't all constants or which have non-~0 114234353Sdim // elements. We have to be a bit careful here, as the type of the constant 115234353Sdim // may not be the same as the type of the vector elements due to type 116234353Sdim // legalization (the elements are promoted to a legal type for the target and 117234353Sdim // a vector of a type may be legal when the base element type is not). 118234353Sdim // We only want to check enough bits to cover the vector elements, because 119234353Sdim // we care if the resultant vector is all ones, not whether the individual 120234353Sdim // constants are. 121193323Sed SDValue NotZero = N->getOperand(i); 122234353Sdim unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 123243830Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 124243830Sdim if (CN->getAPIntValue().countTrailingOnes() < EltSize) 125193323Sed return false; 126243830Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 127243830Sdim if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 128193323Sed return false; 129193323Sed } else 130193323Sed return false; 131193323Sed 132193323Sed // Okay, we have at least one ~0 value, check to see if the rest match or are 133234353Sdim // undefs. Even with the above element type twiddling, this should be OK, as 134234353Sdim // the same type legalization should have applied to all the elements. 135193323Sed for (++i; i != e; ++i) 136193323Sed if (N->getOperand(i) != NotZero && 137193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 138193323Sed return false; 139193323Sed return true; 140193323Sed} 141193323Sed 142193323Sed 143193323Sed/// isBuildVectorAllZeros - Return true if the specified node is a 144193323Sed/// BUILD_VECTOR where all of the elements are 0 or undef. 145193323Sedbool ISD::isBuildVectorAllZeros(const SDNode *N) { 146193323Sed // Look through a bit convert. 147218893Sdim if (N->getOpcode() == ISD::BITCAST) 148193323Sed N = N->getOperand(0).getNode(); 149193323Sed 150193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 151193323Sed 152193323Sed unsigned i = 0, e = N->getNumOperands(); 153193323Sed 154193323Sed // Skip over all of the undef values. 155193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 156193323Sed ++i; 157193323Sed 158193323Sed // Do not accept an all-undef vector. 159193323Sed if (i == e) return false; 160193323Sed 161193574Sed // Do not accept build_vectors that aren't all constants or which have non-0 162193323Sed // elements. 163193323Sed SDValue Zero = N->getOperand(i); 164243830Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) { 165243830Sdim if (!CN->isNullValue()) 166193323Sed return false; 167243830Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) { 168243830Sdim if (!CFPN->getValueAPF().isPosZero()) 169193323Sed return false; 170193323Sed } else 171193323Sed return false; 172193323Sed 173193574Sed // Okay, we have at least one 0 value, check to see if the rest match or are 174193323Sed // undefs. 175193323Sed for (++i; i != e; ++i) 176193323Sed if (N->getOperand(i) != Zero && 177193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 178193323Sed return false; 179193323Sed return true; 180193323Sed} 181193323Sed 182193323Sed/// isScalarToVector - Return true if the specified node is a 183193323Sed/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 184193323Sed/// element is not an undef. 185193323Sedbool ISD::isScalarToVector(const SDNode *N) { 186193323Sed if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 187193323Sed return true; 188193323Sed 189193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) 190193323Sed return false; 191193323Sed if (N->getOperand(0).getOpcode() == ISD::UNDEF) 192193323Sed return false; 193193323Sed unsigned NumElems = N->getNumOperands(); 194218893Sdim if (NumElems == 1) 195218893Sdim return false; 196193323Sed for (unsigned i = 1; i < NumElems; ++i) { 197193323Sed SDValue V = N->getOperand(i); 198193323Sed if (V.getOpcode() != ISD::UNDEF) 199193323Sed return false; 200193323Sed } 201193323Sed return true; 202193323Sed} 203193323Sed 204239462Sdim/// allOperandsUndef - Return true if the node has at least one operand 205239462Sdim/// and all operands of the specified node are ISD::UNDEF. 206239462Sdimbool ISD::allOperandsUndef(const SDNode *N) { 207239462Sdim // Return false if the node has no operands. 208239462Sdim // This is "logically inconsistent" with the definition of "all" but 209239462Sdim // is probably the desired behavior. 210239462Sdim if (N->getNumOperands() == 0) 211239462Sdim return false; 212239462Sdim 213239462Sdim for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i) 214239462Sdim if (N->getOperand(i).getOpcode() != ISD::UNDEF) 215239462Sdim return false; 216239462Sdim 217239462Sdim return true; 218239462Sdim} 219239462Sdim 220193323Sed/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 221193323Sed/// when given the operation for (X op Y). 222193323SedISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 223193323Sed // To perform this operation, we just need to swap the L and G bits of the 224193323Sed // operation. 225193323Sed unsigned OldL = (Operation >> 2) & 1; 226193323Sed unsigned OldG = (Operation >> 1) & 1; 227193323Sed return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 228193323Sed (OldL << 1) | // New G bit 229193323Sed (OldG << 2)); // New L bit. 230193323Sed} 231193323Sed 232193323Sed/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 233193323Sed/// 'op' is a valid SetCC operation. 234193323SedISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 235193323Sed unsigned Operation = Op; 236193323Sed if (isInteger) 237193323Sed Operation ^= 7; // Flip L, G, E bits, but not U. 238193323Sed else 239193323Sed Operation ^= 15; // Flip all of the condition bits. 240193323Sed 241193323Sed if (Operation > ISD::SETTRUE2) 242193323Sed Operation &= ~8; // Don't let N and U bits get set. 243193323Sed 244193323Sed return ISD::CondCode(Operation); 245193323Sed} 246193323Sed 247193323Sed 248193323Sed/// isSignedOp - For an integer comparison, return 1 if the comparison is a 249193323Sed/// signed operation and 2 if the result is an unsigned comparison. Return zero 250193323Sed/// if the operation does not depend on the sign of the input (setne and seteq). 251193323Sedstatic int isSignedOp(ISD::CondCode Opcode) { 252193323Sed switch (Opcode) { 253198090Srdivacky default: llvm_unreachable("Illegal integer setcc operation!"); 254193323Sed case ISD::SETEQ: 255193323Sed case ISD::SETNE: return 0; 256193323Sed case ISD::SETLT: 257193323Sed case ISD::SETLE: 258193323Sed case ISD::SETGT: 259193323Sed case ISD::SETGE: return 1; 260193323Sed case ISD::SETULT: 261193323Sed case ISD::SETULE: 262193323Sed case ISD::SETUGT: 263193323Sed case ISD::SETUGE: return 2; 264193323Sed } 265193323Sed} 266193323Sed 267193323Sed/// getSetCCOrOperation - Return the result of a logical OR between different 268193323Sed/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 269193323Sed/// returns SETCC_INVALID if it is not possible to represent the resultant 270193323Sed/// comparison. 271193323SedISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 272193323Sed bool isInteger) { 273193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 274193323Sed // Cannot fold a signed integer setcc with an unsigned integer setcc. 275193323Sed return ISD::SETCC_INVALID; 276193323Sed 277193323Sed unsigned Op = Op1 | Op2; // Combine all of the condition bits. 278193323Sed 279193323Sed // If the N and U bits get set then the resultant comparison DOES suddenly 280193323Sed // care about orderedness, and is true when ordered. 281193323Sed if (Op > ISD::SETTRUE2) 282193323Sed Op &= ~16; // Clear the U bit if the N bit is set. 283193323Sed 284193323Sed // Canonicalize illegal integer setcc's. 285193323Sed if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 286193323Sed Op = ISD::SETNE; 287193323Sed 288193323Sed return ISD::CondCode(Op); 289193323Sed} 290193323Sed 291193323Sed/// getSetCCAndOperation - Return the result of a logical AND between different 292193323Sed/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 293193323Sed/// function returns zero if it is not possible to represent the resultant 294193323Sed/// comparison. 295193323SedISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 296193323Sed bool isInteger) { 297193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 298193323Sed // Cannot fold a signed setcc with an unsigned setcc. 299193323Sed return ISD::SETCC_INVALID; 300193323Sed 301193323Sed // Combine all of the condition bits. 302193323Sed ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 303193323Sed 304193323Sed // Canonicalize illegal integer setcc's. 305193323Sed if (isInteger) { 306193323Sed switch (Result) { 307193323Sed default: break; 308193323Sed case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 309193323Sed case ISD::SETOEQ: // SETEQ & SETU[LG]E 310193323Sed case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 311193323Sed case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 312193323Sed case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 313193323Sed } 314193323Sed } 315193323Sed 316193323Sed return Result; 317193323Sed} 318193323Sed 319193323Sed//===----------------------------------------------------------------------===// 320193323Sed// SDNode Profile Support 321193323Sed//===----------------------------------------------------------------------===// 322193323Sed 323193323Sed/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 324193323Sed/// 325193323Sedstatic void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 326193323Sed ID.AddInteger(OpC); 327193323Sed} 328193323Sed 329193323Sed/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 330193323Sed/// solely with their pointer. 331193323Sedstatic void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 332193323Sed ID.AddPointer(VTList.VTs); 333193323Sed} 334193323Sed 335193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 336193323Sed/// 337193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 338193323Sed const SDValue *Ops, unsigned NumOps) { 339193323Sed for (; NumOps; --NumOps, ++Ops) { 340193323Sed ID.AddPointer(Ops->getNode()); 341193323Sed ID.AddInteger(Ops->getResNo()); 342193323Sed } 343193323Sed} 344193323Sed 345193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 346193323Sed/// 347193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 348193323Sed const SDUse *Ops, unsigned NumOps) { 349193323Sed for (; NumOps; --NumOps, ++Ops) { 350193323Sed ID.AddPointer(Ops->getNode()); 351193323Sed ID.AddInteger(Ops->getResNo()); 352193323Sed } 353193323Sed} 354193323Sed 355193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, 356193323Sed unsigned short OpC, SDVTList VTList, 357193323Sed const SDValue *OpList, unsigned N) { 358193323Sed AddNodeIDOpcode(ID, OpC); 359193323Sed AddNodeIDValueTypes(ID, VTList); 360193323Sed AddNodeIDOperands(ID, OpList, N); 361193323Sed} 362193323Sed 363193323Sed/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 364193323Sed/// the NodeID data. 365193323Sedstatic void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 366193323Sed switch (N->getOpcode()) { 367195098Sed case ISD::TargetExternalSymbol: 368195098Sed case ISD::ExternalSymbol: 369198090Srdivacky llvm_unreachable("Should only be used on nodes with operands"); 370193323Sed default: break; // Normal nodes don't need extra info. 371193323Sed case ISD::TargetConstant: 372193323Sed case ISD::Constant: 373193323Sed ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 374193323Sed break; 375193323Sed case ISD::TargetConstantFP: 376193323Sed case ISD::ConstantFP: { 377193323Sed ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 378193323Sed break; 379193323Sed } 380193323Sed case ISD::TargetGlobalAddress: 381193323Sed case ISD::GlobalAddress: 382193323Sed case ISD::TargetGlobalTLSAddress: 383193323Sed case ISD::GlobalTLSAddress: { 384193323Sed const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 385193323Sed ID.AddPointer(GA->getGlobal()); 386193323Sed ID.AddInteger(GA->getOffset()); 387195098Sed ID.AddInteger(GA->getTargetFlags()); 388239462Sdim ID.AddInteger(GA->getAddressSpace()); 389193323Sed break; 390193323Sed } 391193323Sed case ISD::BasicBlock: 392193323Sed ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 393193323Sed break; 394193323Sed case ISD::Register: 395193323Sed ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 396193323Sed break; 397234353Sdim case ISD::RegisterMask: 398234353Sdim ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 399234353Sdim break; 400193323Sed case ISD::SRCVALUE: 401193323Sed ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 402193323Sed break; 403193323Sed case ISD::FrameIndex: 404193323Sed case ISD::TargetFrameIndex: 405193323Sed ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 406193323Sed break; 407193323Sed case ISD::JumpTable: 408193323Sed case ISD::TargetJumpTable: 409193323Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 410195098Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 411193323Sed break; 412193323Sed case ISD::ConstantPool: 413193323Sed case ISD::TargetConstantPool: { 414193323Sed const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 415193323Sed ID.AddInteger(CP->getAlignment()); 416193323Sed ID.AddInteger(CP->getOffset()); 417193323Sed if (CP->isMachineConstantPoolEntry()) 418226633Sdim CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 419193323Sed else 420193323Sed ID.AddPointer(CP->getConstVal()); 421195098Sed ID.AddInteger(CP->getTargetFlags()); 422193323Sed break; 423193323Sed } 424239462Sdim case ISD::TargetIndex: { 425239462Sdim const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 426239462Sdim ID.AddInteger(TI->getIndex()); 427239462Sdim ID.AddInteger(TI->getOffset()); 428239462Sdim ID.AddInteger(TI->getTargetFlags()); 429239462Sdim break; 430239462Sdim } 431193323Sed case ISD::LOAD: { 432193323Sed const LoadSDNode *LD = cast<LoadSDNode>(N); 433193323Sed ID.AddInteger(LD->getMemoryVT().getRawBits()); 434193323Sed ID.AddInteger(LD->getRawSubclassData()); 435239462Sdim ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 436193323Sed break; 437193323Sed } 438193323Sed case ISD::STORE: { 439193323Sed const StoreSDNode *ST = cast<StoreSDNode>(N); 440193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 441193323Sed ID.AddInteger(ST->getRawSubclassData()); 442239462Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 443193323Sed break; 444193323Sed } 445193323Sed case ISD::ATOMIC_CMP_SWAP: 446193323Sed case ISD::ATOMIC_SWAP: 447193323Sed case ISD::ATOMIC_LOAD_ADD: 448193323Sed case ISD::ATOMIC_LOAD_SUB: 449193323Sed case ISD::ATOMIC_LOAD_AND: 450193323Sed case ISD::ATOMIC_LOAD_OR: 451193323Sed case ISD::ATOMIC_LOAD_XOR: 452193323Sed case ISD::ATOMIC_LOAD_NAND: 453193323Sed case ISD::ATOMIC_LOAD_MIN: 454193323Sed case ISD::ATOMIC_LOAD_MAX: 455193323Sed case ISD::ATOMIC_LOAD_UMIN: 456226633Sdim case ISD::ATOMIC_LOAD_UMAX: 457226633Sdim case ISD::ATOMIC_LOAD: 458226633Sdim case ISD::ATOMIC_STORE: { 459193323Sed const AtomicSDNode *AT = cast<AtomicSDNode>(N); 460193323Sed ID.AddInteger(AT->getMemoryVT().getRawBits()); 461193323Sed ID.AddInteger(AT->getRawSubclassData()); 462239462Sdim ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 463193323Sed break; 464193323Sed } 465239462Sdim case ISD::PREFETCH: { 466239462Sdim const MemSDNode *PF = cast<MemSDNode>(N); 467239462Sdim ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 468239462Sdim break; 469239462Sdim } 470193323Sed case ISD::VECTOR_SHUFFLE: { 471193323Sed const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 472198090Srdivacky for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 473193323Sed i != e; ++i) 474193323Sed ID.AddInteger(SVN->getMaskElt(i)); 475193323Sed break; 476193323Sed } 477198892Srdivacky case ISD::TargetBlockAddress: 478198892Srdivacky case ISD::BlockAddress: { 479243830Sdim const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 480243830Sdim ID.AddPointer(BA->getBlockAddress()); 481243830Sdim ID.AddInteger(BA->getOffset()); 482243830Sdim ID.AddInteger(BA->getTargetFlags()); 483198892Srdivacky break; 484198892Srdivacky } 485193323Sed } // end switch (N->getOpcode()) 486239462Sdim 487239462Sdim // Target specific memory nodes could also have address spaces to check. 488239462Sdim if (N->isTargetMemoryOpcode()) 489239462Sdim ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 490193323Sed} 491193323Sed 492193323Sed/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 493193323Sed/// data. 494193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 495193323Sed AddNodeIDOpcode(ID, N->getOpcode()); 496193323Sed // Add the return value info. 497193323Sed AddNodeIDValueTypes(ID, N->getVTList()); 498193323Sed // Add the operand info. 499193323Sed AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 500193323Sed 501193323Sed // Handle SDNode leafs with special info. 502193323Sed AddNodeIDCustom(ID, N); 503193323Sed} 504193323Sed 505193323Sed/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 506204642Srdivacky/// the CSE map that carries volatility, temporalness, indexing mode, and 507193323Sed/// extension/truncation information. 508193323Sed/// 509193323Sedstatic inline unsigned 510204642SrdivackyencodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 511234353Sdim bool isNonTemporal, bool isInvariant) { 512193323Sed assert((ConvType & 3) == ConvType && 513193323Sed "ConvType may not require more than 2 bits!"); 514193323Sed assert((AM & 7) == AM && 515193323Sed "AM may not require more than 3 bits!"); 516193323Sed return ConvType | 517193323Sed (AM << 2) | 518204642Srdivacky (isVolatile << 5) | 519234353Sdim (isNonTemporal << 6) | 520234353Sdim (isInvariant << 7); 521193323Sed} 522193323Sed 523193323Sed//===----------------------------------------------------------------------===// 524193323Sed// SelectionDAG Class 525193323Sed//===----------------------------------------------------------------------===// 526193323Sed 527193323Sed/// doNotCSE - Return true if CSE should not be performed for this node. 528193323Sedstatic bool doNotCSE(SDNode *N) { 529218893Sdim if (N->getValueType(0) == MVT::Glue) 530193323Sed return true; // Never CSE anything that produces a flag. 531193323Sed 532193323Sed switch (N->getOpcode()) { 533193323Sed default: break; 534193323Sed case ISD::HANDLENODE: 535193323Sed case ISD::EH_LABEL: 536193323Sed return true; // Never CSE these nodes. 537193323Sed } 538193323Sed 539193323Sed // Check that remaining values produced are not flags. 540193323Sed for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 541218893Sdim if (N->getValueType(i) == MVT::Glue) 542193323Sed return true; // Never CSE anything that produces a flag. 543193323Sed 544193323Sed return false; 545193323Sed} 546193323Sed 547193323Sed/// RemoveDeadNodes - This method deletes all unreachable nodes in the 548193323Sed/// SelectionDAG. 549193323Sedvoid SelectionDAG::RemoveDeadNodes() { 550193323Sed // Create a dummy node (which is not added to allnodes), that adds a reference 551193323Sed // to the root node, preventing it from being deleted. 552193323Sed HandleSDNode Dummy(getRoot()); 553193323Sed 554193323Sed SmallVector<SDNode*, 128> DeadNodes; 555193323Sed 556193323Sed // Add all obviously-dead nodes to the DeadNodes worklist. 557193323Sed for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 558193323Sed if (I->use_empty()) 559193323Sed DeadNodes.push_back(I); 560193323Sed 561193323Sed RemoveDeadNodes(DeadNodes); 562193323Sed 563193323Sed // If the root changed (e.g. it was a dead load, update the root). 564193323Sed setRoot(Dummy.getValue()); 565193323Sed} 566193323Sed 567193323Sed/// RemoveDeadNodes - This method deletes the unreachable nodes in the 568193323Sed/// given list, and any nodes that become unreachable as a result. 569239462Sdimvoid SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 570193323Sed 571193323Sed // Process the worklist, deleting the nodes and adding their uses to the 572193323Sed // worklist. 573193323Sed while (!DeadNodes.empty()) { 574193323Sed SDNode *N = DeadNodes.pop_back_val(); 575193323Sed 576239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 577239462Sdim DUL->NodeDeleted(N, 0); 578193323Sed 579193323Sed // Take the node out of the appropriate CSE map. 580193323Sed RemoveNodeFromCSEMaps(N); 581193323Sed 582193323Sed // Next, brutally remove the operand list. This is safe to do, as there are 583193323Sed // no cycles in the graph. 584193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 585193323Sed SDUse &Use = *I++; 586193323Sed SDNode *Operand = Use.getNode(); 587193323Sed Use.set(SDValue()); 588193323Sed 589193323Sed // Now that we removed this operand, see if there are no uses of it left. 590193323Sed if (Operand->use_empty()) 591193323Sed DeadNodes.push_back(Operand); 592193323Sed } 593193323Sed 594193323Sed DeallocateNode(N); 595193323Sed } 596193323Sed} 597193323Sed 598239462Sdimvoid SelectionDAG::RemoveDeadNode(SDNode *N){ 599193323Sed SmallVector<SDNode*, 16> DeadNodes(1, N); 600234353Sdim 601234353Sdim // Create a dummy node that adds a reference to the root node, preventing 602234353Sdim // it from being deleted. (This matters if the root is an operand of the 603234353Sdim // dead node.) 604234353Sdim HandleSDNode Dummy(getRoot()); 605234353Sdim 606239462Sdim RemoveDeadNodes(DeadNodes); 607193323Sed} 608193323Sed 609193323Sedvoid SelectionDAG::DeleteNode(SDNode *N) { 610193323Sed // First take this out of the appropriate CSE map. 611193323Sed RemoveNodeFromCSEMaps(N); 612193323Sed 613193323Sed // Finally, remove uses due to operands of this node, remove from the 614193323Sed // AllNodes list, and delete the node. 615193323Sed DeleteNodeNotInCSEMaps(N); 616193323Sed} 617193323Sed 618193323Sedvoid SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 619193323Sed assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 620193323Sed assert(N->use_empty() && "Cannot delete a node that is not dead!"); 621193323Sed 622193323Sed // Drop all of the operands and decrement used node's use counts. 623193323Sed N->DropOperands(); 624193323Sed 625193323Sed DeallocateNode(N); 626193323Sed} 627193323Sed 628193323Sedvoid SelectionDAG::DeallocateNode(SDNode *N) { 629193323Sed if (N->OperandsNeedDelete) 630193323Sed delete[] N->OperandList; 631193323Sed 632193323Sed // Set the opcode to DELETED_NODE to help catch bugs when node 633193323Sed // memory is reallocated. 634193323Sed N->NodeType = ISD::DELETED_NODE; 635193323Sed 636193323Sed NodeAllocator.Deallocate(AllNodes.remove(N)); 637200581Srdivacky 638206083Srdivacky // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 639224145Sdim ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 640206083Srdivacky for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 641206083Srdivacky DbgVals[i]->setIsInvalidated(); 642193323Sed} 643193323Sed 644193323Sed/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 645193323Sed/// correspond to it. This is useful when we're about to delete or repurpose 646193323Sed/// the node. We don't want future request for structurally identical nodes 647193323Sed/// to return N anymore. 648193323Sedbool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 649193323Sed bool Erased = false; 650193323Sed switch (N->getOpcode()) { 651193323Sed case ISD::HANDLENODE: return false; // noop. 652193323Sed case ISD::CONDCODE: 653193323Sed assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 654193323Sed "Cond code doesn't exist!"); 655193323Sed Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 656193323Sed CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 657193323Sed break; 658193323Sed case ISD::ExternalSymbol: 659193323Sed Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 660193323Sed break; 661195098Sed case ISD::TargetExternalSymbol: { 662195098Sed ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 663195098Sed Erased = TargetExternalSymbols.erase( 664195098Sed std::pair<std::string,unsigned char>(ESN->getSymbol(), 665195098Sed ESN->getTargetFlags())); 666193323Sed break; 667195098Sed } 668193323Sed case ISD::VALUETYPE: { 669198090Srdivacky EVT VT = cast<VTSDNode>(N)->getVT(); 670193323Sed if (VT.isExtended()) { 671193323Sed Erased = ExtendedValueTypeNodes.erase(VT); 672193323Sed } else { 673198090Srdivacky Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 674198090Srdivacky ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 675193323Sed } 676193323Sed break; 677193323Sed } 678193323Sed default: 679193323Sed // Remove it from the CSE Map. 680218893Sdim assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 681218893Sdim assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 682193323Sed Erased = CSEMap.RemoveNode(N); 683193323Sed break; 684193323Sed } 685193323Sed#ifndef NDEBUG 686193323Sed // Verify that the node was actually in one of the CSE maps, unless it has a 687193323Sed // flag result (which cannot be CSE'd) or is one of the special cases that are 688193323Sed // not subject to CSE. 689218893Sdim if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 690193323Sed !N->isMachineOpcode() && !doNotCSE(N)) { 691193323Sed N->dump(this); 692202375Srdivacky dbgs() << "\n"; 693198090Srdivacky llvm_unreachable("Node is not in map!"); 694193323Sed } 695193323Sed#endif 696193323Sed return Erased; 697193323Sed} 698193323Sed 699193323Sed/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 700193323Sed/// maps and modified in place. Add it back to the CSE maps, unless an identical 701193323Sed/// node already exists, in which case transfer all its users to the existing 702193323Sed/// node. This transfer can potentially trigger recursive merging. 703193323Sed/// 704193323Sedvoid 705239462SdimSelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 706193323Sed // For node types that aren't CSE'd, just act as if no identical node 707193323Sed // already exists. 708193323Sed if (!doNotCSE(N)) { 709193323Sed SDNode *Existing = CSEMap.GetOrInsertNode(N); 710193323Sed if (Existing != N) { 711193323Sed // If there was already an existing matching node, use ReplaceAllUsesWith 712193323Sed // to replace the dead one with the existing one. This can cause 713193323Sed // recursive merging of other unrelated nodes down the line. 714239462Sdim ReplaceAllUsesWith(N, Existing); 715193323Sed 716239462Sdim // N is now dead. Inform the listeners and delete it. 717239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 718239462Sdim DUL->NodeDeleted(N, Existing); 719193323Sed DeleteNodeNotInCSEMaps(N); 720193323Sed return; 721193323Sed } 722193323Sed } 723193323Sed 724239462Sdim // If the node doesn't already exist, we updated it. Inform listeners. 725239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 726239462Sdim DUL->NodeUpdated(N); 727193323Sed} 728193323Sed 729193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 730193323Sed/// were replaced with those specified. If this node is never memoized, 731193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 732193323Sed/// node already exists with these operands, the slot will be non-null. 733193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 734193323Sed void *&InsertPos) { 735193323Sed if (doNotCSE(N)) 736193323Sed return 0; 737193323Sed 738193323Sed SDValue Ops[] = { Op }; 739193323Sed FoldingSetNodeID ID; 740193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 741193323Sed AddNodeIDCustom(ID, N); 742200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 743200581Srdivacky return Node; 744193323Sed} 745193323Sed 746193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 747193323Sed/// were replaced with those specified. If this node is never memoized, 748193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 749193323Sed/// node already exists with these operands, the slot will be non-null. 750193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 751193323Sed SDValue Op1, SDValue Op2, 752193323Sed void *&InsertPos) { 753193323Sed if (doNotCSE(N)) 754193323Sed return 0; 755193323Sed 756193323Sed SDValue Ops[] = { Op1, Op2 }; 757193323Sed FoldingSetNodeID ID; 758193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 759193323Sed AddNodeIDCustom(ID, N); 760200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 761200581Srdivacky return Node; 762193323Sed} 763193323Sed 764193323Sed 765193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 766193323Sed/// were replaced with those specified. If this node is never memoized, 767193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 768193323Sed/// node already exists with these operands, the slot will be non-null. 769193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 770193323Sed const SDValue *Ops,unsigned NumOps, 771193323Sed void *&InsertPos) { 772193323Sed if (doNotCSE(N)) 773193323Sed return 0; 774193323Sed 775193323Sed FoldingSetNodeID ID; 776193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 777193323Sed AddNodeIDCustom(ID, N); 778200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 779200581Srdivacky return Node; 780193323Sed} 781193323Sed 782218893Sdim#ifndef NDEBUG 783218893Sdim/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 784218893Sdimstatic void VerifyNodeCommon(SDNode *N) { 785193323Sed switch (N->getOpcode()) { 786193323Sed default: 787193323Sed break; 788193323Sed case ISD::BUILD_PAIR: { 789198090Srdivacky EVT VT = N->getValueType(0); 790193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 791193323Sed assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 792193323Sed "Wrong return type!"); 793193323Sed assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 794193323Sed assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 795193323Sed "Mismatched operand types!"); 796193323Sed assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 797193323Sed "Wrong operand type!"); 798193323Sed assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 799193323Sed "Wrong return type size"); 800193323Sed break; 801193323Sed } 802193323Sed case ISD::BUILD_VECTOR: { 803193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 804193323Sed assert(N->getValueType(0).isVector() && "Wrong return type!"); 805193323Sed assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 806193323Sed "Wrong number of operands!"); 807198090Srdivacky EVT EltVT = N->getValueType(0).getVectorElementType(); 808226633Sdim for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 809193323Sed assert((I->getValueType() == EltVT || 810193323Sed (EltVT.isInteger() && I->getValueType().isInteger() && 811193323Sed EltVT.bitsLE(I->getValueType()))) && 812193323Sed "Wrong operand type!"); 813226633Sdim assert(I->getValueType() == N->getOperand(0).getValueType() && 814226633Sdim "Operands must all have the same type"); 815226633Sdim } 816193323Sed break; 817193323Sed } 818193323Sed } 819193323Sed} 820193323Sed 821218893Sdim/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 822218893Sdimstatic void VerifySDNode(SDNode *N) { 823218893Sdim // The SDNode allocators cannot be used to allocate nodes with fields that are 824218893Sdim // not present in an SDNode! 825218893Sdim assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 826218893Sdim assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 827218893Sdim assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 828218893Sdim assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 829218893Sdim assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 830218893Sdim assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 831218893Sdim assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 832218893Sdim assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 833218893Sdim assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 834218893Sdim assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 835218893Sdim assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 836218893Sdim assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 837218893Sdim assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 838218893Sdim assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 839218893Sdim assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 840218893Sdim assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 841218893Sdim assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 842218893Sdim assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 843218893Sdim assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 844218893Sdim 845218893Sdim VerifyNodeCommon(N); 846218893Sdim} 847218893Sdim 848218893Sdim/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 849218893Sdim/// invalid. 850218893Sdimstatic void VerifyMachineNode(SDNode *N) { 851218893Sdim // The MachineNode allocators cannot be used to allocate nodes with fields 852218893Sdim // that are not present in a MachineNode! 853218893Sdim // Currently there are no such nodes. 854218893Sdim 855218893Sdim VerifyNodeCommon(N); 856218893Sdim} 857218893Sdim#endif // NDEBUG 858218893Sdim 859198090Srdivacky/// getEVTAlignment - Compute the default alignment value for the 860193323Sed/// given type. 861193323Sed/// 862198090Srdivackyunsigned SelectionDAG::getEVTAlignment(EVT VT) const { 863226633Sdim Type *Ty = VT == MVT::iPTR ? 864198090Srdivacky PointerType::get(Type::getInt8Ty(*getContext()), 0) : 865198090Srdivacky VT.getTypeForEVT(*getContext()); 866193323Sed 867263508Sdim return TM.getTargetLowering()->getDataLayout()->getABITypeAlignment(Ty); 868193323Sed} 869193323Sed 870193323Sed// EntryNode could meaningfully have debug info if we can find it... 871234353SdimSelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 872263508Sdim : TM(tm), TSI(*tm.getSelectionDAGInfo()), TTI(0), TLI(0), OptLevel(OL), 873263508Sdim EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 874263508Sdim Root(getEntryNode()), NewNodesMustHaveLegalTypes(false), 875263508Sdim UpdateListeners(0) { 876193323Sed AllNodes.push_back(&EntryNode); 877205218Srdivacky DbgInfo = new SDDbgInfo(); 878193323Sed} 879193323Sed 880263508Sdimvoid SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti, 881263508Sdim const TargetLowering *tli) { 882193323Sed MF = &mf; 883249423Sdim TTI = tti; 884263508Sdim TLI = tli; 885198090Srdivacky Context = &mf.getFunction()->getContext(); 886193323Sed} 887193323Sed 888193323SedSelectionDAG::~SelectionDAG() { 889239462Sdim assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 890193323Sed allnodes_clear(); 891205218Srdivacky delete DbgInfo; 892193323Sed} 893193323Sed 894193323Sedvoid SelectionDAG::allnodes_clear() { 895193323Sed assert(&*AllNodes.begin() == &EntryNode); 896193323Sed AllNodes.remove(AllNodes.begin()); 897193323Sed while (!AllNodes.empty()) 898193323Sed DeallocateNode(AllNodes.begin()); 899193323Sed} 900193323Sed 901193323Sedvoid SelectionDAG::clear() { 902193323Sed allnodes_clear(); 903193323Sed OperandAllocator.Reset(); 904193323Sed CSEMap.clear(); 905193323Sed 906193323Sed ExtendedValueTypeNodes.clear(); 907193323Sed ExternalSymbols.clear(); 908193323Sed TargetExternalSymbols.clear(); 909193323Sed std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 910193323Sed static_cast<CondCodeSDNode*>(0)); 911193323Sed std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 912193323Sed static_cast<SDNode*>(0)); 913193323Sed 914193323Sed EntryNode.UseList = 0; 915193323Sed AllNodes.push_back(&EntryNode); 916193323Sed Root = getEntryNode(); 917206083Srdivacky DbgInfo->clear(); 918193323Sed} 919193323Sed 920263508SdimSDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 921226633Sdim return VT.bitsGT(Op.getValueType()) ? 922226633Sdim getNode(ISD::ANY_EXTEND, DL, VT, Op) : 923226633Sdim getNode(ISD::TRUNCATE, DL, VT, Op); 924226633Sdim} 925226633Sdim 926263508SdimSDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 927198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 928198090Srdivacky getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 929198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 930198090Srdivacky} 931198090Srdivacky 932263508SdimSDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 933198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 934198090Srdivacky getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 935198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 936198090Srdivacky} 937198090Srdivacky 938263508SdimSDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) { 939200581Srdivacky assert(!VT.isVector() && 940200581Srdivacky "getZeroExtendInReg should use the vector element type instead of " 941200581Srdivacky "the vector type!"); 942193323Sed if (Op.getValueType() == VT) return Op; 943200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 944200581Srdivacky APInt Imm = APInt::getLowBitsSet(BitWidth, 945193323Sed VT.getSizeInBits()); 946193323Sed return getNode(ISD::AND, DL, Op.getValueType(), Op, 947193323Sed getConstant(Imm, Op.getValueType())); 948193323Sed} 949193323Sed 950193323Sed/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 951193323Sed/// 952263508SdimSDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) { 953204642Srdivacky EVT EltVT = VT.getScalarType(); 954193323Sed SDValue NegOne = 955193323Sed getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 956193323Sed return getNode(ISD::XOR, DL, VT, Val, NegOne); 957193323Sed} 958193323Sed 959198090SrdivackySDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 960204642Srdivacky EVT EltVT = VT.getScalarType(); 961193323Sed assert((EltVT.getSizeInBits() >= 64 || 962193323Sed (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 963193323Sed "getConstant with a uint64_t value that doesn't fit in the type!"); 964193323Sed return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 965193323Sed} 966193323Sed 967198090SrdivackySDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 968198090Srdivacky return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 969193323Sed} 970193323Sed 971198090SrdivackySDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 972193323Sed assert(VT.isInteger() && "Cannot create FP integer constant!"); 973193323Sed 974204642Srdivacky EVT EltVT = VT.getScalarType(); 975226633Sdim const ConstantInt *Elt = &Val; 976226633Sdim 977263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 978263508Sdim 979226633Sdim // In some cases the vector type is legal but the element type is illegal and 980226633Sdim // needs to be promoted, for example v8i8 on ARM. In this case, promote the 981226633Sdim // inserted value (the type does not need to match the vector element type). 982226633Sdim // Any extra bits introduced will be truncated away. 983263508Sdim if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 984226633Sdim TargetLowering::TypePromoteInteger) { 985263508Sdim EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 986226633Sdim APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 987226633Sdim Elt = ConstantInt::get(*getContext(), NewVal); 988226633Sdim } 989263508Sdim // In other cases the element type is illegal and needs to be expanded, for 990263508Sdim // example v2i64 on MIPS32. In this case, find the nearest legal type, split 991263508Sdim // the value into n parts and use a vector type with n-times the elements. 992263508Sdim // Then bitcast to the type requested. 993263508Sdim // Legalizing constants too early makes the DAGCombiner's job harder so we 994263508Sdim // only legalize if the DAG tells us we must produce legal types. 995263508Sdim else if (NewNodesMustHaveLegalTypes && VT.isVector() && 996263508Sdim TLI->getTypeAction(*getContext(), EltVT) == 997263508Sdim TargetLowering::TypeExpandInteger) { 998263508Sdim APInt NewVal = Elt->getValue(); 999263508Sdim EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1000263508Sdim unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); 1001263508Sdim unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; 1002263508Sdim EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); 1003226633Sdim 1004263508Sdim // Check the temporary vector is the correct size. If this fails then 1005263508Sdim // getTypeToTransformTo() probably returned a type whose size (in bits) 1006263508Sdim // isn't a power-of-2 factor of the requested type size. 1007263508Sdim assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); 1008263508Sdim 1009263508Sdim SmallVector<SDValue, 2> EltParts; 1010263508Sdim for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { 1011263508Sdim EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) 1012263508Sdim .trunc(ViaEltSizeInBits), 1013263508Sdim ViaEltVT, isT)); 1014263508Sdim } 1015263508Sdim 1016263508Sdim // EltParts is currently in little endian order. If we actually want 1017263508Sdim // big-endian order then reverse it now. 1018263508Sdim if (TLI->isBigEndian()) 1019263508Sdim std::reverse(EltParts.begin(), EltParts.end()); 1020263508Sdim 1021263508Sdim // The elements must be reversed when the element order is different 1022263508Sdim // to the endianness of the elements (because the BITCAST is itself a 1023263508Sdim // vector shuffle in this situation). However, we do not need any code to 1024263508Sdim // perform this reversal because getConstant() is producing a vector 1025263508Sdim // splat. 1026263508Sdim // This situation occurs in MIPS MSA. 1027263508Sdim 1028263508Sdim SmallVector<SDValue, 8> Ops; 1029263508Sdim for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) 1030263508Sdim Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); 1031263508Sdim 1032263508Sdim SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT, 1033263508Sdim getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT, 1034263508Sdim &Ops[0], Ops.size())); 1035263508Sdim return Result; 1036263508Sdim } 1037263508Sdim 1038226633Sdim assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 1039193323Sed "APInt size does not match type size!"); 1040193323Sed unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 1041193323Sed FoldingSetNodeID ID; 1042193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1043226633Sdim ID.AddPointer(Elt); 1044193323Sed void *IP = 0; 1045193323Sed SDNode *N = NULL; 1046201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1047193323Sed if (!VT.isVector()) 1048193323Sed return SDValue(N, 0); 1049201360Srdivacky 1050193323Sed if (!N) { 1051226633Sdim N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 1052193323Sed CSEMap.InsertNode(N, IP); 1053193323Sed AllNodes.push_back(N); 1054193323Sed } 1055193323Sed 1056193323Sed SDValue Result(N, 0); 1057193323Sed if (VT.isVector()) { 1058193323Sed SmallVector<SDValue, 8> Ops; 1059193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1060263508Sdim Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1061193323Sed } 1062193323Sed return Result; 1063193323Sed} 1064193323Sed 1065193323SedSDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 1066263508Sdim return getConstant(Val, TM.getTargetLowering()->getPointerTy(), isTarget); 1067193323Sed} 1068193323Sed 1069193323Sed 1070198090SrdivackySDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 1071198090Srdivacky return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 1072193323Sed} 1073193323Sed 1074198090SrdivackySDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 1075193323Sed assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1076193323Sed 1077204642Srdivacky EVT EltVT = VT.getScalarType(); 1078193323Sed 1079193323Sed // Do the map lookup using the actual bit pattern for the floating point 1080193323Sed // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1081193323Sed // we don't have issues with SNANs. 1082193323Sed unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1083193323Sed FoldingSetNodeID ID; 1084193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1085193323Sed ID.AddPointer(&V); 1086193323Sed void *IP = 0; 1087193323Sed SDNode *N = NULL; 1088201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1089193323Sed if (!VT.isVector()) 1090193323Sed return SDValue(N, 0); 1091201360Srdivacky 1092193323Sed if (!N) { 1093205407Srdivacky N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1094193323Sed CSEMap.InsertNode(N, IP); 1095193323Sed AllNodes.push_back(N); 1096193323Sed } 1097193323Sed 1098193323Sed SDValue Result(N, 0); 1099193323Sed if (VT.isVector()) { 1100193323Sed SmallVector<SDValue, 8> Ops; 1101193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1102263508Sdim // FIXME SDLoc info might be appropriate here 1103263508Sdim Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1104193323Sed } 1105193323Sed return Result; 1106193323Sed} 1107193323Sed 1108198090SrdivackySDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1109204642Srdivacky EVT EltVT = VT.getScalarType(); 1110193323Sed if (EltVT==MVT::f32) 1111193323Sed return getConstantFP(APFloat((float)Val), VT, isTarget); 1112208599Srdivacky else if (EltVT==MVT::f64) 1113193323Sed return getConstantFP(APFloat(Val), VT, isTarget); 1114249423Sdim else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || 1115249423Sdim EltVT==MVT::f16) { 1116208599Srdivacky bool ignored; 1117208599Srdivacky APFloat apf = APFloat(Val); 1118249423Sdim apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1119208599Srdivacky &ignored); 1120208599Srdivacky return getConstantFP(apf, VT, isTarget); 1121234353Sdim } else 1122234353Sdim llvm_unreachable("Unsupported type in getConstantFP"); 1123193323Sed} 1124193323Sed 1125263508SdimSDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL, 1126198090Srdivacky EVT VT, int64_t Offset, 1127195098Sed bool isTargetGA, 1128195098Sed unsigned char TargetFlags) { 1129195098Sed assert((TargetFlags == 0 || isTargetGA) && 1130195098Sed "Cannot set target flags on target-independent globals"); 1131263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1132198090Srdivacky 1133193323Sed // Truncate (with sign-extension) the offset value to the pointer size. 1134263508Sdim unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 1135193323Sed if (BitWidth < 64) 1136243830Sdim Offset = SignExtend64(Offset, BitWidth); 1137193323Sed 1138193323Sed const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1139193323Sed if (!GVar) { 1140193323Sed // If GV is an alias then use the aliasee for determining thread-localness. 1141193323Sed if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1142193323Sed GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1143193323Sed } 1144193323Sed 1145195098Sed unsigned Opc; 1146193323Sed if (GVar && GVar->isThreadLocal()) 1147193323Sed Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1148193323Sed else 1149193323Sed Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1150193323Sed 1151193323Sed FoldingSetNodeID ID; 1152193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1153193323Sed ID.AddPointer(GV); 1154193323Sed ID.AddInteger(Offset); 1155195098Sed ID.AddInteger(TargetFlags); 1156239462Sdim ID.AddInteger(GV->getType()->getAddressSpace()); 1157193323Sed void *IP = 0; 1158201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1159193323Sed return SDValue(E, 0); 1160201360Srdivacky 1161263508Sdim SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(), 1162263508Sdim DL.getDebugLoc(), GV, VT, 1163205407Srdivacky Offset, TargetFlags); 1164193323Sed CSEMap.InsertNode(N, IP); 1165193323Sed AllNodes.push_back(N); 1166193323Sed return SDValue(N, 0); 1167193323Sed} 1168193323Sed 1169198090SrdivackySDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1170193323Sed unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1171193323Sed FoldingSetNodeID ID; 1172193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1173193323Sed ID.AddInteger(FI); 1174193323Sed void *IP = 0; 1175201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1176193323Sed return SDValue(E, 0); 1177201360Srdivacky 1178205407Srdivacky SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1179193323Sed CSEMap.InsertNode(N, IP); 1180193323Sed AllNodes.push_back(N); 1181193323Sed return SDValue(N, 0); 1182193323Sed} 1183193323Sed 1184198090SrdivackySDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1185195098Sed unsigned char TargetFlags) { 1186195098Sed assert((TargetFlags == 0 || isTarget) && 1187195098Sed "Cannot set target flags on target-independent jump tables"); 1188193323Sed unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1189193323Sed FoldingSetNodeID ID; 1190193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1191193323Sed ID.AddInteger(JTI); 1192195098Sed ID.AddInteger(TargetFlags); 1193193323Sed void *IP = 0; 1194201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1195193323Sed return SDValue(E, 0); 1196201360Srdivacky 1197205407Srdivacky SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1198205407Srdivacky TargetFlags); 1199193323Sed CSEMap.InsertNode(N, IP); 1200193323Sed AllNodes.push_back(N); 1201193323Sed return SDValue(N, 0); 1202193323Sed} 1203193323Sed 1204207618SrdivackySDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1205193323Sed unsigned Alignment, int Offset, 1206198090Srdivacky bool isTarget, 1207195098Sed unsigned char TargetFlags) { 1208195098Sed assert((TargetFlags == 0 || isTarget) && 1209195098Sed "Cannot set target flags on target-independent globals"); 1210193323Sed if (Alignment == 0) 1211263508Sdim Alignment = 1212263508Sdim TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1213193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1214193323Sed FoldingSetNodeID ID; 1215193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1216193323Sed ID.AddInteger(Alignment); 1217193323Sed ID.AddInteger(Offset); 1218193323Sed ID.AddPointer(C); 1219195098Sed ID.AddInteger(TargetFlags); 1220193323Sed void *IP = 0; 1221201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1222193323Sed return SDValue(E, 0); 1223201360Srdivacky 1224205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1225205407Srdivacky Alignment, TargetFlags); 1226193323Sed CSEMap.InsertNode(N, IP); 1227193323Sed AllNodes.push_back(N); 1228193323Sed return SDValue(N, 0); 1229193323Sed} 1230193323Sed 1231193323Sed 1232198090SrdivackySDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1233193323Sed unsigned Alignment, int Offset, 1234195098Sed bool isTarget, 1235195098Sed unsigned char TargetFlags) { 1236195098Sed assert((TargetFlags == 0 || isTarget) && 1237195098Sed "Cannot set target flags on target-independent globals"); 1238193323Sed if (Alignment == 0) 1239263508Sdim Alignment = 1240263508Sdim TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1241193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1242193323Sed FoldingSetNodeID ID; 1243193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1244193323Sed ID.AddInteger(Alignment); 1245193323Sed ID.AddInteger(Offset); 1246226633Sdim C->addSelectionDAGCSEId(ID); 1247195098Sed ID.AddInteger(TargetFlags); 1248193323Sed void *IP = 0; 1249201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1250193323Sed return SDValue(E, 0); 1251201360Srdivacky 1252205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1253205407Srdivacky Alignment, TargetFlags); 1254193323Sed CSEMap.InsertNode(N, IP); 1255193323Sed AllNodes.push_back(N); 1256193323Sed return SDValue(N, 0); 1257193323Sed} 1258193323Sed 1259239462SdimSDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1260239462Sdim unsigned char TargetFlags) { 1261239462Sdim FoldingSetNodeID ID; 1262239462Sdim AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0); 1263239462Sdim ID.AddInteger(Index); 1264239462Sdim ID.AddInteger(Offset); 1265239462Sdim ID.AddInteger(TargetFlags); 1266239462Sdim void *IP = 0; 1267239462Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1268239462Sdim return SDValue(E, 0); 1269239462Sdim 1270239462Sdim SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, 1271239462Sdim TargetFlags); 1272239462Sdim CSEMap.InsertNode(N, IP); 1273239462Sdim AllNodes.push_back(N); 1274239462Sdim return SDValue(N, 0); 1275239462Sdim} 1276239462Sdim 1277193323SedSDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1278193323Sed FoldingSetNodeID ID; 1279193323Sed AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1280193323Sed ID.AddPointer(MBB); 1281193323Sed void *IP = 0; 1282201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1283193323Sed return SDValue(E, 0); 1284201360Srdivacky 1285205407Srdivacky SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1286193323Sed CSEMap.InsertNode(N, IP); 1287193323Sed AllNodes.push_back(N); 1288193323Sed return SDValue(N, 0); 1289193323Sed} 1290193323Sed 1291198090SrdivackySDValue SelectionDAG::getValueType(EVT VT) { 1292198090Srdivacky if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1293198090Srdivacky ValueTypeNodes.size()) 1294198090Srdivacky ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1295193323Sed 1296193323Sed SDNode *&N = VT.isExtended() ? 1297198090Srdivacky ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1298193323Sed 1299193323Sed if (N) return SDValue(N, 0); 1300205407Srdivacky N = new (NodeAllocator) VTSDNode(VT); 1301193323Sed AllNodes.push_back(N); 1302193323Sed return SDValue(N, 0); 1303193323Sed} 1304193323Sed 1305198090SrdivackySDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1306193323Sed SDNode *&N = ExternalSymbols[Sym]; 1307193323Sed if (N) return SDValue(N, 0); 1308205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1309193323Sed AllNodes.push_back(N); 1310193323Sed return SDValue(N, 0); 1311193323Sed} 1312193323Sed 1313198090SrdivackySDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1314195098Sed unsigned char TargetFlags) { 1315195098Sed SDNode *&N = 1316195098Sed TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1317195098Sed TargetFlags)]; 1318193323Sed if (N) return SDValue(N, 0); 1319205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1320193323Sed AllNodes.push_back(N); 1321193323Sed return SDValue(N, 0); 1322193323Sed} 1323193323Sed 1324193323SedSDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1325193323Sed if ((unsigned)Cond >= CondCodeNodes.size()) 1326193323Sed CondCodeNodes.resize(Cond+1); 1327193323Sed 1328193323Sed if (CondCodeNodes[Cond] == 0) { 1329205407Srdivacky CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1330193323Sed CondCodeNodes[Cond] = N; 1331193323Sed AllNodes.push_back(N); 1332193323Sed } 1333201360Srdivacky 1334193323Sed return SDValue(CondCodeNodes[Cond], 0); 1335193323Sed} 1336193323Sed 1337193323Sed// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1338193323Sed// the shuffle mask M that point at N1 to point at N2, and indices that point 1339193323Sed// N2 to point at N1. 1340193323Sedstatic void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1341193323Sed std::swap(N1, N2); 1342193323Sed int NElts = M.size(); 1343193323Sed for (int i = 0; i != NElts; ++i) { 1344193323Sed if (M[i] >= NElts) 1345193323Sed M[i] -= NElts; 1346193323Sed else if (M[i] >= 0) 1347193323Sed M[i] += NElts; 1348193323Sed } 1349193323Sed} 1350193323Sed 1351263508SdimSDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, 1352193323Sed SDValue N2, const int *Mask) { 1353263508Sdim assert(VT == N1.getValueType() && VT == N2.getValueType() && 1354263508Sdim "Invalid VECTOR_SHUFFLE"); 1355193323Sed 1356193323Sed // Canonicalize shuffle undef, undef -> undef 1357193323Sed if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1358198090Srdivacky return getUNDEF(VT); 1359193323Sed 1360198090Srdivacky // Validate that all indices in Mask are within the range of the elements 1361193323Sed // input to the shuffle. 1362193323Sed unsigned NElts = VT.getVectorNumElements(); 1363193323Sed SmallVector<int, 8> MaskVec; 1364193323Sed for (unsigned i = 0; i != NElts; ++i) { 1365193323Sed assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1366193323Sed MaskVec.push_back(Mask[i]); 1367193323Sed } 1368198090Srdivacky 1369193323Sed // Canonicalize shuffle v, v -> v, undef 1370193323Sed if (N1 == N2) { 1371193323Sed N2 = getUNDEF(VT); 1372193323Sed for (unsigned i = 0; i != NElts; ++i) 1373193323Sed if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1374193323Sed } 1375198090Srdivacky 1376193323Sed // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1377193323Sed if (N1.getOpcode() == ISD::UNDEF) 1378193323Sed commuteShuffle(N1, N2, MaskVec); 1379198090Srdivacky 1380193323Sed // Canonicalize all index into lhs, -> shuffle lhs, undef 1381193323Sed // Canonicalize all index into rhs, -> shuffle rhs, undef 1382193323Sed bool AllLHS = true, AllRHS = true; 1383193323Sed bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1384193323Sed for (unsigned i = 0; i != NElts; ++i) { 1385193323Sed if (MaskVec[i] >= (int)NElts) { 1386193323Sed if (N2Undef) 1387193323Sed MaskVec[i] = -1; 1388193323Sed else 1389193323Sed AllLHS = false; 1390193323Sed } else if (MaskVec[i] >= 0) { 1391193323Sed AllRHS = false; 1392193323Sed } 1393193323Sed } 1394193323Sed if (AllLHS && AllRHS) 1395193323Sed return getUNDEF(VT); 1396193323Sed if (AllLHS && !N2Undef) 1397193323Sed N2 = getUNDEF(VT); 1398193323Sed if (AllRHS) { 1399193323Sed N1 = getUNDEF(VT); 1400193323Sed commuteShuffle(N1, N2, MaskVec); 1401193323Sed } 1402198090Srdivacky 1403263508Sdim // If Identity shuffle return that node. 1404193323Sed bool Identity = true; 1405193323Sed for (unsigned i = 0; i != NElts; ++i) { 1406193323Sed if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1407193323Sed } 1408263508Sdim if (Identity && NElts) 1409193323Sed return N1; 1410193323Sed 1411193323Sed FoldingSetNodeID ID; 1412193323Sed SDValue Ops[2] = { N1, N2 }; 1413193323Sed AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1414193323Sed for (unsigned i = 0; i != NElts; ++i) 1415193323Sed ID.AddInteger(MaskVec[i]); 1416198090Srdivacky 1417193323Sed void* IP = 0; 1418201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1419193323Sed return SDValue(E, 0); 1420198090Srdivacky 1421193323Sed // Allocate the mask array for the node out of the BumpPtrAllocator, since 1422193323Sed // SDNode doesn't have access to it. This memory will be "leaked" when 1423193323Sed // the node is deallocated, but recovered when the NodeAllocator is released. 1424193323Sed int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1425193323Sed memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1426198090Srdivacky 1427205407Srdivacky ShuffleVectorSDNode *N = 1428263508Sdim new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(), 1429263508Sdim dl.getDebugLoc(), N1, N2, 1430263508Sdim MaskAlloc); 1431193323Sed CSEMap.InsertNode(N, IP); 1432193323Sed AllNodes.push_back(N); 1433193323Sed return SDValue(N, 0); 1434193323Sed} 1435193323Sed 1436263508SdimSDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, 1437193323Sed SDValue Val, SDValue DTy, 1438193323Sed SDValue STy, SDValue Rnd, SDValue Sat, 1439193323Sed ISD::CvtCode Code) { 1440193323Sed // If the src and dest types are the same and the conversion is between 1441193323Sed // integer types of the same sign or two floats, no conversion is necessary. 1442193323Sed if (DTy == STy && 1443193323Sed (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1444193323Sed return Val; 1445193323Sed 1446193323Sed FoldingSetNodeID ID; 1447199481Srdivacky SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1448199481Srdivacky AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1449193323Sed void* IP = 0; 1450201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1451193323Sed return SDValue(E, 0); 1452201360Srdivacky 1453263508Sdim CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(), 1454263508Sdim dl.getDebugLoc(), 1455263508Sdim Ops, 5, Code); 1456193323Sed CSEMap.InsertNode(N, IP); 1457193323Sed AllNodes.push_back(N); 1458193323Sed return SDValue(N, 0); 1459193323Sed} 1460193323Sed 1461198090SrdivackySDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1462193323Sed FoldingSetNodeID ID; 1463193323Sed AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1464193323Sed ID.AddInteger(RegNo); 1465193323Sed void *IP = 0; 1466201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1467193323Sed return SDValue(E, 0); 1468201360Srdivacky 1469205407Srdivacky SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1470193323Sed CSEMap.InsertNode(N, IP); 1471193323Sed AllNodes.push_back(N); 1472193323Sed return SDValue(N, 0); 1473193323Sed} 1474193323Sed 1475234353SdimSDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1476234353Sdim FoldingSetNodeID ID; 1477234353Sdim AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 1478234353Sdim ID.AddPointer(RegMask); 1479234353Sdim void *IP = 0; 1480234353Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1481234353Sdim return SDValue(E, 0); 1482234353Sdim 1483234353Sdim SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 1484234353Sdim CSEMap.InsertNode(N, IP); 1485234353Sdim AllNodes.push_back(N); 1486234353Sdim return SDValue(N, 0); 1487234353Sdim} 1488234353Sdim 1489263508SdimSDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { 1490193323Sed FoldingSetNodeID ID; 1491193323Sed SDValue Ops[] = { Root }; 1492205218Srdivacky AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1493205218Srdivacky ID.AddPointer(Label); 1494193323Sed void *IP = 0; 1495201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1496193323Sed return SDValue(E, 0); 1497218893Sdim 1498263508Sdim SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(), 1499263508Sdim dl.getDebugLoc(), Root, Label); 1500193323Sed CSEMap.InsertNode(N, IP); 1501193323Sed AllNodes.push_back(N); 1502193323Sed return SDValue(N, 0); 1503193323Sed} 1504193323Sed 1505205218Srdivacky 1506207618SrdivackySDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1507243830Sdim int64_t Offset, 1508199989Srdivacky bool isTarget, 1509199989Srdivacky unsigned char TargetFlags) { 1510198892Srdivacky unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1511198892Srdivacky 1512198892Srdivacky FoldingSetNodeID ID; 1513199989Srdivacky AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1514198892Srdivacky ID.AddPointer(BA); 1515243830Sdim ID.AddInteger(Offset); 1516199989Srdivacky ID.AddInteger(TargetFlags); 1517198892Srdivacky void *IP = 0; 1518201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1519198892Srdivacky return SDValue(E, 0); 1520201360Srdivacky 1521243830Sdim SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset, 1522243830Sdim TargetFlags); 1523198892Srdivacky CSEMap.InsertNode(N, IP); 1524198892Srdivacky AllNodes.push_back(N); 1525198892Srdivacky return SDValue(N, 0); 1526198892Srdivacky} 1527198892Srdivacky 1528193323SedSDValue SelectionDAG::getSrcValue(const Value *V) { 1529204642Srdivacky assert((!V || V->getType()->isPointerTy()) && 1530193323Sed "SrcValue is not a pointer?"); 1531193323Sed 1532193323Sed FoldingSetNodeID ID; 1533193323Sed AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1534193323Sed ID.AddPointer(V); 1535193323Sed 1536193323Sed void *IP = 0; 1537201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1538193323Sed return SDValue(E, 0); 1539193323Sed 1540205407Srdivacky SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1541193323Sed CSEMap.InsertNode(N, IP); 1542193323Sed AllNodes.push_back(N); 1543193323Sed return SDValue(N, 0); 1544193323Sed} 1545193323Sed 1546207618Srdivacky/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1547207618SrdivackySDValue SelectionDAG::getMDNode(const MDNode *MD) { 1548207618Srdivacky FoldingSetNodeID ID; 1549207618Srdivacky AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1550207618Srdivacky ID.AddPointer(MD); 1551218893Sdim 1552207618Srdivacky void *IP = 0; 1553207618Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1554207618Srdivacky return SDValue(E, 0); 1555218893Sdim 1556207618Srdivacky SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1557207618Srdivacky CSEMap.InsertNode(N, IP); 1558207618Srdivacky AllNodes.push_back(N); 1559207618Srdivacky return SDValue(N, 0); 1560207618Srdivacky} 1561207618Srdivacky 1562263508Sdim/// getAddrSpaceCast - Return an AddrSpaceCastSDNode. 1563263508SdimSDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, 1564263508Sdim unsigned SrcAS, unsigned DestAS) { 1565263508Sdim SDValue Ops[] = {Ptr}; 1566263508Sdim FoldingSetNodeID ID; 1567263508Sdim AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), &Ops[0], 1); 1568263508Sdim ID.AddInteger(SrcAS); 1569263508Sdim ID.AddInteger(DestAS); 1570207618Srdivacky 1571263508Sdim void *IP = 0; 1572263508Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1573263508Sdim return SDValue(E, 0); 1574263508Sdim 1575263508Sdim SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(), 1576263508Sdim dl.getDebugLoc(), 1577263508Sdim VT, Ptr, SrcAS, DestAS); 1578263508Sdim CSEMap.InsertNode(N, IP); 1579263508Sdim AllNodes.push_back(N); 1580263508Sdim return SDValue(N, 0); 1581263508Sdim} 1582263508Sdim 1583193323Sed/// getShiftAmountOperand - Return the specified value casted to 1584193323Sed/// the target's desired shift amount type. 1585221345SdimSDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1586198090Srdivacky EVT OpTy = Op.getValueType(); 1587263508Sdim EVT ShTy = TM.getTargetLowering()->getShiftAmountTy(LHSTy); 1588193323Sed if (OpTy == ShTy || OpTy.isVector()) return Op; 1589193323Sed 1590193323Sed ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1591263508Sdim return getNode(Opcode, SDLoc(Op), ShTy, Op); 1592193323Sed} 1593193323Sed 1594193323Sed/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1595193323Sed/// specified value type. 1596198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1597193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1598198090Srdivacky unsigned ByteSize = VT.getStoreSize(); 1599226633Sdim Type *Ty = VT.getTypeForEVT(*getContext()); 1600263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1601193323Sed unsigned StackAlign = 1602263508Sdim std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign); 1603193323Sed 1604199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1605263508Sdim return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1606193323Sed} 1607193323Sed 1608193323Sed/// CreateStackTemporary - Create a stack temporary suitable for holding 1609193323Sed/// either of the specified value types. 1610198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1611193323Sed unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1612193323Sed VT2.getStoreSizeInBits())/8; 1613226633Sdim Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1614226633Sdim Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1615263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1616263508Sdim const DataLayout *TD = TLI->getDataLayout(); 1617193323Sed unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1618193323Sed TD->getPrefTypeAlignment(Ty2)); 1619193323Sed 1620193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1621199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1622263508Sdim return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1623193323Sed} 1624193323Sed 1625198090SrdivackySDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1626263508Sdim SDValue N2, ISD::CondCode Cond, SDLoc dl) { 1627193323Sed // These setcc operations always fold. 1628193323Sed switch (Cond) { 1629193323Sed default: break; 1630193323Sed case ISD::SETFALSE: 1631193323Sed case ISD::SETFALSE2: return getConstant(0, VT); 1632193323Sed case ISD::SETTRUE: 1633263508Sdim case ISD::SETTRUE2: { 1634263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1635263508Sdim TargetLowering::BooleanContent Cnt = TLI->getBooleanContents(VT.isVector()); 1636263508Sdim return getConstant( 1637263508Sdim Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT); 1638263508Sdim } 1639193323Sed 1640193323Sed case ISD::SETOEQ: 1641193323Sed case ISD::SETOGT: 1642193323Sed case ISD::SETOGE: 1643193323Sed case ISD::SETOLT: 1644193323Sed case ISD::SETOLE: 1645193323Sed case ISD::SETONE: 1646193323Sed case ISD::SETO: 1647193323Sed case ISD::SETUO: 1648193323Sed case ISD::SETUEQ: 1649193323Sed case ISD::SETUNE: 1650193323Sed assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1651193323Sed break; 1652193323Sed } 1653193323Sed 1654193323Sed if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1655193323Sed const APInt &C2 = N2C->getAPIntValue(); 1656193323Sed if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1657193323Sed const APInt &C1 = N1C->getAPIntValue(); 1658193323Sed 1659193323Sed switch (Cond) { 1660198090Srdivacky default: llvm_unreachable("Unknown integer setcc!"); 1661193323Sed case ISD::SETEQ: return getConstant(C1 == C2, VT); 1662193323Sed case ISD::SETNE: return getConstant(C1 != C2, VT); 1663193323Sed case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1664193323Sed case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1665193323Sed case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1666193323Sed case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1667193323Sed case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1668193323Sed case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1669193323Sed case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1670193323Sed case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1671193323Sed } 1672193323Sed } 1673193323Sed } 1674193323Sed if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1675193323Sed if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1676193323Sed APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1677193323Sed switch (Cond) { 1678193323Sed default: break; 1679193323Sed case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1680193323Sed return getUNDEF(VT); 1681193323Sed // fall through 1682193323Sed case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1683193323Sed case ISD::SETNE: if (R==APFloat::cmpUnordered) 1684193323Sed return getUNDEF(VT); 1685193323Sed // fall through 1686193323Sed case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1687193323Sed R==APFloat::cmpLessThan, VT); 1688193323Sed case ISD::SETLT: if (R==APFloat::cmpUnordered) 1689193323Sed return getUNDEF(VT); 1690193323Sed // fall through 1691193323Sed case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1692193323Sed case ISD::SETGT: if (R==APFloat::cmpUnordered) 1693193323Sed return getUNDEF(VT); 1694193323Sed // fall through 1695193323Sed case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1696193323Sed case ISD::SETLE: if (R==APFloat::cmpUnordered) 1697193323Sed return getUNDEF(VT); 1698193323Sed // fall through 1699193323Sed case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1700193323Sed R==APFloat::cmpEqual, VT); 1701193323Sed case ISD::SETGE: if (R==APFloat::cmpUnordered) 1702193323Sed return getUNDEF(VT); 1703193323Sed // fall through 1704193323Sed case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1705193323Sed R==APFloat::cmpEqual, VT); 1706193323Sed case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1707193323Sed case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1708193323Sed case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1709193323Sed R==APFloat::cmpEqual, VT); 1710193323Sed case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1711193323Sed case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1712193323Sed R==APFloat::cmpLessThan, VT); 1713193323Sed case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1714193323Sed R==APFloat::cmpUnordered, VT); 1715193323Sed case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1716193323Sed case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1717193323Sed } 1718193323Sed } else { 1719193323Sed // Ensure that the constant occurs on the RHS. 1720263508Sdim ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 1721263508Sdim MVT CompVT = N1.getValueType().getSimpleVT(); 1722263508Sdim if (!TM.getTargetLowering()->isCondCodeLegal(SwappedCond, CompVT)) 1723263508Sdim return SDValue(); 1724263508Sdim 1725263508Sdim return getSetCC(dl, VT, N2, N1, SwappedCond); 1726193323Sed } 1727193323Sed } 1728193323Sed 1729193323Sed // Could not fold it. 1730193323Sed return SDValue(); 1731193323Sed} 1732193323Sed 1733193323Sed/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1734193323Sed/// use this predicate to simplify operations downstream. 1735193323Sedbool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1736198090Srdivacky // This predicate is not safe for vector operations. 1737198090Srdivacky if (Op.getValueType().isVector()) 1738198090Srdivacky return false; 1739198090Srdivacky 1740200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1741193323Sed return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1742193323Sed} 1743193323Sed 1744193323Sed/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1745193323Sed/// this predicate to simplify operations downstream. Mask is known to be zero 1746193323Sed/// for bits that V cannot have. 1747193323Sedbool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1748193323Sed unsigned Depth) const { 1749193323Sed APInt KnownZero, KnownOne; 1750234353Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 1751193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1752193323Sed return (KnownZero & Mask) == Mask; 1753193323Sed} 1754193323Sed 1755193323Sed/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1756193323Sed/// known to be either zero or one and return them in the KnownZero/KnownOne 1757193323Sed/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1758193323Sed/// processing. 1759234353Sdimvoid SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, 1760234353Sdim APInt &KnownOne, unsigned Depth) const { 1761263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1762234353Sdim unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1763193323Sed 1764193323Sed KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1765234353Sdim if (Depth == 6) 1766193323Sed return; // Limit search depth. 1767193323Sed 1768193323Sed APInt KnownZero2, KnownOne2; 1769193323Sed 1770193323Sed switch (Op.getOpcode()) { 1771193323Sed case ISD::Constant: 1772193323Sed // We know all of the bits for a constant! 1773234353Sdim KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue(); 1774234353Sdim KnownZero = ~KnownOne; 1775193323Sed return; 1776193323Sed case ISD::AND: 1777193323Sed // If either the LHS or the RHS are Zero, the result is zero. 1778234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1779234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1780193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1781193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1782193323Sed 1783193323Sed // Output known-1 bits are only known if set in both the LHS & RHS. 1784193323Sed KnownOne &= KnownOne2; 1785193323Sed // Output known-0 are known to be clear if zero in either the LHS | RHS. 1786193323Sed KnownZero |= KnownZero2; 1787193323Sed return; 1788193323Sed case ISD::OR: 1789234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1790234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1791193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1792193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1793193323Sed 1794193323Sed // Output known-0 bits are only known if clear in both the LHS & RHS. 1795193323Sed KnownZero &= KnownZero2; 1796193323Sed // Output known-1 are known to be set if set in either the LHS | RHS. 1797193323Sed KnownOne |= KnownOne2; 1798193323Sed return; 1799193323Sed case ISD::XOR: { 1800234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1801234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1802193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1803193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1804193323Sed 1805193323Sed // Output known-0 bits are known if clear or set in both the LHS & RHS. 1806193323Sed APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1807193323Sed // Output known-1 are known to be set if set in only one of the LHS, RHS. 1808193323Sed KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1809193323Sed KnownZero = KnownZeroOut; 1810193323Sed return; 1811193323Sed } 1812193323Sed case ISD::MUL: { 1813234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1814234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1815193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1816193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1817193323Sed 1818193323Sed // If low bits are zero in either operand, output low known-0 bits. 1819193323Sed // Also compute a conserative estimate for high known-0 bits. 1820193323Sed // More trickiness is possible, but this is sufficient for the 1821193323Sed // interesting case of alignment computation. 1822218893Sdim KnownOne.clearAllBits(); 1823193323Sed unsigned TrailZ = KnownZero.countTrailingOnes() + 1824193323Sed KnownZero2.countTrailingOnes(); 1825193323Sed unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1826193323Sed KnownZero2.countLeadingOnes(), 1827193323Sed BitWidth) - BitWidth; 1828193323Sed 1829193323Sed TrailZ = std::min(TrailZ, BitWidth); 1830193323Sed LeadZ = std::min(LeadZ, BitWidth); 1831193323Sed KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1832193323Sed APInt::getHighBitsSet(BitWidth, LeadZ); 1833193323Sed return; 1834193323Sed } 1835193323Sed case ISD::UDIV: { 1836193323Sed // For the purposes of computing leading zeros we can conservatively 1837193323Sed // treat a udiv as a logical right shift by the power of 2 known to 1838193323Sed // be less than the denominator. 1839234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1840193323Sed unsigned LeadZ = KnownZero2.countLeadingOnes(); 1841193323Sed 1842218893Sdim KnownOne2.clearAllBits(); 1843218893Sdim KnownZero2.clearAllBits(); 1844234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1845193323Sed unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1846193323Sed if (RHSUnknownLeadingOnes != BitWidth) 1847193323Sed LeadZ = std::min(BitWidth, 1848193323Sed LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1849193323Sed 1850234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ); 1851193323Sed return; 1852193323Sed } 1853193323Sed case ISD::SELECT: 1854234353Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1); 1855234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1856193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1857193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1858193323Sed 1859193323Sed // Only known if known in both the LHS and RHS. 1860193323Sed KnownOne &= KnownOne2; 1861193323Sed KnownZero &= KnownZero2; 1862193323Sed return; 1863193323Sed case ISD::SELECT_CC: 1864234353Sdim ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1); 1865234353Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1); 1866193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1867193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1868193323Sed 1869193323Sed // Only known if known in both the LHS and RHS. 1870193323Sed KnownOne &= KnownOne2; 1871193323Sed KnownZero &= KnownZero2; 1872193323Sed return; 1873193323Sed case ISD::SADDO: 1874193323Sed case ISD::UADDO: 1875193323Sed case ISD::SSUBO: 1876193323Sed case ISD::USUBO: 1877193323Sed case ISD::SMULO: 1878193323Sed case ISD::UMULO: 1879193323Sed if (Op.getResNo() != 1) 1880193323Sed return; 1881193323Sed // The boolean result conforms to getBooleanContents. Fall through. 1882193323Sed case ISD::SETCC: 1883193323Sed // If we know the result of a setcc has the top bits zero, use this info. 1884263508Sdim if (TLI->getBooleanContents(Op.getValueType().isVector()) == 1885226633Sdim TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1886193323Sed KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1887193323Sed return; 1888193323Sed case ISD::SHL: 1889193323Sed // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1890193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1891193323Sed unsigned ShAmt = SA->getZExtValue(); 1892193323Sed 1893193323Sed // If the shift count is an invalid immediate, don't do anything. 1894193323Sed if (ShAmt >= BitWidth) 1895193323Sed return; 1896193323Sed 1897234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1898193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1899193323Sed KnownZero <<= ShAmt; 1900193323Sed KnownOne <<= ShAmt; 1901193323Sed // low bits known zero. 1902193323Sed KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1903193323Sed } 1904193323Sed return; 1905193323Sed case ISD::SRL: 1906193323Sed // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1907193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1908193323Sed unsigned ShAmt = SA->getZExtValue(); 1909193323Sed 1910193323Sed // If the shift count is an invalid immediate, don't do anything. 1911193323Sed if (ShAmt >= BitWidth) 1912193323Sed return; 1913193323Sed 1914234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1915193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1916193323Sed KnownZero = KnownZero.lshr(ShAmt); 1917193323Sed KnownOne = KnownOne.lshr(ShAmt); 1918193323Sed 1919234353Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1920193323Sed KnownZero |= HighBits; // High bits known zero. 1921193323Sed } 1922193323Sed return; 1923193323Sed case ISD::SRA: 1924193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1925193323Sed unsigned ShAmt = SA->getZExtValue(); 1926193323Sed 1927193323Sed // If the shift count is an invalid immediate, don't do anything. 1928193323Sed if (ShAmt >= BitWidth) 1929193323Sed return; 1930193323Sed 1931193323Sed // If any of the demanded bits are produced by the sign extension, we also 1932193323Sed // demand the input sign bit. 1933234353Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1934193323Sed 1935234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1936193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1937193323Sed KnownZero = KnownZero.lshr(ShAmt); 1938193323Sed KnownOne = KnownOne.lshr(ShAmt); 1939193323Sed 1940193323Sed // Handle the sign bits. 1941193323Sed APInt SignBit = APInt::getSignBit(BitWidth); 1942193323Sed SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1943193323Sed 1944193323Sed if (KnownZero.intersects(SignBit)) { 1945193323Sed KnownZero |= HighBits; // New bits are known zero. 1946193323Sed } else if (KnownOne.intersects(SignBit)) { 1947193323Sed KnownOne |= HighBits; // New bits are known one. 1948193323Sed } 1949193323Sed } 1950193323Sed return; 1951193323Sed case ISD::SIGN_EXTEND_INREG: { 1952198090Srdivacky EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1953202375Srdivacky unsigned EBits = EVT.getScalarType().getSizeInBits(); 1954193323Sed 1955193323Sed // Sign extension. Compute the demanded bits in the result that are not 1956193323Sed // present in the input. 1957234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 1958193323Sed 1959193323Sed APInt InSignBit = APInt::getSignBit(EBits); 1960234353Sdim APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 1961193323Sed 1962193323Sed // If the sign extended bits are demanded, we know that the sign 1963193323Sed // bit is demanded. 1964218893Sdim InSignBit = InSignBit.zext(BitWidth); 1965193323Sed if (NewBits.getBoolValue()) 1966193323Sed InputDemandedBits |= InSignBit; 1967193323Sed 1968234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1969234353Sdim KnownOne &= InputDemandedBits; 1970234353Sdim KnownZero &= InputDemandedBits; 1971193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1972193323Sed 1973193323Sed // If the sign bit of the input is known set or clear, then we know the 1974193323Sed // top bits of the result. 1975193323Sed if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1976193323Sed KnownZero |= NewBits; 1977193323Sed KnownOne &= ~NewBits; 1978193323Sed } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1979193323Sed KnownOne |= NewBits; 1980193323Sed KnownZero &= ~NewBits; 1981193323Sed } else { // Input sign bit unknown 1982193323Sed KnownZero &= ~NewBits; 1983193323Sed KnownOne &= ~NewBits; 1984193323Sed } 1985193323Sed return; 1986193323Sed } 1987193323Sed case ISD::CTTZ: 1988234353Sdim case ISD::CTTZ_ZERO_UNDEF: 1989193323Sed case ISD::CTLZ: 1990234353Sdim case ISD::CTLZ_ZERO_UNDEF: 1991193323Sed case ISD::CTPOP: { 1992193323Sed unsigned LowBits = Log2_32(BitWidth)+1; 1993193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 1994218893Sdim KnownOne.clearAllBits(); 1995193323Sed return; 1996193323Sed } 1997193323Sed case ISD::LOAD: { 1998234353Sdim LoadSDNode *LD = cast<LoadSDNode>(Op); 1999249423Sdim // If this is a ZEXTLoad and we are looking at the loaded value. 2000249423Sdim if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 2001198090Srdivacky EVT VT = LD->getMemoryVT(); 2002202375Srdivacky unsigned MemBits = VT.getScalarType().getSizeInBits(); 2003234353Sdim KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 2004234353Sdim } else if (const MDNode *Ranges = LD->getRanges()) { 2005234353Sdim computeMaskedBitsLoad(*Ranges, KnownZero); 2006193323Sed } 2007193323Sed return; 2008193323Sed } 2009193323Sed case ISD::ZERO_EXTEND: { 2010198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2011200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2012234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2013218893Sdim KnownZero = KnownZero.trunc(InBits); 2014218893Sdim KnownOne = KnownOne.trunc(InBits); 2015234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2016218893Sdim KnownZero = KnownZero.zext(BitWidth); 2017218893Sdim KnownOne = KnownOne.zext(BitWidth); 2018193323Sed KnownZero |= NewBits; 2019193323Sed return; 2020193323Sed } 2021193323Sed case ISD::SIGN_EXTEND: { 2022198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2023200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2024234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2025193323Sed 2026218893Sdim KnownZero = KnownZero.trunc(InBits); 2027218893Sdim KnownOne = KnownOne.trunc(InBits); 2028234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2029193323Sed 2030193323Sed // Note if the sign bit is known to be zero or one. 2031193323Sed bool SignBitKnownZero = KnownZero.isNegative(); 2032193323Sed bool SignBitKnownOne = KnownOne.isNegative(); 2033193323Sed assert(!(SignBitKnownZero && SignBitKnownOne) && 2034193323Sed "Sign bit can't be known to be both zero and one!"); 2035193323Sed 2036218893Sdim KnownZero = KnownZero.zext(BitWidth); 2037218893Sdim KnownOne = KnownOne.zext(BitWidth); 2038193323Sed 2039193323Sed // If the sign bit is known zero or one, the top bits match. 2040193323Sed if (SignBitKnownZero) 2041193323Sed KnownZero |= NewBits; 2042193323Sed else if (SignBitKnownOne) 2043193323Sed KnownOne |= NewBits; 2044193323Sed return; 2045193323Sed } 2046193323Sed case ISD::ANY_EXTEND: { 2047198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2048200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2049218893Sdim KnownZero = KnownZero.trunc(InBits); 2050218893Sdim KnownOne = KnownOne.trunc(InBits); 2051234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2052218893Sdim KnownZero = KnownZero.zext(BitWidth); 2053218893Sdim KnownOne = KnownOne.zext(BitWidth); 2054193323Sed return; 2055193323Sed } 2056193323Sed case ISD::TRUNCATE: { 2057198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2058200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2059218893Sdim KnownZero = KnownZero.zext(InBits); 2060218893Sdim KnownOne = KnownOne.zext(InBits); 2061234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2062193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 2063218893Sdim KnownZero = KnownZero.trunc(BitWidth); 2064218893Sdim KnownOne = KnownOne.trunc(BitWidth); 2065193323Sed break; 2066193323Sed } 2067193323Sed case ISD::AssertZext: { 2068198090Srdivacky EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2069193323Sed APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 2070234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2071234353Sdim KnownZero |= (~InMask); 2072239462Sdim KnownOne &= (~KnownZero); 2073193323Sed return; 2074193323Sed } 2075193323Sed case ISD::FGETSIGN: 2076193323Sed // All bits are zero except the low bit. 2077193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 2078193323Sed return; 2079193323Sed 2080193323Sed case ISD::SUB: { 2081193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 2082193323Sed // We know that the top bits of C-X are clear if X contains less bits 2083193323Sed // than C (i.e. no wrap-around can happen). For example, 20-X is 2084193323Sed // positive if we can prove that X is >= 0 and < 16. 2085193323Sed if (CLHS->getAPIntValue().isNonNegative()) { 2086193323Sed unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 2087193323Sed // NLZ can't be BitWidth with no sign bit 2088193323Sed APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 2089234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2090193323Sed 2091193323Sed // If all of the MaskV bits are known to be zero, then we know the 2092193323Sed // output top bits are zero, because we now know that the output is 2093193323Sed // from [0-C]. 2094193323Sed if ((KnownZero2 & MaskV) == MaskV) { 2095193323Sed unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 2096193323Sed // Top bits known zero. 2097234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2); 2098193323Sed } 2099193323Sed } 2100193323Sed } 2101193323Sed } 2102193323Sed // fall through 2103218893Sdim case ISD::ADD: 2104218893Sdim case ISD::ADDE: { 2105193323Sed // Output known-0 bits are known if clear or set in both the low clear bits 2106193323Sed // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2107193323Sed // low 3 bits clear. 2108234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 2109193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2110193323Sed unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 2111193323Sed 2112234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2113193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2114193323Sed KnownZeroOut = std::min(KnownZeroOut, 2115193323Sed KnownZero2.countTrailingOnes()); 2116193323Sed 2117218893Sdim if (Op.getOpcode() == ISD::ADD) { 2118218893Sdim KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2119218893Sdim return; 2120218893Sdim } 2121218893Sdim 2122218893Sdim // With ADDE, a carry bit may be added in, so we can only use this 2123218893Sdim // information if we know (at least) that the low two bits are clear. We 2124218893Sdim // then return to the caller that the low bit is unknown but that other bits 2125218893Sdim // are known zero. 2126218893Sdim if (KnownZeroOut >= 2) // ADDE 2127218893Sdim KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2128193323Sed return; 2129193323Sed } 2130193323Sed case ISD::SREM: 2131193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2132203954Srdivacky const APInt &RA = Rem->getAPIntValue().abs(); 2133203954Srdivacky if (RA.isPowerOf2()) { 2134203954Srdivacky APInt LowBits = RA - 1; 2135234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1); 2136193323Sed 2137203954Srdivacky // The low bits of the first operand are unchanged by the srem. 2138203954Srdivacky KnownZero = KnownZero2 & LowBits; 2139203954Srdivacky KnownOne = KnownOne2 & LowBits; 2140203954Srdivacky 2141203954Srdivacky // If the first operand is non-negative or has all low bits zero, then 2142203954Srdivacky // the upper bits are all zero. 2143193323Sed if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2144203954Srdivacky KnownZero |= ~LowBits; 2145193323Sed 2146203954Srdivacky // If the first operand is negative and not all low bits are zero, then 2147203954Srdivacky // the upper bits are all one. 2148203954Srdivacky if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2149203954Srdivacky KnownOne |= ~LowBits; 2150193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2151193323Sed } 2152193323Sed } 2153193323Sed return; 2154193323Sed case ISD::UREM: { 2155193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2156193323Sed const APInt &RA = Rem->getAPIntValue(); 2157193323Sed if (RA.isPowerOf2()) { 2158193323Sed APInt LowBits = (RA - 1); 2159234353Sdim KnownZero |= ~LowBits; 2160234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1); 2161193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2162193323Sed break; 2163193323Sed } 2164193323Sed } 2165193323Sed 2166193323Sed // Since the result is less than or equal to either operand, any leading 2167193323Sed // zero bits in either operand must also exist in the result. 2168234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2169234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2170193323Sed 2171193323Sed uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2172193323Sed KnownZero2.countLeadingOnes()); 2173218893Sdim KnownOne.clearAllBits(); 2174234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, Leaders); 2175193323Sed return; 2176193323Sed } 2177218893Sdim case ISD::FrameIndex: 2178218893Sdim case ISD::TargetFrameIndex: 2179218893Sdim if (unsigned Align = InferPtrAlignment(Op)) { 2180218893Sdim // The low bits are known zero if the pointer is aligned. 2181218893Sdim KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2182218893Sdim return; 2183218893Sdim } 2184218893Sdim break; 2185219077Sdim 2186193323Sed default: 2187223017Sdim if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2188223017Sdim break; 2189223017Sdim // Fallthrough 2190193323Sed case ISD::INTRINSIC_WO_CHAIN: 2191193323Sed case ISD::INTRINSIC_W_CHAIN: 2192193323Sed case ISD::INTRINSIC_VOID: 2193223017Sdim // Allow the target to implement this method for its nodes. 2194263508Sdim TLI->computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth); 2195193323Sed return; 2196193323Sed } 2197193323Sed} 2198193323Sed 2199193323Sed/// ComputeNumSignBits - Return the number of times the sign bit of the 2200193323Sed/// register is replicated into the other bits. We know that at least 1 bit 2201193323Sed/// is always equal to the sign bit (itself), but other cases can give us 2202193323Sed/// information. For example, immediately after an "SRA X, 2", we know that 2203193323Sed/// the top 3 bits are all equal to each other, so we return 3. 2204193323Sedunsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2205263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2206198090Srdivacky EVT VT = Op.getValueType(); 2207193323Sed assert(VT.isInteger() && "Invalid VT!"); 2208200581Srdivacky unsigned VTBits = VT.getScalarType().getSizeInBits(); 2209193323Sed unsigned Tmp, Tmp2; 2210193323Sed unsigned FirstAnswer = 1; 2211193323Sed 2212193323Sed if (Depth == 6) 2213193323Sed return 1; // Limit search depth. 2214193323Sed 2215193323Sed switch (Op.getOpcode()) { 2216193323Sed default: break; 2217193323Sed case ISD::AssertSext: 2218193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2219193323Sed return VTBits-Tmp+1; 2220193323Sed case ISD::AssertZext: 2221193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2222193323Sed return VTBits-Tmp; 2223193323Sed 2224193323Sed case ISD::Constant: { 2225193323Sed const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2226219077Sdim return Val.getNumSignBits(); 2227193323Sed } 2228193323Sed 2229193323Sed case ISD::SIGN_EXTEND: 2230263508Sdim Tmp = 2231263508Sdim VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2232193323Sed return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2233193323Sed 2234193323Sed case ISD::SIGN_EXTEND_INREG: 2235193323Sed // Max of the input and what this extends. 2236202375Srdivacky Tmp = 2237202375Srdivacky cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2238193323Sed Tmp = VTBits-Tmp+1; 2239193323Sed 2240193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2241193323Sed return std::max(Tmp, Tmp2); 2242193323Sed 2243193323Sed case ISD::SRA: 2244193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2245193323Sed // SRA X, C -> adds C sign bits. 2246193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2247193323Sed Tmp += C->getZExtValue(); 2248193323Sed if (Tmp > VTBits) Tmp = VTBits; 2249193323Sed } 2250193323Sed return Tmp; 2251193323Sed case ISD::SHL: 2252193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2253193323Sed // shl destroys sign bits. 2254193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2255193323Sed if (C->getZExtValue() >= VTBits || // Bad shift. 2256193323Sed C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2257193323Sed return Tmp - C->getZExtValue(); 2258193323Sed } 2259193323Sed break; 2260193323Sed case ISD::AND: 2261193323Sed case ISD::OR: 2262193323Sed case ISD::XOR: // NOT is handled here. 2263193323Sed // Logical binary ops preserve the number of sign bits at the worst. 2264193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2265193323Sed if (Tmp != 1) { 2266193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2267193323Sed FirstAnswer = std::min(Tmp, Tmp2); 2268193323Sed // We computed what we know about the sign bits as our first 2269193323Sed // answer. Now proceed to the generic code that uses 2270193323Sed // ComputeMaskedBits, and pick whichever answer is better. 2271193323Sed } 2272193323Sed break; 2273193323Sed 2274193323Sed case ISD::SELECT: 2275193323Sed Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2276193323Sed if (Tmp == 1) return 1; // Early out. 2277193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2278193323Sed return std::min(Tmp, Tmp2); 2279193323Sed 2280193323Sed case ISD::SADDO: 2281193323Sed case ISD::UADDO: 2282193323Sed case ISD::SSUBO: 2283193323Sed case ISD::USUBO: 2284193323Sed case ISD::SMULO: 2285193323Sed case ISD::UMULO: 2286193323Sed if (Op.getResNo() != 1) 2287193323Sed break; 2288193323Sed // The boolean result conforms to getBooleanContents. Fall through. 2289193323Sed case ISD::SETCC: 2290193323Sed // If setcc returns 0/-1, all bits are sign bits. 2291263508Sdim if (TLI->getBooleanContents(Op.getValueType().isVector()) == 2292193323Sed TargetLowering::ZeroOrNegativeOneBooleanContent) 2293193323Sed return VTBits; 2294193323Sed break; 2295193323Sed case ISD::ROTL: 2296193323Sed case ISD::ROTR: 2297193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2298193323Sed unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2299193323Sed 2300193323Sed // Handle rotate right by N like a rotate left by 32-N. 2301193323Sed if (Op.getOpcode() == ISD::ROTR) 2302193323Sed RotAmt = (VTBits-RotAmt) & (VTBits-1); 2303193323Sed 2304193323Sed // If we aren't rotating out all of the known-in sign bits, return the 2305193323Sed // number that are left. This handles rotl(sext(x), 1) for example. 2306193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2307193323Sed if (Tmp > RotAmt+1) return Tmp-RotAmt; 2308193323Sed } 2309193323Sed break; 2310193323Sed case ISD::ADD: 2311193323Sed // Add can have at most one carry bit. Thus we know that the output 2312193323Sed // is, at worst, one more bit than the inputs. 2313193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2314193323Sed if (Tmp == 1) return 1; // Early out. 2315193323Sed 2316193323Sed // Special case decrementing a value (ADD X, -1): 2317193323Sed if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2318193323Sed if (CRHS->isAllOnesValue()) { 2319193323Sed APInt KnownZero, KnownOne; 2320234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2321193323Sed 2322193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2323193323Sed // sign bits set. 2324234353Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2325193323Sed return VTBits; 2326193323Sed 2327193323Sed // If we are subtracting one from a positive number, there is no carry 2328193323Sed // out of the result. 2329193323Sed if (KnownZero.isNegative()) 2330193323Sed return Tmp; 2331193323Sed } 2332193323Sed 2333193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2334193323Sed if (Tmp2 == 1) return 1; 2335234353Sdim return std::min(Tmp, Tmp2)-1; 2336193323Sed 2337193323Sed case ISD::SUB: 2338193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2339193323Sed if (Tmp2 == 1) return 1; 2340193323Sed 2341193323Sed // Handle NEG. 2342193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2343193323Sed if (CLHS->isNullValue()) { 2344193323Sed APInt KnownZero, KnownOne; 2345234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 2346193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2347193323Sed // sign bits set. 2348234353Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2349193323Sed return VTBits; 2350193323Sed 2351193323Sed // If the input is known to be positive (the sign bit is known clear), 2352193323Sed // the output of the NEG has the same number of sign bits as the input. 2353193323Sed if (KnownZero.isNegative()) 2354193323Sed return Tmp2; 2355193323Sed 2356193323Sed // Otherwise, we treat this like a SUB. 2357193323Sed } 2358193323Sed 2359193323Sed // Sub can have at most one carry bit. Thus we know that the output 2360193323Sed // is, at worst, one more bit than the inputs. 2361193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2362193323Sed if (Tmp == 1) return 1; // Early out. 2363234353Sdim return std::min(Tmp, Tmp2)-1; 2364193323Sed case ISD::TRUNCATE: 2365193323Sed // FIXME: it's tricky to do anything useful for this, but it is an important 2366193323Sed // case for targets like X86. 2367193323Sed break; 2368193323Sed } 2369193323Sed 2370249423Sdim // If we are looking at the loaded value of the SDNode. 2371249423Sdim if (Op.getResNo() == 0) { 2372249423Sdim // Handle LOADX separately here. EXTLOAD case will fallthrough. 2373249423Sdim if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 2374249423Sdim unsigned ExtType = LD->getExtensionType(); 2375249423Sdim switch (ExtType) { 2376249423Sdim default: break; 2377249423Sdim case ISD::SEXTLOAD: // '17' bits known 2378249423Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2379249423Sdim return VTBits-Tmp+1; 2380249423Sdim case ISD::ZEXTLOAD: // '16' bits known 2381249423Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2382249423Sdim return VTBits-Tmp; 2383249423Sdim } 2384193323Sed } 2385193323Sed } 2386193323Sed 2387193323Sed // Allow the target to implement this method for its nodes. 2388193323Sed if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2389193323Sed Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2390193323Sed Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2391193323Sed Op.getOpcode() == ISD::INTRINSIC_VOID) { 2392263508Sdim unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, Depth); 2393193323Sed if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2394193323Sed } 2395193323Sed 2396193323Sed // Finally, if we can prove that the top bits of the result are 0's or 1's, 2397193323Sed // use this information. 2398193323Sed APInt KnownZero, KnownOne; 2399234353Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 2400193323Sed 2401234353Sdim APInt Mask; 2402193323Sed if (KnownZero.isNegative()) { // sign bit is 0 2403193323Sed Mask = KnownZero; 2404193323Sed } else if (KnownOne.isNegative()) { // sign bit is 1; 2405193323Sed Mask = KnownOne; 2406193323Sed } else { 2407193323Sed // Nothing known. 2408193323Sed return FirstAnswer; 2409193323Sed } 2410193323Sed 2411193323Sed // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2412193323Sed // the number of identical bits in the top of the input value. 2413193323Sed Mask = ~Mask; 2414193323Sed Mask <<= Mask.getBitWidth()-VTBits; 2415193323Sed // Return # leading zeros. We use 'min' here in case Val was zero before 2416193323Sed // shifting. We don't want to return '64' as for an i32 "0". 2417193323Sed return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2418193323Sed} 2419193323Sed 2420218893Sdim/// isBaseWithConstantOffset - Return true if the specified operand is an 2421218893Sdim/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2422218893Sdim/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2423218893Sdim/// semantics as an ADD. This handles the equivalence: 2424218893Sdim/// X|Cst == X+Cst iff X&Cst = 0. 2425218893Sdimbool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2426218893Sdim if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2427218893Sdim !isa<ConstantSDNode>(Op.getOperand(1))) 2428218893Sdim return false; 2429219077Sdim 2430219077Sdim if (Op.getOpcode() == ISD::OR && 2431218893Sdim !MaskedValueIsZero(Op.getOperand(0), 2432218893Sdim cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2433218893Sdim return false; 2434219077Sdim 2435218893Sdim return true; 2436218893Sdim} 2437218893Sdim 2438218893Sdim 2439198090Srdivackybool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2440198090Srdivacky // If we're told that NaNs won't happen, assume they won't. 2441234353Sdim if (getTarget().Options.NoNaNsFPMath) 2442198090Srdivacky return true; 2443193323Sed 2444198090Srdivacky // If the value is a constant, we can obviously see if it is a NaN or not. 2445198090Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2446198090Srdivacky return !C->getValueAPF().isNaN(); 2447198090Srdivacky 2448198090Srdivacky // TODO: Recognize more cases here. 2449198090Srdivacky 2450198090Srdivacky return false; 2451198090Srdivacky} 2452198090Srdivacky 2453204642Srdivackybool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2454204642Srdivacky // If the value is a constant, we can obviously see if it is a zero or not. 2455204642Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2456204642Srdivacky return !C->isZero(); 2457204642Srdivacky 2458204642Srdivacky // TODO: Recognize more cases here. 2459223017Sdim switch (Op.getOpcode()) { 2460223017Sdim default: break; 2461223017Sdim case ISD::OR: 2462223017Sdim if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2463223017Sdim return !C->isNullValue(); 2464223017Sdim break; 2465223017Sdim } 2466204642Srdivacky 2467204642Srdivacky return false; 2468204642Srdivacky} 2469204642Srdivacky 2470204642Srdivackybool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2471204642Srdivacky // Check the obvious case. 2472204642Srdivacky if (A == B) return true; 2473204642Srdivacky 2474204642Srdivacky // For for negative and positive zero. 2475204642Srdivacky if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2476204642Srdivacky if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2477204642Srdivacky if (CA->isZero() && CB->isZero()) return true; 2478204642Srdivacky 2479204642Srdivacky // Otherwise they may not be equal. 2480204642Srdivacky return false; 2481204642Srdivacky} 2482204642Srdivacky 2483193323Sed/// getNode - Gets or creates the specified node. 2484193323Sed/// 2485263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) { 2486193323Sed FoldingSetNodeID ID; 2487193323Sed AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2488193323Sed void *IP = 0; 2489201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2490193323Sed return SDValue(E, 0); 2491201360Srdivacky 2492263508Sdim SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), 2493263508Sdim DL.getDebugLoc(), getVTList(VT)); 2494193323Sed CSEMap.InsertNode(N, IP); 2495193323Sed 2496193323Sed AllNodes.push_back(N); 2497193323Sed#ifndef NDEBUG 2498218893Sdim VerifySDNode(N); 2499193323Sed#endif 2500193323Sed return SDValue(N, 0); 2501193323Sed} 2502193323Sed 2503263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 2504198090Srdivacky EVT VT, SDValue Operand) { 2505193323Sed // Constant fold unary operations with an integer constant operand. 2506193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2507193323Sed const APInt &Val = C->getAPIntValue(); 2508193323Sed switch (Opcode) { 2509193323Sed default: break; 2510193323Sed case ISD::SIGN_EXTEND: 2511218893Sdim return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2512193323Sed case ISD::ANY_EXTEND: 2513193323Sed case ISD::ZERO_EXTEND: 2514193323Sed case ISD::TRUNCATE: 2515218893Sdim return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2516193323Sed case ISD::UINT_TO_FP: 2517193323Sed case ISD::SINT_TO_FP: { 2518249423Sdim APFloat apf(EVTToAPFloatSemantics(VT), 2519249423Sdim APInt::getNullValue(VT.getSizeInBits())); 2520193323Sed (void)apf.convertFromAPInt(Val, 2521193323Sed Opcode==ISD::SINT_TO_FP, 2522193323Sed APFloat::rmNearestTiesToEven); 2523193323Sed return getConstantFP(apf, VT); 2524193323Sed } 2525218893Sdim case ISD::BITCAST: 2526193323Sed if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2527249423Sdim return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); 2528193323Sed else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2529249423Sdim return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); 2530193323Sed break; 2531193323Sed case ISD::BSWAP: 2532193323Sed return getConstant(Val.byteSwap(), VT); 2533193323Sed case ISD::CTPOP: 2534193323Sed return getConstant(Val.countPopulation(), VT); 2535193323Sed case ISD::CTLZ: 2536234353Sdim case ISD::CTLZ_ZERO_UNDEF: 2537193323Sed return getConstant(Val.countLeadingZeros(), VT); 2538193323Sed case ISD::CTTZ: 2539234353Sdim case ISD::CTTZ_ZERO_UNDEF: 2540193323Sed return getConstant(Val.countTrailingZeros(), VT); 2541193323Sed } 2542193323Sed } 2543193323Sed 2544193323Sed // Constant fold unary operations with a floating point constant operand. 2545193323Sed if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2546193323Sed APFloat V = C->getValueAPF(); // make copy 2547243830Sdim switch (Opcode) { 2548243830Sdim case ISD::FNEG: 2549243830Sdim V.changeSign(); 2550243830Sdim return getConstantFP(V, VT); 2551243830Sdim case ISD::FABS: 2552243830Sdim V.clearSign(); 2553243830Sdim return getConstantFP(V, VT); 2554243830Sdim case ISD::FCEIL: { 2555243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 2556243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2557193323Sed return getConstantFP(V, VT); 2558243830Sdim break; 2559243830Sdim } 2560243830Sdim case ISD::FTRUNC: { 2561243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 2562243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2563193323Sed return getConstantFP(V, VT); 2564243830Sdim break; 2565243830Sdim } 2566243830Sdim case ISD::FFLOOR: { 2567243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 2568243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2569193323Sed return getConstantFP(V, VT); 2570243830Sdim break; 2571243830Sdim } 2572243830Sdim case ISD::FP_EXTEND: { 2573243830Sdim bool ignored; 2574243830Sdim // This can return overflow, underflow, or inexact; we don't care. 2575243830Sdim // FIXME need to be more flexible about rounding mode. 2576249423Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 2577243830Sdim APFloat::rmNearestTiesToEven, &ignored); 2578243830Sdim return getConstantFP(V, VT); 2579243830Sdim } 2580243830Sdim case ISD::FP_TO_SINT: 2581243830Sdim case ISD::FP_TO_UINT: { 2582243830Sdim integerPart x[2]; 2583243830Sdim bool ignored; 2584243830Sdim assert(integerPartWidth >= 64); 2585243830Sdim // FIXME need to be more flexible about rounding mode. 2586243830Sdim APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2587243830Sdim Opcode==ISD::FP_TO_SINT, 2588243830Sdim APFloat::rmTowardZero, &ignored); 2589243830Sdim if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2590193323Sed break; 2591243830Sdim APInt api(VT.getSizeInBits(), x); 2592243830Sdim return getConstant(api, VT); 2593193323Sed } 2594243830Sdim case ISD::BITCAST: 2595243830Sdim if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2596243830Sdim return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2597243830Sdim else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2598243830Sdim return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2599243830Sdim break; 2600243830Sdim } 2601193323Sed } 2602193323Sed 2603193323Sed unsigned OpOpcode = Operand.getNode()->getOpcode(); 2604193323Sed switch (Opcode) { 2605193323Sed case ISD::TokenFactor: 2606193323Sed case ISD::MERGE_VALUES: 2607193323Sed case ISD::CONCAT_VECTORS: 2608193323Sed return Operand; // Factor, merge or concat of one node? No need. 2609198090Srdivacky case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2610193323Sed case ISD::FP_EXTEND: 2611193323Sed assert(VT.isFloatingPoint() && 2612193323Sed Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2613193323Sed if (Operand.getValueType() == VT) return Operand; // noop conversion. 2614200581Srdivacky assert((!VT.isVector() || 2615200581Srdivacky VT.getVectorNumElements() == 2616200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2617200581Srdivacky "Vector element count mismatch!"); 2618193323Sed if (Operand.getOpcode() == ISD::UNDEF) 2619193323Sed return getUNDEF(VT); 2620193323Sed break; 2621193323Sed case ISD::SIGN_EXTEND: 2622193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2623193323Sed "Invalid SIGN_EXTEND!"); 2624193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2625200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2626200581Srdivacky "Invalid sext node, dst < src!"); 2627200581Srdivacky assert((!VT.isVector() || 2628200581Srdivacky VT.getVectorNumElements() == 2629200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2630200581Srdivacky "Vector element count mismatch!"); 2631193323Sed if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2632193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2633221345Sdim else if (OpOpcode == ISD::UNDEF) 2634221345Sdim // sext(undef) = 0, because the top bits will all be the same. 2635221345Sdim return getConstant(0, VT); 2636193323Sed break; 2637193323Sed case ISD::ZERO_EXTEND: 2638193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2639193323Sed "Invalid ZERO_EXTEND!"); 2640193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2641200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2642200581Srdivacky "Invalid zext node, dst < src!"); 2643200581Srdivacky assert((!VT.isVector() || 2644200581Srdivacky VT.getVectorNumElements() == 2645200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2646200581Srdivacky "Vector element count mismatch!"); 2647193323Sed if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2648193323Sed return getNode(ISD::ZERO_EXTEND, DL, VT, 2649193323Sed Operand.getNode()->getOperand(0)); 2650221345Sdim else if (OpOpcode == ISD::UNDEF) 2651221345Sdim // zext(undef) = 0, because the top bits will be zero. 2652221345Sdim return getConstant(0, VT); 2653193323Sed break; 2654193323Sed case ISD::ANY_EXTEND: 2655193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2656193323Sed "Invalid ANY_EXTEND!"); 2657193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2658200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2659200581Srdivacky "Invalid anyext node, dst < src!"); 2660200581Srdivacky assert((!VT.isVector() || 2661200581Srdivacky VT.getVectorNumElements() == 2662200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2663200581Srdivacky "Vector element count mismatch!"); 2664210299Sed 2665210299Sed if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2666210299Sed OpOpcode == ISD::ANY_EXTEND) 2667193323Sed // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2668193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2669221345Sdim else if (OpOpcode == ISD::UNDEF) 2670221345Sdim return getUNDEF(VT); 2671210299Sed 2672210299Sed // (ext (trunx x)) -> x 2673210299Sed if (OpOpcode == ISD::TRUNCATE) { 2674210299Sed SDValue OpOp = Operand.getNode()->getOperand(0); 2675210299Sed if (OpOp.getValueType() == VT) 2676210299Sed return OpOp; 2677210299Sed } 2678193323Sed break; 2679193323Sed case ISD::TRUNCATE: 2680193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2681193323Sed "Invalid TRUNCATE!"); 2682193323Sed if (Operand.getValueType() == VT) return Operand; // noop truncate 2683200581Srdivacky assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2684200581Srdivacky "Invalid truncate node, src < dst!"); 2685200581Srdivacky assert((!VT.isVector() || 2686200581Srdivacky VT.getVectorNumElements() == 2687200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2688200581Srdivacky "Vector element count mismatch!"); 2689193323Sed if (OpOpcode == ISD::TRUNCATE) 2690193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2691234353Sdim if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2692234353Sdim OpOpcode == ISD::ANY_EXTEND) { 2693193323Sed // If the source is smaller than the dest, we still need an extend. 2694200581Srdivacky if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2695200581Srdivacky .bitsLT(VT.getScalarType())) 2696193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2697234353Sdim if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2698193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2699234353Sdim return Operand.getNode()->getOperand(0); 2700193323Sed } 2701234353Sdim if (OpOpcode == ISD::UNDEF) 2702234353Sdim return getUNDEF(VT); 2703193323Sed break; 2704218893Sdim case ISD::BITCAST: 2705193323Sed // Basic sanity checking. 2706193323Sed assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2707218893Sdim && "Cannot BITCAST between types of different sizes!"); 2708193323Sed if (VT == Operand.getValueType()) return Operand; // noop conversion. 2709218893Sdim if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2710218893Sdim return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2711193323Sed if (OpOpcode == ISD::UNDEF) 2712193323Sed return getUNDEF(VT); 2713193323Sed break; 2714193323Sed case ISD::SCALAR_TO_VECTOR: 2715193323Sed assert(VT.isVector() && !Operand.getValueType().isVector() && 2716193323Sed (VT.getVectorElementType() == Operand.getValueType() || 2717193323Sed (VT.getVectorElementType().isInteger() && 2718193323Sed Operand.getValueType().isInteger() && 2719193323Sed VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2720193323Sed "Illegal SCALAR_TO_VECTOR node!"); 2721193323Sed if (OpOpcode == ISD::UNDEF) 2722193323Sed return getUNDEF(VT); 2723193323Sed // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2724193323Sed if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2725193323Sed isa<ConstantSDNode>(Operand.getOperand(1)) && 2726193323Sed Operand.getConstantOperandVal(1) == 0 && 2727193323Sed Operand.getOperand(0).getValueType() == VT) 2728193323Sed return Operand.getOperand(0); 2729193323Sed break; 2730193323Sed case ISD::FNEG: 2731193323Sed // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2732234353Sdim if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2733193323Sed return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2734193323Sed Operand.getNode()->getOperand(0)); 2735193323Sed if (OpOpcode == ISD::FNEG) // --X -> X 2736193323Sed return Operand.getNode()->getOperand(0); 2737193323Sed break; 2738193323Sed case ISD::FABS: 2739193323Sed if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2740193323Sed return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2741193323Sed break; 2742193323Sed } 2743193323Sed 2744193323Sed SDNode *N; 2745193323Sed SDVTList VTs = getVTList(VT); 2746218893Sdim if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2747193323Sed FoldingSetNodeID ID; 2748193323Sed SDValue Ops[1] = { Operand }; 2749193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2750193323Sed void *IP = 0; 2751201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2752193323Sed return SDValue(E, 0); 2753201360Srdivacky 2754263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2755263508Sdim DL.getDebugLoc(), VTs, Operand); 2756193323Sed CSEMap.InsertNode(N, IP); 2757193323Sed } else { 2758263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2759263508Sdim DL.getDebugLoc(), VTs, Operand); 2760193323Sed } 2761193323Sed 2762193323Sed AllNodes.push_back(N); 2763193323Sed#ifndef NDEBUG 2764218893Sdim VerifySDNode(N); 2765193323Sed#endif 2766193323Sed return SDValue(N, 0); 2767193323Sed} 2768193323Sed 2769249423SdimSDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, 2770249423Sdim SDNode *Cst1, SDNode *Cst2) { 2771249423Sdim SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; 2772249423Sdim SmallVector<SDValue, 4> Outputs; 2773249423Sdim EVT SVT = VT.getScalarType(); 2774193323Sed 2775249423Sdim ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); 2776249423Sdim ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); 2777249423Sdim if (Scalar1 && Scalar2) { 2778249423Sdim // Scalar instruction. 2779249423Sdim Inputs.push_back(std::make_pair(Scalar1, Scalar2)); 2780249423Sdim } else { 2781249423Sdim // For vectors extract each constant element into Inputs so we can constant 2782249423Sdim // fold them individually. 2783249423Sdim BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 2784249423Sdim BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 2785249423Sdim if (!BV1 || !BV2) 2786249423Sdim return SDValue(); 2787249423Sdim 2788249423Sdim assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 2789249423Sdim 2790249423Sdim for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 2791249423Sdim ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); 2792249423Sdim ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); 2793249423Sdim if (!V1 || !V2) // Not a constant, bail. 2794249423Sdim return SDValue(); 2795249423Sdim 2796249423Sdim // Avoid BUILD_VECTOR nodes that perform implicit truncation. 2797249423Sdim // FIXME: This is valid and could be handled by truncating the APInts. 2798249423Sdim if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 2799249423Sdim return SDValue(); 2800249423Sdim 2801249423Sdim Inputs.push_back(std::make_pair(V1, V2)); 2802249423Sdim } 2803193323Sed } 2804193323Sed 2805249423Sdim // We have a number of constant values, constant fold them element by element. 2806249423Sdim for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { 2807249423Sdim const APInt &C1 = Inputs[I].first->getAPIntValue(); 2808249423Sdim const APInt &C2 = Inputs[I].second->getAPIntValue(); 2809249423Sdim 2810249423Sdim switch (Opcode) { 2811249423Sdim case ISD::ADD: 2812249423Sdim Outputs.push_back(getConstant(C1 + C2, SVT)); 2813249423Sdim break; 2814249423Sdim case ISD::SUB: 2815249423Sdim Outputs.push_back(getConstant(C1 - C2, SVT)); 2816249423Sdim break; 2817249423Sdim case ISD::MUL: 2818249423Sdim Outputs.push_back(getConstant(C1 * C2, SVT)); 2819249423Sdim break; 2820249423Sdim case ISD::UDIV: 2821249423Sdim if (!C2.getBoolValue()) 2822249423Sdim return SDValue(); 2823249423Sdim Outputs.push_back(getConstant(C1.udiv(C2), SVT)); 2824249423Sdim break; 2825249423Sdim case ISD::UREM: 2826249423Sdim if (!C2.getBoolValue()) 2827249423Sdim return SDValue(); 2828249423Sdim Outputs.push_back(getConstant(C1.urem(C2), SVT)); 2829249423Sdim break; 2830249423Sdim case ISD::SDIV: 2831249423Sdim if (!C2.getBoolValue()) 2832249423Sdim return SDValue(); 2833249423Sdim Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); 2834249423Sdim break; 2835249423Sdim case ISD::SREM: 2836249423Sdim if (!C2.getBoolValue()) 2837249423Sdim return SDValue(); 2838249423Sdim Outputs.push_back(getConstant(C1.srem(C2), SVT)); 2839249423Sdim break; 2840249423Sdim case ISD::AND: 2841249423Sdim Outputs.push_back(getConstant(C1 & C2, SVT)); 2842249423Sdim break; 2843249423Sdim case ISD::OR: 2844249423Sdim Outputs.push_back(getConstant(C1 | C2, SVT)); 2845249423Sdim break; 2846249423Sdim case ISD::XOR: 2847249423Sdim Outputs.push_back(getConstant(C1 ^ C2, SVT)); 2848249423Sdim break; 2849249423Sdim case ISD::SHL: 2850249423Sdim Outputs.push_back(getConstant(C1 << C2, SVT)); 2851249423Sdim break; 2852249423Sdim case ISD::SRL: 2853249423Sdim Outputs.push_back(getConstant(C1.lshr(C2), SVT)); 2854249423Sdim break; 2855249423Sdim case ISD::SRA: 2856249423Sdim Outputs.push_back(getConstant(C1.ashr(C2), SVT)); 2857249423Sdim break; 2858249423Sdim case ISD::ROTL: 2859249423Sdim Outputs.push_back(getConstant(C1.rotl(C2), SVT)); 2860249423Sdim break; 2861249423Sdim case ISD::ROTR: 2862249423Sdim Outputs.push_back(getConstant(C1.rotr(C2), SVT)); 2863249423Sdim break; 2864249423Sdim default: 2865249423Sdim return SDValue(); 2866249423Sdim } 2867249423Sdim } 2868249423Sdim 2869249423Sdim // Handle the scalar case first. 2870251662Sdim if (Scalar1 && Scalar2) 2871249423Sdim return Outputs.back(); 2872249423Sdim 2873249423Sdim // Otherwise build a big vector out of the scalar elements we generated. 2874263508Sdim return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(), 2875249423Sdim Outputs.size()); 2876193323Sed} 2877193323Sed 2878263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, 2879249423Sdim SDValue N2) { 2880193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2881193323Sed ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2882193323Sed switch (Opcode) { 2883193323Sed default: break; 2884193323Sed case ISD::TokenFactor: 2885193323Sed assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2886193323Sed N2.getValueType() == MVT::Other && "Invalid token factor!"); 2887193323Sed // Fold trivial token factors. 2888193323Sed if (N1.getOpcode() == ISD::EntryToken) return N2; 2889193323Sed if (N2.getOpcode() == ISD::EntryToken) return N1; 2890193323Sed if (N1 == N2) return N1; 2891193323Sed break; 2892193323Sed case ISD::CONCAT_VECTORS: 2893239462Sdim // Concat of UNDEFs is UNDEF. 2894239462Sdim if (N1.getOpcode() == ISD::UNDEF && 2895239462Sdim N2.getOpcode() == ISD::UNDEF) 2896239462Sdim return getUNDEF(VT); 2897239462Sdim 2898193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2899193323Sed // one big BUILD_VECTOR. 2900193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 2901193323Sed N2.getOpcode() == ISD::BUILD_VECTOR) { 2902212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2903212904Sdim N1.getNode()->op_end()); 2904210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2905193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2906193323Sed } 2907193323Sed break; 2908193323Sed case ISD::AND: 2909208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2910208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2911193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2912193323Sed // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2913193323Sed // worth handling here. 2914193323Sed if (N2C && N2C->isNullValue()) 2915193323Sed return N2; 2916193323Sed if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2917193323Sed return N1; 2918193323Sed break; 2919193323Sed case ISD::OR: 2920193323Sed case ISD::XOR: 2921193323Sed case ISD::ADD: 2922193323Sed case ISD::SUB: 2923208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2924208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2925193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2926193323Sed // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2927193323Sed // it's worth handling here. 2928193323Sed if (N2C && N2C->isNullValue()) 2929193323Sed return N1; 2930193323Sed break; 2931193323Sed case ISD::UDIV: 2932193323Sed case ISD::UREM: 2933193323Sed case ISD::MULHU: 2934193323Sed case ISD::MULHS: 2935193323Sed case ISD::MUL: 2936193323Sed case ISD::SDIV: 2937193323Sed case ISD::SREM: 2938193323Sed assert(VT.isInteger() && "This operator does not apply to FP types!"); 2939208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2940208599Srdivacky N1.getValueType() == VT && "Binary operator types must match!"); 2941208599Srdivacky break; 2942193323Sed case ISD::FADD: 2943193323Sed case ISD::FSUB: 2944193323Sed case ISD::FMUL: 2945193323Sed case ISD::FDIV: 2946193323Sed case ISD::FREM: 2947234353Sdim if (getTarget().Options.UnsafeFPMath) { 2948193323Sed if (Opcode == ISD::FADD) { 2949193323Sed // 0+x --> x 2950193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2951193323Sed if (CFP->getValueAPF().isZero()) 2952193323Sed return N2; 2953193323Sed // x+0 --> x 2954193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2955193323Sed if (CFP->getValueAPF().isZero()) 2956193323Sed return N1; 2957193323Sed } else if (Opcode == ISD::FSUB) { 2958193323Sed // x-0 --> x 2959193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2960193323Sed if (CFP->getValueAPF().isZero()) 2961193323Sed return N1; 2962243830Sdim } else if (Opcode == ISD::FMUL) { 2963243830Sdim ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1); 2964243830Sdim SDValue V = N2; 2965243830Sdim 2966243830Sdim // If the first operand isn't the constant, try the second 2967243830Sdim if (!CFP) { 2968243830Sdim CFP = dyn_cast<ConstantFPSDNode>(N2); 2969243830Sdim V = N1; 2970243830Sdim } 2971243830Sdim 2972243830Sdim if (CFP) { 2973243830Sdim // 0*x --> 0 2974243830Sdim if (CFP->isZero()) 2975243830Sdim return SDValue(CFP,0); 2976243830Sdim // 1*x --> x 2977243830Sdim if (CFP->isExactlyValue(1.0)) 2978243830Sdim return V; 2979243830Sdim } 2980193323Sed } 2981193323Sed } 2982208599Srdivacky assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2983193323Sed assert(N1.getValueType() == N2.getValueType() && 2984193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2985193323Sed break; 2986193323Sed case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2987193323Sed assert(N1.getValueType() == VT && 2988193323Sed N1.getValueType().isFloatingPoint() && 2989193323Sed N2.getValueType().isFloatingPoint() && 2990193323Sed "Invalid FCOPYSIGN!"); 2991193323Sed break; 2992193323Sed case ISD::SHL: 2993193323Sed case ISD::SRA: 2994193323Sed case ISD::SRL: 2995193323Sed case ISD::ROTL: 2996193323Sed case ISD::ROTR: 2997193323Sed assert(VT == N1.getValueType() && 2998193323Sed "Shift operators return type must be the same as their first arg"); 2999193323Sed assert(VT.isInteger() && N2.getValueType().isInteger() && 3000193323Sed "Shifts only work on integers"); 3001249423Sdim assert((!VT.isVector() || VT == N2.getValueType()) && 3002249423Sdim "Vector shift amounts must be in the same as their first arg"); 3003218893Sdim // Verify that the shift amount VT is bit enough to hold valid shift 3004218893Sdim // amounts. This catches things like trying to shift an i1024 value by an 3005218893Sdim // i8, which is easy to fall into in generic code that uses 3006218893Sdim // TLI.getShiftAmount(). 3007218893Sdim assert(N2.getValueType().getSizeInBits() >= 3008219077Sdim Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 3009218893Sdim "Invalid use of small shift amount with oversized value!"); 3010193323Sed 3011193323Sed // Always fold shifts of i1 values so the code generator doesn't need to 3012193323Sed // handle them. Since we know the size of the shift has to be less than the 3013193323Sed // size of the value, the shift/rotate count is guaranteed to be zero. 3014193323Sed if (VT == MVT::i1) 3015193323Sed return N1; 3016202375Srdivacky if (N2C && N2C->isNullValue()) 3017202375Srdivacky return N1; 3018193323Sed break; 3019193323Sed case ISD::FP_ROUND_INREG: { 3020198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3021193323Sed assert(VT == N1.getValueType() && "Not an inreg round!"); 3022193323Sed assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 3023193323Sed "Cannot FP_ROUND_INREG integer types"); 3024202375Srdivacky assert(EVT.isVector() == VT.isVector() && 3025202375Srdivacky "FP_ROUND_INREG type should be vector iff the operand " 3026202375Srdivacky "type is vector!"); 3027202375Srdivacky assert((!EVT.isVector() || 3028202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3029202375Srdivacky "Vector element counts must match in FP_ROUND_INREG"); 3030193323Sed assert(EVT.bitsLE(VT) && "Not rounding down!"); 3031226633Sdim (void)EVT; 3032193323Sed if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 3033193323Sed break; 3034193323Sed } 3035193323Sed case ISD::FP_ROUND: 3036193323Sed assert(VT.isFloatingPoint() && 3037193323Sed N1.getValueType().isFloatingPoint() && 3038193323Sed VT.bitsLE(N1.getValueType()) && 3039193323Sed isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 3040193323Sed if (N1.getValueType() == VT) return N1; // noop conversion. 3041193323Sed break; 3042193323Sed case ISD::AssertSext: 3043193323Sed case ISD::AssertZext: { 3044198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3045193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 3046193323Sed assert(VT.isInteger() && EVT.isInteger() && 3047193323Sed "Cannot *_EXTEND_INREG FP types"); 3048200581Srdivacky assert(!EVT.isVector() && 3049200581Srdivacky "AssertSExt/AssertZExt type should be the vector element type " 3050200581Srdivacky "rather than the vector type!"); 3051193323Sed assert(EVT.bitsLE(VT) && "Not extending!"); 3052193323Sed if (VT == EVT) return N1; // noop assertion. 3053193323Sed break; 3054193323Sed } 3055193323Sed case ISD::SIGN_EXTEND_INREG: { 3056198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3057193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 3058193323Sed assert(VT.isInteger() && EVT.isInteger() && 3059193323Sed "Cannot *_EXTEND_INREG FP types"); 3060202375Srdivacky assert(EVT.isVector() == VT.isVector() && 3061202375Srdivacky "SIGN_EXTEND_INREG type should be vector iff the operand " 3062202375Srdivacky "type is vector!"); 3063202375Srdivacky assert((!EVT.isVector() || 3064202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3065202375Srdivacky "Vector element counts must match in SIGN_EXTEND_INREG"); 3066202375Srdivacky assert(EVT.bitsLE(VT) && "Not extending!"); 3067193323Sed if (EVT == VT) return N1; // Not actually extending 3068193323Sed 3069193323Sed if (N1C) { 3070193323Sed APInt Val = N1C->getAPIntValue(); 3071202375Srdivacky unsigned FromBits = EVT.getScalarType().getSizeInBits(); 3072193323Sed Val <<= Val.getBitWidth()-FromBits; 3073193323Sed Val = Val.ashr(Val.getBitWidth()-FromBits); 3074193323Sed return getConstant(Val, VT); 3075193323Sed } 3076193323Sed break; 3077193323Sed } 3078193323Sed case ISD::EXTRACT_VECTOR_ELT: 3079193323Sed // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 3080193323Sed if (N1.getOpcode() == ISD::UNDEF) 3081193323Sed return getUNDEF(VT); 3082193323Sed 3083193323Sed // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 3084193323Sed // expanding copies of large vectors from registers. 3085193323Sed if (N2C && 3086193323Sed N1.getOpcode() == ISD::CONCAT_VECTORS && 3087193323Sed N1.getNumOperands() > 0) { 3088193323Sed unsigned Factor = 3089193323Sed N1.getOperand(0).getValueType().getVectorNumElements(); 3090193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 3091193323Sed N1.getOperand(N2C->getZExtValue() / Factor), 3092193323Sed getConstant(N2C->getZExtValue() % Factor, 3093193323Sed N2.getValueType())); 3094193323Sed } 3095193323Sed 3096193323Sed // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 3097193323Sed // expanding large vector constants. 3098193323Sed if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 3099193323Sed SDValue Elt = N1.getOperand(N2C->getZExtValue()); 3100243830Sdim 3101243830Sdim if (VT != Elt.getValueType()) 3102193323Sed // If the vector element type is not legal, the BUILD_VECTOR operands 3103243830Sdim // are promoted and implicitly truncated, and the result implicitly 3104243830Sdim // extended. Make that explicit here. 3105243830Sdim Elt = getAnyExtOrTrunc(Elt, DL, VT); 3106243830Sdim 3107193323Sed return Elt; 3108193323Sed } 3109193323Sed 3110193323Sed // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 3111193323Sed // operations are lowered to scalars. 3112193323Sed if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 3113203954Srdivacky // If the indices are the same, return the inserted element else 3114203954Srdivacky // if the indices are known different, extract the element from 3115193323Sed // the original vector. 3116207618Srdivacky SDValue N1Op2 = N1.getOperand(2); 3117207618Srdivacky ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 3118207618Srdivacky 3119207618Srdivacky if (N1Op2C && N2C) { 3120207618Srdivacky if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 3121207618Srdivacky if (VT == N1.getOperand(1).getValueType()) 3122207618Srdivacky return N1.getOperand(1); 3123207618Srdivacky else 3124207618Srdivacky return getSExtOrTrunc(N1.getOperand(1), DL, VT); 3125207618Srdivacky } 3126207618Srdivacky 3127193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 3128207618Srdivacky } 3129193323Sed } 3130193323Sed break; 3131193323Sed case ISD::EXTRACT_ELEMENT: 3132193323Sed assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 3133193323Sed assert(!N1.getValueType().isVector() && !VT.isVector() && 3134193323Sed (N1.getValueType().isInteger() == VT.isInteger()) && 3135226633Sdim N1.getValueType() != VT && 3136193323Sed "Wrong types for EXTRACT_ELEMENT!"); 3137193323Sed 3138193323Sed // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 3139193323Sed // 64-bit integers into 32-bit parts. Instead of building the extract of 3140193323Sed // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 3141193323Sed if (N1.getOpcode() == ISD::BUILD_PAIR) 3142193323Sed return N1.getOperand(N2C->getZExtValue()); 3143193323Sed 3144193323Sed // EXTRACT_ELEMENT of a constant int is also very common. 3145193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 3146193323Sed unsigned ElementSize = VT.getSizeInBits(); 3147193323Sed unsigned Shift = ElementSize * N2C->getZExtValue(); 3148193323Sed APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 3149193323Sed return getConstant(ShiftedVal.trunc(ElementSize), VT); 3150193323Sed } 3151193323Sed break; 3152218893Sdim case ISD::EXTRACT_SUBVECTOR: { 3153218893Sdim SDValue Index = N2; 3154218893Sdim if (VT.isSimple() && N1.getValueType().isSimple()) { 3155218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3156218893Sdim "Extract subvector VTs must be a vectors!"); 3157263508Sdim assert(VT.getVectorElementType() == 3158263508Sdim N1.getValueType().getVectorElementType() && 3159218893Sdim "Extract subvector VTs must have the same element type!"); 3160263508Sdim assert(VT.getSimpleVT() <= N1.getSimpleValueType() && 3161218893Sdim "Extract subvector must be from larger vector to smaller vector!"); 3162218893Sdim 3163218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3164218893Sdim assert((VT.getVectorNumElements() + 3165218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3166218893Sdim <= N1.getValueType().getVectorNumElements()) 3167218893Sdim && "Extract subvector overflow!"); 3168218893Sdim } 3169218893Sdim 3170218893Sdim // Trivial extraction. 3171263508Sdim if (VT.getSimpleVT() == N1.getSimpleValueType()) 3172218893Sdim return N1; 3173218893Sdim } 3174193323Sed break; 3175193323Sed } 3176218893Sdim } 3177193323Sed 3178249423Sdim // Perform trivial constant folding. 3179249423Sdim SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); 3180249423Sdim if (SV.getNode()) return SV; 3181249423Sdim 3182249423Sdim // Canonicalize constant to RHS if commutative. 3183249423Sdim if (N1C && !N2C && isCommutativeBinOp(Opcode)) { 3184249423Sdim std::swap(N1C, N2C); 3185249423Sdim std::swap(N1, N2); 3186193323Sed } 3187193323Sed 3188193323Sed // Constant fold FP operations. 3189193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 3190193323Sed ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 3191193323Sed if (N1CFP) { 3192193323Sed if (!N2CFP && isCommutativeBinOp(Opcode)) { 3193249423Sdim // Canonicalize constant to RHS if commutative. 3194193323Sed std::swap(N1CFP, N2CFP); 3195193323Sed std::swap(N1, N2); 3196243830Sdim } else if (N2CFP) { 3197193323Sed APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 3198193323Sed APFloat::opStatus s; 3199193323Sed switch (Opcode) { 3200193323Sed case ISD::FADD: 3201193323Sed s = V1.add(V2, APFloat::rmNearestTiesToEven); 3202193323Sed if (s != APFloat::opInvalidOp) 3203193323Sed return getConstantFP(V1, VT); 3204193323Sed break; 3205193323Sed case ISD::FSUB: 3206193323Sed s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 3207193323Sed if (s!=APFloat::opInvalidOp) 3208193323Sed return getConstantFP(V1, VT); 3209193323Sed break; 3210193323Sed case ISD::FMUL: 3211193323Sed s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3212193323Sed if (s!=APFloat::opInvalidOp) 3213193323Sed return getConstantFP(V1, VT); 3214193323Sed break; 3215193323Sed case ISD::FDIV: 3216193323Sed s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3217193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3218193323Sed return getConstantFP(V1, VT); 3219193323Sed break; 3220193323Sed case ISD::FREM : 3221193323Sed s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3222193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3223193323Sed return getConstantFP(V1, VT); 3224193323Sed break; 3225193323Sed case ISD::FCOPYSIGN: 3226193323Sed V1.copySign(V2); 3227193323Sed return getConstantFP(V1, VT); 3228193323Sed default: break; 3229193323Sed } 3230193323Sed } 3231234353Sdim 3232234353Sdim if (Opcode == ISD::FP_ROUND) { 3233234353Sdim APFloat V = N1CFP->getValueAPF(); // make copy 3234234353Sdim bool ignored; 3235234353Sdim // This can return overflow, underflow, or inexact; we don't care. 3236234353Sdim // FIXME need to be more flexible about rounding mode. 3237249423Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 3238234353Sdim APFloat::rmNearestTiesToEven, &ignored); 3239234353Sdim return getConstantFP(V, VT); 3240234353Sdim } 3241193323Sed } 3242193323Sed 3243193323Sed // Canonicalize an UNDEF to the RHS, even over a constant. 3244193323Sed if (N1.getOpcode() == ISD::UNDEF) { 3245193323Sed if (isCommutativeBinOp(Opcode)) { 3246193323Sed std::swap(N1, N2); 3247193323Sed } else { 3248193323Sed switch (Opcode) { 3249193323Sed case ISD::FP_ROUND_INREG: 3250193323Sed case ISD::SIGN_EXTEND_INREG: 3251193323Sed case ISD::SUB: 3252193323Sed case ISD::FSUB: 3253193323Sed case ISD::FDIV: 3254193323Sed case ISD::FREM: 3255193323Sed case ISD::SRA: 3256193323Sed return N1; // fold op(undef, arg2) -> undef 3257193323Sed case ISD::UDIV: 3258193323Sed case ISD::SDIV: 3259193323Sed case ISD::UREM: 3260193323Sed case ISD::SREM: 3261193323Sed case ISD::SRL: 3262193323Sed case ISD::SHL: 3263193323Sed if (!VT.isVector()) 3264193323Sed return getConstant(0, VT); // fold op(undef, arg2) -> 0 3265193323Sed // For vectors, we can't easily build an all zero vector, just return 3266193323Sed // the LHS. 3267193323Sed return N2; 3268193323Sed } 3269193323Sed } 3270193323Sed } 3271193323Sed 3272193323Sed // Fold a bunch of operators when the RHS is undef. 3273193323Sed if (N2.getOpcode() == ISD::UNDEF) { 3274193323Sed switch (Opcode) { 3275193323Sed case ISD::XOR: 3276193323Sed if (N1.getOpcode() == ISD::UNDEF) 3277193323Sed // Handle undef ^ undef -> 0 special case. This is a common 3278193323Sed // idiom (misuse). 3279193323Sed return getConstant(0, VT); 3280193323Sed // fallthrough 3281193323Sed case ISD::ADD: 3282193323Sed case ISD::ADDC: 3283193323Sed case ISD::ADDE: 3284193323Sed case ISD::SUB: 3285193574Sed case ISD::UDIV: 3286193574Sed case ISD::SDIV: 3287193574Sed case ISD::UREM: 3288193574Sed case ISD::SREM: 3289193574Sed return N2; // fold op(arg1, undef) -> undef 3290193323Sed case ISD::FADD: 3291193323Sed case ISD::FSUB: 3292193323Sed case ISD::FMUL: 3293193323Sed case ISD::FDIV: 3294193323Sed case ISD::FREM: 3295234353Sdim if (getTarget().Options.UnsafeFPMath) 3296193574Sed return N2; 3297193574Sed break; 3298193323Sed case ISD::MUL: 3299193323Sed case ISD::AND: 3300193323Sed case ISD::SRL: 3301193323Sed case ISD::SHL: 3302193323Sed if (!VT.isVector()) 3303193323Sed return getConstant(0, VT); // fold op(arg1, undef) -> 0 3304193323Sed // For vectors, we can't easily build an all zero vector, just return 3305193323Sed // the LHS. 3306193323Sed return N1; 3307193323Sed case ISD::OR: 3308193323Sed if (!VT.isVector()) 3309193323Sed return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3310193323Sed // For vectors, we can't easily build an all one vector, just return 3311193323Sed // the LHS. 3312193323Sed return N1; 3313193323Sed case ISD::SRA: 3314193323Sed return N1; 3315193323Sed } 3316193323Sed } 3317193323Sed 3318193323Sed // Memoize this node if possible. 3319193323Sed SDNode *N; 3320193323Sed SDVTList VTs = getVTList(VT); 3321218893Sdim if (VT != MVT::Glue) { 3322193323Sed SDValue Ops[] = { N1, N2 }; 3323193323Sed FoldingSetNodeID ID; 3324193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3325193323Sed void *IP = 0; 3326201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3327193323Sed return SDValue(E, 0); 3328201360Srdivacky 3329263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3330263508Sdim DL.getDebugLoc(), VTs, N1, N2); 3331193323Sed CSEMap.InsertNode(N, IP); 3332193323Sed } else { 3333263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3334263508Sdim DL.getDebugLoc(), VTs, N1, N2); 3335193323Sed } 3336193323Sed 3337193323Sed AllNodes.push_back(N); 3338193323Sed#ifndef NDEBUG 3339218893Sdim VerifySDNode(N); 3340193323Sed#endif 3341193323Sed return SDValue(N, 0); 3342193323Sed} 3343193323Sed 3344263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3345193323Sed SDValue N1, SDValue N2, SDValue N3) { 3346193323Sed // Perform various simplifications. 3347193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3348193323Sed switch (Opcode) { 3349263508Sdim case ISD::FMA: { 3350263508Sdim ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 3351263508Sdim ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 3352263508Sdim ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 3353263508Sdim if (N1CFP && N2CFP && N3CFP) { 3354263508Sdim APFloat V1 = N1CFP->getValueAPF(); 3355263508Sdim const APFloat &V2 = N2CFP->getValueAPF(); 3356263508Sdim const APFloat &V3 = N3CFP->getValueAPF(); 3357263508Sdim APFloat::opStatus s = 3358263508Sdim V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 3359263508Sdim if (s != APFloat::opInvalidOp) 3360263508Sdim return getConstantFP(V1, VT); 3361263508Sdim } 3362263508Sdim break; 3363263508Sdim } 3364193323Sed case ISD::CONCAT_VECTORS: 3365193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3366193323Sed // one big BUILD_VECTOR. 3367193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 3368193323Sed N2.getOpcode() == ISD::BUILD_VECTOR && 3369193323Sed N3.getOpcode() == ISD::BUILD_VECTOR) { 3370212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3371212904Sdim N1.getNode()->op_end()); 3372210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3373210299Sed Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3374193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3375193323Sed } 3376193323Sed break; 3377193323Sed case ISD::SETCC: { 3378193323Sed // Use FoldSetCC to simplify SETCC's. 3379193323Sed SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3380193323Sed if (Simp.getNode()) return Simp; 3381193323Sed break; 3382193323Sed } 3383193323Sed case ISD::SELECT: 3384193323Sed if (N1C) { 3385193323Sed if (N1C->getZExtValue()) 3386234353Sdim return N2; // select true, X, Y -> X 3387234353Sdim return N3; // select false, X, Y -> Y 3388193323Sed } 3389193323Sed 3390193323Sed if (N2 == N3) return N2; // select C, X, X -> X 3391193323Sed break; 3392193323Sed case ISD::VECTOR_SHUFFLE: 3393198090Srdivacky llvm_unreachable("should use getVectorShuffle constructor!"); 3394218893Sdim case ISD::INSERT_SUBVECTOR: { 3395218893Sdim SDValue Index = N3; 3396218893Sdim if (VT.isSimple() && N1.getValueType().isSimple() 3397218893Sdim && N2.getValueType().isSimple()) { 3398218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3399218893Sdim N2.getValueType().isVector() && 3400218893Sdim "Insert subvector VTs must be a vectors"); 3401218893Sdim assert(VT == N1.getValueType() && 3402218893Sdim "Dest and insert subvector source types must match!"); 3403263508Sdim assert(N2.getSimpleValueType() <= N1.getSimpleValueType() && 3404218893Sdim "Insert subvector must be from smaller vector to larger vector!"); 3405218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3406218893Sdim assert((N2.getValueType().getVectorNumElements() + 3407218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3408218893Sdim <= VT.getVectorNumElements()) 3409218893Sdim && "Insert subvector overflow!"); 3410218893Sdim } 3411218893Sdim 3412218893Sdim // Trivial insertion. 3413263508Sdim if (VT.getSimpleVT() == N2.getSimpleValueType()) 3414218893Sdim return N2; 3415218893Sdim } 3416218893Sdim break; 3417218893Sdim } 3418218893Sdim case ISD::BITCAST: 3419193323Sed // Fold bit_convert nodes from a type to themselves. 3420193323Sed if (N1.getValueType() == VT) 3421193323Sed return N1; 3422193323Sed break; 3423193323Sed } 3424193323Sed 3425193323Sed // Memoize node if it doesn't produce a flag. 3426193323Sed SDNode *N; 3427193323Sed SDVTList VTs = getVTList(VT); 3428218893Sdim if (VT != MVT::Glue) { 3429193323Sed SDValue Ops[] = { N1, N2, N3 }; 3430193323Sed FoldingSetNodeID ID; 3431193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3432193323Sed void *IP = 0; 3433201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3434193323Sed return SDValue(E, 0); 3435201360Srdivacky 3436263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3437263508Sdim DL.getDebugLoc(), VTs, N1, N2, N3); 3438193323Sed CSEMap.InsertNode(N, IP); 3439193323Sed } else { 3440263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3441263508Sdim DL.getDebugLoc(), VTs, N1, N2, N3); 3442193323Sed } 3443200581Srdivacky 3444193323Sed AllNodes.push_back(N); 3445193323Sed#ifndef NDEBUG 3446218893Sdim VerifySDNode(N); 3447193323Sed#endif 3448193323Sed return SDValue(N, 0); 3449193323Sed} 3450193323Sed 3451263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3452193323Sed SDValue N1, SDValue N2, SDValue N3, 3453193323Sed SDValue N4) { 3454193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 3455193323Sed return getNode(Opcode, DL, VT, Ops, 4); 3456193323Sed} 3457193323Sed 3458263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3459193323Sed SDValue N1, SDValue N2, SDValue N3, 3460193323Sed SDValue N4, SDValue N5) { 3461193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3462193323Sed return getNode(Opcode, DL, VT, Ops, 5); 3463193323Sed} 3464193323Sed 3465198090Srdivacky/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3466198090Srdivacky/// the incoming stack arguments to be loaded from the stack. 3467198090SrdivackySDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3468198090Srdivacky SmallVector<SDValue, 8> ArgChains; 3469198090Srdivacky 3470198090Srdivacky // Include the original chain at the beginning of the list. When this is 3471198090Srdivacky // used by target LowerCall hooks, this helps legalize find the 3472198090Srdivacky // CALLSEQ_BEGIN node. 3473198090Srdivacky ArgChains.push_back(Chain); 3474198090Srdivacky 3475198090Srdivacky // Add a chain value for each stack argument. 3476198090Srdivacky for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3477198090Srdivacky UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3478198090Srdivacky if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3479198090Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3480198090Srdivacky if (FI->getIndex() < 0) 3481198090Srdivacky ArgChains.push_back(SDValue(L, 1)); 3482198090Srdivacky 3483198090Srdivacky // Build a tokenfactor for all the chains. 3484263508Sdim return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, 3485198090Srdivacky &ArgChains[0], ArgChains.size()); 3486198090Srdivacky} 3487198090Srdivacky 3488193323Sed/// getMemsetValue - Vectorized representation of the memset value 3489193323Sed/// operand. 3490198090Srdivackystatic SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3491263508Sdim SDLoc dl) { 3492206124Srdivacky assert(Value.getOpcode() != ISD::UNDEF); 3493206124Srdivacky 3494204642Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits(); 3495193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3496249423Sdim assert(C->getAPIntValue().getBitWidth() == 8); 3497249423Sdim APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 3498193323Sed if (VT.isInteger()) 3499193323Sed return DAG.getConstant(Val, VT); 3500249423Sdim return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); 3501193323Sed } 3502193323Sed 3503193323Sed Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3504218893Sdim if (NumBits > 8) { 3505218893Sdim // Use a multiplication with 0x010101... to extend the input to the 3506218893Sdim // required length. 3507249423Sdim APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 3508218893Sdim Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3509193323Sed } 3510193323Sed 3511193323Sed return Value; 3512193323Sed} 3513193323Sed 3514193323Sed/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3515193323Sed/// used when a memcpy is turned into a memset when the source is a constant 3516193323Sed/// string ptr. 3517263508Sdimstatic SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG, 3518234353Sdim const TargetLowering &TLI, StringRef Str) { 3519193323Sed // Handle vector with all elements zero. 3520193323Sed if (Str.empty()) { 3521193323Sed if (VT.isInteger()) 3522193323Sed return DAG.getConstant(0, VT); 3523218893Sdim else if (VT == MVT::f32 || VT == MVT::f64) 3524206083Srdivacky return DAG.getConstantFP(0.0, VT); 3525206083Srdivacky else if (VT.isVector()) { 3526206083Srdivacky unsigned NumElts = VT.getVectorNumElements(); 3527206083Srdivacky MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3528218893Sdim return DAG.getNode(ISD::BITCAST, dl, VT, 3529206083Srdivacky DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3530206083Srdivacky EltVT, NumElts))); 3531206083Srdivacky } else 3532206083Srdivacky llvm_unreachable("Expected type!"); 3533193323Sed } 3534193323Sed 3535193323Sed assert(!VT.isVector() && "Can't handle vector type here!"); 3536249423Sdim unsigned NumVTBits = VT.getSizeInBits(); 3537249423Sdim unsigned NumVTBytes = NumVTBits / 8; 3538234353Sdim unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); 3539234353Sdim 3540249423Sdim APInt Val(NumVTBits, 0); 3541234353Sdim if (TLI.isLittleEndian()) { 3542234353Sdim for (unsigned i = 0; i != NumBytes; ++i) 3543234353Sdim Val |= (uint64_t)(unsigned char)Str[i] << i*8; 3544234353Sdim } else { 3545234353Sdim for (unsigned i = 0; i != NumBytes; ++i) 3546234353Sdim Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; 3547193323Sed } 3548234353Sdim 3549249423Sdim // If the "cost" of materializing the integer immediate is 1 or free, then 3550249423Sdim // it is cost effective to turn the load into the immediate. 3551249423Sdim const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); 3552249423Sdim if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) 3553249423Sdim return DAG.getConstant(Val, VT); 3554249423Sdim return SDValue(0, 0); 3555193323Sed} 3556193323Sed 3557193323Sed/// getMemBasePlusOffset - Returns base and offset node for the 3558193323Sed/// 3559263508Sdimstatic SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl, 3560193323Sed SelectionDAG &DAG) { 3561198090Srdivacky EVT VT = Base.getValueType(); 3562263508Sdim return DAG.getNode(ISD::ADD, dl, 3563193323Sed VT, Base, DAG.getConstant(Offset, VT)); 3564193323Sed} 3565193323Sed 3566193323Sed/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3567193323Sed/// 3568234353Sdimstatic bool isMemSrcFromString(SDValue Src, StringRef &Str) { 3569193323Sed unsigned SrcDelta = 0; 3570193323Sed GlobalAddressSDNode *G = NULL; 3571193323Sed if (Src.getOpcode() == ISD::GlobalAddress) 3572193323Sed G = cast<GlobalAddressSDNode>(Src); 3573193323Sed else if (Src.getOpcode() == ISD::ADD && 3574193323Sed Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3575193323Sed Src.getOperand(1).getOpcode() == ISD::Constant) { 3576193323Sed G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3577193323Sed SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3578193323Sed } 3579193323Sed if (!G) 3580193323Sed return false; 3581193323Sed 3582234353Sdim return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false); 3583193323Sed} 3584193323Sed 3585206083Srdivacky/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3586206083Srdivacky/// to replace the memset / memcpy. Return true if the number of memory ops 3587206083Srdivacky/// is below the threshold. It returns the types of the sequence of 3588206083Srdivacky/// memory ops to perform memset / memcpy by reference. 3589206083Srdivackystatic bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3590206083Srdivacky unsigned Limit, uint64_t Size, 3591206083Srdivacky unsigned DstAlign, unsigned SrcAlign, 3592249423Sdim bool IsMemset, 3593249423Sdim bool ZeroMemset, 3594207618Srdivacky bool MemcpyStrSrc, 3595249423Sdim bool AllowOverlap, 3596206083Srdivacky SelectionDAG &DAG, 3597206083Srdivacky const TargetLowering &TLI) { 3598206083Srdivacky assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3599206083Srdivacky "Expecting memcpy / memset source to meet alignment requirement!"); 3600224145Sdim // If 'SrcAlign' is zero, that means the memory operation does not need to 3601224145Sdim // load the value, i.e. memset or memcpy from constant string. Otherwise, 3602224145Sdim // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3603224145Sdim // is the specified alignment of the memory operation. If it is zero, that 3604224145Sdim // means it's possible to change the alignment of the destination. 3605224145Sdim // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3606224145Sdim // not need to be loaded. 3607206124Srdivacky EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3608249423Sdim IsMemset, ZeroMemset, MemcpyStrSrc, 3609207618Srdivacky DAG.getMachineFunction()); 3610193323Sed 3611204961Srdivacky if (VT == MVT::Other) { 3612243830Sdim if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() || 3613206083Srdivacky TLI.allowsUnalignedMemoryAccesses(VT)) { 3614206274Srdivacky VT = TLI.getPointerTy(); 3615193323Sed } else { 3616206083Srdivacky switch (DstAlign & 7) { 3617193323Sed case 0: VT = MVT::i64; break; 3618193323Sed case 4: VT = MVT::i32; break; 3619193323Sed case 2: VT = MVT::i16; break; 3620193323Sed default: VT = MVT::i8; break; 3621193323Sed } 3622193323Sed } 3623193323Sed 3624193323Sed MVT LVT = MVT::i64; 3625193323Sed while (!TLI.isTypeLegal(LVT)) 3626198090Srdivacky LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3627193323Sed assert(LVT.isInteger()); 3628193323Sed 3629193323Sed if (VT.bitsGT(LVT)) 3630193323Sed VT = LVT; 3631193323Sed } 3632193323Sed 3633193323Sed unsigned NumMemOps = 0; 3634193323Sed while (Size != 0) { 3635193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3636193323Sed while (VTSize > Size) { 3637193323Sed // For now, only use non-vector load / store's for the left-over pieces. 3638249423Sdim EVT NewVT = VT; 3639249423Sdim unsigned NewVTSize; 3640249423Sdim 3641249423Sdim bool Found = false; 3642206083Srdivacky if (VT.isVector() || VT.isFloatingPoint()) { 3643249423Sdim NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 3644249423Sdim if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 3645249423Sdim TLI.isSafeMemOpType(NewVT.getSimpleVT())) 3646249423Sdim Found = true; 3647249423Sdim else if (NewVT == MVT::i64 && 3648249423Sdim TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 3649249423Sdim TLI.isSafeMemOpType(MVT::f64)) { 3650249423Sdim // i64 is usually not legal on 32-bit targets, but f64 may be. 3651249423Sdim NewVT = MVT::f64; 3652249423Sdim Found = true; 3653249423Sdim } 3654193323Sed } 3655249423Sdim 3656249423Sdim if (!Found) { 3657249423Sdim do { 3658249423Sdim NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 3659249423Sdim if (NewVT == MVT::i8) 3660249423Sdim break; 3661249423Sdim } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 3662249423Sdim } 3663249423Sdim NewVTSize = NewVT.getSizeInBits() / 8; 3664249423Sdim 3665249423Sdim // If the new VT cannot cover all of the remaining bits, then consider 3666249423Sdim // issuing a (or a pair of) unaligned and overlapping load / store. 3667249423Sdim // FIXME: Only does this for 64-bit or more since we don't have proper 3668249423Sdim // cost model for unaligned load / store. 3669249423Sdim bool Fast; 3670249423Sdim if (NumMemOps && AllowOverlap && 3671249423Sdim VTSize >= 8 && NewVTSize < Size && 3672249423Sdim TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) 3673249423Sdim VTSize = Size; 3674249423Sdim else { 3675249423Sdim VT = NewVT; 3676249423Sdim VTSize = NewVTSize; 3677249423Sdim } 3678193323Sed } 3679193323Sed 3680193323Sed if (++NumMemOps > Limit) 3681193323Sed return false; 3682249423Sdim 3683193323Sed MemOps.push_back(VT); 3684193323Sed Size -= VTSize; 3685193323Sed } 3686193323Sed 3687193323Sed return true; 3688193323Sed} 3689193323Sed 3690263508Sdimstatic SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3691206083Srdivacky SDValue Chain, SDValue Dst, 3692206083Srdivacky SDValue Src, uint64_t Size, 3693206274Srdivacky unsigned Align, bool isVol, 3694206274Srdivacky bool AlwaysInline, 3695218893Sdim MachinePointerInfo DstPtrInfo, 3696218893Sdim MachinePointerInfo SrcPtrInfo) { 3697206124Srdivacky // Turn a memcpy of undef to nop. 3698206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3699206124Srdivacky return Chain; 3700193323Sed 3701193323Sed // Expand memcpy to a series of load and store ops if the size operand falls 3702193323Sed // below a certain threshold. 3703218893Sdim // TODO: In the AlwaysInline case, if the size is big then generate a loop 3704218893Sdim // rather than maybe a humongous number of loads and stores. 3705206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3706198090Srdivacky std::vector<EVT> MemOps; 3707206083Srdivacky bool DstAlignCanChange = false; 3708218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3709218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3710243830Sdim bool OptSize = 3711249423Sdim MF.getFunction()->getAttributes(). 3712249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3713206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3714206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3715206083Srdivacky DstAlignCanChange = true; 3716206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3717206083Srdivacky if (Align > SrcAlign) 3718206083Srdivacky SrcAlign = Align; 3719234353Sdim StringRef Str; 3720206083Srdivacky bool CopyFromStr = isMemSrcFromString(Src, Str); 3721206083Srdivacky bool isZeroStr = CopyFromStr && Str.empty(); 3722218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3723218893Sdim 3724206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3725206083Srdivacky (DstAlignCanChange ? 0 : Align), 3726207618Srdivacky (isZeroStr ? 0 : SrcAlign), 3727249423Sdim false, false, CopyFromStr, true, DAG, TLI)) 3728193323Sed return SDValue(); 3729193323Sed 3730206083Srdivacky if (DstAlignCanChange) { 3731226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3732243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3733249423Sdim 3734249423Sdim // Don't promote to an alignment that would require dynamic stack 3735263508Sdim // realignment. 3736249423Sdim const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); 3737249423Sdim if (!TRI->needsStackRealignment(MF)) 3738249423Sdim while (NewAlign > Align && 3739249423Sdim TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) 3740249423Sdim NewAlign /= 2; 3741249423Sdim 3742206083Srdivacky if (NewAlign > Align) { 3743206083Srdivacky // Give the stack frame object a larger alignment if needed. 3744206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3745206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3746206083Srdivacky Align = NewAlign; 3747206083Srdivacky } 3748206083Srdivacky } 3749193323Sed 3750193323Sed SmallVector<SDValue, 8> OutChains; 3751193323Sed unsigned NumMemOps = MemOps.size(); 3752193323Sed uint64_t SrcOff = 0, DstOff = 0; 3753198090Srdivacky for (unsigned i = 0; i != NumMemOps; ++i) { 3754198090Srdivacky EVT VT = MemOps[i]; 3755193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3756193323Sed SDValue Value, Store; 3757193323Sed 3758249423Sdim if (VTSize > Size) { 3759249423Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3760249423Sdim // pair. Adjust the offset accordingly. 3761249423Sdim assert(i == NumMemOps-1 && i != 0); 3762249423Sdim SrcOff -= VTSize - Size; 3763249423Sdim DstOff -= VTSize - Size; 3764249423Sdim } 3765249423Sdim 3766206083Srdivacky if (CopyFromStr && 3767206083Srdivacky (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3768193323Sed // It's unlikely a store of a vector immediate can be done in a single 3769193323Sed // instruction. It would require a load from a constantpool first. 3770206083Srdivacky // We only handle zero vectors here. 3771193323Sed // FIXME: Handle other cases where store of vector immediate is done in 3772193323Sed // a single instruction. 3773234353Sdim Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); 3774249423Sdim if (Value.getNode()) 3775249423Sdim Store = DAG.getStore(Chain, dl, Value, 3776263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3777249423Sdim DstPtrInfo.getWithOffset(DstOff), isVol, 3778249423Sdim false, Align); 3779249423Sdim } 3780249423Sdim 3781249423Sdim if (!Store.getNode()) { 3782194710Sed // The type might not be legal for the target. This should only happen 3783194710Sed // if the type is smaller than a legal type, as on PPC, so the right 3784195098Sed // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3785195098Sed // to Load/Store if NVT==VT. 3786194710Sed // FIXME does the case above also need this? 3787198090Srdivacky EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3788195098Sed assert(NVT.bitsGE(VT)); 3789218893Sdim Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3790263508Sdim getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3791218893Sdim SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3792206083Srdivacky MinAlign(SrcAlign, SrcOff)); 3793195098Sed Store = DAG.getTruncStore(Chain, dl, Value, 3794263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3795218893Sdim DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3796218893Sdim false, Align); 3797193323Sed } 3798193323Sed OutChains.push_back(Store); 3799193323Sed SrcOff += VTSize; 3800193323Sed DstOff += VTSize; 3801249423Sdim Size -= VTSize; 3802193323Sed } 3803193323Sed 3804193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3805193323Sed &OutChains[0], OutChains.size()); 3806193323Sed} 3807193323Sed 3808263508Sdimstatic SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3809206083Srdivacky SDValue Chain, SDValue Dst, 3810206083Srdivacky SDValue Src, uint64_t Size, 3811206274Srdivacky unsigned Align, bool isVol, 3812206274Srdivacky bool AlwaysInline, 3813218893Sdim MachinePointerInfo DstPtrInfo, 3814218893Sdim MachinePointerInfo SrcPtrInfo) { 3815206124Srdivacky // Turn a memmove of undef to nop. 3816206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3817206124Srdivacky return Chain; 3818193323Sed 3819193323Sed // Expand memmove to a series of load and store ops if the size operand falls 3820193323Sed // below a certain threshold. 3821206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3822198090Srdivacky std::vector<EVT> MemOps; 3823206083Srdivacky bool DstAlignCanChange = false; 3824218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3825218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3826249423Sdim bool OptSize = MF.getFunction()->getAttributes(). 3827249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3828206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3829206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3830206083Srdivacky DstAlignCanChange = true; 3831206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3832206083Srdivacky if (Align > SrcAlign) 3833206083Srdivacky SrcAlign = Align; 3834218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3835206083Srdivacky 3836206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3837249423Sdim (DstAlignCanChange ? 0 : Align), SrcAlign, 3838249423Sdim false, false, false, false, DAG, TLI)) 3839193323Sed return SDValue(); 3840193323Sed 3841206083Srdivacky if (DstAlignCanChange) { 3842226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3843243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3844206083Srdivacky if (NewAlign > Align) { 3845206083Srdivacky // Give the stack frame object a larger alignment if needed. 3846206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3847206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3848206083Srdivacky Align = NewAlign; 3849206083Srdivacky } 3850206083Srdivacky } 3851206083Srdivacky 3852193323Sed uint64_t SrcOff = 0, DstOff = 0; 3853193323Sed SmallVector<SDValue, 8> LoadValues; 3854193323Sed SmallVector<SDValue, 8> LoadChains; 3855193323Sed SmallVector<SDValue, 8> OutChains; 3856193323Sed unsigned NumMemOps = MemOps.size(); 3857193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3858198090Srdivacky EVT VT = MemOps[i]; 3859193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3860263508Sdim SDValue Value; 3861193323Sed 3862193323Sed Value = DAG.getLoad(VT, dl, Chain, 3863263508Sdim getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3864218893Sdim SrcPtrInfo.getWithOffset(SrcOff), isVol, 3865234353Sdim false, false, SrcAlign); 3866193323Sed LoadValues.push_back(Value); 3867193323Sed LoadChains.push_back(Value.getValue(1)); 3868193323Sed SrcOff += VTSize; 3869193323Sed } 3870193323Sed Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3871193323Sed &LoadChains[0], LoadChains.size()); 3872193323Sed OutChains.clear(); 3873193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3874198090Srdivacky EVT VT = MemOps[i]; 3875193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3876263508Sdim SDValue Store; 3877193323Sed 3878193323Sed Store = DAG.getStore(Chain, dl, LoadValues[i], 3879263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3880218893Sdim DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3881193323Sed OutChains.push_back(Store); 3882193323Sed DstOff += VTSize; 3883193323Sed } 3884193323Sed 3885193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3886193323Sed &OutChains[0], OutChains.size()); 3887193323Sed} 3888193323Sed 3889263508Sdim/// \brief Lower the call to 'memset' intrinsic function into a series of store 3890263508Sdim/// operations. 3891263508Sdim/// 3892263508Sdim/// \param DAG Selection DAG where lowered code is placed. 3893263508Sdim/// \param dl Link to corresponding IR location. 3894263508Sdim/// \param Chain Control flow dependency. 3895263508Sdim/// \param Dst Pointer to destination memory location. 3896263508Sdim/// \param Src Value of byte to write into the memory. 3897263508Sdim/// \param Size Number of bytes to write. 3898263508Sdim/// \param Align Alignment of the destination in bytes. 3899263508Sdim/// \param isVol True if destination is volatile. 3900263508Sdim/// \param DstPtrInfo IR information on the memory pointer. 3901263508Sdim/// \returns New head in the control flow, if lowering was successful, empty 3902263508Sdim/// SDValue otherwise. 3903263508Sdim/// 3904263508Sdim/// The function tries to replace 'llvm.memset' intrinsic with several store 3905263508Sdim/// operations and value calculation code. This is usually profitable for small 3906263508Sdim/// memory size. 3907263508Sdimstatic SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, 3908206083Srdivacky SDValue Chain, SDValue Dst, 3909206083Srdivacky SDValue Src, uint64_t Size, 3910206274Srdivacky unsigned Align, bool isVol, 3911218893Sdim MachinePointerInfo DstPtrInfo) { 3912206124Srdivacky // Turn a memset of undef to nop. 3913206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3914206124Srdivacky return Chain; 3915193323Sed 3916193323Sed // Expand memset to a series of load/store ops if the size operand 3917193323Sed // falls below a certain threshold. 3918206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3919198090Srdivacky std::vector<EVT> MemOps; 3920206083Srdivacky bool DstAlignCanChange = false; 3921218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3922218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3923249423Sdim bool OptSize = MF.getFunction()->getAttributes(). 3924249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3925206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3926206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3927206083Srdivacky DstAlignCanChange = true; 3928234353Sdim bool IsZeroVal = 3929206124Srdivacky isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3930218893Sdim if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3931206083Srdivacky Size, (DstAlignCanChange ? 0 : Align), 0, 3932249423Sdim true, IsZeroVal, false, true, DAG, TLI)) 3933193323Sed return SDValue(); 3934193323Sed 3935206083Srdivacky if (DstAlignCanChange) { 3936226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3937243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3938206083Srdivacky if (NewAlign > Align) { 3939206083Srdivacky // Give the stack frame object a larger alignment if needed. 3940206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3941206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3942206083Srdivacky Align = NewAlign; 3943206083Srdivacky } 3944206083Srdivacky } 3945206083Srdivacky 3946193323Sed SmallVector<SDValue, 8> OutChains; 3947193323Sed uint64_t DstOff = 0; 3948193323Sed unsigned NumMemOps = MemOps.size(); 3949218893Sdim 3950218893Sdim // Find the largest store and generate the bit pattern for it. 3951218893Sdim EVT LargestVT = MemOps[0]; 3952218893Sdim for (unsigned i = 1; i < NumMemOps; i++) 3953218893Sdim if (MemOps[i].bitsGT(LargestVT)) 3954218893Sdim LargestVT = MemOps[i]; 3955218893Sdim SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3956218893Sdim 3957193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3958198090Srdivacky EVT VT = MemOps[i]; 3959249423Sdim unsigned VTSize = VT.getSizeInBits() / 8; 3960249423Sdim if (VTSize > Size) { 3961249423Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3962249423Sdim // pair. Adjust the offset accordingly. 3963249423Sdim assert(i == NumMemOps-1 && i != 0); 3964249423Sdim DstOff -= VTSize - Size; 3965249423Sdim } 3966218893Sdim 3967218893Sdim // If this store is smaller than the largest store see whether we can get 3968218893Sdim // the smaller value for free with a truncate. 3969218893Sdim SDValue Value = MemSetValue; 3970218893Sdim if (VT.bitsLT(LargestVT)) { 3971218893Sdim if (!LargestVT.isVector() && !VT.isVector() && 3972218893Sdim TLI.isTruncateFree(LargestVT, VT)) 3973218893Sdim Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3974218893Sdim else 3975218893Sdim Value = getMemsetValue(Src, VT, DAG, dl); 3976218893Sdim } 3977218893Sdim assert(Value.getValueType() == VT && "Value with wrong type."); 3978193323Sed SDValue Store = DAG.getStore(Chain, dl, Value, 3979263508Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3980218893Sdim DstPtrInfo.getWithOffset(DstOff), 3981218893Sdim isVol, false, Align); 3982193323Sed OutChains.push_back(Store); 3983218893Sdim DstOff += VT.getSizeInBits() / 8; 3984249423Sdim Size -= VTSize; 3985193323Sed } 3986193323Sed 3987193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3988193323Sed &OutChains[0], OutChains.size()); 3989193323Sed} 3990193323Sed 3991263508SdimSDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, 3992193323Sed SDValue Src, SDValue Size, 3993206274Srdivacky unsigned Align, bool isVol, bool AlwaysInline, 3994218893Sdim MachinePointerInfo DstPtrInfo, 3995218893Sdim MachinePointerInfo SrcPtrInfo) { 3996249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 3997193323Sed 3998193323Sed // Check to see if we should lower the memcpy to loads and stores first. 3999193323Sed // For cases within the target-specified limits, this is the best choice. 4000193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4001193323Sed if (ConstantSize) { 4002193323Sed // Memcpy with size zero? Just return the original chain. 4003193323Sed if (ConstantSize->isNullValue()) 4004193323Sed return Chain; 4005193323Sed 4006206083Srdivacky SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4007206083Srdivacky ConstantSize->getZExtValue(),Align, 4008218893Sdim isVol, false, DstPtrInfo, SrcPtrInfo); 4009193323Sed if (Result.getNode()) 4010193323Sed return Result; 4011193323Sed } 4012193323Sed 4013193323Sed // Then check to see if we should lower the memcpy with target-specific 4014193323Sed // code. If the target chooses to do this, this is the next best. 4015193323Sed SDValue Result = 4016208599Srdivacky TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 4017206274Srdivacky isVol, AlwaysInline, 4018218893Sdim DstPtrInfo, SrcPtrInfo); 4019193323Sed if (Result.getNode()) 4020193323Sed return Result; 4021193323Sed 4022193323Sed // If we really need inline code and the target declined to provide it, 4023193323Sed // use a (potentially long) sequence of loads and stores. 4024193323Sed if (AlwaysInline) { 4025193323Sed assert(ConstantSize && "AlwaysInline requires a constant size!"); 4026193323Sed return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4027206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 4028218893Sdim true, DstPtrInfo, SrcPtrInfo); 4029193323Sed } 4030193323Sed 4031206274Srdivacky // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 4032206274Srdivacky // memcpy is not guaranteed to be safe. libc memcpys aren't required to 4033206274Srdivacky // respect volatile, so they may do things like read or write memory 4034206274Srdivacky // beyond the given memory regions. But fixing this isn't easy, and most 4035206274Srdivacky // people don't care. 4036206274Srdivacky 4037263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4038263508Sdim 4039193323Sed // Emit a library call. 4040193323Sed TargetLowering::ArgListTy Args; 4041193323Sed TargetLowering::ArgListEntry Entry; 4042263508Sdim Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4043193323Sed Entry.Node = Dst; Args.push_back(Entry); 4044193323Sed Entry.Node = Src; Args.push_back(Entry); 4045193323Sed Entry.Node = Size; Args.push_back(Entry); 4046263508Sdim // FIXME: pass in SDLoc 4047239462Sdim TargetLowering:: 4048239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4049198090Srdivacky false, false, false, false, 0, 4050263508Sdim TLI->getLibcallCallingConv(RTLIB::MEMCPY), 4051234353Sdim /*isTailCall=*/false, 4052234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4053263508Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 4054263508Sdim TLI->getPointerTy()), 4055204642Srdivacky Args, *this, dl); 4056263508Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4057239462Sdim 4058193323Sed return CallResult.second; 4059193323Sed} 4060193323Sed 4061263508SdimSDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, 4062193323Sed SDValue Src, SDValue Size, 4063206274Srdivacky unsigned Align, bool isVol, 4064218893Sdim MachinePointerInfo DstPtrInfo, 4065218893Sdim MachinePointerInfo SrcPtrInfo) { 4066249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4067193323Sed 4068193323Sed // Check to see if we should lower the memmove to loads and stores first. 4069193323Sed // For cases within the target-specified limits, this is the best choice. 4070193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4071193323Sed if (ConstantSize) { 4072193323Sed // Memmove with size zero? Just return the original chain. 4073193323Sed if (ConstantSize->isNullValue()) 4074193323Sed return Chain; 4075193323Sed 4076193323Sed SDValue Result = 4077193323Sed getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 4078206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 4079218893Sdim false, DstPtrInfo, SrcPtrInfo); 4080193323Sed if (Result.getNode()) 4081193323Sed return Result; 4082193323Sed } 4083193323Sed 4084193323Sed // Then check to see if we should lower the memmove with target-specific 4085193323Sed // code. If the target chooses to do this, this is the next best. 4086193323Sed SDValue Result = 4087208599Srdivacky TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4088218893Sdim DstPtrInfo, SrcPtrInfo); 4089193323Sed if (Result.getNode()) 4090193323Sed return Result; 4091193323Sed 4092207618Srdivacky // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 4093207618Srdivacky // not be safe. See memcpy above for more details. 4094207618Srdivacky 4095263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4096263508Sdim 4097193323Sed // Emit a library call. 4098193323Sed TargetLowering::ArgListTy Args; 4099193323Sed TargetLowering::ArgListEntry Entry; 4100263508Sdim Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4101193323Sed Entry.Node = Dst; Args.push_back(Entry); 4102193323Sed Entry.Node = Src; Args.push_back(Entry); 4103193323Sed Entry.Node = Size; Args.push_back(Entry); 4104263508Sdim // FIXME: pass in SDLoc 4105239462Sdim TargetLowering:: 4106239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4107198090Srdivacky false, false, false, false, 0, 4108263508Sdim TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 4109234353Sdim /*isTailCall=*/false, 4110234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4111263508Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 4112263508Sdim TLI->getPointerTy()), 4113204642Srdivacky Args, *this, dl); 4114263508Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4115239462Sdim 4116193323Sed return CallResult.second; 4117193323Sed} 4118193323Sed 4119263508SdimSDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, 4120193323Sed SDValue Src, SDValue Size, 4121206274Srdivacky unsigned Align, bool isVol, 4122218893Sdim MachinePointerInfo DstPtrInfo) { 4123249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4124193323Sed 4125193323Sed // Check to see if we should lower the memset to stores first. 4126193323Sed // For cases within the target-specified limits, this is the best choice. 4127193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4128193323Sed if (ConstantSize) { 4129193323Sed // Memset with size zero? Just return the original chain. 4130193323Sed if (ConstantSize->isNullValue()) 4131193323Sed return Chain; 4132193323Sed 4133206274Srdivacky SDValue Result = 4134206274Srdivacky getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 4135218893Sdim Align, isVol, DstPtrInfo); 4136206274Srdivacky 4137193323Sed if (Result.getNode()) 4138193323Sed return Result; 4139193323Sed } 4140193323Sed 4141193323Sed // Then check to see if we should lower the memset with target-specific 4142193323Sed // code. If the target chooses to do this, this is the next best. 4143193323Sed SDValue Result = 4144208599Srdivacky TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4145218893Sdim DstPtrInfo); 4146193323Sed if (Result.getNode()) 4147193323Sed return Result; 4148193323Sed 4149218893Sdim // Emit a library call. 4150263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4151263508Sdim Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext()); 4152193323Sed TargetLowering::ArgListTy Args; 4153193323Sed TargetLowering::ArgListEntry Entry; 4154193323Sed Entry.Node = Dst; Entry.Ty = IntPtrTy; 4155193323Sed Args.push_back(Entry); 4156193323Sed // Extend or truncate the argument to be an i32 value for the call. 4157193323Sed if (Src.getValueType().bitsGT(MVT::i32)) 4158193323Sed Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 4159193323Sed else 4160193323Sed Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 4161198090Srdivacky Entry.Node = Src; 4162198090Srdivacky Entry.Ty = Type::getInt32Ty(*getContext()); 4163198090Srdivacky Entry.isSExt = true; 4164193323Sed Args.push_back(Entry); 4165198090Srdivacky Entry.Node = Size; 4166198090Srdivacky Entry.Ty = IntPtrTy; 4167198090Srdivacky Entry.isSExt = false; 4168193323Sed Args.push_back(Entry); 4169263508Sdim // FIXME: pass in SDLoc 4170239462Sdim TargetLowering:: 4171239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4172198090Srdivacky false, false, false, false, 0, 4173263508Sdim TLI->getLibcallCallingConv(RTLIB::MEMSET), 4174234353Sdim /*isTailCall=*/false, 4175234353Sdim /*doesNotReturn*/false, /*isReturnValueUsed=*/false, 4176263508Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 4177263508Sdim TLI->getPointerTy()), 4178204642Srdivacky Args, *this, dl); 4179263508Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4180239462Sdim 4181193323Sed return CallResult.second; 4182193323Sed} 4183193323Sed 4184263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4185263508Sdim SDVTList VTList, SDValue* Ops, unsigned NumOps, 4186263508Sdim MachineMemOperand *MMO, 4187263508Sdim AtomicOrdering Ordering, 4188263508Sdim SynchronizationScope SynchScope) { 4189263508Sdim FoldingSetNodeID ID; 4190263508Sdim ID.AddInteger(MemVT.getRawBits()); 4191263508Sdim AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4192263508Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4193263508Sdim void* IP = 0; 4194263508Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4195263508Sdim cast<AtomicSDNode>(E)->refineAlignment(MMO); 4196263508Sdim return SDValue(E, 0); 4197263508Sdim } 4198263508Sdim 4199263508Sdim // Allocate the operands array for the node out of the BumpPtrAllocator, since 4200263508Sdim // SDNode doesn't have access to it. This memory will be "leaked" when 4201263508Sdim // the node is deallocated, but recovered when the allocator is released. 4202263508Sdim // If the number of operands is less than 5 we use AtomicSDNode's internal 4203263508Sdim // storage. 4204263508Sdim SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps) : 0; 4205263508Sdim 4206263508Sdim SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), 4207263508Sdim dl.getDebugLoc(), VTList, MemVT, 4208263508Sdim Ops, DynOps, NumOps, MMO, 4209263508Sdim Ordering, SynchScope); 4210263508Sdim CSEMap.InsertNode(N, IP); 4211263508Sdim AllNodes.push_back(N); 4212263508Sdim return SDValue(N, 0); 4213263508Sdim} 4214263508Sdim 4215263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4216218893Sdim SDValue Chain, SDValue Ptr, SDValue Cmp, 4217218893Sdim SDValue Swp, MachinePointerInfo PtrInfo, 4218226633Sdim unsigned Alignment, 4219226633Sdim AtomicOrdering Ordering, 4220243830Sdim SynchronizationScope SynchScope) { 4221198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4222198090Srdivacky Alignment = getEVTAlignment(MemVT); 4223198090Srdivacky 4224198090Srdivacky MachineFunction &MF = getMachineFunction(); 4225198090Srdivacky 4226243830Sdim // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. 4227198090Srdivacky // For now, atomics are considered to be volatile always. 4228226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4229226633Sdim // orderings in the memoperand. 4230243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4231243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4232243830Sdim Flags |= MachineMemOperand::MOLoad; 4233243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4234243830Sdim Flags |= MachineMemOperand::MOStore; 4235198090Srdivacky 4236198090Srdivacky MachineMemOperand *MMO = 4237218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 4238198090Srdivacky 4239226633Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 4240226633Sdim Ordering, SynchScope); 4241198090Srdivacky} 4242198090Srdivacky 4243263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4244198090Srdivacky SDValue Chain, 4245198090Srdivacky SDValue Ptr, SDValue Cmp, 4246226633Sdim SDValue Swp, MachineMemOperand *MMO, 4247226633Sdim AtomicOrdering Ordering, 4248226633Sdim SynchronizationScope SynchScope) { 4249193323Sed assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 4250193323Sed assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 4251193323Sed 4252198090Srdivacky EVT VT = Cmp.getValueType(); 4253193323Sed 4254193323Sed SDVTList VTs = getVTList(VT, MVT::Other); 4255193323Sed SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 4256263508Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, Ordering, SynchScope); 4257193323Sed} 4258193323Sed 4259263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4260193323Sed SDValue Chain, 4261193323Sed SDValue Ptr, SDValue Val, 4262193323Sed const Value* PtrVal, 4263226633Sdim unsigned Alignment, 4264226633Sdim AtomicOrdering Ordering, 4265226633Sdim SynchronizationScope SynchScope) { 4266198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4267198090Srdivacky Alignment = getEVTAlignment(MemVT); 4268198090Srdivacky 4269198090Srdivacky MachineFunction &MF = getMachineFunction(); 4270243830Sdim // An atomic store does not load. An atomic load does not store. 4271226633Sdim // (An atomicrmw obviously both loads and stores.) 4272243830Sdim // For now, atomics are considered to be volatile always, and they are 4273243830Sdim // chained as such. 4274226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4275226633Sdim // orderings in the memoperand. 4276243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4277243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4278243830Sdim Flags |= MachineMemOperand::MOLoad; 4279243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4280243830Sdim Flags |= MachineMemOperand::MOStore; 4281198090Srdivacky 4282198090Srdivacky MachineMemOperand *MMO = 4283218893Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4284198090Srdivacky MemVT.getStoreSize(), Alignment); 4285198090Srdivacky 4286226633Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 4287226633Sdim Ordering, SynchScope); 4288198090Srdivacky} 4289198090Srdivacky 4290263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4291198090Srdivacky SDValue Chain, 4292198090Srdivacky SDValue Ptr, SDValue Val, 4293226633Sdim MachineMemOperand *MMO, 4294226633Sdim AtomicOrdering Ordering, 4295226633Sdim SynchronizationScope SynchScope) { 4296193323Sed assert((Opcode == ISD::ATOMIC_LOAD_ADD || 4297193323Sed Opcode == ISD::ATOMIC_LOAD_SUB || 4298193323Sed Opcode == ISD::ATOMIC_LOAD_AND || 4299193323Sed Opcode == ISD::ATOMIC_LOAD_OR || 4300193323Sed Opcode == ISD::ATOMIC_LOAD_XOR || 4301193323Sed Opcode == ISD::ATOMIC_LOAD_NAND || 4302193323Sed Opcode == ISD::ATOMIC_LOAD_MIN || 4303193323Sed Opcode == ISD::ATOMIC_LOAD_MAX || 4304193323Sed Opcode == ISD::ATOMIC_LOAD_UMIN || 4305193323Sed Opcode == ISD::ATOMIC_LOAD_UMAX || 4306226633Sdim Opcode == ISD::ATOMIC_SWAP || 4307226633Sdim Opcode == ISD::ATOMIC_STORE) && 4308193323Sed "Invalid Atomic Op"); 4309193323Sed 4310198090Srdivacky EVT VT = Val.getValueType(); 4311193323Sed 4312226633Sdim SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 4313226633Sdim getVTList(VT, MVT::Other); 4314193323Sed SDValue Ops[] = {Chain, Ptr, Val}; 4315263508Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 3, MMO, Ordering, SynchScope); 4316193323Sed} 4317193323Sed 4318263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4319226633Sdim EVT VT, SDValue Chain, 4320226633Sdim SDValue Ptr, 4321226633Sdim const Value* PtrVal, 4322226633Sdim unsigned Alignment, 4323226633Sdim AtomicOrdering Ordering, 4324226633Sdim SynchronizationScope SynchScope) { 4325226633Sdim if (Alignment == 0) // Ensure that codegen never sees alignment 0 4326226633Sdim Alignment = getEVTAlignment(MemVT); 4327226633Sdim 4328226633Sdim MachineFunction &MF = getMachineFunction(); 4329243830Sdim // An atomic store does not load. An atomic load does not store. 4330243830Sdim // (An atomicrmw obviously both loads and stores.) 4331243830Sdim // For now, atomics are considered to be volatile always, and they are 4332243830Sdim // chained as such. 4333226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4334226633Sdim // orderings in the memoperand. 4335243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4336243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4337243830Sdim Flags |= MachineMemOperand::MOLoad; 4338243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4339243830Sdim Flags |= MachineMemOperand::MOStore; 4340226633Sdim 4341226633Sdim MachineMemOperand *MMO = 4342226633Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4343226633Sdim MemVT.getStoreSize(), Alignment); 4344226633Sdim 4345226633Sdim return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4346226633Sdim Ordering, SynchScope); 4347226633Sdim} 4348226633Sdim 4349263508SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4350226633Sdim EVT VT, SDValue Chain, 4351226633Sdim SDValue Ptr, 4352226633Sdim MachineMemOperand *MMO, 4353226633Sdim AtomicOrdering Ordering, 4354226633Sdim SynchronizationScope SynchScope) { 4355226633Sdim assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4356226633Sdim 4357226633Sdim SDVTList VTs = getVTList(VT, MVT::Other); 4358226633Sdim SDValue Ops[] = {Chain, Ptr}; 4359263508Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 2, MMO, Ordering, SynchScope); 4360226633Sdim} 4361226633Sdim 4362193323Sed/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4363193323SedSDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4364263508Sdim SDLoc dl) { 4365193323Sed if (NumOps == 1) 4366193323Sed return Ops[0]; 4367193323Sed 4368198090Srdivacky SmallVector<EVT, 4> VTs; 4369193323Sed VTs.reserve(NumOps); 4370193323Sed for (unsigned i = 0; i < NumOps; ++i) 4371193323Sed VTs.push_back(Ops[i].getValueType()); 4372193323Sed return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4373193323Sed Ops, NumOps); 4374193323Sed} 4375193323Sed 4376193323SedSDValue 4377263508SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, 4378198090Srdivacky const EVT *VTs, unsigned NumVTs, 4379193323Sed const SDValue *Ops, unsigned NumOps, 4380218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4381193323Sed unsigned Align, bool Vol, 4382193323Sed bool ReadMem, bool WriteMem) { 4383193323Sed return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4384218893Sdim MemVT, PtrInfo, Align, Vol, 4385193323Sed ReadMem, WriteMem); 4386193323Sed} 4387193323Sed 4388193323SedSDValue 4389263508SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4390193323Sed const SDValue *Ops, unsigned NumOps, 4391218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4392193323Sed unsigned Align, bool Vol, 4393193323Sed bool ReadMem, bool WriteMem) { 4394198090Srdivacky if (Align == 0) // Ensure that codegen never sees alignment 0 4395198090Srdivacky Align = getEVTAlignment(MemVT); 4396198090Srdivacky 4397198090Srdivacky MachineFunction &MF = getMachineFunction(); 4398198090Srdivacky unsigned Flags = 0; 4399198090Srdivacky if (WriteMem) 4400198090Srdivacky Flags |= MachineMemOperand::MOStore; 4401198090Srdivacky if (ReadMem) 4402198090Srdivacky Flags |= MachineMemOperand::MOLoad; 4403198090Srdivacky if (Vol) 4404198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4405198090Srdivacky MachineMemOperand *MMO = 4406218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4407198090Srdivacky 4408198090Srdivacky return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4409198090Srdivacky} 4410198090Srdivacky 4411198090SrdivackySDValue 4412263508SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4413198090Srdivacky const SDValue *Ops, unsigned NumOps, 4414198090Srdivacky EVT MemVT, MachineMemOperand *MMO) { 4415198090Srdivacky assert((Opcode == ISD::INTRINSIC_VOID || 4416198090Srdivacky Opcode == ISD::INTRINSIC_W_CHAIN || 4417218893Sdim Opcode == ISD::PREFETCH || 4418243830Sdim Opcode == ISD::LIFETIME_START || 4419243830Sdim Opcode == ISD::LIFETIME_END || 4420198090Srdivacky (Opcode <= INT_MAX && 4421198090Srdivacky (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4422198090Srdivacky "Opcode is not a memory-accessing opcode!"); 4423198090Srdivacky 4424193323Sed // Memoize the node unless it returns a flag. 4425193323Sed MemIntrinsicSDNode *N; 4426218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4427193323Sed FoldingSetNodeID ID; 4428193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4429239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4430193323Sed void *IP = 0; 4431198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4432198090Srdivacky cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4433193323Sed return SDValue(E, 0); 4434198090Srdivacky } 4435193323Sed 4436263508Sdim N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4437263508Sdim dl.getDebugLoc(), VTList, Ops, 4438263508Sdim NumOps, MemVT, MMO); 4439193323Sed CSEMap.InsertNode(N, IP); 4440193323Sed } else { 4441263508Sdim N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4442263508Sdim dl.getDebugLoc(), VTList, Ops, 4443263508Sdim NumOps, MemVT, MMO); 4444193323Sed } 4445193323Sed AllNodes.push_back(N); 4446193323Sed return SDValue(N, 0); 4447193323Sed} 4448193323Sed 4449218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4450218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4451218893Sdim/// code generator has many cases where it doesn't bother passing in a 4452218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4453218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4454218893Sdim // If this is FI+Offset, we can model it. 4455218893Sdim if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4456218893Sdim return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4457218893Sdim 4458218893Sdim // If this is (FI+Offset1)+Offset2, we can model it. 4459218893Sdim if (Ptr.getOpcode() != ISD::ADD || 4460218893Sdim !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4461218893Sdim !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4462218893Sdim return MachinePointerInfo(); 4463218893Sdim 4464218893Sdim int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4465218893Sdim return MachinePointerInfo::getFixedStack(FI, Offset+ 4466218893Sdim cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4467218893Sdim} 4468218893Sdim 4469218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4470218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4471218893Sdim/// code generator has many cases where it doesn't bother passing in a 4472218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4473218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4474218893Sdim // If the 'Offset' value isn't a constant, we can't handle this. 4475218893Sdim if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4476218893Sdim return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4477218893Sdim if (OffsetOp.getOpcode() == ISD::UNDEF) 4478218893Sdim return InferPointerInfo(Ptr); 4479218893Sdim return MachinePointerInfo(); 4480218893Sdim} 4481218893Sdim 4482218893Sdim 4483193323SedSDValue 4484210299SedSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4485263508Sdim EVT VT, SDLoc dl, SDValue Chain, 4486193323Sed SDValue Ptr, SDValue Offset, 4487218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4488234353Sdim bool isVolatile, bool isNonTemporal, bool isInvariant, 4489234353Sdim unsigned Alignment, const MDNode *TBAAInfo, 4490234353Sdim const MDNode *Ranges) { 4491243830Sdim assert(Chain.getValueType() == MVT::Other && 4492224145Sdim "Invalid chain type"); 4493193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4494198090Srdivacky Alignment = getEVTAlignment(VT); 4495193323Sed 4496198090Srdivacky unsigned Flags = MachineMemOperand::MOLoad; 4497198090Srdivacky if (isVolatile) 4498198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4499203954Srdivacky if (isNonTemporal) 4500203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4501234353Sdim if (isInvariant) 4502234353Sdim Flags |= MachineMemOperand::MOInvariant; 4503218893Sdim 4504218893Sdim // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4505218893Sdim // clients. 4506218893Sdim if (PtrInfo.V == 0) 4507218893Sdim PtrInfo = InferPointerInfo(Ptr, Offset); 4508218893Sdim 4509218893Sdim MachineFunction &MF = getMachineFunction(); 4510198090Srdivacky MachineMemOperand *MMO = 4511218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4512234353Sdim TBAAInfo, Ranges); 4513210299Sed return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4514198090Srdivacky} 4515198090Srdivacky 4516198090SrdivackySDValue 4517218893SdimSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4518263508Sdim EVT VT, SDLoc dl, SDValue Chain, 4519198090Srdivacky SDValue Ptr, SDValue Offset, EVT MemVT, 4520198090Srdivacky MachineMemOperand *MMO) { 4521198090Srdivacky if (VT == MemVT) { 4522193323Sed ExtType = ISD::NON_EXTLOAD; 4523193323Sed } else if (ExtType == ISD::NON_EXTLOAD) { 4524198090Srdivacky assert(VT == MemVT && "Non-extending load from different memory type!"); 4525193323Sed } else { 4526193323Sed // Extending load. 4527200581Srdivacky assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4528200581Srdivacky "Should only be an extending load, not truncating!"); 4529198090Srdivacky assert(VT.isInteger() == MemVT.isInteger() && 4530193323Sed "Cannot convert from FP to Int or Int -> FP!"); 4531200581Srdivacky assert(VT.isVector() == MemVT.isVector() && 4532200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4533200581Srdivacky assert((!VT.isVector() || 4534200581Srdivacky VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4535200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4536193323Sed } 4537193323Sed 4538193323Sed bool Indexed = AM != ISD::UNINDEXED; 4539193323Sed assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4540193323Sed "Unindexed load with an offset!"); 4541193323Sed 4542193323Sed SDVTList VTs = Indexed ? 4543193323Sed getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4544193323Sed SDValue Ops[] = { Chain, Ptr, Offset }; 4545193323Sed FoldingSetNodeID ID; 4546193323Sed AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4547198090Srdivacky ID.AddInteger(MemVT.getRawBits()); 4548204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4549243830Sdim MMO->isNonTemporal(), 4550234353Sdim MMO->isInvariant())); 4551239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4552193323Sed void *IP = 0; 4553198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4554198090Srdivacky cast<LoadSDNode>(E)->refineAlignment(MMO); 4555193323Sed return SDValue(E, 0); 4556198090Srdivacky } 4557263508Sdim SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(), 4558263508Sdim dl.getDebugLoc(), VTs, AM, ExtType, 4559205407Srdivacky MemVT, MMO); 4560193323Sed CSEMap.InsertNode(N, IP); 4561193323Sed AllNodes.push_back(N); 4562193323Sed return SDValue(N, 0); 4563193323Sed} 4564193323Sed 4565263508SdimSDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4566193323Sed SDValue Chain, SDValue Ptr, 4567218893Sdim MachinePointerInfo PtrInfo, 4568203954Srdivacky bool isVolatile, bool isNonTemporal, 4569243830Sdim bool isInvariant, unsigned Alignment, 4570234353Sdim const MDNode *TBAAInfo, 4571234353Sdim const MDNode *Ranges) { 4572193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4573210299Sed return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4574234353Sdim PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4575234353Sdim TBAAInfo, Ranges); 4576193323Sed} 4577193323Sed 4578263508SdimSDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4579263508Sdim SDValue Chain, SDValue Ptr, 4580263508Sdim MachineMemOperand *MMO) { 4581263508Sdim SDValue Undef = getUNDEF(Ptr.getValueType()); 4582263508Sdim return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4583263508Sdim VT, MMO); 4584263508Sdim} 4585263508Sdim 4586263508SdimSDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4587193323Sed SDValue Chain, SDValue Ptr, 4588218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4589203954Srdivacky bool isVolatile, bool isNonTemporal, 4590218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4591193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4592210299Sed return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4593234353Sdim PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4594218893Sdim TBAAInfo); 4595193323Sed} 4596193323Sed 4597218893Sdim 4598263508SdimSDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4599263508Sdim SDValue Chain, SDValue Ptr, EVT MemVT, 4600263508Sdim MachineMemOperand *MMO) { 4601263508Sdim SDValue Undef = getUNDEF(Ptr.getValueType()); 4602263508Sdim return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4603263508Sdim MemVT, MMO); 4604263508Sdim} 4605263508Sdim 4606193323SedSDValue 4607263508SdimSelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base, 4608193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4609193323Sed LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4610193323Sed assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4611193323Sed "Load is already a indexed load!"); 4612210299Sed return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4613218893Sdim LD->getChain(), Base, Offset, LD->getPointerInfo(), 4614243830Sdim LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4615234353Sdim false, LD->getAlignment()); 4616193323Sed} 4617193323Sed 4618263508SdimSDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4619218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4620203954Srdivacky bool isVolatile, bool isNonTemporal, 4621218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4622243830Sdim assert(Chain.getValueType() == MVT::Other && 4623224145Sdim "Invalid chain type"); 4624193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4625198090Srdivacky Alignment = getEVTAlignment(Val.getValueType()); 4626193323Sed 4627198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4628198090Srdivacky if (isVolatile) 4629198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4630203954Srdivacky if (isNonTemporal) 4631203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4632218893Sdim 4633218893Sdim if (PtrInfo.V == 0) 4634218893Sdim PtrInfo = InferPointerInfo(Ptr); 4635218893Sdim 4636218893Sdim MachineFunction &MF = getMachineFunction(); 4637198090Srdivacky MachineMemOperand *MMO = 4638218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, 4639218893Sdim Val.getValueType().getStoreSize(), Alignment, 4640218893Sdim TBAAInfo); 4641198090Srdivacky 4642198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4643198090Srdivacky} 4644198090Srdivacky 4645263508SdimSDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4646198090Srdivacky SDValue Ptr, MachineMemOperand *MMO) { 4647243830Sdim assert(Chain.getValueType() == MVT::Other && 4648224145Sdim "Invalid chain type"); 4649198090Srdivacky EVT VT = Val.getValueType(); 4650193323Sed SDVTList VTs = getVTList(MVT::Other); 4651193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4652193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4653193323Sed FoldingSetNodeID ID; 4654193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4655193323Sed ID.AddInteger(VT.getRawBits()); 4656204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4657234353Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4658239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4659193323Sed void *IP = 0; 4660198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4661198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4662193323Sed return SDValue(E, 0); 4663198090Srdivacky } 4664263508Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4665263508Sdim dl.getDebugLoc(), VTs, 4666263508Sdim ISD::UNINDEXED, false, VT, MMO); 4667193323Sed CSEMap.InsertNode(N, IP); 4668193323Sed AllNodes.push_back(N); 4669193323Sed return SDValue(N, 0); 4670193323Sed} 4671193323Sed 4672263508SdimSDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4673218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4674218893Sdim EVT SVT,bool isVolatile, bool isNonTemporal, 4675218893Sdim unsigned Alignment, 4676218893Sdim const MDNode *TBAAInfo) { 4677243830Sdim assert(Chain.getValueType() == MVT::Other && 4678224145Sdim "Invalid chain type"); 4679198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4680198090Srdivacky Alignment = getEVTAlignment(SVT); 4681193323Sed 4682198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4683198090Srdivacky if (isVolatile) 4684198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4685203954Srdivacky if (isNonTemporal) 4686203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4687218893Sdim 4688218893Sdim if (PtrInfo.V == 0) 4689218893Sdim PtrInfo = InferPointerInfo(Ptr); 4690218893Sdim 4691218893Sdim MachineFunction &MF = getMachineFunction(); 4692198090Srdivacky MachineMemOperand *MMO = 4693218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4694218893Sdim TBAAInfo); 4695198090Srdivacky 4696198090Srdivacky return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4697198090Srdivacky} 4698198090Srdivacky 4699263508SdimSDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4700198090Srdivacky SDValue Ptr, EVT SVT, 4701198090Srdivacky MachineMemOperand *MMO) { 4702198090Srdivacky EVT VT = Val.getValueType(); 4703198090Srdivacky 4704243830Sdim assert(Chain.getValueType() == MVT::Other && 4705224145Sdim "Invalid chain type"); 4706193323Sed if (VT == SVT) 4707198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4708193323Sed 4709200581Srdivacky assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4710200581Srdivacky "Should only be a truncating store, not extending!"); 4711193323Sed assert(VT.isInteger() == SVT.isInteger() && 4712193323Sed "Can't do FP-INT conversion!"); 4713200581Srdivacky assert(VT.isVector() == SVT.isVector() && 4714200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4715200581Srdivacky assert((!VT.isVector() || 4716200581Srdivacky VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4717200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4718193323Sed 4719193323Sed SDVTList VTs = getVTList(MVT::Other); 4720193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4721193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4722193323Sed FoldingSetNodeID ID; 4723193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4724193323Sed ID.AddInteger(SVT.getRawBits()); 4725204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4726234353Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4727239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4728193323Sed void *IP = 0; 4729198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4730198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4731193323Sed return SDValue(E, 0); 4732198090Srdivacky } 4733263508Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4734263508Sdim dl.getDebugLoc(), VTs, 4735263508Sdim ISD::UNINDEXED, true, SVT, MMO); 4736193323Sed CSEMap.InsertNode(N, IP); 4737193323Sed AllNodes.push_back(N); 4738193323Sed return SDValue(N, 0); 4739193323Sed} 4740193323Sed 4741193323SedSDValue 4742263508SdimSelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, 4743193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4744193323Sed StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4745193323Sed assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4746193323Sed "Store is already a indexed store!"); 4747193323Sed SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4748193323Sed SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4749193323Sed FoldingSetNodeID ID; 4750193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4751193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 4752193323Sed ID.AddInteger(ST->getRawSubclassData()); 4753239462Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 4754193323Sed void *IP = 0; 4755201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4756193323Sed return SDValue(E, 0); 4757201360Srdivacky 4758263508Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4759263508Sdim dl.getDebugLoc(), VTs, AM, 4760205407Srdivacky ST->isTruncatingStore(), 4761205407Srdivacky ST->getMemoryVT(), 4762205407Srdivacky ST->getMemOperand()); 4763193323Sed CSEMap.InsertNode(N, IP); 4764193323Sed AllNodes.push_back(N); 4765193323Sed return SDValue(N, 0); 4766193323Sed} 4767193323Sed 4768263508SdimSDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl, 4769193323Sed SDValue Chain, SDValue Ptr, 4770210299Sed SDValue SV, 4771210299Sed unsigned Align) { 4772210299Sed SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4773210299Sed return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4774193323Sed} 4775193323Sed 4776263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4777193323Sed const SDUse *Ops, unsigned NumOps) { 4778193323Sed switch (NumOps) { 4779193323Sed case 0: return getNode(Opcode, DL, VT); 4780193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4781193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4782193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4783193323Sed default: break; 4784193323Sed } 4785193323Sed 4786193323Sed // Copy from an SDUse array into an SDValue array for use with 4787193323Sed // the regular getNode logic. 4788193323Sed SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4789193323Sed return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4790193323Sed} 4791193323Sed 4792263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4793193323Sed const SDValue *Ops, unsigned NumOps) { 4794193323Sed switch (NumOps) { 4795193323Sed case 0: return getNode(Opcode, DL, VT); 4796193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4797193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4798193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4799193323Sed default: break; 4800193323Sed } 4801193323Sed 4802193323Sed switch (Opcode) { 4803193323Sed default: break; 4804193323Sed case ISD::SELECT_CC: { 4805193323Sed assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4806193323Sed assert(Ops[0].getValueType() == Ops[1].getValueType() && 4807193323Sed "LHS and RHS of condition must have same type!"); 4808193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4809193323Sed "True and False arms of SelectCC must have same type!"); 4810193323Sed assert(Ops[2].getValueType() == VT && 4811193323Sed "select_cc node must be of same type as true and false value!"); 4812193323Sed break; 4813193323Sed } 4814193323Sed case ISD::BR_CC: { 4815193323Sed assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4816193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4817193323Sed "LHS/RHS of comparison should match types!"); 4818193323Sed break; 4819193323Sed } 4820193323Sed } 4821193323Sed 4822193323Sed // Memoize nodes. 4823193323Sed SDNode *N; 4824193323Sed SDVTList VTs = getVTList(VT); 4825193323Sed 4826218893Sdim if (VT != MVT::Glue) { 4827193323Sed FoldingSetNodeID ID; 4828193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4829193323Sed void *IP = 0; 4830193323Sed 4831201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4832193323Sed return SDValue(E, 0); 4833193323Sed 4834263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4835263508Sdim VTs, Ops, NumOps); 4836193323Sed CSEMap.InsertNode(N, IP); 4837193323Sed } else { 4838263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4839263508Sdim VTs, Ops, NumOps); 4840193323Sed } 4841193323Sed 4842193323Sed AllNodes.push_back(N); 4843193323Sed#ifndef NDEBUG 4844218893Sdim VerifySDNode(N); 4845193323Sed#endif 4846193323Sed return SDValue(N, 0); 4847193323Sed} 4848193323Sed 4849263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4850249423Sdim ArrayRef<EVT> ResultTys, 4851193323Sed const SDValue *Ops, unsigned NumOps) { 4852193323Sed return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4853193323Sed Ops, NumOps); 4854193323Sed} 4855193323Sed 4856263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4857198090Srdivacky const EVT *VTs, unsigned NumVTs, 4858193323Sed const SDValue *Ops, unsigned NumOps) { 4859193323Sed if (NumVTs == 1) 4860193323Sed return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4861193323Sed return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4862193323Sed} 4863193323Sed 4864263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4865193323Sed const SDValue *Ops, unsigned NumOps) { 4866193323Sed if (VTList.NumVTs == 1) 4867193323Sed return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4868193323Sed 4869198090Srdivacky#if 0 4870193323Sed switch (Opcode) { 4871193323Sed // FIXME: figure out how to safely handle things like 4872193323Sed // int foo(int x) { return 1 << (x & 255); } 4873193323Sed // int bar() { return foo(256); } 4874193323Sed case ISD::SRA_PARTS: 4875193323Sed case ISD::SRL_PARTS: 4876193323Sed case ISD::SHL_PARTS: 4877193323Sed if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4878193323Sed cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4879193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4880193323Sed else if (N3.getOpcode() == ISD::AND) 4881193323Sed if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4882193323Sed // If the and is only masking out bits that cannot effect the shift, 4883193323Sed // eliminate the and. 4884202375Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4885193323Sed if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4886193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4887193323Sed } 4888193323Sed break; 4889198090Srdivacky } 4890193323Sed#endif 4891193323Sed 4892193323Sed // Memoize the node unless it returns a flag. 4893193323Sed SDNode *N; 4894218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4895193323Sed FoldingSetNodeID ID; 4896193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4897193323Sed void *IP = 0; 4898201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4899193323Sed return SDValue(E, 0); 4900201360Srdivacky 4901193323Sed if (NumOps == 1) { 4902263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4903263508Sdim DL.getDebugLoc(), VTList, Ops[0]); 4904193323Sed } else if (NumOps == 2) { 4905263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4906263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4907263508Sdim Ops[1]); 4908193323Sed } else if (NumOps == 3) { 4909263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4910263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4911263508Sdim Ops[1], Ops[2]); 4912193323Sed } else { 4913263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4914263508Sdim VTList, Ops, NumOps); 4915193323Sed } 4916193323Sed CSEMap.InsertNode(N, IP); 4917193323Sed } else { 4918193323Sed if (NumOps == 1) { 4919263508Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4920263508Sdim DL.getDebugLoc(), VTList, Ops[0]); 4921193323Sed } else if (NumOps == 2) { 4922263508Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4923263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4924263508Sdim Ops[1]); 4925193323Sed } else if (NumOps == 3) { 4926263508Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4927263508Sdim DL.getDebugLoc(), VTList, Ops[0], 4928263508Sdim Ops[1], Ops[2]); 4929193323Sed } else { 4930263508Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4931263508Sdim VTList, Ops, NumOps); 4932193323Sed } 4933193323Sed } 4934193323Sed AllNodes.push_back(N); 4935193323Sed#ifndef NDEBUG 4936218893Sdim VerifySDNode(N); 4937193323Sed#endif 4938193323Sed return SDValue(N, 0); 4939193323Sed} 4940193323Sed 4941263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) { 4942193323Sed return getNode(Opcode, DL, VTList, 0, 0); 4943193323Sed} 4944193323Sed 4945263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4946193323Sed SDValue N1) { 4947193323Sed SDValue Ops[] = { N1 }; 4948193323Sed return getNode(Opcode, DL, VTList, Ops, 1); 4949193323Sed} 4950193323Sed 4951263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4952193323Sed SDValue N1, SDValue N2) { 4953193323Sed SDValue Ops[] = { N1, N2 }; 4954193323Sed return getNode(Opcode, DL, VTList, Ops, 2); 4955193323Sed} 4956193323Sed 4957263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4958193323Sed SDValue N1, SDValue N2, SDValue N3) { 4959193323Sed SDValue Ops[] = { N1, N2, N3 }; 4960193323Sed return getNode(Opcode, DL, VTList, Ops, 3); 4961193323Sed} 4962193323Sed 4963263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4964193323Sed SDValue N1, SDValue N2, SDValue N3, 4965193323Sed SDValue N4) { 4966193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 4967193323Sed return getNode(Opcode, DL, VTList, Ops, 4); 4968193323Sed} 4969193323Sed 4970263508SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4971193323Sed SDValue N1, SDValue N2, SDValue N3, 4972193323Sed SDValue N4, SDValue N5) { 4973193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4974193323Sed return getNode(Opcode, DL, VTList, Ops, 5); 4975193323Sed} 4976193323Sed 4977198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT) { 4978193323Sed return makeVTList(SDNode::getValueTypeList(VT), 1); 4979193323Sed} 4980193323Sed 4981198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4982263508Sdim FoldingSetNodeID ID; 4983263508Sdim ID.AddInteger(2U); 4984263508Sdim ID.AddInteger(VT1.getRawBits()); 4985263508Sdim ID.AddInteger(VT2.getRawBits()); 4986193323Sed 4987263508Sdim void *IP = 0; 4988263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4989263508Sdim if (Result == NULL) { 4990263508Sdim EVT *Array = Allocator.Allocate<EVT>(2); 4991263508Sdim Array[0] = VT1; 4992263508Sdim Array[1] = VT2; 4993263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 4994263508Sdim VTListMap.InsertNode(Result, IP); 4995263508Sdim } 4996263508Sdim return Result->getSDVTList(); 4997193323Sed} 4998193323Sed 4999198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 5000263508Sdim FoldingSetNodeID ID; 5001263508Sdim ID.AddInteger(3U); 5002263508Sdim ID.AddInteger(VT1.getRawBits()); 5003263508Sdim ID.AddInteger(VT2.getRawBits()); 5004263508Sdim ID.AddInteger(VT3.getRawBits()); 5005193323Sed 5006263508Sdim void *IP = 0; 5007263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5008263508Sdim if (Result == NULL) { 5009263508Sdim EVT *Array = Allocator.Allocate<EVT>(3); 5010263508Sdim Array[0] = VT1; 5011263508Sdim Array[1] = VT2; 5012263508Sdim Array[2] = VT3; 5013263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 5014263508Sdim VTListMap.InsertNode(Result, IP); 5015263508Sdim } 5016263508Sdim return Result->getSDVTList(); 5017193323Sed} 5018193323Sed 5019198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 5020263508Sdim FoldingSetNodeID ID; 5021263508Sdim ID.AddInteger(4U); 5022263508Sdim ID.AddInteger(VT1.getRawBits()); 5023263508Sdim ID.AddInteger(VT2.getRawBits()); 5024263508Sdim ID.AddInteger(VT3.getRawBits()); 5025263508Sdim ID.AddInteger(VT4.getRawBits()); 5026193323Sed 5027263508Sdim void *IP = 0; 5028263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5029263508Sdim if (Result == NULL) { 5030263508Sdim EVT *Array = Allocator.Allocate<EVT>(4); 5031263508Sdim Array[0] = VT1; 5032263508Sdim Array[1] = VT2; 5033263508Sdim Array[2] = VT3; 5034263508Sdim Array[3] = VT4; 5035263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 5036263508Sdim VTListMap.InsertNode(Result, IP); 5037263508Sdim } 5038263508Sdim return Result->getSDVTList(); 5039193323Sed} 5040193323Sed 5041198090SrdivackySDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 5042263508Sdim FoldingSetNodeID ID; 5043263508Sdim ID.AddInteger(NumVTs); 5044263508Sdim for (unsigned index = 0; index < NumVTs; index++) { 5045263508Sdim ID.AddInteger(VTs[index].getRawBits()); 5046193323Sed } 5047193323Sed 5048263508Sdim void *IP = 0; 5049263508Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5050263508Sdim if (Result == NULL) { 5051263508Sdim EVT *Array = Allocator.Allocate<EVT>(NumVTs); 5052263508Sdim std::copy(VTs, VTs + NumVTs, Array); 5053263508Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 5054263508Sdim VTListMap.InsertNode(Result, IP); 5055193323Sed } 5056263508Sdim return Result->getSDVTList(); 5057193323Sed} 5058193323Sed 5059193323Sed 5060193323Sed/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 5061193323Sed/// specified operands. If the resultant node already exists in the DAG, 5062193323Sed/// this does not modify the specified node, instead it returns the node that 5063193323Sed/// already exists. If the resultant node does not exist in the DAG, the 5064193323Sed/// input node is returned. As a degenerate case, if you specify the same 5065193323Sed/// input operands as the node already has, the input node is returned. 5066210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 5067193323Sed assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 5068193323Sed 5069193323Sed // Check to see if there is no change. 5070210299Sed if (Op == N->getOperand(0)) return N; 5071193323Sed 5072193323Sed // See if the modified node already exists. 5073193323Sed void *InsertPos = 0; 5074193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 5075210299Sed return Existing; 5076193323Sed 5077193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5078193323Sed if (InsertPos) 5079193323Sed if (!RemoveNodeFromCSEMaps(N)) 5080193323Sed InsertPos = 0; 5081193323Sed 5082193323Sed // Now we update the operands. 5083193323Sed N->OperandList[0].set(Op); 5084193323Sed 5085193323Sed // If this gets put into a CSE map, add it. 5086193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5087210299Sed return N; 5088193323Sed} 5089193323Sed 5090210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 5091193323Sed assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 5092193323Sed 5093193323Sed // Check to see if there is no change. 5094193323Sed if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 5095210299Sed return N; // No operands changed, just return the input node. 5096193323Sed 5097193323Sed // See if the modified node already exists. 5098193323Sed void *InsertPos = 0; 5099193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 5100210299Sed return Existing; 5101193323Sed 5102193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5103193323Sed if (InsertPos) 5104193323Sed if (!RemoveNodeFromCSEMaps(N)) 5105193323Sed InsertPos = 0; 5106193323Sed 5107193323Sed // Now we update the operands. 5108193323Sed if (N->OperandList[0] != Op1) 5109193323Sed N->OperandList[0].set(Op1); 5110193323Sed if (N->OperandList[1] != Op2) 5111193323Sed N->OperandList[1].set(Op2); 5112193323Sed 5113193323Sed // If this gets put into a CSE map, add it. 5114193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5115210299Sed return N; 5116193323Sed} 5117193323Sed 5118210299SedSDNode *SelectionDAG:: 5119210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 5120193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5121193323Sed return UpdateNodeOperands(N, Ops, 3); 5122193323Sed} 5123193323Sed 5124210299SedSDNode *SelectionDAG:: 5125210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5126193323Sed SDValue Op3, SDValue Op4) { 5127193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 5128193323Sed return UpdateNodeOperands(N, Ops, 4); 5129193323Sed} 5130193323Sed 5131210299SedSDNode *SelectionDAG:: 5132210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5133193323Sed SDValue Op3, SDValue Op4, SDValue Op5) { 5134193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 5135193323Sed return UpdateNodeOperands(N, Ops, 5); 5136193323Sed} 5137193323Sed 5138210299SedSDNode *SelectionDAG:: 5139210299SedUpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 5140193323Sed assert(N->getNumOperands() == NumOps && 5141193323Sed "Update with wrong number of operands"); 5142193323Sed 5143193323Sed // Check to see if there is no change. 5144193323Sed bool AnyChange = false; 5145193323Sed for (unsigned i = 0; i != NumOps; ++i) { 5146193323Sed if (Ops[i] != N->getOperand(i)) { 5147193323Sed AnyChange = true; 5148193323Sed break; 5149193323Sed } 5150193323Sed } 5151193323Sed 5152193323Sed // No operands changed, just return the input node. 5153210299Sed if (!AnyChange) return N; 5154193323Sed 5155193323Sed // See if the modified node already exists. 5156193323Sed void *InsertPos = 0; 5157193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 5158210299Sed return Existing; 5159193323Sed 5160193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5161193323Sed if (InsertPos) 5162193323Sed if (!RemoveNodeFromCSEMaps(N)) 5163193323Sed InsertPos = 0; 5164193323Sed 5165193323Sed // Now we update the operands. 5166193323Sed for (unsigned i = 0; i != NumOps; ++i) 5167193323Sed if (N->OperandList[i] != Ops[i]) 5168193323Sed N->OperandList[i].set(Ops[i]); 5169193323Sed 5170193323Sed // If this gets put into a CSE map, add it. 5171193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5172210299Sed return N; 5173193323Sed} 5174193323Sed 5175193323Sed/// DropOperands - Release the operands and set this node to have 5176193323Sed/// zero operands. 5177193323Sedvoid SDNode::DropOperands() { 5178193323Sed // Unlike the code in MorphNodeTo that does this, we don't need to 5179193323Sed // watch for dead nodes here. 5180193323Sed for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 5181193323Sed SDUse &Use = *I++; 5182193323Sed Use.set(SDValue()); 5183193323Sed } 5184193323Sed} 5185193323Sed 5186193323Sed/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 5187193323Sed/// machine opcode. 5188193323Sed/// 5189193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5190198090Srdivacky EVT VT) { 5191193323Sed SDVTList VTs = getVTList(VT); 5192193323Sed return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 5193193323Sed} 5194193323Sed 5195193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5196198090Srdivacky EVT VT, SDValue Op1) { 5197193323Sed SDVTList VTs = getVTList(VT); 5198193323Sed SDValue Ops[] = { Op1 }; 5199193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5200193323Sed} 5201193323Sed 5202193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5203198090Srdivacky EVT VT, SDValue Op1, 5204193323Sed SDValue Op2) { 5205193323Sed SDVTList VTs = getVTList(VT); 5206193323Sed SDValue Ops[] = { Op1, Op2 }; 5207193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5208193323Sed} 5209193323Sed 5210193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5211198090Srdivacky EVT VT, SDValue Op1, 5212193323Sed SDValue Op2, SDValue Op3) { 5213193323Sed SDVTList VTs = getVTList(VT); 5214193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5215193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5216193323Sed} 5217193323Sed 5218193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5219198090Srdivacky EVT VT, const SDValue *Ops, 5220193323Sed unsigned NumOps) { 5221193323Sed SDVTList VTs = getVTList(VT); 5222193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5223193323Sed} 5224193323Sed 5225193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5226198090Srdivacky EVT VT1, EVT VT2, const SDValue *Ops, 5227193323Sed unsigned NumOps) { 5228193323Sed SDVTList VTs = getVTList(VT1, VT2); 5229193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5230193323Sed} 5231193323Sed 5232193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5233198090Srdivacky EVT VT1, EVT VT2) { 5234193323Sed SDVTList VTs = getVTList(VT1, VT2); 5235193323Sed return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 5236193323Sed} 5237193323Sed 5238193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5239198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5240193323Sed const SDValue *Ops, unsigned NumOps) { 5241193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5242193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5243193323Sed} 5244193323Sed 5245193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5246198090Srdivacky EVT VT1, EVT VT2, EVT VT3, EVT VT4, 5247193323Sed const SDValue *Ops, unsigned NumOps) { 5248193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5249193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5250193323Sed} 5251193323Sed 5252193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5253198090Srdivacky EVT VT1, EVT VT2, 5254193323Sed SDValue Op1) { 5255193323Sed SDVTList VTs = getVTList(VT1, VT2); 5256193323Sed SDValue Ops[] = { Op1 }; 5257193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5258193323Sed} 5259193323Sed 5260193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5261198090Srdivacky EVT VT1, EVT VT2, 5262193323Sed SDValue Op1, SDValue Op2) { 5263193323Sed SDVTList VTs = getVTList(VT1, VT2); 5264193323Sed SDValue Ops[] = { Op1, Op2 }; 5265193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5266193323Sed} 5267193323Sed 5268193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5269198090Srdivacky EVT VT1, EVT VT2, 5270193323Sed SDValue Op1, SDValue Op2, 5271193323Sed SDValue Op3) { 5272193323Sed SDVTList VTs = getVTList(VT1, VT2); 5273193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5274193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5275193323Sed} 5276193323Sed 5277193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5278198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5279193323Sed SDValue Op1, SDValue Op2, 5280193323Sed SDValue Op3) { 5281193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5282193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5283193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5284193323Sed} 5285193323Sed 5286193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5287193323Sed SDVTList VTs, const SDValue *Ops, 5288193323Sed unsigned NumOps) { 5289204642Srdivacky N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 5290204642Srdivacky // Reset the NodeID to -1. 5291204642Srdivacky N->setNodeId(-1); 5292204642Srdivacky return N; 5293193323Sed} 5294193323Sed 5295263508Sdim/// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away 5296234353Sdim/// the line number information on the merged node since it is not possible to 5297234353Sdim/// preserve the information that operation is associated with multiple lines. 5298234353Sdim/// This will make the debugger working better at -O0, were there is a higher 5299234353Sdim/// probability having other instructions associated with that line. 5300234353Sdim/// 5301263508Sdim/// For IROrder, we keep the smaller of the two 5302263508SdimSDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { 5303234353Sdim DebugLoc NLoc = N->getDebugLoc(); 5304263508Sdim if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && 5305263508Sdim (OLoc.getDebugLoc() != NLoc)) { 5306234353Sdim N->setDebugLoc(DebugLoc()); 5307234353Sdim } 5308263508Sdim unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 5309263508Sdim N->setIROrder(Order); 5310234353Sdim return N; 5311234353Sdim} 5312234353Sdim 5313204642Srdivacky/// MorphNodeTo - This *mutates* the specified node to have the specified 5314193323Sed/// return type, opcode, and operands. 5315193323Sed/// 5316193323Sed/// Note that MorphNodeTo returns the resultant node. If there is already a 5317193323Sed/// node of the specified opcode and operands, it returns that node instead of 5318263508Sdim/// the current one. Note that the SDLoc need not be the same. 5319193323Sed/// 5320193323Sed/// Using MorphNodeTo is faster than creating a new node and swapping it in 5321193323Sed/// with ReplaceAllUsesWith both because it often avoids allocating a new 5322193323Sed/// node, and because it doesn't require CSE recalculation for any of 5323193323Sed/// the node's users. 5324193323Sed/// 5325193323SedSDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 5326193323Sed SDVTList VTs, const SDValue *Ops, 5327193323Sed unsigned NumOps) { 5328193323Sed // If an identical node already exists, use it. 5329193323Sed void *IP = 0; 5330218893Sdim if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 5331193323Sed FoldingSetNodeID ID; 5332193323Sed AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 5333201360Srdivacky if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 5334263508Sdim return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N)); 5335193323Sed } 5336193323Sed 5337193323Sed if (!RemoveNodeFromCSEMaps(N)) 5338193323Sed IP = 0; 5339193323Sed 5340193323Sed // Start the morphing. 5341193323Sed N->NodeType = Opc; 5342193323Sed N->ValueList = VTs.VTs; 5343193323Sed N->NumValues = VTs.NumVTs; 5344193323Sed 5345193323Sed // Clear the operands list, updating used nodes to remove this from their 5346193323Sed // use list. Keep track of any operands that become dead as a result. 5347193323Sed SmallPtrSet<SDNode*, 16> DeadNodeSet; 5348193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 5349193323Sed SDUse &Use = *I++; 5350193323Sed SDNode *Used = Use.getNode(); 5351193323Sed Use.set(SDValue()); 5352193323Sed if (Used->use_empty()) 5353193323Sed DeadNodeSet.insert(Used); 5354193323Sed } 5355193323Sed 5356198090Srdivacky if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 5357198090Srdivacky // Initialize the memory references information. 5358198090Srdivacky MN->setMemRefs(0, 0); 5359198090Srdivacky // If NumOps is larger than the # of operands we can have in a 5360198090Srdivacky // MachineSDNode, reallocate the operand list. 5361198090Srdivacky if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 5362198090Srdivacky if (MN->OperandsNeedDelete) 5363198090Srdivacky delete[] MN->OperandList; 5364198090Srdivacky if (NumOps > array_lengthof(MN->LocalOperands)) 5365198090Srdivacky // We're creating a final node that will live unmorphed for the 5366198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5367198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5368198090Srdivacky MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5369205407Srdivacky Ops, NumOps); 5370198090Srdivacky else 5371198090Srdivacky MN->InitOperands(MN->LocalOperands, Ops, NumOps); 5372198090Srdivacky MN->OperandsNeedDelete = false; 5373198090Srdivacky } else 5374198090Srdivacky MN->InitOperands(MN->OperandList, Ops, NumOps); 5375198090Srdivacky } else { 5376198090Srdivacky // If NumOps is larger than the # of operands we currently have, reallocate 5377198090Srdivacky // the operand list. 5378198090Srdivacky if (NumOps > N->NumOperands) { 5379198090Srdivacky if (N->OperandsNeedDelete) 5380198090Srdivacky delete[] N->OperandList; 5381198090Srdivacky N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5382193323Sed N->OperandsNeedDelete = true; 5383198090Srdivacky } else 5384198396Srdivacky N->InitOperands(N->OperandList, Ops, NumOps); 5385193323Sed } 5386193323Sed 5387193323Sed // Delete any nodes that are still dead after adding the uses for the 5388193323Sed // new operands. 5389204642Srdivacky if (!DeadNodeSet.empty()) { 5390204642Srdivacky SmallVector<SDNode *, 16> DeadNodes; 5391204642Srdivacky for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5392204642Srdivacky E = DeadNodeSet.end(); I != E; ++I) 5393204642Srdivacky if ((*I)->use_empty()) 5394204642Srdivacky DeadNodes.push_back(*I); 5395204642Srdivacky RemoveDeadNodes(DeadNodes); 5396204642Srdivacky } 5397193323Sed 5398193323Sed if (IP) 5399193323Sed CSEMap.InsertNode(N, IP); // Memoize the new node. 5400193323Sed return N; 5401193323Sed} 5402193323Sed 5403193323Sed 5404198090Srdivacky/// getMachineNode - These are used for target selectors to create a new node 5405198090Srdivacky/// with specified return type(s), MachineInstr opcode, and operands. 5406193323Sed/// 5407198090Srdivacky/// Note that getMachineNode returns the resultant node. If there is already a 5408193323Sed/// node of the specified opcode and operands, it returns that node instead of 5409193323Sed/// the current one. 5410198090SrdivackyMachineSDNode * 5411263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) { 5412198090Srdivacky SDVTList VTs = getVTList(VT); 5413251662Sdim return getMachineNode(Opcode, dl, VTs, None); 5414193323Sed} 5415193323Sed 5416198090SrdivackyMachineSDNode * 5417263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) { 5418198090Srdivacky SDVTList VTs = getVTList(VT); 5419198090Srdivacky SDValue Ops[] = { Op1 }; 5420251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5421193323Sed} 5422193323Sed 5423198090SrdivackyMachineSDNode * 5424263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5425198090Srdivacky SDValue Op1, SDValue Op2) { 5426198090Srdivacky SDVTList VTs = getVTList(VT); 5427198090Srdivacky SDValue Ops[] = { Op1, Op2 }; 5428251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5429193323Sed} 5430193323Sed 5431198090SrdivackyMachineSDNode * 5432263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5433198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5434198090Srdivacky SDVTList VTs = getVTList(VT); 5435198090Srdivacky SDValue Ops[] = { Op1, Op2, Op3 }; 5436251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5437193323Sed} 5438193323Sed 5439198090SrdivackyMachineSDNode * 5440263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5441251662Sdim ArrayRef<SDValue> Ops) { 5442198090Srdivacky SDVTList VTs = getVTList(VT); 5443251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5444193323Sed} 5445193323Sed 5446198090SrdivackyMachineSDNode * 5447263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) { 5448193323Sed SDVTList VTs = getVTList(VT1, VT2); 5449251662Sdim return getMachineNode(Opcode, dl, VTs, None); 5450193323Sed} 5451193323Sed 5452198090SrdivackyMachineSDNode * 5453263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5454198090Srdivacky EVT VT1, EVT VT2, SDValue Op1) { 5455193323Sed SDVTList VTs = getVTList(VT1, VT2); 5456198090Srdivacky SDValue Ops[] = { Op1 }; 5457251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5458193323Sed} 5459193323Sed 5460198090SrdivackyMachineSDNode * 5461263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5462198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5463193323Sed SDVTList VTs = getVTList(VT1, VT2); 5464193323Sed SDValue Ops[] = { Op1, Op2 }; 5465251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5466193323Sed} 5467193323Sed 5468198090SrdivackyMachineSDNode * 5469263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5470198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, 5471198090Srdivacky SDValue Op2, SDValue Op3) { 5472193323Sed SDVTList VTs = getVTList(VT1, VT2); 5473193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5474251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5475193323Sed} 5476193323Sed 5477198090SrdivackyMachineSDNode * 5478263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5479198090Srdivacky EVT VT1, EVT VT2, 5480251662Sdim ArrayRef<SDValue> Ops) { 5481193323Sed SDVTList VTs = getVTList(VT1, VT2); 5482251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5483193323Sed} 5484193323Sed 5485198090SrdivackyMachineSDNode * 5486263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5487198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5488198090Srdivacky SDValue Op1, SDValue Op2) { 5489193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5490193323Sed SDValue Ops[] = { Op1, Op2 }; 5491251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5492193323Sed} 5493193323Sed 5494198090SrdivackyMachineSDNode * 5495263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5496198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5497198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5498193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5499193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5500251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5501193323Sed} 5502193323Sed 5503198090SrdivackyMachineSDNode * 5504263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5505198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5506251662Sdim ArrayRef<SDValue> Ops) { 5507193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5508251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5509193323Sed} 5510193323Sed 5511198090SrdivackyMachineSDNode * 5512263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, 5513198090Srdivacky EVT VT2, EVT VT3, EVT VT4, 5514251662Sdim ArrayRef<SDValue> Ops) { 5515193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5516251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5517193323Sed} 5518193323Sed 5519198090SrdivackyMachineSDNode * 5520263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5521249423Sdim ArrayRef<EVT> ResultTys, 5522251662Sdim ArrayRef<SDValue> Ops) { 5523198090Srdivacky SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5524251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5525193323Sed} 5526193323Sed 5527198090SrdivackyMachineSDNode * 5528263508SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, 5529251662Sdim ArrayRef<SDValue> OpsArray) { 5530218893Sdim bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5531198090Srdivacky MachineSDNode *N; 5532218893Sdim void *IP = 0; 5533251662Sdim const SDValue *Ops = OpsArray.data(); 5534251662Sdim unsigned NumOps = OpsArray.size(); 5535198090Srdivacky 5536198090Srdivacky if (DoCSE) { 5537198090Srdivacky FoldingSetNodeID ID; 5538198090Srdivacky AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5539198090Srdivacky IP = 0; 5540234353Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5541263508Sdim return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); 5542234353Sdim } 5543198090Srdivacky } 5544198090Srdivacky 5545198090Srdivacky // Allocate a new MachineSDNode. 5546263508Sdim N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(), 5547263508Sdim DL.getDebugLoc(), VTs); 5548198090Srdivacky 5549198090Srdivacky // Initialize the operands list. 5550198090Srdivacky if (NumOps > array_lengthof(N->LocalOperands)) 5551198090Srdivacky // We're creating a final node that will live unmorphed for the 5552198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5553198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5554198090Srdivacky N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5555198090Srdivacky Ops, NumOps); 5556198090Srdivacky else 5557198090Srdivacky N->InitOperands(N->LocalOperands, Ops, NumOps); 5558198090Srdivacky N->OperandsNeedDelete = false; 5559198090Srdivacky 5560198090Srdivacky if (DoCSE) 5561198090Srdivacky CSEMap.InsertNode(N, IP); 5562198090Srdivacky 5563198090Srdivacky AllNodes.push_back(N); 5564198090Srdivacky#ifndef NDEBUG 5565218893Sdim VerifyMachineNode(N); 5566198090Srdivacky#endif 5567198090Srdivacky return N; 5568198090Srdivacky} 5569198090Srdivacky 5570198090Srdivacky/// getTargetExtractSubreg - A convenience function for creating 5571203954Srdivacky/// TargetOpcode::EXTRACT_SUBREG nodes. 5572198090SrdivackySDValue 5573263508SdimSelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, 5574198090Srdivacky SDValue Operand) { 5575198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5576203954Srdivacky SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5577198090Srdivacky VT, Operand, SRIdxVal); 5578198090Srdivacky return SDValue(Subreg, 0); 5579198090Srdivacky} 5580198090Srdivacky 5581198090Srdivacky/// getTargetInsertSubreg - A convenience function for creating 5582203954Srdivacky/// TargetOpcode::INSERT_SUBREG nodes. 5583198090SrdivackySDValue 5584263508SdimSelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, 5585198090Srdivacky SDValue Operand, SDValue Subreg) { 5586198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5587203954Srdivacky SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5588198090Srdivacky VT, Operand, Subreg, SRIdxVal); 5589198090Srdivacky return SDValue(Result, 0); 5590198090Srdivacky} 5591198090Srdivacky 5592193323Sed/// getNodeIfExists - Get the specified node if it's already available, or 5593193323Sed/// else return NULL. 5594193323SedSDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5595193323Sed const SDValue *Ops, unsigned NumOps) { 5596218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5597193323Sed FoldingSetNodeID ID; 5598193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5599193323Sed void *IP = 0; 5600201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5601193323Sed return E; 5602193323Sed } 5603193323Sed return NULL; 5604193323Sed} 5605193323Sed 5606206083Srdivacky/// getDbgValue - Creates a SDDbgValue node. 5607206083Srdivacky/// 5608206083SrdivackySDDbgValue * 5609206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5610206083Srdivacky DebugLoc DL, unsigned O) { 5611206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5612206083Srdivacky} 5613206083Srdivacky 5614206083SrdivackySDDbgValue * 5615207618SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5616206083Srdivacky DebugLoc DL, unsigned O) { 5617206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5618206083Srdivacky} 5619206083Srdivacky 5620206083SrdivackySDDbgValue * 5621206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5622206083Srdivacky DebugLoc DL, unsigned O) { 5623206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5624206083Srdivacky} 5625206083Srdivacky 5626204792Srdivackynamespace { 5627204792Srdivacky 5628204792Srdivacky/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5629204792Srdivacky/// pointed to by a use iterator is deleted, increment the use iterator 5630204792Srdivacky/// so that it doesn't dangle. 5631204792Srdivacky/// 5632204792Srdivackyclass RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5633204792Srdivacky SDNode::use_iterator &UI; 5634204792Srdivacky SDNode::use_iterator &UE; 5635204792Srdivacky 5636204792Srdivacky virtual void NodeDeleted(SDNode *N, SDNode *E) { 5637204792Srdivacky // Increment the iterator as needed. 5638204792Srdivacky while (UI != UE && N == *UI) 5639204792Srdivacky ++UI; 5640204792Srdivacky } 5641204792Srdivacky 5642204792Srdivackypublic: 5643239462Sdim RAUWUpdateListener(SelectionDAG &d, 5644204792Srdivacky SDNode::use_iterator &ui, 5645204792Srdivacky SDNode::use_iterator &ue) 5646239462Sdim : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 5647204792Srdivacky}; 5648204792Srdivacky 5649204792Srdivacky} 5650204792Srdivacky 5651193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5652193323Sed/// This can cause recursive merging of nodes in the DAG. 5653193323Sed/// 5654193323Sed/// This version assumes From has a single result value. 5655193323Sed/// 5656239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 5657193323Sed SDNode *From = FromN.getNode(); 5658193323Sed assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5659193323Sed "Cannot replace with this method!"); 5660193323Sed assert(From != To.getNode() && "Cannot replace uses of with self"); 5661193323Sed 5662193323Sed // Iterate over all the existing uses of From. New uses will be added 5663193323Sed // to the beginning of the use list, which we avoid visiting. 5664193323Sed // This specifically avoids visiting uses of From that arise while the 5665193323Sed // replacement is happening, because any such uses would be the result 5666193323Sed // of CSE: If an existing node looks like From after one of its operands 5667193323Sed // is replaced by To, we don't want to replace of all its users with To 5668193323Sed // too. See PR3018 for more info. 5669193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5670239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5671193323Sed while (UI != UE) { 5672193323Sed SDNode *User = *UI; 5673193323Sed 5674193323Sed // This node is about to morph, remove its old self from the CSE maps. 5675193323Sed RemoveNodeFromCSEMaps(User); 5676193323Sed 5677193323Sed // A user can appear in a use list multiple times, and when this 5678193323Sed // happens the uses are usually next to each other in the list. 5679193323Sed // To help reduce the number of CSE recomputations, process all 5680193323Sed // the uses of this user that we can find this way. 5681193323Sed do { 5682193323Sed SDUse &Use = UI.getUse(); 5683193323Sed ++UI; 5684193323Sed Use.set(To); 5685193323Sed } while (UI != UE && *UI == User); 5686193323Sed 5687193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5688193323Sed // already exists there, recursively merge the results together. 5689239462Sdim AddModifiedNodeToCSEMaps(User); 5690193323Sed } 5691234353Sdim 5692234353Sdim // If we just RAUW'd the root, take note. 5693234353Sdim if (FromN == getRoot()) 5694234353Sdim setRoot(To); 5695193323Sed} 5696193323Sed 5697193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5698193323Sed/// This can cause recursive merging of nodes in the DAG. 5699193323Sed/// 5700193323Sed/// This version assumes that for each value of From, there is a 5701193323Sed/// corresponding value in To in the same position with the same type. 5702193323Sed/// 5703239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 5704193323Sed#ifndef NDEBUG 5705193323Sed for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5706193323Sed assert((!From->hasAnyUseOfValue(i) || 5707193323Sed From->getValueType(i) == To->getValueType(i)) && 5708193323Sed "Cannot use this version of ReplaceAllUsesWith!"); 5709193323Sed#endif 5710193323Sed 5711193323Sed // Handle the trivial case. 5712193323Sed if (From == To) 5713193323Sed return; 5714193323Sed 5715193323Sed // Iterate over just the existing users of From. See the comments in 5716193323Sed // the ReplaceAllUsesWith above. 5717193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5718239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5719193323Sed while (UI != UE) { 5720193323Sed SDNode *User = *UI; 5721193323Sed 5722193323Sed // This node is about to morph, remove its old self from the CSE maps. 5723193323Sed RemoveNodeFromCSEMaps(User); 5724193323Sed 5725193323Sed // A user can appear in a use list multiple times, and when this 5726193323Sed // happens the uses are usually next to each other in the list. 5727193323Sed // To help reduce the number of CSE recomputations, process all 5728193323Sed // the uses of this user that we can find this way. 5729193323Sed do { 5730193323Sed SDUse &Use = UI.getUse(); 5731193323Sed ++UI; 5732193323Sed Use.setNode(To); 5733193323Sed } while (UI != UE && *UI == User); 5734193323Sed 5735193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5736193323Sed // already exists there, recursively merge the results together. 5737239462Sdim AddModifiedNodeToCSEMaps(User); 5738193323Sed } 5739234353Sdim 5740234353Sdim // If we just RAUW'd the root, take note. 5741234353Sdim if (From == getRoot().getNode()) 5742234353Sdim setRoot(SDValue(To, getRoot().getResNo())); 5743193323Sed} 5744193323Sed 5745193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5746193323Sed/// This can cause recursive merging of nodes in the DAG. 5747193323Sed/// 5748193323Sed/// This version can replace From with any result values. To must match the 5749193323Sed/// number and types of values returned by From. 5750239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 5751193323Sed if (From->getNumValues() == 1) // Handle the simple case efficiently. 5752239462Sdim return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 5753193323Sed 5754193323Sed // Iterate over just the existing users of From. See the comments in 5755193323Sed // the ReplaceAllUsesWith above. 5756193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5757239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5758193323Sed while (UI != UE) { 5759193323Sed SDNode *User = *UI; 5760193323Sed 5761193323Sed // This node is about to morph, remove its old self from the CSE maps. 5762193323Sed RemoveNodeFromCSEMaps(User); 5763193323Sed 5764193323Sed // A user can appear in a use list multiple times, and when this 5765193323Sed // happens the uses are usually next to each other in the list. 5766193323Sed // To help reduce the number of CSE recomputations, process all 5767193323Sed // the uses of this user that we can find this way. 5768193323Sed do { 5769193323Sed SDUse &Use = UI.getUse(); 5770193323Sed const SDValue &ToOp = To[Use.getResNo()]; 5771193323Sed ++UI; 5772193323Sed Use.set(ToOp); 5773193323Sed } while (UI != UE && *UI == User); 5774193323Sed 5775193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5776193323Sed // already exists there, recursively merge the results together. 5777239462Sdim AddModifiedNodeToCSEMaps(User); 5778193323Sed } 5779234353Sdim 5780234353Sdim // If we just RAUW'd the root, take note. 5781234353Sdim if (From == getRoot().getNode()) 5782234353Sdim setRoot(SDValue(To[getRoot().getResNo()])); 5783193323Sed} 5784193323Sed 5785193323Sed/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5786193323Sed/// uses of other values produced by From.getNode() alone. The Deleted 5787193323Sed/// vector is handled the same way as for ReplaceAllUsesWith. 5788239462Sdimvoid SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 5789193323Sed // Handle the really simple, really trivial case efficiently. 5790193323Sed if (From == To) return; 5791193323Sed 5792193323Sed // Handle the simple, trivial, case efficiently. 5793193323Sed if (From.getNode()->getNumValues() == 1) { 5794239462Sdim ReplaceAllUsesWith(From, To); 5795193323Sed return; 5796193323Sed } 5797193323Sed 5798193323Sed // Iterate over just the existing users of From. See the comments in 5799193323Sed // the ReplaceAllUsesWith above. 5800193323Sed SDNode::use_iterator UI = From.getNode()->use_begin(), 5801193323Sed UE = From.getNode()->use_end(); 5802239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5803193323Sed while (UI != UE) { 5804193323Sed SDNode *User = *UI; 5805193323Sed bool UserRemovedFromCSEMaps = false; 5806193323Sed 5807193323Sed // A user can appear in a use list multiple times, and when this 5808193323Sed // happens the uses are usually next to each other in the list. 5809193323Sed // To help reduce the number of CSE recomputations, process all 5810193323Sed // the uses of this user that we can find this way. 5811193323Sed do { 5812193323Sed SDUse &Use = UI.getUse(); 5813193323Sed 5814193323Sed // Skip uses of different values from the same node. 5815193323Sed if (Use.getResNo() != From.getResNo()) { 5816193323Sed ++UI; 5817193323Sed continue; 5818193323Sed } 5819193323Sed 5820193323Sed // If this node hasn't been modified yet, it's still in the CSE maps, 5821193323Sed // so remove its old self from the CSE maps. 5822193323Sed if (!UserRemovedFromCSEMaps) { 5823193323Sed RemoveNodeFromCSEMaps(User); 5824193323Sed UserRemovedFromCSEMaps = true; 5825193323Sed } 5826193323Sed 5827193323Sed ++UI; 5828193323Sed Use.set(To); 5829193323Sed } while (UI != UE && *UI == User); 5830193323Sed 5831193323Sed // We are iterating over all uses of the From node, so if a use 5832193323Sed // doesn't use the specific value, no changes are made. 5833193323Sed if (!UserRemovedFromCSEMaps) 5834193323Sed continue; 5835193323Sed 5836193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5837193323Sed // already exists there, recursively merge the results together. 5838239462Sdim AddModifiedNodeToCSEMaps(User); 5839193323Sed } 5840234353Sdim 5841234353Sdim // If we just RAUW'd the root, take note. 5842234353Sdim if (From == getRoot()) 5843234353Sdim setRoot(To); 5844193323Sed} 5845193323Sed 5846193323Sednamespace { 5847193323Sed /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5848193323Sed /// to record information about a use. 5849193323Sed struct UseMemo { 5850193323Sed SDNode *User; 5851193323Sed unsigned Index; 5852193323Sed SDUse *Use; 5853193323Sed }; 5854193323Sed 5855193323Sed /// operator< - Sort Memos by User. 5856193323Sed bool operator<(const UseMemo &L, const UseMemo &R) { 5857193323Sed return (intptr_t)L.User < (intptr_t)R.User; 5858193323Sed } 5859193323Sed} 5860193323Sed 5861193323Sed/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5862193323Sed/// uses of other values produced by From.getNode() alone. The same value 5863193323Sed/// may appear in both the From and To list. The Deleted vector is 5864193323Sed/// handled the same way as for ReplaceAllUsesWith. 5865193323Sedvoid SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5866193323Sed const SDValue *To, 5867239462Sdim unsigned Num){ 5868193323Sed // Handle the simple, trivial case efficiently. 5869193323Sed if (Num == 1) 5870239462Sdim return ReplaceAllUsesOfValueWith(*From, *To); 5871193323Sed 5872193323Sed // Read up all the uses and make records of them. This helps 5873193323Sed // processing new uses that are introduced during the 5874193323Sed // replacement process. 5875193323Sed SmallVector<UseMemo, 4> Uses; 5876193323Sed for (unsigned i = 0; i != Num; ++i) { 5877193323Sed unsigned FromResNo = From[i].getResNo(); 5878193323Sed SDNode *FromNode = From[i].getNode(); 5879193323Sed for (SDNode::use_iterator UI = FromNode->use_begin(), 5880193323Sed E = FromNode->use_end(); UI != E; ++UI) { 5881193323Sed SDUse &Use = UI.getUse(); 5882193323Sed if (Use.getResNo() == FromResNo) { 5883193323Sed UseMemo Memo = { *UI, i, &Use }; 5884193323Sed Uses.push_back(Memo); 5885193323Sed } 5886193323Sed } 5887193323Sed } 5888193323Sed 5889193323Sed // Sort the uses, so that all the uses from a given User are together. 5890193323Sed std::sort(Uses.begin(), Uses.end()); 5891193323Sed 5892193323Sed for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5893193323Sed UseIndex != UseIndexEnd; ) { 5894193323Sed // We know that this user uses some value of From. If it is the right 5895193323Sed // value, update it. 5896193323Sed SDNode *User = Uses[UseIndex].User; 5897193323Sed 5898193323Sed // This node is about to morph, remove its old self from the CSE maps. 5899193323Sed RemoveNodeFromCSEMaps(User); 5900193323Sed 5901193323Sed // The Uses array is sorted, so all the uses for a given User 5902193323Sed // are next to each other in the list. 5903193323Sed // To help reduce the number of CSE recomputations, process all 5904193323Sed // the uses of this user that we can find this way. 5905193323Sed do { 5906193323Sed unsigned i = Uses[UseIndex].Index; 5907193323Sed SDUse &Use = *Uses[UseIndex].Use; 5908193323Sed ++UseIndex; 5909193323Sed 5910193323Sed Use.set(To[i]); 5911193323Sed } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5912193323Sed 5913193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5914193323Sed // already exists there, recursively merge the results together. 5915239462Sdim AddModifiedNodeToCSEMaps(User); 5916193323Sed } 5917193323Sed} 5918193323Sed 5919193323Sed/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5920193323Sed/// based on their topological order. It returns the maximum id and a vector 5921193323Sed/// of the SDNodes* in assigned order by reference. 5922193323Sedunsigned SelectionDAG::AssignTopologicalOrder() { 5923193323Sed 5924193323Sed unsigned DAGSize = 0; 5925193323Sed 5926193323Sed // SortedPos tracks the progress of the algorithm. Nodes before it are 5927193323Sed // sorted, nodes after it are unsorted. When the algorithm completes 5928193323Sed // it is at the end of the list. 5929193323Sed allnodes_iterator SortedPos = allnodes_begin(); 5930193323Sed 5931193323Sed // Visit all the nodes. Move nodes with no operands to the front of 5932193323Sed // the list immediately. Annotate nodes that do have operands with their 5933193323Sed // operand count. Before we do this, the Node Id fields of the nodes 5934193323Sed // may contain arbitrary values. After, the Node Id fields for nodes 5935193323Sed // before SortedPos will contain the topological sort index, and the 5936193323Sed // Node Id fields for nodes At SortedPos and after will contain the 5937193323Sed // count of outstanding operands. 5938193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5939193323Sed SDNode *N = I++; 5940202878Srdivacky checkForCycles(N); 5941193323Sed unsigned Degree = N->getNumOperands(); 5942193323Sed if (Degree == 0) { 5943193323Sed // A node with no uses, add it to the result array immediately. 5944193323Sed N->setNodeId(DAGSize++); 5945193323Sed allnodes_iterator Q = N; 5946193323Sed if (Q != SortedPos) 5947193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5948202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5949193323Sed ++SortedPos; 5950193323Sed } else { 5951193323Sed // Temporarily use the Node Id as scratch space for the degree count. 5952193323Sed N->setNodeId(Degree); 5953193323Sed } 5954193323Sed } 5955193323Sed 5956239462Sdim // Visit all the nodes. As we iterate, move nodes into sorted order, 5957193323Sed // such that by the time the end is reached all nodes will be sorted. 5958193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5959193323Sed SDNode *N = I; 5960202878Srdivacky checkForCycles(N); 5961202878Srdivacky // N is in sorted position, so all its uses have one less operand 5962202878Srdivacky // that needs to be sorted. 5963193323Sed for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5964193323Sed UI != UE; ++UI) { 5965193323Sed SDNode *P = *UI; 5966193323Sed unsigned Degree = P->getNodeId(); 5967202878Srdivacky assert(Degree != 0 && "Invalid node degree"); 5968193323Sed --Degree; 5969193323Sed if (Degree == 0) { 5970193323Sed // All of P's operands are sorted, so P may sorted now. 5971193323Sed P->setNodeId(DAGSize++); 5972193323Sed if (P != SortedPos) 5973193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5974202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5975193323Sed ++SortedPos; 5976193323Sed } else { 5977193323Sed // Update P's outstanding operand count. 5978193323Sed P->setNodeId(Degree); 5979193323Sed } 5980193323Sed } 5981202878Srdivacky if (I == SortedPos) { 5982203954Srdivacky#ifndef NDEBUG 5983203954Srdivacky SDNode *S = ++I; 5984203954Srdivacky dbgs() << "Overran sorted position:\n"; 5985202878Srdivacky S->dumprFull(); 5986203954Srdivacky#endif 5987203954Srdivacky llvm_unreachable(0); 5988202878Srdivacky } 5989193323Sed } 5990193323Sed 5991193323Sed assert(SortedPos == AllNodes.end() && 5992193323Sed "Topological sort incomplete!"); 5993193323Sed assert(AllNodes.front().getOpcode() == ISD::EntryToken && 5994193323Sed "First node in topological sort is not the entry token!"); 5995193323Sed assert(AllNodes.front().getNodeId() == 0 && 5996193323Sed "First node in topological sort has non-zero id!"); 5997193323Sed assert(AllNodes.front().getNumOperands() == 0 && 5998193323Sed "First node in topological sort has operands!"); 5999193323Sed assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 6000193323Sed "Last node in topologic sort has unexpected id!"); 6001193323Sed assert(AllNodes.back().use_empty() && 6002193323Sed "Last node in topologic sort has users!"); 6003193323Sed assert(DAGSize == allnodes_size() && "Node count mismatch!"); 6004193323Sed return DAGSize; 6005193323Sed} 6006193323Sed 6007206083Srdivacky/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 6008206083Srdivacky/// value is produced by SD. 6009207618Srdivackyvoid SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 6010207618Srdivacky DbgInfo->add(DB, SD, isParameter); 6011206083Srdivacky if (SD) 6012206083Srdivacky SD->setHasDebugValue(true); 6013205218Srdivacky} 6014201360Srdivacky 6015218893Sdim/// TransferDbgValues - Transfer SDDbgValues. 6016218893Sdimvoid SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 6017218893Sdim if (From == To || !From.getNode()->getHasDebugValue()) 6018218893Sdim return; 6019218893Sdim SDNode *FromNode = From.getNode(); 6020218893Sdim SDNode *ToNode = To.getNode(); 6021224145Sdim ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 6022218893Sdim SmallVector<SDDbgValue *, 2> ClonedDVs; 6023224145Sdim for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 6024218893Sdim I != E; ++I) { 6025218893Sdim SDDbgValue *Dbg = *I; 6026218893Sdim if (Dbg->getKind() == SDDbgValue::SDNODE) { 6027218893Sdim SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 6028218893Sdim Dbg->getOffset(), Dbg->getDebugLoc(), 6029218893Sdim Dbg->getOrder()); 6030218893Sdim ClonedDVs.push_back(Clone); 6031218893Sdim } 6032218893Sdim } 6033263508Sdim for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(), 6034218893Sdim E = ClonedDVs.end(); I != E; ++I) 6035218893Sdim AddDbgValue(*I, ToNode, false); 6036218893Sdim} 6037218893Sdim 6038193323Sed//===----------------------------------------------------------------------===// 6039193323Sed// SDNode Class 6040193323Sed//===----------------------------------------------------------------------===// 6041193323Sed 6042193323SedHandleSDNode::~HandleSDNode() { 6043193323Sed DropOperands(); 6044193323Sed} 6045193323Sed 6046263508SdimGlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 6047263508Sdim DebugLoc DL, const GlobalValue *GA, 6048198090Srdivacky EVT VT, int64_t o, unsigned char TF) 6049263508Sdim : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 6050207618Srdivacky TheGlobal = GA; 6051193323Sed} 6052193323Sed 6053263508SdimAddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, 6054263508Sdim SDValue X, unsigned SrcAS, 6055263508Sdim unsigned DestAS) 6056263508Sdim : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), 6057263508Sdim SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 6058263508Sdim 6059263508SdimMemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6060263508Sdim EVT memvt, MachineMemOperand *mmo) 6061263508Sdim : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 6062204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6063234353Sdim MMO->isNonTemporal(), MMO->isInvariant()); 6064198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6065204642Srdivacky assert(isNonTemporal() == MMO->isNonTemporal() && 6066204642Srdivacky "Non-temporal encoding error!"); 6067198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6068193323Sed} 6069193323Sed 6070263508SdimMemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6071218893Sdim const SDValue *Ops, unsigned NumOps, EVT memvt, 6072198090Srdivacky MachineMemOperand *mmo) 6073263508Sdim : SDNode(Opc, Order, dl, VTs, Ops, NumOps), 6074198090Srdivacky MemoryVT(memvt), MMO(mmo) { 6075204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6076234353Sdim MMO->isNonTemporal(), MMO->isInvariant()); 6077198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6078198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6079193323Sed} 6080193323Sed 6081193323Sed/// Profile - Gather unique data for the node. 6082193323Sed/// 6083193323Sedvoid SDNode::Profile(FoldingSetNodeID &ID) const { 6084193323Sed AddNodeIDNode(ID, this); 6085193323Sed} 6086193323Sed 6087198090Srdivackynamespace { 6088198090Srdivacky struct EVTArray { 6089198090Srdivacky std::vector<EVT> VTs; 6090218893Sdim 6091198090Srdivacky EVTArray() { 6092198090Srdivacky VTs.reserve(MVT::LAST_VALUETYPE); 6093198090Srdivacky for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 6094198090Srdivacky VTs.push_back(MVT((MVT::SimpleValueType)i)); 6095198090Srdivacky } 6096198090Srdivacky }; 6097198090Srdivacky} 6098198090Srdivacky 6099198090Srdivackystatic ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 6100198090Srdivackystatic ManagedStatic<EVTArray> SimpleVTArray; 6101195098Sedstatic ManagedStatic<sys::SmartMutex<true> > VTMutex; 6102195098Sed 6103193323Sed/// getValueTypeList - Return a pointer to the specified value type. 6104193323Sed/// 6105198090Srdivackyconst EVT *SDNode::getValueTypeList(EVT VT) { 6106193323Sed if (VT.isExtended()) { 6107198090Srdivacky sys::SmartScopedLock<true> Lock(*VTMutex); 6108195098Sed return &(*EVTs->insert(VT).first); 6109193323Sed } else { 6110218893Sdim assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 6111208599Srdivacky "Value type out of range!"); 6112198090Srdivacky return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 6113193323Sed } 6114193323Sed} 6115193323Sed 6116193323Sed/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 6117193323Sed/// indicated value. This method ignores uses of other values defined by this 6118193323Sed/// operation. 6119193323Sedbool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 6120193323Sed assert(Value < getNumValues() && "Bad value!"); 6121193323Sed 6122193323Sed // TODO: Only iterate over uses of a given value of the node 6123193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 6124193323Sed if (UI.getUse().getResNo() == Value) { 6125193323Sed if (NUses == 0) 6126193323Sed return false; 6127193323Sed --NUses; 6128193323Sed } 6129193323Sed } 6130193323Sed 6131193323Sed // Found exactly the right number of uses? 6132193323Sed return NUses == 0; 6133193323Sed} 6134193323Sed 6135193323Sed 6136193323Sed/// hasAnyUseOfValue - Return true if there are any use of the indicated 6137193323Sed/// value. This method ignores uses of other values defined by this operation. 6138193323Sedbool SDNode::hasAnyUseOfValue(unsigned Value) const { 6139193323Sed assert(Value < getNumValues() && "Bad value!"); 6140193323Sed 6141193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 6142193323Sed if (UI.getUse().getResNo() == Value) 6143193323Sed return true; 6144193323Sed 6145193323Sed return false; 6146193323Sed} 6147193323Sed 6148193323Sed 6149193323Sed/// isOnlyUserOf - Return true if this node is the only use of N. 6150193323Sed/// 6151193323Sedbool SDNode::isOnlyUserOf(SDNode *N) const { 6152193323Sed bool Seen = false; 6153193323Sed for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 6154193323Sed SDNode *User = *I; 6155193323Sed if (User == this) 6156193323Sed Seen = true; 6157193323Sed else 6158193323Sed return false; 6159193323Sed } 6160193323Sed 6161193323Sed return Seen; 6162193323Sed} 6163193323Sed 6164193323Sed/// isOperand - Return true if this node is an operand of N. 6165193323Sed/// 6166193323Sedbool SDValue::isOperandOf(SDNode *N) const { 6167193323Sed for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6168193323Sed if (*this == N->getOperand(i)) 6169193323Sed return true; 6170193323Sed return false; 6171193323Sed} 6172193323Sed 6173193323Sedbool SDNode::isOperandOf(SDNode *N) const { 6174193323Sed for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 6175193323Sed if (this == N->OperandList[i].getNode()) 6176193323Sed return true; 6177193323Sed return false; 6178193323Sed} 6179193323Sed 6180193323Sed/// reachesChainWithoutSideEffects - Return true if this operand (which must 6181193323Sed/// be a chain) reaches the specified operand without crossing any 6182218893Sdim/// side-effecting instructions on any chain path. In practice, this looks 6183218893Sdim/// through token factors and non-volatile loads. In order to remain efficient, 6184218893Sdim/// this only looks a couple of nodes in, it does not do an exhaustive search. 6185193323Sedbool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 6186193323Sed unsigned Depth) const { 6187193323Sed if (*this == Dest) return true; 6188193323Sed 6189193323Sed // Don't search too deeply, we just want to be able to see through 6190193323Sed // TokenFactor's etc. 6191193323Sed if (Depth == 0) return false; 6192193323Sed 6193193323Sed // If this is a token factor, all inputs to the TF happen in parallel. If any 6194218893Sdim // of the operands of the TF does not reach dest, then we cannot do the xform. 6195193323Sed if (getOpcode() == ISD::TokenFactor) { 6196193323Sed for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6197218893Sdim if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 6198218893Sdim return false; 6199218893Sdim return true; 6200193323Sed } 6201193323Sed 6202193323Sed // Loads don't have side effects, look through them. 6203193323Sed if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 6204193323Sed if (!Ld->isVolatile()) 6205193323Sed return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 6206193323Sed } 6207193323Sed return false; 6208193323Sed} 6209193323Sed 6210224145Sdim/// hasPredecessor - Return true if N is a predecessor of this node. 6211224145Sdim/// N is either an operand of this node, or can be reached by recursively 6212224145Sdim/// traversing up the operands. 6213224145Sdim/// NOTE: This is an expensive method. Use it carefully. 6214224145Sdimbool SDNode::hasPredecessor(const SDNode *N) const { 6215224145Sdim SmallPtrSet<const SDNode *, 32> Visited; 6216224145Sdim SmallVector<const SDNode *, 16> Worklist; 6217224145Sdim return hasPredecessorHelper(N, Visited, Worklist); 6218224145Sdim} 6219198892Srdivacky 6220263508Sdimbool 6221263508SdimSDNode::hasPredecessorHelper(const SDNode *N, 6222263508Sdim SmallPtrSet<const SDNode *, 32> &Visited, 6223263508Sdim SmallVectorImpl<const SDNode *> &Worklist) const { 6224224145Sdim if (Visited.empty()) { 6225224145Sdim Worklist.push_back(this); 6226224145Sdim } else { 6227224145Sdim // Take a look in the visited set. If we've already encountered this node 6228224145Sdim // we needn't search further. 6229224145Sdim if (Visited.count(N)) 6230224145Sdim return true; 6231224145Sdim } 6232224145Sdim 6233224145Sdim // Haven't visited N yet. Continue the search. 6234224145Sdim while (!Worklist.empty()) { 6235224145Sdim const SDNode *M = Worklist.pop_back_val(); 6236224145Sdim for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 6237224145Sdim SDNode *Op = M->getOperand(i).getNode(); 6238198892Srdivacky if (Visited.insert(Op)) 6239198892Srdivacky Worklist.push_back(Op); 6240224145Sdim if (Op == N) 6241224145Sdim return true; 6242198892Srdivacky } 6243224145Sdim } 6244198892Srdivacky 6245198892Srdivacky return false; 6246193323Sed} 6247193323Sed 6248193323Seduint64_t SDNode::getConstantOperandVal(unsigned Num) const { 6249193323Sed assert(Num < NumOperands && "Invalid child # of SDNode!"); 6250193323Sed return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 6251193323Sed} 6252193323Sed 6253199989SrdivackySDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6254199989Srdivacky assert(N->getNumValues() == 1 && 6255199989Srdivacky "Can't unroll a vector with multiple results!"); 6256199989Srdivacky 6257199989Srdivacky EVT VT = N->getValueType(0); 6258199989Srdivacky unsigned NE = VT.getVectorNumElements(); 6259199989Srdivacky EVT EltVT = VT.getVectorElementType(); 6260263508Sdim SDLoc dl(N); 6261199989Srdivacky 6262199989Srdivacky SmallVector<SDValue, 8> Scalars; 6263199989Srdivacky SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6264199989Srdivacky 6265199989Srdivacky // If ResNE is 0, fully unroll the vector op. 6266199989Srdivacky if (ResNE == 0) 6267199989Srdivacky ResNE = NE; 6268199989Srdivacky else if (NE > ResNE) 6269199989Srdivacky NE = ResNE; 6270199989Srdivacky 6271199989Srdivacky unsigned i; 6272199989Srdivacky for (i= 0; i != NE; ++i) { 6273207618Srdivacky for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6274199989Srdivacky SDValue Operand = N->getOperand(j); 6275199989Srdivacky EVT OperandVT = Operand.getValueType(); 6276199989Srdivacky if (OperandVT.isVector()) { 6277199989Srdivacky // A vector operand; extract a single element. 6278263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6279199989Srdivacky EVT OperandEltVT = OperandVT.getVectorElementType(); 6280199989Srdivacky Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6281199989Srdivacky OperandEltVT, 6282199989Srdivacky Operand, 6283263508Sdim getConstant(i, TLI->getVectorIdxTy())); 6284199989Srdivacky } else { 6285199989Srdivacky // A scalar operand; just use it as is. 6286199989Srdivacky Operands[j] = Operand; 6287199989Srdivacky } 6288199989Srdivacky } 6289199989Srdivacky 6290199989Srdivacky switch (N->getOpcode()) { 6291199989Srdivacky default: 6292199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6293199989Srdivacky &Operands[0], Operands.size())); 6294199989Srdivacky break; 6295226633Sdim case ISD::VSELECT: 6296226633Sdim Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6297226633Sdim &Operands[0], Operands.size())); 6298226633Sdim break; 6299199989Srdivacky case ISD::SHL: 6300199989Srdivacky case ISD::SRA: 6301199989Srdivacky case ISD::SRL: 6302199989Srdivacky case ISD::ROTL: 6303199989Srdivacky case ISD::ROTR: 6304199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6305263508Sdim getShiftAmountOperand(Operands[0].getValueType(), 6306263508Sdim Operands[1]))); 6307199989Srdivacky break; 6308202375Srdivacky case ISD::SIGN_EXTEND_INREG: 6309202375Srdivacky case ISD::FP_ROUND_INREG: { 6310202375Srdivacky EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6311202375Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6312202375Srdivacky Operands[0], 6313202375Srdivacky getValueType(ExtVT))); 6314199989Srdivacky } 6315202375Srdivacky } 6316199989Srdivacky } 6317199989Srdivacky 6318199989Srdivacky for (; i < ResNE; ++i) 6319199989Srdivacky Scalars.push_back(getUNDEF(EltVT)); 6320199989Srdivacky 6321199989Srdivacky return getNode(ISD::BUILD_VECTOR, dl, 6322199989Srdivacky EVT::getVectorVT(*getContext(), EltVT, ResNE), 6323199989Srdivacky &Scalars[0], Scalars.size()); 6324199989Srdivacky} 6325199989Srdivacky 6326200581Srdivacky 6327218893Sdim/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6328218893Sdim/// location that is 'Dist' units away from the location that the 'Base' load 6329200581Srdivacky/// is loading from. 6330218893Sdimbool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6331200581Srdivacky unsigned Bytes, int Dist) const { 6332200581Srdivacky if (LD->getChain() != Base->getChain()) 6333200581Srdivacky return false; 6334200581Srdivacky EVT VT = LD->getValueType(0); 6335200581Srdivacky if (VT.getSizeInBits() / 8 != Bytes) 6336200581Srdivacky return false; 6337200581Srdivacky 6338200581Srdivacky SDValue Loc = LD->getOperand(1); 6339200581Srdivacky SDValue BaseLoc = Base->getOperand(1); 6340200581Srdivacky if (Loc.getOpcode() == ISD::FrameIndex) { 6341200581Srdivacky if (BaseLoc.getOpcode() != ISD::FrameIndex) 6342200581Srdivacky return false; 6343200581Srdivacky const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6344200581Srdivacky int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6345200581Srdivacky int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6346200581Srdivacky int FS = MFI->getObjectSize(FI); 6347200581Srdivacky int BFS = MFI->getObjectSize(BFI); 6348200581Srdivacky if (FS != BFS || FS != (int)Bytes) return false; 6349200581Srdivacky return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6350200581Srdivacky } 6351200581Srdivacky 6352218893Sdim // Handle X+C 6353218893Sdim if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6354218893Sdim cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6355218893Sdim return true; 6356218893Sdim 6357207618Srdivacky const GlobalValue *GV1 = NULL; 6358207618Srdivacky const GlobalValue *GV2 = NULL; 6359200581Srdivacky int64_t Offset1 = 0; 6360200581Srdivacky int64_t Offset2 = 0; 6361263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6362263508Sdim bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6363263508Sdim bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6364200581Srdivacky if (isGA1 && isGA2 && GV1 == GV2) 6365200581Srdivacky return Offset1 == (Offset2 + Dist*Bytes); 6366200581Srdivacky return false; 6367200581Srdivacky} 6368200581Srdivacky 6369200581Srdivacky 6370200581Srdivacky/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6371200581Srdivacky/// it cannot be inferred. 6372200581Srdivackyunsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6373200581Srdivacky // If this is a GlobalAddress + cst, return the alignment. 6374207618Srdivacky const GlobalValue *GV; 6375200581Srdivacky int64_t GVOffset = 0; 6376263508Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6377263508Sdim if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6378263508Sdim unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 6379234353Sdim APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6380234353Sdim llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne, 6381263508Sdim TLI->getDataLayout()); 6382234353Sdim unsigned AlignBits = KnownZero.countTrailingOnes(); 6383234353Sdim unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6384234353Sdim if (Align) 6385234353Sdim return MinAlign(Align, GVOffset); 6386206083Srdivacky } 6387200581Srdivacky 6388200581Srdivacky // If this is a direct reference to a stack slot, use information about the 6389200581Srdivacky // stack slot's alignment. 6390200581Srdivacky int FrameIdx = 1 << 31; 6391200581Srdivacky int64_t FrameOffset = 0; 6392200581Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6393200581Srdivacky FrameIdx = FI->getIndex(); 6394218893Sdim } else if (isBaseWithConstantOffset(Ptr) && 6395200581Srdivacky isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6396218893Sdim // Handle FI+Cst 6397200581Srdivacky FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6398200581Srdivacky FrameOffset = Ptr.getConstantOperandVal(1); 6399200581Srdivacky } 6400200581Srdivacky 6401200581Srdivacky if (FrameIdx != (1 << 31)) { 6402200581Srdivacky const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6403200581Srdivacky unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6404200581Srdivacky FrameOffset); 6405200581Srdivacky return FIInfoAlign; 6406200581Srdivacky } 6407200581Srdivacky 6408200581Srdivacky return 0; 6409200581Srdivacky} 6410200581Srdivacky 6411263508Sdim/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 6412263508Sdim/// which is split (or expanded) into two not necessarily identical pieces. 6413263508Sdimstd::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { 6414263508Sdim // Currently all types are split in half. 6415263508Sdim EVT LoVT, HiVT; 6416263508Sdim if (!VT.isVector()) { 6417263508Sdim LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); 6418263508Sdim } else { 6419263508Sdim unsigned NumElements = VT.getVectorNumElements(); 6420263508Sdim assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 6421263508Sdim LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), 6422263508Sdim NumElements/2); 6423263508Sdim } 6424263508Sdim return std::make_pair(LoVT, HiVT); 6425263508Sdim} 6426263508Sdim 6427263508Sdim/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 6428263508Sdim/// low/high part. 6429263508Sdimstd::pair<SDValue, SDValue> 6430263508SdimSelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, 6431263508Sdim const EVT &HiVT) { 6432263508Sdim assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <= 6433263508Sdim N.getValueType().getVectorNumElements() && 6434263508Sdim "More vector elements requested than available!"); 6435263508Sdim SDValue Lo, Hi; 6436263508Sdim Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, 6437263508Sdim getConstant(0, TLI->getVectorIdxTy())); 6438263508Sdim Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, 6439263508Sdim getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy())); 6440263508Sdim return std::make_pair(Lo, Hi); 6441263508Sdim} 6442263508Sdim 6443193323Sed// getAddressSpace - Return the address space this GlobalAddress belongs to. 6444193323Sedunsigned GlobalAddressSDNode::getAddressSpace() const { 6445193323Sed return getGlobal()->getType()->getAddressSpace(); 6446193323Sed} 6447193323Sed 6448193323Sed 6449226633SdimType *ConstantPoolSDNode::getType() const { 6450193323Sed if (isMachineConstantPoolEntry()) 6451193323Sed return Val.MachineCPVal->getType(); 6452193323Sed return Val.ConstVal->getType(); 6453193323Sed} 6454193323Sed 6455193323Sedbool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6456193323Sed APInt &SplatUndef, 6457193323Sed unsigned &SplatBitSize, 6458193323Sed bool &HasAnyUndefs, 6459199481Srdivacky unsigned MinSplatBits, 6460199481Srdivacky bool isBigEndian) { 6461198090Srdivacky EVT VT = getValueType(0); 6462193323Sed assert(VT.isVector() && "Expected a vector type"); 6463193323Sed unsigned sz = VT.getSizeInBits(); 6464193323Sed if (MinSplatBits > sz) 6465193323Sed return false; 6466193323Sed 6467193323Sed SplatValue = APInt(sz, 0); 6468193323Sed SplatUndef = APInt(sz, 0); 6469193323Sed 6470193323Sed // Get the bits. Bits with undefined values (when the corresponding element 6471193323Sed // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6472193323Sed // in SplatValue. If any of the values are not constant, give up and return 6473193323Sed // false. 6474193323Sed unsigned int nOps = getNumOperands(); 6475193323Sed assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6476193323Sed unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6477199481Srdivacky 6478199481Srdivacky for (unsigned j = 0; j < nOps; ++j) { 6479199481Srdivacky unsigned i = isBigEndian ? nOps-1-j : j; 6480193323Sed SDValue OpVal = getOperand(i); 6481199481Srdivacky unsigned BitPos = j * EltBitSize; 6482193323Sed 6483193323Sed if (OpVal.getOpcode() == ISD::UNDEF) 6484199481Srdivacky SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6485193323Sed else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6486218893Sdim SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6487207618Srdivacky zextOrTrunc(sz) << BitPos; 6488193323Sed else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6489193323Sed SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6490193323Sed else 6491193323Sed return false; 6492193323Sed } 6493193323Sed 6494193323Sed // The build_vector is all constants or undefs. Find the smallest element 6495193323Sed // size that splats the vector. 6496193323Sed 6497193323Sed HasAnyUndefs = (SplatUndef != 0); 6498193323Sed while (sz > 8) { 6499193323Sed 6500193323Sed unsigned HalfSize = sz / 2; 6501218893Sdim APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6502218893Sdim APInt LowValue = SplatValue.trunc(HalfSize); 6503218893Sdim APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6504218893Sdim APInt LowUndef = SplatUndef.trunc(HalfSize); 6505193323Sed 6506193323Sed // If the two halves do not match (ignoring undef bits), stop here. 6507193323Sed if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6508193323Sed MinSplatBits > HalfSize) 6509193323Sed break; 6510193323Sed 6511193323Sed SplatValue = HighValue | LowValue; 6512193323Sed SplatUndef = HighUndef & LowUndef; 6513198090Srdivacky 6514193323Sed sz = HalfSize; 6515193323Sed } 6516193323Sed 6517193323Sed SplatBitSize = sz; 6518193323Sed return true; 6519193323Sed} 6520193323Sed 6521198090Srdivackybool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6522193323Sed // Find the first non-undef value in the shuffle mask. 6523193323Sed unsigned i, e; 6524193323Sed for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6525193323Sed /* search */; 6526193323Sed 6527193323Sed assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6528198090Srdivacky 6529193323Sed // Make sure all remaining elements are either undef or the same as the first 6530193323Sed // non-undef value. 6531193323Sed for (int Idx = Mask[i]; i != e; ++i) 6532193323Sed if (Mask[i] >= 0 && Mask[i] != Idx) 6533193323Sed return false; 6534193323Sed return true; 6535193323Sed} 6536202878Srdivacky 6537204642Srdivacky#ifdef XDEBUG 6538202878Srdivackystatic void checkForCyclesHelper(const SDNode *N, 6539204642Srdivacky SmallPtrSet<const SDNode*, 32> &Visited, 6540204642Srdivacky SmallPtrSet<const SDNode*, 32> &Checked) { 6541204642Srdivacky // If this node has already been checked, don't check it again. 6542204642Srdivacky if (Checked.count(N)) 6543204642Srdivacky return; 6544218893Sdim 6545204642Srdivacky // If a node has already been visited on this depth-first walk, reject it as 6546204642Srdivacky // a cycle. 6547204642Srdivacky if (!Visited.insert(N)) { 6548202878Srdivacky dbgs() << "Offending node:\n"; 6549202878Srdivacky N->dumprFull(); 6550204642Srdivacky errs() << "Detected cycle in SelectionDAG\n"; 6551204642Srdivacky abort(); 6552202878Srdivacky } 6553218893Sdim 6554204642Srdivacky for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6555204642Srdivacky checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6556218893Sdim 6557204642Srdivacky Checked.insert(N); 6558204642Srdivacky Visited.erase(N); 6559202878Srdivacky} 6560204642Srdivacky#endif 6561202878Srdivacky 6562202878Srdivackyvoid llvm::checkForCycles(const llvm::SDNode *N) { 6563202878Srdivacky#ifdef XDEBUG 6564263508Sdim assert(N && "Checking nonexistent SDNode"); 6565204642Srdivacky SmallPtrSet<const SDNode*, 32> visited; 6566204642Srdivacky SmallPtrSet<const SDNode*, 32> checked; 6567204642Srdivacky checkForCyclesHelper(N, visited, checked); 6568202878Srdivacky#endif 6569202878Srdivacky} 6570202878Srdivacky 6571202878Srdivackyvoid llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6572202878Srdivacky checkForCycles(DAG->getRoot().getNode()); 6573202878Srdivacky} 6574