1193323Sed//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This implements the SelectionDAG class. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13201360Srdivacky 14193323Sed#include "llvm/CodeGen/SelectionDAG.h" 15249423Sdim#include "SDNodeDbgValue.h" 16201360Srdivacky#include "SDNodeOrdering.h" 17249423Sdim#include "llvm/ADT/SetVector.h" 18249423Sdim#include "llvm/ADT/SmallPtrSet.h" 19249423Sdim#include "llvm/ADT/SmallSet.h" 20249423Sdim#include "llvm/ADT/SmallVector.h" 21249423Sdim#include "llvm/ADT/StringExtras.h" 22249423Sdim#include "llvm/Analysis/TargetTransformInfo.h" 23239462Sdim#include "llvm/Analysis/ValueTracking.h" 24193323Sed#include "llvm/Assembly/Writer.h" 25193323Sed#include "llvm/CodeGen/MachineBasicBlock.h" 26193323Sed#include "llvm/CodeGen/MachineConstantPool.h" 27193323Sed#include "llvm/CodeGen/MachineFrameInfo.h" 28193323Sed#include "llvm/CodeGen/MachineModuleInfo.h" 29249423Sdim#include "llvm/DebugInfo.h" 30249423Sdim#include "llvm/IR/CallingConv.h" 31249423Sdim#include "llvm/IR/Constants.h" 32249423Sdim#include "llvm/IR/DataLayout.h" 33249423Sdim#include "llvm/IR/DerivedTypes.h" 34249423Sdim#include "llvm/IR/Function.h" 35249423Sdim#include "llvm/IR/GlobalAlias.h" 36249423Sdim#include "llvm/IR/GlobalVariable.h" 37249423Sdim#include "llvm/IR/Intrinsics.h" 38193323Sed#include "llvm/Support/CommandLine.h" 39202375Srdivacky#include "llvm/Support/Debug.h" 40198090Srdivacky#include "llvm/Support/ErrorHandling.h" 41195098Sed#include "llvm/Support/ManagedStatic.h" 42193323Sed#include "llvm/Support/MathExtras.h" 43249423Sdim#include "llvm/Support/Mutex.h" 44193323Sed#include "llvm/Support/raw_ostream.h" 45249423Sdim#include "llvm/Target/TargetInstrInfo.h" 46249423Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 47249423Sdim#include "llvm/Target/TargetLowering.h" 48249423Sdim#include "llvm/Target/TargetMachine.h" 49249423Sdim#include "llvm/Target/TargetOptions.h" 50249423Sdim#include "llvm/Target/TargetRegisterInfo.h" 51249423Sdim#include "llvm/Target/TargetSelectionDAGInfo.h" 52193323Sed#include <algorithm> 53193323Sed#include <cmath> 54193323Sedusing namespace llvm; 55193323Sed 56193323Sed/// makeVTList - Return an instance of the SDVTList struct initialized with the 57193323Sed/// specified members. 58198090Srdivackystatic SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 59193323Sed SDVTList Res = {VTs, NumVTs}; 60193323Sed return Res; 61193323Sed} 62193323Sed 63239462Sdim// Default null implementations of the callbacks. 64239462Sdimvoid SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 65239462Sdimvoid SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 66193323Sed 67193323Sed//===----------------------------------------------------------------------===// 68193323Sed// ConstantFPSDNode Class 69193323Sed//===----------------------------------------------------------------------===// 70193323Sed 71193323Sed/// isExactlyValue - We don't rely on operator== working on double values, as 72193323Sed/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 73193323Sed/// As such, this method can be used to do an exact bit-for-bit comparison of 74193323Sed/// two floating point values. 75193323Sedbool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 76193323Sed return getValueAPF().bitwiseIsEqual(V); 77193323Sed} 78193323Sed 79198090Srdivackybool ConstantFPSDNode::isValueValidForType(EVT VT, 80193323Sed const APFloat& Val) { 81193323Sed assert(VT.isFloatingPoint() && "Can only convert between FP types"); 82193323Sed 83193323Sed // convert modifies in place, so make a copy. 84193323Sed APFloat Val2 = APFloat(Val); 85193323Sed bool losesInfo; 86249423Sdim (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 87249423Sdim APFloat::rmNearestTiesToEven, 88193323Sed &losesInfo); 89193323Sed return !losesInfo; 90193323Sed} 91193323Sed 92193323Sed//===----------------------------------------------------------------------===// 93193323Sed// ISD Namespace 94193323Sed//===----------------------------------------------------------------------===// 95193323Sed 96193323Sed/// isBuildVectorAllOnes - Return true if the specified node is a 97193323Sed/// BUILD_VECTOR where all of the elements are ~0 or undef. 98193323Sedbool ISD::isBuildVectorAllOnes(const SDNode *N) { 99193323Sed // Look through a bit convert. 100218893Sdim if (N->getOpcode() == ISD::BITCAST) 101193323Sed N = N->getOperand(0).getNode(); 102193323Sed 103193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 104193323Sed 105193323Sed unsigned i = 0, e = N->getNumOperands(); 106193323Sed 107193323Sed // Skip over all of the undef values. 108193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 109193323Sed ++i; 110193323Sed 111193323Sed // Do not accept an all-undef vector. 112193323Sed if (i == e) return false; 113193323Sed 114193323Sed // Do not accept build_vectors that aren't all constants or which have non-~0 115234353Sdim // elements. We have to be a bit careful here, as the type of the constant 116234353Sdim // may not be the same as the type of the vector elements due to type 117234353Sdim // legalization (the elements are promoted to a legal type for the target and 118234353Sdim // a vector of a type may be legal when the base element type is not). 119234353Sdim // We only want to check enough bits to cover the vector elements, because 120234353Sdim // we care if the resultant vector is all ones, not whether the individual 121234353Sdim // constants are. 122193323Sed SDValue NotZero = N->getOperand(i); 123234353Sdim unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 124243830Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 125243830Sdim if (CN->getAPIntValue().countTrailingOnes() < EltSize) 126193323Sed return false; 127243830Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 128243830Sdim if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 129193323Sed return false; 130193323Sed } else 131193323Sed return false; 132193323Sed 133193323Sed // Okay, we have at least one ~0 value, check to see if the rest match or are 134234353Sdim // undefs. Even with the above element type twiddling, this should be OK, as 135234353Sdim // the same type legalization should have applied to all the elements. 136193323Sed for (++i; i != e; ++i) 137193323Sed if (N->getOperand(i) != NotZero && 138193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 139193323Sed return false; 140193323Sed return true; 141193323Sed} 142193323Sed 143193323Sed 144193323Sed/// isBuildVectorAllZeros - Return true if the specified node is a 145193323Sed/// BUILD_VECTOR where all of the elements are 0 or undef. 146193323Sedbool ISD::isBuildVectorAllZeros(const SDNode *N) { 147193323Sed // Look through a bit convert. 148218893Sdim if (N->getOpcode() == ISD::BITCAST) 149193323Sed N = N->getOperand(0).getNode(); 150193323Sed 151193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 152193323Sed 153193323Sed unsigned i = 0, e = N->getNumOperands(); 154193323Sed 155193323Sed // Skip over all of the undef values. 156193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 157193323Sed ++i; 158193323Sed 159193323Sed // Do not accept an all-undef vector. 160193323Sed if (i == e) return false; 161193323Sed 162193574Sed // Do not accept build_vectors that aren't all constants or which have non-0 163193323Sed // elements. 164193323Sed SDValue Zero = N->getOperand(i); 165243830Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) { 166243830Sdim if (!CN->isNullValue()) 167193323Sed return false; 168243830Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) { 169243830Sdim if (!CFPN->getValueAPF().isPosZero()) 170193323Sed return false; 171193323Sed } else 172193323Sed return false; 173193323Sed 174193574Sed // Okay, we have at least one 0 value, check to see if the rest match or are 175193323Sed // undefs. 176193323Sed for (++i; i != e; ++i) 177193323Sed if (N->getOperand(i) != Zero && 178193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 179193323Sed return false; 180193323Sed return true; 181193323Sed} 182193323Sed 183193323Sed/// isScalarToVector - Return true if the specified node is a 184193323Sed/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 185193323Sed/// element is not an undef. 186193323Sedbool ISD::isScalarToVector(const SDNode *N) { 187193323Sed if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 188193323Sed return true; 189193323Sed 190193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) 191193323Sed return false; 192193323Sed if (N->getOperand(0).getOpcode() == ISD::UNDEF) 193193323Sed return false; 194193323Sed unsigned NumElems = N->getNumOperands(); 195218893Sdim if (NumElems == 1) 196218893Sdim return false; 197193323Sed for (unsigned i = 1; i < NumElems; ++i) { 198193323Sed SDValue V = N->getOperand(i); 199193323Sed if (V.getOpcode() != ISD::UNDEF) 200193323Sed return false; 201193323Sed } 202193323Sed return true; 203193323Sed} 204193323Sed 205239462Sdim/// allOperandsUndef - Return true if the node has at least one operand 206239462Sdim/// and all operands of the specified node are ISD::UNDEF. 207239462Sdimbool ISD::allOperandsUndef(const SDNode *N) { 208239462Sdim // Return false if the node has no operands. 209239462Sdim // This is "logically inconsistent" with the definition of "all" but 210239462Sdim // is probably the desired behavior. 211239462Sdim if (N->getNumOperands() == 0) 212239462Sdim return false; 213239462Sdim 214239462Sdim for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i) 215239462Sdim if (N->getOperand(i).getOpcode() != ISD::UNDEF) 216239462Sdim return false; 217239462Sdim 218239462Sdim return true; 219239462Sdim} 220239462Sdim 221193323Sed/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 222193323Sed/// when given the operation for (X op Y). 223193323SedISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 224193323Sed // To perform this operation, we just need to swap the L and G bits of the 225193323Sed // operation. 226193323Sed unsigned OldL = (Operation >> 2) & 1; 227193323Sed unsigned OldG = (Operation >> 1) & 1; 228193323Sed return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 229193323Sed (OldL << 1) | // New G bit 230193323Sed (OldG << 2)); // New L bit. 231193323Sed} 232193323Sed 233193323Sed/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 234193323Sed/// 'op' is a valid SetCC operation. 235193323SedISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 236193323Sed unsigned Operation = Op; 237193323Sed if (isInteger) 238193323Sed Operation ^= 7; // Flip L, G, E bits, but not U. 239193323Sed else 240193323Sed Operation ^= 15; // Flip all of the condition bits. 241193323Sed 242193323Sed if (Operation > ISD::SETTRUE2) 243193323Sed Operation &= ~8; // Don't let N and U bits get set. 244193323Sed 245193323Sed return ISD::CondCode(Operation); 246193323Sed} 247193323Sed 248193323Sed 249193323Sed/// isSignedOp - For an integer comparison, return 1 if the comparison is a 250193323Sed/// signed operation and 2 if the result is an unsigned comparison. Return zero 251193323Sed/// if the operation does not depend on the sign of the input (setne and seteq). 252193323Sedstatic int isSignedOp(ISD::CondCode Opcode) { 253193323Sed switch (Opcode) { 254198090Srdivacky default: llvm_unreachable("Illegal integer setcc operation!"); 255193323Sed case ISD::SETEQ: 256193323Sed case ISD::SETNE: return 0; 257193323Sed case ISD::SETLT: 258193323Sed case ISD::SETLE: 259193323Sed case ISD::SETGT: 260193323Sed case ISD::SETGE: return 1; 261193323Sed case ISD::SETULT: 262193323Sed case ISD::SETULE: 263193323Sed case ISD::SETUGT: 264193323Sed case ISD::SETUGE: return 2; 265193323Sed } 266193323Sed} 267193323Sed 268193323Sed/// getSetCCOrOperation - Return the result of a logical OR between different 269193323Sed/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 270193323Sed/// returns SETCC_INVALID if it is not possible to represent the resultant 271193323Sed/// comparison. 272193323SedISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 273193323Sed bool isInteger) { 274193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 275193323Sed // Cannot fold a signed integer setcc with an unsigned integer setcc. 276193323Sed return ISD::SETCC_INVALID; 277193323Sed 278193323Sed unsigned Op = Op1 | Op2; // Combine all of the condition bits. 279193323Sed 280193323Sed // If the N and U bits get set then the resultant comparison DOES suddenly 281193323Sed // care about orderedness, and is true when ordered. 282193323Sed if (Op > ISD::SETTRUE2) 283193323Sed Op &= ~16; // Clear the U bit if the N bit is set. 284193323Sed 285193323Sed // Canonicalize illegal integer setcc's. 286193323Sed if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 287193323Sed Op = ISD::SETNE; 288193323Sed 289193323Sed return ISD::CondCode(Op); 290193323Sed} 291193323Sed 292193323Sed/// getSetCCAndOperation - Return the result of a logical AND between different 293193323Sed/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 294193323Sed/// function returns zero if it is not possible to represent the resultant 295193323Sed/// comparison. 296193323SedISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 297193323Sed bool isInteger) { 298193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 299193323Sed // Cannot fold a signed setcc with an unsigned setcc. 300193323Sed return ISD::SETCC_INVALID; 301193323Sed 302193323Sed // Combine all of the condition bits. 303193323Sed ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 304193323Sed 305193323Sed // Canonicalize illegal integer setcc's. 306193323Sed if (isInteger) { 307193323Sed switch (Result) { 308193323Sed default: break; 309193323Sed case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 310193323Sed case ISD::SETOEQ: // SETEQ & SETU[LG]E 311193323Sed case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 312193323Sed case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 313193323Sed case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 314193323Sed } 315193323Sed } 316193323Sed 317193323Sed return Result; 318193323Sed} 319193323Sed 320193323Sed//===----------------------------------------------------------------------===// 321193323Sed// SDNode Profile Support 322193323Sed//===----------------------------------------------------------------------===// 323193323Sed 324193323Sed/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 325193323Sed/// 326193323Sedstatic void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 327193323Sed ID.AddInteger(OpC); 328193323Sed} 329193323Sed 330193323Sed/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 331193323Sed/// solely with their pointer. 332193323Sedstatic void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 333193323Sed ID.AddPointer(VTList.VTs); 334193323Sed} 335193323Sed 336193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 337193323Sed/// 338193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 339193323Sed const SDValue *Ops, unsigned NumOps) { 340193323Sed for (; NumOps; --NumOps, ++Ops) { 341193323Sed ID.AddPointer(Ops->getNode()); 342193323Sed ID.AddInteger(Ops->getResNo()); 343193323Sed } 344193323Sed} 345193323Sed 346193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 347193323Sed/// 348193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 349193323Sed const SDUse *Ops, unsigned NumOps) { 350193323Sed for (; NumOps; --NumOps, ++Ops) { 351193323Sed ID.AddPointer(Ops->getNode()); 352193323Sed ID.AddInteger(Ops->getResNo()); 353193323Sed } 354193323Sed} 355193323Sed 356193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, 357193323Sed unsigned short OpC, SDVTList VTList, 358193323Sed const SDValue *OpList, unsigned N) { 359193323Sed AddNodeIDOpcode(ID, OpC); 360193323Sed AddNodeIDValueTypes(ID, VTList); 361193323Sed AddNodeIDOperands(ID, OpList, N); 362193323Sed} 363193323Sed 364193323Sed/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 365193323Sed/// the NodeID data. 366193323Sedstatic void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 367193323Sed switch (N->getOpcode()) { 368195098Sed case ISD::TargetExternalSymbol: 369195098Sed case ISD::ExternalSymbol: 370198090Srdivacky llvm_unreachable("Should only be used on nodes with operands"); 371193323Sed default: break; // Normal nodes don't need extra info. 372193323Sed case ISD::TargetConstant: 373193323Sed case ISD::Constant: 374193323Sed ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 375193323Sed break; 376193323Sed case ISD::TargetConstantFP: 377193323Sed case ISD::ConstantFP: { 378193323Sed ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 379193323Sed break; 380193323Sed } 381193323Sed case ISD::TargetGlobalAddress: 382193323Sed case ISD::GlobalAddress: 383193323Sed case ISD::TargetGlobalTLSAddress: 384193323Sed case ISD::GlobalTLSAddress: { 385193323Sed const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 386193323Sed ID.AddPointer(GA->getGlobal()); 387193323Sed ID.AddInteger(GA->getOffset()); 388195098Sed ID.AddInteger(GA->getTargetFlags()); 389239462Sdim ID.AddInteger(GA->getAddressSpace()); 390193323Sed break; 391193323Sed } 392193323Sed case ISD::BasicBlock: 393193323Sed ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 394193323Sed break; 395193323Sed case ISD::Register: 396193323Sed ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 397193323Sed break; 398234353Sdim case ISD::RegisterMask: 399234353Sdim ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 400234353Sdim break; 401193323Sed case ISD::SRCVALUE: 402193323Sed ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 403193323Sed break; 404193323Sed case ISD::FrameIndex: 405193323Sed case ISD::TargetFrameIndex: 406193323Sed ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 407193323Sed break; 408193323Sed case ISD::JumpTable: 409193323Sed case ISD::TargetJumpTable: 410193323Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 411195098Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 412193323Sed break; 413193323Sed case ISD::ConstantPool: 414193323Sed case ISD::TargetConstantPool: { 415193323Sed const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 416193323Sed ID.AddInteger(CP->getAlignment()); 417193323Sed ID.AddInteger(CP->getOffset()); 418193323Sed if (CP->isMachineConstantPoolEntry()) 419226633Sdim CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 420193323Sed else 421193323Sed ID.AddPointer(CP->getConstVal()); 422195098Sed ID.AddInteger(CP->getTargetFlags()); 423193323Sed break; 424193323Sed } 425239462Sdim case ISD::TargetIndex: { 426239462Sdim const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 427239462Sdim ID.AddInteger(TI->getIndex()); 428239462Sdim ID.AddInteger(TI->getOffset()); 429239462Sdim ID.AddInteger(TI->getTargetFlags()); 430239462Sdim break; 431239462Sdim } 432193323Sed case ISD::LOAD: { 433193323Sed const LoadSDNode *LD = cast<LoadSDNode>(N); 434193323Sed ID.AddInteger(LD->getMemoryVT().getRawBits()); 435193323Sed ID.AddInteger(LD->getRawSubclassData()); 436239462Sdim ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 437193323Sed break; 438193323Sed } 439193323Sed case ISD::STORE: { 440193323Sed const StoreSDNode *ST = cast<StoreSDNode>(N); 441193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 442193323Sed ID.AddInteger(ST->getRawSubclassData()); 443239462Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 444193323Sed break; 445193323Sed } 446193323Sed case ISD::ATOMIC_CMP_SWAP: 447193323Sed case ISD::ATOMIC_SWAP: 448193323Sed case ISD::ATOMIC_LOAD_ADD: 449193323Sed case ISD::ATOMIC_LOAD_SUB: 450193323Sed case ISD::ATOMIC_LOAD_AND: 451193323Sed case ISD::ATOMIC_LOAD_OR: 452193323Sed case ISD::ATOMIC_LOAD_XOR: 453193323Sed case ISD::ATOMIC_LOAD_NAND: 454193323Sed case ISD::ATOMIC_LOAD_MIN: 455193323Sed case ISD::ATOMIC_LOAD_MAX: 456193323Sed case ISD::ATOMIC_LOAD_UMIN: 457226633Sdim case ISD::ATOMIC_LOAD_UMAX: 458226633Sdim case ISD::ATOMIC_LOAD: 459226633Sdim case ISD::ATOMIC_STORE: { 460193323Sed const AtomicSDNode *AT = cast<AtomicSDNode>(N); 461193323Sed ID.AddInteger(AT->getMemoryVT().getRawBits()); 462193323Sed ID.AddInteger(AT->getRawSubclassData()); 463239462Sdim ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 464193323Sed break; 465193323Sed } 466239462Sdim case ISD::PREFETCH: { 467239462Sdim const MemSDNode *PF = cast<MemSDNode>(N); 468239462Sdim ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 469239462Sdim break; 470239462Sdim } 471193323Sed case ISD::VECTOR_SHUFFLE: { 472193323Sed const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 473198090Srdivacky for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 474193323Sed i != e; ++i) 475193323Sed ID.AddInteger(SVN->getMaskElt(i)); 476193323Sed break; 477193323Sed } 478198892Srdivacky case ISD::TargetBlockAddress: 479198892Srdivacky case ISD::BlockAddress: { 480243830Sdim const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 481243830Sdim ID.AddPointer(BA->getBlockAddress()); 482243830Sdim ID.AddInteger(BA->getOffset()); 483243830Sdim ID.AddInteger(BA->getTargetFlags()); 484198892Srdivacky break; 485198892Srdivacky } 486193323Sed } // end switch (N->getOpcode()) 487239462Sdim 488239462Sdim // Target specific memory nodes could also have address spaces to check. 489239462Sdim if (N->isTargetMemoryOpcode()) 490239462Sdim ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 491193323Sed} 492193323Sed 493193323Sed/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 494193323Sed/// data. 495193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 496193323Sed AddNodeIDOpcode(ID, N->getOpcode()); 497193323Sed // Add the return value info. 498193323Sed AddNodeIDValueTypes(ID, N->getVTList()); 499193323Sed // Add the operand info. 500193323Sed AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 501193323Sed 502193323Sed // Handle SDNode leafs with special info. 503193323Sed AddNodeIDCustom(ID, N); 504193323Sed} 505193323Sed 506193323Sed/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 507204642Srdivacky/// the CSE map that carries volatility, temporalness, indexing mode, and 508193323Sed/// extension/truncation information. 509193323Sed/// 510193323Sedstatic inline unsigned 511204642SrdivackyencodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 512234353Sdim bool isNonTemporal, bool isInvariant) { 513193323Sed assert((ConvType & 3) == ConvType && 514193323Sed "ConvType may not require more than 2 bits!"); 515193323Sed assert((AM & 7) == AM && 516193323Sed "AM may not require more than 3 bits!"); 517193323Sed return ConvType | 518193323Sed (AM << 2) | 519204642Srdivacky (isVolatile << 5) | 520234353Sdim (isNonTemporal << 6) | 521234353Sdim (isInvariant << 7); 522193323Sed} 523193323Sed 524193323Sed//===----------------------------------------------------------------------===// 525193323Sed// SelectionDAG Class 526193323Sed//===----------------------------------------------------------------------===// 527193323Sed 528193323Sed/// doNotCSE - Return true if CSE should not be performed for this node. 529193323Sedstatic bool doNotCSE(SDNode *N) { 530218893Sdim if (N->getValueType(0) == MVT::Glue) 531193323Sed return true; // Never CSE anything that produces a flag. 532193323Sed 533193323Sed switch (N->getOpcode()) { 534193323Sed default: break; 535193323Sed case ISD::HANDLENODE: 536193323Sed case ISD::EH_LABEL: 537193323Sed return true; // Never CSE these nodes. 538193323Sed } 539193323Sed 540193323Sed // Check that remaining values produced are not flags. 541193323Sed for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 542218893Sdim if (N->getValueType(i) == MVT::Glue) 543193323Sed return true; // Never CSE anything that produces a flag. 544193323Sed 545193323Sed return false; 546193323Sed} 547193323Sed 548193323Sed/// RemoveDeadNodes - This method deletes all unreachable nodes in the 549193323Sed/// SelectionDAG. 550193323Sedvoid SelectionDAG::RemoveDeadNodes() { 551193323Sed // Create a dummy node (which is not added to allnodes), that adds a reference 552193323Sed // to the root node, preventing it from being deleted. 553193323Sed HandleSDNode Dummy(getRoot()); 554193323Sed 555193323Sed SmallVector<SDNode*, 128> DeadNodes; 556193323Sed 557193323Sed // Add all obviously-dead nodes to the DeadNodes worklist. 558193323Sed for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 559193323Sed if (I->use_empty()) 560193323Sed DeadNodes.push_back(I); 561193323Sed 562193323Sed RemoveDeadNodes(DeadNodes); 563193323Sed 564193323Sed // If the root changed (e.g. it was a dead load, update the root). 565193323Sed setRoot(Dummy.getValue()); 566193323Sed} 567193323Sed 568193323Sed/// RemoveDeadNodes - This method deletes the unreachable nodes in the 569193323Sed/// given list, and any nodes that become unreachable as a result. 570239462Sdimvoid SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 571193323Sed 572193323Sed // Process the worklist, deleting the nodes and adding their uses to the 573193323Sed // worklist. 574193323Sed while (!DeadNodes.empty()) { 575193323Sed SDNode *N = DeadNodes.pop_back_val(); 576193323Sed 577239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 578239462Sdim DUL->NodeDeleted(N, 0); 579193323Sed 580193323Sed // Take the node out of the appropriate CSE map. 581193323Sed RemoveNodeFromCSEMaps(N); 582193323Sed 583193323Sed // Next, brutally remove the operand list. This is safe to do, as there are 584193323Sed // no cycles in the graph. 585193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 586193323Sed SDUse &Use = *I++; 587193323Sed SDNode *Operand = Use.getNode(); 588193323Sed Use.set(SDValue()); 589193323Sed 590193323Sed // Now that we removed this operand, see if there are no uses of it left. 591193323Sed if (Operand->use_empty()) 592193323Sed DeadNodes.push_back(Operand); 593193323Sed } 594193323Sed 595193323Sed DeallocateNode(N); 596193323Sed } 597193323Sed} 598193323Sed 599239462Sdimvoid SelectionDAG::RemoveDeadNode(SDNode *N){ 600193323Sed SmallVector<SDNode*, 16> DeadNodes(1, N); 601234353Sdim 602234353Sdim // Create a dummy node that adds a reference to the root node, preventing 603234353Sdim // it from being deleted. (This matters if the root is an operand of the 604234353Sdim // dead node.) 605234353Sdim HandleSDNode Dummy(getRoot()); 606234353Sdim 607239462Sdim RemoveDeadNodes(DeadNodes); 608193323Sed} 609193323Sed 610193323Sedvoid SelectionDAG::DeleteNode(SDNode *N) { 611193323Sed // First take this out of the appropriate CSE map. 612193323Sed RemoveNodeFromCSEMaps(N); 613193323Sed 614193323Sed // Finally, remove uses due to operands of this node, remove from the 615193323Sed // AllNodes list, and delete the node. 616193323Sed DeleteNodeNotInCSEMaps(N); 617193323Sed} 618193323Sed 619193323Sedvoid SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 620193323Sed assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 621193323Sed assert(N->use_empty() && "Cannot delete a node that is not dead!"); 622193323Sed 623193323Sed // Drop all of the operands and decrement used node's use counts. 624193323Sed N->DropOperands(); 625193323Sed 626193323Sed DeallocateNode(N); 627193323Sed} 628193323Sed 629193323Sedvoid SelectionDAG::DeallocateNode(SDNode *N) { 630193323Sed if (N->OperandsNeedDelete) 631193323Sed delete[] N->OperandList; 632193323Sed 633193323Sed // Set the opcode to DELETED_NODE to help catch bugs when node 634193323Sed // memory is reallocated. 635193323Sed N->NodeType = ISD::DELETED_NODE; 636193323Sed 637193323Sed NodeAllocator.Deallocate(AllNodes.remove(N)); 638200581Srdivacky 639200581Srdivacky // Remove the ordering of this node. 640202878Srdivacky Ordering->remove(N); 641205218Srdivacky 642206083Srdivacky // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 643224145Sdim ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 644206083Srdivacky for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 645206083Srdivacky DbgVals[i]->setIsInvalidated(); 646193323Sed} 647193323Sed 648193323Sed/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 649193323Sed/// correspond to it. This is useful when we're about to delete or repurpose 650193323Sed/// the node. We don't want future request for structurally identical nodes 651193323Sed/// to return N anymore. 652193323Sedbool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 653193323Sed bool Erased = false; 654193323Sed switch (N->getOpcode()) { 655193323Sed case ISD::HANDLENODE: return false; // noop. 656193323Sed case ISD::CONDCODE: 657193323Sed assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 658193323Sed "Cond code doesn't exist!"); 659193323Sed Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 660193323Sed CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 661193323Sed break; 662193323Sed case ISD::ExternalSymbol: 663193323Sed Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 664193323Sed break; 665195098Sed case ISD::TargetExternalSymbol: { 666195098Sed ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 667195098Sed Erased = TargetExternalSymbols.erase( 668195098Sed std::pair<std::string,unsigned char>(ESN->getSymbol(), 669195098Sed ESN->getTargetFlags())); 670193323Sed break; 671195098Sed } 672193323Sed case ISD::VALUETYPE: { 673198090Srdivacky EVT VT = cast<VTSDNode>(N)->getVT(); 674193323Sed if (VT.isExtended()) { 675193323Sed Erased = ExtendedValueTypeNodes.erase(VT); 676193323Sed } else { 677198090Srdivacky Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 678198090Srdivacky ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 679193323Sed } 680193323Sed break; 681193323Sed } 682193323Sed default: 683193323Sed // Remove it from the CSE Map. 684218893Sdim assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 685218893Sdim assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 686193323Sed Erased = CSEMap.RemoveNode(N); 687193323Sed break; 688193323Sed } 689193323Sed#ifndef NDEBUG 690193323Sed // Verify that the node was actually in one of the CSE maps, unless it has a 691193323Sed // flag result (which cannot be CSE'd) or is one of the special cases that are 692193323Sed // not subject to CSE. 693218893Sdim if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 694193323Sed !N->isMachineOpcode() && !doNotCSE(N)) { 695193323Sed N->dump(this); 696202375Srdivacky dbgs() << "\n"; 697198090Srdivacky llvm_unreachable("Node is not in map!"); 698193323Sed } 699193323Sed#endif 700193323Sed return Erased; 701193323Sed} 702193323Sed 703193323Sed/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 704193323Sed/// maps and modified in place. Add it back to the CSE maps, unless an identical 705193323Sed/// node already exists, in which case transfer all its users to the existing 706193323Sed/// node. This transfer can potentially trigger recursive merging. 707193323Sed/// 708193323Sedvoid 709239462SdimSelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 710193323Sed // For node types that aren't CSE'd, just act as if no identical node 711193323Sed // already exists. 712193323Sed if (!doNotCSE(N)) { 713193323Sed SDNode *Existing = CSEMap.GetOrInsertNode(N); 714193323Sed if (Existing != N) { 715193323Sed // If there was already an existing matching node, use ReplaceAllUsesWith 716193323Sed // to replace the dead one with the existing one. This can cause 717193323Sed // recursive merging of other unrelated nodes down the line. 718239462Sdim ReplaceAllUsesWith(N, Existing); 719193323Sed 720239462Sdim // N is now dead. Inform the listeners and delete it. 721239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 722239462Sdim DUL->NodeDeleted(N, Existing); 723193323Sed DeleteNodeNotInCSEMaps(N); 724193323Sed return; 725193323Sed } 726193323Sed } 727193323Sed 728239462Sdim // If the node doesn't already exist, we updated it. Inform listeners. 729239462Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 730239462Sdim DUL->NodeUpdated(N); 731193323Sed} 732193323Sed 733193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 734193323Sed/// were replaced with those specified. If this node is never memoized, 735193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 736193323Sed/// node already exists with these operands, the slot will be non-null. 737193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 738193323Sed void *&InsertPos) { 739193323Sed if (doNotCSE(N)) 740193323Sed return 0; 741193323Sed 742193323Sed SDValue Ops[] = { Op }; 743193323Sed FoldingSetNodeID ID; 744193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 745193323Sed AddNodeIDCustom(ID, N); 746200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 747200581Srdivacky return Node; 748193323Sed} 749193323Sed 750193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 751193323Sed/// were replaced with those specified. If this node is never memoized, 752193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 753193323Sed/// node already exists with these operands, the slot will be non-null. 754193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 755193323Sed SDValue Op1, SDValue Op2, 756193323Sed void *&InsertPos) { 757193323Sed if (doNotCSE(N)) 758193323Sed return 0; 759193323Sed 760193323Sed SDValue Ops[] = { Op1, Op2 }; 761193323Sed FoldingSetNodeID ID; 762193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 763193323Sed AddNodeIDCustom(ID, N); 764200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 765200581Srdivacky return Node; 766193323Sed} 767193323Sed 768193323Sed 769193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 770193323Sed/// were replaced with those specified. If this node is never memoized, 771193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 772193323Sed/// node already exists with these operands, the slot will be non-null. 773193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 774193323Sed const SDValue *Ops,unsigned NumOps, 775193323Sed void *&InsertPos) { 776193323Sed if (doNotCSE(N)) 777193323Sed return 0; 778193323Sed 779193323Sed FoldingSetNodeID ID; 780193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 781193323Sed AddNodeIDCustom(ID, N); 782200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 783200581Srdivacky return Node; 784193323Sed} 785193323Sed 786218893Sdim#ifndef NDEBUG 787218893Sdim/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 788218893Sdimstatic void VerifyNodeCommon(SDNode *N) { 789193323Sed switch (N->getOpcode()) { 790193323Sed default: 791193323Sed break; 792193323Sed case ISD::BUILD_PAIR: { 793198090Srdivacky EVT VT = N->getValueType(0); 794193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 795193323Sed assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 796193323Sed "Wrong return type!"); 797193323Sed assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 798193323Sed assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 799193323Sed "Mismatched operand types!"); 800193323Sed assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 801193323Sed "Wrong operand type!"); 802193323Sed assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 803193323Sed "Wrong return type size"); 804193323Sed break; 805193323Sed } 806193323Sed case ISD::BUILD_VECTOR: { 807193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 808193323Sed assert(N->getValueType(0).isVector() && "Wrong return type!"); 809193323Sed assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 810193323Sed "Wrong number of operands!"); 811198090Srdivacky EVT EltVT = N->getValueType(0).getVectorElementType(); 812226633Sdim for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 813193323Sed assert((I->getValueType() == EltVT || 814193323Sed (EltVT.isInteger() && I->getValueType().isInteger() && 815193323Sed EltVT.bitsLE(I->getValueType()))) && 816193323Sed "Wrong operand type!"); 817226633Sdim assert(I->getValueType() == N->getOperand(0).getValueType() && 818226633Sdim "Operands must all have the same type"); 819226633Sdim } 820193323Sed break; 821193323Sed } 822193323Sed } 823193323Sed} 824193323Sed 825218893Sdim/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 826218893Sdimstatic void VerifySDNode(SDNode *N) { 827218893Sdim // The SDNode allocators cannot be used to allocate nodes with fields that are 828218893Sdim // not present in an SDNode! 829218893Sdim assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 830218893Sdim assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 831218893Sdim assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 832218893Sdim assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 833218893Sdim assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 834218893Sdim assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 835218893Sdim assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 836218893Sdim assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 837218893Sdim assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 838218893Sdim assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 839218893Sdim assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 840218893Sdim assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 841218893Sdim assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 842218893Sdim assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 843218893Sdim assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 844218893Sdim assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 845218893Sdim assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 846218893Sdim assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 847218893Sdim assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 848218893Sdim 849218893Sdim VerifyNodeCommon(N); 850218893Sdim} 851218893Sdim 852218893Sdim/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 853218893Sdim/// invalid. 854218893Sdimstatic void VerifyMachineNode(SDNode *N) { 855218893Sdim // The MachineNode allocators cannot be used to allocate nodes with fields 856218893Sdim // that are not present in a MachineNode! 857218893Sdim // Currently there are no such nodes. 858218893Sdim 859218893Sdim VerifyNodeCommon(N); 860218893Sdim} 861218893Sdim#endif // NDEBUG 862218893Sdim 863198090Srdivacky/// getEVTAlignment - Compute the default alignment value for the 864193323Sed/// given type. 865193323Sed/// 866198090Srdivackyunsigned SelectionDAG::getEVTAlignment(EVT VT) const { 867226633Sdim Type *Ty = VT == MVT::iPTR ? 868198090Srdivacky PointerType::get(Type::getInt8Ty(*getContext()), 0) : 869198090Srdivacky VT.getTypeForEVT(*getContext()); 870193323Sed 871243830Sdim return TLI.getDataLayout()->getABITypeAlignment(Ty); 872193323Sed} 873193323Sed 874193323Sed// EntryNode could meaningfully have debug info if we can find it... 875234353SdimSelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 876208599Srdivacky : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), 877249423Sdim TTI(0), OptLevel(OL), EntryNode(ISD::EntryToken, DebugLoc(), 878249423Sdim getVTList(MVT::Other)), 879239462Sdim Root(getEntryNode()), Ordering(0), UpdateListeners(0) { 880193323Sed AllNodes.push_back(&EntryNode); 881202878Srdivacky Ordering = new SDNodeOrdering(); 882205218Srdivacky DbgInfo = new SDDbgInfo(); 883193323Sed} 884193323Sed 885249423Sdimvoid SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti) { 886193323Sed MF = &mf; 887249423Sdim TTI = tti; 888198090Srdivacky Context = &mf.getFunction()->getContext(); 889193323Sed} 890193323Sed 891193323SedSelectionDAG::~SelectionDAG() { 892239462Sdim assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 893193323Sed allnodes_clear(); 894200581Srdivacky delete Ordering; 895205218Srdivacky delete DbgInfo; 896193323Sed} 897193323Sed 898193323Sedvoid SelectionDAG::allnodes_clear() { 899193323Sed assert(&*AllNodes.begin() == &EntryNode); 900193323Sed AllNodes.remove(AllNodes.begin()); 901193323Sed while (!AllNodes.empty()) 902193323Sed DeallocateNode(AllNodes.begin()); 903193323Sed} 904193323Sed 905193323Sedvoid SelectionDAG::clear() { 906193323Sed allnodes_clear(); 907193323Sed OperandAllocator.Reset(); 908193323Sed CSEMap.clear(); 909193323Sed 910193323Sed ExtendedValueTypeNodes.clear(); 911193323Sed ExternalSymbols.clear(); 912193323Sed TargetExternalSymbols.clear(); 913193323Sed std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 914193323Sed static_cast<CondCodeSDNode*>(0)); 915193323Sed std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 916193323Sed static_cast<SDNode*>(0)); 917193323Sed 918193323Sed EntryNode.UseList = 0; 919193323Sed AllNodes.push_back(&EntryNode); 920193323Sed Root = getEntryNode(); 921210299Sed Ordering->clear(); 922206083Srdivacky DbgInfo->clear(); 923193323Sed} 924193323Sed 925226633SdimSDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 926226633Sdim return VT.bitsGT(Op.getValueType()) ? 927226633Sdim getNode(ISD::ANY_EXTEND, DL, VT, Op) : 928226633Sdim getNode(ISD::TRUNCATE, DL, VT, Op); 929226633Sdim} 930226633Sdim 931198090SrdivackySDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 932198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 933198090Srdivacky getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 934198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 935198090Srdivacky} 936198090Srdivacky 937198090SrdivackySDValue SelectionDAG::getZExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { 938198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 939198090Srdivacky getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 940198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 941198090Srdivacky} 942198090Srdivacky 943198090SrdivackySDValue SelectionDAG::getZeroExtendInReg(SDValue Op, DebugLoc DL, EVT VT) { 944200581Srdivacky assert(!VT.isVector() && 945200581Srdivacky "getZeroExtendInReg should use the vector element type instead of " 946200581Srdivacky "the vector type!"); 947193323Sed if (Op.getValueType() == VT) return Op; 948200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 949200581Srdivacky APInt Imm = APInt::getLowBitsSet(BitWidth, 950193323Sed VT.getSizeInBits()); 951193323Sed return getNode(ISD::AND, DL, Op.getValueType(), Op, 952193323Sed getConstant(Imm, Op.getValueType())); 953193323Sed} 954193323Sed 955193323Sed/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 956193323Sed/// 957198090SrdivackySDValue SelectionDAG::getNOT(DebugLoc DL, SDValue Val, EVT VT) { 958204642Srdivacky EVT EltVT = VT.getScalarType(); 959193323Sed SDValue NegOne = 960193323Sed getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 961193323Sed return getNode(ISD::XOR, DL, VT, Val, NegOne); 962193323Sed} 963193323Sed 964198090SrdivackySDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 965204642Srdivacky EVT EltVT = VT.getScalarType(); 966193323Sed assert((EltVT.getSizeInBits() >= 64 || 967193323Sed (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 968193323Sed "getConstant with a uint64_t value that doesn't fit in the type!"); 969193323Sed return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 970193323Sed} 971193323Sed 972198090SrdivackySDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 973198090Srdivacky return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 974193323Sed} 975193323Sed 976198090SrdivackySDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 977193323Sed assert(VT.isInteger() && "Cannot create FP integer constant!"); 978193323Sed 979204642Srdivacky EVT EltVT = VT.getScalarType(); 980226633Sdim const ConstantInt *Elt = &Val; 981226633Sdim 982226633Sdim // In some cases the vector type is legal but the element type is illegal and 983226633Sdim // needs to be promoted, for example v8i8 on ARM. In this case, promote the 984226633Sdim // inserted value (the type does not need to match the vector element type). 985226633Sdim // Any extra bits introduced will be truncated away. 986226633Sdim if (VT.isVector() && TLI.getTypeAction(*getContext(), EltVT) == 987226633Sdim TargetLowering::TypePromoteInteger) { 988226633Sdim EltVT = TLI.getTypeToTransformTo(*getContext(), EltVT); 989226633Sdim APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 990226633Sdim Elt = ConstantInt::get(*getContext(), NewVal); 991226633Sdim } 992226633Sdim 993226633Sdim assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 994193323Sed "APInt size does not match type size!"); 995193323Sed unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 996193323Sed FoldingSetNodeID ID; 997193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 998226633Sdim ID.AddPointer(Elt); 999193323Sed void *IP = 0; 1000193323Sed SDNode *N = NULL; 1001201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1002193323Sed if (!VT.isVector()) 1003193323Sed return SDValue(N, 0); 1004201360Srdivacky 1005193323Sed if (!N) { 1006226633Sdim N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 1007193323Sed CSEMap.InsertNode(N, IP); 1008193323Sed AllNodes.push_back(N); 1009193323Sed } 1010193323Sed 1011193323Sed SDValue Result(N, 0); 1012193323Sed if (VT.isVector()) { 1013193323Sed SmallVector<SDValue, 8> Ops; 1014193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1015206124Srdivacky Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 1016193323Sed } 1017193323Sed return Result; 1018193323Sed} 1019193323Sed 1020193323SedSDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 1021193323Sed return getConstant(Val, TLI.getPointerTy(), isTarget); 1022193323Sed} 1023193323Sed 1024193323Sed 1025198090SrdivackySDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 1026198090Srdivacky return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 1027193323Sed} 1028193323Sed 1029198090SrdivackySDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 1030193323Sed assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1031193323Sed 1032204642Srdivacky EVT EltVT = VT.getScalarType(); 1033193323Sed 1034193323Sed // Do the map lookup using the actual bit pattern for the floating point 1035193323Sed // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1036193323Sed // we don't have issues with SNANs. 1037193323Sed unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1038193323Sed FoldingSetNodeID ID; 1039193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1040193323Sed ID.AddPointer(&V); 1041193323Sed void *IP = 0; 1042193323Sed SDNode *N = NULL; 1043201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1044193323Sed if (!VT.isVector()) 1045193323Sed return SDValue(N, 0); 1046201360Srdivacky 1047193323Sed if (!N) { 1048205407Srdivacky N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1049193323Sed CSEMap.InsertNode(N, IP); 1050193323Sed AllNodes.push_back(N); 1051193323Sed } 1052193323Sed 1053193323Sed SDValue Result(N, 0); 1054193323Sed if (VT.isVector()) { 1055193323Sed SmallVector<SDValue, 8> Ops; 1056193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1057193323Sed // FIXME DebugLoc info might be appropriate here 1058206124Srdivacky Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); 1059193323Sed } 1060193323Sed return Result; 1061193323Sed} 1062193323Sed 1063198090SrdivackySDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1064204642Srdivacky EVT EltVT = VT.getScalarType(); 1065193323Sed if (EltVT==MVT::f32) 1066193323Sed return getConstantFP(APFloat((float)Val), VT, isTarget); 1067208599Srdivacky else if (EltVT==MVT::f64) 1068193323Sed return getConstantFP(APFloat(Val), VT, isTarget); 1069249423Sdim else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || 1070249423Sdim EltVT==MVT::f16) { 1071208599Srdivacky bool ignored; 1072208599Srdivacky APFloat apf = APFloat(Val); 1073249423Sdim apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1074208599Srdivacky &ignored); 1075208599Srdivacky return getConstantFP(apf, VT, isTarget); 1076234353Sdim } else 1077234353Sdim llvm_unreachable("Unsupported type in getConstantFP"); 1078193323Sed} 1079193323Sed 1080210299SedSDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, DebugLoc DL, 1081198090Srdivacky EVT VT, int64_t Offset, 1082195098Sed bool isTargetGA, 1083195098Sed unsigned char TargetFlags) { 1084195098Sed assert((TargetFlags == 0 || isTargetGA) && 1085195098Sed "Cannot set target flags on target-independent globals"); 1086198090Srdivacky 1087193323Sed // Truncate (with sign-extension) the offset value to the pointer size. 1088243830Sdim unsigned BitWidth = TLI.getPointerTy().getSizeInBits(); 1089193323Sed if (BitWidth < 64) 1090243830Sdim Offset = SignExtend64(Offset, BitWidth); 1091193323Sed 1092193323Sed const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1093193323Sed if (!GVar) { 1094193323Sed // If GV is an alias then use the aliasee for determining thread-localness. 1095193323Sed if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1096193323Sed GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1097193323Sed } 1098193323Sed 1099195098Sed unsigned Opc; 1100193323Sed if (GVar && GVar->isThreadLocal()) 1101193323Sed Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1102193323Sed else 1103193323Sed Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1104193323Sed 1105193323Sed FoldingSetNodeID ID; 1106193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1107193323Sed ID.AddPointer(GV); 1108193323Sed ID.AddInteger(Offset); 1109195098Sed ID.AddInteger(TargetFlags); 1110239462Sdim ID.AddInteger(GV->getType()->getAddressSpace()); 1111193323Sed void *IP = 0; 1112201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1113193323Sed return SDValue(E, 0); 1114201360Srdivacky 1115210299Sed SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL, GV, VT, 1116205407Srdivacky Offset, TargetFlags); 1117193323Sed CSEMap.InsertNode(N, IP); 1118193323Sed AllNodes.push_back(N); 1119193323Sed return SDValue(N, 0); 1120193323Sed} 1121193323Sed 1122198090SrdivackySDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1123193323Sed unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1124193323Sed FoldingSetNodeID ID; 1125193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1126193323Sed ID.AddInteger(FI); 1127193323Sed void *IP = 0; 1128201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1129193323Sed return SDValue(E, 0); 1130201360Srdivacky 1131205407Srdivacky SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1132193323Sed CSEMap.InsertNode(N, IP); 1133193323Sed AllNodes.push_back(N); 1134193323Sed return SDValue(N, 0); 1135193323Sed} 1136193323Sed 1137198090SrdivackySDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1138195098Sed unsigned char TargetFlags) { 1139195098Sed assert((TargetFlags == 0 || isTarget) && 1140195098Sed "Cannot set target flags on target-independent jump tables"); 1141193323Sed unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1142193323Sed FoldingSetNodeID ID; 1143193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1144193323Sed ID.AddInteger(JTI); 1145195098Sed ID.AddInteger(TargetFlags); 1146193323Sed void *IP = 0; 1147201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1148193323Sed return SDValue(E, 0); 1149201360Srdivacky 1150205407Srdivacky SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1151205407Srdivacky TargetFlags); 1152193323Sed CSEMap.InsertNode(N, IP); 1153193323Sed AllNodes.push_back(N); 1154193323Sed return SDValue(N, 0); 1155193323Sed} 1156193323Sed 1157207618SrdivackySDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1158193323Sed unsigned Alignment, int Offset, 1159198090Srdivacky bool isTarget, 1160195098Sed unsigned char TargetFlags) { 1161195098Sed assert((TargetFlags == 0 || isTarget) && 1162195098Sed "Cannot set target flags on target-independent globals"); 1163193323Sed if (Alignment == 0) 1164243830Sdim Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType()); 1165193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1166193323Sed FoldingSetNodeID ID; 1167193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1168193323Sed ID.AddInteger(Alignment); 1169193323Sed ID.AddInteger(Offset); 1170193323Sed ID.AddPointer(C); 1171195098Sed ID.AddInteger(TargetFlags); 1172193323Sed void *IP = 0; 1173201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1174193323Sed return SDValue(E, 0); 1175201360Srdivacky 1176205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1177205407Srdivacky Alignment, TargetFlags); 1178193323Sed CSEMap.InsertNode(N, IP); 1179193323Sed AllNodes.push_back(N); 1180193323Sed return SDValue(N, 0); 1181193323Sed} 1182193323Sed 1183193323Sed 1184198090SrdivackySDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1185193323Sed unsigned Alignment, int Offset, 1186195098Sed bool isTarget, 1187195098Sed unsigned char TargetFlags) { 1188195098Sed assert((TargetFlags == 0 || isTarget) && 1189195098Sed "Cannot set target flags on target-independent globals"); 1190193323Sed if (Alignment == 0) 1191243830Sdim Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType()); 1192193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1193193323Sed FoldingSetNodeID ID; 1194193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1195193323Sed ID.AddInteger(Alignment); 1196193323Sed ID.AddInteger(Offset); 1197226633Sdim C->addSelectionDAGCSEId(ID); 1198195098Sed ID.AddInteger(TargetFlags); 1199193323Sed void *IP = 0; 1200201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1201193323Sed return SDValue(E, 0); 1202201360Srdivacky 1203205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1204205407Srdivacky Alignment, TargetFlags); 1205193323Sed CSEMap.InsertNode(N, IP); 1206193323Sed AllNodes.push_back(N); 1207193323Sed return SDValue(N, 0); 1208193323Sed} 1209193323Sed 1210239462SdimSDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1211239462Sdim unsigned char TargetFlags) { 1212239462Sdim FoldingSetNodeID ID; 1213239462Sdim AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0); 1214239462Sdim ID.AddInteger(Index); 1215239462Sdim ID.AddInteger(Offset); 1216239462Sdim ID.AddInteger(TargetFlags); 1217239462Sdim void *IP = 0; 1218239462Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1219239462Sdim return SDValue(E, 0); 1220239462Sdim 1221239462Sdim SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, 1222239462Sdim TargetFlags); 1223239462Sdim CSEMap.InsertNode(N, IP); 1224239462Sdim AllNodes.push_back(N); 1225239462Sdim return SDValue(N, 0); 1226239462Sdim} 1227239462Sdim 1228193323SedSDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1229193323Sed FoldingSetNodeID ID; 1230193323Sed AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1231193323Sed ID.AddPointer(MBB); 1232193323Sed void *IP = 0; 1233201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1234193323Sed return SDValue(E, 0); 1235201360Srdivacky 1236205407Srdivacky SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1237193323Sed CSEMap.InsertNode(N, IP); 1238193323Sed AllNodes.push_back(N); 1239193323Sed return SDValue(N, 0); 1240193323Sed} 1241193323Sed 1242198090SrdivackySDValue SelectionDAG::getValueType(EVT VT) { 1243198090Srdivacky if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1244198090Srdivacky ValueTypeNodes.size()) 1245198090Srdivacky ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1246193323Sed 1247193323Sed SDNode *&N = VT.isExtended() ? 1248198090Srdivacky ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1249193323Sed 1250193323Sed if (N) return SDValue(N, 0); 1251205407Srdivacky N = new (NodeAllocator) VTSDNode(VT); 1252193323Sed AllNodes.push_back(N); 1253193323Sed return SDValue(N, 0); 1254193323Sed} 1255193323Sed 1256198090SrdivackySDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1257193323Sed SDNode *&N = ExternalSymbols[Sym]; 1258193323Sed if (N) return SDValue(N, 0); 1259205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1260193323Sed AllNodes.push_back(N); 1261193323Sed return SDValue(N, 0); 1262193323Sed} 1263193323Sed 1264198090SrdivackySDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1265195098Sed unsigned char TargetFlags) { 1266195098Sed SDNode *&N = 1267195098Sed TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1268195098Sed TargetFlags)]; 1269193323Sed if (N) return SDValue(N, 0); 1270205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1271193323Sed AllNodes.push_back(N); 1272193323Sed return SDValue(N, 0); 1273193323Sed} 1274193323Sed 1275193323SedSDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1276193323Sed if ((unsigned)Cond >= CondCodeNodes.size()) 1277193323Sed CondCodeNodes.resize(Cond+1); 1278193323Sed 1279193323Sed if (CondCodeNodes[Cond] == 0) { 1280205407Srdivacky CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1281193323Sed CondCodeNodes[Cond] = N; 1282193323Sed AllNodes.push_back(N); 1283193323Sed } 1284201360Srdivacky 1285193323Sed return SDValue(CondCodeNodes[Cond], 0); 1286193323Sed} 1287193323Sed 1288193323Sed// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1289193323Sed// the shuffle mask M that point at N1 to point at N2, and indices that point 1290193323Sed// N2 to point at N1. 1291193323Sedstatic void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1292193323Sed std::swap(N1, N2); 1293193323Sed int NElts = M.size(); 1294193323Sed for (int i = 0; i != NElts; ++i) { 1295193323Sed if (M[i] >= NElts) 1296193323Sed M[i] -= NElts; 1297193323Sed else if (M[i] >= 0) 1298193323Sed M[i] += NElts; 1299193323Sed } 1300193323Sed} 1301193323Sed 1302198090SrdivackySDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, 1303193323Sed SDValue N2, const int *Mask) { 1304193323Sed assert(N1.getValueType() == N2.getValueType() && "Invalid VECTOR_SHUFFLE"); 1305198090Srdivacky assert(VT.isVector() && N1.getValueType().isVector() && 1306193323Sed "Vector Shuffle VTs must be a vectors"); 1307193323Sed assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() 1308193323Sed && "Vector Shuffle VTs must have same element type"); 1309193323Sed 1310193323Sed // Canonicalize shuffle undef, undef -> undef 1311193323Sed if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1312198090Srdivacky return getUNDEF(VT); 1313193323Sed 1314198090Srdivacky // Validate that all indices in Mask are within the range of the elements 1315193323Sed // input to the shuffle. 1316193323Sed unsigned NElts = VT.getVectorNumElements(); 1317193323Sed SmallVector<int, 8> MaskVec; 1318193323Sed for (unsigned i = 0; i != NElts; ++i) { 1319193323Sed assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1320193323Sed MaskVec.push_back(Mask[i]); 1321193323Sed } 1322198090Srdivacky 1323193323Sed // Canonicalize shuffle v, v -> v, undef 1324193323Sed if (N1 == N2) { 1325193323Sed N2 = getUNDEF(VT); 1326193323Sed for (unsigned i = 0; i != NElts; ++i) 1327193323Sed if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1328193323Sed } 1329198090Srdivacky 1330193323Sed // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1331193323Sed if (N1.getOpcode() == ISD::UNDEF) 1332193323Sed commuteShuffle(N1, N2, MaskVec); 1333198090Srdivacky 1334193323Sed // Canonicalize all index into lhs, -> shuffle lhs, undef 1335193323Sed // Canonicalize all index into rhs, -> shuffle rhs, undef 1336193323Sed bool AllLHS = true, AllRHS = true; 1337193323Sed bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1338193323Sed for (unsigned i = 0; i != NElts; ++i) { 1339193323Sed if (MaskVec[i] >= (int)NElts) { 1340193323Sed if (N2Undef) 1341193323Sed MaskVec[i] = -1; 1342193323Sed else 1343193323Sed AllLHS = false; 1344193323Sed } else if (MaskVec[i] >= 0) { 1345193323Sed AllRHS = false; 1346193323Sed } 1347193323Sed } 1348193323Sed if (AllLHS && AllRHS) 1349193323Sed return getUNDEF(VT); 1350193323Sed if (AllLHS && !N2Undef) 1351193323Sed N2 = getUNDEF(VT); 1352193323Sed if (AllRHS) { 1353193323Sed N1 = getUNDEF(VT); 1354193323Sed commuteShuffle(N1, N2, MaskVec); 1355193323Sed } 1356198090Srdivacky 1357193323Sed // If Identity shuffle, or all shuffle in to undef, return that node. 1358193323Sed bool AllUndef = true; 1359193323Sed bool Identity = true; 1360193323Sed for (unsigned i = 0; i != NElts; ++i) { 1361193323Sed if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1362193323Sed if (MaskVec[i] >= 0) AllUndef = false; 1363193323Sed } 1364198090Srdivacky if (Identity && NElts == N1.getValueType().getVectorNumElements()) 1365193323Sed return N1; 1366193323Sed if (AllUndef) 1367193323Sed return getUNDEF(VT); 1368193323Sed 1369193323Sed FoldingSetNodeID ID; 1370193323Sed SDValue Ops[2] = { N1, N2 }; 1371193323Sed AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1372193323Sed for (unsigned i = 0; i != NElts; ++i) 1373193323Sed ID.AddInteger(MaskVec[i]); 1374198090Srdivacky 1375193323Sed void* IP = 0; 1376201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1377193323Sed return SDValue(E, 0); 1378198090Srdivacky 1379193323Sed // Allocate the mask array for the node out of the BumpPtrAllocator, since 1380193323Sed // SDNode doesn't have access to it. This memory will be "leaked" when 1381193323Sed // the node is deallocated, but recovered when the NodeAllocator is released. 1382193323Sed int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1383193323Sed memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1384198090Srdivacky 1385205407Srdivacky ShuffleVectorSDNode *N = 1386205407Srdivacky new (NodeAllocator) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc); 1387193323Sed CSEMap.InsertNode(N, IP); 1388193323Sed AllNodes.push_back(N); 1389193323Sed return SDValue(N, 0); 1390193323Sed} 1391193323Sed 1392198090SrdivackySDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl, 1393193323Sed SDValue Val, SDValue DTy, 1394193323Sed SDValue STy, SDValue Rnd, SDValue Sat, 1395193323Sed ISD::CvtCode Code) { 1396193323Sed // If the src and dest types are the same and the conversion is between 1397193323Sed // integer types of the same sign or two floats, no conversion is necessary. 1398193323Sed if (DTy == STy && 1399193323Sed (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1400193323Sed return Val; 1401193323Sed 1402193323Sed FoldingSetNodeID ID; 1403199481Srdivacky SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1404199481Srdivacky AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1405193323Sed void* IP = 0; 1406201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1407193323Sed return SDValue(E, 0); 1408201360Srdivacky 1409205407Srdivacky CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl, Ops, 5, 1410205407Srdivacky Code); 1411193323Sed CSEMap.InsertNode(N, IP); 1412193323Sed AllNodes.push_back(N); 1413193323Sed return SDValue(N, 0); 1414193323Sed} 1415193323Sed 1416198090SrdivackySDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1417193323Sed FoldingSetNodeID ID; 1418193323Sed AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1419193323Sed ID.AddInteger(RegNo); 1420193323Sed void *IP = 0; 1421201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1422193323Sed return SDValue(E, 0); 1423201360Srdivacky 1424205407Srdivacky SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1425193323Sed CSEMap.InsertNode(N, IP); 1426193323Sed AllNodes.push_back(N); 1427193323Sed return SDValue(N, 0); 1428193323Sed} 1429193323Sed 1430234353SdimSDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1431234353Sdim FoldingSetNodeID ID; 1432234353Sdim AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 1433234353Sdim ID.AddPointer(RegMask); 1434234353Sdim void *IP = 0; 1435234353Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1436234353Sdim return SDValue(E, 0); 1437234353Sdim 1438234353Sdim SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 1439234353Sdim CSEMap.InsertNode(N, IP); 1440234353Sdim AllNodes.push_back(N); 1441234353Sdim return SDValue(N, 0); 1442234353Sdim} 1443234353Sdim 1444205218SrdivackySDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { 1445193323Sed FoldingSetNodeID ID; 1446193323Sed SDValue Ops[] = { Root }; 1447205218Srdivacky AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1448205218Srdivacky ID.AddPointer(Label); 1449193323Sed void *IP = 0; 1450201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1451193323Sed return SDValue(E, 0); 1452218893Sdim 1453205407Srdivacky SDNode *N = new (NodeAllocator) EHLabelSDNode(dl, Root, Label); 1454193323Sed CSEMap.InsertNode(N, IP); 1455193323Sed AllNodes.push_back(N); 1456193323Sed return SDValue(N, 0); 1457193323Sed} 1458193323Sed 1459205218Srdivacky 1460207618SrdivackySDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1461243830Sdim int64_t Offset, 1462199989Srdivacky bool isTarget, 1463199989Srdivacky unsigned char TargetFlags) { 1464198892Srdivacky unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1465198892Srdivacky 1466198892Srdivacky FoldingSetNodeID ID; 1467199989Srdivacky AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1468198892Srdivacky ID.AddPointer(BA); 1469243830Sdim ID.AddInteger(Offset); 1470199989Srdivacky ID.AddInteger(TargetFlags); 1471198892Srdivacky void *IP = 0; 1472201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1473198892Srdivacky return SDValue(E, 0); 1474201360Srdivacky 1475243830Sdim SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset, 1476243830Sdim TargetFlags); 1477198892Srdivacky CSEMap.InsertNode(N, IP); 1478198892Srdivacky AllNodes.push_back(N); 1479198892Srdivacky return SDValue(N, 0); 1480198892Srdivacky} 1481198892Srdivacky 1482193323SedSDValue SelectionDAG::getSrcValue(const Value *V) { 1483204642Srdivacky assert((!V || V->getType()->isPointerTy()) && 1484193323Sed "SrcValue is not a pointer?"); 1485193323Sed 1486193323Sed FoldingSetNodeID ID; 1487193323Sed AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1488193323Sed ID.AddPointer(V); 1489193323Sed 1490193323Sed void *IP = 0; 1491201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1492193323Sed return SDValue(E, 0); 1493193323Sed 1494205407Srdivacky SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1495193323Sed CSEMap.InsertNode(N, IP); 1496193323Sed AllNodes.push_back(N); 1497193323Sed return SDValue(N, 0); 1498193323Sed} 1499193323Sed 1500207618Srdivacky/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1501207618SrdivackySDValue SelectionDAG::getMDNode(const MDNode *MD) { 1502207618Srdivacky FoldingSetNodeID ID; 1503207618Srdivacky AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1504207618Srdivacky ID.AddPointer(MD); 1505218893Sdim 1506207618Srdivacky void *IP = 0; 1507207618Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1508207618Srdivacky return SDValue(E, 0); 1509218893Sdim 1510207618Srdivacky SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1511207618Srdivacky CSEMap.InsertNode(N, IP); 1512207618Srdivacky AllNodes.push_back(N); 1513207618Srdivacky return SDValue(N, 0); 1514207618Srdivacky} 1515207618Srdivacky 1516207618Srdivacky 1517193323Sed/// getShiftAmountOperand - Return the specified value casted to 1518193323Sed/// the target's desired shift amount type. 1519221345SdimSDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1520198090Srdivacky EVT OpTy = Op.getValueType(); 1521249423Sdim EVT ShTy = TLI.getShiftAmountTy(LHSTy); 1522193323Sed if (OpTy == ShTy || OpTy.isVector()) return Op; 1523193323Sed 1524193323Sed ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1525193323Sed return getNode(Opcode, Op.getDebugLoc(), ShTy, Op); 1526193323Sed} 1527193323Sed 1528193323Sed/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1529193323Sed/// specified value type. 1530198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1531193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1532198090Srdivacky unsigned ByteSize = VT.getStoreSize(); 1533226633Sdim Type *Ty = VT.getTypeForEVT(*getContext()); 1534193323Sed unsigned StackAlign = 1535243830Sdim std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty), minAlign); 1536193323Sed 1537199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1538193323Sed return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1539193323Sed} 1540193323Sed 1541193323Sed/// CreateStackTemporary - Create a stack temporary suitable for holding 1542193323Sed/// either of the specified value types. 1543198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1544193323Sed unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1545193323Sed VT2.getStoreSizeInBits())/8; 1546226633Sdim Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1547226633Sdim Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1548243830Sdim const DataLayout *TD = TLI.getDataLayout(); 1549193323Sed unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1550193323Sed TD->getPrefTypeAlignment(Ty2)); 1551193323Sed 1552193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1553199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1554193323Sed return getFrameIndex(FrameIdx, TLI.getPointerTy()); 1555193323Sed} 1556193323Sed 1557198090SrdivackySDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1558193323Sed SDValue N2, ISD::CondCode Cond, DebugLoc dl) { 1559193323Sed // These setcc operations always fold. 1560193323Sed switch (Cond) { 1561193323Sed default: break; 1562193323Sed case ISD::SETFALSE: 1563193323Sed case ISD::SETFALSE2: return getConstant(0, VT); 1564193323Sed case ISD::SETTRUE: 1565193323Sed case ISD::SETTRUE2: return getConstant(1, VT); 1566193323Sed 1567193323Sed case ISD::SETOEQ: 1568193323Sed case ISD::SETOGT: 1569193323Sed case ISD::SETOGE: 1570193323Sed case ISD::SETOLT: 1571193323Sed case ISD::SETOLE: 1572193323Sed case ISD::SETONE: 1573193323Sed case ISD::SETO: 1574193323Sed case ISD::SETUO: 1575193323Sed case ISD::SETUEQ: 1576193323Sed case ISD::SETUNE: 1577193323Sed assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1578193323Sed break; 1579193323Sed } 1580193323Sed 1581193323Sed if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1582193323Sed const APInt &C2 = N2C->getAPIntValue(); 1583193323Sed if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1584193323Sed const APInt &C1 = N1C->getAPIntValue(); 1585193323Sed 1586193323Sed switch (Cond) { 1587198090Srdivacky default: llvm_unreachable("Unknown integer setcc!"); 1588193323Sed case ISD::SETEQ: return getConstant(C1 == C2, VT); 1589193323Sed case ISD::SETNE: return getConstant(C1 != C2, VT); 1590193323Sed case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1591193323Sed case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1592193323Sed case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1593193323Sed case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1594193323Sed case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1595193323Sed case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1596193323Sed case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1597193323Sed case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1598193323Sed } 1599193323Sed } 1600193323Sed } 1601193323Sed if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1602193323Sed if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1603193323Sed APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1604193323Sed switch (Cond) { 1605193323Sed default: break; 1606193323Sed case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1607193323Sed return getUNDEF(VT); 1608193323Sed // fall through 1609193323Sed case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1610193323Sed case ISD::SETNE: if (R==APFloat::cmpUnordered) 1611193323Sed return getUNDEF(VT); 1612193323Sed // fall through 1613193323Sed case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1614193323Sed R==APFloat::cmpLessThan, VT); 1615193323Sed case ISD::SETLT: if (R==APFloat::cmpUnordered) 1616193323Sed return getUNDEF(VT); 1617193323Sed // fall through 1618193323Sed case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1619193323Sed case ISD::SETGT: if (R==APFloat::cmpUnordered) 1620193323Sed return getUNDEF(VT); 1621193323Sed // fall through 1622193323Sed case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1623193323Sed case ISD::SETLE: if (R==APFloat::cmpUnordered) 1624193323Sed return getUNDEF(VT); 1625193323Sed // fall through 1626193323Sed case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1627193323Sed R==APFloat::cmpEqual, VT); 1628193323Sed case ISD::SETGE: if (R==APFloat::cmpUnordered) 1629193323Sed return getUNDEF(VT); 1630193323Sed // fall through 1631193323Sed case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1632193323Sed R==APFloat::cmpEqual, VT); 1633193323Sed case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1634193323Sed case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1635193323Sed case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1636193323Sed R==APFloat::cmpEqual, VT); 1637193323Sed case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1638193323Sed case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1639193323Sed R==APFloat::cmpLessThan, VT); 1640193323Sed case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1641193323Sed R==APFloat::cmpUnordered, VT); 1642193323Sed case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1643193323Sed case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1644193323Sed } 1645193323Sed } else { 1646193323Sed // Ensure that the constant occurs on the RHS. 1647193323Sed return getSetCC(dl, VT, N2, N1, ISD::getSetCCSwappedOperands(Cond)); 1648193323Sed } 1649193323Sed } 1650193323Sed 1651193323Sed // Could not fold it. 1652193323Sed return SDValue(); 1653193323Sed} 1654193323Sed 1655193323Sed/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1656193323Sed/// use this predicate to simplify operations downstream. 1657193323Sedbool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1658198090Srdivacky // This predicate is not safe for vector operations. 1659198090Srdivacky if (Op.getValueType().isVector()) 1660198090Srdivacky return false; 1661198090Srdivacky 1662200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1663193323Sed return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1664193323Sed} 1665193323Sed 1666193323Sed/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1667193323Sed/// this predicate to simplify operations downstream. Mask is known to be zero 1668193323Sed/// for bits that V cannot have. 1669193323Sedbool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1670193323Sed unsigned Depth) const { 1671193323Sed APInt KnownZero, KnownOne; 1672234353Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 1673193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1674193323Sed return (KnownZero & Mask) == Mask; 1675193323Sed} 1676193323Sed 1677193323Sed/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1678193323Sed/// known to be either zero or one and return them in the KnownZero/KnownOne 1679193323Sed/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1680193323Sed/// processing. 1681234353Sdimvoid SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, 1682234353Sdim APInt &KnownOne, unsigned Depth) const { 1683234353Sdim unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1684193323Sed 1685193323Sed KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1686234353Sdim if (Depth == 6) 1687193323Sed return; // Limit search depth. 1688193323Sed 1689193323Sed APInt KnownZero2, KnownOne2; 1690193323Sed 1691193323Sed switch (Op.getOpcode()) { 1692193323Sed case ISD::Constant: 1693193323Sed // We know all of the bits for a constant! 1694234353Sdim KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue(); 1695234353Sdim KnownZero = ~KnownOne; 1696193323Sed return; 1697193323Sed case ISD::AND: 1698193323Sed // If either the LHS or the RHS are Zero, the result is zero. 1699234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1700234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1701193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1702193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1703193323Sed 1704193323Sed // Output known-1 bits are only known if set in both the LHS & RHS. 1705193323Sed KnownOne &= KnownOne2; 1706193323Sed // Output known-0 are known to be clear if zero in either the LHS | RHS. 1707193323Sed KnownZero |= KnownZero2; 1708193323Sed return; 1709193323Sed case ISD::OR: 1710234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1711234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1712193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1713193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1714193323Sed 1715193323Sed // Output known-0 bits are only known if clear in both the LHS & RHS. 1716193323Sed KnownZero &= KnownZero2; 1717193323Sed // Output known-1 are known to be set if set in either the LHS | RHS. 1718193323Sed KnownOne |= KnownOne2; 1719193323Sed return; 1720193323Sed case ISD::XOR: { 1721234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1722234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1723193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1724193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1725193323Sed 1726193323Sed // Output known-0 bits are known if clear or set in both the LHS & RHS. 1727193323Sed APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1728193323Sed // Output known-1 are known to be set if set in only one of the LHS, RHS. 1729193323Sed KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1730193323Sed KnownZero = KnownZeroOut; 1731193323Sed return; 1732193323Sed } 1733193323Sed case ISD::MUL: { 1734234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1735234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1736193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1737193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1738193323Sed 1739193323Sed // If low bits are zero in either operand, output low known-0 bits. 1740193323Sed // Also compute a conserative estimate for high known-0 bits. 1741193323Sed // More trickiness is possible, but this is sufficient for the 1742193323Sed // interesting case of alignment computation. 1743218893Sdim KnownOne.clearAllBits(); 1744193323Sed unsigned TrailZ = KnownZero.countTrailingOnes() + 1745193323Sed KnownZero2.countTrailingOnes(); 1746193323Sed unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1747193323Sed KnownZero2.countLeadingOnes(), 1748193323Sed BitWidth) - BitWidth; 1749193323Sed 1750193323Sed TrailZ = std::min(TrailZ, BitWidth); 1751193323Sed LeadZ = std::min(LeadZ, BitWidth); 1752193323Sed KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1753193323Sed APInt::getHighBitsSet(BitWidth, LeadZ); 1754193323Sed return; 1755193323Sed } 1756193323Sed case ISD::UDIV: { 1757193323Sed // For the purposes of computing leading zeros we can conservatively 1758193323Sed // treat a udiv as a logical right shift by the power of 2 known to 1759193323Sed // be less than the denominator. 1760234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1761193323Sed unsigned LeadZ = KnownZero2.countLeadingOnes(); 1762193323Sed 1763218893Sdim KnownOne2.clearAllBits(); 1764218893Sdim KnownZero2.clearAllBits(); 1765234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1766193323Sed unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1767193323Sed if (RHSUnknownLeadingOnes != BitWidth) 1768193323Sed LeadZ = std::min(BitWidth, 1769193323Sed LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1770193323Sed 1771234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ); 1772193323Sed return; 1773193323Sed } 1774193323Sed case ISD::SELECT: 1775234353Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1); 1776234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1777193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1778193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1779193323Sed 1780193323Sed // Only known if known in both the LHS and RHS. 1781193323Sed KnownOne &= KnownOne2; 1782193323Sed KnownZero &= KnownZero2; 1783193323Sed return; 1784193323Sed case ISD::SELECT_CC: 1785234353Sdim ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1); 1786234353Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1); 1787193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1788193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1789193323Sed 1790193323Sed // Only known if known in both the LHS and RHS. 1791193323Sed KnownOne &= KnownOne2; 1792193323Sed KnownZero &= KnownZero2; 1793193323Sed return; 1794193323Sed case ISD::SADDO: 1795193323Sed case ISD::UADDO: 1796193323Sed case ISD::SSUBO: 1797193323Sed case ISD::USUBO: 1798193323Sed case ISD::SMULO: 1799193323Sed case ISD::UMULO: 1800193323Sed if (Op.getResNo() != 1) 1801193323Sed return; 1802193323Sed // The boolean result conforms to getBooleanContents. Fall through. 1803193323Sed case ISD::SETCC: 1804193323Sed // If we know the result of a setcc has the top bits zero, use this info. 1805226633Sdim if (TLI.getBooleanContents(Op.getValueType().isVector()) == 1806226633Sdim TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1807193323Sed KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1808193323Sed return; 1809193323Sed case ISD::SHL: 1810193323Sed // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1811193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1812193323Sed unsigned ShAmt = SA->getZExtValue(); 1813193323Sed 1814193323Sed // If the shift count is an invalid immediate, don't do anything. 1815193323Sed if (ShAmt >= BitWidth) 1816193323Sed return; 1817193323Sed 1818234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1819193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1820193323Sed KnownZero <<= ShAmt; 1821193323Sed KnownOne <<= ShAmt; 1822193323Sed // low bits known zero. 1823193323Sed KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1824193323Sed } 1825193323Sed return; 1826193323Sed case ISD::SRL: 1827193323Sed // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1828193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1829193323Sed unsigned ShAmt = SA->getZExtValue(); 1830193323Sed 1831193323Sed // If the shift count is an invalid immediate, don't do anything. 1832193323Sed if (ShAmt >= BitWidth) 1833193323Sed return; 1834193323Sed 1835234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1836193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1837193323Sed KnownZero = KnownZero.lshr(ShAmt); 1838193323Sed KnownOne = KnownOne.lshr(ShAmt); 1839193323Sed 1840234353Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1841193323Sed KnownZero |= HighBits; // High bits known zero. 1842193323Sed } 1843193323Sed return; 1844193323Sed case ISD::SRA: 1845193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1846193323Sed unsigned ShAmt = SA->getZExtValue(); 1847193323Sed 1848193323Sed // If the shift count is an invalid immediate, don't do anything. 1849193323Sed if (ShAmt >= BitWidth) 1850193323Sed return; 1851193323Sed 1852193323Sed // If any of the demanded bits are produced by the sign extension, we also 1853193323Sed // demand the input sign bit. 1854234353Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1855193323Sed 1856234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1857193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1858193323Sed KnownZero = KnownZero.lshr(ShAmt); 1859193323Sed KnownOne = KnownOne.lshr(ShAmt); 1860193323Sed 1861193323Sed // Handle the sign bits. 1862193323Sed APInt SignBit = APInt::getSignBit(BitWidth); 1863193323Sed SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1864193323Sed 1865193323Sed if (KnownZero.intersects(SignBit)) { 1866193323Sed KnownZero |= HighBits; // New bits are known zero. 1867193323Sed } else if (KnownOne.intersects(SignBit)) { 1868193323Sed KnownOne |= HighBits; // New bits are known one. 1869193323Sed } 1870193323Sed } 1871193323Sed return; 1872193323Sed case ISD::SIGN_EXTEND_INREG: { 1873198090Srdivacky EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1874202375Srdivacky unsigned EBits = EVT.getScalarType().getSizeInBits(); 1875193323Sed 1876193323Sed // Sign extension. Compute the demanded bits in the result that are not 1877193323Sed // present in the input. 1878234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 1879193323Sed 1880193323Sed APInt InSignBit = APInt::getSignBit(EBits); 1881234353Sdim APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 1882193323Sed 1883193323Sed // If the sign extended bits are demanded, we know that the sign 1884193323Sed // bit is demanded. 1885218893Sdim InSignBit = InSignBit.zext(BitWidth); 1886193323Sed if (NewBits.getBoolValue()) 1887193323Sed InputDemandedBits |= InSignBit; 1888193323Sed 1889234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1890234353Sdim KnownOne &= InputDemandedBits; 1891234353Sdim KnownZero &= InputDemandedBits; 1892193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1893193323Sed 1894193323Sed // If the sign bit of the input is known set or clear, then we know the 1895193323Sed // top bits of the result. 1896193323Sed if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1897193323Sed KnownZero |= NewBits; 1898193323Sed KnownOne &= ~NewBits; 1899193323Sed } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1900193323Sed KnownOne |= NewBits; 1901193323Sed KnownZero &= ~NewBits; 1902193323Sed } else { // Input sign bit unknown 1903193323Sed KnownZero &= ~NewBits; 1904193323Sed KnownOne &= ~NewBits; 1905193323Sed } 1906193323Sed return; 1907193323Sed } 1908193323Sed case ISD::CTTZ: 1909234353Sdim case ISD::CTTZ_ZERO_UNDEF: 1910193323Sed case ISD::CTLZ: 1911234353Sdim case ISD::CTLZ_ZERO_UNDEF: 1912193323Sed case ISD::CTPOP: { 1913193323Sed unsigned LowBits = Log2_32(BitWidth)+1; 1914193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 1915218893Sdim KnownOne.clearAllBits(); 1916193323Sed return; 1917193323Sed } 1918193323Sed case ISD::LOAD: { 1919234353Sdim LoadSDNode *LD = cast<LoadSDNode>(Op); 1920249423Sdim // If this is a ZEXTLoad and we are looking at the loaded value. 1921249423Sdim if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 1922198090Srdivacky EVT VT = LD->getMemoryVT(); 1923202375Srdivacky unsigned MemBits = VT.getScalarType().getSizeInBits(); 1924234353Sdim KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 1925234353Sdim } else if (const MDNode *Ranges = LD->getRanges()) { 1926234353Sdim computeMaskedBitsLoad(*Ranges, KnownZero); 1927193323Sed } 1928193323Sed return; 1929193323Sed } 1930193323Sed case ISD::ZERO_EXTEND: { 1931198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 1932200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 1933234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 1934218893Sdim KnownZero = KnownZero.trunc(InBits); 1935218893Sdim KnownOne = KnownOne.trunc(InBits); 1936234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1937218893Sdim KnownZero = KnownZero.zext(BitWidth); 1938218893Sdim KnownOne = KnownOne.zext(BitWidth); 1939193323Sed KnownZero |= NewBits; 1940193323Sed return; 1941193323Sed } 1942193323Sed case ISD::SIGN_EXTEND: { 1943198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 1944200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 1945193323Sed APInt InSignBit = APInt::getSignBit(InBits); 1946234353Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 1947193323Sed 1948218893Sdim KnownZero = KnownZero.trunc(InBits); 1949218893Sdim KnownOne = KnownOne.trunc(InBits); 1950234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1951193323Sed 1952193323Sed // Note if the sign bit is known to be zero or one. 1953193323Sed bool SignBitKnownZero = KnownZero.isNegative(); 1954193323Sed bool SignBitKnownOne = KnownOne.isNegative(); 1955193323Sed assert(!(SignBitKnownZero && SignBitKnownOne) && 1956193323Sed "Sign bit can't be known to be both zero and one!"); 1957193323Sed 1958218893Sdim KnownZero = KnownZero.zext(BitWidth); 1959218893Sdim KnownOne = KnownOne.zext(BitWidth); 1960193323Sed 1961193323Sed // If the sign bit is known zero or one, the top bits match. 1962193323Sed if (SignBitKnownZero) 1963193323Sed KnownZero |= NewBits; 1964193323Sed else if (SignBitKnownOne) 1965193323Sed KnownOne |= NewBits; 1966193323Sed return; 1967193323Sed } 1968193323Sed case ISD::ANY_EXTEND: { 1969198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 1970200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 1971218893Sdim KnownZero = KnownZero.trunc(InBits); 1972218893Sdim KnownOne = KnownOne.trunc(InBits); 1973234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1974218893Sdim KnownZero = KnownZero.zext(BitWidth); 1975218893Sdim KnownOne = KnownOne.zext(BitWidth); 1976193323Sed return; 1977193323Sed } 1978193323Sed case ISD::TRUNCATE: { 1979198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 1980200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 1981218893Sdim KnownZero = KnownZero.zext(InBits); 1982218893Sdim KnownOne = KnownOne.zext(InBits); 1983234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1984193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1985218893Sdim KnownZero = KnownZero.trunc(BitWidth); 1986218893Sdim KnownOne = KnownOne.trunc(BitWidth); 1987193323Sed break; 1988193323Sed } 1989193323Sed case ISD::AssertZext: { 1990198090Srdivacky EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1991193323Sed APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 1992234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1993234353Sdim KnownZero |= (~InMask); 1994239462Sdim KnownOne &= (~KnownZero); 1995193323Sed return; 1996193323Sed } 1997193323Sed case ISD::FGETSIGN: 1998193323Sed // All bits are zero except the low bit. 1999193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 2000193323Sed return; 2001193323Sed 2002193323Sed case ISD::SUB: { 2003193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 2004193323Sed // We know that the top bits of C-X are clear if X contains less bits 2005193323Sed // than C (i.e. no wrap-around can happen). For example, 20-X is 2006193323Sed // positive if we can prove that X is >= 0 and < 16. 2007193323Sed if (CLHS->getAPIntValue().isNonNegative()) { 2008193323Sed unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 2009193323Sed // NLZ can't be BitWidth with no sign bit 2010193323Sed APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 2011234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2012193323Sed 2013193323Sed // If all of the MaskV bits are known to be zero, then we know the 2014193323Sed // output top bits are zero, because we now know that the output is 2015193323Sed // from [0-C]. 2016193323Sed if ((KnownZero2 & MaskV) == MaskV) { 2017193323Sed unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 2018193323Sed // Top bits known zero. 2019234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2); 2020193323Sed } 2021193323Sed } 2022193323Sed } 2023193323Sed } 2024193323Sed // fall through 2025218893Sdim case ISD::ADD: 2026218893Sdim case ISD::ADDE: { 2027193323Sed // Output known-0 bits are known if clear or set in both the low clear bits 2028193323Sed // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2029193323Sed // low 3 bits clear. 2030234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 2031193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2032193323Sed unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 2033193323Sed 2034234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2035193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2036193323Sed KnownZeroOut = std::min(KnownZeroOut, 2037193323Sed KnownZero2.countTrailingOnes()); 2038193323Sed 2039218893Sdim if (Op.getOpcode() == ISD::ADD) { 2040218893Sdim KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2041218893Sdim return; 2042218893Sdim } 2043218893Sdim 2044218893Sdim // With ADDE, a carry bit may be added in, so we can only use this 2045218893Sdim // information if we know (at least) that the low two bits are clear. We 2046218893Sdim // then return to the caller that the low bit is unknown but that other bits 2047218893Sdim // are known zero. 2048218893Sdim if (KnownZeroOut >= 2) // ADDE 2049218893Sdim KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2050193323Sed return; 2051193323Sed } 2052193323Sed case ISD::SREM: 2053193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2054203954Srdivacky const APInt &RA = Rem->getAPIntValue().abs(); 2055203954Srdivacky if (RA.isPowerOf2()) { 2056203954Srdivacky APInt LowBits = RA - 1; 2057193323Sed APInt Mask2 = LowBits | APInt::getSignBit(BitWidth); 2058234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1); 2059193323Sed 2060203954Srdivacky // The low bits of the first operand are unchanged by the srem. 2061203954Srdivacky KnownZero = KnownZero2 & LowBits; 2062203954Srdivacky KnownOne = KnownOne2 & LowBits; 2063203954Srdivacky 2064203954Srdivacky // If the first operand is non-negative or has all low bits zero, then 2065203954Srdivacky // the upper bits are all zero. 2066193323Sed if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2067203954Srdivacky KnownZero |= ~LowBits; 2068193323Sed 2069203954Srdivacky // If the first operand is negative and not all low bits are zero, then 2070203954Srdivacky // the upper bits are all one. 2071203954Srdivacky if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2072203954Srdivacky KnownOne |= ~LowBits; 2073193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2074193323Sed } 2075193323Sed } 2076193323Sed return; 2077193323Sed case ISD::UREM: { 2078193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2079193323Sed const APInt &RA = Rem->getAPIntValue(); 2080193323Sed if (RA.isPowerOf2()) { 2081193323Sed APInt LowBits = (RA - 1); 2082234353Sdim KnownZero |= ~LowBits; 2083234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1); 2084193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2085193323Sed break; 2086193323Sed } 2087193323Sed } 2088193323Sed 2089193323Sed // Since the result is less than or equal to either operand, any leading 2090193323Sed // zero bits in either operand must also exist in the result. 2091234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2092234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2093193323Sed 2094193323Sed uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2095193323Sed KnownZero2.countLeadingOnes()); 2096218893Sdim KnownOne.clearAllBits(); 2097234353Sdim KnownZero = APInt::getHighBitsSet(BitWidth, Leaders); 2098193323Sed return; 2099193323Sed } 2100218893Sdim case ISD::FrameIndex: 2101218893Sdim case ISD::TargetFrameIndex: 2102218893Sdim if (unsigned Align = InferPtrAlignment(Op)) { 2103218893Sdim // The low bits are known zero if the pointer is aligned. 2104218893Sdim KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2105218893Sdim return; 2106218893Sdim } 2107218893Sdim break; 2108219077Sdim 2109193323Sed default: 2110223017Sdim if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2111223017Sdim break; 2112223017Sdim // Fallthrough 2113193323Sed case ISD::INTRINSIC_WO_CHAIN: 2114193323Sed case ISD::INTRINSIC_W_CHAIN: 2115193323Sed case ISD::INTRINSIC_VOID: 2116223017Sdim // Allow the target to implement this method for its nodes. 2117234353Sdim TLI.computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth); 2118193323Sed return; 2119193323Sed } 2120193323Sed} 2121193323Sed 2122193323Sed/// ComputeNumSignBits - Return the number of times the sign bit of the 2123193323Sed/// register is replicated into the other bits. We know that at least 1 bit 2124193323Sed/// is always equal to the sign bit (itself), but other cases can give us 2125193323Sed/// information. For example, immediately after an "SRA X, 2", we know that 2126193323Sed/// the top 3 bits are all equal to each other, so we return 3. 2127193323Sedunsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2128198090Srdivacky EVT VT = Op.getValueType(); 2129193323Sed assert(VT.isInteger() && "Invalid VT!"); 2130200581Srdivacky unsigned VTBits = VT.getScalarType().getSizeInBits(); 2131193323Sed unsigned Tmp, Tmp2; 2132193323Sed unsigned FirstAnswer = 1; 2133193323Sed 2134193323Sed if (Depth == 6) 2135193323Sed return 1; // Limit search depth. 2136193323Sed 2137193323Sed switch (Op.getOpcode()) { 2138193323Sed default: break; 2139193323Sed case ISD::AssertSext: 2140193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2141193323Sed return VTBits-Tmp+1; 2142193323Sed case ISD::AssertZext: 2143193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2144193323Sed return VTBits-Tmp; 2145193323Sed 2146193323Sed case ISD::Constant: { 2147193323Sed const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2148219077Sdim return Val.getNumSignBits(); 2149193323Sed } 2150193323Sed 2151193323Sed case ISD::SIGN_EXTEND: 2152200581Srdivacky Tmp = VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2153193323Sed return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2154193323Sed 2155193323Sed case ISD::SIGN_EXTEND_INREG: 2156193323Sed // Max of the input and what this extends. 2157202375Srdivacky Tmp = 2158202375Srdivacky cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2159193323Sed Tmp = VTBits-Tmp+1; 2160193323Sed 2161193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2162193323Sed return std::max(Tmp, Tmp2); 2163193323Sed 2164193323Sed case ISD::SRA: 2165193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2166193323Sed // SRA X, C -> adds C sign bits. 2167193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2168193323Sed Tmp += C->getZExtValue(); 2169193323Sed if (Tmp > VTBits) Tmp = VTBits; 2170193323Sed } 2171193323Sed return Tmp; 2172193323Sed case ISD::SHL: 2173193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2174193323Sed // shl destroys sign bits. 2175193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2176193323Sed if (C->getZExtValue() >= VTBits || // Bad shift. 2177193323Sed C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2178193323Sed return Tmp - C->getZExtValue(); 2179193323Sed } 2180193323Sed break; 2181193323Sed case ISD::AND: 2182193323Sed case ISD::OR: 2183193323Sed case ISD::XOR: // NOT is handled here. 2184193323Sed // Logical binary ops preserve the number of sign bits at the worst. 2185193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2186193323Sed if (Tmp != 1) { 2187193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2188193323Sed FirstAnswer = std::min(Tmp, Tmp2); 2189193323Sed // We computed what we know about the sign bits as our first 2190193323Sed // answer. Now proceed to the generic code that uses 2191193323Sed // ComputeMaskedBits, and pick whichever answer is better. 2192193323Sed } 2193193323Sed break; 2194193323Sed 2195193323Sed case ISD::SELECT: 2196193323Sed Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2197193323Sed if (Tmp == 1) return 1; // Early out. 2198193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2199193323Sed return std::min(Tmp, Tmp2); 2200193323Sed 2201193323Sed case ISD::SADDO: 2202193323Sed case ISD::UADDO: 2203193323Sed case ISD::SSUBO: 2204193323Sed case ISD::USUBO: 2205193323Sed case ISD::SMULO: 2206193323Sed case ISD::UMULO: 2207193323Sed if (Op.getResNo() != 1) 2208193323Sed break; 2209193323Sed // The boolean result conforms to getBooleanContents. Fall through. 2210193323Sed case ISD::SETCC: 2211193323Sed // If setcc returns 0/-1, all bits are sign bits. 2212226633Sdim if (TLI.getBooleanContents(Op.getValueType().isVector()) == 2213193323Sed TargetLowering::ZeroOrNegativeOneBooleanContent) 2214193323Sed return VTBits; 2215193323Sed break; 2216193323Sed case ISD::ROTL: 2217193323Sed case ISD::ROTR: 2218193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2219193323Sed unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2220193323Sed 2221193323Sed // Handle rotate right by N like a rotate left by 32-N. 2222193323Sed if (Op.getOpcode() == ISD::ROTR) 2223193323Sed RotAmt = (VTBits-RotAmt) & (VTBits-1); 2224193323Sed 2225193323Sed // If we aren't rotating out all of the known-in sign bits, return the 2226193323Sed // number that are left. This handles rotl(sext(x), 1) for example. 2227193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2228193323Sed if (Tmp > RotAmt+1) return Tmp-RotAmt; 2229193323Sed } 2230193323Sed break; 2231193323Sed case ISD::ADD: 2232193323Sed // Add can have at most one carry bit. Thus we know that the output 2233193323Sed // is, at worst, one more bit than the inputs. 2234193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2235193323Sed if (Tmp == 1) return 1; // Early out. 2236193323Sed 2237193323Sed // Special case decrementing a value (ADD X, -1): 2238193323Sed if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2239193323Sed if (CRHS->isAllOnesValue()) { 2240193323Sed APInt KnownZero, KnownOne; 2241234353Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2242193323Sed 2243193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2244193323Sed // sign bits set. 2245234353Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2246193323Sed return VTBits; 2247193323Sed 2248193323Sed // If we are subtracting one from a positive number, there is no carry 2249193323Sed // out of the result. 2250193323Sed if (KnownZero.isNegative()) 2251193323Sed return Tmp; 2252193323Sed } 2253193323Sed 2254193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2255193323Sed if (Tmp2 == 1) return 1; 2256234353Sdim return std::min(Tmp, Tmp2)-1; 2257193323Sed 2258193323Sed case ISD::SUB: 2259193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2260193323Sed if (Tmp2 == 1) return 1; 2261193323Sed 2262193323Sed // Handle NEG. 2263193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2264193323Sed if (CLHS->isNullValue()) { 2265193323Sed APInt KnownZero, KnownOne; 2266234353Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 2267193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2268193323Sed // sign bits set. 2269234353Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2270193323Sed return VTBits; 2271193323Sed 2272193323Sed // If the input is known to be positive (the sign bit is known clear), 2273193323Sed // the output of the NEG has the same number of sign bits as the input. 2274193323Sed if (KnownZero.isNegative()) 2275193323Sed return Tmp2; 2276193323Sed 2277193323Sed // Otherwise, we treat this like a SUB. 2278193323Sed } 2279193323Sed 2280193323Sed // Sub can have at most one carry bit. Thus we know that the output 2281193323Sed // is, at worst, one more bit than the inputs. 2282193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2283193323Sed if (Tmp == 1) return 1; // Early out. 2284234353Sdim return std::min(Tmp, Tmp2)-1; 2285193323Sed case ISD::TRUNCATE: 2286193323Sed // FIXME: it's tricky to do anything useful for this, but it is an important 2287193323Sed // case for targets like X86. 2288193323Sed break; 2289193323Sed } 2290193323Sed 2291249423Sdim // If we are looking at the loaded value of the SDNode. 2292249423Sdim if (Op.getResNo() == 0) { 2293249423Sdim // Handle LOADX separately here. EXTLOAD case will fallthrough. 2294249423Sdim if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 2295249423Sdim unsigned ExtType = LD->getExtensionType(); 2296249423Sdim switch (ExtType) { 2297249423Sdim default: break; 2298249423Sdim case ISD::SEXTLOAD: // '17' bits known 2299249423Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2300249423Sdim return VTBits-Tmp+1; 2301249423Sdim case ISD::ZEXTLOAD: // '16' bits known 2302249423Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2303249423Sdim return VTBits-Tmp; 2304249423Sdim } 2305193323Sed } 2306193323Sed } 2307193323Sed 2308193323Sed // Allow the target to implement this method for its nodes. 2309193323Sed if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2310193323Sed Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2311193323Sed Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2312193323Sed Op.getOpcode() == ISD::INTRINSIC_VOID) { 2313193323Sed unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth); 2314193323Sed if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2315193323Sed } 2316193323Sed 2317193323Sed // Finally, if we can prove that the top bits of the result are 0's or 1's, 2318193323Sed // use this information. 2319193323Sed APInt KnownZero, KnownOne; 2320234353Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 2321193323Sed 2322234353Sdim APInt Mask; 2323193323Sed if (KnownZero.isNegative()) { // sign bit is 0 2324193323Sed Mask = KnownZero; 2325193323Sed } else if (KnownOne.isNegative()) { // sign bit is 1; 2326193323Sed Mask = KnownOne; 2327193323Sed } else { 2328193323Sed // Nothing known. 2329193323Sed return FirstAnswer; 2330193323Sed } 2331193323Sed 2332193323Sed // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2333193323Sed // the number of identical bits in the top of the input value. 2334193323Sed Mask = ~Mask; 2335193323Sed Mask <<= Mask.getBitWidth()-VTBits; 2336193323Sed // Return # leading zeros. We use 'min' here in case Val was zero before 2337193323Sed // shifting. We don't want to return '64' as for an i32 "0". 2338193323Sed return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2339193323Sed} 2340193323Sed 2341218893Sdim/// isBaseWithConstantOffset - Return true if the specified operand is an 2342218893Sdim/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2343218893Sdim/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2344218893Sdim/// semantics as an ADD. This handles the equivalence: 2345218893Sdim/// X|Cst == X+Cst iff X&Cst = 0. 2346218893Sdimbool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2347218893Sdim if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2348218893Sdim !isa<ConstantSDNode>(Op.getOperand(1))) 2349218893Sdim return false; 2350219077Sdim 2351219077Sdim if (Op.getOpcode() == ISD::OR && 2352218893Sdim !MaskedValueIsZero(Op.getOperand(0), 2353218893Sdim cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2354218893Sdim return false; 2355219077Sdim 2356218893Sdim return true; 2357218893Sdim} 2358218893Sdim 2359218893Sdim 2360198090Srdivackybool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2361198090Srdivacky // If we're told that NaNs won't happen, assume they won't. 2362234353Sdim if (getTarget().Options.NoNaNsFPMath) 2363198090Srdivacky return true; 2364193323Sed 2365198090Srdivacky // If the value is a constant, we can obviously see if it is a NaN or not. 2366198090Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2367198090Srdivacky return !C->getValueAPF().isNaN(); 2368198090Srdivacky 2369198090Srdivacky // TODO: Recognize more cases here. 2370198090Srdivacky 2371198090Srdivacky return false; 2372198090Srdivacky} 2373198090Srdivacky 2374204642Srdivackybool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2375204642Srdivacky // If the value is a constant, we can obviously see if it is a zero or not. 2376204642Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2377204642Srdivacky return !C->isZero(); 2378204642Srdivacky 2379204642Srdivacky // TODO: Recognize more cases here. 2380223017Sdim switch (Op.getOpcode()) { 2381223017Sdim default: break; 2382223017Sdim case ISD::OR: 2383223017Sdim if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2384223017Sdim return !C->isNullValue(); 2385223017Sdim break; 2386223017Sdim } 2387204642Srdivacky 2388204642Srdivacky return false; 2389204642Srdivacky} 2390204642Srdivacky 2391204642Srdivackybool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2392204642Srdivacky // Check the obvious case. 2393204642Srdivacky if (A == B) return true; 2394204642Srdivacky 2395204642Srdivacky // For for negative and positive zero. 2396204642Srdivacky if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2397204642Srdivacky if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2398204642Srdivacky if (CA->isZero() && CB->isZero()) return true; 2399204642Srdivacky 2400204642Srdivacky // Otherwise they may not be equal. 2401204642Srdivacky return false; 2402204642Srdivacky} 2403204642Srdivacky 2404193323Sed/// getNode - Gets or creates the specified node. 2405193323Sed/// 2406198090SrdivackySDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { 2407193323Sed FoldingSetNodeID ID; 2408193323Sed AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2409193323Sed void *IP = 0; 2410201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2411193323Sed return SDValue(E, 0); 2412201360Srdivacky 2413205407Srdivacky SDNode *N = new (NodeAllocator) SDNode(Opcode, DL, getVTList(VT)); 2414193323Sed CSEMap.InsertNode(N, IP); 2415193323Sed 2416193323Sed AllNodes.push_back(N); 2417193323Sed#ifndef NDEBUG 2418218893Sdim VerifySDNode(N); 2419193323Sed#endif 2420193323Sed return SDValue(N, 0); 2421193323Sed} 2422193323Sed 2423193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 2424198090Srdivacky EVT VT, SDValue Operand) { 2425193323Sed // Constant fold unary operations with an integer constant operand. 2426193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2427193323Sed const APInt &Val = C->getAPIntValue(); 2428193323Sed switch (Opcode) { 2429193323Sed default: break; 2430193323Sed case ISD::SIGN_EXTEND: 2431218893Sdim return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2432193323Sed case ISD::ANY_EXTEND: 2433193323Sed case ISD::ZERO_EXTEND: 2434193323Sed case ISD::TRUNCATE: 2435218893Sdim return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2436193323Sed case ISD::UINT_TO_FP: 2437193323Sed case ISD::SINT_TO_FP: { 2438249423Sdim APFloat apf(EVTToAPFloatSemantics(VT), 2439249423Sdim APInt::getNullValue(VT.getSizeInBits())); 2440193323Sed (void)apf.convertFromAPInt(Val, 2441193323Sed Opcode==ISD::SINT_TO_FP, 2442193323Sed APFloat::rmNearestTiesToEven); 2443193323Sed return getConstantFP(apf, VT); 2444193323Sed } 2445218893Sdim case ISD::BITCAST: 2446193323Sed if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2447249423Sdim return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); 2448193323Sed else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2449249423Sdim return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); 2450193323Sed break; 2451193323Sed case ISD::BSWAP: 2452193323Sed return getConstant(Val.byteSwap(), VT); 2453193323Sed case ISD::CTPOP: 2454193323Sed return getConstant(Val.countPopulation(), VT); 2455193323Sed case ISD::CTLZ: 2456234353Sdim case ISD::CTLZ_ZERO_UNDEF: 2457193323Sed return getConstant(Val.countLeadingZeros(), VT); 2458193323Sed case ISD::CTTZ: 2459234353Sdim case ISD::CTTZ_ZERO_UNDEF: 2460193323Sed return getConstant(Val.countTrailingZeros(), VT); 2461193323Sed } 2462193323Sed } 2463193323Sed 2464193323Sed // Constant fold unary operations with a floating point constant operand. 2465193323Sed if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2466193323Sed APFloat V = C->getValueAPF(); // make copy 2467243830Sdim switch (Opcode) { 2468243830Sdim case ISD::FNEG: 2469243830Sdim V.changeSign(); 2470243830Sdim return getConstantFP(V, VT); 2471243830Sdim case ISD::FABS: 2472243830Sdim V.clearSign(); 2473243830Sdim return getConstantFP(V, VT); 2474243830Sdim case ISD::FCEIL: { 2475243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 2476243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2477193323Sed return getConstantFP(V, VT); 2478243830Sdim break; 2479243830Sdim } 2480243830Sdim case ISD::FTRUNC: { 2481243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 2482243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2483193323Sed return getConstantFP(V, VT); 2484243830Sdim break; 2485243830Sdim } 2486243830Sdim case ISD::FFLOOR: { 2487243830Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 2488243830Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2489193323Sed return getConstantFP(V, VT); 2490243830Sdim break; 2491243830Sdim } 2492243830Sdim case ISD::FP_EXTEND: { 2493243830Sdim bool ignored; 2494243830Sdim // This can return overflow, underflow, or inexact; we don't care. 2495243830Sdim // FIXME need to be more flexible about rounding mode. 2496249423Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 2497243830Sdim APFloat::rmNearestTiesToEven, &ignored); 2498243830Sdim return getConstantFP(V, VT); 2499243830Sdim } 2500243830Sdim case ISD::FP_TO_SINT: 2501243830Sdim case ISD::FP_TO_UINT: { 2502243830Sdim integerPart x[2]; 2503243830Sdim bool ignored; 2504243830Sdim assert(integerPartWidth >= 64); 2505243830Sdim // FIXME need to be more flexible about rounding mode. 2506243830Sdim APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2507243830Sdim Opcode==ISD::FP_TO_SINT, 2508243830Sdim APFloat::rmTowardZero, &ignored); 2509243830Sdim if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2510193323Sed break; 2511243830Sdim APInt api(VT.getSizeInBits(), x); 2512243830Sdim return getConstant(api, VT); 2513193323Sed } 2514243830Sdim case ISD::BITCAST: 2515243830Sdim if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2516243830Sdim return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2517243830Sdim else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2518243830Sdim return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2519243830Sdim break; 2520243830Sdim } 2521193323Sed } 2522193323Sed 2523193323Sed unsigned OpOpcode = Operand.getNode()->getOpcode(); 2524193323Sed switch (Opcode) { 2525193323Sed case ISD::TokenFactor: 2526193323Sed case ISD::MERGE_VALUES: 2527193323Sed case ISD::CONCAT_VECTORS: 2528193323Sed return Operand; // Factor, merge or concat of one node? No need. 2529198090Srdivacky case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2530193323Sed case ISD::FP_EXTEND: 2531193323Sed assert(VT.isFloatingPoint() && 2532193323Sed Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2533193323Sed if (Operand.getValueType() == VT) return Operand; // noop conversion. 2534200581Srdivacky assert((!VT.isVector() || 2535200581Srdivacky VT.getVectorNumElements() == 2536200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2537200581Srdivacky "Vector element count mismatch!"); 2538193323Sed if (Operand.getOpcode() == ISD::UNDEF) 2539193323Sed return getUNDEF(VT); 2540193323Sed break; 2541193323Sed case ISD::SIGN_EXTEND: 2542193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2543193323Sed "Invalid SIGN_EXTEND!"); 2544193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2545200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2546200581Srdivacky "Invalid sext node, dst < src!"); 2547200581Srdivacky assert((!VT.isVector() || 2548200581Srdivacky VT.getVectorNumElements() == 2549200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2550200581Srdivacky "Vector element count mismatch!"); 2551193323Sed if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2552193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2553221345Sdim else if (OpOpcode == ISD::UNDEF) 2554221345Sdim // sext(undef) = 0, because the top bits will all be the same. 2555221345Sdim return getConstant(0, VT); 2556193323Sed break; 2557193323Sed case ISD::ZERO_EXTEND: 2558193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2559193323Sed "Invalid ZERO_EXTEND!"); 2560193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2561200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2562200581Srdivacky "Invalid zext node, dst < src!"); 2563200581Srdivacky assert((!VT.isVector() || 2564200581Srdivacky VT.getVectorNumElements() == 2565200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2566200581Srdivacky "Vector element count mismatch!"); 2567193323Sed if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2568193323Sed return getNode(ISD::ZERO_EXTEND, DL, VT, 2569193323Sed Operand.getNode()->getOperand(0)); 2570221345Sdim else if (OpOpcode == ISD::UNDEF) 2571221345Sdim // zext(undef) = 0, because the top bits will be zero. 2572221345Sdim return getConstant(0, VT); 2573193323Sed break; 2574193323Sed case ISD::ANY_EXTEND: 2575193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2576193323Sed "Invalid ANY_EXTEND!"); 2577193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2578200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2579200581Srdivacky "Invalid anyext node, dst < src!"); 2580200581Srdivacky assert((!VT.isVector() || 2581200581Srdivacky VT.getVectorNumElements() == 2582200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2583200581Srdivacky "Vector element count mismatch!"); 2584210299Sed 2585210299Sed if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2586210299Sed OpOpcode == ISD::ANY_EXTEND) 2587193323Sed // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2588193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2589221345Sdim else if (OpOpcode == ISD::UNDEF) 2590221345Sdim return getUNDEF(VT); 2591210299Sed 2592210299Sed // (ext (trunx x)) -> x 2593210299Sed if (OpOpcode == ISD::TRUNCATE) { 2594210299Sed SDValue OpOp = Operand.getNode()->getOperand(0); 2595210299Sed if (OpOp.getValueType() == VT) 2596210299Sed return OpOp; 2597210299Sed } 2598193323Sed break; 2599193323Sed case ISD::TRUNCATE: 2600193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2601193323Sed "Invalid TRUNCATE!"); 2602193323Sed if (Operand.getValueType() == VT) return Operand; // noop truncate 2603200581Srdivacky assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2604200581Srdivacky "Invalid truncate node, src < dst!"); 2605200581Srdivacky assert((!VT.isVector() || 2606200581Srdivacky VT.getVectorNumElements() == 2607200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2608200581Srdivacky "Vector element count mismatch!"); 2609193323Sed if (OpOpcode == ISD::TRUNCATE) 2610193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2611234353Sdim if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2612234353Sdim OpOpcode == ISD::ANY_EXTEND) { 2613193323Sed // If the source is smaller than the dest, we still need an extend. 2614200581Srdivacky if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2615200581Srdivacky .bitsLT(VT.getScalarType())) 2616193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2617234353Sdim if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2618193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2619234353Sdim return Operand.getNode()->getOperand(0); 2620193323Sed } 2621234353Sdim if (OpOpcode == ISD::UNDEF) 2622234353Sdim return getUNDEF(VT); 2623193323Sed break; 2624218893Sdim case ISD::BITCAST: 2625193323Sed // Basic sanity checking. 2626193323Sed assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2627218893Sdim && "Cannot BITCAST between types of different sizes!"); 2628193323Sed if (VT == Operand.getValueType()) return Operand; // noop conversion. 2629218893Sdim if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2630218893Sdim return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2631193323Sed if (OpOpcode == ISD::UNDEF) 2632193323Sed return getUNDEF(VT); 2633193323Sed break; 2634193323Sed case ISD::SCALAR_TO_VECTOR: 2635193323Sed assert(VT.isVector() && !Operand.getValueType().isVector() && 2636193323Sed (VT.getVectorElementType() == Operand.getValueType() || 2637193323Sed (VT.getVectorElementType().isInteger() && 2638193323Sed Operand.getValueType().isInteger() && 2639193323Sed VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2640193323Sed "Illegal SCALAR_TO_VECTOR node!"); 2641193323Sed if (OpOpcode == ISD::UNDEF) 2642193323Sed return getUNDEF(VT); 2643193323Sed // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2644193323Sed if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2645193323Sed isa<ConstantSDNode>(Operand.getOperand(1)) && 2646193323Sed Operand.getConstantOperandVal(1) == 0 && 2647193323Sed Operand.getOperand(0).getValueType() == VT) 2648193323Sed return Operand.getOperand(0); 2649193323Sed break; 2650193323Sed case ISD::FNEG: 2651193323Sed // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2652234353Sdim if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2653193323Sed return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2654193323Sed Operand.getNode()->getOperand(0)); 2655193323Sed if (OpOpcode == ISD::FNEG) // --X -> X 2656193323Sed return Operand.getNode()->getOperand(0); 2657193323Sed break; 2658193323Sed case ISD::FABS: 2659193323Sed if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2660193323Sed return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2661193323Sed break; 2662193323Sed } 2663193323Sed 2664193323Sed SDNode *N; 2665193323Sed SDVTList VTs = getVTList(VT); 2666218893Sdim if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2667193323Sed FoldingSetNodeID ID; 2668193323Sed SDValue Ops[1] = { Operand }; 2669193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2670193323Sed void *IP = 0; 2671201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2672193323Sed return SDValue(E, 0); 2673201360Srdivacky 2674205407Srdivacky N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2675193323Sed CSEMap.InsertNode(N, IP); 2676193323Sed } else { 2677205407Srdivacky N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); 2678193323Sed } 2679193323Sed 2680193323Sed AllNodes.push_back(N); 2681193323Sed#ifndef NDEBUG 2682218893Sdim VerifySDNode(N); 2683193323Sed#endif 2684193323Sed return SDValue(N, 0); 2685193323Sed} 2686193323Sed 2687249423SdimSDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, 2688249423Sdim SDNode *Cst1, SDNode *Cst2) { 2689249423Sdim SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; 2690249423Sdim SmallVector<SDValue, 4> Outputs; 2691249423Sdim EVT SVT = VT.getScalarType(); 2692193323Sed 2693249423Sdim ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); 2694249423Sdim ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); 2695249423Sdim if (Scalar1 && Scalar2) { 2696249423Sdim // Scalar instruction. 2697249423Sdim Inputs.push_back(std::make_pair(Scalar1, Scalar2)); 2698249423Sdim } else { 2699249423Sdim // For vectors extract each constant element into Inputs so we can constant 2700249423Sdim // fold them individually. 2701249423Sdim BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 2702249423Sdim BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 2703249423Sdim if (!BV1 || !BV2) 2704249423Sdim return SDValue(); 2705249423Sdim 2706249423Sdim assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 2707249423Sdim 2708249423Sdim for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 2709249423Sdim ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); 2710249423Sdim ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); 2711249423Sdim if (!V1 || !V2) // Not a constant, bail. 2712249423Sdim return SDValue(); 2713249423Sdim 2714249423Sdim // Avoid BUILD_VECTOR nodes that perform implicit truncation. 2715249423Sdim // FIXME: This is valid and could be handled by truncating the APInts. 2716249423Sdim if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 2717249423Sdim return SDValue(); 2718249423Sdim 2719249423Sdim Inputs.push_back(std::make_pair(V1, V2)); 2720249423Sdim } 2721193323Sed } 2722193323Sed 2723249423Sdim // We have a number of constant values, constant fold them element by element. 2724249423Sdim for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { 2725249423Sdim const APInt &C1 = Inputs[I].first->getAPIntValue(); 2726249423Sdim const APInt &C2 = Inputs[I].second->getAPIntValue(); 2727249423Sdim 2728249423Sdim switch (Opcode) { 2729249423Sdim case ISD::ADD: 2730249423Sdim Outputs.push_back(getConstant(C1 + C2, SVT)); 2731249423Sdim break; 2732249423Sdim case ISD::SUB: 2733249423Sdim Outputs.push_back(getConstant(C1 - C2, SVT)); 2734249423Sdim break; 2735249423Sdim case ISD::MUL: 2736249423Sdim Outputs.push_back(getConstant(C1 * C2, SVT)); 2737249423Sdim break; 2738249423Sdim case ISD::UDIV: 2739249423Sdim if (!C2.getBoolValue()) 2740249423Sdim return SDValue(); 2741249423Sdim Outputs.push_back(getConstant(C1.udiv(C2), SVT)); 2742249423Sdim break; 2743249423Sdim case ISD::UREM: 2744249423Sdim if (!C2.getBoolValue()) 2745249423Sdim return SDValue(); 2746249423Sdim Outputs.push_back(getConstant(C1.urem(C2), SVT)); 2747249423Sdim break; 2748249423Sdim case ISD::SDIV: 2749249423Sdim if (!C2.getBoolValue()) 2750249423Sdim return SDValue(); 2751249423Sdim Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); 2752249423Sdim break; 2753249423Sdim case ISD::SREM: 2754249423Sdim if (!C2.getBoolValue()) 2755249423Sdim return SDValue(); 2756249423Sdim Outputs.push_back(getConstant(C1.srem(C2), SVT)); 2757249423Sdim break; 2758249423Sdim case ISD::AND: 2759249423Sdim Outputs.push_back(getConstant(C1 & C2, SVT)); 2760249423Sdim break; 2761249423Sdim case ISD::OR: 2762249423Sdim Outputs.push_back(getConstant(C1 | C2, SVT)); 2763249423Sdim break; 2764249423Sdim case ISD::XOR: 2765249423Sdim Outputs.push_back(getConstant(C1 ^ C2, SVT)); 2766249423Sdim break; 2767249423Sdim case ISD::SHL: 2768249423Sdim Outputs.push_back(getConstant(C1 << C2, SVT)); 2769249423Sdim break; 2770249423Sdim case ISD::SRL: 2771249423Sdim Outputs.push_back(getConstant(C1.lshr(C2), SVT)); 2772249423Sdim break; 2773249423Sdim case ISD::SRA: 2774249423Sdim Outputs.push_back(getConstant(C1.ashr(C2), SVT)); 2775249423Sdim break; 2776249423Sdim case ISD::ROTL: 2777249423Sdim Outputs.push_back(getConstant(C1.rotl(C2), SVT)); 2778249423Sdim break; 2779249423Sdim case ISD::ROTR: 2780249423Sdim Outputs.push_back(getConstant(C1.rotr(C2), SVT)); 2781249423Sdim break; 2782249423Sdim default: 2783249423Sdim return SDValue(); 2784249423Sdim } 2785249423Sdim } 2786249423Sdim 2787249423Sdim // Handle the scalar case first. 2788251662Sdim if (Scalar1 && Scalar2) 2789249423Sdim return Outputs.back(); 2790249423Sdim 2791249423Sdim // Otherwise build a big vector out of the scalar elements we generated. 2792249423Sdim return getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, Outputs.data(), 2793249423Sdim Outputs.size()); 2794193323Sed} 2795193323Sed 2796249423SdimSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, SDValue N1, 2797249423Sdim SDValue N2) { 2798193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2799193323Sed ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2800193323Sed switch (Opcode) { 2801193323Sed default: break; 2802193323Sed case ISD::TokenFactor: 2803193323Sed assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2804193323Sed N2.getValueType() == MVT::Other && "Invalid token factor!"); 2805193323Sed // Fold trivial token factors. 2806193323Sed if (N1.getOpcode() == ISD::EntryToken) return N2; 2807193323Sed if (N2.getOpcode() == ISD::EntryToken) return N1; 2808193323Sed if (N1 == N2) return N1; 2809193323Sed break; 2810193323Sed case ISD::CONCAT_VECTORS: 2811239462Sdim // Concat of UNDEFs is UNDEF. 2812239462Sdim if (N1.getOpcode() == ISD::UNDEF && 2813239462Sdim N2.getOpcode() == ISD::UNDEF) 2814239462Sdim return getUNDEF(VT); 2815239462Sdim 2816193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2817193323Sed // one big BUILD_VECTOR. 2818193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 2819193323Sed N2.getOpcode() == ISD::BUILD_VECTOR) { 2820212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2821212904Sdim N1.getNode()->op_end()); 2822210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2823193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2824193323Sed } 2825193323Sed break; 2826193323Sed case ISD::AND: 2827208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2828208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2829193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2830193323Sed // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2831193323Sed // worth handling here. 2832193323Sed if (N2C && N2C->isNullValue()) 2833193323Sed return N2; 2834193323Sed if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2835193323Sed return N1; 2836193323Sed break; 2837193323Sed case ISD::OR: 2838193323Sed case ISD::XOR: 2839193323Sed case ISD::ADD: 2840193323Sed case ISD::SUB: 2841208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2842208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2843193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2844193323Sed // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2845193323Sed // it's worth handling here. 2846193323Sed if (N2C && N2C->isNullValue()) 2847193323Sed return N1; 2848193323Sed break; 2849193323Sed case ISD::UDIV: 2850193323Sed case ISD::UREM: 2851193323Sed case ISD::MULHU: 2852193323Sed case ISD::MULHS: 2853193323Sed case ISD::MUL: 2854193323Sed case ISD::SDIV: 2855193323Sed case ISD::SREM: 2856193323Sed assert(VT.isInteger() && "This operator does not apply to FP types!"); 2857208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2858208599Srdivacky N1.getValueType() == VT && "Binary operator types must match!"); 2859208599Srdivacky break; 2860193323Sed case ISD::FADD: 2861193323Sed case ISD::FSUB: 2862193323Sed case ISD::FMUL: 2863193323Sed case ISD::FDIV: 2864193323Sed case ISD::FREM: 2865234353Sdim if (getTarget().Options.UnsafeFPMath) { 2866193323Sed if (Opcode == ISD::FADD) { 2867193323Sed // 0+x --> x 2868193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2869193323Sed if (CFP->getValueAPF().isZero()) 2870193323Sed return N2; 2871193323Sed // x+0 --> x 2872193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2873193323Sed if (CFP->getValueAPF().isZero()) 2874193323Sed return N1; 2875193323Sed } else if (Opcode == ISD::FSUB) { 2876193323Sed // x-0 --> x 2877193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2878193323Sed if (CFP->getValueAPF().isZero()) 2879193323Sed return N1; 2880243830Sdim } else if (Opcode == ISD::FMUL) { 2881243830Sdim ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1); 2882243830Sdim SDValue V = N2; 2883243830Sdim 2884243830Sdim // If the first operand isn't the constant, try the second 2885243830Sdim if (!CFP) { 2886243830Sdim CFP = dyn_cast<ConstantFPSDNode>(N2); 2887243830Sdim V = N1; 2888243830Sdim } 2889243830Sdim 2890243830Sdim if (CFP) { 2891243830Sdim // 0*x --> 0 2892243830Sdim if (CFP->isZero()) 2893243830Sdim return SDValue(CFP,0); 2894243830Sdim // 1*x --> x 2895243830Sdim if (CFP->isExactlyValue(1.0)) 2896243830Sdim return V; 2897243830Sdim } 2898193323Sed } 2899193323Sed } 2900208599Srdivacky assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2901193323Sed assert(N1.getValueType() == N2.getValueType() && 2902193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2903193323Sed break; 2904193323Sed case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2905193323Sed assert(N1.getValueType() == VT && 2906193323Sed N1.getValueType().isFloatingPoint() && 2907193323Sed N2.getValueType().isFloatingPoint() && 2908193323Sed "Invalid FCOPYSIGN!"); 2909193323Sed break; 2910193323Sed case ISD::SHL: 2911193323Sed case ISD::SRA: 2912193323Sed case ISD::SRL: 2913193323Sed case ISD::ROTL: 2914193323Sed case ISD::ROTR: 2915193323Sed assert(VT == N1.getValueType() && 2916193323Sed "Shift operators return type must be the same as their first arg"); 2917193323Sed assert(VT.isInteger() && N2.getValueType().isInteger() && 2918193323Sed "Shifts only work on integers"); 2919249423Sdim assert((!VT.isVector() || VT == N2.getValueType()) && 2920249423Sdim "Vector shift amounts must be in the same as their first arg"); 2921218893Sdim // Verify that the shift amount VT is bit enough to hold valid shift 2922218893Sdim // amounts. This catches things like trying to shift an i1024 value by an 2923218893Sdim // i8, which is easy to fall into in generic code that uses 2924218893Sdim // TLI.getShiftAmount(). 2925218893Sdim assert(N2.getValueType().getSizeInBits() >= 2926219077Sdim Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 2927218893Sdim "Invalid use of small shift amount with oversized value!"); 2928193323Sed 2929193323Sed // Always fold shifts of i1 values so the code generator doesn't need to 2930193323Sed // handle them. Since we know the size of the shift has to be less than the 2931193323Sed // size of the value, the shift/rotate count is guaranteed to be zero. 2932193323Sed if (VT == MVT::i1) 2933193323Sed return N1; 2934202375Srdivacky if (N2C && N2C->isNullValue()) 2935202375Srdivacky return N1; 2936193323Sed break; 2937193323Sed case ISD::FP_ROUND_INREG: { 2938198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 2939193323Sed assert(VT == N1.getValueType() && "Not an inreg round!"); 2940193323Sed assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 2941193323Sed "Cannot FP_ROUND_INREG integer types"); 2942202375Srdivacky assert(EVT.isVector() == VT.isVector() && 2943202375Srdivacky "FP_ROUND_INREG type should be vector iff the operand " 2944202375Srdivacky "type is vector!"); 2945202375Srdivacky assert((!EVT.isVector() || 2946202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2947202375Srdivacky "Vector element counts must match in FP_ROUND_INREG"); 2948193323Sed assert(EVT.bitsLE(VT) && "Not rounding down!"); 2949226633Sdim (void)EVT; 2950193323Sed if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 2951193323Sed break; 2952193323Sed } 2953193323Sed case ISD::FP_ROUND: 2954193323Sed assert(VT.isFloatingPoint() && 2955193323Sed N1.getValueType().isFloatingPoint() && 2956193323Sed VT.bitsLE(N1.getValueType()) && 2957193323Sed isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 2958193323Sed if (N1.getValueType() == VT) return N1; // noop conversion. 2959193323Sed break; 2960193323Sed case ISD::AssertSext: 2961193323Sed case ISD::AssertZext: { 2962198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 2963193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 2964193323Sed assert(VT.isInteger() && EVT.isInteger() && 2965193323Sed "Cannot *_EXTEND_INREG FP types"); 2966200581Srdivacky assert(!EVT.isVector() && 2967200581Srdivacky "AssertSExt/AssertZExt type should be the vector element type " 2968200581Srdivacky "rather than the vector type!"); 2969193323Sed assert(EVT.bitsLE(VT) && "Not extending!"); 2970193323Sed if (VT == EVT) return N1; // noop assertion. 2971193323Sed break; 2972193323Sed } 2973193323Sed case ISD::SIGN_EXTEND_INREG: { 2974198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 2975193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 2976193323Sed assert(VT.isInteger() && EVT.isInteger() && 2977193323Sed "Cannot *_EXTEND_INREG FP types"); 2978202375Srdivacky assert(EVT.isVector() == VT.isVector() && 2979202375Srdivacky "SIGN_EXTEND_INREG type should be vector iff the operand " 2980202375Srdivacky "type is vector!"); 2981202375Srdivacky assert((!EVT.isVector() || 2982202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2983202375Srdivacky "Vector element counts must match in SIGN_EXTEND_INREG"); 2984202375Srdivacky assert(EVT.bitsLE(VT) && "Not extending!"); 2985193323Sed if (EVT == VT) return N1; // Not actually extending 2986193323Sed 2987193323Sed if (N1C) { 2988193323Sed APInt Val = N1C->getAPIntValue(); 2989202375Srdivacky unsigned FromBits = EVT.getScalarType().getSizeInBits(); 2990193323Sed Val <<= Val.getBitWidth()-FromBits; 2991193323Sed Val = Val.ashr(Val.getBitWidth()-FromBits); 2992193323Sed return getConstant(Val, VT); 2993193323Sed } 2994193323Sed break; 2995193323Sed } 2996193323Sed case ISD::EXTRACT_VECTOR_ELT: 2997193323Sed // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 2998193323Sed if (N1.getOpcode() == ISD::UNDEF) 2999193323Sed return getUNDEF(VT); 3000193323Sed 3001193323Sed // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 3002193323Sed // expanding copies of large vectors from registers. 3003193323Sed if (N2C && 3004193323Sed N1.getOpcode() == ISD::CONCAT_VECTORS && 3005193323Sed N1.getNumOperands() > 0) { 3006193323Sed unsigned Factor = 3007193323Sed N1.getOperand(0).getValueType().getVectorNumElements(); 3008193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 3009193323Sed N1.getOperand(N2C->getZExtValue() / Factor), 3010193323Sed getConstant(N2C->getZExtValue() % Factor, 3011193323Sed N2.getValueType())); 3012193323Sed } 3013193323Sed 3014193323Sed // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 3015193323Sed // expanding large vector constants. 3016193323Sed if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 3017193323Sed SDValue Elt = N1.getOperand(N2C->getZExtValue()); 3018243830Sdim 3019243830Sdim if (VT != Elt.getValueType()) 3020193323Sed // If the vector element type is not legal, the BUILD_VECTOR operands 3021243830Sdim // are promoted and implicitly truncated, and the result implicitly 3022243830Sdim // extended. Make that explicit here. 3023243830Sdim Elt = getAnyExtOrTrunc(Elt, DL, VT); 3024243830Sdim 3025193323Sed return Elt; 3026193323Sed } 3027193323Sed 3028193323Sed // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 3029193323Sed // operations are lowered to scalars. 3030193323Sed if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 3031203954Srdivacky // If the indices are the same, return the inserted element else 3032203954Srdivacky // if the indices are known different, extract the element from 3033193323Sed // the original vector. 3034207618Srdivacky SDValue N1Op2 = N1.getOperand(2); 3035207618Srdivacky ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 3036207618Srdivacky 3037207618Srdivacky if (N1Op2C && N2C) { 3038207618Srdivacky if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 3039207618Srdivacky if (VT == N1.getOperand(1).getValueType()) 3040207618Srdivacky return N1.getOperand(1); 3041207618Srdivacky else 3042207618Srdivacky return getSExtOrTrunc(N1.getOperand(1), DL, VT); 3043207618Srdivacky } 3044207618Srdivacky 3045193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 3046207618Srdivacky } 3047193323Sed } 3048193323Sed break; 3049193323Sed case ISD::EXTRACT_ELEMENT: 3050193323Sed assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 3051193323Sed assert(!N1.getValueType().isVector() && !VT.isVector() && 3052193323Sed (N1.getValueType().isInteger() == VT.isInteger()) && 3053226633Sdim N1.getValueType() != VT && 3054193323Sed "Wrong types for EXTRACT_ELEMENT!"); 3055193323Sed 3056193323Sed // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 3057193323Sed // 64-bit integers into 32-bit parts. Instead of building the extract of 3058193323Sed // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 3059193323Sed if (N1.getOpcode() == ISD::BUILD_PAIR) 3060193323Sed return N1.getOperand(N2C->getZExtValue()); 3061193323Sed 3062193323Sed // EXTRACT_ELEMENT of a constant int is also very common. 3063193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 3064193323Sed unsigned ElementSize = VT.getSizeInBits(); 3065193323Sed unsigned Shift = ElementSize * N2C->getZExtValue(); 3066193323Sed APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 3067193323Sed return getConstant(ShiftedVal.trunc(ElementSize), VT); 3068193323Sed } 3069193323Sed break; 3070218893Sdim case ISD::EXTRACT_SUBVECTOR: { 3071218893Sdim SDValue Index = N2; 3072218893Sdim if (VT.isSimple() && N1.getValueType().isSimple()) { 3073218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3074218893Sdim "Extract subvector VTs must be a vectors!"); 3075218893Sdim assert(VT.getVectorElementType() == N1.getValueType().getVectorElementType() && 3076218893Sdim "Extract subvector VTs must have the same element type!"); 3077218893Sdim assert(VT.getSimpleVT() <= N1.getValueType().getSimpleVT() && 3078218893Sdim "Extract subvector must be from larger vector to smaller vector!"); 3079218893Sdim 3080218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3081218893Sdim assert((VT.getVectorNumElements() + 3082218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3083218893Sdim <= N1.getValueType().getVectorNumElements()) 3084218893Sdim && "Extract subvector overflow!"); 3085218893Sdim } 3086218893Sdim 3087218893Sdim // Trivial extraction. 3088218893Sdim if (VT.getSimpleVT() == N1.getValueType().getSimpleVT()) 3089218893Sdim return N1; 3090218893Sdim } 3091193323Sed break; 3092193323Sed } 3093218893Sdim } 3094193323Sed 3095249423Sdim // Perform trivial constant folding. 3096249423Sdim SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); 3097249423Sdim if (SV.getNode()) return SV; 3098249423Sdim 3099249423Sdim // Canonicalize constant to RHS if commutative. 3100249423Sdim if (N1C && !N2C && isCommutativeBinOp(Opcode)) { 3101249423Sdim std::swap(N1C, N2C); 3102249423Sdim std::swap(N1, N2); 3103193323Sed } 3104193323Sed 3105193323Sed // Constant fold FP operations. 3106193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 3107193323Sed ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 3108193323Sed if (N1CFP) { 3109193323Sed if (!N2CFP && isCommutativeBinOp(Opcode)) { 3110249423Sdim // Canonicalize constant to RHS if commutative. 3111193323Sed std::swap(N1CFP, N2CFP); 3112193323Sed std::swap(N1, N2); 3113243830Sdim } else if (N2CFP) { 3114193323Sed APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 3115193323Sed APFloat::opStatus s; 3116193323Sed switch (Opcode) { 3117193323Sed case ISD::FADD: 3118193323Sed s = V1.add(V2, APFloat::rmNearestTiesToEven); 3119193323Sed if (s != APFloat::opInvalidOp) 3120193323Sed return getConstantFP(V1, VT); 3121193323Sed break; 3122193323Sed case ISD::FSUB: 3123193323Sed s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 3124193323Sed if (s!=APFloat::opInvalidOp) 3125193323Sed return getConstantFP(V1, VT); 3126193323Sed break; 3127193323Sed case ISD::FMUL: 3128193323Sed s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3129193323Sed if (s!=APFloat::opInvalidOp) 3130193323Sed return getConstantFP(V1, VT); 3131193323Sed break; 3132193323Sed case ISD::FDIV: 3133193323Sed s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3134193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3135193323Sed return getConstantFP(V1, VT); 3136193323Sed break; 3137193323Sed case ISD::FREM : 3138193323Sed s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3139193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3140193323Sed return getConstantFP(V1, VT); 3141193323Sed break; 3142193323Sed case ISD::FCOPYSIGN: 3143193323Sed V1.copySign(V2); 3144193323Sed return getConstantFP(V1, VT); 3145193323Sed default: break; 3146193323Sed } 3147193323Sed } 3148234353Sdim 3149234353Sdim if (Opcode == ISD::FP_ROUND) { 3150234353Sdim APFloat V = N1CFP->getValueAPF(); // make copy 3151234353Sdim bool ignored; 3152234353Sdim // This can return overflow, underflow, or inexact; we don't care. 3153234353Sdim // FIXME need to be more flexible about rounding mode. 3154249423Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 3155234353Sdim APFloat::rmNearestTiesToEven, &ignored); 3156234353Sdim return getConstantFP(V, VT); 3157234353Sdim } 3158193323Sed } 3159193323Sed 3160193323Sed // Canonicalize an UNDEF to the RHS, even over a constant. 3161193323Sed if (N1.getOpcode() == ISD::UNDEF) { 3162193323Sed if (isCommutativeBinOp(Opcode)) { 3163193323Sed std::swap(N1, N2); 3164193323Sed } else { 3165193323Sed switch (Opcode) { 3166193323Sed case ISD::FP_ROUND_INREG: 3167193323Sed case ISD::SIGN_EXTEND_INREG: 3168193323Sed case ISD::SUB: 3169193323Sed case ISD::FSUB: 3170193323Sed case ISD::FDIV: 3171193323Sed case ISD::FREM: 3172193323Sed case ISD::SRA: 3173193323Sed return N1; // fold op(undef, arg2) -> undef 3174193323Sed case ISD::UDIV: 3175193323Sed case ISD::SDIV: 3176193323Sed case ISD::UREM: 3177193323Sed case ISD::SREM: 3178193323Sed case ISD::SRL: 3179193323Sed case ISD::SHL: 3180193323Sed if (!VT.isVector()) 3181193323Sed return getConstant(0, VT); // fold op(undef, arg2) -> 0 3182193323Sed // For vectors, we can't easily build an all zero vector, just return 3183193323Sed // the LHS. 3184193323Sed return N2; 3185193323Sed } 3186193323Sed } 3187193323Sed } 3188193323Sed 3189193323Sed // Fold a bunch of operators when the RHS is undef. 3190193323Sed if (N2.getOpcode() == ISD::UNDEF) { 3191193323Sed switch (Opcode) { 3192193323Sed case ISD::XOR: 3193193323Sed if (N1.getOpcode() == ISD::UNDEF) 3194193323Sed // Handle undef ^ undef -> 0 special case. This is a common 3195193323Sed // idiom (misuse). 3196193323Sed return getConstant(0, VT); 3197193323Sed // fallthrough 3198193323Sed case ISD::ADD: 3199193323Sed case ISD::ADDC: 3200193323Sed case ISD::ADDE: 3201193323Sed case ISD::SUB: 3202193574Sed case ISD::UDIV: 3203193574Sed case ISD::SDIV: 3204193574Sed case ISD::UREM: 3205193574Sed case ISD::SREM: 3206193574Sed return N2; // fold op(arg1, undef) -> undef 3207193323Sed case ISD::FADD: 3208193323Sed case ISD::FSUB: 3209193323Sed case ISD::FMUL: 3210193323Sed case ISD::FDIV: 3211193323Sed case ISD::FREM: 3212234353Sdim if (getTarget().Options.UnsafeFPMath) 3213193574Sed return N2; 3214193574Sed break; 3215193323Sed case ISD::MUL: 3216193323Sed case ISD::AND: 3217193323Sed case ISD::SRL: 3218193323Sed case ISD::SHL: 3219193323Sed if (!VT.isVector()) 3220193323Sed return getConstant(0, VT); // fold op(arg1, undef) -> 0 3221193323Sed // For vectors, we can't easily build an all zero vector, just return 3222193323Sed // the LHS. 3223193323Sed return N1; 3224193323Sed case ISD::OR: 3225193323Sed if (!VT.isVector()) 3226193323Sed return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3227193323Sed // For vectors, we can't easily build an all one vector, just return 3228193323Sed // the LHS. 3229193323Sed return N1; 3230193323Sed case ISD::SRA: 3231193323Sed return N1; 3232193323Sed } 3233193323Sed } 3234193323Sed 3235193323Sed // Memoize this node if possible. 3236193323Sed SDNode *N; 3237193323Sed SDVTList VTs = getVTList(VT); 3238218893Sdim if (VT != MVT::Glue) { 3239193323Sed SDValue Ops[] = { N1, N2 }; 3240193323Sed FoldingSetNodeID ID; 3241193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3242193323Sed void *IP = 0; 3243201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3244193323Sed return SDValue(E, 0); 3245201360Srdivacky 3246205407Srdivacky N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3247193323Sed CSEMap.InsertNode(N, IP); 3248193323Sed } else { 3249205407Srdivacky N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); 3250193323Sed } 3251193323Sed 3252193323Sed AllNodes.push_back(N); 3253193323Sed#ifndef NDEBUG 3254218893Sdim VerifySDNode(N); 3255193323Sed#endif 3256193323Sed return SDValue(N, 0); 3257193323Sed} 3258193323Sed 3259198090SrdivackySDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3260193323Sed SDValue N1, SDValue N2, SDValue N3) { 3261193323Sed // Perform various simplifications. 3262193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3263193323Sed switch (Opcode) { 3264193323Sed case ISD::CONCAT_VECTORS: 3265193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3266193323Sed // one big BUILD_VECTOR. 3267193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 3268193323Sed N2.getOpcode() == ISD::BUILD_VECTOR && 3269193323Sed N3.getOpcode() == ISD::BUILD_VECTOR) { 3270212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3271212904Sdim N1.getNode()->op_end()); 3272210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3273210299Sed Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3274193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3275193323Sed } 3276193323Sed break; 3277193323Sed case ISD::SETCC: { 3278193323Sed // Use FoldSetCC to simplify SETCC's. 3279193323Sed SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3280193323Sed if (Simp.getNode()) return Simp; 3281193323Sed break; 3282193323Sed } 3283193323Sed case ISD::SELECT: 3284193323Sed if (N1C) { 3285193323Sed if (N1C->getZExtValue()) 3286234353Sdim return N2; // select true, X, Y -> X 3287234353Sdim return N3; // select false, X, Y -> Y 3288193323Sed } 3289193323Sed 3290193323Sed if (N2 == N3) return N2; // select C, X, X -> X 3291193323Sed break; 3292193323Sed case ISD::VECTOR_SHUFFLE: 3293198090Srdivacky llvm_unreachable("should use getVectorShuffle constructor!"); 3294218893Sdim case ISD::INSERT_SUBVECTOR: { 3295218893Sdim SDValue Index = N3; 3296218893Sdim if (VT.isSimple() && N1.getValueType().isSimple() 3297218893Sdim && N2.getValueType().isSimple()) { 3298218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3299218893Sdim N2.getValueType().isVector() && 3300218893Sdim "Insert subvector VTs must be a vectors"); 3301218893Sdim assert(VT == N1.getValueType() && 3302218893Sdim "Dest and insert subvector source types must match!"); 3303218893Sdim assert(N2.getValueType().getSimpleVT() <= N1.getValueType().getSimpleVT() && 3304218893Sdim "Insert subvector must be from smaller vector to larger vector!"); 3305218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3306218893Sdim assert((N2.getValueType().getVectorNumElements() + 3307218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3308218893Sdim <= VT.getVectorNumElements()) 3309218893Sdim && "Insert subvector overflow!"); 3310218893Sdim } 3311218893Sdim 3312218893Sdim // Trivial insertion. 3313218893Sdim if (VT.getSimpleVT() == N2.getValueType().getSimpleVT()) 3314218893Sdim return N2; 3315218893Sdim } 3316218893Sdim break; 3317218893Sdim } 3318218893Sdim case ISD::BITCAST: 3319193323Sed // Fold bit_convert nodes from a type to themselves. 3320193323Sed if (N1.getValueType() == VT) 3321193323Sed return N1; 3322193323Sed break; 3323193323Sed } 3324193323Sed 3325193323Sed // Memoize node if it doesn't produce a flag. 3326193323Sed SDNode *N; 3327193323Sed SDVTList VTs = getVTList(VT); 3328218893Sdim if (VT != MVT::Glue) { 3329193323Sed SDValue Ops[] = { N1, N2, N3 }; 3330193323Sed FoldingSetNodeID ID; 3331193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3332193323Sed void *IP = 0; 3333201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3334193323Sed return SDValue(E, 0); 3335201360Srdivacky 3336205407Srdivacky N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3337193323Sed CSEMap.InsertNode(N, IP); 3338193323Sed } else { 3339205407Srdivacky N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); 3340193323Sed } 3341200581Srdivacky 3342193323Sed AllNodes.push_back(N); 3343193323Sed#ifndef NDEBUG 3344218893Sdim VerifySDNode(N); 3345193323Sed#endif 3346193323Sed return SDValue(N, 0); 3347193323Sed} 3348193323Sed 3349198090SrdivackySDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3350193323Sed SDValue N1, SDValue N2, SDValue N3, 3351193323Sed SDValue N4) { 3352193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 3353193323Sed return getNode(Opcode, DL, VT, Ops, 4); 3354193323Sed} 3355193323Sed 3356198090SrdivackySDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 3357193323Sed SDValue N1, SDValue N2, SDValue N3, 3358193323Sed SDValue N4, SDValue N5) { 3359193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3360193323Sed return getNode(Opcode, DL, VT, Ops, 5); 3361193323Sed} 3362193323Sed 3363198090Srdivacky/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3364198090Srdivacky/// the incoming stack arguments to be loaded from the stack. 3365198090SrdivackySDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3366198090Srdivacky SmallVector<SDValue, 8> ArgChains; 3367198090Srdivacky 3368198090Srdivacky // Include the original chain at the beginning of the list. When this is 3369198090Srdivacky // used by target LowerCall hooks, this helps legalize find the 3370198090Srdivacky // CALLSEQ_BEGIN node. 3371198090Srdivacky ArgChains.push_back(Chain); 3372198090Srdivacky 3373198090Srdivacky // Add a chain value for each stack argument. 3374198090Srdivacky for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3375198090Srdivacky UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3376198090Srdivacky if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3377198090Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3378198090Srdivacky if (FI->getIndex() < 0) 3379198090Srdivacky ArgChains.push_back(SDValue(L, 1)); 3380198090Srdivacky 3381198090Srdivacky // Build a tokenfactor for all the chains. 3382198090Srdivacky return getNode(ISD::TokenFactor, Chain.getDebugLoc(), MVT::Other, 3383198090Srdivacky &ArgChains[0], ArgChains.size()); 3384198090Srdivacky} 3385198090Srdivacky 3386193323Sed/// getMemsetValue - Vectorized representation of the memset value 3387193323Sed/// operand. 3388198090Srdivackystatic SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3389193323Sed DebugLoc dl) { 3390206124Srdivacky assert(Value.getOpcode() != ISD::UNDEF); 3391206124Srdivacky 3392204642Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits(); 3393193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3394249423Sdim assert(C->getAPIntValue().getBitWidth() == 8); 3395249423Sdim APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 3396193323Sed if (VT.isInteger()) 3397193323Sed return DAG.getConstant(Val, VT); 3398249423Sdim return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); 3399193323Sed } 3400193323Sed 3401193323Sed Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3402218893Sdim if (NumBits > 8) { 3403218893Sdim // Use a multiplication with 0x010101... to extend the input to the 3404218893Sdim // required length. 3405249423Sdim APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 3406218893Sdim Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3407193323Sed } 3408193323Sed 3409193323Sed return Value; 3410193323Sed} 3411193323Sed 3412193323Sed/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3413193323Sed/// used when a memcpy is turned into a memset when the source is a constant 3414193323Sed/// string ptr. 3415198090Srdivackystatic SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, 3416234353Sdim const TargetLowering &TLI, StringRef Str) { 3417193323Sed // Handle vector with all elements zero. 3418193323Sed if (Str.empty()) { 3419193323Sed if (VT.isInteger()) 3420193323Sed return DAG.getConstant(0, VT); 3421218893Sdim else if (VT == MVT::f32 || VT == MVT::f64) 3422206083Srdivacky return DAG.getConstantFP(0.0, VT); 3423206083Srdivacky else if (VT.isVector()) { 3424206083Srdivacky unsigned NumElts = VT.getVectorNumElements(); 3425206083Srdivacky MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3426218893Sdim return DAG.getNode(ISD::BITCAST, dl, VT, 3427206083Srdivacky DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3428206083Srdivacky EltVT, NumElts))); 3429206083Srdivacky } else 3430206083Srdivacky llvm_unreachable("Expected type!"); 3431193323Sed } 3432193323Sed 3433193323Sed assert(!VT.isVector() && "Can't handle vector type here!"); 3434249423Sdim unsigned NumVTBits = VT.getSizeInBits(); 3435249423Sdim unsigned NumVTBytes = NumVTBits / 8; 3436234353Sdim unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); 3437234353Sdim 3438249423Sdim APInt Val(NumVTBits, 0); 3439234353Sdim if (TLI.isLittleEndian()) { 3440234353Sdim for (unsigned i = 0; i != NumBytes; ++i) 3441234353Sdim Val |= (uint64_t)(unsigned char)Str[i] << i*8; 3442234353Sdim } else { 3443234353Sdim for (unsigned i = 0; i != NumBytes; ++i) 3444234353Sdim Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; 3445193323Sed } 3446234353Sdim 3447249423Sdim // If the "cost" of materializing the integer immediate is 1 or free, then 3448249423Sdim // it is cost effective to turn the load into the immediate. 3449249423Sdim const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); 3450249423Sdim if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) 3451249423Sdim return DAG.getConstant(Val, VT); 3452249423Sdim return SDValue(0, 0); 3453193323Sed} 3454193323Sed 3455193323Sed/// getMemBasePlusOffset - Returns base and offset node for the 3456193323Sed/// 3457193323Sedstatic SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, 3458193323Sed SelectionDAG &DAG) { 3459198090Srdivacky EVT VT = Base.getValueType(); 3460193323Sed return DAG.getNode(ISD::ADD, Base.getDebugLoc(), 3461193323Sed VT, Base, DAG.getConstant(Offset, VT)); 3462193323Sed} 3463193323Sed 3464193323Sed/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3465193323Sed/// 3466234353Sdimstatic bool isMemSrcFromString(SDValue Src, StringRef &Str) { 3467193323Sed unsigned SrcDelta = 0; 3468193323Sed GlobalAddressSDNode *G = NULL; 3469193323Sed if (Src.getOpcode() == ISD::GlobalAddress) 3470193323Sed G = cast<GlobalAddressSDNode>(Src); 3471193323Sed else if (Src.getOpcode() == ISD::ADD && 3472193323Sed Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3473193323Sed Src.getOperand(1).getOpcode() == ISD::Constant) { 3474193323Sed G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3475193323Sed SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3476193323Sed } 3477193323Sed if (!G) 3478193323Sed return false; 3479193323Sed 3480234353Sdim return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false); 3481193323Sed} 3482193323Sed 3483206083Srdivacky/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3484206083Srdivacky/// to replace the memset / memcpy. Return true if the number of memory ops 3485206083Srdivacky/// is below the threshold. It returns the types of the sequence of 3486206083Srdivacky/// memory ops to perform memset / memcpy by reference. 3487206083Srdivackystatic bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3488206083Srdivacky unsigned Limit, uint64_t Size, 3489206083Srdivacky unsigned DstAlign, unsigned SrcAlign, 3490249423Sdim bool IsMemset, 3491249423Sdim bool ZeroMemset, 3492207618Srdivacky bool MemcpyStrSrc, 3493249423Sdim bool AllowOverlap, 3494206083Srdivacky SelectionDAG &DAG, 3495206083Srdivacky const TargetLowering &TLI) { 3496206083Srdivacky assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3497206083Srdivacky "Expecting memcpy / memset source to meet alignment requirement!"); 3498224145Sdim // If 'SrcAlign' is zero, that means the memory operation does not need to 3499224145Sdim // load the value, i.e. memset or memcpy from constant string. Otherwise, 3500224145Sdim // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3501224145Sdim // is the specified alignment of the memory operation. If it is zero, that 3502224145Sdim // means it's possible to change the alignment of the destination. 3503224145Sdim // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3504224145Sdim // not need to be loaded. 3505206124Srdivacky EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3506249423Sdim IsMemset, ZeroMemset, MemcpyStrSrc, 3507207618Srdivacky DAG.getMachineFunction()); 3508193323Sed 3509204961Srdivacky if (VT == MVT::Other) { 3510243830Sdim if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() || 3511206083Srdivacky TLI.allowsUnalignedMemoryAccesses(VT)) { 3512206274Srdivacky VT = TLI.getPointerTy(); 3513193323Sed } else { 3514206083Srdivacky switch (DstAlign & 7) { 3515193323Sed case 0: VT = MVT::i64; break; 3516193323Sed case 4: VT = MVT::i32; break; 3517193323Sed case 2: VT = MVT::i16; break; 3518193323Sed default: VT = MVT::i8; break; 3519193323Sed } 3520193323Sed } 3521193323Sed 3522193323Sed MVT LVT = MVT::i64; 3523193323Sed while (!TLI.isTypeLegal(LVT)) 3524198090Srdivacky LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3525193323Sed assert(LVT.isInteger()); 3526193323Sed 3527193323Sed if (VT.bitsGT(LVT)) 3528193323Sed VT = LVT; 3529193323Sed } 3530193323Sed 3531193323Sed unsigned NumMemOps = 0; 3532193323Sed while (Size != 0) { 3533193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3534193323Sed while (VTSize > Size) { 3535193323Sed // For now, only use non-vector load / store's for the left-over pieces. 3536249423Sdim EVT NewVT = VT; 3537249423Sdim unsigned NewVTSize; 3538249423Sdim 3539249423Sdim bool Found = false; 3540206083Srdivacky if (VT.isVector() || VT.isFloatingPoint()) { 3541249423Sdim NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 3542249423Sdim if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 3543249423Sdim TLI.isSafeMemOpType(NewVT.getSimpleVT())) 3544249423Sdim Found = true; 3545249423Sdim else if (NewVT == MVT::i64 && 3546249423Sdim TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 3547249423Sdim TLI.isSafeMemOpType(MVT::f64)) { 3548249423Sdim // i64 is usually not legal on 32-bit targets, but f64 may be. 3549249423Sdim NewVT = MVT::f64; 3550249423Sdim Found = true; 3551249423Sdim } 3552193323Sed } 3553249423Sdim 3554249423Sdim if (!Found) { 3555249423Sdim do { 3556249423Sdim NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 3557249423Sdim if (NewVT == MVT::i8) 3558249423Sdim break; 3559249423Sdim } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 3560249423Sdim } 3561249423Sdim NewVTSize = NewVT.getSizeInBits() / 8; 3562249423Sdim 3563249423Sdim // If the new VT cannot cover all of the remaining bits, then consider 3564249423Sdim // issuing a (or a pair of) unaligned and overlapping load / store. 3565249423Sdim // FIXME: Only does this for 64-bit or more since we don't have proper 3566249423Sdim // cost model for unaligned load / store. 3567249423Sdim bool Fast; 3568249423Sdim if (NumMemOps && AllowOverlap && 3569249423Sdim VTSize >= 8 && NewVTSize < Size && 3570249423Sdim TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) 3571249423Sdim VTSize = Size; 3572249423Sdim else { 3573249423Sdim VT = NewVT; 3574249423Sdim VTSize = NewVTSize; 3575249423Sdim } 3576193323Sed } 3577193323Sed 3578193323Sed if (++NumMemOps > Limit) 3579193323Sed return false; 3580249423Sdim 3581193323Sed MemOps.push_back(VT); 3582193323Sed Size -= VTSize; 3583193323Sed } 3584193323Sed 3585193323Sed return true; 3586193323Sed} 3587193323Sed 3588193323Sedstatic SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3589206083Srdivacky SDValue Chain, SDValue Dst, 3590206083Srdivacky SDValue Src, uint64_t Size, 3591206274Srdivacky unsigned Align, bool isVol, 3592206274Srdivacky bool AlwaysInline, 3593218893Sdim MachinePointerInfo DstPtrInfo, 3594218893Sdim MachinePointerInfo SrcPtrInfo) { 3595206124Srdivacky // Turn a memcpy of undef to nop. 3596206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3597206124Srdivacky return Chain; 3598193323Sed 3599193323Sed // Expand memcpy to a series of load and store ops if the size operand falls 3600193323Sed // below a certain threshold. 3601218893Sdim // TODO: In the AlwaysInline case, if the size is big then generate a loop 3602218893Sdim // rather than maybe a humongous number of loads and stores. 3603206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3604198090Srdivacky std::vector<EVT> MemOps; 3605206083Srdivacky bool DstAlignCanChange = false; 3606218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3607218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3608243830Sdim bool OptSize = 3609249423Sdim MF.getFunction()->getAttributes(). 3610249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3611206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3612206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3613206083Srdivacky DstAlignCanChange = true; 3614206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3615206083Srdivacky if (Align > SrcAlign) 3616206083Srdivacky SrcAlign = Align; 3617234353Sdim StringRef Str; 3618206083Srdivacky bool CopyFromStr = isMemSrcFromString(Src, Str); 3619206083Srdivacky bool isZeroStr = CopyFromStr && Str.empty(); 3620218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3621218893Sdim 3622206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3623206083Srdivacky (DstAlignCanChange ? 0 : Align), 3624207618Srdivacky (isZeroStr ? 0 : SrcAlign), 3625249423Sdim false, false, CopyFromStr, true, DAG, TLI)) 3626193323Sed return SDValue(); 3627193323Sed 3628206083Srdivacky if (DstAlignCanChange) { 3629226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3630243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3631249423Sdim 3632249423Sdim // Don't promote to an alignment that would require dynamic stack 3633249423Sdim // realignment. 3634249423Sdim const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); 3635249423Sdim if (!TRI->needsStackRealignment(MF)) 3636249423Sdim while (NewAlign > Align && 3637249423Sdim TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) 3638249423Sdim NewAlign /= 2; 3639249423Sdim 3640206083Srdivacky if (NewAlign > Align) { 3641206083Srdivacky // Give the stack frame object a larger alignment if needed. 3642206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3643206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3644206083Srdivacky Align = NewAlign; 3645206083Srdivacky } 3646206083Srdivacky } 3647193323Sed 3648193323Sed SmallVector<SDValue, 8> OutChains; 3649193323Sed unsigned NumMemOps = MemOps.size(); 3650193323Sed uint64_t SrcOff = 0, DstOff = 0; 3651198090Srdivacky for (unsigned i = 0; i != NumMemOps; ++i) { 3652198090Srdivacky EVT VT = MemOps[i]; 3653193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3654193323Sed SDValue Value, Store; 3655193323Sed 3656249423Sdim if (VTSize > Size) { 3657249423Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3658249423Sdim // pair. Adjust the offset accordingly. 3659249423Sdim assert(i == NumMemOps-1 && i != 0); 3660249423Sdim SrcOff -= VTSize - Size; 3661249423Sdim DstOff -= VTSize - Size; 3662249423Sdim } 3663249423Sdim 3664206083Srdivacky if (CopyFromStr && 3665206083Srdivacky (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3666193323Sed // It's unlikely a store of a vector immediate can be done in a single 3667193323Sed // instruction. It would require a load from a constantpool first. 3668206083Srdivacky // We only handle zero vectors here. 3669193323Sed // FIXME: Handle other cases where store of vector immediate is done in 3670193323Sed // a single instruction. 3671234353Sdim Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); 3672249423Sdim if (Value.getNode()) 3673249423Sdim Store = DAG.getStore(Chain, dl, Value, 3674249423Sdim getMemBasePlusOffset(Dst, DstOff, DAG), 3675249423Sdim DstPtrInfo.getWithOffset(DstOff), isVol, 3676249423Sdim false, Align); 3677249423Sdim } 3678249423Sdim 3679249423Sdim if (!Store.getNode()) { 3680194710Sed // The type might not be legal for the target. This should only happen 3681194710Sed // if the type is smaller than a legal type, as on PPC, so the right 3682195098Sed // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3683195098Sed // to Load/Store if NVT==VT. 3684194710Sed // FIXME does the case above also need this? 3685198090Srdivacky EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3686195098Sed assert(NVT.bitsGE(VT)); 3687218893Sdim Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3688195098Sed getMemBasePlusOffset(Src, SrcOff, DAG), 3689218893Sdim SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3690206083Srdivacky MinAlign(SrcAlign, SrcOff)); 3691195098Sed Store = DAG.getTruncStore(Chain, dl, Value, 3692203954Srdivacky getMemBasePlusOffset(Dst, DstOff, DAG), 3693218893Sdim DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3694218893Sdim false, Align); 3695193323Sed } 3696193323Sed OutChains.push_back(Store); 3697193323Sed SrcOff += VTSize; 3698193323Sed DstOff += VTSize; 3699249423Sdim Size -= VTSize; 3700193323Sed } 3701193323Sed 3702193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3703193323Sed &OutChains[0], OutChains.size()); 3704193323Sed} 3705193323Sed 3706193323Sedstatic SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, 3707206083Srdivacky SDValue Chain, SDValue Dst, 3708206083Srdivacky SDValue Src, uint64_t Size, 3709206274Srdivacky unsigned Align, bool isVol, 3710206274Srdivacky bool AlwaysInline, 3711218893Sdim MachinePointerInfo DstPtrInfo, 3712218893Sdim MachinePointerInfo SrcPtrInfo) { 3713206124Srdivacky // Turn a memmove of undef to nop. 3714206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3715206124Srdivacky return Chain; 3716193323Sed 3717193323Sed // Expand memmove to a series of load and store ops if the size operand falls 3718193323Sed // below a certain threshold. 3719206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3720198090Srdivacky std::vector<EVT> MemOps; 3721206083Srdivacky bool DstAlignCanChange = false; 3722218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3723218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3724249423Sdim bool OptSize = MF.getFunction()->getAttributes(). 3725249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3726206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3727206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3728206083Srdivacky DstAlignCanChange = true; 3729206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3730206083Srdivacky if (Align > SrcAlign) 3731206083Srdivacky SrcAlign = Align; 3732218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3733206083Srdivacky 3734206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3735249423Sdim (DstAlignCanChange ? 0 : Align), SrcAlign, 3736249423Sdim false, false, false, false, DAG, TLI)) 3737193323Sed return SDValue(); 3738193323Sed 3739206083Srdivacky if (DstAlignCanChange) { 3740226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3741243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3742206083Srdivacky if (NewAlign > Align) { 3743206083Srdivacky // Give the stack frame object a larger alignment if needed. 3744206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3745206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3746206083Srdivacky Align = NewAlign; 3747206083Srdivacky } 3748206083Srdivacky } 3749206083Srdivacky 3750193323Sed uint64_t SrcOff = 0, DstOff = 0; 3751193323Sed SmallVector<SDValue, 8> LoadValues; 3752193323Sed SmallVector<SDValue, 8> LoadChains; 3753193323Sed SmallVector<SDValue, 8> OutChains; 3754193323Sed unsigned NumMemOps = MemOps.size(); 3755193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3756198090Srdivacky EVT VT = MemOps[i]; 3757193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3758193323Sed SDValue Value, Store; 3759193323Sed 3760193323Sed Value = DAG.getLoad(VT, dl, Chain, 3761193323Sed getMemBasePlusOffset(Src, SrcOff, DAG), 3762218893Sdim SrcPtrInfo.getWithOffset(SrcOff), isVol, 3763234353Sdim false, false, SrcAlign); 3764193323Sed LoadValues.push_back(Value); 3765193323Sed LoadChains.push_back(Value.getValue(1)); 3766193323Sed SrcOff += VTSize; 3767193323Sed } 3768193323Sed Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3769193323Sed &LoadChains[0], LoadChains.size()); 3770193323Sed OutChains.clear(); 3771193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3772198090Srdivacky EVT VT = MemOps[i]; 3773193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3774193323Sed SDValue Value, Store; 3775193323Sed 3776193323Sed Store = DAG.getStore(Chain, dl, LoadValues[i], 3777193323Sed getMemBasePlusOffset(Dst, DstOff, DAG), 3778218893Sdim DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3779193323Sed OutChains.push_back(Store); 3780193323Sed DstOff += VTSize; 3781193323Sed } 3782193323Sed 3783193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3784193323Sed &OutChains[0], OutChains.size()); 3785193323Sed} 3786193323Sed 3787193323Sedstatic SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, 3788206083Srdivacky SDValue Chain, SDValue Dst, 3789206083Srdivacky SDValue Src, uint64_t Size, 3790206274Srdivacky unsigned Align, bool isVol, 3791218893Sdim MachinePointerInfo DstPtrInfo) { 3792206124Srdivacky // Turn a memset of undef to nop. 3793206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3794206124Srdivacky return Chain; 3795193323Sed 3796193323Sed // Expand memset to a series of load/store ops if the size operand 3797193323Sed // falls below a certain threshold. 3798206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3799198090Srdivacky std::vector<EVT> MemOps; 3800206083Srdivacky bool DstAlignCanChange = false; 3801218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3802218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3803249423Sdim bool OptSize = MF.getFunction()->getAttributes(). 3804249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3805206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3806206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3807206083Srdivacky DstAlignCanChange = true; 3808234353Sdim bool IsZeroVal = 3809206124Srdivacky isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3810218893Sdim if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3811206083Srdivacky Size, (DstAlignCanChange ? 0 : Align), 0, 3812249423Sdim true, IsZeroVal, false, true, DAG, TLI)) 3813193323Sed return SDValue(); 3814193323Sed 3815206083Srdivacky if (DstAlignCanChange) { 3816226633Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3817243830Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3818206083Srdivacky if (NewAlign > Align) { 3819206083Srdivacky // Give the stack frame object a larger alignment if needed. 3820206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3821206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3822206083Srdivacky Align = NewAlign; 3823206083Srdivacky } 3824206083Srdivacky } 3825206083Srdivacky 3826193323Sed SmallVector<SDValue, 8> OutChains; 3827193323Sed uint64_t DstOff = 0; 3828193323Sed unsigned NumMemOps = MemOps.size(); 3829218893Sdim 3830218893Sdim // Find the largest store and generate the bit pattern for it. 3831218893Sdim EVT LargestVT = MemOps[0]; 3832218893Sdim for (unsigned i = 1; i < NumMemOps; i++) 3833218893Sdim if (MemOps[i].bitsGT(LargestVT)) 3834218893Sdim LargestVT = MemOps[i]; 3835218893Sdim SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3836218893Sdim 3837193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3838198090Srdivacky EVT VT = MemOps[i]; 3839249423Sdim unsigned VTSize = VT.getSizeInBits() / 8; 3840249423Sdim if (VTSize > Size) { 3841249423Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3842249423Sdim // pair. Adjust the offset accordingly. 3843249423Sdim assert(i == NumMemOps-1 && i != 0); 3844249423Sdim DstOff -= VTSize - Size; 3845249423Sdim } 3846218893Sdim 3847218893Sdim // If this store is smaller than the largest store see whether we can get 3848218893Sdim // the smaller value for free with a truncate. 3849218893Sdim SDValue Value = MemSetValue; 3850218893Sdim if (VT.bitsLT(LargestVT)) { 3851218893Sdim if (!LargestVT.isVector() && !VT.isVector() && 3852218893Sdim TLI.isTruncateFree(LargestVT, VT)) 3853218893Sdim Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3854218893Sdim else 3855218893Sdim Value = getMemsetValue(Src, VT, DAG, dl); 3856218893Sdim } 3857218893Sdim assert(Value.getValueType() == VT && "Value with wrong type."); 3858193323Sed SDValue Store = DAG.getStore(Chain, dl, Value, 3859193323Sed getMemBasePlusOffset(Dst, DstOff, DAG), 3860218893Sdim DstPtrInfo.getWithOffset(DstOff), 3861218893Sdim isVol, false, Align); 3862193323Sed OutChains.push_back(Store); 3863218893Sdim DstOff += VT.getSizeInBits() / 8; 3864249423Sdim Size -= VTSize; 3865193323Sed } 3866193323Sed 3867193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3868193323Sed &OutChains[0], OutChains.size()); 3869193323Sed} 3870193323Sed 3871193323SedSDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, 3872193323Sed SDValue Src, SDValue Size, 3873206274Srdivacky unsigned Align, bool isVol, bool AlwaysInline, 3874218893Sdim MachinePointerInfo DstPtrInfo, 3875218893Sdim MachinePointerInfo SrcPtrInfo) { 3876249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 3877193323Sed 3878193323Sed // Check to see if we should lower the memcpy to loads and stores first. 3879193323Sed // For cases within the target-specified limits, this is the best choice. 3880193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3881193323Sed if (ConstantSize) { 3882193323Sed // Memcpy with size zero? Just return the original chain. 3883193323Sed if (ConstantSize->isNullValue()) 3884193323Sed return Chain; 3885193323Sed 3886206083Srdivacky SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3887206083Srdivacky ConstantSize->getZExtValue(),Align, 3888218893Sdim isVol, false, DstPtrInfo, SrcPtrInfo); 3889193323Sed if (Result.getNode()) 3890193323Sed return Result; 3891193323Sed } 3892193323Sed 3893193323Sed // Then check to see if we should lower the memcpy with target-specific 3894193323Sed // code. If the target chooses to do this, this is the next best. 3895193323Sed SDValue Result = 3896208599Srdivacky TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 3897206274Srdivacky isVol, AlwaysInline, 3898218893Sdim DstPtrInfo, SrcPtrInfo); 3899193323Sed if (Result.getNode()) 3900193323Sed return Result; 3901193323Sed 3902193323Sed // If we really need inline code and the target declined to provide it, 3903193323Sed // use a (potentially long) sequence of loads and stores. 3904193323Sed if (AlwaysInline) { 3905193323Sed assert(ConstantSize && "AlwaysInline requires a constant size!"); 3906193323Sed return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3907206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 3908218893Sdim true, DstPtrInfo, SrcPtrInfo); 3909193323Sed } 3910193323Sed 3911206274Srdivacky // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 3912206274Srdivacky // memcpy is not guaranteed to be safe. libc memcpys aren't required to 3913206274Srdivacky // respect volatile, so they may do things like read or write memory 3914206274Srdivacky // beyond the given memory regions. But fixing this isn't easy, and most 3915206274Srdivacky // people don't care. 3916206274Srdivacky 3917193323Sed // Emit a library call. 3918193323Sed TargetLowering::ArgListTy Args; 3919193323Sed TargetLowering::ArgListEntry Entry; 3920243830Sdim Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext()); 3921193323Sed Entry.Node = Dst; Args.push_back(Entry); 3922193323Sed Entry.Node = Src; Args.push_back(Entry); 3923193323Sed Entry.Node = Size; Args.push_back(Entry); 3924193323Sed // FIXME: pass in DebugLoc 3925239462Sdim TargetLowering:: 3926239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 3927198090Srdivacky false, false, false, false, 0, 3928234353Sdim TLI.getLibcallCallingConv(RTLIB::MEMCPY), 3929234353Sdim /*isTailCall=*/false, 3930234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 3931198090Srdivacky getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY), 3932198090Srdivacky TLI.getPointerTy()), 3933204642Srdivacky Args, *this, dl); 3934239462Sdim std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI); 3935239462Sdim 3936193323Sed return CallResult.second; 3937193323Sed} 3938193323Sed 3939193323SedSDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, 3940193323Sed SDValue Src, SDValue Size, 3941206274Srdivacky unsigned Align, bool isVol, 3942218893Sdim MachinePointerInfo DstPtrInfo, 3943218893Sdim MachinePointerInfo SrcPtrInfo) { 3944249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 3945193323Sed 3946193323Sed // Check to see if we should lower the memmove to loads and stores first. 3947193323Sed // For cases within the target-specified limits, this is the best choice. 3948193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3949193323Sed if (ConstantSize) { 3950193323Sed // Memmove with size zero? Just return the original chain. 3951193323Sed if (ConstantSize->isNullValue()) 3952193323Sed return Chain; 3953193323Sed 3954193323Sed SDValue Result = 3955193323Sed getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 3956206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 3957218893Sdim false, DstPtrInfo, SrcPtrInfo); 3958193323Sed if (Result.getNode()) 3959193323Sed return Result; 3960193323Sed } 3961193323Sed 3962193323Sed // Then check to see if we should lower the memmove with target-specific 3963193323Sed // code. If the target chooses to do this, this is the next best. 3964193323Sed SDValue Result = 3965208599Srdivacky TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 3966218893Sdim DstPtrInfo, SrcPtrInfo); 3967193323Sed if (Result.getNode()) 3968193323Sed return Result; 3969193323Sed 3970207618Srdivacky // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 3971207618Srdivacky // not be safe. See memcpy above for more details. 3972207618Srdivacky 3973193323Sed // Emit a library call. 3974193323Sed TargetLowering::ArgListTy Args; 3975193323Sed TargetLowering::ArgListEntry Entry; 3976243830Sdim Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext()); 3977193323Sed Entry.Node = Dst; Args.push_back(Entry); 3978193323Sed Entry.Node = Src; Args.push_back(Entry); 3979193323Sed Entry.Node = Size; Args.push_back(Entry); 3980193323Sed // FIXME: pass in DebugLoc 3981239462Sdim TargetLowering:: 3982239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 3983198090Srdivacky false, false, false, false, 0, 3984234353Sdim TLI.getLibcallCallingConv(RTLIB::MEMMOVE), 3985234353Sdim /*isTailCall=*/false, 3986234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 3987198090Srdivacky getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE), 3988198090Srdivacky TLI.getPointerTy()), 3989204642Srdivacky Args, *this, dl); 3990239462Sdim std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI); 3991239462Sdim 3992193323Sed return CallResult.second; 3993193323Sed} 3994193323Sed 3995193323SedSDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, 3996193323Sed SDValue Src, SDValue Size, 3997206274Srdivacky unsigned Align, bool isVol, 3998218893Sdim MachinePointerInfo DstPtrInfo) { 3999249423Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4000193323Sed 4001193323Sed // Check to see if we should lower the memset to stores first. 4002193323Sed // For cases within the target-specified limits, this is the best choice. 4003193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4004193323Sed if (ConstantSize) { 4005193323Sed // Memset with size zero? Just return the original chain. 4006193323Sed if (ConstantSize->isNullValue()) 4007193323Sed return Chain; 4008193323Sed 4009206274Srdivacky SDValue Result = 4010206274Srdivacky getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 4011218893Sdim Align, isVol, DstPtrInfo); 4012206274Srdivacky 4013193323Sed if (Result.getNode()) 4014193323Sed return Result; 4015193323Sed } 4016193323Sed 4017193323Sed // Then check to see if we should lower the memset with target-specific 4018193323Sed // code. If the target chooses to do this, this is the next best. 4019193323Sed SDValue Result = 4020208599Srdivacky TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4021218893Sdim DstPtrInfo); 4022193323Sed if (Result.getNode()) 4023193323Sed return Result; 4024193323Sed 4025218893Sdim // Emit a library call. 4026243830Sdim Type *IntPtrTy = TLI.getDataLayout()->getIntPtrType(*getContext()); 4027193323Sed TargetLowering::ArgListTy Args; 4028193323Sed TargetLowering::ArgListEntry Entry; 4029193323Sed Entry.Node = Dst; Entry.Ty = IntPtrTy; 4030193323Sed Args.push_back(Entry); 4031193323Sed // Extend or truncate the argument to be an i32 value for the call. 4032193323Sed if (Src.getValueType().bitsGT(MVT::i32)) 4033193323Sed Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 4034193323Sed else 4035193323Sed Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 4036198090Srdivacky Entry.Node = Src; 4037198090Srdivacky Entry.Ty = Type::getInt32Ty(*getContext()); 4038198090Srdivacky Entry.isSExt = true; 4039193323Sed Args.push_back(Entry); 4040198090Srdivacky Entry.Node = Size; 4041198090Srdivacky Entry.Ty = IntPtrTy; 4042198090Srdivacky Entry.isSExt = false; 4043193323Sed Args.push_back(Entry); 4044193323Sed // FIXME: pass in DebugLoc 4045239462Sdim TargetLowering:: 4046239462Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4047198090Srdivacky false, false, false, false, 0, 4048234353Sdim TLI.getLibcallCallingConv(RTLIB::MEMSET), 4049234353Sdim /*isTailCall=*/false, 4050234353Sdim /*doesNotReturn*/false, /*isReturnValueUsed=*/false, 4051198090Srdivacky getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET), 4052198090Srdivacky TLI.getPointerTy()), 4053204642Srdivacky Args, *this, dl); 4054239462Sdim std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI); 4055239462Sdim 4056193323Sed return CallResult.second; 4057193323Sed} 4058193323Sed 4059198090SrdivackySDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4060218893Sdim SDValue Chain, SDValue Ptr, SDValue Cmp, 4061218893Sdim SDValue Swp, MachinePointerInfo PtrInfo, 4062226633Sdim unsigned Alignment, 4063226633Sdim AtomicOrdering Ordering, 4064243830Sdim SynchronizationScope SynchScope) { 4065198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4066198090Srdivacky Alignment = getEVTAlignment(MemVT); 4067198090Srdivacky 4068198090Srdivacky MachineFunction &MF = getMachineFunction(); 4069198090Srdivacky 4070243830Sdim // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. 4071198090Srdivacky // For now, atomics are considered to be volatile always. 4072226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4073226633Sdim // orderings in the memoperand. 4074243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4075243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4076243830Sdim Flags |= MachineMemOperand::MOLoad; 4077243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4078243830Sdim Flags |= MachineMemOperand::MOStore; 4079198090Srdivacky 4080198090Srdivacky MachineMemOperand *MMO = 4081218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 4082198090Srdivacky 4083226633Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 4084226633Sdim Ordering, SynchScope); 4085198090Srdivacky} 4086198090Srdivacky 4087198090SrdivackySDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4088198090Srdivacky SDValue Chain, 4089198090Srdivacky SDValue Ptr, SDValue Cmp, 4090226633Sdim SDValue Swp, MachineMemOperand *MMO, 4091226633Sdim AtomicOrdering Ordering, 4092226633Sdim SynchronizationScope SynchScope) { 4093193323Sed assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 4094193323Sed assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 4095193323Sed 4096198090Srdivacky EVT VT = Cmp.getValueType(); 4097193323Sed 4098193323Sed SDVTList VTs = getVTList(VT, MVT::Other); 4099193323Sed FoldingSetNodeID ID; 4100193323Sed ID.AddInteger(MemVT.getRawBits()); 4101193323Sed SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 4102193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 4); 4103239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4104193323Sed void* IP = 0; 4105198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4106198090Srdivacky cast<AtomicSDNode>(E)->refineAlignment(MMO); 4107193323Sed return SDValue(E, 0); 4108198090Srdivacky } 4109205407Srdivacky SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 4110226633Sdim Ptr, Cmp, Swp, MMO, Ordering, 4111226633Sdim SynchScope); 4112193323Sed CSEMap.InsertNode(N, IP); 4113193323Sed AllNodes.push_back(N); 4114193323Sed return SDValue(N, 0); 4115193323Sed} 4116193323Sed 4117198090SrdivackySDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4118193323Sed SDValue Chain, 4119193323Sed SDValue Ptr, SDValue Val, 4120193323Sed const Value* PtrVal, 4121226633Sdim unsigned Alignment, 4122226633Sdim AtomicOrdering Ordering, 4123226633Sdim SynchronizationScope SynchScope) { 4124198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4125198090Srdivacky Alignment = getEVTAlignment(MemVT); 4126198090Srdivacky 4127198090Srdivacky MachineFunction &MF = getMachineFunction(); 4128243830Sdim // An atomic store does not load. An atomic load does not store. 4129226633Sdim // (An atomicrmw obviously both loads and stores.) 4130243830Sdim // For now, atomics are considered to be volatile always, and they are 4131243830Sdim // chained as such. 4132226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4133226633Sdim // orderings in the memoperand. 4134243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4135243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4136243830Sdim Flags |= MachineMemOperand::MOLoad; 4137243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4138243830Sdim Flags |= MachineMemOperand::MOStore; 4139198090Srdivacky 4140198090Srdivacky MachineMemOperand *MMO = 4141218893Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4142198090Srdivacky MemVT.getStoreSize(), Alignment); 4143198090Srdivacky 4144226633Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 4145226633Sdim Ordering, SynchScope); 4146198090Srdivacky} 4147198090Srdivacky 4148198090SrdivackySDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4149198090Srdivacky SDValue Chain, 4150198090Srdivacky SDValue Ptr, SDValue Val, 4151226633Sdim MachineMemOperand *MMO, 4152226633Sdim AtomicOrdering Ordering, 4153226633Sdim SynchronizationScope SynchScope) { 4154193323Sed assert((Opcode == ISD::ATOMIC_LOAD_ADD || 4155193323Sed Opcode == ISD::ATOMIC_LOAD_SUB || 4156193323Sed Opcode == ISD::ATOMIC_LOAD_AND || 4157193323Sed Opcode == ISD::ATOMIC_LOAD_OR || 4158193323Sed Opcode == ISD::ATOMIC_LOAD_XOR || 4159193323Sed Opcode == ISD::ATOMIC_LOAD_NAND || 4160193323Sed Opcode == ISD::ATOMIC_LOAD_MIN || 4161193323Sed Opcode == ISD::ATOMIC_LOAD_MAX || 4162193323Sed Opcode == ISD::ATOMIC_LOAD_UMIN || 4163193323Sed Opcode == ISD::ATOMIC_LOAD_UMAX || 4164226633Sdim Opcode == ISD::ATOMIC_SWAP || 4165226633Sdim Opcode == ISD::ATOMIC_STORE) && 4166193323Sed "Invalid Atomic Op"); 4167193323Sed 4168198090Srdivacky EVT VT = Val.getValueType(); 4169193323Sed 4170226633Sdim SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 4171226633Sdim getVTList(VT, MVT::Other); 4172193323Sed FoldingSetNodeID ID; 4173193323Sed ID.AddInteger(MemVT.getRawBits()); 4174193323Sed SDValue Ops[] = {Chain, Ptr, Val}; 4175193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 4176239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4177193323Sed void* IP = 0; 4178198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4179198090Srdivacky cast<AtomicSDNode>(E)->refineAlignment(MMO); 4180193323Sed return SDValue(E, 0); 4181198090Srdivacky } 4182205407Srdivacky SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 4183226633Sdim Ptr, Val, MMO, 4184226633Sdim Ordering, SynchScope); 4185193323Sed CSEMap.InsertNode(N, IP); 4186193323Sed AllNodes.push_back(N); 4187193323Sed return SDValue(N, 0); 4188193323Sed} 4189193323Sed 4190226633SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4191226633Sdim EVT VT, SDValue Chain, 4192226633Sdim SDValue Ptr, 4193226633Sdim const Value* PtrVal, 4194226633Sdim unsigned Alignment, 4195226633Sdim AtomicOrdering Ordering, 4196226633Sdim SynchronizationScope SynchScope) { 4197226633Sdim if (Alignment == 0) // Ensure that codegen never sees alignment 0 4198226633Sdim Alignment = getEVTAlignment(MemVT); 4199226633Sdim 4200226633Sdim MachineFunction &MF = getMachineFunction(); 4201243830Sdim // An atomic store does not load. An atomic load does not store. 4202243830Sdim // (An atomicrmw obviously both loads and stores.) 4203243830Sdim // For now, atomics are considered to be volatile always, and they are 4204243830Sdim // chained as such. 4205226633Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4206226633Sdim // orderings in the memoperand. 4207243830Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4208243830Sdim if (Opcode != ISD::ATOMIC_STORE) 4209243830Sdim Flags |= MachineMemOperand::MOLoad; 4210243830Sdim if (Opcode != ISD::ATOMIC_LOAD) 4211243830Sdim Flags |= MachineMemOperand::MOStore; 4212226633Sdim 4213226633Sdim MachineMemOperand *MMO = 4214226633Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4215226633Sdim MemVT.getStoreSize(), Alignment); 4216226633Sdim 4217226633Sdim return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4218226633Sdim Ordering, SynchScope); 4219226633Sdim} 4220226633Sdim 4221226633SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, 4222226633Sdim EVT VT, SDValue Chain, 4223226633Sdim SDValue Ptr, 4224226633Sdim MachineMemOperand *MMO, 4225226633Sdim AtomicOrdering Ordering, 4226226633Sdim SynchronizationScope SynchScope) { 4227226633Sdim assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4228226633Sdim 4229226633Sdim SDVTList VTs = getVTList(VT, MVT::Other); 4230226633Sdim FoldingSetNodeID ID; 4231226633Sdim ID.AddInteger(MemVT.getRawBits()); 4232226633Sdim SDValue Ops[] = {Chain, Ptr}; 4233226633Sdim AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 4234239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4235226633Sdim void* IP = 0; 4236226633Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4237226633Sdim cast<AtomicSDNode>(E)->refineAlignment(MMO); 4238226633Sdim return SDValue(E, 0); 4239226633Sdim } 4240226633Sdim SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, 4241226633Sdim Ptr, MMO, Ordering, SynchScope); 4242226633Sdim CSEMap.InsertNode(N, IP); 4243226633Sdim AllNodes.push_back(N); 4244226633Sdim return SDValue(N, 0); 4245226633Sdim} 4246226633Sdim 4247193323Sed/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4248193323SedSDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4249193323Sed DebugLoc dl) { 4250193323Sed if (NumOps == 1) 4251193323Sed return Ops[0]; 4252193323Sed 4253198090Srdivacky SmallVector<EVT, 4> VTs; 4254193323Sed VTs.reserve(NumOps); 4255193323Sed for (unsigned i = 0; i < NumOps; ++i) 4256193323Sed VTs.push_back(Ops[i].getValueType()); 4257193323Sed return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4258193323Sed Ops, NumOps); 4259193323Sed} 4260193323Sed 4261193323SedSDValue 4262193323SedSelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, 4263198090Srdivacky const EVT *VTs, unsigned NumVTs, 4264193323Sed const SDValue *Ops, unsigned NumOps, 4265218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4266193323Sed unsigned Align, bool Vol, 4267193323Sed bool ReadMem, bool WriteMem) { 4268193323Sed return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4269218893Sdim MemVT, PtrInfo, Align, Vol, 4270193323Sed ReadMem, WriteMem); 4271193323Sed} 4272193323Sed 4273193323SedSDValue 4274193323SedSelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4275193323Sed const SDValue *Ops, unsigned NumOps, 4276218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4277193323Sed unsigned Align, bool Vol, 4278193323Sed bool ReadMem, bool WriteMem) { 4279198090Srdivacky if (Align == 0) // Ensure that codegen never sees alignment 0 4280198090Srdivacky Align = getEVTAlignment(MemVT); 4281198090Srdivacky 4282198090Srdivacky MachineFunction &MF = getMachineFunction(); 4283198090Srdivacky unsigned Flags = 0; 4284198090Srdivacky if (WriteMem) 4285198090Srdivacky Flags |= MachineMemOperand::MOStore; 4286198090Srdivacky if (ReadMem) 4287198090Srdivacky Flags |= MachineMemOperand::MOLoad; 4288198090Srdivacky if (Vol) 4289198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4290198090Srdivacky MachineMemOperand *MMO = 4291218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4292198090Srdivacky 4293198090Srdivacky return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4294198090Srdivacky} 4295198090Srdivacky 4296198090SrdivackySDValue 4297198090SrdivackySelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, 4298198090Srdivacky const SDValue *Ops, unsigned NumOps, 4299198090Srdivacky EVT MemVT, MachineMemOperand *MMO) { 4300198090Srdivacky assert((Opcode == ISD::INTRINSIC_VOID || 4301198090Srdivacky Opcode == ISD::INTRINSIC_W_CHAIN || 4302218893Sdim Opcode == ISD::PREFETCH || 4303243830Sdim Opcode == ISD::LIFETIME_START || 4304243830Sdim Opcode == ISD::LIFETIME_END || 4305198090Srdivacky (Opcode <= INT_MAX && 4306198090Srdivacky (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4307198090Srdivacky "Opcode is not a memory-accessing opcode!"); 4308198090Srdivacky 4309193323Sed // Memoize the node unless it returns a flag. 4310193323Sed MemIntrinsicSDNode *N; 4311218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4312193323Sed FoldingSetNodeID ID; 4313193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4314239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4315193323Sed void *IP = 0; 4316198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4317198090Srdivacky cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4318193323Sed return SDValue(E, 0); 4319198090Srdivacky } 4320193323Sed 4321205407Srdivacky N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4322205407Srdivacky MemVT, MMO); 4323193323Sed CSEMap.InsertNode(N, IP); 4324193323Sed } else { 4325205407Srdivacky N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, 4326205407Srdivacky MemVT, MMO); 4327193323Sed } 4328193323Sed AllNodes.push_back(N); 4329193323Sed return SDValue(N, 0); 4330193323Sed} 4331193323Sed 4332218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4333218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4334218893Sdim/// code generator has many cases where it doesn't bother passing in a 4335218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4336218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4337218893Sdim // If this is FI+Offset, we can model it. 4338218893Sdim if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4339218893Sdim return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4340218893Sdim 4341218893Sdim // If this is (FI+Offset1)+Offset2, we can model it. 4342218893Sdim if (Ptr.getOpcode() != ISD::ADD || 4343218893Sdim !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4344218893Sdim !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4345218893Sdim return MachinePointerInfo(); 4346218893Sdim 4347218893Sdim int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4348218893Sdim return MachinePointerInfo::getFixedStack(FI, Offset+ 4349218893Sdim cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4350218893Sdim} 4351218893Sdim 4352218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4353218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4354218893Sdim/// code generator has many cases where it doesn't bother passing in a 4355218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4356218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4357218893Sdim // If the 'Offset' value isn't a constant, we can't handle this. 4358218893Sdim if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4359218893Sdim return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4360218893Sdim if (OffsetOp.getOpcode() == ISD::UNDEF) 4361218893Sdim return InferPointerInfo(Ptr); 4362218893Sdim return MachinePointerInfo(); 4363218893Sdim} 4364218893Sdim 4365218893Sdim 4366193323SedSDValue 4367210299SedSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4368210299Sed EVT VT, DebugLoc dl, SDValue Chain, 4369193323Sed SDValue Ptr, SDValue Offset, 4370218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4371234353Sdim bool isVolatile, bool isNonTemporal, bool isInvariant, 4372234353Sdim unsigned Alignment, const MDNode *TBAAInfo, 4373234353Sdim const MDNode *Ranges) { 4374243830Sdim assert(Chain.getValueType() == MVT::Other && 4375224145Sdim "Invalid chain type"); 4376193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4377198090Srdivacky Alignment = getEVTAlignment(VT); 4378193323Sed 4379198090Srdivacky unsigned Flags = MachineMemOperand::MOLoad; 4380198090Srdivacky if (isVolatile) 4381198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4382203954Srdivacky if (isNonTemporal) 4383203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4384234353Sdim if (isInvariant) 4385234353Sdim Flags |= MachineMemOperand::MOInvariant; 4386218893Sdim 4387218893Sdim // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4388218893Sdim // clients. 4389218893Sdim if (PtrInfo.V == 0) 4390218893Sdim PtrInfo = InferPointerInfo(Ptr, Offset); 4391218893Sdim 4392218893Sdim MachineFunction &MF = getMachineFunction(); 4393198090Srdivacky MachineMemOperand *MMO = 4394218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4395234353Sdim TBAAInfo, Ranges); 4396210299Sed return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4397198090Srdivacky} 4398198090Srdivacky 4399198090SrdivackySDValue 4400218893SdimSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4401210299Sed EVT VT, DebugLoc dl, SDValue Chain, 4402198090Srdivacky SDValue Ptr, SDValue Offset, EVT MemVT, 4403198090Srdivacky MachineMemOperand *MMO) { 4404198090Srdivacky if (VT == MemVT) { 4405193323Sed ExtType = ISD::NON_EXTLOAD; 4406193323Sed } else if (ExtType == ISD::NON_EXTLOAD) { 4407198090Srdivacky assert(VT == MemVT && "Non-extending load from different memory type!"); 4408193323Sed } else { 4409193323Sed // Extending load. 4410200581Srdivacky assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4411200581Srdivacky "Should only be an extending load, not truncating!"); 4412198090Srdivacky assert(VT.isInteger() == MemVT.isInteger() && 4413193323Sed "Cannot convert from FP to Int or Int -> FP!"); 4414200581Srdivacky assert(VT.isVector() == MemVT.isVector() && 4415200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4416200581Srdivacky assert((!VT.isVector() || 4417200581Srdivacky VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4418200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4419193323Sed } 4420193323Sed 4421193323Sed bool Indexed = AM != ISD::UNINDEXED; 4422193323Sed assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4423193323Sed "Unindexed load with an offset!"); 4424193323Sed 4425193323Sed SDVTList VTs = Indexed ? 4426193323Sed getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4427193323Sed SDValue Ops[] = { Chain, Ptr, Offset }; 4428193323Sed FoldingSetNodeID ID; 4429193323Sed AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4430198090Srdivacky ID.AddInteger(MemVT.getRawBits()); 4431204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4432243830Sdim MMO->isNonTemporal(), 4433234353Sdim MMO->isInvariant())); 4434239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4435193323Sed void *IP = 0; 4436198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4437198090Srdivacky cast<LoadSDNode>(E)->refineAlignment(MMO); 4438193323Sed return SDValue(E, 0); 4439198090Srdivacky } 4440205407Srdivacky SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl, VTs, AM, ExtType, 4441205407Srdivacky MemVT, MMO); 4442193323Sed CSEMap.InsertNode(N, IP); 4443193323Sed AllNodes.push_back(N); 4444193323Sed return SDValue(N, 0); 4445193323Sed} 4446193323Sed 4447198090SrdivackySDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, 4448193323Sed SDValue Chain, SDValue Ptr, 4449218893Sdim MachinePointerInfo PtrInfo, 4450203954Srdivacky bool isVolatile, bool isNonTemporal, 4451243830Sdim bool isInvariant, unsigned Alignment, 4452234353Sdim const MDNode *TBAAInfo, 4453234353Sdim const MDNode *Ranges) { 4454193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4455210299Sed return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4456234353Sdim PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4457234353Sdim TBAAInfo, Ranges); 4458193323Sed} 4459193323Sed 4460218893SdimSDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, 4461193323Sed SDValue Chain, SDValue Ptr, 4462218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4463203954Srdivacky bool isVolatile, bool isNonTemporal, 4464218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4465193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4466210299Sed return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4467234353Sdim PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4468218893Sdim TBAAInfo); 4469193323Sed} 4470193323Sed 4471218893Sdim 4472193323SedSDValue 4473193323SedSelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, 4474193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4475193323Sed LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4476193323Sed assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4477193323Sed "Load is already a indexed load!"); 4478210299Sed return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4479218893Sdim LD->getChain(), Base, Offset, LD->getPointerInfo(), 4480243830Sdim LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4481234353Sdim false, LD->getAlignment()); 4482193323Sed} 4483193323Sed 4484193323SedSDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4485218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4486203954Srdivacky bool isVolatile, bool isNonTemporal, 4487218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4488243830Sdim assert(Chain.getValueType() == MVT::Other && 4489224145Sdim "Invalid chain type"); 4490193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4491198090Srdivacky Alignment = getEVTAlignment(Val.getValueType()); 4492193323Sed 4493198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4494198090Srdivacky if (isVolatile) 4495198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4496203954Srdivacky if (isNonTemporal) 4497203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4498218893Sdim 4499218893Sdim if (PtrInfo.V == 0) 4500218893Sdim PtrInfo = InferPointerInfo(Ptr); 4501218893Sdim 4502218893Sdim MachineFunction &MF = getMachineFunction(); 4503198090Srdivacky MachineMemOperand *MMO = 4504218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, 4505218893Sdim Val.getValueType().getStoreSize(), Alignment, 4506218893Sdim TBAAInfo); 4507198090Srdivacky 4508198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4509198090Srdivacky} 4510198090Srdivacky 4511198090SrdivackySDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, 4512198090Srdivacky SDValue Ptr, MachineMemOperand *MMO) { 4513243830Sdim assert(Chain.getValueType() == MVT::Other && 4514224145Sdim "Invalid chain type"); 4515198090Srdivacky EVT VT = Val.getValueType(); 4516193323Sed SDVTList VTs = getVTList(MVT::Other); 4517193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4518193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4519193323Sed FoldingSetNodeID ID; 4520193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4521193323Sed ID.AddInteger(VT.getRawBits()); 4522204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4523234353Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4524239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4525193323Sed void *IP = 0; 4526198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4527198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4528193323Sed return SDValue(E, 0); 4529198090Srdivacky } 4530205407Srdivacky SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4531205407Srdivacky false, VT, MMO); 4532193323Sed CSEMap.InsertNode(N, IP); 4533193323Sed AllNodes.push_back(N); 4534193323Sed return SDValue(N, 0); 4535193323Sed} 4536193323Sed 4537193323SedSDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4538218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4539218893Sdim EVT SVT,bool isVolatile, bool isNonTemporal, 4540218893Sdim unsigned Alignment, 4541218893Sdim const MDNode *TBAAInfo) { 4542243830Sdim assert(Chain.getValueType() == MVT::Other && 4543224145Sdim "Invalid chain type"); 4544198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4545198090Srdivacky Alignment = getEVTAlignment(SVT); 4546193323Sed 4547198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4548198090Srdivacky if (isVolatile) 4549198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4550203954Srdivacky if (isNonTemporal) 4551203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4552218893Sdim 4553218893Sdim if (PtrInfo.V == 0) 4554218893Sdim PtrInfo = InferPointerInfo(Ptr); 4555218893Sdim 4556218893Sdim MachineFunction &MF = getMachineFunction(); 4557198090Srdivacky MachineMemOperand *MMO = 4558218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4559218893Sdim TBAAInfo); 4560198090Srdivacky 4561198090Srdivacky return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4562198090Srdivacky} 4563198090Srdivacky 4564198090SrdivackySDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, 4565198090Srdivacky SDValue Ptr, EVT SVT, 4566198090Srdivacky MachineMemOperand *MMO) { 4567198090Srdivacky EVT VT = Val.getValueType(); 4568198090Srdivacky 4569243830Sdim assert(Chain.getValueType() == MVT::Other && 4570224145Sdim "Invalid chain type"); 4571193323Sed if (VT == SVT) 4572198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4573193323Sed 4574200581Srdivacky assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4575200581Srdivacky "Should only be a truncating store, not extending!"); 4576193323Sed assert(VT.isInteger() == SVT.isInteger() && 4577193323Sed "Can't do FP-INT conversion!"); 4578200581Srdivacky assert(VT.isVector() == SVT.isVector() && 4579200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4580200581Srdivacky assert((!VT.isVector() || 4581200581Srdivacky VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4582200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4583193323Sed 4584193323Sed SDVTList VTs = getVTList(MVT::Other); 4585193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4586193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4587193323Sed FoldingSetNodeID ID; 4588193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4589193323Sed ID.AddInteger(SVT.getRawBits()); 4590204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4591234353Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4592239462Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4593193323Sed void *IP = 0; 4594198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4595198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4596193323Sed return SDValue(E, 0); 4597198090Srdivacky } 4598205407Srdivacky SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, 4599205407Srdivacky true, SVT, MMO); 4600193323Sed CSEMap.InsertNode(N, IP); 4601193323Sed AllNodes.push_back(N); 4602193323Sed return SDValue(N, 0); 4603193323Sed} 4604193323Sed 4605193323SedSDValue 4606193323SedSelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base, 4607193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4608193323Sed StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4609193323Sed assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4610193323Sed "Store is already a indexed store!"); 4611193323Sed SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4612193323Sed SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4613193323Sed FoldingSetNodeID ID; 4614193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4615193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 4616193323Sed ID.AddInteger(ST->getRawSubclassData()); 4617239462Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 4618193323Sed void *IP = 0; 4619201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4620193323Sed return SDValue(E, 0); 4621201360Srdivacky 4622205407Srdivacky SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, AM, 4623205407Srdivacky ST->isTruncatingStore(), 4624205407Srdivacky ST->getMemoryVT(), 4625205407Srdivacky ST->getMemOperand()); 4626193323Sed CSEMap.InsertNode(N, IP); 4627193323Sed AllNodes.push_back(N); 4628193323Sed return SDValue(N, 0); 4629193323Sed} 4630193323Sed 4631198090SrdivackySDValue SelectionDAG::getVAArg(EVT VT, DebugLoc dl, 4632193323Sed SDValue Chain, SDValue Ptr, 4633210299Sed SDValue SV, 4634210299Sed unsigned Align) { 4635210299Sed SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4636210299Sed return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4637193323Sed} 4638193323Sed 4639198090SrdivackySDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4640193323Sed const SDUse *Ops, unsigned NumOps) { 4641193323Sed switch (NumOps) { 4642193323Sed case 0: return getNode(Opcode, DL, VT); 4643193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4644193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4645193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4646193323Sed default: break; 4647193323Sed } 4648193323Sed 4649193323Sed // Copy from an SDUse array into an SDValue array for use with 4650193323Sed // the regular getNode logic. 4651193323Sed SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4652193323Sed return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4653193323Sed} 4654193323Sed 4655198090SrdivackySDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, 4656193323Sed const SDValue *Ops, unsigned NumOps) { 4657193323Sed switch (NumOps) { 4658193323Sed case 0: return getNode(Opcode, DL, VT); 4659193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4660193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4661193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4662193323Sed default: break; 4663193323Sed } 4664193323Sed 4665193323Sed switch (Opcode) { 4666193323Sed default: break; 4667193323Sed case ISD::SELECT_CC: { 4668193323Sed assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4669193323Sed assert(Ops[0].getValueType() == Ops[1].getValueType() && 4670193323Sed "LHS and RHS of condition must have same type!"); 4671193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4672193323Sed "True and False arms of SelectCC must have same type!"); 4673193323Sed assert(Ops[2].getValueType() == VT && 4674193323Sed "select_cc node must be of same type as true and false value!"); 4675193323Sed break; 4676193323Sed } 4677193323Sed case ISD::BR_CC: { 4678193323Sed assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4679193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4680193323Sed "LHS/RHS of comparison should match types!"); 4681193323Sed break; 4682193323Sed } 4683193323Sed } 4684193323Sed 4685193323Sed // Memoize nodes. 4686193323Sed SDNode *N; 4687193323Sed SDVTList VTs = getVTList(VT); 4688193323Sed 4689218893Sdim if (VT != MVT::Glue) { 4690193323Sed FoldingSetNodeID ID; 4691193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4692193323Sed void *IP = 0; 4693193323Sed 4694201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4695193323Sed return SDValue(E, 0); 4696193323Sed 4697205407Srdivacky N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4698193323Sed CSEMap.InsertNode(N, IP); 4699193323Sed } else { 4700205407Srdivacky N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); 4701193323Sed } 4702193323Sed 4703193323Sed AllNodes.push_back(N); 4704193323Sed#ifndef NDEBUG 4705218893Sdim VerifySDNode(N); 4706193323Sed#endif 4707193323Sed return SDValue(N, 0); 4708193323Sed} 4709193323Sed 4710193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4711249423Sdim ArrayRef<EVT> ResultTys, 4712193323Sed const SDValue *Ops, unsigned NumOps) { 4713193323Sed return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4714193323Sed Ops, NumOps); 4715193323Sed} 4716193323Sed 4717193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, 4718198090Srdivacky const EVT *VTs, unsigned NumVTs, 4719193323Sed const SDValue *Ops, unsigned NumOps) { 4720193323Sed if (NumVTs == 1) 4721193323Sed return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4722193323Sed return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4723193323Sed} 4724193323Sed 4725193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4726193323Sed const SDValue *Ops, unsigned NumOps) { 4727193323Sed if (VTList.NumVTs == 1) 4728193323Sed return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4729193323Sed 4730198090Srdivacky#if 0 4731193323Sed switch (Opcode) { 4732193323Sed // FIXME: figure out how to safely handle things like 4733193323Sed // int foo(int x) { return 1 << (x & 255); } 4734193323Sed // int bar() { return foo(256); } 4735193323Sed case ISD::SRA_PARTS: 4736193323Sed case ISD::SRL_PARTS: 4737193323Sed case ISD::SHL_PARTS: 4738193323Sed if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4739193323Sed cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4740193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4741193323Sed else if (N3.getOpcode() == ISD::AND) 4742193323Sed if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4743193323Sed // If the and is only masking out bits that cannot effect the shift, 4744193323Sed // eliminate the and. 4745202375Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4746193323Sed if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4747193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4748193323Sed } 4749193323Sed break; 4750198090Srdivacky } 4751193323Sed#endif 4752193323Sed 4753193323Sed // Memoize the node unless it returns a flag. 4754193323Sed SDNode *N; 4755218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4756193323Sed FoldingSetNodeID ID; 4757193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4758193323Sed void *IP = 0; 4759201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4760193323Sed return SDValue(E, 0); 4761201360Srdivacky 4762193323Sed if (NumOps == 1) { 4763205407Srdivacky N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4764193323Sed } else if (NumOps == 2) { 4765205407Srdivacky N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4766193323Sed } else if (NumOps == 3) { 4767205407Srdivacky N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4768205407Srdivacky Ops[2]); 4769193323Sed } else { 4770205407Srdivacky N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4771193323Sed } 4772193323Sed CSEMap.InsertNode(N, IP); 4773193323Sed } else { 4774193323Sed if (NumOps == 1) { 4775205407Srdivacky N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); 4776193323Sed } else if (NumOps == 2) { 4777205407Srdivacky N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); 4778193323Sed } else if (NumOps == 3) { 4779205407Srdivacky N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], 4780205407Srdivacky Ops[2]); 4781193323Sed } else { 4782205407Srdivacky N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); 4783193323Sed } 4784193323Sed } 4785193323Sed AllNodes.push_back(N); 4786193323Sed#ifndef NDEBUG 4787218893Sdim VerifySDNode(N); 4788193323Sed#endif 4789193323Sed return SDValue(N, 0); 4790193323Sed} 4791193323Sed 4792193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList) { 4793193323Sed return getNode(Opcode, DL, VTList, 0, 0); 4794193323Sed} 4795193323Sed 4796193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4797193323Sed SDValue N1) { 4798193323Sed SDValue Ops[] = { N1 }; 4799193323Sed return getNode(Opcode, DL, VTList, Ops, 1); 4800193323Sed} 4801193323Sed 4802193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4803193323Sed SDValue N1, SDValue N2) { 4804193323Sed SDValue Ops[] = { N1, N2 }; 4805193323Sed return getNode(Opcode, DL, VTList, Ops, 2); 4806193323Sed} 4807193323Sed 4808193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4809193323Sed SDValue N1, SDValue N2, SDValue N3) { 4810193323Sed SDValue Ops[] = { N1, N2, N3 }; 4811193323Sed return getNode(Opcode, DL, VTList, Ops, 3); 4812193323Sed} 4813193323Sed 4814193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4815193323Sed SDValue N1, SDValue N2, SDValue N3, 4816193323Sed SDValue N4) { 4817193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 4818193323Sed return getNode(Opcode, DL, VTList, Ops, 4); 4819193323Sed} 4820193323Sed 4821193323SedSDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, 4822193323Sed SDValue N1, SDValue N2, SDValue N3, 4823193323Sed SDValue N4, SDValue N5) { 4824193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4825193323Sed return getNode(Opcode, DL, VTList, Ops, 5); 4826193323Sed} 4827193323Sed 4828198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT) { 4829193323Sed return makeVTList(SDNode::getValueTypeList(VT), 1); 4830193323Sed} 4831193323Sed 4832198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4833193323Sed for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4834193323Sed E = VTList.rend(); I != E; ++I) 4835193323Sed if (I->NumVTs == 2 && I->VTs[0] == VT1 && I->VTs[1] == VT2) 4836193323Sed return *I; 4837193323Sed 4838198090Srdivacky EVT *Array = Allocator.Allocate<EVT>(2); 4839193323Sed Array[0] = VT1; 4840193323Sed Array[1] = VT2; 4841193323Sed SDVTList Result = makeVTList(Array, 2); 4842193323Sed VTList.push_back(Result); 4843193323Sed return Result; 4844193323Sed} 4845193323Sed 4846198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 4847193323Sed for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4848193323Sed E = VTList.rend(); I != E; ++I) 4849193323Sed if (I->NumVTs == 3 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4850193323Sed I->VTs[2] == VT3) 4851193323Sed return *I; 4852193323Sed 4853198090Srdivacky EVT *Array = Allocator.Allocate<EVT>(3); 4854193323Sed Array[0] = VT1; 4855193323Sed Array[1] = VT2; 4856193323Sed Array[2] = VT3; 4857193323Sed SDVTList Result = makeVTList(Array, 3); 4858193323Sed VTList.push_back(Result); 4859193323Sed return Result; 4860193323Sed} 4861193323Sed 4862198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 4863193323Sed for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4864193323Sed E = VTList.rend(); I != E; ++I) 4865193323Sed if (I->NumVTs == 4 && I->VTs[0] == VT1 && I->VTs[1] == VT2 && 4866193323Sed I->VTs[2] == VT3 && I->VTs[3] == VT4) 4867193323Sed return *I; 4868193323Sed 4869200581Srdivacky EVT *Array = Allocator.Allocate<EVT>(4); 4870193323Sed Array[0] = VT1; 4871193323Sed Array[1] = VT2; 4872193323Sed Array[2] = VT3; 4873193323Sed Array[3] = VT4; 4874193323Sed SDVTList Result = makeVTList(Array, 4); 4875193323Sed VTList.push_back(Result); 4876193323Sed return Result; 4877193323Sed} 4878193323Sed 4879198090SrdivackySDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 4880193323Sed switch (NumVTs) { 4881198090Srdivacky case 0: llvm_unreachable("Cannot have nodes without results!"); 4882193323Sed case 1: return getVTList(VTs[0]); 4883193323Sed case 2: return getVTList(VTs[0], VTs[1]); 4884193323Sed case 3: return getVTList(VTs[0], VTs[1], VTs[2]); 4885201360Srdivacky case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]); 4886193323Sed default: break; 4887193323Sed } 4888193323Sed 4889193323Sed for (std::vector<SDVTList>::reverse_iterator I = VTList.rbegin(), 4890193323Sed E = VTList.rend(); I != E; ++I) { 4891193323Sed if (I->NumVTs != NumVTs || VTs[0] != I->VTs[0] || VTs[1] != I->VTs[1]) 4892193323Sed continue; 4893193323Sed 4894239462Sdim if (std::equal(&VTs[2], &VTs[NumVTs], &I->VTs[2])) 4895193323Sed return *I; 4896193323Sed } 4897193323Sed 4898198090Srdivacky EVT *Array = Allocator.Allocate<EVT>(NumVTs); 4899193323Sed std::copy(VTs, VTs+NumVTs, Array); 4900193323Sed SDVTList Result = makeVTList(Array, NumVTs); 4901193323Sed VTList.push_back(Result); 4902193323Sed return Result; 4903193323Sed} 4904193323Sed 4905193323Sed 4906193323Sed/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 4907193323Sed/// specified operands. If the resultant node already exists in the DAG, 4908193323Sed/// this does not modify the specified node, instead it returns the node that 4909193323Sed/// already exists. If the resultant node does not exist in the DAG, the 4910193323Sed/// input node is returned. As a degenerate case, if you specify the same 4911193323Sed/// input operands as the node already has, the input node is returned. 4912210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 4913193323Sed assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 4914193323Sed 4915193323Sed // Check to see if there is no change. 4916210299Sed if (Op == N->getOperand(0)) return N; 4917193323Sed 4918193323Sed // See if the modified node already exists. 4919193323Sed void *InsertPos = 0; 4920193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 4921210299Sed return Existing; 4922193323Sed 4923193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 4924193323Sed if (InsertPos) 4925193323Sed if (!RemoveNodeFromCSEMaps(N)) 4926193323Sed InsertPos = 0; 4927193323Sed 4928193323Sed // Now we update the operands. 4929193323Sed N->OperandList[0].set(Op); 4930193323Sed 4931193323Sed // If this gets put into a CSE map, add it. 4932193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4933210299Sed return N; 4934193323Sed} 4935193323Sed 4936210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 4937193323Sed assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 4938193323Sed 4939193323Sed // Check to see if there is no change. 4940193323Sed if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 4941210299Sed return N; // No operands changed, just return the input node. 4942193323Sed 4943193323Sed // See if the modified node already exists. 4944193323Sed void *InsertPos = 0; 4945193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 4946210299Sed return Existing; 4947193323Sed 4948193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 4949193323Sed if (InsertPos) 4950193323Sed if (!RemoveNodeFromCSEMaps(N)) 4951193323Sed InsertPos = 0; 4952193323Sed 4953193323Sed // Now we update the operands. 4954193323Sed if (N->OperandList[0] != Op1) 4955193323Sed N->OperandList[0].set(Op1); 4956193323Sed if (N->OperandList[1] != Op2) 4957193323Sed N->OperandList[1].set(Op2); 4958193323Sed 4959193323Sed // If this gets put into a CSE map, add it. 4960193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 4961210299Sed return N; 4962193323Sed} 4963193323Sed 4964210299SedSDNode *SelectionDAG:: 4965210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 4966193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 4967193323Sed return UpdateNodeOperands(N, Ops, 3); 4968193323Sed} 4969193323Sed 4970210299SedSDNode *SelectionDAG:: 4971210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4972193323Sed SDValue Op3, SDValue Op4) { 4973193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 4974193323Sed return UpdateNodeOperands(N, Ops, 4); 4975193323Sed} 4976193323Sed 4977210299SedSDNode *SelectionDAG:: 4978210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 4979193323Sed SDValue Op3, SDValue Op4, SDValue Op5) { 4980193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 4981193323Sed return UpdateNodeOperands(N, Ops, 5); 4982193323Sed} 4983193323Sed 4984210299SedSDNode *SelectionDAG:: 4985210299SedUpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 4986193323Sed assert(N->getNumOperands() == NumOps && 4987193323Sed "Update with wrong number of operands"); 4988193323Sed 4989193323Sed // Check to see if there is no change. 4990193323Sed bool AnyChange = false; 4991193323Sed for (unsigned i = 0; i != NumOps; ++i) { 4992193323Sed if (Ops[i] != N->getOperand(i)) { 4993193323Sed AnyChange = true; 4994193323Sed break; 4995193323Sed } 4996193323Sed } 4997193323Sed 4998193323Sed // No operands changed, just return the input node. 4999210299Sed if (!AnyChange) return N; 5000193323Sed 5001193323Sed // See if the modified node already exists. 5002193323Sed void *InsertPos = 0; 5003193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 5004210299Sed return Existing; 5005193323Sed 5006193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5007193323Sed if (InsertPos) 5008193323Sed if (!RemoveNodeFromCSEMaps(N)) 5009193323Sed InsertPos = 0; 5010193323Sed 5011193323Sed // Now we update the operands. 5012193323Sed for (unsigned i = 0; i != NumOps; ++i) 5013193323Sed if (N->OperandList[i] != Ops[i]) 5014193323Sed N->OperandList[i].set(Ops[i]); 5015193323Sed 5016193323Sed // If this gets put into a CSE map, add it. 5017193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5018210299Sed return N; 5019193323Sed} 5020193323Sed 5021193323Sed/// DropOperands - Release the operands and set this node to have 5022193323Sed/// zero operands. 5023193323Sedvoid SDNode::DropOperands() { 5024193323Sed // Unlike the code in MorphNodeTo that does this, we don't need to 5025193323Sed // watch for dead nodes here. 5026193323Sed for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 5027193323Sed SDUse &Use = *I++; 5028193323Sed Use.set(SDValue()); 5029193323Sed } 5030193323Sed} 5031193323Sed 5032193323Sed/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 5033193323Sed/// machine opcode. 5034193323Sed/// 5035193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5036198090Srdivacky EVT VT) { 5037193323Sed SDVTList VTs = getVTList(VT); 5038193323Sed return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 5039193323Sed} 5040193323Sed 5041193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5042198090Srdivacky EVT VT, SDValue Op1) { 5043193323Sed SDVTList VTs = getVTList(VT); 5044193323Sed SDValue Ops[] = { Op1 }; 5045193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5046193323Sed} 5047193323Sed 5048193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5049198090Srdivacky EVT VT, SDValue Op1, 5050193323Sed SDValue Op2) { 5051193323Sed SDVTList VTs = getVTList(VT); 5052193323Sed SDValue Ops[] = { Op1, Op2 }; 5053193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5054193323Sed} 5055193323Sed 5056193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5057198090Srdivacky EVT VT, SDValue Op1, 5058193323Sed SDValue Op2, SDValue Op3) { 5059193323Sed SDVTList VTs = getVTList(VT); 5060193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5061193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5062193323Sed} 5063193323Sed 5064193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5065198090Srdivacky EVT VT, const SDValue *Ops, 5066193323Sed unsigned NumOps) { 5067193323Sed SDVTList VTs = getVTList(VT); 5068193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5069193323Sed} 5070193323Sed 5071193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5072198090Srdivacky EVT VT1, EVT VT2, const SDValue *Ops, 5073193323Sed unsigned NumOps) { 5074193323Sed SDVTList VTs = getVTList(VT1, VT2); 5075193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5076193323Sed} 5077193323Sed 5078193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5079198090Srdivacky EVT VT1, EVT VT2) { 5080193323Sed SDVTList VTs = getVTList(VT1, VT2); 5081193323Sed return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 5082193323Sed} 5083193323Sed 5084193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5085198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5086193323Sed const SDValue *Ops, unsigned NumOps) { 5087193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5088193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5089193323Sed} 5090193323Sed 5091193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5092198090Srdivacky EVT VT1, EVT VT2, EVT VT3, EVT VT4, 5093193323Sed const SDValue *Ops, unsigned NumOps) { 5094193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5095193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5096193323Sed} 5097193323Sed 5098193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5099198090Srdivacky EVT VT1, EVT VT2, 5100193323Sed SDValue Op1) { 5101193323Sed SDVTList VTs = getVTList(VT1, VT2); 5102193323Sed SDValue Ops[] = { Op1 }; 5103193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5104193323Sed} 5105193323Sed 5106193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5107198090Srdivacky EVT VT1, EVT VT2, 5108193323Sed SDValue Op1, SDValue Op2) { 5109193323Sed SDVTList VTs = getVTList(VT1, VT2); 5110193323Sed SDValue Ops[] = { Op1, Op2 }; 5111193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5112193323Sed} 5113193323Sed 5114193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5115198090Srdivacky EVT VT1, EVT VT2, 5116193323Sed SDValue Op1, SDValue Op2, 5117193323Sed SDValue Op3) { 5118193323Sed SDVTList VTs = getVTList(VT1, VT2); 5119193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5120193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5121193323Sed} 5122193323Sed 5123193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5124198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5125193323Sed SDValue Op1, SDValue Op2, 5126193323Sed SDValue Op3) { 5127193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5128193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5129193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5130193323Sed} 5131193323Sed 5132193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5133193323Sed SDVTList VTs, const SDValue *Ops, 5134193323Sed unsigned NumOps) { 5135204642Srdivacky N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 5136204642Srdivacky // Reset the NodeID to -1. 5137204642Srdivacky N->setNodeId(-1); 5138204642Srdivacky return N; 5139193323Sed} 5140193323Sed 5141234353Sdim/// UpdadeDebugLocOnMergedSDNode - If the opt level is -O0 then it throws away 5142234353Sdim/// the line number information on the merged node since it is not possible to 5143234353Sdim/// preserve the information that operation is associated with multiple lines. 5144234353Sdim/// This will make the debugger working better at -O0, were there is a higher 5145234353Sdim/// probability having other instructions associated with that line. 5146234353Sdim/// 5147234353SdimSDNode *SelectionDAG::UpdadeDebugLocOnMergedSDNode(SDNode *N, DebugLoc OLoc) { 5148234353Sdim DebugLoc NLoc = N->getDebugLoc(); 5149234353Sdim if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && (OLoc != NLoc)) { 5150234353Sdim N->setDebugLoc(DebugLoc()); 5151234353Sdim } 5152234353Sdim return N; 5153234353Sdim} 5154234353Sdim 5155204642Srdivacky/// MorphNodeTo - This *mutates* the specified node to have the specified 5156193323Sed/// return type, opcode, and operands. 5157193323Sed/// 5158193323Sed/// Note that MorphNodeTo returns the resultant node. If there is already a 5159193323Sed/// node of the specified opcode and operands, it returns that node instead of 5160193323Sed/// the current one. Note that the DebugLoc need not be the same. 5161193323Sed/// 5162193323Sed/// Using MorphNodeTo is faster than creating a new node and swapping it in 5163193323Sed/// with ReplaceAllUsesWith both because it often avoids allocating a new 5164193323Sed/// node, and because it doesn't require CSE recalculation for any of 5165193323Sed/// the node's users. 5166193323Sed/// 5167193323SedSDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 5168193323Sed SDVTList VTs, const SDValue *Ops, 5169193323Sed unsigned NumOps) { 5170193323Sed // If an identical node already exists, use it. 5171193323Sed void *IP = 0; 5172218893Sdim if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 5173193323Sed FoldingSetNodeID ID; 5174193323Sed AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 5175201360Srdivacky if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 5176234353Sdim return UpdadeDebugLocOnMergedSDNode(ON, N->getDebugLoc()); 5177193323Sed } 5178193323Sed 5179193323Sed if (!RemoveNodeFromCSEMaps(N)) 5180193323Sed IP = 0; 5181193323Sed 5182193323Sed // Start the morphing. 5183193323Sed N->NodeType = Opc; 5184193323Sed N->ValueList = VTs.VTs; 5185193323Sed N->NumValues = VTs.NumVTs; 5186193323Sed 5187193323Sed // Clear the operands list, updating used nodes to remove this from their 5188193323Sed // use list. Keep track of any operands that become dead as a result. 5189193323Sed SmallPtrSet<SDNode*, 16> DeadNodeSet; 5190193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 5191193323Sed SDUse &Use = *I++; 5192193323Sed SDNode *Used = Use.getNode(); 5193193323Sed Use.set(SDValue()); 5194193323Sed if (Used->use_empty()) 5195193323Sed DeadNodeSet.insert(Used); 5196193323Sed } 5197193323Sed 5198198090Srdivacky if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 5199198090Srdivacky // Initialize the memory references information. 5200198090Srdivacky MN->setMemRefs(0, 0); 5201198090Srdivacky // If NumOps is larger than the # of operands we can have in a 5202198090Srdivacky // MachineSDNode, reallocate the operand list. 5203198090Srdivacky if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 5204198090Srdivacky if (MN->OperandsNeedDelete) 5205198090Srdivacky delete[] MN->OperandList; 5206198090Srdivacky if (NumOps > array_lengthof(MN->LocalOperands)) 5207198090Srdivacky // We're creating a final node that will live unmorphed for the 5208198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5209198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5210198090Srdivacky MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5211205407Srdivacky Ops, NumOps); 5212198090Srdivacky else 5213198090Srdivacky MN->InitOperands(MN->LocalOperands, Ops, NumOps); 5214198090Srdivacky MN->OperandsNeedDelete = false; 5215198090Srdivacky } else 5216198090Srdivacky MN->InitOperands(MN->OperandList, Ops, NumOps); 5217198090Srdivacky } else { 5218198090Srdivacky // If NumOps is larger than the # of operands we currently have, reallocate 5219198090Srdivacky // the operand list. 5220198090Srdivacky if (NumOps > N->NumOperands) { 5221198090Srdivacky if (N->OperandsNeedDelete) 5222198090Srdivacky delete[] N->OperandList; 5223198090Srdivacky N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5224193323Sed N->OperandsNeedDelete = true; 5225198090Srdivacky } else 5226198396Srdivacky N->InitOperands(N->OperandList, Ops, NumOps); 5227193323Sed } 5228193323Sed 5229193323Sed // Delete any nodes that are still dead after adding the uses for the 5230193323Sed // new operands. 5231204642Srdivacky if (!DeadNodeSet.empty()) { 5232204642Srdivacky SmallVector<SDNode *, 16> DeadNodes; 5233204642Srdivacky for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5234204642Srdivacky E = DeadNodeSet.end(); I != E; ++I) 5235204642Srdivacky if ((*I)->use_empty()) 5236204642Srdivacky DeadNodes.push_back(*I); 5237204642Srdivacky RemoveDeadNodes(DeadNodes); 5238204642Srdivacky } 5239193323Sed 5240193323Sed if (IP) 5241193323Sed CSEMap.InsertNode(N, IP); // Memoize the new node. 5242193323Sed return N; 5243193323Sed} 5244193323Sed 5245193323Sed 5246198090Srdivacky/// getMachineNode - These are used for target selectors to create a new node 5247198090Srdivacky/// with specified return type(s), MachineInstr opcode, and operands. 5248193323Sed/// 5249198090Srdivacky/// Note that getMachineNode returns the resultant node. If there is already a 5250193323Sed/// node of the specified opcode and operands, it returns that node instead of 5251193323Sed/// the current one. 5252198090SrdivackyMachineSDNode * 5253198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) { 5254198090Srdivacky SDVTList VTs = getVTList(VT); 5255251662Sdim return getMachineNode(Opcode, dl, VTs, None); 5256193323Sed} 5257193323Sed 5258198090SrdivackyMachineSDNode * 5259198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, SDValue Op1) { 5260198090Srdivacky SDVTList VTs = getVTList(VT); 5261198090Srdivacky SDValue Ops[] = { Op1 }; 5262251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5263193323Sed} 5264193323Sed 5265198090SrdivackyMachineSDNode * 5266198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5267198090Srdivacky SDValue Op1, SDValue Op2) { 5268198090Srdivacky SDVTList VTs = getVTList(VT); 5269198090Srdivacky SDValue Ops[] = { Op1, Op2 }; 5270251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5271193323Sed} 5272193323Sed 5273198090SrdivackyMachineSDNode * 5274198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5275198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5276198090Srdivacky SDVTList VTs = getVTList(VT); 5277198090Srdivacky SDValue Ops[] = { Op1, Op2, Op3 }; 5278251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5279193323Sed} 5280193323Sed 5281198090SrdivackyMachineSDNode * 5282198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, 5283251662Sdim ArrayRef<SDValue> Ops) { 5284198090Srdivacky SDVTList VTs = getVTList(VT); 5285251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5286193323Sed} 5287193323Sed 5288198090SrdivackyMachineSDNode * 5289198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, EVT VT2) { 5290193323Sed SDVTList VTs = getVTList(VT1, VT2); 5291251662Sdim return getMachineNode(Opcode, dl, VTs, None); 5292193323Sed} 5293193323Sed 5294198090SrdivackyMachineSDNode * 5295198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5296198090Srdivacky EVT VT1, EVT VT2, SDValue Op1) { 5297193323Sed SDVTList VTs = getVTList(VT1, VT2); 5298198090Srdivacky SDValue Ops[] = { Op1 }; 5299251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5300193323Sed} 5301193323Sed 5302198090SrdivackyMachineSDNode * 5303198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5304198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5305193323Sed SDVTList VTs = getVTList(VT1, VT2); 5306193323Sed SDValue Ops[] = { Op1, Op2 }; 5307251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5308193323Sed} 5309193323Sed 5310198090SrdivackyMachineSDNode * 5311198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5312198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, 5313198090Srdivacky SDValue Op2, SDValue Op3) { 5314193323Sed SDVTList VTs = getVTList(VT1, VT2); 5315193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5316251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5317193323Sed} 5318193323Sed 5319198090SrdivackyMachineSDNode * 5320198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5321198090Srdivacky EVT VT1, EVT VT2, 5322251662Sdim ArrayRef<SDValue> Ops) { 5323193323Sed SDVTList VTs = getVTList(VT1, VT2); 5324251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5325193323Sed} 5326193323Sed 5327198090SrdivackyMachineSDNode * 5328198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5329198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5330198090Srdivacky SDValue Op1, SDValue Op2) { 5331193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5332193323Sed SDValue Ops[] = { Op1, Op2 }; 5333251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5334193323Sed} 5335193323Sed 5336198090SrdivackyMachineSDNode * 5337198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5338198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5339198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5340193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5341193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5342251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5343193323Sed} 5344193323Sed 5345198090SrdivackyMachineSDNode * 5346198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5347198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5348251662Sdim ArrayRef<SDValue> Ops) { 5349193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5350251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5351193323Sed} 5352193323Sed 5353198090SrdivackyMachineSDNode * 5354198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, 5355198090Srdivacky EVT VT2, EVT VT3, EVT VT4, 5356251662Sdim ArrayRef<SDValue> Ops) { 5357193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5358251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5359193323Sed} 5360193323Sed 5361198090SrdivackyMachineSDNode * 5362198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, 5363249423Sdim ArrayRef<EVT> ResultTys, 5364251662Sdim ArrayRef<SDValue> Ops) { 5365198090Srdivacky SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5366251662Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5367193323Sed} 5368193323Sed 5369198090SrdivackyMachineSDNode * 5370198090SrdivackySelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, 5371251662Sdim ArrayRef<SDValue> OpsArray) { 5372218893Sdim bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5373198090Srdivacky MachineSDNode *N; 5374218893Sdim void *IP = 0; 5375251662Sdim const SDValue *Ops = OpsArray.data(); 5376251662Sdim unsigned NumOps = OpsArray.size(); 5377198090Srdivacky 5378198090Srdivacky if (DoCSE) { 5379198090Srdivacky FoldingSetNodeID ID; 5380198090Srdivacky AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5381198090Srdivacky IP = 0; 5382234353Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5383234353Sdim return cast<MachineSDNode>(UpdadeDebugLocOnMergedSDNode(E, DL)); 5384234353Sdim } 5385198090Srdivacky } 5386198090Srdivacky 5387198090Srdivacky // Allocate a new MachineSDNode. 5388205407Srdivacky N = new (NodeAllocator) MachineSDNode(~Opcode, DL, VTs); 5389198090Srdivacky 5390198090Srdivacky // Initialize the operands list. 5391198090Srdivacky if (NumOps > array_lengthof(N->LocalOperands)) 5392198090Srdivacky // We're creating a final node that will live unmorphed for the 5393198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5394198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5395198090Srdivacky N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5396198090Srdivacky Ops, NumOps); 5397198090Srdivacky else 5398198090Srdivacky N->InitOperands(N->LocalOperands, Ops, NumOps); 5399198090Srdivacky N->OperandsNeedDelete = false; 5400198090Srdivacky 5401198090Srdivacky if (DoCSE) 5402198090Srdivacky CSEMap.InsertNode(N, IP); 5403198090Srdivacky 5404198090Srdivacky AllNodes.push_back(N); 5405198090Srdivacky#ifndef NDEBUG 5406218893Sdim VerifyMachineNode(N); 5407198090Srdivacky#endif 5408198090Srdivacky return N; 5409198090Srdivacky} 5410198090Srdivacky 5411198090Srdivacky/// getTargetExtractSubreg - A convenience function for creating 5412203954Srdivacky/// TargetOpcode::EXTRACT_SUBREG nodes. 5413198090SrdivackySDValue 5414198090SrdivackySelectionDAG::getTargetExtractSubreg(int SRIdx, DebugLoc DL, EVT VT, 5415198090Srdivacky SDValue Operand) { 5416198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5417203954Srdivacky SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5418198090Srdivacky VT, Operand, SRIdxVal); 5419198090Srdivacky return SDValue(Subreg, 0); 5420198090Srdivacky} 5421198090Srdivacky 5422198090Srdivacky/// getTargetInsertSubreg - A convenience function for creating 5423203954Srdivacky/// TargetOpcode::INSERT_SUBREG nodes. 5424198090SrdivackySDValue 5425198090SrdivackySelectionDAG::getTargetInsertSubreg(int SRIdx, DebugLoc DL, EVT VT, 5426198090Srdivacky SDValue Operand, SDValue Subreg) { 5427198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5428203954Srdivacky SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5429198090Srdivacky VT, Operand, Subreg, SRIdxVal); 5430198090Srdivacky return SDValue(Result, 0); 5431198090Srdivacky} 5432198090Srdivacky 5433193323Sed/// getNodeIfExists - Get the specified node if it's already available, or 5434193323Sed/// else return NULL. 5435193323SedSDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5436193323Sed const SDValue *Ops, unsigned NumOps) { 5437218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5438193323Sed FoldingSetNodeID ID; 5439193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5440193323Sed void *IP = 0; 5441201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5442193323Sed return E; 5443193323Sed } 5444193323Sed return NULL; 5445193323Sed} 5446193323Sed 5447206083Srdivacky/// getDbgValue - Creates a SDDbgValue node. 5448206083Srdivacky/// 5449206083SrdivackySDDbgValue * 5450206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5451206083Srdivacky DebugLoc DL, unsigned O) { 5452206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5453206083Srdivacky} 5454206083Srdivacky 5455206083SrdivackySDDbgValue * 5456207618SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5457206083Srdivacky DebugLoc DL, unsigned O) { 5458206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5459206083Srdivacky} 5460206083Srdivacky 5461206083SrdivackySDDbgValue * 5462206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5463206083Srdivacky DebugLoc DL, unsigned O) { 5464206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5465206083Srdivacky} 5466206083Srdivacky 5467204792Srdivackynamespace { 5468204792Srdivacky 5469204792Srdivacky/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5470204792Srdivacky/// pointed to by a use iterator is deleted, increment the use iterator 5471204792Srdivacky/// so that it doesn't dangle. 5472204792Srdivacky/// 5473204792Srdivackyclass RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5474204792Srdivacky SDNode::use_iterator &UI; 5475204792Srdivacky SDNode::use_iterator &UE; 5476204792Srdivacky 5477204792Srdivacky virtual void NodeDeleted(SDNode *N, SDNode *E) { 5478204792Srdivacky // Increment the iterator as needed. 5479204792Srdivacky while (UI != UE && N == *UI) 5480204792Srdivacky ++UI; 5481204792Srdivacky } 5482204792Srdivacky 5483204792Srdivackypublic: 5484239462Sdim RAUWUpdateListener(SelectionDAG &d, 5485204792Srdivacky SDNode::use_iterator &ui, 5486204792Srdivacky SDNode::use_iterator &ue) 5487239462Sdim : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 5488204792Srdivacky}; 5489204792Srdivacky 5490204792Srdivacky} 5491204792Srdivacky 5492193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5493193323Sed/// This can cause recursive merging of nodes in the DAG. 5494193323Sed/// 5495193323Sed/// This version assumes From has a single result value. 5496193323Sed/// 5497239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 5498193323Sed SDNode *From = FromN.getNode(); 5499193323Sed assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5500193323Sed "Cannot replace with this method!"); 5501193323Sed assert(From != To.getNode() && "Cannot replace uses of with self"); 5502193323Sed 5503193323Sed // Iterate over all the existing uses of From. New uses will be added 5504193323Sed // to the beginning of the use list, which we avoid visiting. 5505193323Sed // This specifically avoids visiting uses of From that arise while the 5506193323Sed // replacement is happening, because any such uses would be the result 5507193323Sed // of CSE: If an existing node looks like From after one of its operands 5508193323Sed // is replaced by To, we don't want to replace of all its users with To 5509193323Sed // too. See PR3018 for more info. 5510193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5511239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5512193323Sed while (UI != UE) { 5513193323Sed SDNode *User = *UI; 5514193323Sed 5515193323Sed // This node is about to morph, remove its old self from the CSE maps. 5516193323Sed RemoveNodeFromCSEMaps(User); 5517193323Sed 5518193323Sed // A user can appear in a use list multiple times, and when this 5519193323Sed // happens the uses are usually next to each other in the list. 5520193323Sed // To help reduce the number of CSE recomputations, process all 5521193323Sed // the uses of this user that we can find this way. 5522193323Sed do { 5523193323Sed SDUse &Use = UI.getUse(); 5524193323Sed ++UI; 5525193323Sed Use.set(To); 5526193323Sed } while (UI != UE && *UI == User); 5527193323Sed 5528193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5529193323Sed // already exists there, recursively merge the results together. 5530239462Sdim AddModifiedNodeToCSEMaps(User); 5531193323Sed } 5532234353Sdim 5533234353Sdim // If we just RAUW'd the root, take note. 5534234353Sdim if (FromN == getRoot()) 5535234353Sdim setRoot(To); 5536193323Sed} 5537193323Sed 5538193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5539193323Sed/// This can cause recursive merging of nodes in the DAG. 5540193323Sed/// 5541193323Sed/// This version assumes that for each value of From, there is a 5542193323Sed/// corresponding value in To in the same position with the same type. 5543193323Sed/// 5544239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 5545193323Sed#ifndef NDEBUG 5546193323Sed for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5547193323Sed assert((!From->hasAnyUseOfValue(i) || 5548193323Sed From->getValueType(i) == To->getValueType(i)) && 5549193323Sed "Cannot use this version of ReplaceAllUsesWith!"); 5550193323Sed#endif 5551193323Sed 5552193323Sed // Handle the trivial case. 5553193323Sed if (From == To) 5554193323Sed return; 5555193323Sed 5556193323Sed // Iterate over just the existing users of From. See the comments in 5557193323Sed // the ReplaceAllUsesWith above. 5558193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5559239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5560193323Sed while (UI != UE) { 5561193323Sed SDNode *User = *UI; 5562193323Sed 5563193323Sed // This node is about to morph, remove its old self from the CSE maps. 5564193323Sed RemoveNodeFromCSEMaps(User); 5565193323Sed 5566193323Sed // A user can appear in a use list multiple times, and when this 5567193323Sed // happens the uses are usually next to each other in the list. 5568193323Sed // To help reduce the number of CSE recomputations, process all 5569193323Sed // the uses of this user that we can find this way. 5570193323Sed do { 5571193323Sed SDUse &Use = UI.getUse(); 5572193323Sed ++UI; 5573193323Sed Use.setNode(To); 5574193323Sed } while (UI != UE && *UI == User); 5575193323Sed 5576193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5577193323Sed // already exists there, recursively merge the results together. 5578239462Sdim AddModifiedNodeToCSEMaps(User); 5579193323Sed } 5580234353Sdim 5581234353Sdim // If we just RAUW'd the root, take note. 5582234353Sdim if (From == getRoot().getNode()) 5583234353Sdim setRoot(SDValue(To, getRoot().getResNo())); 5584193323Sed} 5585193323Sed 5586193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5587193323Sed/// This can cause recursive merging of nodes in the DAG. 5588193323Sed/// 5589193323Sed/// This version can replace From with any result values. To must match the 5590193323Sed/// number and types of values returned by From. 5591239462Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 5592193323Sed if (From->getNumValues() == 1) // Handle the simple case efficiently. 5593239462Sdim return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 5594193323Sed 5595193323Sed // Iterate over just the existing users of From. See the comments in 5596193323Sed // the ReplaceAllUsesWith above. 5597193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5598239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5599193323Sed while (UI != UE) { 5600193323Sed SDNode *User = *UI; 5601193323Sed 5602193323Sed // This node is about to morph, remove its old self from the CSE maps. 5603193323Sed RemoveNodeFromCSEMaps(User); 5604193323Sed 5605193323Sed // A user can appear in a use list multiple times, and when this 5606193323Sed // happens the uses are usually next to each other in the list. 5607193323Sed // To help reduce the number of CSE recomputations, process all 5608193323Sed // the uses of this user that we can find this way. 5609193323Sed do { 5610193323Sed SDUse &Use = UI.getUse(); 5611193323Sed const SDValue &ToOp = To[Use.getResNo()]; 5612193323Sed ++UI; 5613193323Sed Use.set(ToOp); 5614193323Sed } while (UI != UE && *UI == User); 5615193323Sed 5616193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5617193323Sed // already exists there, recursively merge the results together. 5618239462Sdim AddModifiedNodeToCSEMaps(User); 5619193323Sed } 5620234353Sdim 5621234353Sdim // If we just RAUW'd the root, take note. 5622234353Sdim if (From == getRoot().getNode()) 5623234353Sdim setRoot(SDValue(To[getRoot().getResNo()])); 5624193323Sed} 5625193323Sed 5626193323Sed/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5627193323Sed/// uses of other values produced by From.getNode() alone. The Deleted 5628193323Sed/// vector is handled the same way as for ReplaceAllUsesWith. 5629239462Sdimvoid SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 5630193323Sed // Handle the really simple, really trivial case efficiently. 5631193323Sed if (From == To) return; 5632193323Sed 5633193323Sed // Handle the simple, trivial, case efficiently. 5634193323Sed if (From.getNode()->getNumValues() == 1) { 5635239462Sdim ReplaceAllUsesWith(From, To); 5636193323Sed return; 5637193323Sed } 5638193323Sed 5639193323Sed // Iterate over just the existing users of From. See the comments in 5640193323Sed // the ReplaceAllUsesWith above. 5641193323Sed SDNode::use_iterator UI = From.getNode()->use_begin(), 5642193323Sed UE = From.getNode()->use_end(); 5643239462Sdim RAUWUpdateListener Listener(*this, UI, UE); 5644193323Sed while (UI != UE) { 5645193323Sed SDNode *User = *UI; 5646193323Sed bool UserRemovedFromCSEMaps = false; 5647193323Sed 5648193323Sed // A user can appear in a use list multiple times, and when this 5649193323Sed // happens the uses are usually next to each other in the list. 5650193323Sed // To help reduce the number of CSE recomputations, process all 5651193323Sed // the uses of this user that we can find this way. 5652193323Sed do { 5653193323Sed SDUse &Use = UI.getUse(); 5654193323Sed 5655193323Sed // Skip uses of different values from the same node. 5656193323Sed if (Use.getResNo() != From.getResNo()) { 5657193323Sed ++UI; 5658193323Sed continue; 5659193323Sed } 5660193323Sed 5661193323Sed // If this node hasn't been modified yet, it's still in the CSE maps, 5662193323Sed // so remove its old self from the CSE maps. 5663193323Sed if (!UserRemovedFromCSEMaps) { 5664193323Sed RemoveNodeFromCSEMaps(User); 5665193323Sed UserRemovedFromCSEMaps = true; 5666193323Sed } 5667193323Sed 5668193323Sed ++UI; 5669193323Sed Use.set(To); 5670193323Sed } while (UI != UE && *UI == User); 5671193323Sed 5672193323Sed // We are iterating over all uses of the From node, so if a use 5673193323Sed // doesn't use the specific value, no changes are made. 5674193323Sed if (!UserRemovedFromCSEMaps) 5675193323Sed continue; 5676193323Sed 5677193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5678193323Sed // already exists there, recursively merge the results together. 5679239462Sdim AddModifiedNodeToCSEMaps(User); 5680193323Sed } 5681234353Sdim 5682234353Sdim // If we just RAUW'd the root, take note. 5683234353Sdim if (From == getRoot()) 5684234353Sdim setRoot(To); 5685193323Sed} 5686193323Sed 5687193323Sednamespace { 5688193323Sed /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5689193323Sed /// to record information about a use. 5690193323Sed struct UseMemo { 5691193323Sed SDNode *User; 5692193323Sed unsigned Index; 5693193323Sed SDUse *Use; 5694193323Sed }; 5695193323Sed 5696193323Sed /// operator< - Sort Memos by User. 5697193323Sed bool operator<(const UseMemo &L, const UseMemo &R) { 5698193323Sed return (intptr_t)L.User < (intptr_t)R.User; 5699193323Sed } 5700193323Sed} 5701193323Sed 5702193323Sed/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5703193323Sed/// uses of other values produced by From.getNode() alone. The same value 5704193323Sed/// may appear in both the From and To list. The Deleted vector is 5705193323Sed/// handled the same way as for ReplaceAllUsesWith. 5706193323Sedvoid SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5707193323Sed const SDValue *To, 5708239462Sdim unsigned Num){ 5709193323Sed // Handle the simple, trivial case efficiently. 5710193323Sed if (Num == 1) 5711239462Sdim return ReplaceAllUsesOfValueWith(*From, *To); 5712193323Sed 5713193323Sed // Read up all the uses and make records of them. This helps 5714193323Sed // processing new uses that are introduced during the 5715193323Sed // replacement process. 5716193323Sed SmallVector<UseMemo, 4> Uses; 5717193323Sed for (unsigned i = 0; i != Num; ++i) { 5718193323Sed unsigned FromResNo = From[i].getResNo(); 5719193323Sed SDNode *FromNode = From[i].getNode(); 5720193323Sed for (SDNode::use_iterator UI = FromNode->use_begin(), 5721193323Sed E = FromNode->use_end(); UI != E; ++UI) { 5722193323Sed SDUse &Use = UI.getUse(); 5723193323Sed if (Use.getResNo() == FromResNo) { 5724193323Sed UseMemo Memo = { *UI, i, &Use }; 5725193323Sed Uses.push_back(Memo); 5726193323Sed } 5727193323Sed } 5728193323Sed } 5729193323Sed 5730193323Sed // Sort the uses, so that all the uses from a given User are together. 5731193323Sed std::sort(Uses.begin(), Uses.end()); 5732193323Sed 5733193323Sed for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5734193323Sed UseIndex != UseIndexEnd; ) { 5735193323Sed // We know that this user uses some value of From. If it is the right 5736193323Sed // value, update it. 5737193323Sed SDNode *User = Uses[UseIndex].User; 5738193323Sed 5739193323Sed // This node is about to morph, remove its old self from the CSE maps. 5740193323Sed RemoveNodeFromCSEMaps(User); 5741193323Sed 5742193323Sed // The Uses array is sorted, so all the uses for a given User 5743193323Sed // are next to each other in the list. 5744193323Sed // To help reduce the number of CSE recomputations, process all 5745193323Sed // the uses of this user that we can find this way. 5746193323Sed do { 5747193323Sed unsigned i = Uses[UseIndex].Index; 5748193323Sed SDUse &Use = *Uses[UseIndex].Use; 5749193323Sed ++UseIndex; 5750193323Sed 5751193323Sed Use.set(To[i]); 5752193323Sed } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5753193323Sed 5754193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5755193323Sed // already exists there, recursively merge the results together. 5756239462Sdim AddModifiedNodeToCSEMaps(User); 5757193323Sed } 5758193323Sed} 5759193323Sed 5760193323Sed/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5761193323Sed/// based on their topological order. It returns the maximum id and a vector 5762193323Sed/// of the SDNodes* in assigned order by reference. 5763193323Sedunsigned SelectionDAG::AssignTopologicalOrder() { 5764193323Sed 5765193323Sed unsigned DAGSize = 0; 5766193323Sed 5767193323Sed // SortedPos tracks the progress of the algorithm. Nodes before it are 5768193323Sed // sorted, nodes after it are unsorted. When the algorithm completes 5769193323Sed // it is at the end of the list. 5770193323Sed allnodes_iterator SortedPos = allnodes_begin(); 5771193323Sed 5772193323Sed // Visit all the nodes. Move nodes with no operands to the front of 5773193323Sed // the list immediately. Annotate nodes that do have operands with their 5774193323Sed // operand count. Before we do this, the Node Id fields of the nodes 5775193323Sed // may contain arbitrary values. After, the Node Id fields for nodes 5776193323Sed // before SortedPos will contain the topological sort index, and the 5777193323Sed // Node Id fields for nodes At SortedPos and after will contain the 5778193323Sed // count of outstanding operands. 5779193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5780193323Sed SDNode *N = I++; 5781202878Srdivacky checkForCycles(N); 5782193323Sed unsigned Degree = N->getNumOperands(); 5783193323Sed if (Degree == 0) { 5784193323Sed // A node with no uses, add it to the result array immediately. 5785193323Sed N->setNodeId(DAGSize++); 5786193323Sed allnodes_iterator Q = N; 5787193323Sed if (Q != SortedPos) 5788193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5789202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5790193323Sed ++SortedPos; 5791193323Sed } else { 5792193323Sed // Temporarily use the Node Id as scratch space for the degree count. 5793193323Sed N->setNodeId(Degree); 5794193323Sed } 5795193323Sed } 5796193323Sed 5797239462Sdim // Visit all the nodes. As we iterate, move nodes into sorted order, 5798193323Sed // such that by the time the end is reached all nodes will be sorted. 5799193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5800193323Sed SDNode *N = I; 5801202878Srdivacky checkForCycles(N); 5802202878Srdivacky // N is in sorted position, so all its uses have one less operand 5803202878Srdivacky // that needs to be sorted. 5804193323Sed for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5805193323Sed UI != UE; ++UI) { 5806193323Sed SDNode *P = *UI; 5807193323Sed unsigned Degree = P->getNodeId(); 5808202878Srdivacky assert(Degree != 0 && "Invalid node degree"); 5809193323Sed --Degree; 5810193323Sed if (Degree == 0) { 5811193323Sed // All of P's operands are sorted, so P may sorted now. 5812193323Sed P->setNodeId(DAGSize++); 5813193323Sed if (P != SortedPos) 5814193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5815202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5816193323Sed ++SortedPos; 5817193323Sed } else { 5818193323Sed // Update P's outstanding operand count. 5819193323Sed P->setNodeId(Degree); 5820193323Sed } 5821193323Sed } 5822202878Srdivacky if (I == SortedPos) { 5823203954Srdivacky#ifndef NDEBUG 5824203954Srdivacky SDNode *S = ++I; 5825203954Srdivacky dbgs() << "Overran sorted position:\n"; 5826202878Srdivacky S->dumprFull(); 5827203954Srdivacky#endif 5828203954Srdivacky llvm_unreachable(0); 5829202878Srdivacky } 5830193323Sed } 5831193323Sed 5832193323Sed assert(SortedPos == AllNodes.end() && 5833193323Sed "Topological sort incomplete!"); 5834193323Sed assert(AllNodes.front().getOpcode() == ISD::EntryToken && 5835193323Sed "First node in topological sort is not the entry token!"); 5836193323Sed assert(AllNodes.front().getNodeId() == 0 && 5837193323Sed "First node in topological sort has non-zero id!"); 5838193323Sed assert(AllNodes.front().getNumOperands() == 0 && 5839193323Sed "First node in topological sort has operands!"); 5840193323Sed assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 5841193323Sed "Last node in topologic sort has unexpected id!"); 5842193323Sed assert(AllNodes.back().use_empty() && 5843193323Sed "Last node in topologic sort has users!"); 5844193323Sed assert(DAGSize == allnodes_size() && "Node count mismatch!"); 5845193323Sed return DAGSize; 5846193323Sed} 5847193323Sed 5848201360Srdivacky/// AssignOrdering - Assign an order to the SDNode. 5849203954Srdivackyvoid SelectionDAG::AssignOrdering(const SDNode *SD, unsigned Order) { 5850201360Srdivacky assert(SD && "Trying to assign an order to a null node!"); 5851202878Srdivacky Ordering->add(SD, Order); 5852201360Srdivacky} 5853193323Sed 5854201360Srdivacky/// GetOrdering - Get the order for the SDNode. 5855201360Srdivackyunsigned SelectionDAG::GetOrdering(const SDNode *SD) const { 5856201360Srdivacky assert(SD && "Trying to get the order of a null node!"); 5857202878Srdivacky return Ordering->getOrder(SD); 5858201360Srdivacky} 5859193323Sed 5860206083Srdivacky/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 5861206083Srdivacky/// value is produced by SD. 5862207618Srdivackyvoid SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 5863207618Srdivacky DbgInfo->add(DB, SD, isParameter); 5864206083Srdivacky if (SD) 5865206083Srdivacky SD->setHasDebugValue(true); 5866205218Srdivacky} 5867201360Srdivacky 5868218893Sdim/// TransferDbgValues - Transfer SDDbgValues. 5869218893Sdimvoid SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 5870218893Sdim if (From == To || !From.getNode()->getHasDebugValue()) 5871218893Sdim return; 5872218893Sdim SDNode *FromNode = From.getNode(); 5873218893Sdim SDNode *ToNode = To.getNode(); 5874224145Sdim ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 5875218893Sdim SmallVector<SDDbgValue *, 2> ClonedDVs; 5876224145Sdim for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 5877218893Sdim I != E; ++I) { 5878218893Sdim SDDbgValue *Dbg = *I; 5879218893Sdim if (Dbg->getKind() == SDDbgValue::SDNODE) { 5880218893Sdim SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 5881218893Sdim Dbg->getOffset(), Dbg->getDebugLoc(), 5882218893Sdim Dbg->getOrder()); 5883218893Sdim ClonedDVs.push_back(Clone); 5884218893Sdim } 5885218893Sdim } 5886218893Sdim for (SmallVector<SDDbgValue *, 2>::iterator I = ClonedDVs.begin(), 5887218893Sdim E = ClonedDVs.end(); I != E; ++I) 5888218893Sdim AddDbgValue(*I, ToNode, false); 5889218893Sdim} 5890218893Sdim 5891193323Sed//===----------------------------------------------------------------------===// 5892193323Sed// SDNode Class 5893193323Sed//===----------------------------------------------------------------------===// 5894193323Sed 5895193323SedHandleSDNode::~HandleSDNode() { 5896193323Sed DropOperands(); 5897193323Sed} 5898193323Sed 5899210299SedGlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, DebugLoc DL, 5900210299Sed const GlobalValue *GA, 5901198090Srdivacky EVT VT, int64_t o, unsigned char TF) 5902210299Sed : SDNode(Opc, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 5903207618Srdivacky TheGlobal = GA; 5904193323Sed} 5905193323Sed 5906198090SrdivackyMemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, 5907198090Srdivacky MachineMemOperand *mmo) 5908198090Srdivacky : SDNode(Opc, dl, VTs), MemoryVT(memvt), MMO(mmo) { 5909204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5910234353Sdim MMO->isNonTemporal(), MMO->isInvariant()); 5911198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5912204642Srdivacky assert(isNonTemporal() == MMO->isNonTemporal() && 5913204642Srdivacky "Non-temporal encoding error!"); 5914198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5915193323Sed} 5916193323Sed 5917193323SedMemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, 5918218893Sdim const SDValue *Ops, unsigned NumOps, EVT memvt, 5919198090Srdivacky MachineMemOperand *mmo) 5920193323Sed : SDNode(Opc, dl, VTs, Ops, NumOps), 5921198090Srdivacky MemoryVT(memvt), MMO(mmo) { 5922204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 5923234353Sdim MMO->isNonTemporal(), MMO->isInvariant()); 5924198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 5925198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 5926193323Sed} 5927193323Sed 5928193323Sed/// Profile - Gather unique data for the node. 5929193323Sed/// 5930193323Sedvoid SDNode::Profile(FoldingSetNodeID &ID) const { 5931193323Sed AddNodeIDNode(ID, this); 5932193323Sed} 5933193323Sed 5934198090Srdivackynamespace { 5935198090Srdivacky struct EVTArray { 5936198090Srdivacky std::vector<EVT> VTs; 5937218893Sdim 5938198090Srdivacky EVTArray() { 5939198090Srdivacky VTs.reserve(MVT::LAST_VALUETYPE); 5940198090Srdivacky for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 5941198090Srdivacky VTs.push_back(MVT((MVT::SimpleValueType)i)); 5942198090Srdivacky } 5943198090Srdivacky }; 5944198090Srdivacky} 5945198090Srdivacky 5946198090Srdivackystatic ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 5947198090Srdivackystatic ManagedStatic<EVTArray> SimpleVTArray; 5948195098Sedstatic ManagedStatic<sys::SmartMutex<true> > VTMutex; 5949195098Sed 5950193323Sed/// getValueTypeList - Return a pointer to the specified value type. 5951193323Sed/// 5952198090Srdivackyconst EVT *SDNode::getValueTypeList(EVT VT) { 5953193323Sed if (VT.isExtended()) { 5954198090Srdivacky sys::SmartScopedLock<true> Lock(*VTMutex); 5955195098Sed return &(*EVTs->insert(VT).first); 5956193323Sed } else { 5957218893Sdim assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 5958208599Srdivacky "Value type out of range!"); 5959198090Srdivacky return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 5960193323Sed } 5961193323Sed} 5962193323Sed 5963193323Sed/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 5964193323Sed/// indicated value. This method ignores uses of other values defined by this 5965193323Sed/// operation. 5966193323Sedbool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 5967193323Sed assert(Value < getNumValues() && "Bad value!"); 5968193323Sed 5969193323Sed // TODO: Only iterate over uses of a given value of the node 5970193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 5971193323Sed if (UI.getUse().getResNo() == Value) { 5972193323Sed if (NUses == 0) 5973193323Sed return false; 5974193323Sed --NUses; 5975193323Sed } 5976193323Sed } 5977193323Sed 5978193323Sed // Found exactly the right number of uses? 5979193323Sed return NUses == 0; 5980193323Sed} 5981193323Sed 5982193323Sed 5983193323Sed/// hasAnyUseOfValue - Return true if there are any use of the indicated 5984193323Sed/// value. This method ignores uses of other values defined by this operation. 5985193323Sedbool SDNode::hasAnyUseOfValue(unsigned Value) const { 5986193323Sed assert(Value < getNumValues() && "Bad value!"); 5987193323Sed 5988193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 5989193323Sed if (UI.getUse().getResNo() == Value) 5990193323Sed return true; 5991193323Sed 5992193323Sed return false; 5993193323Sed} 5994193323Sed 5995193323Sed 5996193323Sed/// isOnlyUserOf - Return true if this node is the only use of N. 5997193323Sed/// 5998193323Sedbool SDNode::isOnlyUserOf(SDNode *N) const { 5999193323Sed bool Seen = false; 6000193323Sed for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 6001193323Sed SDNode *User = *I; 6002193323Sed if (User == this) 6003193323Sed Seen = true; 6004193323Sed else 6005193323Sed return false; 6006193323Sed } 6007193323Sed 6008193323Sed return Seen; 6009193323Sed} 6010193323Sed 6011193323Sed/// isOperand - Return true if this node is an operand of N. 6012193323Sed/// 6013193323Sedbool SDValue::isOperandOf(SDNode *N) const { 6014193323Sed for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6015193323Sed if (*this == N->getOperand(i)) 6016193323Sed return true; 6017193323Sed return false; 6018193323Sed} 6019193323Sed 6020193323Sedbool SDNode::isOperandOf(SDNode *N) const { 6021193323Sed for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 6022193323Sed if (this == N->OperandList[i].getNode()) 6023193323Sed return true; 6024193323Sed return false; 6025193323Sed} 6026193323Sed 6027193323Sed/// reachesChainWithoutSideEffects - Return true if this operand (which must 6028193323Sed/// be a chain) reaches the specified operand without crossing any 6029218893Sdim/// side-effecting instructions on any chain path. In practice, this looks 6030218893Sdim/// through token factors and non-volatile loads. In order to remain efficient, 6031218893Sdim/// this only looks a couple of nodes in, it does not do an exhaustive search. 6032193323Sedbool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 6033193323Sed unsigned Depth) const { 6034193323Sed if (*this == Dest) return true; 6035193323Sed 6036193323Sed // Don't search too deeply, we just want to be able to see through 6037193323Sed // TokenFactor's etc. 6038193323Sed if (Depth == 0) return false; 6039193323Sed 6040193323Sed // If this is a token factor, all inputs to the TF happen in parallel. If any 6041218893Sdim // of the operands of the TF does not reach dest, then we cannot do the xform. 6042193323Sed if (getOpcode() == ISD::TokenFactor) { 6043193323Sed for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6044218893Sdim if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 6045218893Sdim return false; 6046218893Sdim return true; 6047193323Sed } 6048193323Sed 6049193323Sed // Loads don't have side effects, look through them. 6050193323Sed if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 6051193323Sed if (!Ld->isVolatile()) 6052193323Sed return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 6053193323Sed } 6054193323Sed return false; 6055193323Sed} 6056193323Sed 6057224145Sdim/// hasPredecessor - Return true if N is a predecessor of this node. 6058224145Sdim/// N is either an operand of this node, or can be reached by recursively 6059224145Sdim/// traversing up the operands. 6060224145Sdim/// NOTE: This is an expensive method. Use it carefully. 6061224145Sdimbool SDNode::hasPredecessor(const SDNode *N) const { 6062224145Sdim SmallPtrSet<const SDNode *, 32> Visited; 6063224145Sdim SmallVector<const SDNode *, 16> Worklist; 6064224145Sdim return hasPredecessorHelper(N, Visited, Worklist); 6065224145Sdim} 6066198892Srdivacky 6067224145Sdimbool SDNode::hasPredecessorHelper(const SDNode *N, 6068224145Sdim SmallPtrSet<const SDNode *, 32> &Visited, 6069224145Sdim SmallVector<const SDNode *, 16> &Worklist) const { 6070224145Sdim if (Visited.empty()) { 6071224145Sdim Worklist.push_back(this); 6072224145Sdim } else { 6073224145Sdim // Take a look in the visited set. If we've already encountered this node 6074224145Sdim // we needn't search further. 6075224145Sdim if (Visited.count(N)) 6076224145Sdim return true; 6077224145Sdim } 6078224145Sdim 6079224145Sdim // Haven't visited N yet. Continue the search. 6080224145Sdim while (!Worklist.empty()) { 6081224145Sdim const SDNode *M = Worklist.pop_back_val(); 6082224145Sdim for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 6083224145Sdim SDNode *Op = M->getOperand(i).getNode(); 6084198892Srdivacky if (Visited.insert(Op)) 6085198892Srdivacky Worklist.push_back(Op); 6086224145Sdim if (Op == N) 6087224145Sdim return true; 6088198892Srdivacky } 6089224145Sdim } 6090198892Srdivacky 6091198892Srdivacky return false; 6092193323Sed} 6093193323Sed 6094193323Seduint64_t SDNode::getConstantOperandVal(unsigned Num) const { 6095193323Sed assert(Num < NumOperands && "Invalid child # of SDNode!"); 6096193323Sed return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 6097193323Sed} 6098193323Sed 6099199989SrdivackySDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6100199989Srdivacky assert(N->getNumValues() == 1 && 6101199989Srdivacky "Can't unroll a vector with multiple results!"); 6102199989Srdivacky 6103199989Srdivacky EVT VT = N->getValueType(0); 6104199989Srdivacky unsigned NE = VT.getVectorNumElements(); 6105199989Srdivacky EVT EltVT = VT.getVectorElementType(); 6106199989Srdivacky DebugLoc dl = N->getDebugLoc(); 6107199989Srdivacky 6108199989Srdivacky SmallVector<SDValue, 8> Scalars; 6109199989Srdivacky SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6110199989Srdivacky 6111199989Srdivacky // If ResNE is 0, fully unroll the vector op. 6112199989Srdivacky if (ResNE == 0) 6113199989Srdivacky ResNE = NE; 6114199989Srdivacky else if (NE > ResNE) 6115199989Srdivacky NE = ResNE; 6116199989Srdivacky 6117199989Srdivacky unsigned i; 6118199989Srdivacky for (i= 0; i != NE; ++i) { 6119207618Srdivacky for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6120199989Srdivacky SDValue Operand = N->getOperand(j); 6121199989Srdivacky EVT OperandVT = Operand.getValueType(); 6122199989Srdivacky if (OperandVT.isVector()) { 6123199989Srdivacky // A vector operand; extract a single element. 6124199989Srdivacky EVT OperandEltVT = OperandVT.getVectorElementType(); 6125199989Srdivacky Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6126199989Srdivacky OperandEltVT, 6127199989Srdivacky Operand, 6128223017Sdim getConstant(i, TLI.getPointerTy())); 6129199989Srdivacky } else { 6130199989Srdivacky // A scalar operand; just use it as is. 6131199989Srdivacky Operands[j] = Operand; 6132199989Srdivacky } 6133199989Srdivacky } 6134199989Srdivacky 6135199989Srdivacky switch (N->getOpcode()) { 6136199989Srdivacky default: 6137199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6138199989Srdivacky &Operands[0], Operands.size())); 6139199989Srdivacky break; 6140226633Sdim case ISD::VSELECT: 6141226633Sdim Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6142226633Sdim &Operands[0], Operands.size())); 6143226633Sdim break; 6144199989Srdivacky case ISD::SHL: 6145199989Srdivacky case ISD::SRA: 6146199989Srdivacky case ISD::SRL: 6147199989Srdivacky case ISD::ROTL: 6148199989Srdivacky case ISD::ROTR: 6149199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6150221345Sdim getShiftAmountOperand(Operands[0].getValueType(), 6151221345Sdim Operands[1]))); 6152199989Srdivacky break; 6153202375Srdivacky case ISD::SIGN_EXTEND_INREG: 6154202375Srdivacky case ISD::FP_ROUND_INREG: { 6155202375Srdivacky EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6156202375Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6157202375Srdivacky Operands[0], 6158202375Srdivacky getValueType(ExtVT))); 6159199989Srdivacky } 6160202375Srdivacky } 6161199989Srdivacky } 6162199989Srdivacky 6163199989Srdivacky for (; i < ResNE; ++i) 6164199989Srdivacky Scalars.push_back(getUNDEF(EltVT)); 6165199989Srdivacky 6166199989Srdivacky return getNode(ISD::BUILD_VECTOR, dl, 6167199989Srdivacky EVT::getVectorVT(*getContext(), EltVT, ResNE), 6168199989Srdivacky &Scalars[0], Scalars.size()); 6169199989Srdivacky} 6170199989Srdivacky 6171200581Srdivacky 6172218893Sdim/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6173218893Sdim/// location that is 'Dist' units away from the location that the 'Base' load 6174200581Srdivacky/// is loading from. 6175218893Sdimbool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6176200581Srdivacky unsigned Bytes, int Dist) const { 6177200581Srdivacky if (LD->getChain() != Base->getChain()) 6178200581Srdivacky return false; 6179200581Srdivacky EVT VT = LD->getValueType(0); 6180200581Srdivacky if (VT.getSizeInBits() / 8 != Bytes) 6181200581Srdivacky return false; 6182200581Srdivacky 6183200581Srdivacky SDValue Loc = LD->getOperand(1); 6184200581Srdivacky SDValue BaseLoc = Base->getOperand(1); 6185200581Srdivacky if (Loc.getOpcode() == ISD::FrameIndex) { 6186200581Srdivacky if (BaseLoc.getOpcode() != ISD::FrameIndex) 6187200581Srdivacky return false; 6188200581Srdivacky const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6189200581Srdivacky int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6190200581Srdivacky int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6191200581Srdivacky int FS = MFI->getObjectSize(FI); 6192200581Srdivacky int BFS = MFI->getObjectSize(BFI); 6193200581Srdivacky if (FS != BFS || FS != (int)Bytes) return false; 6194200581Srdivacky return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6195200581Srdivacky } 6196200581Srdivacky 6197218893Sdim // Handle X+C 6198218893Sdim if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6199218893Sdim cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6200218893Sdim return true; 6201218893Sdim 6202207618Srdivacky const GlobalValue *GV1 = NULL; 6203207618Srdivacky const GlobalValue *GV2 = NULL; 6204200581Srdivacky int64_t Offset1 = 0; 6205200581Srdivacky int64_t Offset2 = 0; 6206200581Srdivacky bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6207200581Srdivacky bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6208200581Srdivacky if (isGA1 && isGA2 && GV1 == GV2) 6209200581Srdivacky return Offset1 == (Offset2 + Dist*Bytes); 6210200581Srdivacky return false; 6211200581Srdivacky} 6212200581Srdivacky 6213200581Srdivacky 6214200581Srdivacky/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6215200581Srdivacky/// it cannot be inferred. 6216200581Srdivackyunsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6217200581Srdivacky // If this is a GlobalAddress + cst, return the alignment. 6218207618Srdivacky const GlobalValue *GV; 6219200581Srdivacky int64_t GVOffset = 0; 6220206083Srdivacky if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6221234353Sdim unsigned PtrWidth = TLI.getPointerTy().getSizeInBits(); 6222234353Sdim APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6223234353Sdim llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne, 6224243830Sdim TLI.getDataLayout()); 6225234353Sdim unsigned AlignBits = KnownZero.countTrailingOnes(); 6226234353Sdim unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6227234353Sdim if (Align) 6228234353Sdim return MinAlign(Align, GVOffset); 6229206083Srdivacky } 6230200581Srdivacky 6231200581Srdivacky // If this is a direct reference to a stack slot, use information about the 6232200581Srdivacky // stack slot's alignment. 6233200581Srdivacky int FrameIdx = 1 << 31; 6234200581Srdivacky int64_t FrameOffset = 0; 6235200581Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6236200581Srdivacky FrameIdx = FI->getIndex(); 6237218893Sdim } else if (isBaseWithConstantOffset(Ptr) && 6238200581Srdivacky isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6239218893Sdim // Handle FI+Cst 6240200581Srdivacky FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6241200581Srdivacky FrameOffset = Ptr.getConstantOperandVal(1); 6242200581Srdivacky } 6243200581Srdivacky 6244200581Srdivacky if (FrameIdx != (1 << 31)) { 6245200581Srdivacky const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6246200581Srdivacky unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6247200581Srdivacky FrameOffset); 6248200581Srdivacky return FIInfoAlign; 6249200581Srdivacky } 6250200581Srdivacky 6251200581Srdivacky return 0; 6252200581Srdivacky} 6253200581Srdivacky 6254193323Sed// getAddressSpace - Return the address space this GlobalAddress belongs to. 6255193323Sedunsigned GlobalAddressSDNode::getAddressSpace() const { 6256193323Sed return getGlobal()->getType()->getAddressSpace(); 6257193323Sed} 6258193323Sed 6259193323Sed 6260226633SdimType *ConstantPoolSDNode::getType() const { 6261193323Sed if (isMachineConstantPoolEntry()) 6262193323Sed return Val.MachineCPVal->getType(); 6263193323Sed return Val.ConstVal->getType(); 6264193323Sed} 6265193323Sed 6266193323Sedbool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6267193323Sed APInt &SplatUndef, 6268193323Sed unsigned &SplatBitSize, 6269193323Sed bool &HasAnyUndefs, 6270199481Srdivacky unsigned MinSplatBits, 6271199481Srdivacky bool isBigEndian) { 6272198090Srdivacky EVT VT = getValueType(0); 6273193323Sed assert(VT.isVector() && "Expected a vector type"); 6274193323Sed unsigned sz = VT.getSizeInBits(); 6275193323Sed if (MinSplatBits > sz) 6276193323Sed return false; 6277193323Sed 6278193323Sed SplatValue = APInt(sz, 0); 6279193323Sed SplatUndef = APInt(sz, 0); 6280193323Sed 6281193323Sed // Get the bits. Bits with undefined values (when the corresponding element 6282193323Sed // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6283193323Sed // in SplatValue. If any of the values are not constant, give up and return 6284193323Sed // false. 6285193323Sed unsigned int nOps = getNumOperands(); 6286193323Sed assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6287193323Sed unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6288199481Srdivacky 6289199481Srdivacky for (unsigned j = 0; j < nOps; ++j) { 6290199481Srdivacky unsigned i = isBigEndian ? nOps-1-j : j; 6291193323Sed SDValue OpVal = getOperand(i); 6292199481Srdivacky unsigned BitPos = j * EltBitSize; 6293193323Sed 6294193323Sed if (OpVal.getOpcode() == ISD::UNDEF) 6295199481Srdivacky SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6296193323Sed else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6297218893Sdim SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6298207618Srdivacky zextOrTrunc(sz) << BitPos; 6299193323Sed else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6300193323Sed SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6301193323Sed else 6302193323Sed return false; 6303193323Sed } 6304193323Sed 6305193323Sed // The build_vector is all constants or undefs. Find the smallest element 6306193323Sed // size that splats the vector. 6307193323Sed 6308193323Sed HasAnyUndefs = (SplatUndef != 0); 6309193323Sed while (sz > 8) { 6310193323Sed 6311193323Sed unsigned HalfSize = sz / 2; 6312218893Sdim APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6313218893Sdim APInt LowValue = SplatValue.trunc(HalfSize); 6314218893Sdim APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6315218893Sdim APInt LowUndef = SplatUndef.trunc(HalfSize); 6316193323Sed 6317193323Sed // If the two halves do not match (ignoring undef bits), stop here. 6318193323Sed if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6319193323Sed MinSplatBits > HalfSize) 6320193323Sed break; 6321193323Sed 6322193323Sed SplatValue = HighValue | LowValue; 6323193323Sed SplatUndef = HighUndef & LowUndef; 6324198090Srdivacky 6325193323Sed sz = HalfSize; 6326193323Sed } 6327193323Sed 6328193323Sed SplatBitSize = sz; 6329193323Sed return true; 6330193323Sed} 6331193323Sed 6332198090Srdivackybool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6333193323Sed // Find the first non-undef value in the shuffle mask. 6334193323Sed unsigned i, e; 6335193323Sed for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6336193323Sed /* search */; 6337193323Sed 6338193323Sed assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6339198090Srdivacky 6340193323Sed // Make sure all remaining elements are either undef or the same as the first 6341193323Sed // non-undef value. 6342193323Sed for (int Idx = Mask[i]; i != e; ++i) 6343193323Sed if (Mask[i] >= 0 && Mask[i] != Idx) 6344193323Sed return false; 6345193323Sed return true; 6346193323Sed} 6347202878Srdivacky 6348204642Srdivacky#ifdef XDEBUG 6349202878Srdivackystatic void checkForCyclesHelper(const SDNode *N, 6350204642Srdivacky SmallPtrSet<const SDNode*, 32> &Visited, 6351204642Srdivacky SmallPtrSet<const SDNode*, 32> &Checked) { 6352204642Srdivacky // If this node has already been checked, don't check it again. 6353204642Srdivacky if (Checked.count(N)) 6354204642Srdivacky return; 6355218893Sdim 6356204642Srdivacky // If a node has already been visited on this depth-first walk, reject it as 6357204642Srdivacky // a cycle. 6358204642Srdivacky if (!Visited.insert(N)) { 6359202878Srdivacky dbgs() << "Offending node:\n"; 6360202878Srdivacky N->dumprFull(); 6361204642Srdivacky errs() << "Detected cycle in SelectionDAG\n"; 6362204642Srdivacky abort(); 6363202878Srdivacky } 6364218893Sdim 6365204642Srdivacky for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6366204642Srdivacky checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6367218893Sdim 6368204642Srdivacky Checked.insert(N); 6369204642Srdivacky Visited.erase(N); 6370202878Srdivacky} 6371204642Srdivacky#endif 6372202878Srdivacky 6373202878Srdivackyvoid llvm::checkForCycles(const llvm::SDNode *N) { 6374202878Srdivacky#ifdef XDEBUG 6375202878Srdivacky assert(N && "Checking nonexistant SDNode"); 6376204642Srdivacky SmallPtrSet<const SDNode*, 32> visited; 6377204642Srdivacky SmallPtrSet<const SDNode*, 32> checked; 6378204642Srdivacky checkForCyclesHelper(N, visited, checked); 6379202878Srdivacky#endif 6380202878Srdivacky} 6381202878Srdivacky 6382202878Srdivackyvoid llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6383202878Srdivacky checkForCycles(DAG->getRoot().getNode()); 6384202878Srdivacky} 6385