1193323Sed//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This implements the SelectionDAG class. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13201360Srdivacky 14193323Sed#include "llvm/CodeGen/SelectionDAG.h" 15252723Sdim#include "SDNodeDbgValue.h" 16252723Sdim#include "llvm/ADT/SetVector.h" 17252723Sdim#include "llvm/ADT/SmallPtrSet.h" 18252723Sdim#include "llvm/ADT/SmallSet.h" 19252723Sdim#include "llvm/ADT/SmallVector.h" 20252723Sdim#include "llvm/ADT/StringExtras.h" 21252723Sdim#include "llvm/Analysis/TargetTransformInfo.h" 22245431Sdim#include "llvm/Analysis/ValueTracking.h" 23193323Sed#include "llvm/Assembly/Writer.h" 24193323Sed#include "llvm/CodeGen/MachineBasicBlock.h" 25193323Sed#include "llvm/CodeGen/MachineConstantPool.h" 26193323Sed#include "llvm/CodeGen/MachineFrameInfo.h" 27193323Sed#include "llvm/CodeGen/MachineModuleInfo.h" 28252723Sdim#include "llvm/DebugInfo.h" 29252723Sdim#include "llvm/IR/CallingConv.h" 30252723Sdim#include "llvm/IR/Constants.h" 31252723Sdim#include "llvm/IR/DataLayout.h" 32252723Sdim#include "llvm/IR/DerivedTypes.h" 33252723Sdim#include "llvm/IR/Function.h" 34252723Sdim#include "llvm/IR/GlobalAlias.h" 35252723Sdim#include "llvm/IR/GlobalVariable.h" 36252723Sdim#include "llvm/IR/Intrinsics.h" 37193323Sed#include "llvm/Support/CommandLine.h" 38202375Srdivacky#include "llvm/Support/Debug.h" 39198090Srdivacky#include "llvm/Support/ErrorHandling.h" 40195098Sed#include "llvm/Support/ManagedStatic.h" 41193323Sed#include "llvm/Support/MathExtras.h" 42252723Sdim#include "llvm/Support/Mutex.h" 43193323Sed#include "llvm/Support/raw_ostream.h" 44252723Sdim#include "llvm/Target/TargetInstrInfo.h" 45252723Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 46252723Sdim#include "llvm/Target/TargetLowering.h" 47252723Sdim#include "llvm/Target/TargetMachine.h" 48252723Sdim#include "llvm/Target/TargetOptions.h" 49252723Sdim#include "llvm/Target/TargetRegisterInfo.h" 50252723Sdim#include "llvm/Target/TargetSelectionDAGInfo.h" 51193323Sed#include <algorithm> 52193323Sed#include <cmath> 53193323Sedusing namespace llvm; 54193323Sed 55193323Sed/// makeVTList - Return an instance of the SDVTList struct initialized with the 56193323Sed/// specified members. 57198090Srdivackystatic SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 58193323Sed SDVTList Res = {VTs, NumVTs}; 59193323Sed return Res; 60193323Sed} 61193323Sed 62245431Sdim// Default null implementations of the callbacks. 63245431Sdimvoid SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 64245431Sdimvoid SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 65193323Sed 66193323Sed//===----------------------------------------------------------------------===// 67193323Sed// ConstantFPSDNode Class 68193323Sed//===----------------------------------------------------------------------===// 69193323Sed 70193323Sed/// isExactlyValue - We don't rely on operator== working on double values, as 71193323Sed/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 72193323Sed/// As such, this method can be used to do an exact bit-for-bit comparison of 73193323Sed/// two floating point values. 74193323Sedbool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 75193323Sed return getValueAPF().bitwiseIsEqual(V); 76193323Sed} 77193323Sed 78198090Srdivackybool ConstantFPSDNode::isValueValidForType(EVT VT, 79193323Sed const APFloat& Val) { 80193323Sed assert(VT.isFloatingPoint() && "Can only convert between FP types"); 81193323Sed 82193323Sed // convert modifies in place, so make a copy. 83193323Sed APFloat Val2 = APFloat(Val); 84193323Sed bool losesInfo; 85252723Sdim (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 86252723Sdim APFloat::rmNearestTiesToEven, 87193323Sed &losesInfo); 88193323Sed return !losesInfo; 89193323Sed} 90193323Sed 91193323Sed//===----------------------------------------------------------------------===// 92193323Sed// ISD Namespace 93193323Sed//===----------------------------------------------------------------------===// 94193323Sed 95193323Sed/// isBuildVectorAllOnes - Return true if the specified node is a 96193323Sed/// BUILD_VECTOR where all of the elements are ~0 or undef. 97193323Sedbool ISD::isBuildVectorAllOnes(const SDNode *N) { 98193323Sed // Look through a bit convert. 99218893Sdim if (N->getOpcode() == ISD::BITCAST) 100193323Sed N = N->getOperand(0).getNode(); 101193323Sed 102193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 103193323Sed 104193323Sed unsigned i = 0, e = N->getNumOperands(); 105193323Sed 106193323Sed // Skip over all of the undef values. 107193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 108193323Sed ++i; 109193323Sed 110193323Sed // Do not accept an all-undef vector. 111193323Sed if (i == e) return false; 112193323Sed 113193323Sed // Do not accept build_vectors that aren't all constants or which have non-~0 114235633Sdim // elements. We have to be a bit careful here, as the type of the constant 115235633Sdim // may not be the same as the type of the vector elements due to type 116235633Sdim // legalization (the elements are promoted to a legal type for the target and 117235633Sdim // a vector of a type may be legal when the base element type is not). 118235633Sdim // We only want to check enough bits to cover the vector elements, because 119235633Sdim // we care if the resultant vector is all ones, not whether the individual 120235633Sdim // constants are. 121193323Sed SDValue NotZero = N->getOperand(i); 122235633Sdim unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 123245431Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 124245431Sdim if (CN->getAPIntValue().countTrailingOnes() < EltSize) 125193323Sed return false; 126245431Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 127245431Sdim if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 128193323Sed return false; 129193323Sed } else 130193323Sed return false; 131193323Sed 132193323Sed // Okay, we have at least one ~0 value, check to see if the rest match or are 133235633Sdim // undefs. Even with the above element type twiddling, this should be OK, as 134235633Sdim // the same type legalization should have applied to all the elements. 135193323Sed for (++i; i != e; ++i) 136193323Sed if (N->getOperand(i) != NotZero && 137193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 138193323Sed return false; 139193323Sed return true; 140193323Sed} 141193323Sed 142193323Sed 143193323Sed/// isBuildVectorAllZeros - Return true if the specified node is a 144193323Sed/// BUILD_VECTOR where all of the elements are 0 or undef. 145193323Sedbool ISD::isBuildVectorAllZeros(const SDNode *N) { 146193323Sed // Look through a bit convert. 147218893Sdim if (N->getOpcode() == ISD::BITCAST) 148193323Sed N = N->getOperand(0).getNode(); 149193323Sed 150193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 151193323Sed 152193323Sed unsigned i = 0, e = N->getNumOperands(); 153193323Sed 154193323Sed // Skip over all of the undef values. 155193323Sed while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 156193323Sed ++i; 157193323Sed 158193323Sed // Do not accept an all-undef vector. 159193323Sed if (i == e) return false; 160193323Sed 161193574Sed // Do not accept build_vectors that aren't all constants or which have non-0 162193323Sed // elements. 163193323Sed SDValue Zero = N->getOperand(i); 164245431Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) { 165245431Sdim if (!CN->isNullValue()) 166193323Sed return false; 167245431Sdim } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) { 168245431Sdim if (!CFPN->getValueAPF().isPosZero()) 169193323Sed return false; 170193323Sed } else 171193323Sed return false; 172193323Sed 173193574Sed // Okay, we have at least one 0 value, check to see if the rest match or are 174193323Sed // undefs. 175193323Sed for (++i; i != e; ++i) 176193323Sed if (N->getOperand(i) != Zero && 177193323Sed N->getOperand(i).getOpcode() != ISD::UNDEF) 178193323Sed return false; 179193323Sed return true; 180193323Sed} 181193323Sed 182193323Sed/// isScalarToVector - Return true if the specified node is a 183193323Sed/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 184193323Sed/// element is not an undef. 185193323Sedbool ISD::isScalarToVector(const SDNode *N) { 186193323Sed if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 187193323Sed return true; 188193323Sed 189193323Sed if (N->getOpcode() != ISD::BUILD_VECTOR) 190193323Sed return false; 191193323Sed if (N->getOperand(0).getOpcode() == ISD::UNDEF) 192193323Sed return false; 193193323Sed unsigned NumElems = N->getNumOperands(); 194218893Sdim if (NumElems == 1) 195218893Sdim return false; 196193323Sed for (unsigned i = 1; i < NumElems; ++i) { 197193323Sed SDValue V = N->getOperand(i); 198193323Sed if (V.getOpcode() != ISD::UNDEF) 199193323Sed return false; 200193323Sed } 201193323Sed return true; 202193323Sed} 203193323Sed 204245431Sdim/// allOperandsUndef - Return true if the node has at least one operand 205245431Sdim/// and all operands of the specified node are ISD::UNDEF. 206245431Sdimbool ISD::allOperandsUndef(const SDNode *N) { 207245431Sdim // Return false if the node has no operands. 208245431Sdim // This is "logically inconsistent" with the definition of "all" but 209245431Sdim // is probably the desired behavior. 210245431Sdim if (N->getNumOperands() == 0) 211245431Sdim return false; 212245431Sdim 213245431Sdim for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i) 214245431Sdim if (N->getOperand(i).getOpcode() != ISD::UNDEF) 215245431Sdim return false; 216245431Sdim 217245431Sdim return true; 218245431Sdim} 219245431Sdim 220193323Sed/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 221193323Sed/// when given the operation for (X op Y). 222193323SedISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 223193323Sed // To perform this operation, we just need to swap the L and G bits of the 224193323Sed // operation. 225193323Sed unsigned OldL = (Operation >> 2) & 1; 226193323Sed unsigned OldG = (Operation >> 1) & 1; 227193323Sed return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 228193323Sed (OldL << 1) | // New G bit 229193323Sed (OldG << 2)); // New L bit. 230193323Sed} 231193323Sed 232193323Sed/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 233193323Sed/// 'op' is a valid SetCC operation. 234193323SedISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 235193323Sed unsigned Operation = Op; 236193323Sed if (isInteger) 237193323Sed Operation ^= 7; // Flip L, G, E bits, but not U. 238193323Sed else 239193323Sed Operation ^= 15; // Flip all of the condition bits. 240193323Sed 241193323Sed if (Operation > ISD::SETTRUE2) 242193323Sed Operation &= ~8; // Don't let N and U bits get set. 243193323Sed 244193323Sed return ISD::CondCode(Operation); 245193323Sed} 246193323Sed 247193323Sed 248193323Sed/// isSignedOp - For an integer comparison, return 1 if the comparison is a 249193323Sed/// signed operation and 2 if the result is an unsigned comparison. Return zero 250193323Sed/// if the operation does not depend on the sign of the input (setne and seteq). 251193323Sedstatic int isSignedOp(ISD::CondCode Opcode) { 252193323Sed switch (Opcode) { 253198090Srdivacky default: llvm_unreachable("Illegal integer setcc operation!"); 254193323Sed case ISD::SETEQ: 255193323Sed case ISD::SETNE: return 0; 256193323Sed case ISD::SETLT: 257193323Sed case ISD::SETLE: 258193323Sed case ISD::SETGT: 259193323Sed case ISD::SETGE: return 1; 260193323Sed case ISD::SETULT: 261193323Sed case ISD::SETULE: 262193323Sed case ISD::SETUGT: 263193323Sed case ISD::SETUGE: return 2; 264193323Sed } 265193323Sed} 266193323Sed 267193323Sed/// getSetCCOrOperation - Return the result of a logical OR between different 268193323Sed/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 269193323Sed/// returns SETCC_INVALID if it is not possible to represent the resultant 270193323Sed/// comparison. 271193323SedISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 272193323Sed bool isInteger) { 273193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 274193323Sed // Cannot fold a signed integer setcc with an unsigned integer setcc. 275193323Sed return ISD::SETCC_INVALID; 276193323Sed 277193323Sed unsigned Op = Op1 | Op2; // Combine all of the condition bits. 278193323Sed 279193323Sed // If the N and U bits get set then the resultant comparison DOES suddenly 280193323Sed // care about orderedness, and is true when ordered. 281193323Sed if (Op > ISD::SETTRUE2) 282193323Sed Op &= ~16; // Clear the U bit if the N bit is set. 283193323Sed 284193323Sed // Canonicalize illegal integer setcc's. 285193323Sed if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 286193323Sed Op = ISD::SETNE; 287193323Sed 288193323Sed return ISD::CondCode(Op); 289193323Sed} 290193323Sed 291193323Sed/// getSetCCAndOperation - Return the result of a logical AND between different 292193323Sed/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 293193323Sed/// function returns zero if it is not possible to represent the resultant 294193323Sed/// comparison. 295193323SedISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 296193323Sed bool isInteger) { 297193323Sed if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 298193323Sed // Cannot fold a signed setcc with an unsigned setcc. 299193323Sed return ISD::SETCC_INVALID; 300193323Sed 301193323Sed // Combine all of the condition bits. 302193323Sed ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 303193323Sed 304193323Sed // Canonicalize illegal integer setcc's. 305193323Sed if (isInteger) { 306193323Sed switch (Result) { 307193323Sed default: break; 308193323Sed case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 309193323Sed case ISD::SETOEQ: // SETEQ & SETU[LG]E 310193323Sed case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 311193323Sed case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 312193323Sed case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 313193323Sed } 314193323Sed } 315193323Sed 316193323Sed return Result; 317193323Sed} 318193323Sed 319193323Sed//===----------------------------------------------------------------------===// 320193323Sed// SDNode Profile Support 321193323Sed//===----------------------------------------------------------------------===// 322193323Sed 323193323Sed/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 324193323Sed/// 325193323Sedstatic void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 326193323Sed ID.AddInteger(OpC); 327193323Sed} 328193323Sed 329193323Sed/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 330193323Sed/// solely with their pointer. 331193323Sedstatic void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 332193323Sed ID.AddPointer(VTList.VTs); 333193323Sed} 334193323Sed 335193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 336193323Sed/// 337193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 338193323Sed const SDValue *Ops, unsigned NumOps) { 339193323Sed for (; NumOps; --NumOps, ++Ops) { 340193323Sed ID.AddPointer(Ops->getNode()); 341193323Sed ID.AddInteger(Ops->getResNo()); 342193323Sed } 343193323Sed} 344193323Sed 345193323Sed/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 346193323Sed/// 347193323Sedstatic void AddNodeIDOperands(FoldingSetNodeID &ID, 348193323Sed const SDUse *Ops, unsigned NumOps) { 349193323Sed for (; NumOps; --NumOps, ++Ops) { 350193323Sed ID.AddPointer(Ops->getNode()); 351193323Sed ID.AddInteger(Ops->getResNo()); 352193323Sed } 353193323Sed} 354193323Sed 355193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, 356193323Sed unsigned short OpC, SDVTList VTList, 357193323Sed const SDValue *OpList, unsigned N) { 358193323Sed AddNodeIDOpcode(ID, OpC); 359193323Sed AddNodeIDValueTypes(ID, VTList); 360193323Sed AddNodeIDOperands(ID, OpList, N); 361193323Sed} 362193323Sed 363193323Sed/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 364193323Sed/// the NodeID data. 365193323Sedstatic void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 366193323Sed switch (N->getOpcode()) { 367195098Sed case ISD::TargetExternalSymbol: 368195098Sed case ISD::ExternalSymbol: 369198090Srdivacky llvm_unreachable("Should only be used on nodes with operands"); 370193323Sed default: break; // Normal nodes don't need extra info. 371193323Sed case ISD::TargetConstant: 372193323Sed case ISD::Constant: 373193323Sed ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 374193323Sed break; 375193323Sed case ISD::TargetConstantFP: 376193323Sed case ISD::ConstantFP: { 377193323Sed ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 378193323Sed break; 379193323Sed } 380193323Sed case ISD::TargetGlobalAddress: 381193323Sed case ISD::GlobalAddress: 382193323Sed case ISD::TargetGlobalTLSAddress: 383193323Sed case ISD::GlobalTLSAddress: { 384193323Sed const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 385193323Sed ID.AddPointer(GA->getGlobal()); 386193323Sed ID.AddInteger(GA->getOffset()); 387195098Sed ID.AddInteger(GA->getTargetFlags()); 388245431Sdim ID.AddInteger(GA->getAddressSpace()); 389193323Sed break; 390193323Sed } 391193323Sed case ISD::BasicBlock: 392193323Sed ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 393193323Sed break; 394193323Sed case ISD::Register: 395193323Sed ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 396193323Sed break; 397235633Sdim case ISD::RegisterMask: 398235633Sdim ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 399235633Sdim break; 400193323Sed case ISD::SRCVALUE: 401193323Sed ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 402193323Sed break; 403193323Sed case ISD::FrameIndex: 404193323Sed case ISD::TargetFrameIndex: 405193323Sed ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 406193323Sed break; 407193323Sed case ISD::JumpTable: 408193323Sed case ISD::TargetJumpTable: 409193323Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 410195098Sed ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 411193323Sed break; 412193323Sed case ISD::ConstantPool: 413193323Sed case ISD::TargetConstantPool: { 414193323Sed const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 415193323Sed ID.AddInteger(CP->getAlignment()); 416193323Sed ID.AddInteger(CP->getOffset()); 417193323Sed if (CP->isMachineConstantPoolEntry()) 418226890Sdim CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 419193323Sed else 420193323Sed ID.AddPointer(CP->getConstVal()); 421195098Sed ID.AddInteger(CP->getTargetFlags()); 422193323Sed break; 423193323Sed } 424245431Sdim case ISD::TargetIndex: { 425245431Sdim const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 426245431Sdim ID.AddInteger(TI->getIndex()); 427245431Sdim ID.AddInteger(TI->getOffset()); 428245431Sdim ID.AddInteger(TI->getTargetFlags()); 429245431Sdim break; 430245431Sdim } 431193323Sed case ISD::LOAD: { 432193323Sed const LoadSDNode *LD = cast<LoadSDNode>(N); 433193323Sed ID.AddInteger(LD->getMemoryVT().getRawBits()); 434193323Sed ID.AddInteger(LD->getRawSubclassData()); 435245431Sdim ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 436193323Sed break; 437193323Sed } 438193323Sed case ISD::STORE: { 439193323Sed const StoreSDNode *ST = cast<StoreSDNode>(N); 440193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 441193323Sed ID.AddInteger(ST->getRawSubclassData()); 442245431Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 443193323Sed break; 444193323Sed } 445193323Sed case ISD::ATOMIC_CMP_SWAP: 446193323Sed case ISD::ATOMIC_SWAP: 447193323Sed case ISD::ATOMIC_LOAD_ADD: 448193323Sed case ISD::ATOMIC_LOAD_SUB: 449193323Sed case ISD::ATOMIC_LOAD_AND: 450193323Sed case ISD::ATOMIC_LOAD_OR: 451193323Sed case ISD::ATOMIC_LOAD_XOR: 452193323Sed case ISD::ATOMIC_LOAD_NAND: 453193323Sed case ISD::ATOMIC_LOAD_MIN: 454193323Sed case ISD::ATOMIC_LOAD_MAX: 455193323Sed case ISD::ATOMIC_LOAD_UMIN: 456226890Sdim case ISD::ATOMIC_LOAD_UMAX: 457226890Sdim case ISD::ATOMIC_LOAD: 458226890Sdim case ISD::ATOMIC_STORE: { 459193323Sed const AtomicSDNode *AT = cast<AtomicSDNode>(N); 460193323Sed ID.AddInteger(AT->getMemoryVT().getRawBits()); 461193323Sed ID.AddInteger(AT->getRawSubclassData()); 462245431Sdim ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 463193323Sed break; 464193323Sed } 465245431Sdim case ISD::PREFETCH: { 466245431Sdim const MemSDNode *PF = cast<MemSDNode>(N); 467245431Sdim ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 468245431Sdim break; 469245431Sdim } 470193323Sed case ISD::VECTOR_SHUFFLE: { 471193323Sed const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 472198090Srdivacky for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 473193323Sed i != e; ++i) 474193323Sed ID.AddInteger(SVN->getMaskElt(i)); 475193323Sed break; 476193323Sed } 477198892Srdivacky case ISD::TargetBlockAddress: 478198892Srdivacky case ISD::BlockAddress: { 479245431Sdim const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 480245431Sdim ID.AddPointer(BA->getBlockAddress()); 481245431Sdim ID.AddInteger(BA->getOffset()); 482245431Sdim ID.AddInteger(BA->getTargetFlags()); 483198892Srdivacky break; 484198892Srdivacky } 485193323Sed } // end switch (N->getOpcode()) 486245431Sdim 487245431Sdim // Target specific memory nodes could also have address spaces to check. 488245431Sdim if (N->isTargetMemoryOpcode()) 489245431Sdim ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 490193323Sed} 491193323Sed 492193323Sed/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 493193323Sed/// data. 494193323Sedstatic void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 495193323Sed AddNodeIDOpcode(ID, N->getOpcode()); 496193323Sed // Add the return value info. 497193323Sed AddNodeIDValueTypes(ID, N->getVTList()); 498193323Sed // Add the operand info. 499193323Sed AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 500193323Sed 501193323Sed // Handle SDNode leafs with special info. 502193323Sed AddNodeIDCustom(ID, N); 503193323Sed} 504193323Sed 505193323Sed/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 506204642Srdivacky/// the CSE map that carries volatility, temporalness, indexing mode, and 507193323Sed/// extension/truncation information. 508193323Sed/// 509193323Sedstatic inline unsigned 510204642SrdivackyencodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 511235633Sdim bool isNonTemporal, bool isInvariant) { 512193323Sed assert((ConvType & 3) == ConvType && 513193323Sed "ConvType may not require more than 2 bits!"); 514193323Sed assert((AM & 7) == AM && 515193323Sed "AM may not require more than 3 bits!"); 516193323Sed return ConvType | 517193323Sed (AM << 2) | 518204642Srdivacky (isVolatile << 5) | 519235633Sdim (isNonTemporal << 6) | 520235633Sdim (isInvariant << 7); 521193323Sed} 522193323Sed 523193323Sed//===----------------------------------------------------------------------===// 524193323Sed// SelectionDAG Class 525193323Sed//===----------------------------------------------------------------------===// 526193323Sed 527193323Sed/// doNotCSE - Return true if CSE should not be performed for this node. 528193323Sedstatic bool doNotCSE(SDNode *N) { 529218893Sdim if (N->getValueType(0) == MVT::Glue) 530193323Sed return true; // Never CSE anything that produces a flag. 531193323Sed 532193323Sed switch (N->getOpcode()) { 533193323Sed default: break; 534193323Sed case ISD::HANDLENODE: 535193323Sed case ISD::EH_LABEL: 536193323Sed return true; // Never CSE these nodes. 537193323Sed } 538193323Sed 539193323Sed // Check that remaining values produced are not flags. 540193323Sed for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 541218893Sdim if (N->getValueType(i) == MVT::Glue) 542193323Sed return true; // Never CSE anything that produces a flag. 543193323Sed 544193323Sed return false; 545193323Sed} 546193323Sed 547193323Sed/// RemoveDeadNodes - This method deletes all unreachable nodes in the 548193323Sed/// SelectionDAG. 549193323Sedvoid SelectionDAG::RemoveDeadNodes() { 550193323Sed // Create a dummy node (which is not added to allnodes), that adds a reference 551193323Sed // to the root node, preventing it from being deleted. 552193323Sed HandleSDNode Dummy(getRoot()); 553193323Sed 554193323Sed SmallVector<SDNode*, 128> DeadNodes; 555193323Sed 556193323Sed // Add all obviously-dead nodes to the DeadNodes worklist. 557193323Sed for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 558193323Sed if (I->use_empty()) 559193323Sed DeadNodes.push_back(I); 560193323Sed 561193323Sed RemoveDeadNodes(DeadNodes); 562193323Sed 563193323Sed // If the root changed (e.g. it was a dead load, update the root). 564193323Sed setRoot(Dummy.getValue()); 565193323Sed} 566193323Sed 567193323Sed/// RemoveDeadNodes - This method deletes the unreachable nodes in the 568193323Sed/// given list, and any nodes that become unreachable as a result. 569245431Sdimvoid SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 570193323Sed 571193323Sed // Process the worklist, deleting the nodes and adding their uses to the 572193323Sed // worklist. 573193323Sed while (!DeadNodes.empty()) { 574193323Sed SDNode *N = DeadNodes.pop_back_val(); 575193323Sed 576245431Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 577245431Sdim DUL->NodeDeleted(N, 0); 578193323Sed 579193323Sed // Take the node out of the appropriate CSE map. 580193323Sed RemoveNodeFromCSEMaps(N); 581193323Sed 582193323Sed // Next, brutally remove the operand list. This is safe to do, as there are 583193323Sed // no cycles in the graph. 584193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 585193323Sed SDUse &Use = *I++; 586193323Sed SDNode *Operand = Use.getNode(); 587193323Sed Use.set(SDValue()); 588193323Sed 589193323Sed // Now that we removed this operand, see if there are no uses of it left. 590193323Sed if (Operand->use_empty()) 591193323Sed DeadNodes.push_back(Operand); 592193323Sed } 593193323Sed 594193323Sed DeallocateNode(N); 595193323Sed } 596193323Sed} 597193323Sed 598245431Sdimvoid SelectionDAG::RemoveDeadNode(SDNode *N){ 599193323Sed SmallVector<SDNode*, 16> DeadNodes(1, N); 600235633Sdim 601235633Sdim // Create a dummy node that adds a reference to the root node, preventing 602235633Sdim // it from being deleted. (This matters if the root is an operand of the 603235633Sdim // dead node.) 604235633Sdim HandleSDNode Dummy(getRoot()); 605235633Sdim 606245431Sdim RemoveDeadNodes(DeadNodes); 607193323Sed} 608193323Sed 609193323Sedvoid SelectionDAG::DeleteNode(SDNode *N) { 610193323Sed // First take this out of the appropriate CSE map. 611193323Sed RemoveNodeFromCSEMaps(N); 612193323Sed 613193323Sed // Finally, remove uses due to operands of this node, remove from the 614193323Sed // AllNodes list, and delete the node. 615193323Sed DeleteNodeNotInCSEMaps(N); 616193323Sed} 617193323Sed 618193323Sedvoid SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 619193323Sed assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 620193323Sed assert(N->use_empty() && "Cannot delete a node that is not dead!"); 621193323Sed 622193323Sed // Drop all of the operands and decrement used node's use counts. 623193323Sed N->DropOperands(); 624193323Sed 625193323Sed DeallocateNode(N); 626193323Sed} 627193323Sed 628193323Sedvoid SelectionDAG::DeallocateNode(SDNode *N) { 629193323Sed if (N->OperandsNeedDelete) 630193323Sed delete[] N->OperandList; 631193323Sed 632193323Sed // Set the opcode to DELETED_NODE to help catch bugs when node 633193323Sed // memory is reallocated. 634193323Sed N->NodeType = ISD::DELETED_NODE; 635193323Sed 636193323Sed NodeAllocator.Deallocate(AllNodes.remove(N)); 637200581Srdivacky 638206083Srdivacky // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 639224145Sdim ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 640206083Srdivacky for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 641206083Srdivacky DbgVals[i]->setIsInvalidated(); 642193323Sed} 643193323Sed 644193323Sed/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 645193323Sed/// correspond to it. This is useful when we're about to delete or repurpose 646193323Sed/// the node. We don't want future request for structurally identical nodes 647193323Sed/// to return N anymore. 648193323Sedbool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 649193323Sed bool Erased = false; 650193323Sed switch (N->getOpcode()) { 651193323Sed case ISD::HANDLENODE: return false; // noop. 652193323Sed case ISD::CONDCODE: 653193323Sed assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 654193323Sed "Cond code doesn't exist!"); 655193323Sed Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 656193323Sed CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 657193323Sed break; 658193323Sed case ISD::ExternalSymbol: 659193323Sed Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 660193323Sed break; 661195098Sed case ISD::TargetExternalSymbol: { 662195098Sed ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 663195098Sed Erased = TargetExternalSymbols.erase( 664195098Sed std::pair<std::string,unsigned char>(ESN->getSymbol(), 665195098Sed ESN->getTargetFlags())); 666193323Sed break; 667195098Sed } 668193323Sed case ISD::VALUETYPE: { 669198090Srdivacky EVT VT = cast<VTSDNode>(N)->getVT(); 670193323Sed if (VT.isExtended()) { 671193323Sed Erased = ExtendedValueTypeNodes.erase(VT); 672193323Sed } else { 673198090Srdivacky Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 674198090Srdivacky ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 675193323Sed } 676193323Sed break; 677193323Sed } 678193323Sed default: 679193323Sed // Remove it from the CSE Map. 680218893Sdim assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 681218893Sdim assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 682193323Sed Erased = CSEMap.RemoveNode(N); 683193323Sed break; 684193323Sed } 685193323Sed#ifndef NDEBUG 686193323Sed // Verify that the node was actually in one of the CSE maps, unless it has a 687193323Sed // flag result (which cannot be CSE'd) or is one of the special cases that are 688193323Sed // not subject to CSE. 689218893Sdim if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 690193323Sed !N->isMachineOpcode() && !doNotCSE(N)) { 691193323Sed N->dump(this); 692202375Srdivacky dbgs() << "\n"; 693198090Srdivacky llvm_unreachable("Node is not in map!"); 694193323Sed } 695193323Sed#endif 696193323Sed return Erased; 697193323Sed} 698193323Sed 699193323Sed/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 700193323Sed/// maps and modified in place. Add it back to the CSE maps, unless an identical 701193323Sed/// node already exists, in which case transfer all its users to the existing 702193323Sed/// node. This transfer can potentially trigger recursive merging. 703193323Sed/// 704193323Sedvoid 705245431SdimSelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 706193323Sed // For node types that aren't CSE'd, just act as if no identical node 707193323Sed // already exists. 708193323Sed if (!doNotCSE(N)) { 709193323Sed SDNode *Existing = CSEMap.GetOrInsertNode(N); 710193323Sed if (Existing != N) { 711193323Sed // If there was already an existing matching node, use ReplaceAllUsesWith 712193323Sed // to replace the dead one with the existing one. This can cause 713193323Sed // recursive merging of other unrelated nodes down the line. 714245431Sdim ReplaceAllUsesWith(N, Existing); 715193323Sed 716245431Sdim // N is now dead. Inform the listeners and delete it. 717245431Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 718245431Sdim DUL->NodeDeleted(N, Existing); 719193323Sed DeleteNodeNotInCSEMaps(N); 720193323Sed return; 721193323Sed } 722193323Sed } 723193323Sed 724245431Sdim // If the node doesn't already exist, we updated it. Inform listeners. 725245431Sdim for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 726245431Sdim DUL->NodeUpdated(N); 727193323Sed} 728193323Sed 729193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 730193323Sed/// were replaced with those specified. If this node is never memoized, 731193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 732193323Sed/// node already exists with these operands, the slot will be non-null. 733193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 734193323Sed void *&InsertPos) { 735193323Sed if (doNotCSE(N)) 736193323Sed return 0; 737193323Sed 738193323Sed SDValue Ops[] = { Op }; 739193323Sed FoldingSetNodeID ID; 740193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 741193323Sed AddNodeIDCustom(ID, N); 742200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 743200581Srdivacky return Node; 744193323Sed} 745193323Sed 746193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 747193323Sed/// were replaced with those specified. If this node is never memoized, 748193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 749193323Sed/// node already exists with these operands, the slot will be non-null. 750193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 751193323Sed SDValue Op1, SDValue Op2, 752193323Sed void *&InsertPos) { 753193323Sed if (doNotCSE(N)) 754193323Sed return 0; 755193323Sed 756193323Sed SDValue Ops[] = { Op1, Op2 }; 757193323Sed FoldingSetNodeID ID; 758193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 759193323Sed AddNodeIDCustom(ID, N); 760200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 761200581Srdivacky return Node; 762193323Sed} 763193323Sed 764193323Sed 765193323Sed/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 766193323Sed/// were replaced with those specified. If this node is never memoized, 767193323Sed/// return null, otherwise return a pointer to the slot it would take. If a 768193323Sed/// node already exists with these operands, the slot will be non-null. 769193323SedSDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 770193323Sed const SDValue *Ops,unsigned NumOps, 771193323Sed void *&InsertPos) { 772193323Sed if (doNotCSE(N)) 773193323Sed return 0; 774193323Sed 775193323Sed FoldingSetNodeID ID; 776193323Sed AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 777193323Sed AddNodeIDCustom(ID, N); 778200581Srdivacky SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 779200581Srdivacky return Node; 780193323Sed} 781193323Sed 782218893Sdim#ifndef NDEBUG 783218893Sdim/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 784218893Sdimstatic void VerifyNodeCommon(SDNode *N) { 785193323Sed switch (N->getOpcode()) { 786193323Sed default: 787193323Sed break; 788193323Sed case ISD::BUILD_PAIR: { 789198090Srdivacky EVT VT = N->getValueType(0); 790193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 791193323Sed assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 792193323Sed "Wrong return type!"); 793193323Sed assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 794193323Sed assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 795193323Sed "Mismatched operand types!"); 796193323Sed assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 797193323Sed "Wrong operand type!"); 798193323Sed assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 799193323Sed "Wrong return type size"); 800193323Sed break; 801193323Sed } 802193323Sed case ISD::BUILD_VECTOR: { 803193323Sed assert(N->getNumValues() == 1 && "Too many results!"); 804193323Sed assert(N->getValueType(0).isVector() && "Wrong return type!"); 805193323Sed assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 806193323Sed "Wrong number of operands!"); 807198090Srdivacky EVT EltVT = N->getValueType(0).getVectorElementType(); 808226890Sdim for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 809193323Sed assert((I->getValueType() == EltVT || 810193323Sed (EltVT.isInteger() && I->getValueType().isInteger() && 811193323Sed EltVT.bitsLE(I->getValueType()))) && 812193323Sed "Wrong operand type!"); 813226890Sdim assert(I->getValueType() == N->getOperand(0).getValueType() && 814226890Sdim "Operands must all have the same type"); 815226890Sdim } 816193323Sed break; 817193323Sed } 818193323Sed } 819193323Sed} 820193323Sed 821218893Sdim/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 822218893Sdimstatic void VerifySDNode(SDNode *N) { 823218893Sdim // The SDNode allocators cannot be used to allocate nodes with fields that are 824218893Sdim // not present in an SDNode! 825218893Sdim assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 826218893Sdim assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 827218893Sdim assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 828218893Sdim assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 829218893Sdim assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 830218893Sdim assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 831218893Sdim assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 832218893Sdim assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 833218893Sdim assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 834218893Sdim assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 835218893Sdim assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 836218893Sdim assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 837218893Sdim assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 838218893Sdim assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 839218893Sdim assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 840218893Sdim assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 841218893Sdim assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 842218893Sdim assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 843218893Sdim assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 844218893Sdim 845218893Sdim VerifyNodeCommon(N); 846218893Sdim} 847218893Sdim 848218893Sdim/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 849218893Sdim/// invalid. 850218893Sdimstatic void VerifyMachineNode(SDNode *N) { 851218893Sdim // The MachineNode allocators cannot be used to allocate nodes with fields 852218893Sdim // that are not present in a MachineNode! 853218893Sdim // Currently there are no such nodes. 854218893Sdim 855218893Sdim VerifyNodeCommon(N); 856218893Sdim} 857218893Sdim#endif // NDEBUG 858218893Sdim 859198090Srdivacky/// getEVTAlignment - Compute the default alignment value for the 860193323Sed/// given type. 861193323Sed/// 862198090Srdivackyunsigned SelectionDAG::getEVTAlignment(EVT VT) const { 863226890Sdim Type *Ty = VT == MVT::iPTR ? 864198090Srdivacky PointerType::get(Type::getInt8Ty(*getContext()), 0) : 865198090Srdivacky VT.getTypeForEVT(*getContext()); 866193323Sed 867263509Sdim return TM.getTargetLowering()->getDataLayout()->getABITypeAlignment(Ty); 868193323Sed} 869193323Sed 870193323Sed// EntryNode could meaningfully have debug info if we can find it... 871235633SdimSelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 872263509Sdim : TM(tm), TSI(*tm.getSelectionDAGInfo()), TTI(0), TLI(0), OptLevel(OL), 873263509Sdim EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 874263509Sdim Root(getEntryNode()), NewNodesMustHaveLegalTypes(false), 875263509Sdim UpdateListeners(0) { 876193323Sed AllNodes.push_back(&EntryNode); 877205218Srdivacky DbgInfo = new SDDbgInfo(); 878193323Sed} 879193323Sed 880263509Sdimvoid SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti, 881263509Sdim const TargetLowering *tli) { 882193323Sed MF = &mf; 883252723Sdim TTI = tti; 884263509Sdim TLI = tli; 885198090Srdivacky Context = &mf.getFunction()->getContext(); 886193323Sed} 887193323Sed 888193323SedSelectionDAG::~SelectionDAG() { 889245431Sdim assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 890193323Sed allnodes_clear(); 891205218Srdivacky delete DbgInfo; 892193323Sed} 893193323Sed 894193323Sedvoid SelectionDAG::allnodes_clear() { 895193323Sed assert(&*AllNodes.begin() == &EntryNode); 896193323Sed AllNodes.remove(AllNodes.begin()); 897193323Sed while (!AllNodes.empty()) 898193323Sed DeallocateNode(AllNodes.begin()); 899193323Sed} 900193323Sed 901193323Sedvoid SelectionDAG::clear() { 902193323Sed allnodes_clear(); 903193323Sed OperandAllocator.Reset(); 904193323Sed CSEMap.clear(); 905193323Sed 906193323Sed ExtendedValueTypeNodes.clear(); 907193323Sed ExternalSymbols.clear(); 908193323Sed TargetExternalSymbols.clear(); 909193323Sed std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 910193323Sed static_cast<CondCodeSDNode*>(0)); 911193323Sed std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 912193323Sed static_cast<SDNode*>(0)); 913193323Sed 914193323Sed EntryNode.UseList = 0; 915193323Sed AllNodes.push_back(&EntryNode); 916193323Sed Root = getEntryNode(); 917206083Srdivacky DbgInfo->clear(); 918193323Sed} 919193323Sed 920263509SdimSDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 921226890Sdim return VT.bitsGT(Op.getValueType()) ? 922226890Sdim getNode(ISD::ANY_EXTEND, DL, VT, Op) : 923226890Sdim getNode(ISD::TRUNCATE, DL, VT, Op); 924226890Sdim} 925226890Sdim 926263509SdimSDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 927198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 928198090Srdivacky getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 929198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 930198090Srdivacky} 931198090Srdivacky 932263509SdimSDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 933198090Srdivacky return VT.bitsGT(Op.getValueType()) ? 934198090Srdivacky getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 935198090Srdivacky getNode(ISD::TRUNCATE, DL, VT, Op); 936198090Srdivacky} 937198090Srdivacky 938263509SdimSDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) { 939200581Srdivacky assert(!VT.isVector() && 940200581Srdivacky "getZeroExtendInReg should use the vector element type instead of " 941200581Srdivacky "the vector type!"); 942193323Sed if (Op.getValueType() == VT) return Op; 943200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 944200581Srdivacky APInt Imm = APInt::getLowBitsSet(BitWidth, 945193323Sed VT.getSizeInBits()); 946193323Sed return getNode(ISD::AND, DL, Op.getValueType(), Op, 947193323Sed getConstant(Imm, Op.getValueType())); 948193323Sed} 949193323Sed 950193323Sed/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 951193323Sed/// 952263509SdimSDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) { 953204642Srdivacky EVT EltVT = VT.getScalarType(); 954193323Sed SDValue NegOne = 955193323Sed getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 956193323Sed return getNode(ISD::XOR, DL, VT, Val, NegOne); 957193323Sed} 958193323Sed 959198090SrdivackySDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 960204642Srdivacky EVT EltVT = VT.getScalarType(); 961193323Sed assert((EltVT.getSizeInBits() >= 64 || 962193323Sed (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 963193323Sed "getConstant with a uint64_t value that doesn't fit in the type!"); 964193323Sed return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 965193323Sed} 966193323Sed 967198090SrdivackySDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 968198090Srdivacky return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 969193323Sed} 970193323Sed 971198090SrdivackySDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 972193323Sed assert(VT.isInteger() && "Cannot create FP integer constant!"); 973193323Sed 974204642Srdivacky EVT EltVT = VT.getScalarType(); 975226890Sdim const ConstantInt *Elt = &Val; 976226890Sdim 977263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 978263509Sdim 979226890Sdim // In some cases the vector type is legal but the element type is illegal and 980226890Sdim // needs to be promoted, for example v8i8 on ARM. In this case, promote the 981226890Sdim // inserted value (the type does not need to match the vector element type). 982226890Sdim // Any extra bits introduced will be truncated away. 983263509Sdim if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 984226890Sdim TargetLowering::TypePromoteInteger) { 985263509Sdim EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 986226890Sdim APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 987226890Sdim Elt = ConstantInt::get(*getContext(), NewVal); 988226890Sdim } 989263509Sdim // In other cases the element type is illegal and needs to be expanded, for 990263509Sdim // example v2i64 on MIPS32. In this case, find the nearest legal type, split 991263509Sdim // the value into n parts and use a vector type with n-times the elements. 992263509Sdim // Then bitcast to the type requested. 993263509Sdim // Legalizing constants too early makes the DAGCombiner's job harder so we 994263509Sdim // only legalize if the DAG tells us we must produce legal types. 995263509Sdim else if (NewNodesMustHaveLegalTypes && VT.isVector() && 996263509Sdim TLI->getTypeAction(*getContext(), EltVT) == 997263509Sdim TargetLowering::TypeExpandInteger) { 998263509Sdim APInt NewVal = Elt->getValue(); 999263509Sdim EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 1000263509Sdim unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); 1001263509Sdim unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; 1002263509Sdim EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); 1003226890Sdim 1004263509Sdim // Check the temporary vector is the correct size. If this fails then 1005263509Sdim // getTypeToTransformTo() probably returned a type whose size (in bits) 1006263509Sdim // isn't a power-of-2 factor of the requested type size. 1007263509Sdim assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); 1008263509Sdim 1009263509Sdim SmallVector<SDValue, 2> EltParts; 1010263509Sdim for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { 1011263509Sdim EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) 1012263509Sdim .trunc(ViaEltSizeInBits), 1013263509Sdim ViaEltVT, isT)); 1014263509Sdim } 1015263509Sdim 1016263509Sdim // EltParts is currently in little endian order. If we actually want 1017263509Sdim // big-endian order then reverse it now. 1018263509Sdim if (TLI->isBigEndian()) 1019263509Sdim std::reverse(EltParts.begin(), EltParts.end()); 1020263509Sdim 1021263509Sdim // The elements must be reversed when the element order is different 1022263509Sdim // to the endianness of the elements (because the BITCAST is itself a 1023263509Sdim // vector shuffle in this situation). However, we do not need any code to 1024263509Sdim // perform this reversal because getConstant() is producing a vector 1025263509Sdim // splat. 1026263509Sdim // This situation occurs in MIPS MSA. 1027263509Sdim 1028263509Sdim SmallVector<SDValue, 8> Ops; 1029263509Sdim for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) 1030263509Sdim Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); 1031263509Sdim 1032263509Sdim SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT, 1033263509Sdim getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT, 1034263509Sdim &Ops[0], Ops.size())); 1035263509Sdim return Result; 1036263509Sdim } 1037263509Sdim 1038226890Sdim assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 1039193323Sed "APInt size does not match type size!"); 1040193323Sed unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 1041193323Sed FoldingSetNodeID ID; 1042193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1043226890Sdim ID.AddPointer(Elt); 1044193323Sed void *IP = 0; 1045193323Sed SDNode *N = NULL; 1046201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1047193323Sed if (!VT.isVector()) 1048193323Sed return SDValue(N, 0); 1049201360Srdivacky 1050193323Sed if (!N) { 1051226890Sdim N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 1052193323Sed CSEMap.InsertNode(N, IP); 1053193323Sed AllNodes.push_back(N); 1054193323Sed } 1055193323Sed 1056193323Sed SDValue Result(N, 0); 1057193323Sed if (VT.isVector()) { 1058193323Sed SmallVector<SDValue, 8> Ops; 1059193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1060263509Sdim Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1061193323Sed } 1062193323Sed return Result; 1063193323Sed} 1064193323Sed 1065193323SedSDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 1066263509Sdim return getConstant(Val, TM.getTargetLowering()->getPointerTy(), isTarget); 1067193323Sed} 1068193323Sed 1069193323Sed 1070198090SrdivackySDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 1071198090Srdivacky return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 1072193323Sed} 1073193323Sed 1074198090SrdivackySDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 1075193323Sed assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1076193323Sed 1077204642Srdivacky EVT EltVT = VT.getScalarType(); 1078193323Sed 1079193323Sed // Do the map lookup using the actual bit pattern for the floating point 1080193323Sed // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1081193323Sed // we don't have issues with SNANs. 1082193323Sed unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1083193323Sed FoldingSetNodeID ID; 1084193323Sed AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1085193323Sed ID.AddPointer(&V); 1086193323Sed void *IP = 0; 1087193323Sed SDNode *N = NULL; 1088201360Srdivacky if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1089193323Sed if (!VT.isVector()) 1090193323Sed return SDValue(N, 0); 1091201360Srdivacky 1092193323Sed if (!N) { 1093205407Srdivacky N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1094193323Sed CSEMap.InsertNode(N, IP); 1095193323Sed AllNodes.push_back(N); 1096193323Sed } 1097193323Sed 1098193323Sed SDValue Result(N, 0); 1099193323Sed if (VT.isVector()) { 1100193323Sed SmallVector<SDValue, 8> Ops; 1101193323Sed Ops.assign(VT.getVectorNumElements(), Result); 1102263509Sdim // FIXME SDLoc info might be appropriate here 1103263509Sdim Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1104193323Sed } 1105193323Sed return Result; 1106193323Sed} 1107193323Sed 1108198090SrdivackySDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1109204642Srdivacky EVT EltVT = VT.getScalarType(); 1110193323Sed if (EltVT==MVT::f32) 1111193323Sed return getConstantFP(APFloat((float)Val), VT, isTarget); 1112208599Srdivacky else if (EltVT==MVT::f64) 1113193323Sed return getConstantFP(APFloat(Val), VT, isTarget); 1114252723Sdim else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || 1115252723Sdim EltVT==MVT::f16) { 1116208599Srdivacky bool ignored; 1117208599Srdivacky APFloat apf = APFloat(Val); 1118252723Sdim apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1119208599Srdivacky &ignored); 1120208599Srdivacky return getConstantFP(apf, VT, isTarget); 1121235633Sdim } else 1122235633Sdim llvm_unreachable("Unsupported type in getConstantFP"); 1123193323Sed} 1124193323Sed 1125263509SdimSDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL, 1126198090Srdivacky EVT VT, int64_t Offset, 1127195098Sed bool isTargetGA, 1128195098Sed unsigned char TargetFlags) { 1129195098Sed assert((TargetFlags == 0 || isTargetGA) && 1130195098Sed "Cannot set target flags on target-independent globals"); 1131263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1132198090Srdivacky 1133193323Sed // Truncate (with sign-extension) the offset value to the pointer size. 1134263509Sdim unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 1135193323Sed if (BitWidth < 64) 1136245431Sdim Offset = SignExtend64(Offset, BitWidth); 1137193323Sed 1138193323Sed const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1139193323Sed if (!GVar) { 1140193323Sed // If GV is an alias then use the aliasee for determining thread-localness. 1141193323Sed if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1142193323Sed GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1143193323Sed } 1144193323Sed 1145195098Sed unsigned Opc; 1146193323Sed if (GVar && GVar->isThreadLocal()) 1147193323Sed Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1148193323Sed else 1149193323Sed Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1150193323Sed 1151193323Sed FoldingSetNodeID ID; 1152193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1153193323Sed ID.AddPointer(GV); 1154193323Sed ID.AddInteger(Offset); 1155195098Sed ID.AddInteger(TargetFlags); 1156245431Sdim ID.AddInteger(GV->getType()->getAddressSpace()); 1157193323Sed void *IP = 0; 1158201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1159193323Sed return SDValue(E, 0); 1160201360Srdivacky 1161263509Sdim SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(), 1162263509Sdim DL.getDebugLoc(), GV, VT, 1163205407Srdivacky Offset, TargetFlags); 1164193323Sed CSEMap.InsertNode(N, IP); 1165193323Sed AllNodes.push_back(N); 1166193323Sed return SDValue(N, 0); 1167193323Sed} 1168193323Sed 1169198090SrdivackySDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1170193323Sed unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1171193323Sed FoldingSetNodeID ID; 1172193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1173193323Sed ID.AddInteger(FI); 1174193323Sed void *IP = 0; 1175201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1176193323Sed return SDValue(E, 0); 1177201360Srdivacky 1178205407Srdivacky SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1179193323Sed CSEMap.InsertNode(N, IP); 1180193323Sed AllNodes.push_back(N); 1181193323Sed return SDValue(N, 0); 1182193323Sed} 1183193323Sed 1184198090SrdivackySDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1185195098Sed unsigned char TargetFlags) { 1186195098Sed assert((TargetFlags == 0 || isTarget) && 1187195098Sed "Cannot set target flags on target-independent jump tables"); 1188193323Sed unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1189193323Sed FoldingSetNodeID ID; 1190193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1191193323Sed ID.AddInteger(JTI); 1192195098Sed ID.AddInteger(TargetFlags); 1193193323Sed void *IP = 0; 1194201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1195193323Sed return SDValue(E, 0); 1196201360Srdivacky 1197205407Srdivacky SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1198205407Srdivacky TargetFlags); 1199193323Sed CSEMap.InsertNode(N, IP); 1200193323Sed AllNodes.push_back(N); 1201193323Sed return SDValue(N, 0); 1202193323Sed} 1203193323Sed 1204207618SrdivackySDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1205193323Sed unsigned Alignment, int Offset, 1206198090Srdivacky bool isTarget, 1207195098Sed unsigned char TargetFlags) { 1208195098Sed assert((TargetFlags == 0 || isTarget) && 1209195098Sed "Cannot set target flags on target-independent globals"); 1210193323Sed if (Alignment == 0) 1211263509Sdim Alignment = 1212263509Sdim TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1213193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1214193323Sed FoldingSetNodeID ID; 1215193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1216193323Sed ID.AddInteger(Alignment); 1217193323Sed ID.AddInteger(Offset); 1218193323Sed ID.AddPointer(C); 1219195098Sed ID.AddInteger(TargetFlags); 1220193323Sed void *IP = 0; 1221201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1222193323Sed return SDValue(E, 0); 1223201360Srdivacky 1224205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1225205407Srdivacky Alignment, TargetFlags); 1226193323Sed CSEMap.InsertNode(N, IP); 1227193323Sed AllNodes.push_back(N); 1228193323Sed return SDValue(N, 0); 1229193323Sed} 1230193323Sed 1231193323Sed 1232198090SrdivackySDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1233193323Sed unsigned Alignment, int Offset, 1234195098Sed bool isTarget, 1235195098Sed unsigned char TargetFlags) { 1236195098Sed assert((TargetFlags == 0 || isTarget) && 1237195098Sed "Cannot set target flags on target-independent globals"); 1238193323Sed if (Alignment == 0) 1239263509Sdim Alignment = 1240263509Sdim TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1241193323Sed unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1242193323Sed FoldingSetNodeID ID; 1243193323Sed AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1244193323Sed ID.AddInteger(Alignment); 1245193323Sed ID.AddInteger(Offset); 1246226890Sdim C->addSelectionDAGCSEId(ID); 1247195098Sed ID.AddInteger(TargetFlags); 1248193323Sed void *IP = 0; 1249201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1250193323Sed return SDValue(E, 0); 1251201360Srdivacky 1252205407Srdivacky SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1253205407Srdivacky Alignment, TargetFlags); 1254193323Sed CSEMap.InsertNode(N, IP); 1255193323Sed AllNodes.push_back(N); 1256193323Sed return SDValue(N, 0); 1257193323Sed} 1258193323Sed 1259245431SdimSDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1260245431Sdim unsigned char TargetFlags) { 1261245431Sdim FoldingSetNodeID ID; 1262245431Sdim AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0); 1263245431Sdim ID.AddInteger(Index); 1264245431Sdim ID.AddInteger(Offset); 1265245431Sdim ID.AddInteger(TargetFlags); 1266245431Sdim void *IP = 0; 1267245431Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1268245431Sdim return SDValue(E, 0); 1269245431Sdim 1270245431Sdim SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, 1271245431Sdim TargetFlags); 1272245431Sdim CSEMap.InsertNode(N, IP); 1273245431Sdim AllNodes.push_back(N); 1274245431Sdim return SDValue(N, 0); 1275245431Sdim} 1276245431Sdim 1277193323SedSDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1278193323Sed FoldingSetNodeID ID; 1279193323Sed AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1280193323Sed ID.AddPointer(MBB); 1281193323Sed void *IP = 0; 1282201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1283193323Sed return SDValue(E, 0); 1284201360Srdivacky 1285205407Srdivacky SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1286193323Sed CSEMap.InsertNode(N, IP); 1287193323Sed AllNodes.push_back(N); 1288193323Sed return SDValue(N, 0); 1289193323Sed} 1290193323Sed 1291198090SrdivackySDValue SelectionDAG::getValueType(EVT VT) { 1292198090Srdivacky if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1293198090Srdivacky ValueTypeNodes.size()) 1294198090Srdivacky ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1295193323Sed 1296193323Sed SDNode *&N = VT.isExtended() ? 1297198090Srdivacky ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1298193323Sed 1299193323Sed if (N) return SDValue(N, 0); 1300205407Srdivacky N = new (NodeAllocator) VTSDNode(VT); 1301193323Sed AllNodes.push_back(N); 1302193323Sed return SDValue(N, 0); 1303193323Sed} 1304193323Sed 1305198090SrdivackySDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1306193323Sed SDNode *&N = ExternalSymbols[Sym]; 1307193323Sed if (N) return SDValue(N, 0); 1308205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1309193323Sed AllNodes.push_back(N); 1310193323Sed return SDValue(N, 0); 1311193323Sed} 1312193323Sed 1313198090SrdivackySDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1314195098Sed unsigned char TargetFlags) { 1315195098Sed SDNode *&N = 1316195098Sed TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1317195098Sed TargetFlags)]; 1318193323Sed if (N) return SDValue(N, 0); 1319205407Srdivacky N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1320193323Sed AllNodes.push_back(N); 1321193323Sed return SDValue(N, 0); 1322193323Sed} 1323193323Sed 1324193323SedSDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1325193323Sed if ((unsigned)Cond >= CondCodeNodes.size()) 1326193323Sed CondCodeNodes.resize(Cond+1); 1327193323Sed 1328193323Sed if (CondCodeNodes[Cond] == 0) { 1329205407Srdivacky CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1330193323Sed CondCodeNodes[Cond] = N; 1331193323Sed AllNodes.push_back(N); 1332193323Sed } 1333201360Srdivacky 1334193323Sed return SDValue(CondCodeNodes[Cond], 0); 1335193323Sed} 1336193323Sed 1337193323Sed// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1338193323Sed// the shuffle mask M that point at N1 to point at N2, and indices that point 1339193323Sed// N2 to point at N1. 1340193323Sedstatic void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1341193323Sed std::swap(N1, N2); 1342193323Sed int NElts = M.size(); 1343193323Sed for (int i = 0; i != NElts; ++i) { 1344193323Sed if (M[i] >= NElts) 1345193323Sed M[i] -= NElts; 1346193323Sed else if (M[i] >= 0) 1347193323Sed M[i] += NElts; 1348193323Sed } 1349193323Sed} 1350193323Sed 1351263509SdimSDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, 1352193323Sed SDValue N2, const int *Mask) { 1353263509Sdim assert(VT == N1.getValueType() && VT == N2.getValueType() && 1354263509Sdim "Invalid VECTOR_SHUFFLE"); 1355193323Sed 1356193323Sed // Canonicalize shuffle undef, undef -> undef 1357193323Sed if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1358198090Srdivacky return getUNDEF(VT); 1359193323Sed 1360198090Srdivacky // Validate that all indices in Mask are within the range of the elements 1361193323Sed // input to the shuffle. 1362193323Sed unsigned NElts = VT.getVectorNumElements(); 1363193323Sed SmallVector<int, 8> MaskVec; 1364193323Sed for (unsigned i = 0; i != NElts; ++i) { 1365193323Sed assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1366193323Sed MaskVec.push_back(Mask[i]); 1367193323Sed } 1368198090Srdivacky 1369193323Sed // Canonicalize shuffle v, v -> v, undef 1370193323Sed if (N1 == N2) { 1371193323Sed N2 = getUNDEF(VT); 1372193323Sed for (unsigned i = 0; i != NElts; ++i) 1373193323Sed if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1374193323Sed } 1375198090Srdivacky 1376193323Sed // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1377193323Sed if (N1.getOpcode() == ISD::UNDEF) 1378193323Sed commuteShuffle(N1, N2, MaskVec); 1379198090Srdivacky 1380193323Sed // Canonicalize all index into lhs, -> shuffle lhs, undef 1381193323Sed // Canonicalize all index into rhs, -> shuffle rhs, undef 1382193323Sed bool AllLHS = true, AllRHS = true; 1383193323Sed bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1384193323Sed for (unsigned i = 0; i != NElts; ++i) { 1385193323Sed if (MaskVec[i] >= (int)NElts) { 1386193323Sed if (N2Undef) 1387193323Sed MaskVec[i] = -1; 1388193323Sed else 1389193323Sed AllLHS = false; 1390193323Sed } else if (MaskVec[i] >= 0) { 1391193323Sed AllRHS = false; 1392193323Sed } 1393193323Sed } 1394193323Sed if (AllLHS && AllRHS) 1395193323Sed return getUNDEF(VT); 1396193323Sed if (AllLHS && !N2Undef) 1397193323Sed N2 = getUNDEF(VT); 1398193323Sed if (AllRHS) { 1399193323Sed N1 = getUNDEF(VT); 1400193323Sed commuteShuffle(N1, N2, MaskVec); 1401193323Sed } 1402198090Srdivacky 1403263509Sdim // If Identity shuffle return that node. 1404193323Sed bool Identity = true; 1405193323Sed for (unsigned i = 0; i != NElts; ++i) { 1406193323Sed if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1407193323Sed } 1408263509Sdim if (Identity && NElts) 1409193323Sed return N1; 1410193323Sed 1411193323Sed FoldingSetNodeID ID; 1412193323Sed SDValue Ops[2] = { N1, N2 }; 1413193323Sed AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1414193323Sed for (unsigned i = 0; i != NElts; ++i) 1415193323Sed ID.AddInteger(MaskVec[i]); 1416198090Srdivacky 1417193323Sed void* IP = 0; 1418201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1419193323Sed return SDValue(E, 0); 1420198090Srdivacky 1421193323Sed // Allocate the mask array for the node out of the BumpPtrAllocator, since 1422193323Sed // SDNode doesn't have access to it. This memory will be "leaked" when 1423193323Sed // the node is deallocated, but recovered when the NodeAllocator is released. 1424193323Sed int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1425193323Sed memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1426198090Srdivacky 1427205407Srdivacky ShuffleVectorSDNode *N = 1428263509Sdim new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(), 1429263509Sdim dl.getDebugLoc(), N1, N2, 1430263509Sdim MaskAlloc); 1431193323Sed CSEMap.InsertNode(N, IP); 1432193323Sed AllNodes.push_back(N); 1433193323Sed return SDValue(N, 0); 1434193323Sed} 1435193323Sed 1436263509SdimSDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, 1437193323Sed SDValue Val, SDValue DTy, 1438193323Sed SDValue STy, SDValue Rnd, SDValue Sat, 1439193323Sed ISD::CvtCode Code) { 1440193323Sed // If the src and dest types are the same and the conversion is between 1441193323Sed // integer types of the same sign or two floats, no conversion is necessary. 1442193323Sed if (DTy == STy && 1443193323Sed (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1444193323Sed return Val; 1445193323Sed 1446193323Sed FoldingSetNodeID ID; 1447199481Srdivacky SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1448199481Srdivacky AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1449193323Sed void* IP = 0; 1450201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1451193323Sed return SDValue(E, 0); 1452201360Srdivacky 1453263509Sdim CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(), 1454263509Sdim dl.getDebugLoc(), 1455263509Sdim Ops, 5, Code); 1456193323Sed CSEMap.InsertNode(N, IP); 1457193323Sed AllNodes.push_back(N); 1458193323Sed return SDValue(N, 0); 1459193323Sed} 1460193323Sed 1461198090SrdivackySDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1462193323Sed FoldingSetNodeID ID; 1463193323Sed AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1464193323Sed ID.AddInteger(RegNo); 1465193323Sed void *IP = 0; 1466201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1467193323Sed return SDValue(E, 0); 1468201360Srdivacky 1469205407Srdivacky SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1470193323Sed CSEMap.InsertNode(N, IP); 1471193323Sed AllNodes.push_back(N); 1472193323Sed return SDValue(N, 0); 1473193323Sed} 1474193323Sed 1475235633SdimSDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1476235633Sdim FoldingSetNodeID ID; 1477235633Sdim AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 1478235633Sdim ID.AddPointer(RegMask); 1479235633Sdim void *IP = 0; 1480235633Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1481235633Sdim return SDValue(E, 0); 1482235633Sdim 1483235633Sdim SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 1484235633Sdim CSEMap.InsertNode(N, IP); 1485235633Sdim AllNodes.push_back(N); 1486235633Sdim return SDValue(N, 0); 1487235633Sdim} 1488235633Sdim 1489263509SdimSDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { 1490193323Sed FoldingSetNodeID ID; 1491193323Sed SDValue Ops[] = { Root }; 1492205218Srdivacky AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1493205218Srdivacky ID.AddPointer(Label); 1494193323Sed void *IP = 0; 1495201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1496193323Sed return SDValue(E, 0); 1497218893Sdim 1498263509Sdim SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(), 1499263509Sdim dl.getDebugLoc(), Root, Label); 1500193323Sed CSEMap.InsertNode(N, IP); 1501193323Sed AllNodes.push_back(N); 1502193323Sed return SDValue(N, 0); 1503193323Sed} 1504193323Sed 1505205218Srdivacky 1506207618SrdivackySDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1507245431Sdim int64_t Offset, 1508199989Srdivacky bool isTarget, 1509199989Srdivacky unsigned char TargetFlags) { 1510198892Srdivacky unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1511198892Srdivacky 1512198892Srdivacky FoldingSetNodeID ID; 1513199989Srdivacky AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1514198892Srdivacky ID.AddPointer(BA); 1515245431Sdim ID.AddInteger(Offset); 1516199989Srdivacky ID.AddInteger(TargetFlags); 1517198892Srdivacky void *IP = 0; 1518201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1519198892Srdivacky return SDValue(E, 0); 1520201360Srdivacky 1521245431Sdim SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset, 1522245431Sdim TargetFlags); 1523198892Srdivacky CSEMap.InsertNode(N, IP); 1524198892Srdivacky AllNodes.push_back(N); 1525198892Srdivacky return SDValue(N, 0); 1526198892Srdivacky} 1527198892Srdivacky 1528193323SedSDValue SelectionDAG::getSrcValue(const Value *V) { 1529204642Srdivacky assert((!V || V->getType()->isPointerTy()) && 1530193323Sed "SrcValue is not a pointer?"); 1531193323Sed 1532193323Sed FoldingSetNodeID ID; 1533193323Sed AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1534193323Sed ID.AddPointer(V); 1535193323Sed 1536193323Sed void *IP = 0; 1537201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1538193323Sed return SDValue(E, 0); 1539193323Sed 1540205407Srdivacky SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1541193323Sed CSEMap.InsertNode(N, IP); 1542193323Sed AllNodes.push_back(N); 1543193323Sed return SDValue(N, 0); 1544193323Sed} 1545193323Sed 1546207618Srdivacky/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1547207618SrdivackySDValue SelectionDAG::getMDNode(const MDNode *MD) { 1548207618Srdivacky FoldingSetNodeID ID; 1549207618Srdivacky AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1550207618Srdivacky ID.AddPointer(MD); 1551218893Sdim 1552207618Srdivacky void *IP = 0; 1553207618Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1554207618Srdivacky return SDValue(E, 0); 1555218893Sdim 1556207618Srdivacky SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1557207618Srdivacky CSEMap.InsertNode(N, IP); 1558207618Srdivacky AllNodes.push_back(N); 1559207618Srdivacky return SDValue(N, 0); 1560207618Srdivacky} 1561207618Srdivacky 1562263509Sdim/// getAddrSpaceCast - Return an AddrSpaceCastSDNode. 1563263509SdimSDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, 1564263509Sdim unsigned SrcAS, unsigned DestAS) { 1565263509Sdim SDValue Ops[] = {Ptr}; 1566263509Sdim FoldingSetNodeID ID; 1567263509Sdim AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), &Ops[0], 1); 1568263509Sdim ID.AddInteger(SrcAS); 1569263509Sdim ID.AddInteger(DestAS); 1570207618Srdivacky 1571263509Sdim void *IP = 0; 1572263509Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1573263509Sdim return SDValue(E, 0); 1574263509Sdim 1575263509Sdim SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(), 1576263509Sdim dl.getDebugLoc(), 1577263509Sdim VT, Ptr, SrcAS, DestAS); 1578263509Sdim CSEMap.InsertNode(N, IP); 1579263509Sdim AllNodes.push_back(N); 1580263509Sdim return SDValue(N, 0); 1581263509Sdim} 1582263509Sdim 1583193323Sed/// getShiftAmountOperand - Return the specified value casted to 1584193323Sed/// the target's desired shift amount type. 1585221345SdimSDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1586198090Srdivacky EVT OpTy = Op.getValueType(); 1587263509Sdim EVT ShTy = TM.getTargetLowering()->getShiftAmountTy(LHSTy); 1588193323Sed if (OpTy == ShTy || OpTy.isVector()) return Op; 1589193323Sed 1590193323Sed ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1591263509Sdim return getNode(Opcode, SDLoc(Op), ShTy, Op); 1592193323Sed} 1593193323Sed 1594193323Sed/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1595193323Sed/// specified value type. 1596198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1597193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1598198090Srdivacky unsigned ByteSize = VT.getStoreSize(); 1599226890Sdim Type *Ty = VT.getTypeForEVT(*getContext()); 1600263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1601193323Sed unsigned StackAlign = 1602263509Sdim std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign); 1603193323Sed 1604199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1605263509Sdim return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1606193323Sed} 1607193323Sed 1608193323Sed/// CreateStackTemporary - Create a stack temporary suitable for holding 1609193323Sed/// either of the specified value types. 1610198090SrdivackySDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1611193323Sed unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1612193323Sed VT2.getStoreSizeInBits())/8; 1613226890Sdim Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1614226890Sdim Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1615263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1616263509Sdim const DataLayout *TD = TLI->getDataLayout(); 1617193323Sed unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1618193323Sed TD->getPrefTypeAlignment(Ty2)); 1619193323Sed 1620193323Sed MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1621199481Srdivacky int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1622263509Sdim return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1623193323Sed} 1624193323Sed 1625198090SrdivackySDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1626263509Sdim SDValue N2, ISD::CondCode Cond, SDLoc dl) { 1627193323Sed // These setcc operations always fold. 1628193323Sed switch (Cond) { 1629193323Sed default: break; 1630193323Sed case ISD::SETFALSE: 1631193323Sed case ISD::SETFALSE2: return getConstant(0, VT); 1632193323Sed case ISD::SETTRUE: 1633263509Sdim case ISD::SETTRUE2: { 1634263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1635263509Sdim TargetLowering::BooleanContent Cnt = TLI->getBooleanContents(VT.isVector()); 1636263509Sdim return getConstant( 1637263509Sdim Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT); 1638263509Sdim } 1639193323Sed 1640193323Sed case ISD::SETOEQ: 1641193323Sed case ISD::SETOGT: 1642193323Sed case ISD::SETOGE: 1643193323Sed case ISD::SETOLT: 1644193323Sed case ISD::SETOLE: 1645193323Sed case ISD::SETONE: 1646193323Sed case ISD::SETO: 1647193323Sed case ISD::SETUO: 1648193323Sed case ISD::SETUEQ: 1649193323Sed case ISD::SETUNE: 1650193323Sed assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1651193323Sed break; 1652193323Sed } 1653193323Sed 1654193323Sed if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1655193323Sed const APInt &C2 = N2C->getAPIntValue(); 1656193323Sed if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1657193323Sed const APInt &C1 = N1C->getAPIntValue(); 1658193323Sed 1659193323Sed switch (Cond) { 1660198090Srdivacky default: llvm_unreachable("Unknown integer setcc!"); 1661193323Sed case ISD::SETEQ: return getConstant(C1 == C2, VT); 1662193323Sed case ISD::SETNE: return getConstant(C1 != C2, VT); 1663193323Sed case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1664193323Sed case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1665193323Sed case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1666193323Sed case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1667193323Sed case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1668193323Sed case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1669193323Sed case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1670193323Sed case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1671193323Sed } 1672193323Sed } 1673193323Sed } 1674193323Sed if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1675193323Sed if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1676193323Sed APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1677193323Sed switch (Cond) { 1678193323Sed default: break; 1679193323Sed case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1680193323Sed return getUNDEF(VT); 1681193323Sed // fall through 1682193323Sed case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1683193323Sed case ISD::SETNE: if (R==APFloat::cmpUnordered) 1684193323Sed return getUNDEF(VT); 1685193323Sed // fall through 1686193323Sed case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1687193323Sed R==APFloat::cmpLessThan, VT); 1688193323Sed case ISD::SETLT: if (R==APFloat::cmpUnordered) 1689193323Sed return getUNDEF(VT); 1690193323Sed // fall through 1691193323Sed case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1692193323Sed case ISD::SETGT: if (R==APFloat::cmpUnordered) 1693193323Sed return getUNDEF(VT); 1694193323Sed // fall through 1695193323Sed case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1696193323Sed case ISD::SETLE: if (R==APFloat::cmpUnordered) 1697193323Sed return getUNDEF(VT); 1698193323Sed // fall through 1699193323Sed case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1700193323Sed R==APFloat::cmpEqual, VT); 1701193323Sed case ISD::SETGE: if (R==APFloat::cmpUnordered) 1702193323Sed return getUNDEF(VT); 1703193323Sed // fall through 1704193323Sed case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1705193323Sed R==APFloat::cmpEqual, VT); 1706193323Sed case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1707193323Sed case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1708193323Sed case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1709193323Sed R==APFloat::cmpEqual, VT); 1710193323Sed case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1711193323Sed case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1712193323Sed R==APFloat::cmpLessThan, VT); 1713193323Sed case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1714193323Sed R==APFloat::cmpUnordered, VT); 1715193323Sed case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1716193323Sed case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1717193323Sed } 1718193323Sed } else { 1719193323Sed // Ensure that the constant occurs on the RHS. 1720263509Sdim ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 1721263509Sdim MVT CompVT = N1.getValueType().getSimpleVT(); 1722263509Sdim if (!TM.getTargetLowering()->isCondCodeLegal(SwappedCond, CompVT)) 1723263509Sdim return SDValue(); 1724263509Sdim 1725263509Sdim return getSetCC(dl, VT, N2, N1, SwappedCond); 1726193323Sed } 1727193323Sed } 1728193323Sed 1729193323Sed // Could not fold it. 1730193323Sed return SDValue(); 1731193323Sed} 1732193323Sed 1733193323Sed/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1734193323Sed/// use this predicate to simplify operations downstream. 1735193323Sedbool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1736198090Srdivacky // This predicate is not safe for vector operations. 1737198090Srdivacky if (Op.getValueType().isVector()) 1738198090Srdivacky return false; 1739198090Srdivacky 1740200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1741193323Sed return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1742193323Sed} 1743193323Sed 1744193323Sed/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1745193323Sed/// this predicate to simplify operations downstream. Mask is known to be zero 1746193323Sed/// for bits that V cannot have. 1747193323Sedbool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1748193323Sed unsigned Depth) const { 1749193323Sed APInt KnownZero, KnownOne; 1750235633Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 1751193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1752193323Sed return (KnownZero & Mask) == Mask; 1753193323Sed} 1754193323Sed 1755193323Sed/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1756193323Sed/// known to be either zero or one and return them in the KnownZero/KnownOne 1757193323Sed/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1758193323Sed/// processing. 1759235633Sdimvoid SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, 1760235633Sdim APInt &KnownOne, unsigned Depth) const { 1761263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 1762235633Sdim unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1763193323Sed 1764193323Sed KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1765235633Sdim if (Depth == 6) 1766193323Sed return; // Limit search depth. 1767193323Sed 1768193323Sed APInt KnownZero2, KnownOne2; 1769193323Sed 1770193323Sed switch (Op.getOpcode()) { 1771193323Sed case ISD::Constant: 1772193323Sed // We know all of the bits for a constant! 1773235633Sdim KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue(); 1774235633Sdim KnownZero = ~KnownOne; 1775193323Sed return; 1776193323Sed case ISD::AND: 1777193323Sed // If either the LHS or the RHS are Zero, the result is zero. 1778235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1779235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1780193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1781193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1782193323Sed 1783193323Sed // Output known-1 bits are only known if set in both the LHS & RHS. 1784193323Sed KnownOne &= KnownOne2; 1785193323Sed // Output known-0 are known to be clear if zero in either the LHS | RHS. 1786193323Sed KnownZero |= KnownZero2; 1787193323Sed return; 1788193323Sed case ISD::OR: 1789235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1790235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1791193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1792193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1793193323Sed 1794193323Sed // Output known-0 bits are only known if clear in both the LHS & RHS. 1795193323Sed KnownZero &= KnownZero2; 1796193323Sed // Output known-1 are known to be set if set in either the LHS | RHS. 1797193323Sed KnownOne |= KnownOne2; 1798193323Sed return; 1799193323Sed case ISD::XOR: { 1800235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1801235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1802193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1803193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1804193323Sed 1805193323Sed // Output known-0 bits are known if clear or set in both the LHS & RHS. 1806193323Sed APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1807193323Sed // Output known-1 are known to be set if set in only one of the LHS, RHS. 1808193323Sed KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1809193323Sed KnownZero = KnownZeroOut; 1810193323Sed return; 1811193323Sed } 1812193323Sed case ISD::MUL: { 1813235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1814235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1815193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1816193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1817193323Sed 1818193323Sed // If low bits are zero in either operand, output low known-0 bits. 1819193323Sed // Also compute a conserative estimate for high known-0 bits. 1820193323Sed // More trickiness is possible, but this is sufficient for the 1821193323Sed // interesting case of alignment computation. 1822218893Sdim KnownOne.clearAllBits(); 1823193323Sed unsigned TrailZ = KnownZero.countTrailingOnes() + 1824193323Sed KnownZero2.countTrailingOnes(); 1825193323Sed unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1826193323Sed KnownZero2.countLeadingOnes(), 1827193323Sed BitWidth) - BitWidth; 1828193323Sed 1829193323Sed TrailZ = std::min(TrailZ, BitWidth); 1830193323Sed LeadZ = std::min(LeadZ, BitWidth); 1831193323Sed KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1832193323Sed APInt::getHighBitsSet(BitWidth, LeadZ); 1833193323Sed return; 1834193323Sed } 1835193323Sed case ISD::UDIV: { 1836193323Sed // For the purposes of computing leading zeros we can conservatively 1837193323Sed // treat a udiv as a logical right shift by the power of 2 known to 1838193323Sed // be less than the denominator. 1839235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1840193323Sed unsigned LeadZ = KnownZero2.countLeadingOnes(); 1841193323Sed 1842218893Sdim KnownOne2.clearAllBits(); 1843218893Sdim KnownZero2.clearAllBits(); 1844235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1845193323Sed unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1846193323Sed if (RHSUnknownLeadingOnes != BitWidth) 1847193323Sed LeadZ = std::min(BitWidth, 1848193323Sed LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1849193323Sed 1850235633Sdim KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ); 1851193323Sed return; 1852193323Sed } 1853193323Sed case ISD::SELECT: 1854235633Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1); 1855235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1856193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1857193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1858193323Sed 1859193323Sed // Only known if known in both the LHS and RHS. 1860193323Sed KnownOne &= KnownOne2; 1861193323Sed KnownZero &= KnownZero2; 1862193323Sed return; 1863193323Sed case ISD::SELECT_CC: 1864235633Sdim ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1); 1865235633Sdim ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1); 1866193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1867193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1868193323Sed 1869193323Sed // Only known if known in both the LHS and RHS. 1870193323Sed KnownOne &= KnownOne2; 1871193323Sed KnownZero &= KnownZero2; 1872193323Sed return; 1873193323Sed case ISD::SADDO: 1874193323Sed case ISD::UADDO: 1875193323Sed case ISD::SSUBO: 1876193323Sed case ISD::USUBO: 1877193323Sed case ISD::SMULO: 1878193323Sed case ISD::UMULO: 1879193323Sed if (Op.getResNo() != 1) 1880193323Sed return; 1881193323Sed // The boolean result conforms to getBooleanContents. Fall through. 1882193323Sed case ISD::SETCC: 1883193323Sed // If we know the result of a setcc has the top bits zero, use this info. 1884263509Sdim if (TLI->getBooleanContents(Op.getValueType().isVector()) == 1885226890Sdim TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1886193323Sed KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1887193323Sed return; 1888193323Sed case ISD::SHL: 1889193323Sed // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1890193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1891193323Sed unsigned ShAmt = SA->getZExtValue(); 1892193323Sed 1893193323Sed // If the shift count is an invalid immediate, don't do anything. 1894193323Sed if (ShAmt >= BitWidth) 1895193323Sed return; 1896193323Sed 1897235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1898193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1899193323Sed KnownZero <<= ShAmt; 1900193323Sed KnownOne <<= ShAmt; 1901193323Sed // low bits known zero. 1902193323Sed KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1903193323Sed } 1904193323Sed return; 1905193323Sed case ISD::SRL: 1906193323Sed // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1907193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1908193323Sed unsigned ShAmt = SA->getZExtValue(); 1909193323Sed 1910193323Sed // If the shift count is an invalid immediate, don't do anything. 1911193323Sed if (ShAmt >= BitWidth) 1912193323Sed return; 1913193323Sed 1914235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1915193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1916193323Sed KnownZero = KnownZero.lshr(ShAmt); 1917193323Sed KnownOne = KnownOne.lshr(ShAmt); 1918193323Sed 1919235633Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1920193323Sed KnownZero |= HighBits; // High bits known zero. 1921193323Sed } 1922193323Sed return; 1923193323Sed case ISD::SRA: 1924193323Sed if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1925193323Sed unsigned ShAmt = SA->getZExtValue(); 1926193323Sed 1927193323Sed // If the shift count is an invalid immediate, don't do anything. 1928193323Sed if (ShAmt >= BitWidth) 1929193323Sed return; 1930193323Sed 1931193323Sed // If any of the demanded bits are produced by the sign extension, we also 1932193323Sed // demand the input sign bit. 1933235633Sdim APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1934193323Sed 1935235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1936193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1937193323Sed KnownZero = KnownZero.lshr(ShAmt); 1938193323Sed KnownOne = KnownOne.lshr(ShAmt); 1939193323Sed 1940193323Sed // Handle the sign bits. 1941193323Sed APInt SignBit = APInt::getSignBit(BitWidth); 1942193323Sed SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1943193323Sed 1944193323Sed if (KnownZero.intersects(SignBit)) { 1945193323Sed KnownZero |= HighBits; // New bits are known zero. 1946193323Sed } else if (KnownOne.intersects(SignBit)) { 1947193323Sed KnownOne |= HighBits; // New bits are known one. 1948193323Sed } 1949193323Sed } 1950193323Sed return; 1951193323Sed case ISD::SIGN_EXTEND_INREG: { 1952198090Srdivacky EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1953202375Srdivacky unsigned EBits = EVT.getScalarType().getSizeInBits(); 1954193323Sed 1955193323Sed // Sign extension. Compute the demanded bits in the result that are not 1956193323Sed // present in the input. 1957235633Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 1958193323Sed 1959193323Sed APInt InSignBit = APInt::getSignBit(EBits); 1960235633Sdim APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 1961193323Sed 1962193323Sed // If the sign extended bits are demanded, we know that the sign 1963193323Sed // bit is demanded. 1964218893Sdim InSignBit = InSignBit.zext(BitWidth); 1965193323Sed if (NewBits.getBoolValue()) 1966193323Sed InputDemandedBits |= InSignBit; 1967193323Sed 1968235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1969235633Sdim KnownOne &= InputDemandedBits; 1970235633Sdim KnownZero &= InputDemandedBits; 1971193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1972193323Sed 1973193323Sed // If the sign bit of the input is known set or clear, then we know the 1974193323Sed // top bits of the result. 1975193323Sed if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1976193323Sed KnownZero |= NewBits; 1977193323Sed KnownOne &= ~NewBits; 1978193323Sed } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1979193323Sed KnownOne |= NewBits; 1980193323Sed KnownZero &= ~NewBits; 1981193323Sed } else { // Input sign bit unknown 1982193323Sed KnownZero &= ~NewBits; 1983193323Sed KnownOne &= ~NewBits; 1984193323Sed } 1985193323Sed return; 1986193323Sed } 1987193323Sed case ISD::CTTZ: 1988235633Sdim case ISD::CTTZ_ZERO_UNDEF: 1989193323Sed case ISD::CTLZ: 1990235633Sdim case ISD::CTLZ_ZERO_UNDEF: 1991193323Sed case ISD::CTPOP: { 1992193323Sed unsigned LowBits = Log2_32(BitWidth)+1; 1993193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 1994218893Sdim KnownOne.clearAllBits(); 1995193323Sed return; 1996193323Sed } 1997193323Sed case ISD::LOAD: { 1998235633Sdim LoadSDNode *LD = cast<LoadSDNode>(Op); 1999252723Sdim // If this is a ZEXTLoad and we are looking at the loaded value. 2000252723Sdim if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 2001198090Srdivacky EVT VT = LD->getMemoryVT(); 2002202375Srdivacky unsigned MemBits = VT.getScalarType().getSizeInBits(); 2003235633Sdim KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 2004235633Sdim } else if (const MDNode *Ranges = LD->getRanges()) { 2005235633Sdim computeMaskedBitsLoad(*Ranges, KnownZero); 2006193323Sed } 2007193323Sed return; 2008193323Sed } 2009193323Sed case ISD::ZERO_EXTEND: { 2010198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2011200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2012235633Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2013218893Sdim KnownZero = KnownZero.trunc(InBits); 2014218893Sdim KnownOne = KnownOne.trunc(InBits); 2015235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2016218893Sdim KnownZero = KnownZero.zext(BitWidth); 2017218893Sdim KnownOne = KnownOne.zext(BitWidth); 2018193323Sed KnownZero |= NewBits; 2019193323Sed return; 2020193323Sed } 2021193323Sed case ISD::SIGN_EXTEND: { 2022198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2023200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2024235633Sdim APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 2025193323Sed 2026218893Sdim KnownZero = KnownZero.trunc(InBits); 2027218893Sdim KnownOne = KnownOne.trunc(InBits); 2028235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2029193323Sed 2030193323Sed // Note if the sign bit is known to be zero or one. 2031193323Sed bool SignBitKnownZero = KnownZero.isNegative(); 2032193323Sed bool SignBitKnownOne = KnownOne.isNegative(); 2033193323Sed assert(!(SignBitKnownZero && SignBitKnownOne) && 2034193323Sed "Sign bit can't be known to be both zero and one!"); 2035193323Sed 2036218893Sdim KnownZero = KnownZero.zext(BitWidth); 2037218893Sdim KnownOne = KnownOne.zext(BitWidth); 2038193323Sed 2039193323Sed // If the sign bit is known zero or one, the top bits match. 2040193323Sed if (SignBitKnownZero) 2041193323Sed KnownZero |= NewBits; 2042193323Sed else if (SignBitKnownOne) 2043193323Sed KnownOne |= NewBits; 2044193323Sed return; 2045193323Sed } 2046193323Sed case ISD::ANY_EXTEND: { 2047198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2048200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2049218893Sdim KnownZero = KnownZero.trunc(InBits); 2050218893Sdim KnownOne = KnownOne.trunc(InBits); 2051235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2052218893Sdim KnownZero = KnownZero.zext(BitWidth); 2053218893Sdim KnownOne = KnownOne.zext(BitWidth); 2054193323Sed return; 2055193323Sed } 2056193323Sed case ISD::TRUNCATE: { 2057198090Srdivacky EVT InVT = Op.getOperand(0).getValueType(); 2058200581Srdivacky unsigned InBits = InVT.getScalarType().getSizeInBits(); 2059218893Sdim KnownZero = KnownZero.zext(InBits); 2060218893Sdim KnownOne = KnownOne.zext(InBits); 2061235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2062193323Sed assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 2063218893Sdim KnownZero = KnownZero.trunc(BitWidth); 2064218893Sdim KnownOne = KnownOne.trunc(BitWidth); 2065193323Sed break; 2066193323Sed } 2067193323Sed case ISD::AssertZext: { 2068198090Srdivacky EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2069193323Sed APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 2070235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2071235633Sdim KnownZero |= (~InMask); 2072245431Sdim KnownOne &= (~KnownZero); 2073193323Sed return; 2074193323Sed } 2075193323Sed case ISD::FGETSIGN: 2076193323Sed // All bits are zero except the low bit. 2077193323Sed KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 2078193323Sed return; 2079193323Sed 2080193323Sed case ISD::SUB: { 2081193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 2082193323Sed // We know that the top bits of C-X are clear if X contains less bits 2083193323Sed // than C (i.e. no wrap-around can happen). For example, 20-X is 2084193323Sed // positive if we can prove that X is >= 0 and < 16. 2085193323Sed if (CLHS->getAPIntValue().isNonNegative()) { 2086193323Sed unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 2087193323Sed // NLZ can't be BitWidth with no sign bit 2088193323Sed APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 2089235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2090193323Sed 2091193323Sed // If all of the MaskV bits are known to be zero, then we know the 2092193323Sed // output top bits are zero, because we now know that the output is 2093193323Sed // from [0-C]. 2094193323Sed if ((KnownZero2 & MaskV) == MaskV) { 2095193323Sed unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 2096193323Sed // Top bits known zero. 2097235633Sdim KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2); 2098193323Sed } 2099193323Sed } 2100193323Sed } 2101193323Sed } 2102193323Sed // fall through 2103218893Sdim case ISD::ADD: 2104218893Sdim case ISD::ADDE: { 2105193323Sed // Output known-0 bits are known if clear or set in both the low clear bits 2106193323Sed // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2107193323Sed // low 3 bits clear. 2108235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 2109193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2110193323Sed unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 2111193323Sed 2112235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2113193323Sed assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2114193323Sed KnownZeroOut = std::min(KnownZeroOut, 2115193323Sed KnownZero2.countTrailingOnes()); 2116193323Sed 2117218893Sdim if (Op.getOpcode() == ISD::ADD) { 2118218893Sdim KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2119218893Sdim return; 2120218893Sdim } 2121218893Sdim 2122218893Sdim // With ADDE, a carry bit may be added in, so we can only use this 2123218893Sdim // information if we know (at least) that the low two bits are clear. We 2124218893Sdim // then return to the caller that the low bit is unknown but that other bits 2125218893Sdim // are known zero. 2126218893Sdim if (KnownZeroOut >= 2) // ADDE 2127218893Sdim KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2128193323Sed return; 2129193323Sed } 2130193323Sed case ISD::SREM: 2131193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2132203954Srdivacky const APInt &RA = Rem->getAPIntValue().abs(); 2133203954Srdivacky if (RA.isPowerOf2()) { 2134203954Srdivacky APInt LowBits = RA - 1; 2135235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1); 2136193323Sed 2137203954Srdivacky // The low bits of the first operand are unchanged by the srem. 2138203954Srdivacky KnownZero = KnownZero2 & LowBits; 2139203954Srdivacky KnownOne = KnownOne2 & LowBits; 2140203954Srdivacky 2141203954Srdivacky // If the first operand is non-negative or has all low bits zero, then 2142203954Srdivacky // the upper bits are all zero. 2143193323Sed if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2144203954Srdivacky KnownZero |= ~LowBits; 2145193323Sed 2146203954Srdivacky // If the first operand is negative and not all low bits are zero, then 2147203954Srdivacky // the upper bits are all one. 2148203954Srdivacky if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2149203954Srdivacky KnownOne |= ~LowBits; 2150193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2151193323Sed } 2152193323Sed } 2153193323Sed return; 2154193323Sed case ISD::UREM: { 2155193323Sed if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2156193323Sed const APInt &RA = Rem->getAPIntValue(); 2157193323Sed if (RA.isPowerOf2()) { 2158193323Sed APInt LowBits = (RA - 1); 2159235633Sdim KnownZero |= ~LowBits; 2160235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1); 2161193323Sed assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2162193323Sed break; 2163193323Sed } 2164193323Sed } 2165193323Sed 2166193323Sed // Since the result is less than or equal to either operand, any leading 2167193323Sed // zero bits in either operand must also exist in the result. 2168235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2169235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2170193323Sed 2171193323Sed uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2172193323Sed KnownZero2.countLeadingOnes()); 2173218893Sdim KnownOne.clearAllBits(); 2174235633Sdim KnownZero = APInt::getHighBitsSet(BitWidth, Leaders); 2175193323Sed return; 2176193323Sed } 2177218893Sdim case ISD::FrameIndex: 2178218893Sdim case ISD::TargetFrameIndex: 2179218893Sdim if (unsigned Align = InferPtrAlignment(Op)) { 2180218893Sdim // The low bits are known zero if the pointer is aligned. 2181218893Sdim KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2182218893Sdim return; 2183218893Sdim } 2184218893Sdim break; 2185219077Sdim 2186193323Sed default: 2187223017Sdim if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2188223017Sdim break; 2189223017Sdim // Fallthrough 2190193323Sed case ISD::INTRINSIC_WO_CHAIN: 2191193323Sed case ISD::INTRINSIC_W_CHAIN: 2192193323Sed case ISD::INTRINSIC_VOID: 2193223017Sdim // Allow the target to implement this method for its nodes. 2194263509Sdim TLI->computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth); 2195193323Sed return; 2196193323Sed } 2197193323Sed} 2198193323Sed 2199193323Sed/// ComputeNumSignBits - Return the number of times the sign bit of the 2200193323Sed/// register is replicated into the other bits. We know that at least 1 bit 2201193323Sed/// is always equal to the sign bit (itself), but other cases can give us 2202193323Sed/// information. For example, immediately after an "SRA X, 2", we know that 2203193323Sed/// the top 3 bits are all equal to each other, so we return 3. 2204193323Sedunsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2205263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 2206198090Srdivacky EVT VT = Op.getValueType(); 2207193323Sed assert(VT.isInteger() && "Invalid VT!"); 2208200581Srdivacky unsigned VTBits = VT.getScalarType().getSizeInBits(); 2209193323Sed unsigned Tmp, Tmp2; 2210193323Sed unsigned FirstAnswer = 1; 2211193323Sed 2212193323Sed if (Depth == 6) 2213193323Sed return 1; // Limit search depth. 2214193323Sed 2215193323Sed switch (Op.getOpcode()) { 2216193323Sed default: break; 2217193323Sed case ISD::AssertSext: 2218193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2219193323Sed return VTBits-Tmp+1; 2220193323Sed case ISD::AssertZext: 2221193323Sed Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2222193323Sed return VTBits-Tmp; 2223193323Sed 2224193323Sed case ISD::Constant: { 2225193323Sed const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2226219077Sdim return Val.getNumSignBits(); 2227193323Sed } 2228193323Sed 2229193323Sed case ISD::SIGN_EXTEND: 2230263509Sdim Tmp = 2231263509Sdim VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2232193323Sed return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2233193323Sed 2234193323Sed case ISD::SIGN_EXTEND_INREG: 2235193323Sed // Max of the input and what this extends. 2236202375Srdivacky Tmp = 2237202375Srdivacky cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2238193323Sed Tmp = VTBits-Tmp+1; 2239193323Sed 2240193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2241193323Sed return std::max(Tmp, Tmp2); 2242193323Sed 2243193323Sed case ISD::SRA: 2244193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2245193323Sed // SRA X, C -> adds C sign bits. 2246193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2247193323Sed Tmp += C->getZExtValue(); 2248193323Sed if (Tmp > VTBits) Tmp = VTBits; 2249193323Sed } 2250193323Sed return Tmp; 2251193323Sed case ISD::SHL: 2252193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2253193323Sed // shl destroys sign bits. 2254193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2255193323Sed if (C->getZExtValue() >= VTBits || // Bad shift. 2256193323Sed C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2257193323Sed return Tmp - C->getZExtValue(); 2258193323Sed } 2259193323Sed break; 2260193323Sed case ISD::AND: 2261193323Sed case ISD::OR: 2262193323Sed case ISD::XOR: // NOT is handled here. 2263193323Sed // Logical binary ops preserve the number of sign bits at the worst. 2264193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2265193323Sed if (Tmp != 1) { 2266193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2267193323Sed FirstAnswer = std::min(Tmp, Tmp2); 2268193323Sed // We computed what we know about the sign bits as our first 2269193323Sed // answer. Now proceed to the generic code that uses 2270193323Sed // ComputeMaskedBits, and pick whichever answer is better. 2271193323Sed } 2272193323Sed break; 2273193323Sed 2274193323Sed case ISD::SELECT: 2275193323Sed Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2276193323Sed if (Tmp == 1) return 1; // Early out. 2277193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2278193323Sed return std::min(Tmp, Tmp2); 2279193323Sed 2280193323Sed case ISD::SADDO: 2281193323Sed case ISD::UADDO: 2282193323Sed case ISD::SSUBO: 2283193323Sed case ISD::USUBO: 2284193323Sed case ISD::SMULO: 2285193323Sed case ISD::UMULO: 2286193323Sed if (Op.getResNo() != 1) 2287193323Sed break; 2288193323Sed // The boolean result conforms to getBooleanContents. Fall through. 2289193323Sed case ISD::SETCC: 2290193323Sed // If setcc returns 0/-1, all bits are sign bits. 2291263509Sdim if (TLI->getBooleanContents(Op.getValueType().isVector()) == 2292193323Sed TargetLowering::ZeroOrNegativeOneBooleanContent) 2293193323Sed return VTBits; 2294193323Sed break; 2295193323Sed case ISD::ROTL: 2296193323Sed case ISD::ROTR: 2297193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2298193323Sed unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2299193323Sed 2300193323Sed // Handle rotate right by N like a rotate left by 32-N. 2301193323Sed if (Op.getOpcode() == ISD::ROTR) 2302193323Sed RotAmt = (VTBits-RotAmt) & (VTBits-1); 2303193323Sed 2304193323Sed // If we aren't rotating out all of the known-in sign bits, return the 2305193323Sed // number that are left. This handles rotl(sext(x), 1) for example. 2306193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2307193323Sed if (Tmp > RotAmt+1) return Tmp-RotAmt; 2308193323Sed } 2309193323Sed break; 2310193323Sed case ISD::ADD: 2311193323Sed // Add can have at most one carry bit. Thus we know that the output 2312193323Sed // is, at worst, one more bit than the inputs. 2313193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2314193323Sed if (Tmp == 1) return 1; // Early out. 2315193323Sed 2316193323Sed // Special case decrementing a value (ADD X, -1): 2317193323Sed if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2318193323Sed if (CRHS->isAllOnesValue()) { 2319193323Sed APInt KnownZero, KnownOne; 2320235633Sdim ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2321193323Sed 2322193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2323193323Sed // sign bits set. 2324235633Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2325193323Sed return VTBits; 2326193323Sed 2327193323Sed // If we are subtracting one from a positive number, there is no carry 2328193323Sed // out of the result. 2329193323Sed if (KnownZero.isNegative()) 2330193323Sed return Tmp; 2331193323Sed } 2332193323Sed 2333193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2334193323Sed if (Tmp2 == 1) return 1; 2335235633Sdim return std::min(Tmp, Tmp2)-1; 2336193323Sed 2337193323Sed case ISD::SUB: 2338193323Sed Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2339193323Sed if (Tmp2 == 1) return 1; 2340193323Sed 2341193323Sed // Handle NEG. 2342193323Sed if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2343193323Sed if (CLHS->isNullValue()) { 2344193323Sed APInt KnownZero, KnownOne; 2345235633Sdim ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 2346193323Sed // If the input is known to be 0 or 1, the output is 0/-1, which is all 2347193323Sed // sign bits set. 2348235633Sdim if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2349193323Sed return VTBits; 2350193323Sed 2351193323Sed // If the input is known to be positive (the sign bit is known clear), 2352193323Sed // the output of the NEG has the same number of sign bits as the input. 2353193323Sed if (KnownZero.isNegative()) 2354193323Sed return Tmp2; 2355193323Sed 2356193323Sed // Otherwise, we treat this like a SUB. 2357193323Sed } 2358193323Sed 2359193323Sed // Sub can have at most one carry bit. Thus we know that the output 2360193323Sed // is, at worst, one more bit than the inputs. 2361193323Sed Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2362193323Sed if (Tmp == 1) return 1; // Early out. 2363235633Sdim return std::min(Tmp, Tmp2)-1; 2364193323Sed case ISD::TRUNCATE: 2365193323Sed // FIXME: it's tricky to do anything useful for this, but it is an important 2366193323Sed // case for targets like X86. 2367193323Sed break; 2368193323Sed } 2369193323Sed 2370252723Sdim // If we are looking at the loaded value of the SDNode. 2371252723Sdim if (Op.getResNo() == 0) { 2372252723Sdim // Handle LOADX separately here. EXTLOAD case will fallthrough. 2373252723Sdim if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 2374252723Sdim unsigned ExtType = LD->getExtensionType(); 2375252723Sdim switch (ExtType) { 2376252723Sdim default: break; 2377252723Sdim case ISD::SEXTLOAD: // '17' bits known 2378252723Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2379252723Sdim return VTBits-Tmp+1; 2380252723Sdim case ISD::ZEXTLOAD: // '16' bits known 2381252723Sdim Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2382252723Sdim return VTBits-Tmp; 2383252723Sdim } 2384193323Sed } 2385193323Sed } 2386193323Sed 2387193323Sed // Allow the target to implement this method for its nodes. 2388193323Sed if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2389193323Sed Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2390193323Sed Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2391193323Sed Op.getOpcode() == ISD::INTRINSIC_VOID) { 2392263509Sdim unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, Depth); 2393193323Sed if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2394193323Sed } 2395193323Sed 2396193323Sed // Finally, if we can prove that the top bits of the result are 0's or 1's, 2397193323Sed // use this information. 2398193323Sed APInt KnownZero, KnownOne; 2399235633Sdim ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 2400193323Sed 2401235633Sdim APInt Mask; 2402193323Sed if (KnownZero.isNegative()) { // sign bit is 0 2403193323Sed Mask = KnownZero; 2404193323Sed } else if (KnownOne.isNegative()) { // sign bit is 1; 2405193323Sed Mask = KnownOne; 2406193323Sed } else { 2407193323Sed // Nothing known. 2408193323Sed return FirstAnswer; 2409193323Sed } 2410193323Sed 2411193323Sed // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2412193323Sed // the number of identical bits in the top of the input value. 2413193323Sed Mask = ~Mask; 2414193323Sed Mask <<= Mask.getBitWidth()-VTBits; 2415193323Sed // Return # leading zeros. We use 'min' here in case Val was zero before 2416193323Sed // shifting. We don't want to return '64' as for an i32 "0". 2417193323Sed return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2418193323Sed} 2419193323Sed 2420218893Sdim/// isBaseWithConstantOffset - Return true if the specified operand is an 2421218893Sdim/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2422218893Sdim/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2423218893Sdim/// semantics as an ADD. This handles the equivalence: 2424218893Sdim/// X|Cst == X+Cst iff X&Cst = 0. 2425218893Sdimbool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2426218893Sdim if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2427218893Sdim !isa<ConstantSDNode>(Op.getOperand(1))) 2428218893Sdim return false; 2429219077Sdim 2430219077Sdim if (Op.getOpcode() == ISD::OR && 2431218893Sdim !MaskedValueIsZero(Op.getOperand(0), 2432218893Sdim cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2433218893Sdim return false; 2434219077Sdim 2435218893Sdim return true; 2436218893Sdim} 2437218893Sdim 2438218893Sdim 2439198090Srdivackybool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2440198090Srdivacky // If we're told that NaNs won't happen, assume they won't. 2441235633Sdim if (getTarget().Options.NoNaNsFPMath) 2442198090Srdivacky return true; 2443193323Sed 2444198090Srdivacky // If the value is a constant, we can obviously see if it is a NaN or not. 2445198090Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2446198090Srdivacky return !C->getValueAPF().isNaN(); 2447198090Srdivacky 2448198090Srdivacky // TODO: Recognize more cases here. 2449198090Srdivacky 2450198090Srdivacky return false; 2451198090Srdivacky} 2452198090Srdivacky 2453204642Srdivackybool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2454204642Srdivacky // If the value is a constant, we can obviously see if it is a zero or not. 2455204642Srdivacky if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2456204642Srdivacky return !C->isZero(); 2457204642Srdivacky 2458204642Srdivacky // TODO: Recognize more cases here. 2459223017Sdim switch (Op.getOpcode()) { 2460223017Sdim default: break; 2461223017Sdim case ISD::OR: 2462223017Sdim if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2463223017Sdim return !C->isNullValue(); 2464223017Sdim break; 2465223017Sdim } 2466204642Srdivacky 2467204642Srdivacky return false; 2468204642Srdivacky} 2469204642Srdivacky 2470204642Srdivackybool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2471204642Srdivacky // Check the obvious case. 2472204642Srdivacky if (A == B) return true; 2473204642Srdivacky 2474204642Srdivacky // For for negative and positive zero. 2475204642Srdivacky if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2476204642Srdivacky if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2477204642Srdivacky if (CA->isZero() && CB->isZero()) return true; 2478204642Srdivacky 2479204642Srdivacky // Otherwise they may not be equal. 2480204642Srdivacky return false; 2481204642Srdivacky} 2482204642Srdivacky 2483193323Sed/// getNode - Gets or creates the specified node. 2484193323Sed/// 2485263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) { 2486193323Sed FoldingSetNodeID ID; 2487193323Sed AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2488193323Sed void *IP = 0; 2489201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2490193323Sed return SDValue(E, 0); 2491201360Srdivacky 2492263509Sdim SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), 2493263509Sdim DL.getDebugLoc(), getVTList(VT)); 2494193323Sed CSEMap.InsertNode(N, IP); 2495193323Sed 2496193323Sed AllNodes.push_back(N); 2497193323Sed#ifndef NDEBUG 2498218893Sdim VerifySDNode(N); 2499193323Sed#endif 2500193323Sed return SDValue(N, 0); 2501193323Sed} 2502193323Sed 2503263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 2504198090Srdivacky EVT VT, SDValue Operand) { 2505193323Sed // Constant fold unary operations with an integer constant operand. 2506193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2507193323Sed const APInt &Val = C->getAPIntValue(); 2508193323Sed switch (Opcode) { 2509193323Sed default: break; 2510193323Sed case ISD::SIGN_EXTEND: 2511218893Sdim return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2512193323Sed case ISD::ANY_EXTEND: 2513193323Sed case ISD::ZERO_EXTEND: 2514193323Sed case ISD::TRUNCATE: 2515218893Sdim return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2516193323Sed case ISD::UINT_TO_FP: 2517193323Sed case ISD::SINT_TO_FP: { 2518252723Sdim APFloat apf(EVTToAPFloatSemantics(VT), 2519252723Sdim APInt::getNullValue(VT.getSizeInBits())); 2520193323Sed (void)apf.convertFromAPInt(Val, 2521193323Sed Opcode==ISD::SINT_TO_FP, 2522193323Sed APFloat::rmNearestTiesToEven); 2523193323Sed return getConstantFP(apf, VT); 2524193323Sed } 2525218893Sdim case ISD::BITCAST: 2526193323Sed if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2527252723Sdim return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); 2528193323Sed else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2529252723Sdim return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); 2530193323Sed break; 2531193323Sed case ISD::BSWAP: 2532193323Sed return getConstant(Val.byteSwap(), VT); 2533193323Sed case ISD::CTPOP: 2534193323Sed return getConstant(Val.countPopulation(), VT); 2535193323Sed case ISD::CTLZ: 2536235633Sdim case ISD::CTLZ_ZERO_UNDEF: 2537193323Sed return getConstant(Val.countLeadingZeros(), VT); 2538193323Sed case ISD::CTTZ: 2539235633Sdim case ISD::CTTZ_ZERO_UNDEF: 2540193323Sed return getConstant(Val.countTrailingZeros(), VT); 2541193323Sed } 2542193323Sed } 2543193323Sed 2544193323Sed // Constant fold unary operations with a floating point constant operand. 2545193323Sed if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2546193323Sed APFloat V = C->getValueAPF(); // make copy 2547245431Sdim switch (Opcode) { 2548245431Sdim case ISD::FNEG: 2549245431Sdim V.changeSign(); 2550245431Sdim return getConstantFP(V, VT); 2551245431Sdim case ISD::FABS: 2552245431Sdim V.clearSign(); 2553245431Sdim return getConstantFP(V, VT); 2554245431Sdim case ISD::FCEIL: { 2555245431Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 2556245431Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2557193323Sed return getConstantFP(V, VT); 2558245431Sdim break; 2559245431Sdim } 2560245431Sdim case ISD::FTRUNC: { 2561245431Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 2562245431Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2563193323Sed return getConstantFP(V, VT); 2564245431Sdim break; 2565245431Sdim } 2566245431Sdim case ISD::FFLOOR: { 2567245431Sdim APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 2568245431Sdim if (fs == APFloat::opOK || fs == APFloat::opInexact) 2569193323Sed return getConstantFP(V, VT); 2570245431Sdim break; 2571245431Sdim } 2572245431Sdim case ISD::FP_EXTEND: { 2573245431Sdim bool ignored; 2574245431Sdim // This can return overflow, underflow, or inexact; we don't care. 2575245431Sdim // FIXME need to be more flexible about rounding mode. 2576252723Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 2577245431Sdim APFloat::rmNearestTiesToEven, &ignored); 2578245431Sdim return getConstantFP(V, VT); 2579245431Sdim } 2580245431Sdim case ISD::FP_TO_SINT: 2581245431Sdim case ISD::FP_TO_UINT: { 2582245431Sdim integerPart x[2]; 2583245431Sdim bool ignored; 2584245431Sdim assert(integerPartWidth >= 64); 2585245431Sdim // FIXME need to be more flexible about rounding mode. 2586245431Sdim APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2587245431Sdim Opcode==ISD::FP_TO_SINT, 2588245431Sdim APFloat::rmTowardZero, &ignored); 2589245431Sdim if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2590193323Sed break; 2591245431Sdim APInt api(VT.getSizeInBits(), x); 2592245431Sdim return getConstant(api, VT); 2593193323Sed } 2594245431Sdim case ISD::BITCAST: 2595245431Sdim if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2596245431Sdim return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2597245431Sdim else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2598245431Sdim return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2599245431Sdim break; 2600245431Sdim } 2601193323Sed } 2602193323Sed 2603193323Sed unsigned OpOpcode = Operand.getNode()->getOpcode(); 2604193323Sed switch (Opcode) { 2605193323Sed case ISD::TokenFactor: 2606193323Sed case ISD::MERGE_VALUES: 2607193323Sed case ISD::CONCAT_VECTORS: 2608193323Sed return Operand; // Factor, merge or concat of one node? No need. 2609198090Srdivacky case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2610193323Sed case ISD::FP_EXTEND: 2611193323Sed assert(VT.isFloatingPoint() && 2612193323Sed Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2613193323Sed if (Operand.getValueType() == VT) return Operand; // noop conversion. 2614200581Srdivacky assert((!VT.isVector() || 2615200581Srdivacky VT.getVectorNumElements() == 2616200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2617200581Srdivacky "Vector element count mismatch!"); 2618193323Sed if (Operand.getOpcode() == ISD::UNDEF) 2619193323Sed return getUNDEF(VT); 2620193323Sed break; 2621193323Sed case ISD::SIGN_EXTEND: 2622193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2623193323Sed "Invalid SIGN_EXTEND!"); 2624193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2625200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2626200581Srdivacky "Invalid sext node, dst < src!"); 2627200581Srdivacky assert((!VT.isVector() || 2628200581Srdivacky VT.getVectorNumElements() == 2629200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2630200581Srdivacky "Vector element count mismatch!"); 2631193323Sed if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2632193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2633221345Sdim else if (OpOpcode == ISD::UNDEF) 2634221345Sdim // sext(undef) = 0, because the top bits will all be the same. 2635221345Sdim return getConstant(0, VT); 2636193323Sed break; 2637193323Sed case ISD::ZERO_EXTEND: 2638193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2639193323Sed "Invalid ZERO_EXTEND!"); 2640193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2641200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2642200581Srdivacky "Invalid zext node, dst < src!"); 2643200581Srdivacky assert((!VT.isVector() || 2644200581Srdivacky VT.getVectorNumElements() == 2645200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2646200581Srdivacky "Vector element count mismatch!"); 2647193323Sed if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2648193323Sed return getNode(ISD::ZERO_EXTEND, DL, VT, 2649193323Sed Operand.getNode()->getOperand(0)); 2650221345Sdim else if (OpOpcode == ISD::UNDEF) 2651221345Sdim // zext(undef) = 0, because the top bits will be zero. 2652221345Sdim return getConstant(0, VT); 2653193323Sed break; 2654193323Sed case ISD::ANY_EXTEND: 2655193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2656193323Sed "Invalid ANY_EXTEND!"); 2657193323Sed if (Operand.getValueType() == VT) return Operand; // noop extension 2658200581Srdivacky assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2659200581Srdivacky "Invalid anyext node, dst < src!"); 2660200581Srdivacky assert((!VT.isVector() || 2661200581Srdivacky VT.getVectorNumElements() == 2662200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2663200581Srdivacky "Vector element count mismatch!"); 2664210299Sed 2665210299Sed if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2666210299Sed OpOpcode == ISD::ANY_EXTEND) 2667193323Sed // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2668193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2669221345Sdim else if (OpOpcode == ISD::UNDEF) 2670221345Sdim return getUNDEF(VT); 2671210299Sed 2672210299Sed // (ext (trunx x)) -> x 2673210299Sed if (OpOpcode == ISD::TRUNCATE) { 2674210299Sed SDValue OpOp = Operand.getNode()->getOperand(0); 2675210299Sed if (OpOp.getValueType() == VT) 2676210299Sed return OpOp; 2677210299Sed } 2678193323Sed break; 2679193323Sed case ISD::TRUNCATE: 2680193323Sed assert(VT.isInteger() && Operand.getValueType().isInteger() && 2681193323Sed "Invalid TRUNCATE!"); 2682193323Sed if (Operand.getValueType() == VT) return Operand; // noop truncate 2683200581Srdivacky assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2684200581Srdivacky "Invalid truncate node, src < dst!"); 2685200581Srdivacky assert((!VT.isVector() || 2686200581Srdivacky VT.getVectorNumElements() == 2687200581Srdivacky Operand.getValueType().getVectorNumElements()) && 2688200581Srdivacky "Vector element count mismatch!"); 2689193323Sed if (OpOpcode == ISD::TRUNCATE) 2690193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2691235633Sdim if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2692235633Sdim OpOpcode == ISD::ANY_EXTEND) { 2693193323Sed // If the source is smaller than the dest, we still need an extend. 2694200581Srdivacky if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2695200581Srdivacky .bitsLT(VT.getScalarType())) 2696193323Sed return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2697235633Sdim if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2698193323Sed return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2699235633Sdim return Operand.getNode()->getOperand(0); 2700193323Sed } 2701235633Sdim if (OpOpcode == ISD::UNDEF) 2702235633Sdim return getUNDEF(VT); 2703193323Sed break; 2704218893Sdim case ISD::BITCAST: 2705193323Sed // Basic sanity checking. 2706193323Sed assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2707218893Sdim && "Cannot BITCAST between types of different sizes!"); 2708193323Sed if (VT == Operand.getValueType()) return Operand; // noop conversion. 2709218893Sdim if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2710218893Sdim return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2711193323Sed if (OpOpcode == ISD::UNDEF) 2712193323Sed return getUNDEF(VT); 2713193323Sed break; 2714193323Sed case ISD::SCALAR_TO_VECTOR: 2715193323Sed assert(VT.isVector() && !Operand.getValueType().isVector() && 2716193323Sed (VT.getVectorElementType() == Operand.getValueType() || 2717193323Sed (VT.getVectorElementType().isInteger() && 2718193323Sed Operand.getValueType().isInteger() && 2719193323Sed VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2720193323Sed "Illegal SCALAR_TO_VECTOR node!"); 2721193323Sed if (OpOpcode == ISD::UNDEF) 2722193323Sed return getUNDEF(VT); 2723193323Sed // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2724193323Sed if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2725193323Sed isa<ConstantSDNode>(Operand.getOperand(1)) && 2726193323Sed Operand.getConstantOperandVal(1) == 0 && 2727193323Sed Operand.getOperand(0).getValueType() == VT) 2728193323Sed return Operand.getOperand(0); 2729193323Sed break; 2730193323Sed case ISD::FNEG: 2731193323Sed // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2732235633Sdim if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2733193323Sed return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2734193323Sed Operand.getNode()->getOperand(0)); 2735193323Sed if (OpOpcode == ISD::FNEG) // --X -> X 2736193323Sed return Operand.getNode()->getOperand(0); 2737193323Sed break; 2738193323Sed case ISD::FABS: 2739193323Sed if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2740193323Sed return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2741193323Sed break; 2742193323Sed } 2743193323Sed 2744193323Sed SDNode *N; 2745193323Sed SDVTList VTs = getVTList(VT); 2746218893Sdim if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2747193323Sed FoldingSetNodeID ID; 2748193323Sed SDValue Ops[1] = { Operand }; 2749193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2750193323Sed void *IP = 0; 2751201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2752193323Sed return SDValue(E, 0); 2753201360Srdivacky 2754263509Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2755263509Sdim DL.getDebugLoc(), VTs, Operand); 2756193323Sed CSEMap.InsertNode(N, IP); 2757193323Sed } else { 2758263509Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2759263509Sdim DL.getDebugLoc(), VTs, Operand); 2760193323Sed } 2761193323Sed 2762193323Sed AllNodes.push_back(N); 2763193323Sed#ifndef NDEBUG 2764218893Sdim VerifySDNode(N); 2765193323Sed#endif 2766193323Sed return SDValue(N, 0); 2767193323Sed} 2768193323Sed 2769252723SdimSDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, 2770252723Sdim SDNode *Cst1, SDNode *Cst2) { 2771252723Sdim SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; 2772252723Sdim SmallVector<SDValue, 4> Outputs; 2773252723Sdim EVT SVT = VT.getScalarType(); 2774193323Sed 2775252723Sdim ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); 2776252723Sdim ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); 2777252723Sdim if (Scalar1 && Scalar2) { 2778252723Sdim // Scalar instruction. 2779252723Sdim Inputs.push_back(std::make_pair(Scalar1, Scalar2)); 2780252723Sdim } else { 2781252723Sdim // For vectors extract each constant element into Inputs so we can constant 2782252723Sdim // fold them individually. 2783252723Sdim BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 2784252723Sdim BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 2785252723Sdim if (!BV1 || !BV2) 2786252723Sdim return SDValue(); 2787252723Sdim 2788252723Sdim assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 2789252723Sdim 2790252723Sdim for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 2791252723Sdim ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); 2792252723Sdim ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); 2793252723Sdim if (!V1 || !V2) // Not a constant, bail. 2794252723Sdim return SDValue(); 2795252723Sdim 2796252723Sdim // Avoid BUILD_VECTOR nodes that perform implicit truncation. 2797252723Sdim // FIXME: This is valid and could be handled by truncating the APInts. 2798252723Sdim if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 2799252723Sdim return SDValue(); 2800252723Sdim 2801252723Sdim Inputs.push_back(std::make_pair(V1, V2)); 2802252723Sdim } 2803193323Sed } 2804193323Sed 2805252723Sdim // We have a number of constant values, constant fold them element by element. 2806252723Sdim for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { 2807252723Sdim const APInt &C1 = Inputs[I].first->getAPIntValue(); 2808252723Sdim const APInt &C2 = Inputs[I].second->getAPIntValue(); 2809252723Sdim 2810252723Sdim switch (Opcode) { 2811252723Sdim case ISD::ADD: 2812252723Sdim Outputs.push_back(getConstant(C1 + C2, SVT)); 2813252723Sdim break; 2814252723Sdim case ISD::SUB: 2815252723Sdim Outputs.push_back(getConstant(C1 - C2, SVT)); 2816252723Sdim break; 2817252723Sdim case ISD::MUL: 2818252723Sdim Outputs.push_back(getConstant(C1 * C2, SVT)); 2819252723Sdim break; 2820252723Sdim case ISD::UDIV: 2821252723Sdim if (!C2.getBoolValue()) 2822252723Sdim return SDValue(); 2823252723Sdim Outputs.push_back(getConstant(C1.udiv(C2), SVT)); 2824252723Sdim break; 2825252723Sdim case ISD::UREM: 2826252723Sdim if (!C2.getBoolValue()) 2827252723Sdim return SDValue(); 2828252723Sdim Outputs.push_back(getConstant(C1.urem(C2), SVT)); 2829252723Sdim break; 2830252723Sdim case ISD::SDIV: 2831252723Sdim if (!C2.getBoolValue()) 2832252723Sdim return SDValue(); 2833252723Sdim Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); 2834252723Sdim break; 2835252723Sdim case ISD::SREM: 2836252723Sdim if (!C2.getBoolValue()) 2837252723Sdim return SDValue(); 2838252723Sdim Outputs.push_back(getConstant(C1.srem(C2), SVT)); 2839252723Sdim break; 2840252723Sdim case ISD::AND: 2841252723Sdim Outputs.push_back(getConstant(C1 & C2, SVT)); 2842252723Sdim break; 2843252723Sdim case ISD::OR: 2844252723Sdim Outputs.push_back(getConstant(C1 | C2, SVT)); 2845252723Sdim break; 2846252723Sdim case ISD::XOR: 2847252723Sdim Outputs.push_back(getConstant(C1 ^ C2, SVT)); 2848252723Sdim break; 2849252723Sdim case ISD::SHL: 2850252723Sdim Outputs.push_back(getConstant(C1 << C2, SVT)); 2851252723Sdim break; 2852252723Sdim case ISD::SRL: 2853252723Sdim Outputs.push_back(getConstant(C1.lshr(C2), SVT)); 2854252723Sdim break; 2855252723Sdim case ISD::SRA: 2856252723Sdim Outputs.push_back(getConstant(C1.ashr(C2), SVT)); 2857252723Sdim break; 2858252723Sdim case ISD::ROTL: 2859252723Sdim Outputs.push_back(getConstant(C1.rotl(C2), SVT)); 2860252723Sdim break; 2861252723Sdim case ISD::ROTR: 2862252723Sdim Outputs.push_back(getConstant(C1.rotr(C2), SVT)); 2863252723Sdim break; 2864252723Sdim default: 2865252723Sdim return SDValue(); 2866252723Sdim } 2867252723Sdim } 2868252723Sdim 2869252723Sdim // Handle the scalar case first. 2870252723Sdim if (Scalar1 && Scalar2) 2871252723Sdim return Outputs.back(); 2872252723Sdim 2873252723Sdim // Otherwise build a big vector out of the scalar elements we generated. 2874263509Sdim return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(), 2875252723Sdim Outputs.size()); 2876193323Sed} 2877193323Sed 2878263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, 2879252723Sdim SDValue N2) { 2880193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2881193323Sed ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2882193323Sed switch (Opcode) { 2883193323Sed default: break; 2884193323Sed case ISD::TokenFactor: 2885193323Sed assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2886193323Sed N2.getValueType() == MVT::Other && "Invalid token factor!"); 2887193323Sed // Fold trivial token factors. 2888193323Sed if (N1.getOpcode() == ISD::EntryToken) return N2; 2889193323Sed if (N2.getOpcode() == ISD::EntryToken) return N1; 2890193323Sed if (N1 == N2) return N1; 2891193323Sed break; 2892193323Sed case ISD::CONCAT_VECTORS: 2893245431Sdim // Concat of UNDEFs is UNDEF. 2894245431Sdim if (N1.getOpcode() == ISD::UNDEF && 2895245431Sdim N2.getOpcode() == ISD::UNDEF) 2896245431Sdim return getUNDEF(VT); 2897245431Sdim 2898193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2899193323Sed // one big BUILD_VECTOR. 2900193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 2901193323Sed N2.getOpcode() == ISD::BUILD_VECTOR) { 2902212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2903212904Sdim N1.getNode()->op_end()); 2904210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2905193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2906193323Sed } 2907193323Sed break; 2908193323Sed case ISD::AND: 2909208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2910208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2911193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2912193323Sed // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2913193323Sed // worth handling here. 2914193323Sed if (N2C && N2C->isNullValue()) 2915193323Sed return N2; 2916193323Sed if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2917193323Sed return N1; 2918193323Sed break; 2919193323Sed case ISD::OR: 2920193323Sed case ISD::XOR: 2921193323Sed case ISD::ADD: 2922193323Sed case ISD::SUB: 2923208599Srdivacky assert(VT.isInteger() && "This operator does not apply to FP types!"); 2924208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2925193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2926193323Sed // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2927193323Sed // it's worth handling here. 2928193323Sed if (N2C && N2C->isNullValue()) 2929193323Sed return N1; 2930193323Sed break; 2931193323Sed case ISD::UDIV: 2932193323Sed case ISD::UREM: 2933193323Sed case ISD::MULHU: 2934193323Sed case ISD::MULHS: 2935193323Sed case ISD::MUL: 2936193323Sed case ISD::SDIV: 2937193323Sed case ISD::SREM: 2938193323Sed assert(VT.isInteger() && "This operator does not apply to FP types!"); 2939208599Srdivacky assert(N1.getValueType() == N2.getValueType() && 2940208599Srdivacky N1.getValueType() == VT && "Binary operator types must match!"); 2941208599Srdivacky break; 2942193323Sed case ISD::FADD: 2943193323Sed case ISD::FSUB: 2944193323Sed case ISD::FMUL: 2945193323Sed case ISD::FDIV: 2946193323Sed case ISD::FREM: 2947235633Sdim if (getTarget().Options.UnsafeFPMath) { 2948193323Sed if (Opcode == ISD::FADD) { 2949193323Sed // 0+x --> x 2950193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2951193323Sed if (CFP->getValueAPF().isZero()) 2952193323Sed return N2; 2953193323Sed // x+0 --> x 2954193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2955193323Sed if (CFP->getValueAPF().isZero()) 2956193323Sed return N1; 2957193323Sed } else if (Opcode == ISD::FSUB) { 2958193323Sed // x-0 --> x 2959193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2960193323Sed if (CFP->getValueAPF().isZero()) 2961193323Sed return N1; 2962245431Sdim } else if (Opcode == ISD::FMUL) { 2963245431Sdim ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1); 2964245431Sdim SDValue V = N2; 2965245431Sdim 2966245431Sdim // If the first operand isn't the constant, try the second 2967245431Sdim if (!CFP) { 2968245431Sdim CFP = dyn_cast<ConstantFPSDNode>(N2); 2969245431Sdim V = N1; 2970245431Sdim } 2971245431Sdim 2972245431Sdim if (CFP) { 2973245431Sdim // 0*x --> 0 2974245431Sdim if (CFP->isZero()) 2975245431Sdim return SDValue(CFP,0); 2976245431Sdim // 1*x --> x 2977245431Sdim if (CFP->isExactlyValue(1.0)) 2978245431Sdim return V; 2979245431Sdim } 2980193323Sed } 2981193323Sed } 2982208599Srdivacky assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2983193323Sed assert(N1.getValueType() == N2.getValueType() && 2984193323Sed N1.getValueType() == VT && "Binary operator types must match!"); 2985193323Sed break; 2986193323Sed case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2987193323Sed assert(N1.getValueType() == VT && 2988193323Sed N1.getValueType().isFloatingPoint() && 2989193323Sed N2.getValueType().isFloatingPoint() && 2990193323Sed "Invalid FCOPYSIGN!"); 2991193323Sed break; 2992193323Sed case ISD::SHL: 2993193323Sed case ISD::SRA: 2994193323Sed case ISD::SRL: 2995193323Sed case ISD::ROTL: 2996193323Sed case ISD::ROTR: 2997193323Sed assert(VT == N1.getValueType() && 2998193323Sed "Shift operators return type must be the same as their first arg"); 2999193323Sed assert(VT.isInteger() && N2.getValueType().isInteger() && 3000193323Sed "Shifts only work on integers"); 3001252723Sdim assert((!VT.isVector() || VT == N2.getValueType()) && 3002252723Sdim "Vector shift amounts must be in the same as their first arg"); 3003218893Sdim // Verify that the shift amount VT is bit enough to hold valid shift 3004218893Sdim // amounts. This catches things like trying to shift an i1024 value by an 3005218893Sdim // i8, which is easy to fall into in generic code that uses 3006218893Sdim // TLI.getShiftAmount(). 3007218893Sdim assert(N2.getValueType().getSizeInBits() >= 3008219077Sdim Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 3009218893Sdim "Invalid use of small shift amount with oversized value!"); 3010193323Sed 3011193323Sed // Always fold shifts of i1 values so the code generator doesn't need to 3012193323Sed // handle them. Since we know the size of the shift has to be less than the 3013193323Sed // size of the value, the shift/rotate count is guaranteed to be zero. 3014193323Sed if (VT == MVT::i1) 3015193323Sed return N1; 3016202375Srdivacky if (N2C && N2C->isNullValue()) 3017202375Srdivacky return N1; 3018193323Sed break; 3019193323Sed case ISD::FP_ROUND_INREG: { 3020198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3021193323Sed assert(VT == N1.getValueType() && "Not an inreg round!"); 3022193323Sed assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 3023193323Sed "Cannot FP_ROUND_INREG integer types"); 3024202375Srdivacky assert(EVT.isVector() == VT.isVector() && 3025202375Srdivacky "FP_ROUND_INREG type should be vector iff the operand " 3026202375Srdivacky "type is vector!"); 3027202375Srdivacky assert((!EVT.isVector() || 3028202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3029202375Srdivacky "Vector element counts must match in FP_ROUND_INREG"); 3030193323Sed assert(EVT.bitsLE(VT) && "Not rounding down!"); 3031226890Sdim (void)EVT; 3032193323Sed if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 3033193323Sed break; 3034193323Sed } 3035193323Sed case ISD::FP_ROUND: 3036193323Sed assert(VT.isFloatingPoint() && 3037193323Sed N1.getValueType().isFloatingPoint() && 3038193323Sed VT.bitsLE(N1.getValueType()) && 3039193323Sed isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 3040193323Sed if (N1.getValueType() == VT) return N1; // noop conversion. 3041193323Sed break; 3042193323Sed case ISD::AssertSext: 3043193323Sed case ISD::AssertZext: { 3044198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3045193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 3046193323Sed assert(VT.isInteger() && EVT.isInteger() && 3047193323Sed "Cannot *_EXTEND_INREG FP types"); 3048200581Srdivacky assert(!EVT.isVector() && 3049200581Srdivacky "AssertSExt/AssertZExt type should be the vector element type " 3050200581Srdivacky "rather than the vector type!"); 3051193323Sed assert(EVT.bitsLE(VT) && "Not extending!"); 3052193323Sed if (VT == EVT) return N1; // noop assertion. 3053193323Sed break; 3054193323Sed } 3055193323Sed case ISD::SIGN_EXTEND_INREG: { 3056198090Srdivacky EVT EVT = cast<VTSDNode>(N2)->getVT(); 3057193323Sed assert(VT == N1.getValueType() && "Not an inreg extend!"); 3058193323Sed assert(VT.isInteger() && EVT.isInteger() && 3059193323Sed "Cannot *_EXTEND_INREG FP types"); 3060202375Srdivacky assert(EVT.isVector() == VT.isVector() && 3061202375Srdivacky "SIGN_EXTEND_INREG type should be vector iff the operand " 3062202375Srdivacky "type is vector!"); 3063202375Srdivacky assert((!EVT.isVector() || 3064202375Srdivacky EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3065202375Srdivacky "Vector element counts must match in SIGN_EXTEND_INREG"); 3066202375Srdivacky assert(EVT.bitsLE(VT) && "Not extending!"); 3067193323Sed if (EVT == VT) return N1; // Not actually extending 3068193323Sed 3069193323Sed if (N1C) { 3070193323Sed APInt Val = N1C->getAPIntValue(); 3071202375Srdivacky unsigned FromBits = EVT.getScalarType().getSizeInBits(); 3072193323Sed Val <<= Val.getBitWidth()-FromBits; 3073193323Sed Val = Val.ashr(Val.getBitWidth()-FromBits); 3074193323Sed return getConstant(Val, VT); 3075193323Sed } 3076193323Sed break; 3077193323Sed } 3078193323Sed case ISD::EXTRACT_VECTOR_ELT: 3079193323Sed // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 3080193323Sed if (N1.getOpcode() == ISD::UNDEF) 3081193323Sed return getUNDEF(VT); 3082193323Sed 3083193323Sed // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 3084193323Sed // expanding copies of large vectors from registers. 3085193323Sed if (N2C && 3086193323Sed N1.getOpcode() == ISD::CONCAT_VECTORS && 3087193323Sed N1.getNumOperands() > 0) { 3088193323Sed unsigned Factor = 3089193323Sed N1.getOperand(0).getValueType().getVectorNumElements(); 3090193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 3091193323Sed N1.getOperand(N2C->getZExtValue() / Factor), 3092193323Sed getConstant(N2C->getZExtValue() % Factor, 3093193323Sed N2.getValueType())); 3094193323Sed } 3095193323Sed 3096193323Sed // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 3097193323Sed // expanding large vector constants. 3098193323Sed if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 3099193323Sed SDValue Elt = N1.getOperand(N2C->getZExtValue()); 3100245431Sdim 3101245431Sdim if (VT != Elt.getValueType()) 3102193323Sed // If the vector element type is not legal, the BUILD_VECTOR operands 3103245431Sdim // are promoted and implicitly truncated, and the result implicitly 3104245431Sdim // extended. Make that explicit here. 3105245431Sdim Elt = getAnyExtOrTrunc(Elt, DL, VT); 3106245431Sdim 3107193323Sed return Elt; 3108193323Sed } 3109193323Sed 3110193323Sed // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 3111193323Sed // operations are lowered to scalars. 3112193323Sed if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 3113203954Srdivacky // If the indices are the same, return the inserted element else 3114203954Srdivacky // if the indices are known different, extract the element from 3115193323Sed // the original vector. 3116207618Srdivacky SDValue N1Op2 = N1.getOperand(2); 3117207618Srdivacky ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 3118207618Srdivacky 3119207618Srdivacky if (N1Op2C && N2C) { 3120207618Srdivacky if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 3121207618Srdivacky if (VT == N1.getOperand(1).getValueType()) 3122207618Srdivacky return N1.getOperand(1); 3123207618Srdivacky else 3124207618Srdivacky return getSExtOrTrunc(N1.getOperand(1), DL, VT); 3125207618Srdivacky } 3126207618Srdivacky 3127193323Sed return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 3128207618Srdivacky } 3129193323Sed } 3130193323Sed break; 3131193323Sed case ISD::EXTRACT_ELEMENT: 3132193323Sed assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 3133193323Sed assert(!N1.getValueType().isVector() && !VT.isVector() && 3134193323Sed (N1.getValueType().isInteger() == VT.isInteger()) && 3135226890Sdim N1.getValueType() != VT && 3136193323Sed "Wrong types for EXTRACT_ELEMENT!"); 3137193323Sed 3138193323Sed // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 3139193323Sed // 64-bit integers into 32-bit parts. Instead of building the extract of 3140193323Sed // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 3141193323Sed if (N1.getOpcode() == ISD::BUILD_PAIR) 3142193323Sed return N1.getOperand(N2C->getZExtValue()); 3143193323Sed 3144193323Sed // EXTRACT_ELEMENT of a constant int is also very common. 3145193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 3146193323Sed unsigned ElementSize = VT.getSizeInBits(); 3147193323Sed unsigned Shift = ElementSize * N2C->getZExtValue(); 3148193323Sed APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 3149193323Sed return getConstant(ShiftedVal.trunc(ElementSize), VT); 3150193323Sed } 3151193323Sed break; 3152218893Sdim case ISD::EXTRACT_SUBVECTOR: { 3153218893Sdim SDValue Index = N2; 3154218893Sdim if (VT.isSimple() && N1.getValueType().isSimple()) { 3155218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3156218893Sdim "Extract subvector VTs must be a vectors!"); 3157263509Sdim assert(VT.getVectorElementType() == 3158263509Sdim N1.getValueType().getVectorElementType() && 3159218893Sdim "Extract subvector VTs must have the same element type!"); 3160263509Sdim assert(VT.getSimpleVT() <= N1.getSimpleValueType() && 3161218893Sdim "Extract subvector must be from larger vector to smaller vector!"); 3162218893Sdim 3163218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3164218893Sdim assert((VT.getVectorNumElements() + 3165218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3166218893Sdim <= N1.getValueType().getVectorNumElements()) 3167218893Sdim && "Extract subvector overflow!"); 3168218893Sdim } 3169218893Sdim 3170218893Sdim // Trivial extraction. 3171263509Sdim if (VT.getSimpleVT() == N1.getSimpleValueType()) 3172218893Sdim return N1; 3173218893Sdim } 3174193323Sed break; 3175193323Sed } 3176218893Sdim } 3177193323Sed 3178252723Sdim // Perform trivial constant folding. 3179252723Sdim SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); 3180252723Sdim if (SV.getNode()) return SV; 3181252723Sdim 3182252723Sdim // Canonicalize constant to RHS if commutative. 3183252723Sdim if (N1C && !N2C && isCommutativeBinOp(Opcode)) { 3184252723Sdim std::swap(N1C, N2C); 3185252723Sdim std::swap(N1, N2); 3186193323Sed } 3187193323Sed 3188193323Sed // Constant fold FP operations. 3189193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 3190193323Sed ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 3191193323Sed if (N1CFP) { 3192193323Sed if (!N2CFP && isCommutativeBinOp(Opcode)) { 3193252723Sdim // Canonicalize constant to RHS if commutative. 3194193323Sed std::swap(N1CFP, N2CFP); 3195193323Sed std::swap(N1, N2); 3196245431Sdim } else if (N2CFP) { 3197193323Sed APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 3198193323Sed APFloat::opStatus s; 3199193323Sed switch (Opcode) { 3200193323Sed case ISD::FADD: 3201193323Sed s = V1.add(V2, APFloat::rmNearestTiesToEven); 3202193323Sed if (s != APFloat::opInvalidOp) 3203193323Sed return getConstantFP(V1, VT); 3204193323Sed break; 3205193323Sed case ISD::FSUB: 3206193323Sed s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 3207193323Sed if (s!=APFloat::opInvalidOp) 3208193323Sed return getConstantFP(V1, VT); 3209193323Sed break; 3210193323Sed case ISD::FMUL: 3211193323Sed s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3212193323Sed if (s!=APFloat::opInvalidOp) 3213193323Sed return getConstantFP(V1, VT); 3214193323Sed break; 3215193323Sed case ISD::FDIV: 3216193323Sed s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3217193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3218193323Sed return getConstantFP(V1, VT); 3219193323Sed break; 3220193323Sed case ISD::FREM : 3221193323Sed s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3222193323Sed if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3223193323Sed return getConstantFP(V1, VT); 3224193323Sed break; 3225193323Sed case ISD::FCOPYSIGN: 3226193323Sed V1.copySign(V2); 3227193323Sed return getConstantFP(V1, VT); 3228193323Sed default: break; 3229193323Sed } 3230193323Sed } 3231235633Sdim 3232235633Sdim if (Opcode == ISD::FP_ROUND) { 3233235633Sdim APFloat V = N1CFP->getValueAPF(); // make copy 3234235633Sdim bool ignored; 3235235633Sdim // This can return overflow, underflow, or inexact; we don't care. 3236235633Sdim // FIXME need to be more flexible about rounding mode. 3237252723Sdim (void)V.convert(EVTToAPFloatSemantics(VT), 3238235633Sdim APFloat::rmNearestTiesToEven, &ignored); 3239235633Sdim return getConstantFP(V, VT); 3240235633Sdim } 3241193323Sed } 3242193323Sed 3243193323Sed // Canonicalize an UNDEF to the RHS, even over a constant. 3244193323Sed if (N1.getOpcode() == ISD::UNDEF) { 3245193323Sed if (isCommutativeBinOp(Opcode)) { 3246193323Sed std::swap(N1, N2); 3247193323Sed } else { 3248193323Sed switch (Opcode) { 3249193323Sed case ISD::FP_ROUND_INREG: 3250193323Sed case ISD::SIGN_EXTEND_INREG: 3251193323Sed case ISD::SUB: 3252193323Sed case ISD::FSUB: 3253193323Sed case ISD::FDIV: 3254193323Sed case ISD::FREM: 3255193323Sed case ISD::SRA: 3256193323Sed return N1; // fold op(undef, arg2) -> undef 3257193323Sed case ISD::UDIV: 3258193323Sed case ISD::SDIV: 3259193323Sed case ISD::UREM: 3260193323Sed case ISD::SREM: 3261193323Sed case ISD::SRL: 3262193323Sed case ISD::SHL: 3263193323Sed if (!VT.isVector()) 3264193323Sed return getConstant(0, VT); // fold op(undef, arg2) -> 0 3265193323Sed // For vectors, we can't easily build an all zero vector, just return 3266193323Sed // the LHS. 3267193323Sed return N2; 3268193323Sed } 3269193323Sed } 3270193323Sed } 3271193323Sed 3272193323Sed // Fold a bunch of operators when the RHS is undef. 3273193323Sed if (N2.getOpcode() == ISD::UNDEF) { 3274193323Sed switch (Opcode) { 3275193323Sed case ISD::XOR: 3276193323Sed if (N1.getOpcode() == ISD::UNDEF) 3277193323Sed // Handle undef ^ undef -> 0 special case. This is a common 3278193323Sed // idiom (misuse). 3279193323Sed return getConstant(0, VT); 3280193323Sed // fallthrough 3281193323Sed case ISD::ADD: 3282193323Sed case ISD::ADDC: 3283193323Sed case ISD::ADDE: 3284193323Sed case ISD::SUB: 3285193574Sed case ISD::UDIV: 3286193574Sed case ISD::SDIV: 3287193574Sed case ISD::UREM: 3288193574Sed case ISD::SREM: 3289193574Sed return N2; // fold op(arg1, undef) -> undef 3290193323Sed case ISD::FADD: 3291193323Sed case ISD::FSUB: 3292193323Sed case ISD::FMUL: 3293193323Sed case ISD::FDIV: 3294193323Sed case ISD::FREM: 3295235633Sdim if (getTarget().Options.UnsafeFPMath) 3296193574Sed return N2; 3297193574Sed break; 3298193323Sed case ISD::MUL: 3299193323Sed case ISD::AND: 3300193323Sed case ISD::SRL: 3301193323Sed case ISD::SHL: 3302193323Sed if (!VT.isVector()) 3303193323Sed return getConstant(0, VT); // fold op(arg1, undef) -> 0 3304193323Sed // For vectors, we can't easily build an all zero vector, just return 3305193323Sed // the LHS. 3306193323Sed return N1; 3307193323Sed case ISD::OR: 3308193323Sed if (!VT.isVector()) 3309193323Sed return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3310193323Sed // For vectors, we can't easily build an all one vector, just return 3311193323Sed // the LHS. 3312193323Sed return N1; 3313193323Sed case ISD::SRA: 3314193323Sed return N1; 3315193323Sed } 3316193323Sed } 3317193323Sed 3318193323Sed // Memoize this node if possible. 3319193323Sed SDNode *N; 3320193323Sed SDVTList VTs = getVTList(VT); 3321218893Sdim if (VT != MVT::Glue) { 3322193323Sed SDValue Ops[] = { N1, N2 }; 3323193323Sed FoldingSetNodeID ID; 3324193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3325193323Sed void *IP = 0; 3326201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3327193323Sed return SDValue(E, 0); 3328201360Srdivacky 3329263509Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3330263509Sdim DL.getDebugLoc(), VTs, N1, N2); 3331193323Sed CSEMap.InsertNode(N, IP); 3332193323Sed } else { 3333263509Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3334263509Sdim DL.getDebugLoc(), VTs, N1, N2); 3335193323Sed } 3336193323Sed 3337193323Sed AllNodes.push_back(N); 3338193323Sed#ifndef NDEBUG 3339218893Sdim VerifySDNode(N); 3340193323Sed#endif 3341193323Sed return SDValue(N, 0); 3342193323Sed} 3343193323Sed 3344263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3345193323Sed SDValue N1, SDValue N2, SDValue N3) { 3346193323Sed // Perform various simplifications. 3347193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3348193323Sed switch (Opcode) { 3349263509Sdim case ISD::FMA: { 3350263509Sdim ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 3351263509Sdim ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 3352263509Sdim ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 3353263509Sdim if (N1CFP && N2CFP && N3CFP) { 3354263509Sdim APFloat V1 = N1CFP->getValueAPF(); 3355263509Sdim const APFloat &V2 = N2CFP->getValueAPF(); 3356263509Sdim const APFloat &V3 = N3CFP->getValueAPF(); 3357263509Sdim APFloat::opStatus s = 3358263509Sdim V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 3359263509Sdim if (s != APFloat::opInvalidOp) 3360263509Sdim return getConstantFP(V1, VT); 3361263509Sdim } 3362263509Sdim break; 3363263509Sdim } 3364193323Sed case ISD::CONCAT_VECTORS: 3365193323Sed // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3366193323Sed // one big BUILD_VECTOR. 3367193323Sed if (N1.getOpcode() == ISD::BUILD_VECTOR && 3368193323Sed N2.getOpcode() == ISD::BUILD_VECTOR && 3369193323Sed N3.getOpcode() == ISD::BUILD_VECTOR) { 3370212904Sdim SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3371212904Sdim N1.getNode()->op_end()); 3372210299Sed Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3373210299Sed Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3374193323Sed return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3375193323Sed } 3376193323Sed break; 3377193323Sed case ISD::SETCC: { 3378193323Sed // Use FoldSetCC to simplify SETCC's. 3379193323Sed SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3380193323Sed if (Simp.getNode()) return Simp; 3381193323Sed break; 3382193323Sed } 3383193323Sed case ISD::SELECT: 3384193323Sed if (N1C) { 3385193323Sed if (N1C->getZExtValue()) 3386235633Sdim return N2; // select true, X, Y -> X 3387235633Sdim return N3; // select false, X, Y -> Y 3388193323Sed } 3389193323Sed 3390193323Sed if (N2 == N3) return N2; // select C, X, X -> X 3391193323Sed break; 3392193323Sed case ISD::VECTOR_SHUFFLE: 3393198090Srdivacky llvm_unreachable("should use getVectorShuffle constructor!"); 3394218893Sdim case ISD::INSERT_SUBVECTOR: { 3395218893Sdim SDValue Index = N3; 3396218893Sdim if (VT.isSimple() && N1.getValueType().isSimple() 3397218893Sdim && N2.getValueType().isSimple()) { 3398218893Sdim assert(VT.isVector() && N1.getValueType().isVector() && 3399218893Sdim N2.getValueType().isVector() && 3400218893Sdim "Insert subvector VTs must be a vectors"); 3401218893Sdim assert(VT == N1.getValueType() && 3402218893Sdim "Dest and insert subvector source types must match!"); 3403263509Sdim assert(N2.getSimpleValueType() <= N1.getSimpleValueType() && 3404218893Sdim "Insert subvector must be from smaller vector to larger vector!"); 3405218893Sdim if (isa<ConstantSDNode>(Index.getNode())) { 3406218893Sdim assert((N2.getValueType().getVectorNumElements() + 3407218893Sdim cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3408218893Sdim <= VT.getVectorNumElements()) 3409218893Sdim && "Insert subvector overflow!"); 3410218893Sdim } 3411218893Sdim 3412218893Sdim // Trivial insertion. 3413263509Sdim if (VT.getSimpleVT() == N2.getSimpleValueType()) 3414218893Sdim return N2; 3415218893Sdim } 3416218893Sdim break; 3417218893Sdim } 3418218893Sdim case ISD::BITCAST: 3419193323Sed // Fold bit_convert nodes from a type to themselves. 3420193323Sed if (N1.getValueType() == VT) 3421193323Sed return N1; 3422193323Sed break; 3423193323Sed } 3424193323Sed 3425193323Sed // Memoize node if it doesn't produce a flag. 3426193323Sed SDNode *N; 3427193323Sed SDVTList VTs = getVTList(VT); 3428218893Sdim if (VT != MVT::Glue) { 3429193323Sed SDValue Ops[] = { N1, N2, N3 }; 3430193323Sed FoldingSetNodeID ID; 3431193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3432193323Sed void *IP = 0; 3433201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3434193323Sed return SDValue(E, 0); 3435201360Srdivacky 3436263509Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3437263509Sdim DL.getDebugLoc(), VTs, N1, N2, N3); 3438193323Sed CSEMap.InsertNode(N, IP); 3439193323Sed } else { 3440263509Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3441263509Sdim DL.getDebugLoc(), VTs, N1, N2, N3); 3442193323Sed } 3443200581Srdivacky 3444193323Sed AllNodes.push_back(N); 3445193323Sed#ifndef NDEBUG 3446218893Sdim VerifySDNode(N); 3447193323Sed#endif 3448193323Sed return SDValue(N, 0); 3449193323Sed} 3450193323Sed 3451263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3452193323Sed SDValue N1, SDValue N2, SDValue N3, 3453193323Sed SDValue N4) { 3454193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 3455193323Sed return getNode(Opcode, DL, VT, Ops, 4); 3456193323Sed} 3457193323Sed 3458263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3459193323Sed SDValue N1, SDValue N2, SDValue N3, 3460193323Sed SDValue N4, SDValue N5) { 3461193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3462193323Sed return getNode(Opcode, DL, VT, Ops, 5); 3463193323Sed} 3464193323Sed 3465198090Srdivacky/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3466198090Srdivacky/// the incoming stack arguments to be loaded from the stack. 3467198090SrdivackySDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3468198090Srdivacky SmallVector<SDValue, 8> ArgChains; 3469198090Srdivacky 3470198090Srdivacky // Include the original chain at the beginning of the list. When this is 3471198090Srdivacky // used by target LowerCall hooks, this helps legalize find the 3472198090Srdivacky // CALLSEQ_BEGIN node. 3473198090Srdivacky ArgChains.push_back(Chain); 3474198090Srdivacky 3475198090Srdivacky // Add a chain value for each stack argument. 3476198090Srdivacky for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3477198090Srdivacky UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3478198090Srdivacky if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3479198090Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3480198090Srdivacky if (FI->getIndex() < 0) 3481198090Srdivacky ArgChains.push_back(SDValue(L, 1)); 3482198090Srdivacky 3483198090Srdivacky // Build a tokenfactor for all the chains. 3484263509Sdim return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, 3485198090Srdivacky &ArgChains[0], ArgChains.size()); 3486198090Srdivacky} 3487198090Srdivacky 3488193323Sed/// getMemsetValue - Vectorized representation of the memset value 3489193323Sed/// operand. 3490198090Srdivackystatic SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3491263509Sdim SDLoc dl) { 3492206124Srdivacky assert(Value.getOpcode() != ISD::UNDEF); 3493206124Srdivacky 3494204642Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits(); 3495193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3496252723Sdim assert(C->getAPIntValue().getBitWidth() == 8); 3497252723Sdim APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 3498193323Sed if (VT.isInteger()) 3499193323Sed return DAG.getConstant(Val, VT); 3500252723Sdim return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); 3501193323Sed } 3502193323Sed 3503193323Sed Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3504218893Sdim if (NumBits > 8) { 3505218893Sdim // Use a multiplication with 0x010101... to extend the input to the 3506218893Sdim // required length. 3507252723Sdim APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 3508218893Sdim Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3509193323Sed } 3510193323Sed 3511193323Sed return Value; 3512193323Sed} 3513193323Sed 3514193323Sed/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3515193323Sed/// used when a memcpy is turned into a memset when the source is a constant 3516193323Sed/// string ptr. 3517263509Sdimstatic SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG, 3518235633Sdim const TargetLowering &TLI, StringRef Str) { 3519193323Sed // Handle vector with all elements zero. 3520193323Sed if (Str.empty()) { 3521193323Sed if (VT.isInteger()) 3522193323Sed return DAG.getConstant(0, VT); 3523218893Sdim else if (VT == MVT::f32 || VT == MVT::f64) 3524206083Srdivacky return DAG.getConstantFP(0.0, VT); 3525206083Srdivacky else if (VT.isVector()) { 3526206083Srdivacky unsigned NumElts = VT.getVectorNumElements(); 3527206083Srdivacky MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3528218893Sdim return DAG.getNode(ISD::BITCAST, dl, VT, 3529206083Srdivacky DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3530206083Srdivacky EltVT, NumElts))); 3531206083Srdivacky } else 3532206083Srdivacky llvm_unreachable("Expected type!"); 3533193323Sed } 3534193323Sed 3535193323Sed assert(!VT.isVector() && "Can't handle vector type here!"); 3536252723Sdim unsigned NumVTBits = VT.getSizeInBits(); 3537252723Sdim unsigned NumVTBytes = NumVTBits / 8; 3538235633Sdim unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); 3539235633Sdim 3540252723Sdim APInt Val(NumVTBits, 0); 3541235633Sdim if (TLI.isLittleEndian()) { 3542235633Sdim for (unsigned i = 0; i != NumBytes; ++i) 3543235633Sdim Val |= (uint64_t)(unsigned char)Str[i] << i*8; 3544235633Sdim } else { 3545235633Sdim for (unsigned i = 0; i != NumBytes; ++i) 3546235633Sdim Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; 3547193323Sed } 3548235633Sdim 3549252723Sdim // If the "cost" of materializing the integer immediate is 1 or free, then 3550252723Sdim // it is cost effective to turn the load into the immediate. 3551252723Sdim const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); 3552252723Sdim if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) 3553252723Sdim return DAG.getConstant(Val, VT); 3554252723Sdim return SDValue(0, 0); 3555193323Sed} 3556193323Sed 3557193323Sed/// getMemBasePlusOffset - Returns base and offset node for the 3558193323Sed/// 3559263509Sdimstatic SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl, 3560193323Sed SelectionDAG &DAG) { 3561198090Srdivacky EVT VT = Base.getValueType(); 3562263509Sdim return DAG.getNode(ISD::ADD, dl, 3563193323Sed VT, Base, DAG.getConstant(Offset, VT)); 3564193323Sed} 3565193323Sed 3566193323Sed/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3567193323Sed/// 3568235633Sdimstatic bool isMemSrcFromString(SDValue Src, StringRef &Str) { 3569193323Sed unsigned SrcDelta = 0; 3570193323Sed GlobalAddressSDNode *G = NULL; 3571193323Sed if (Src.getOpcode() == ISD::GlobalAddress) 3572193323Sed G = cast<GlobalAddressSDNode>(Src); 3573193323Sed else if (Src.getOpcode() == ISD::ADD && 3574193323Sed Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3575193323Sed Src.getOperand(1).getOpcode() == ISD::Constant) { 3576193323Sed G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3577193323Sed SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3578193323Sed } 3579193323Sed if (!G) 3580193323Sed return false; 3581193323Sed 3582235633Sdim return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false); 3583193323Sed} 3584193323Sed 3585206083Srdivacky/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3586206083Srdivacky/// to replace the memset / memcpy. Return true if the number of memory ops 3587206083Srdivacky/// is below the threshold. It returns the types of the sequence of 3588206083Srdivacky/// memory ops to perform memset / memcpy by reference. 3589206083Srdivackystatic bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3590206083Srdivacky unsigned Limit, uint64_t Size, 3591206083Srdivacky unsigned DstAlign, unsigned SrcAlign, 3592252723Sdim bool IsMemset, 3593252723Sdim bool ZeroMemset, 3594207618Srdivacky bool MemcpyStrSrc, 3595252723Sdim bool AllowOverlap, 3596206083Srdivacky SelectionDAG &DAG, 3597206083Srdivacky const TargetLowering &TLI) { 3598206083Srdivacky assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3599206083Srdivacky "Expecting memcpy / memset source to meet alignment requirement!"); 3600224145Sdim // If 'SrcAlign' is zero, that means the memory operation does not need to 3601224145Sdim // load the value, i.e. memset or memcpy from constant string. Otherwise, 3602224145Sdim // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3603224145Sdim // is the specified alignment of the memory operation. If it is zero, that 3604224145Sdim // means it's possible to change the alignment of the destination. 3605224145Sdim // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3606224145Sdim // not need to be loaded. 3607206124Srdivacky EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3608252723Sdim IsMemset, ZeroMemset, MemcpyStrSrc, 3609207618Srdivacky DAG.getMachineFunction()); 3610193323Sed 3611204961Srdivacky if (VT == MVT::Other) { 3612245431Sdim if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() || 3613206083Srdivacky TLI.allowsUnalignedMemoryAccesses(VT)) { 3614206274Srdivacky VT = TLI.getPointerTy(); 3615193323Sed } else { 3616206083Srdivacky switch (DstAlign & 7) { 3617193323Sed case 0: VT = MVT::i64; break; 3618193323Sed case 4: VT = MVT::i32; break; 3619193323Sed case 2: VT = MVT::i16; break; 3620193323Sed default: VT = MVT::i8; break; 3621193323Sed } 3622193323Sed } 3623193323Sed 3624193323Sed MVT LVT = MVT::i64; 3625193323Sed while (!TLI.isTypeLegal(LVT)) 3626198090Srdivacky LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3627193323Sed assert(LVT.isInteger()); 3628193323Sed 3629193323Sed if (VT.bitsGT(LVT)) 3630193323Sed VT = LVT; 3631193323Sed } 3632193323Sed 3633193323Sed unsigned NumMemOps = 0; 3634193323Sed while (Size != 0) { 3635193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3636193323Sed while (VTSize > Size) { 3637193323Sed // For now, only use non-vector load / store's for the left-over pieces. 3638252723Sdim EVT NewVT = VT; 3639252723Sdim unsigned NewVTSize; 3640252723Sdim 3641252723Sdim bool Found = false; 3642206083Srdivacky if (VT.isVector() || VT.isFloatingPoint()) { 3643252723Sdim NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 3644252723Sdim if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 3645252723Sdim TLI.isSafeMemOpType(NewVT.getSimpleVT())) 3646252723Sdim Found = true; 3647252723Sdim else if (NewVT == MVT::i64 && 3648252723Sdim TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 3649252723Sdim TLI.isSafeMemOpType(MVT::f64)) { 3650252723Sdim // i64 is usually not legal on 32-bit targets, but f64 may be. 3651252723Sdim NewVT = MVT::f64; 3652252723Sdim Found = true; 3653252723Sdim } 3654193323Sed } 3655252723Sdim 3656252723Sdim if (!Found) { 3657252723Sdim do { 3658252723Sdim NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 3659252723Sdim if (NewVT == MVT::i8) 3660252723Sdim break; 3661252723Sdim } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 3662252723Sdim } 3663252723Sdim NewVTSize = NewVT.getSizeInBits() / 8; 3664252723Sdim 3665252723Sdim // If the new VT cannot cover all of the remaining bits, then consider 3666252723Sdim // issuing a (or a pair of) unaligned and overlapping load / store. 3667252723Sdim // FIXME: Only does this for 64-bit or more since we don't have proper 3668252723Sdim // cost model for unaligned load / store. 3669252723Sdim bool Fast; 3670252723Sdim if (NumMemOps && AllowOverlap && 3671252723Sdim VTSize >= 8 && NewVTSize < Size && 3672252723Sdim TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) 3673252723Sdim VTSize = Size; 3674252723Sdim else { 3675252723Sdim VT = NewVT; 3676252723Sdim VTSize = NewVTSize; 3677252723Sdim } 3678193323Sed } 3679193323Sed 3680193323Sed if (++NumMemOps > Limit) 3681193323Sed return false; 3682252723Sdim 3683193323Sed MemOps.push_back(VT); 3684193323Sed Size -= VTSize; 3685193323Sed } 3686193323Sed 3687193323Sed return true; 3688193323Sed} 3689193323Sed 3690263509Sdimstatic SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3691206083Srdivacky SDValue Chain, SDValue Dst, 3692206083Srdivacky SDValue Src, uint64_t Size, 3693206274Srdivacky unsigned Align, bool isVol, 3694206274Srdivacky bool AlwaysInline, 3695218893Sdim MachinePointerInfo DstPtrInfo, 3696218893Sdim MachinePointerInfo SrcPtrInfo) { 3697206124Srdivacky // Turn a memcpy of undef to nop. 3698206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3699206124Srdivacky return Chain; 3700193323Sed 3701193323Sed // Expand memcpy to a series of load and store ops if the size operand falls 3702193323Sed // below a certain threshold. 3703218893Sdim // TODO: In the AlwaysInline case, if the size is big then generate a loop 3704218893Sdim // rather than maybe a humongous number of loads and stores. 3705206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3706198090Srdivacky std::vector<EVT> MemOps; 3707206083Srdivacky bool DstAlignCanChange = false; 3708218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3709218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3710245431Sdim bool OptSize = 3711252723Sdim MF.getFunction()->getAttributes(). 3712252723Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3713206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3714206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3715206083Srdivacky DstAlignCanChange = true; 3716206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3717206083Srdivacky if (Align > SrcAlign) 3718206083Srdivacky SrcAlign = Align; 3719235633Sdim StringRef Str; 3720206083Srdivacky bool CopyFromStr = isMemSrcFromString(Src, Str); 3721206083Srdivacky bool isZeroStr = CopyFromStr && Str.empty(); 3722218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3723218893Sdim 3724206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3725206083Srdivacky (DstAlignCanChange ? 0 : Align), 3726207618Srdivacky (isZeroStr ? 0 : SrcAlign), 3727252723Sdim false, false, CopyFromStr, true, DAG, TLI)) 3728193323Sed return SDValue(); 3729193323Sed 3730206083Srdivacky if (DstAlignCanChange) { 3731226890Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3732245431Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3733252723Sdim 3734252723Sdim // Don't promote to an alignment that would require dynamic stack 3735263509Sdim // realignment. 3736252723Sdim const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); 3737252723Sdim if (!TRI->needsStackRealignment(MF)) 3738252723Sdim while (NewAlign > Align && 3739252723Sdim TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) 3740252723Sdim NewAlign /= 2; 3741252723Sdim 3742206083Srdivacky if (NewAlign > Align) { 3743206083Srdivacky // Give the stack frame object a larger alignment if needed. 3744206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3745206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3746206083Srdivacky Align = NewAlign; 3747206083Srdivacky } 3748206083Srdivacky } 3749193323Sed 3750193323Sed SmallVector<SDValue, 8> OutChains; 3751193323Sed unsigned NumMemOps = MemOps.size(); 3752193323Sed uint64_t SrcOff = 0, DstOff = 0; 3753198090Srdivacky for (unsigned i = 0; i != NumMemOps; ++i) { 3754198090Srdivacky EVT VT = MemOps[i]; 3755193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3756193323Sed SDValue Value, Store; 3757193323Sed 3758252723Sdim if (VTSize > Size) { 3759252723Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3760252723Sdim // pair. Adjust the offset accordingly. 3761252723Sdim assert(i == NumMemOps-1 && i != 0); 3762252723Sdim SrcOff -= VTSize - Size; 3763252723Sdim DstOff -= VTSize - Size; 3764252723Sdim } 3765252723Sdim 3766206083Srdivacky if (CopyFromStr && 3767206083Srdivacky (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3768193323Sed // It's unlikely a store of a vector immediate can be done in a single 3769193323Sed // instruction. It would require a load from a constantpool first. 3770206083Srdivacky // We only handle zero vectors here. 3771193323Sed // FIXME: Handle other cases where store of vector immediate is done in 3772193323Sed // a single instruction. 3773235633Sdim Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); 3774252723Sdim if (Value.getNode()) 3775252723Sdim Store = DAG.getStore(Chain, dl, Value, 3776263509Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3777252723Sdim DstPtrInfo.getWithOffset(DstOff), isVol, 3778252723Sdim false, Align); 3779252723Sdim } 3780252723Sdim 3781252723Sdim if (!Store.getNode()) { 3782194710Sed // The type might not be legal for the target. This should only happen 3783194710Sed // if the type is smaller than a legal type, as on PPC, so the right 3784195098Sed // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3785195098Sed // to Load/Store if NVT==VT. 3786194710Sed // FIXME does the case above also need this? 3787198090Srdivacky EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3788195098Sed assert(NVT.bitsGE(VT)); 3789218893Sdim Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3790263509Sdim getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3791218893Sdim SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3792206083Srdivacky MinAlign(SrcAlign, SrcOff)); 3793195098Sed Store = DAG.getTruncStore(Chain, dl, Value, 3794263509Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3795218893Sdim DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3796218893Sdim false, Align); 3797193323Sed } 3798193323Sed OutChains.push_back(Store); 3799193323Sed SrcOff += VTSize; 3800193323Sed DstOff += VTSize; 3801252723Sdim Size -= VTSize; 3802193323Sed } 3803193323Sed 3804193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3805193323Sed &OutChains[0], OutChains.size()); 3806193323Sed} 3807193323Sed 3808263509Sdimstatic SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3809206083Srdivacky SDValue Chain, SDValue Dst, 3810206083Srdivacky SDValue Src, uint64_t Size, 3811206274Srdivacky unsigned Align, bool isVol, 3812206274Srdivacky bool AlwaysInline, 3813218893Sdim MachinePointerInfo DstPtrInfo, 3814218893Sdim MachinePointerInfo SrcPtrInfo) { 3815206124Srdivacky // Turn a memmove of undef to nop. 3816206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3817206124Srdivacky return Chain; 3818193323Sed 3819193323Sed // Expand memmove to a series of load and store ops if the size operand falls 3820193323Sed // below a certain threshold. 3821206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3822198090Srdivacky std::vector<EVT> MemOps; 3823206083Srdivacky bool DstAlignCanChange = false; 3824218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3825218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3826252723Sdim bool OptSize = MF.getFunction()->getAttributes(). 3827252723Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3828206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3829206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3830206083Srdivacky DstAlignCanChange = true; 3831206083Srdivacky unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3832206083Srdivacky if (Align > SrcAlign) 3833206083Srdivacky SrcAlign = Align; 3834218893Sdim unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3835206083Srdivacky 3836206083Srdivacky if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3837252723Sdim (DstAlignCanChange ? 0 : Align), SrcAlign, 3838252723Sdim false, false, false, false, DAG, TLI)) 3839193323Sed return SDValue(); 3840193323Sed 3841206083Srdivacky if (DstAlignCanChange) { 3842226890Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3843245431Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3844206083Srdivacky if (NewAlign > Align) { 3845206083Srdivacky // Give the stack frame object a larger alignment if needed. 3846206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3847206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3848206083Srdivacky Align = NewAlign; 3849206083Srdivacky } 3850206083Srdivacky } 3851206083Srdivacky 3852193323Sed uint64_t SrcOff = 0, DstOff = 0; 3853193323Sed SmallVector<SDValue, 8> LoadValues; 3854193323Sed SmallVector<SDValue, 8> LoadChains; 3855193323Sed SmallVector<SDValue, 8> OutChains; 3856193323Sed unsigned NumMemOps = MemOps.size(); 3857193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3858198090Srdivacky EVT VT = MemOps[i]; 3859193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3860263509Sdim SDValue Value; 3861193323Sed 3862193323Sed Value = DAG.getLoad(VT, dl, Chain, 3863263509Sdim getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3864218893Sdim SrcPtrInfo.getWithOffset(SrcOff), isVol, 3865235633Sdim false, false, SrcAlign); 3866193323Sed LoadValues.push_back(Value); 3867193323Sed LoadChains.push_back(Value.getValue(1)); 3868193323Sed SrcOff += VTSize; 3869193323Sed } 3870193323Sed Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3871193323Sed &LoadChains[0], LoadChains.size()); 3872193323Sed OutChains.clear(); 3873193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3874198090Srdivacky EVT VT = MemOps[i]; 3875193323Sed unsigned VTSize = VT.getSizeInBits() / 8; 3876263509Sdim SDValue Store; 3877193323Sed 3878193323Sed Store = DAG.getStore(Chain, dl, LoadValues[i], 3879263509Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3880218893Sdim DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3881193323Sed OutChains.push_back(Store); 3882193323Sed DstOff += VTSize; 3883193323Sed } 3884193323Sed 3885193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3886193323Sed &OutChains[0], OutChains.size()); 3887193323Sed} 3888193323Sed 3889263509Sdim/// \brief Lower the call to 'memset' intrinsic function into a series of store 3890263509Sdim/// operations. 3891263509Sdim/// 3892263509Sdim/// \param DAG Selection DAG where lowered code is placed. 3893263509Sdim/// \param dl Link to corresponding IR location. 3894263509Sdim/// \param Chain Control flow dependency. 3895263509Sdim/// \param Dst Pointer to destination memory location. 3896263509Sdim/// \param Src Value of byte to write into the memory. 3897263509Sdim/// \param Size Number of bytes to write. 3898263509Sdim/// \param Align Alignment of the destination in bytes. 3899263509Sdim/// \param isVol True if destination is volatile. 3900263509Sdim/// \param DstPtrInfo IR information on the memory pointer. 3901263509Sdim/// \returns New head in the control flow, if lowering was successful, empty 3902263509Sdim/// SDValue otherwise. 3903263509Sdim/// 3904263509Sdim/// The function tries to replace 'llvm.memset' intrinsic with several store 3905263509Sdim/// operations and value calculation code. This is usually profitable for small 3906263509Sdim/// memory size. 3907263509Sdimstatic SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, 3908206083Srdivacky SDValue Chain, SDValue Dst, 3909206083Srdivacky SDValue Src, uint64_t Size, 3910206274Srdivacky unsigned Align, bool isVol, 3911218893Sdim MachinePointerInfo DstPtrInfo) { 3912206124Srdivacky // Turn a memset of undef to nop. 3913206124Srdivacky if (Src.getOpcode() == ISD::UNDEF) 3914206124Srdivacky return Chain; 3915193323Sed 3916193323Sed // Expand memset to a series of load/store ops if the size operand 3917193323Sed // falls below a certain threshold. 3918206124Srdivacky const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3919198090Srdivacky std::vector<EVT> MemOps; 3920206083Srdivacky bool DstAlignCanChange = false; 3921218893Sdim MachineFunction &MF = DAG.getMachineFunction(); 3922218893Sdim MachineFrameInfo *MFI = MF.getFrameInfo(); 3923252723Sdim bool OptSize = MF.getFunction()->getAttributes(). 3924252723Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3925206083Srdivacky FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3926206083Srdivacky if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3927206083Srdivacky DstAlignCanChange = true; 3928235633Sdim bool IsZeroVal = 3929206124Srdivacky isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3930218893Sdim if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3931206083Srdivacky Size, (DstAlignCanChange ? 0 : Align), 0, 3932252723Sdim true, IsZeroVal, false, true, DAG, TLI)) 3933193323Sed return SDValue(); 3934193323Sed 3935206083Srdivacky if (DstAlignCanChange) { 3936226890Sdim Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3937245431Sdim unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3938206083Srdivacky if (NewAlign > Align) { 3939206083Srdivacky // Give the stack frame object a larger alignment if needed. 3940206083Srdivacky if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3941206083Srdivacky MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3942206083Srdivacky Align = NewAlign; 3943206083Srdivacky } 3944206083Srdivacky } 3945206083Srdivacky 3946193323Sed SmallVector<SDValue, 8> OutChains; 3947193323Sed uint64_t DstOff = 0; 3948193323Sed unsigned NumMemOps = MemOps.size(); 3949218893Sdim 3950218893Sdim // Find the largest store and generate the bit pattern for it. 3951218893Sdim EVT LargestVT = MemOps[0]; 3952218893Sdim for (unsigned i = 1; i < NumMemOps; i++) 3953218893Sdim if (MemOps[i].bitsGT(LargestVT)) 3954218893Sdim LargestVT = MemOps[i]; 3955218893Sdim SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3956218893Sdim 3957193323Sed for (unsigned i = 0; i < NumMemOps; i++) { 3958198090Srdivacky EVT VT = MemOps[i]; 3959252723Sdim unsigned VTSize = VT.getSizeInBits() / 8; 3960252723Sdim if (VTSize > Size) { 3961252723Sdim // Issuing an unaligned load / store pair that overlaps with the previous 3962252723Sdim // pair. Adjust the offset accordingly. 3963252723Sdim assert(i == NumMemOps-1 && i != 0); 3964252723Sdim DstOff -= VTSize - Size; 3965252723Sdim } 3966218893Sdim 3967218893Sdim // If this store is smaller than the largest store see whether we can get 3968218893Sdim // the smaller value for free with a truncate. 3969218893Sdim SDValue Value = MemSetValue; 3970218893Sdim if (VT.bitsLT(LargestVT)) { 3971218893Sdim if (!LargestVT.isVector() && !VT.isVector() && 3972218893Sdim TLI.isTruncateFree(LargestVT, VT)) 3973218893Sdim Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3974218893Sdim else 3975218893Sdim Value = getMemsetValue(Src, VT, DAG, dl); 3976218893Sdim } 3977218893Sdim assert(Value.getValueType() == VT && "Value with wrong type."); 3978193323Sed SDValue Store = DAG.getStore(Chain, dl, Value, 3979263509Sdim getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3980218893Sdim DstPtrInfo.getWithOffset(DstOff), 3981218893Sdim isVol, false, Align); 3982193323Sed OutChains.push_back(Store); 3983218893Sdim DstOff += VT.getSizeInBits() / 8; 3984252723Sdim Size -= VTSize; 3985193323Sed } 3986193323Sed 3987193323Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3988193323Sed &OutChains[0], OutChains.size()); 3989193323Sed} 3990193323Sed 3991263509SdimSDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, 3992193323Sed SDValue Src, SDValue Size, 3993206274Srdivacky unsigned Align, bool isVol, bool AlwaysInline, 3994218893Sdim MachinePointerInfo DstPtrInfo, 3995218893Sdim MachinePointerInfo SrcPtrInfo) { 3996252723Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 3997193323Sed 3998193323Sed // Check to see if we should lower the memcpy to loads and stores first. 3999193323Sed // For cases within the target-specified limits, this is the best choice. 4000193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4001193323Sed if (ConstantSize) { 4002193323Sed // Memcpy with size zero? Just return the original chain. 4003193323Sed if (ConstantSize->isNullValue()) 4004193323Sed return Chain; 4005193323Sed 4006206083Srdivacky SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4007206083Srdivacky ConstantSize->getZExtValue(),Align, 4008218893Sdim isVol, false, DstPtrInfo, SrcPtrInfo); 4009193323Sed if (Result.getNode()) 4010193323Sed return Result; 4011193323Sed } 4012193323Sed 4013193323Sed // Then check to see if we should lower the memcpy with target-specific 4014193323Sed // code. If the target chooses to do this, this is the next best. 4015193323Sed SDValue Result = 4016208599Srdivacky TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 4017206274Srdivacky isVol, AlwaysInline, 4018218893Sdim DstPtrInfo, SrcPtrInfo); 4019193323Sed if (Result.getNode()) 4020193323Sed return Result; 4021193323Sed 4022193323Sed // If we really need inline code and the target declined to provide it, 4023193323Sed // use a (potentially long) sequence of loads and stores. 4024193323Sed if (AlwaysInline) { 4025193323Sed assert(ConstantSize && "AlwaysInline requires a constant size!"); 4026193323Sed return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 4027206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 4028218893Sdim true, DstPtrInfo, SrcPtrInfo); 4029193323Sed } 4030193323Sed 4031206274Srdivacky // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 4032206274Srdivacky // memcpy is not guaranteed to be safe. libc memcpys aren't required to 4033206274Srdivacky // respect volatile, so they may do things like read or write memory 4034206274Srdivacky // beyond the given memory regions. But fixing this isn't easy, and most 4035206274Srdivacky // people don't care. 4036206274Srdivacky 4037263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4038263509Sdim 4039193323Sed // Emit a library call. 4040193323Sed TargetLowering::ArgListTy Args; 4041193323Sed TargetLowering::ArgListEntry Entry; 4042263509Sdim Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4043193323Sed Entry.Node = Dst; Args.push_back(Entry); 4044193323Sed Entry.Node = Src; Args.push_back(Entry); 4045193323Sed Entry.Node = Size; Args.push_back(Entry); 4046263509Sdim // FIXME: pass in SDLoc 4047245431Sdim TargetLowering:: 4048245431Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4049198090Srdivacky false, false, false, false, 0, 4050263509Sdim TLI->getLibcallCallingConv(RTLIB::MEMCPY), 4051235633Sdim /*isTailCall=*/false, 4052235633Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4053263509Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 4054263509Sdim TLI->getPointerTy()), 4055204642Srdivacky Args, *this, dl); 4056263509Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4057245431Sdim 4058193323Sed return CallResult.second; 4059193323Sed} 4060193323Sed 4061263509SdimSDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, 4062193323Sed SDValue Src, SDValue Size, 4063206274Srdivacky unsigned Align, bool isVol, 4064218893Sdim MachinePointerInfo DstPtrInfo, 4065218893Sdim MachinePointerInfo SrcPtrInfo) { 4066252723Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4067193323Sed 4068193323Sed // Check to see if we should lower the memmove to loads and stores first. 4069193323Sed // For cases within the target-specified limits, this is the best choice. 4070193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4071193323Sed if (ConstantSize) { 4072193323Sed // Memmove with size zero? Just return the original chain. 4073193323Sed if (ConstantSize->isNullValue()) 4074193323Sed return Chain; 4075193323Sed 4076193323Sed SDValue Result = 4077193323Sed getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 4078206274Srdivacky ConstantSize->getZExtValue(), Align, isVol, 4079218893Sdim false, DstPtrInfo, SrcPtrInfo); 4080193323Sed if (Result.getNode()) 4081193323Sed return Result; 4082193323Sed } 4083193323Sed 4084193323Sed // Then check to see if we should lower the memmove with target-specific 4085193323Sed // code. If the target chooses to do this, this is the next best. 4086193323Sed SDValue Result = 4087208599Srdivacky TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4088218893Sdim DstPtrInfo, SrcPtrInfo); 4089193323Sed if (Result.getNode()) 4090193323Sed return Result; 4091193323Sed 4092207618Srdivacky // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 4093207618Srdivacky // not be safe. See memcpy above for more details. 4094207618Srdivacky 4095263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4096263509Sdim 4097193323Sed // Emit a library call. 4098193323Sed TargetLowering::ArgListTy Args; 4099193323Sed TargetLowering::ArgListEntry Entry; 4100263509Sdim Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4101193323Sed Entry.Node = Dst; Args.push_back(Entry); 4102193323Sed Entry.Node = Src; Args.push_back(Entry); 4103193323Sed Entry.Node = Size; Args.push_back(Entry); 4104263509Sdim // FIXME: pass in SDLoc 4105245431Sdim TargetLowering:: 4106245431Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4107198090Srdivacky false, false, false, false, 0, 4108263509Sdim TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 4109235633Sdim /*isTailCall=*/false, 4110235633Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4111263509Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 4112263509Sdim TLI->getPointerTy()), 4113204642Srdivacky Args, *this, dl); 4114263509Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4115245431Sdim 4116193323Sed return CallResult.second; 4117193323Sed} 4118193323Sed 4119263509SdimSDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, 4120193323Sed SDValue Src, SDValue Size, 4121206274Srdivacky unsigned Align, bool isVol, 4122218893Sdim MachinePointerInfo DstPtrInfo) { 4123252723Sdim assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4124193323Sed 4125193323Sed // Check to see if we should lower the memset to stores first. 4126193323Sed // For cases within the target-specified limits, this is the best choice. 4127193323Sed ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4128193323Sed if (ConstantSize) { 4129193323Sed // Memset with size zero? Just return the original chain. 4130193323Sed if (ConstantSize->isNullValue()) 4131193323Sed return Chain; 4132193323Sed 4133206274Srdivacky SDValue Result = 4134206274Srdivacky getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 4135218893Sdim Align, isVol, DstPtrInfo); 4136206274Srdivacky 4137193323Sed if (Result.getNode()) 4138193323Sed return Result; 4139193323Sed } 4140193323Sed 4141193323Sed // Then check to see if we should lower the memset with target-specific 4142193323Sed // code. If the target chooses to do this, this is the next best. 4143193323Sed SDValue Result = 4144208599Srdivacky TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4145218893Sdim DstPtrInfo); 4146193323Sed if (Result.getNode()) 4147193323Sed return Result; 4148193323Sed 4149218893Sdim // Emit a library call. 4150263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 4151263509Sdim Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext()); 4152193323Sed TargetLowering::ArgListTy Args; 4153193323Sed TargetLowering::ArgListEntry Entry; 4154193323Sed Entry.Node = Dst; Entry.Ty = IntPtrTy; 4155193323Sed Args.push_back(Entry); 4156193323Sed // Extend or truncate the argument to be an i32 value for the call. 4157193323Sed if (Src.getValueType().bitsGT(MVT::i32)) 4158193323Sed Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 4159193323Sed else 4160193323Sed Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 4161198090Srdivacky Entry.Node = Src; 4162198090Srdivacky Entry.Ty = Type::getInt32Ty(*getContext()); 4163198090Srdivacky Entry.isSExt = true; 4164193323Sed Args.push_back(Entry); 4165198090Srdivacky Entry.Node = Size; 4166198090Srdivacky Entry.Ty = IntPtrTy; 4167198090Srdivacky Entry.isSExt = false; 4168193323Sed Args.push_back(Entry); 4169263509Sdim // FIXME: pass in SDLoc 4170245431Sdim TargetLowering:: 4171245431Sdim CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4172198090Srdivacky false, false, false, false, 0, 4173263509Sdim TLI->getLibcallCallingConv(RTLIB::MEMSET), 4174235633Sdim /*isTailCall=*/false, 4175235633Sdim /*doesNotReturn*/false, /*isReturnValueUsed=*/false, 4176263509Sdim getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 4177263509Sdim TLI->getPointerTy()), 4178204642Srdivacky Args, *this, dl); 4179263509Sdim std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4180245431Sdim 4181193323Sed return CallResult.second; 4182193323Sed} 4183193323Sed 4184263509SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4185263509Sdim SDVTList VTList, SDValue* Ops, unsigned NumOps, 4186263509Sdim MachineMemOperand *MMO, 4187263509Sdim AtomicOrdering Ordering, 4188263509Sdim SynchronizationScope SynchScope) { 4189263509Sdim FoldingSetNodeID ID; 4190263509Sdim ID.AddInteger(MemVT.getRawBits()); 4191263509Sdim AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4192263509Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4193263509Sdim void* IP = 0; 4194263509Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4195263509Sdim cast<AtomicSDNode>(E)->refineAlignment(MMO); 4196263509Sdim return SDValue(E, 0); 4197263509Sdim } 4198263509Sdim 4199263509Sdim // Allocate the operands array for the node out of the BumpPtrAllocator, since 4200263509Sdim // SDNode doesn't have access to it. This memory will be "leaked" when 4201263509Sdim // the node is deallocated, but recovered when the allocator is released. 4202263509Sdim // If the number of operands is less than 5 we use AtomicSDNode's internal 4203263509Sdim // storage. 4204263509Sdim SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps) : 0; 4205263509Sdim 4206263509Sdim SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), 4207263509Sdim dl.getDebugLoc(), VTList, MemVT, 4208263509Sdim Ops, DynOps, NumOps, MMO, 4209263509Sdim Ordering, SynchScope); 4210263509Sdim CSEMap.InsertNode(N, IP); 4211263509Sdim AllNodes.push_back(N); 4212263509Sdim return SDValue(N, 0); 4213263509Sdim} 4214263509Sdim 4215263509SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4216218893Sdim SDValue Chain, SDValue Ptr, SDValue Cmp, 4217218893Sdim SDValue Swp, MachinePointerInfo PtrInfo, 4218226890Sdim unsigned Alignment, 4219226890Sdim AtomicOrdering Ordering, 4220245431Sdim SynchronizationScope SynchScope) { 4221198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4222198090Srdivacky Alignment = getEVTAlignment(MemVT); 4223198090Srdivacky 4224198090Srdivacky MachineFunction &MF = getMachineFunction(); 4225198090Srdivacky 4226245431Sdim // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. 4227198090Srdivacky // For now, atomics are considered to be volatile always. 4228226890Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4229226890Sdim // orderings in the memoperand. 4230245431Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4231245431Sdim if (Opcode != ISD::ATOMIC_STORE) 4232245431Sdim Flags |= MachineMemOperand::MOLoad; 4233245431Sdim if (Opcode != ISD::ATOMIC_LOAD) 4234245431Sdim Flags |= MachineMemOperand::MOStore; 4235198090Srdivacky 4236198090Srdivacky MachineMemOperand *MMO = 4237218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 4238198090Srdivacky 4239226890Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 4240226890Sdim Ordering, SynchScope); 4241198090Srdivacky} 4242198090Srdivacky 4243263509SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4244198090Srdivacky SDValue Chain, 4245198090Srdivacky SDValue Ptr, SDValue Cmp, 4246226890Sdim SDValue Swp, MachineMemOperand *MMO, 4247226890Sdim AtomicOrdering Ordering, 4248226890Sdim SynchronizationScope SynchScope) { 4249193323Sed assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 4250193323Sed assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 4251193323Sed 4252198090Srdivacky EVT VT = Cmp.getValueType(); 4253193323Sed 4254193323Sed SDVTList VTs = getVTList(VT, MVT::Other); 4255193323Sed SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 4256263509Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, Ordering, SynchScope); 4257193323Sed} 4258193323Sed 4259263509SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4260193323Sed SDValue Chain, 4261193323Sed SDValue Ptr, SDValue Val, 4262193323Sed const Value* PtrVal, 4263226890Sdim unsigned Alignment, 4264226890Sdim AtomicOrdering Ordering, 4265226890Sdim SynchronizationScope SynchScope) { 4266198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4267198090Srdivacky Alignment = getEVTAlignment(MemVT); 4268198090Srdivacky 4269198090Srdivacky MachineFunction &MF = getMachineFunction(); 4270245431Sdim // An atomic store does not load. An atomic load does not store. 4271226890Sdim // (An atomicrmw obviously both loads and stores.) 4272245431Sdim // For now, atomics are considered to be volatile always, and they are 4273245431Sdim // chained as such. 4274226890Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4275226890Sdim // orderings in the memoperand. 4276245431Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4277245431Sdim if (Opcode != ISD::ATOMIC_STORE) 4278245431Sdim Flags |= MachineMemOperand::MOLoad; 4279245431Sdim if (Opcode != ISD::ATOMIC_LOAD) 4280245431Sdim Flags |= MachineMemOperand::MOStore; 4281198090Srdivacky 4282198090Srdivacky MachineMemOperand *MMO = 4283218893Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4284198090Srdivacky MemVT.getStoreSize(), Alignment); 4285198090Srdivacky 4286226890Sdim return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 4287226890Sdim Ordering, SynchScope); 4288198090Srdivacky} 4289198090Srdivacky 4290263509SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4291198090Srdivacky SDValue Chain, 4292198090Srdivacky SDValue Ptr, SDValue Val, 4293226890Sdim MachineMemOperand *MMO, 4294226890Sdim AtomicOrdering Ordering, 4295226890Sdim SynchronizationScope SynchScope) { 4296193323Sed assert((Opcode == ISD::ATOMIC_LOAD_ADD || 4297193323Sed Opcode == ISD::ATOMIC_LOAD_SUB || 4298193323Sed Opcode == ISD::ATOMIC_LOAD_AND || 4299193323Sed Opcode == ISD::ATOMIC_LOAD_OR || 4300193323Sed Opcode == ISD::ATOMIC_LOAD_XOR || 4301193323Sed Opcode == ISD::ATOMIC_LOAD_NAND || 4302193323Sed Opcode == ISD::ATOMIC_LOAD_MIN || 4303193323Sed Opcode == ISD::ATOMIC_LOAD_MAX || 4304193323Sed Opcode == ISD::ATOMIC_LOAD_UMIN || 4305193323Sed Opcode == ISD::ATOMIC_LOAD_UMAX || 4306226890Sdim Opcode == ISD::ATOMIC_SWAP || 4307226890Sdim Opcode == ISD::ATOMIC_STORE) && 4308193323Sed "Invalid Atomic Op"); 4309193323Sed 4310198090Srdivacky EVT VT = Val.getValueType(); 4311193323Sed 4312226890Sdim SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 4313226890Sdim getVTList(VT, MVT::Other); 4314193323Sed SDValue Ops[] = {Chain, Ptr, Val}; 4315263509Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 3, MMO, Ordering, SynchScope); 4316193323Sed} 4317193323Sed 4318263509SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4319226890Sdim EVT VT, SDValue Chain, 4320226890Sdim SDValue Ptr, 4321226890Sdim const Value* PtrVal, 4322226890Sdim unsigned Alignment, 4323226890Sdim AtomicOrdering Ordering, 4324226890Sdim SynchronizationScope SynchScope) { 4325226890Sdim if (Alignment == 0) // Ensure that codegen never sees alignment 0 4326226890Sdim Alignment = getEVTAlignment(MemVT); 4327226890Sdim 4328226890Sdim MachineFunction &MF = getMachineFunction(); 4329245431Sdim // An atomic store does not load. An atomic load does not store. 4330245431Sdim // (An atomicrmw obviously both loads and stores.) 4331245431Sdim // For now, atomics are considered to be volatile always, and they are 4332245431Sdim // chained as such. 4333226890Sdim // FIXME: Volatile isn't really correct; we should keep track of atomic 4334226890Sdim // orderings in the memoperand. 4335245431Sdim unsigned Flags = MachineMemOperand::MOVolatile; 4336245431Sdim if (Opcode != ISD::ATOMIC_STORE) 4337245431Sdim Flags |= MachineMemOperand::MOLoad; 4338245431Sdim if (Opcode != ISD::ATOMIC_LOAD) 4339245431Sdim Flags |= MachineMemOperand::MOStore; 4340226890Sdim 4341226890Sdim MachineMemOperand *MMO = 4342226890Sdim MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4343226890Sdim MemVT.getStoreSize(), Alignment); 4344226890Sdim 4345226890Sdim return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4346226890Sdim Ordering, SynchScope); 4347226890Sdim} 4348226890Sdim 4349263509SdimSDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4350226890Sdim EVT VT, SDValue Chain, 4351226890Sdim SDValue Ptr, 4352226890Sdim MachineMemOperand *MMO, 4353226890Sdim AtomicOrdering Ordering, 4354226890Sdim SynchronizationScope SynchScope) { 4355226890Sdim assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4356226890Sdim 4357226890Sdim SDVTList VTs = getVTList(VT, MVT::Other); 4358226890Sdim SDValue Ops[] = {Chain, Ptr}; 4359263509Sdim return getAtomic(Opcode, dl, MemVT, VTs, Ops, 2, MMO, Ordering, SynchScope); 4360226890Sdim} 4361226890Sdim 4362193323Sed/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4363193323SedSDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4364263509Sdim SDLoc dl) { 4365193323Sed if (NumOps == 1) 4366193323Sed return Ops[0]; 4367193323Sed 4368198090Srdivacky SmallVector<EVT, 4> VTs; 4369193323Sed VTs.reserve(NumOps); 4370193323Sed for (unsigned i = 0; i < NumOps; ++i) 4371193323Sed VTs.push_back(Ops[i].getValueType()); 4372193323Sed return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4373193323Sed Ops, NumOps); 4374193323Sed} 4375193323Sed 4376193323SedSDValue 4377263509SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, 4378198090Srdivacky const EVT *VTs, unsigned NumVTs, 4379193323Sed const SDValue *Ops, unsigned NumOps, 4380218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4381193323Sed unsigned Align, bool Vol, 4382193323Sed bool ReadMem, bool WriteMem) { 4383193323Sed return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4384218893Sdim MemVT, PtrInfo, Align, Vol, 4385193323Sed ReadMem, WriteMem); 4386193323Sed} 4387193323Sed 4388193323SedSDValue 4389263509SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4390193323Sed const SDValue *Ops, unsigned NumOps, 4391218893Sdim EVT MemVT, MachinePointerInfo PtrInfo, 4392193323Sed unsigned Align, bool Vol, 4393193323Sed bool ReadMem, bool WriteMem) { 4394198090Srdivacky if (Align == 0) // Ensure that codegen never sees alignment 0 4395198090Srdivacky Align = getEVTAlignment(MemVT); 4396198090Srdivacky 4397198090Srdivacky MachineFunction &MF = getMachineFunction(); 4398198090Srdivacky unsigned Flags = 0; 4399198090Srdivacky if (WriteMem) 4400198090Srdivacky Flags |= MachineMemOperand::MOStore; 4401198090Srdivacky if (ReadMem) 4402198090Srdivacky Flags |= MachineMemOperand::MOLoad; 4403198090Srdivacky if (Vol) 4404198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4405198090Srdivacky MachineMemOperand *MMO = 4406218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4407198090Srdivacky 4408198090Srdivacky return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4409198090Srdivacky} 4410198090Srdivacky 4411198090SrdivackySDValue 4412263509SdimSelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4413198090Srdivacky const SDValue *Ops, unsigned NumOps, 4414198090Srdivacky EVT MemVT, MachineMemOperand *MMO) { 4415198090Srdivacky assert((Opcode == ISD::INTRINSIC_VOID || 4416198090Srdivacky Opcode == ISD::INTRINSIC_W_CHAIN || 4417218893Sdim Opcode == ISD::PREFETCH || 4418245431Sdim Opcode == ISD::LIFETIME_START || 4419245431Sdim Opcode == ISD::LIFETIME_END || 4420198090Srdivacky (Opcode <= INT_MAX && 4421198090Srdivacky (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4422198090Srdivacky "Opcode is not a memory-accessing opcode!"); 4423198090Srdivacky 4424193323Sed // Memoize the node unless it returns a flag. 4425193323Sed MemIntrinsicSDNode *N; 4426218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4427193323Sed FoldingSetNodeID ID; 4428193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4429245431Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4430193323Sed void *IP = 0; 4431198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4432198090Srdivacky cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4433193323Sed return SDValue(E, 0); 4434198090Srdivacky } 4435193323Sed 4436263509Sdim N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4437263509Sdim dl.getDebugLoc(), VTList, Ops, 4438263509Sdim NumOps, MemVT, MMO); 4439193323Sed CSEMap.InsertNode(N, IP); 4440193323Sed } else { 4441263509Sdim N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4442263509Sdim dl.getDebugLoc(), VTList, Ops, 4443263509Sdim NumOps, MemVT, MMO); 4444193323Sed } 4445193323Sed AllNodes.push_back(N); 4446193323Sed return SDValue(N, 0); 4447193323Sed} 4448193323Sed 4449218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4450218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4451218893Sdim/// code generator has many cases where it doesn't bother passing in a 4452218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4453218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4454218893Sdim // If this is FI+Offset, we can model it. 4455218893Sdim if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4456218893Sdim return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4457218893Sdim 4458218893Sdim // If this is (FI+Offset1)+Offset2, we can model it. 4459218893Sdim if (Ptr.getOpcode() != ISD::ADD || 4460218893Sdim !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4461218893Sdim !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4462218893Sdim return MachinePointerInfo(); 4463218893Sdim 4464218893Sdim int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4465218893Sdim return MachinePointerInfo::getFixedStack(FI, Offset+ 4466218893Sdim cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4467218893Sdim} 4468218893Sdim 4469218893Sdim/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4470218893Sdim/// MachinePointerInfo record from it. This is particularly useful because the 4471218893Sdim/// code generator has many cases where it doesn't bother passing in a 4472218893Sdim/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4473218893Sdimstatic MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4474218893Sdim // If the 'Offset' value isn't a constant, we can't handle this. 4475218893Sdim if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4476218893Sdim return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4477218893Sdim if (OffsetOp.getOpcode() == ISD::UNDEF) 4478218893Sdim return InferPointerInfo(Ptr); 4479218893Sdim return MachinePointerInfo(); 4480218893Sdim} 4481218893Sdim 4482218893Sdim 4483193323SedSDValue 4484210299SedSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4485263509Sdim EVT VT, SDLoc dl, SDValue Chain, 4486193323Sed SDValue Ptr, SDValue Offset, 4487218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4488235633Sdim bool isVolatile, bool isNonTemporal, bool isInvariant, 4489235633Sdim unsigned Alignment, const MDNode *TBAAInfo, 4490235633Sdim const MDNode *Ranges) { 4491245431Sdim assert(Chain.getValueType() == MVT::Other && 4492224145Sdim "Invalid chain type"); 4493193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4494198090Srdivacky Alignment = getEVTAlignment(VT); 4495193323Sed 4496198090Srdivacky unsigned Flags = MachineMemOperand::MOLoad; 4497198090Srdivacky if (isVolatile) 4498198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4499203954Srdivacky if (isNonTemporal) 4500203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4501235633Sdim if (isInvariant) 4502235633Sdim Flags |= MachineMemOperand::MOInvariant; 4503218893Sdim 4504218893Sdim // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4505218893Sdim // clients. 4506218893Sdim if (PtrInfo.V == 0) 4507218893Sdim PtrInfo = InferPointerInfo(Ptr, Offset); 4508218893Sdim 4509218893Sdim MachineFunction &MF = getMachineFunction(); 4510198090Srdivacky MachineMemOperand *MMO = 4511218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4512235633Sdim TBAAInfo, Ranges); 4513210299Sed return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4514198090Srdivacky} 4515198090Srdivacky 4516198090SrdivackySDValue 4517218893SdimSelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4518263509Sdim EVT VT, SDLoc dl, SDValue Chain, 4519198090Srdivacky SDValue Ptr, SDValue Offset, EVT MemVT, 4520198090Srdivacky MachineMemOperand *MMO) { 4521198090Srdivacky if (VT == MemVT) { 4522193323Sed ExtType = ISD::NON_EXTLOAD; 4523193323Sed } else if (ExtType == ISD::NON_EXTLOAD) { 4524198090Srdivacky assert(VT == MemVT && "Non-extending load from different memory type!"); 4525193323Sed } else { 4526193323Sed // Extending load. 4527200581Srdivacky assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4528200581Srdivacky "Should only be an extending load, not truncating!"); 4529198090Srdivacky assert(VT.isInteger() == MemVT.isInteger() && 4530193323Sed "Cannot convert from FP to Int or Int -> FP!"); 4531200581Srdivacky assert(VT.isVector() == MemVT.isVector() && 4532200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4533200581Srdivacky assert((!VT.isVector() || 4534200581Srdivacky VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4535200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4536193323Sed } 4537193323Sed 4538193323Sed bool Indexed = AM != ISD::UNINDEXED; 4539193323Sed assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4540193323Sed "Unindexed load with an offset!"); 4541193323Sed 4542193323Sed SDVTList VTs = Indexed ? 4543193323Sed getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4544193323Sed SDValue Ops[] = { Chain, Ptr, Offset }; 4545193323Sed FoldingSetNodeID ID; 4546193323Sed AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4547198090Srdivacky ID.AddInteger(MemVT.getRawBits()); 4548204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4549245431Sdim MMO->isNonTemporal(), 4550235633Sdim MMO->isInvariant())); 4551245431Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4552193323Sed void *IP = 0; 4553198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4554198090Srdivacky cast<LoadSDNode>(E)->refineAlignment(MMO); 4555193323Sed return SDValue(E, 0); 4556198090Srdivacky } 4557263509Sdim SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(), 4558263509Sdim dl.getDebugLoc(), VTs, AM, ExtType, 4559205407Srdivacky MemVT, MMO); 4560193323Sed CSEMap.InsertNode(N, IP); 4561193323Sed AllNodes.push_back(N); 4562193323Sed return SDValue(N, 0); 4563193323Sed} 4564193323Sed 4565263509SdimSDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4566193323Sed SDValue Chain, SDValue Ptr, 4567218893Sdim MachinePointerInfo PtrInfo, 4568203954Srdivacky bool isVolatile, bool isNonTemporal, 4569245431Sdim bool isInvariant, unsigned Alignment, 4570235633Sdim const MDNode *TBAAInfo, 4571235633Sdim const MDNode *Ranges) { 4572193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4573210299Sed return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4574235633Sdim PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4575235633Sdim TBAAInfo, Ranges); 4576193323Sed} 4577193323Sed 4578263509SdimSDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4579263509Sdim SDValue Chain, SDValue Ptr, 4580263509Sdim MachineMemOperand *MMO) { 4581263509Sdim SDValue Undef = getUNDEF(Ptr.getValueType()); 4582263509Sdim return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4583263509Sdim VT, MMO); 4584263509Sdim} 4585263509Sdim 4586263509SdimSDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4587193323Sed SDValue Chain, SDValue Ptr, 4588218893Sdim MachinePointerInfo PtrInfo, EVT MemVT, 4589203954Srdivacky bool isVolatile, bool isNonTemporal, 4590218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4591193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4592210299Sed return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4593235633Sdim PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4594218893Sdim TBAAInfo); 4595193323Sed} 4596193323Sed 4597218893Sdim 4598263509SdimSDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4599263509Sdim SDValue Chain, SDValue Ptr, EVT MemVT, 4600263509Sdim MachineMemOperand *MMO) { 4601263509Sdim SDValue Undef = getUNDEF(Ptr.getValueType()); 4602263509Sdim return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4603263509Sdim MemVT, MMO); 4604263509Sdim} 4605263509Sdim 4606193323SedSDValue 4607263509SdimSelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base, 4608193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4609193323Sed LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4610193323Sed assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4611193323Sed "Load is already a indexed load!"); 4612210299Sed return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4613218893Sdim LD->getChain(), Base, Offset, LD->getPointerInfo(), 4614245431Sdim LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4615235633Sdim false, LD->getAlignment()); 4616193323Sed} 4617193323Sed 4618263509SdimSDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4619218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4620203954Srdivacky bool isVolatile, bool isNonTemporal, 4621218893Sdim unsigned Alignment, const MDNode *TBAAInfo) { 4622245431Sdim assert(Chain.getValueType() == MVT::Other && 4623224145Sdim "Invalid chain type"); 4624193323Sed if (Alignment == 0) // Ensure that codegen never sees alignment 0 4625198090Srdivacky Alignment = getEVTAlignment(Val.getValueType()); 4626193323Sed 4627198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4628198090Srdivacky if (isVolatile) 4629198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4630203954Srdivacky if (isNonTemporal) 4631203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4632218893Sdim 4633218893Sdim if (PtrInfo.V == 0) 4634218893Sdim PtrInfo = InferPointerInfo(Ptr); 4635218893Sdim 4636218893Sdim MachineFunction &MF = getMachineFunction(); 4637198090Srdivacky MachineMemOperand *MMO = 4638218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, 4639218893Sdim Val.getValueType().getStoreSize(), Alignment, 4640218893Sdim TBAAInfo); 4641198090Srdivacky 4642198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4643198090Srdivacky} 4644198090Srdivacky 4645263509SdimSDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4646198090Srdivacky SDValue Ptr, MachineMemOperand *MMO) { 4647245431Sdim assert(Chain.getValueType() == MVT::Other && 4648224145Sdim "Invalid chain type"); 4649198090Srdivacky EVT VT = Val.getValueType(); 4650193323Sed SDVTList VTs = getVTList(MVT::Other); 4651193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4652193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4653193323Sed FoldingSetNodeID ID; 4654193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4655193323Sed ID.AddInteger(VT.getRawBits()); 4656204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4657235633Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4658245431Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4659193323Sed void *IP = 0; 4660198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4661198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4662193323Sed return SDValue(E, 0); 4663198090Srdivacky } 4664263509Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4665263509Sdim dl.getDebugLoc(), VTs, 4666263509Sdim ISD::UNINDEXED, false, VT, MMO); 4667193323Sed CSEMap.InsertNode(N, IP); 4668193323Sed AllNodes.push_back(N); 4669193323Sed return SDValue(N, 0); 4670193323Sed} 4671193323Sed 4672263509SdimSDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4673218893Sdim SDValue Ptr, MachinePointerInfo PtrInfo, 4674218893Sdim EVT SVT,bool isVolatile, bool isNonTemporal, 4675218893Sdim unsigned Alignment, 4676218893Sdim const MDNode *TBAAInfo) { 4677245431Sdim assert(Chain.getValueType() == MVT::Other && 4678224145Sdim "Invalid chain type"); 4679198090Srdivacky if (Alignment == 0) // Ensure that codegen never sees alignment 0 4680198090Srdivacky Alignment = getEVTAlignment(SVT); 4681193323Sed 4682198090Srdivacky unsigned Flags = MachineMemOperand::MOStore; 4683198090Srdivacky if (isVolatile) 4684198090Srdivacky Flags |= MachineMemOperand::MOVolatile; 4685203954Srdivacky if (isNonTemporal) 4686203954Srdivacky Flags |= MachineMemOperand::MONonTemporal; 4687218893Sdim 4688218893Sdim if (PtrInfo.V == 0) 4689218893Sdim PtrInfo = InferPointerInfo(Ptr); 4690218893Sdim 4691218893Sdim MachineFunction &MF = getMachineFunction(); 4692198090Srdivacky MachineMemOperand *MMO = 4693218893Sdim MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4694218893Sdim TBAAInfo); 4695198090Srdivacky 4696198090Srdivacky return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4697198090Srdivacky} 4698198090Srdivacky 4699263509SdimSDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4700198090Srdivacky SDValue Ptr, EVT SVT, 4701198090Srdivacky MachineMemOperand *MMO) { 4702198090Srdivacky EVT VT = Val.getValueType(); 4703198090Srdivacky 4704245431Sdim assert(Chain.getValueType() == MVT::Other && 4705224145Sdim "Invalid chain type"); 4706193323Sed if (VT == SVT) 4707198090Srdivacky return getStore(Chain, dl, Val, Ptr, MMO); 4708193323Sed 4709200581Srdivacky assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4710200581Srdivacky "Should only be a truncating store, not extending!"); 4711193323Sed assert(VT.isInteger() == SVT.isInteger() && 4712193323Sed "Can't do FP-INT conversion!"); 4713200581Srdivacky assert(VT.isVector() == SVT.isVector() && 4714200581Srdivacky "Cannot use trunc store to convert to or from a vector!"); 4715200581Srdivacky assert((!VT.isVector() || 4716200581Srdivacky VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4717200581Srdivacky "Cannot use trunc store to change the number of vector elements!"); 4718193323Sed 4719193323Sed SDVTList VTs = getVTList(MVT::Other); 4720193323Sed SDValue Undef = getUNDEF(Ptr.getValueType()); 4721193323Sed SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4722193323Sed FoldingSetNodeID ID; 4723193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4724193323Sed ID.AddInteger(SVT.getRawBits()); 4725204642Srdivacky ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4726235633Sdim MMO->isNonTemporal(), MMO->isInvariant())); 4727245431Sdim ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4728193323Sed void *IP = 0; 4729198090Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4730198090Srdivacky cast<StoreSDNode>(E)->refineAlignment(MMO); 4731193323Sed return SDValue(E, 0); 4732198090Srdivacky } 4733263509Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4734263509Sdim dl.getDebugLoc(), VTs, 4735263509Sdim ISD::UNINDEXED, true, SVT, MMO); 4736193323Sed CSEMap.InsertNode(N, IP); 4737193323Sed AllNodes.push_back(N); 4738193323Sed return SDValue(N, 0); 4739193323Sed} 4740193323Sed 4741193323SedSDValue 4742263509SdimSelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, 4743193323Sed SDValue Offset, ISD::MemIndexedMode AM) { 4744193323Sed StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4745193323Sed assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4746193323Sed "Store is already a indexed store!"); 4747193323Sed SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4748193323Sed SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4749193323Sed FoldingSetNodeID ID; 4750193323Sed AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4751193323Sed ID.AddInteger(ST->getMemoryVT().getRawBits()); 4752193323Sed ID.AddInteger(ST->getRawSubclassData()); 4753245431Sdim ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 4754193323Sed void *IP = 0; 4755201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4756193323Sed return SDValue(E, 0); 4757201360Srdivacky 4758263509Sdim SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4759263509Sdim dl.getDebugLoc(), VTs, AM, 4760205407Srdivacky ST->isTruncatingStore(), 4761205407Srdivacky ST->getMemoryVT(), 4762205407Srdivacky ST->getMemOperand()); 4763193323Sed CSEMap.InsertNode(N, IP); 4764193323Sed AllNodes.push_back(N); 4765193323Sed return SDValue(N, 0); 4766193323Sed} 4767193323Sed 4768263509SdimSDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl, 4769193323Sed SDValue Chain, SDValue Ptr, 4770210299Sed SDValue SV, 4771210299Sed unsigned Align) { 4772210299Sed SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4773210299Sed return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4774193323Sed} 4775193323Sed 4776263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4777193323Sed const SDUse *Ops, unsigned NumOps) { 4778193323Sed switch (NumOps) { 4779193323Sed case 0: return getNode(Opcode, DL, VT); 4780193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4781193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4782193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4783193323Sed default: break; 4784193323Sed } 4785193323Sed 4786193323Sed // Copy from an SDUse array into an SDValue array for use with 4787193323Sed // the regular getNode logic. 4788193323Sed SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4789193323Sed return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4790193323Sed} 4791193323Sed 4792263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4793193323Sed const SDValue *Ops, unsigned NumOps) { 4794193323Sed switch (NumOps) { 4795193323Sed case 0: return getNode(Opcode, DL, VT); 4796193323Sed case 1: return getNode(Opcode, DL, VT, Ops[0]); 4797193323Sed case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4798193323Sed case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4799193323Sed default: break; 4800193323Sed } 4801193323Sed 4802193323Sed switch (Opcode) { 4803193323Sed default: break; 4804193323Sed case ISD::SELECT_CC: { 4805193323Sed assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4806193323Sed assert(Ops[0].getValueType() == Ops[1].getValueType() && 4807193323Sed "LHS and RHS of condition must have same type!"); 4808193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4809193323Sed "True and False arms of SelectCC must have same type!"); 4810193323Sed assert(Ops[2].getValueType() == VT && 4811193323Sed "select_cc node must be of same type as true and false value!"); 4812193323Sed break; 4813193323Sed } 4814193323Sed case ISD::BR_CC: { 4815193323Sed assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4816193323Sed assert(Ops[2].getValueType() == Ops[3].getValueType() && 4817193323Sed "LHS/RHS of comparison should match types!"); 4818193323Sed break; 4819193323Sed } 4820193323Sed } 4821193323Sed 4822193323Sed // Memoize nodes. 4823193323Sed SDNode *N; 4824193323Sed SDVTList VTs = getVTList(VT); 4825193323Sed 4826218893Sdim if (VT != MVT::Glue) { 4827193323Sed FoldingSetNodeID ID; 4828193323Sed AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4829193323Sed void *IP = 0; 4830193323Sed 4831201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4832193323Sed return SDValue(E, 0); 4833193323Sed 4834263509Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4835263509Sdim VTs, Ops, NumOps); 4836193323Sed CSEMap.InsertNode(N, IP); 4837193323Sed } else { 4838263509Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4839263509Sdim VTs, Ops, NumOps); 4840193323Sed } 4841193323Sed 4842193323Sed AllNodes.push_back(N); 4843193323Sed#ifndef NDEBUG 4844218893Sdim VerifySDNode(N); 4845193323Sed#endif 4846193323Sed return SDValue(N, 0); 4847193323Sed} 4848193323Sed 4849263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4850252723Sdim ArrayRef<EVT> ResultTys, 4851193323Sed const SDValue *Ops, unsigned NumOps) { 4852193323Sed return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4853193323Sed Ops, NumOps); 4854193323Sed} 4855193323Sed 4856263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4857198090Srdivacky const EVT *VTs, unsigned NumVTs, 4858193323Sed const SDValue *Ops, unsigned NumOps) { 4859193323Sed if (NumVTs == 1) 4860193323Sed return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4861193323Sed return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4862193323Sed} 4863193323Sed 4864263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4865193323Sed const SDValue *Ops, unsigned NumOps) { 4866193323Sed if (VTList.NumVTs == 1) 4867193323Sed return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4868193323Sed 4869198090Srdivacky#if 0 4870193323Sed switch (Opcode) { 4871193323Sed // FIXME: figure out how to safely handle things like 4872193323Sed // int foo(int x) { return 1 << (x & 255); } 4873193323Sed // int bar() { return foo(256); } 4874193323Sed case ISD::SRA_PARTS: 4875193323Sed case ISD::SRL_PARTS: 4876193323Sed case ISD::SHL_PARTS: 4877193323Sed if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4878193323Sed cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4879193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4880193323Sed else if (N3.getOpcode() == ISD::AND) 4881193323Sed if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4882193323Sed // If the and is only masking out bits that cannot effect the shift, 4883193323Sed // eliminate the and. 4884202375Srdivacky unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4885193323Sed if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4886193323Sed return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4887193323Sed } 4888193323Sed break; 4889198090Srdivacky } 4890193323Sed#endif 4891193323Sed 4892193323Sed // Memoize the node unless it returns a flag. 4893193323Sed SDNode *N; 4894218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4895193323Sed FoldingSetNodeID ID; 4896193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4897193323Sed void *IP = 0; 4898201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4899193323Sed return SDValue(E, 0); 4900201360Srdivacky 4901193323Sed if (NumOps == 1) { 4902263509Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4903263509Sdim DL.getDebugLoc(), VTList, Ops[0]); 4904193323Sed } else if (NumOps == 2) { 4905263509Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4906263509Sdim DL.getDebugLoc(), VTList, Ops[0], 4907263509Sdim Ops[1]); 4908193323Sed } else if (NumOps == 3) { 4909263509Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4910263509Sdim DL.getDebugLoc(), VTList, Ops[0], 4911263509Sdim Ops[1], Ops[2]); 4912193323Sed } else { 4913263509Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4914263509Sdim VTList, Ops, NumOps); 4915193323Sed } 4916193323Sed CSEMap.InsertNode(N, IP); 4917193323Sed } else { 4918193323Sed if (NumOps == 1) { 4919263509Sdim N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4920263509Sdim DL.getDebugLoc(), VTList, Ops[0]); 4921193323Sed } else if (NumOps == 2) { 4922263509Sdim N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4923263509Sdim DL.getDebugLoc(), VTList, Ops[0], 4924263509Sdim Ops[1]); 4925193323Sed } else if (NumOps == 3) { 4926263509Sdim N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4927263509Sdim DL.getDebugLoc(), VTList, Ops[0], 4928263509Sdim Ops[1], Ops[2]); 4929193323Sed } else { 4930263509Sdim N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4931263509Sdim VTList, Ops, NumOps); 4932193323Sed } 4933193323Sed } 4934193323Sed AllNodes.push_back(N); 4935193323Sed#ifndef NDEBUG 4936218893Sdim VerifySDNode(N); 4937193323Sed#endif 4938193323Sed return SDValue(N, 0); 4939193323Sed} 4940193323Sed 4941263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) { 4942193323Sed return getNode(Opcode, DL, VTList, 0, 0); 4943193323Sed} 4944193323Sed 4945263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4946193323Sed SDValue N1) { 4947193323Sed SDValue Ops[] = { N1 }; 4948193323Sed return getNode(Opcode, DL, VTList, Ops, 1); 4949193323Sed} 4950193323Sed 4951263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4952193323Sed SDValue N1, SDValue N2) { 4953193323Sed SDValue Ops[] = { N1, N2 }; 4954193323Sed return getNode(Opcode, DL, VTList, Ops, 2); 4955193323Sed} 4956193323Sed 4957263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4958193323Sed SDValue N1, SDValue N2, SDValue N3) { 4959193323Sed SDValue Ops[] = { N1, N2, N3 }; 4960193323Sed return getNode(Opcode, DL, VTList, Ops, 3); 4961193323Sed} 4962193323Sed 4963263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4964193323Sed SDValue N1, SDValue N2, SDValue N3, 4965193323Sed SDValue N4) { 4966193323Sed SDValue Ops[] = { N1, N2, N3, N4 }; 4967193323Sed return getNode(Opcode, DL, VTList, Ops, 4); 4968193323Sed} 4969193323Sed 4970263509SdimSDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4971193323Sed SDValue N1, SDValue N2, SDValue N3, 4972193323Sed SDValue N4, SDValue N5) { 4973193323Sed SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4974193323Sed return getNode(Opcode, DL, VTList, Ops, 5); 4975193323Sed} 4976193323Sed 4977198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT) { 4978193323Sed return makeVTList(SDNode::getValueTypeList(VT), 1); 4979193323Sed} 4980193323Sed 4981198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4982263509Sdim FoldingSetNodeID ID; 4983263509Sdim ID.AddInteger(2U); 4984263509Sdim ID.AddInteger(VT1.getRawBits()); 4985263509Sdim ID.AddInteger(VT2.getRawBits()); 4986193323Sed 4987263509Sdim void *IP = 0; 4988263509Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4989263509Sdim if (Result == NULL) { 4990263509Sdim EVT *Array = Allocator.Allocate<EVT>(2); 4991263509Sdim Array[0] = VT1; 4992263509Sdim Array[1] = VT2; 4993263509Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 4994263509Sdim VTListMap.InsertNode(Result, IP); 4995263509Sdim } 4996263509Sdim return Result->getSDVTList(); 4997193323Sed} 4998193323Sed 4999198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 5000263509Sdim FoldingSetNodeID ID; 5001263509Sdim ID.AddInteger(3U); 5002263509Sdim ID.AddInteger(VT1.getRawBits()); 5003263509Sdim ID.AddInteger(VT2.getRawBits()); 5004263509Sdim ID.AddInteger(VT3.getRawBits()); 5005193323Sed 5006263509Sdim void *IP = 0; 5007263509Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5008263509Sdim if (Result == NULL) { 5009263509Sdim EVT *Array = Allocator.Allocate<EVT>(3); 5010263509Sdim Array[0] = VT1; 5011263509Sdim Array[1] = VT2; 5012263509Sdim Array[2] = VT3; 5013263509Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 5014263509Sdim VTListMap.InsertNode(Result, IP); 5015263509Sdim } 5016263509Sdim return Result->getSDVTList(); 5017193323Sed} 5018193323Sed 5019198090SrdivackySDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 5020263509Sdim FoldingSetNodeID ID; 5021263509Sdim ID.AddInteger(4U); 5022263509Sdim ID.AddInteger(VT1.getRawBits()); 5023263509Sdim ID.AddInteger(VT2.getRawBits()); 5024263509Sdim ID.AddInteger(VT3.getRawBits()); 5025263509Sdim ID.AddInteger(VT4.getRawBits()); 5026193323Sed 5027263509Sdim void *IP = 0; 5028263509Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5029263509Sdim if (Result == NULL) { 5030263509Sdim EVT *Array = Allocator.Allocate<EVT>(4); 5031263509Sdim Array[0] = VT1; 5032263509Sdim Array[1] = VT2; 5033263509Sdim Array[2] = VT3; 5034263509Sdim Array[3] = VT4; 5035263509Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 5036263509Sdim VTListMap.InsertNode(Result, IP); 5037263509Sdim } 5038263509Sdim return Result->getSDVTList(); 5039193323Sed} 5040193323Sed 5041198090SrdivackySDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 5042263509Sdim FoldingSetNodeID ID; 5043263509Sdim ID.AddInteger(NumVTs); 5044263509Sdim for (unsigned index = 0; index < NumVTs; index++) { 5045263509Sdim ID.AddInteger(VTs[index].getRawBits()); 5046193323Sed } 5047193323Sed 5048263509Sdim void *IP = 0; 5049263509Sdim SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 5050263509Sdim if (Result == NULL) { 5051263509Sdim EVT *Array = Allocator.Allocate<EVT>(NumVTs); 5052263509Sdim std::copy(VTs, VTs + NumVTs, Array); 5053263509Sdim Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 5054263509Sdim VTListMap.InsertNode(Result, IP); 5055193323Sed } 5056263509Sdim return Result->getSDVTList(); 5057193323Sed} 5058193323Sed 5059193323Sed 5060193323Sed/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 5061193323Sed/// specified operands. If the resultant node already exists in the DAG, 5062193323Sed/// this does not modify the specified node, instead it returns the node that 5063193323Sed/// already exists. If the resultant node does not exist in the DAG, the 5064193323Sed/// input node is returned. As a degenerate case, if you specify the same 5065193323Sed/// input operands as the node already has, the input node is returned. 5066210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 5067193323Sed assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 5068193323Sed 5069193323Sed // Check to see if there is no change. 5070210299Sed if (Op == N->getOperand(0)) return N; 5071193323Sed 5072193323Sed // See if the modified node already exists. 5073193323Sed void *InsertPos = 0; 5074193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 5075210299Sed return Existing; 5076193323Sed 5077193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5078193323Sed if (InsertPos) 5079193323Sed if (!RemoveNodeFromCSEMaps(N)) 5080193323Sed InsertPos = 0; 5081193323Sed 5082193323Sed // Now we update the operands. 5083193323Sed N->OperandList[0].set(Op); 5084193323Sed 5085193323Sed // If this gets put into a CSE map, add it. 5086193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5087210299Sed return N; 5088193323Sed} 5089193323Sed 5090210299SedSDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 5091193323Sed assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 5092193323Sed 5093193323Sed // Check to see if there is no change. 5094193323Sed if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 5095210299Sed return N; // No operands changed, just return the input node. 5096193323Sed 5097193323Sed // See if the modified node already exists. 5098193323Sed void *InsertPos = 0; 5099193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 5100210299Sed return Existing; 5101193323Sed 5102193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5103193323Sed if (InsertPos) 5104193323Sed if (!RemoveNodeFromCSEMaps(N)) 5105193323Sed InsertPos = 0; 5106193323Sed 5107193323Sed // Now we update the operands. 5108193323Sed if (N->OperandList[0] != Op1) 5109193323Sed N->OperandList[0].set(Op1); 5110193323Sed if (N->OperandList[1] != Op2) 5111193323Sed N->OperandList[1].set(Op2); 5112193323Sed 5113193323Sed // If this gets put into a CSE map, add it. 5114193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5115210299Sed return N; 5116193323Sed} 5117193323Sed 5118210299SedSDNode *SelectionDAG:: 5119210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 5120193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5121193323Sed return UpdateNodeOperands(N, Ops, 3); 5122193323Sed} 5123193323Sed 5124210299SedSDNode *SelectionDAG:: 5125210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5126193323Sed SDValue Op3, SDValue Op4) { 5127193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 5128193323Sed return UpdateNodeOperands(N, Ops, 4); 5129193323Sed} 5130193323Sed 5131210299SedSDNode *SelectionDAG:: 5132210299SedUpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5133193323Sed SDValue Op3, SDValue Op4, SDValue Op5) { 5134193323Sed SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 5135193323Sed return UpdateNodeOperands(N, Ops, 5); 5136193323Sed} 5137193323Sed 5138210299SedSDNode *SelectionDAG:: 5139210299SedUpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 5140193323Sed assert(N->getNumOperands() == NumOps && 5141193323Sed "Update with wrong number of operands"); 5142193323Sed 5143193323Sed // Check to see if there is no change. 5144193323Sed bool AnyChange = false; 5145193323Sed for (unsigned i = 0; i != NumOps; ++i) { 5146193323Sed if (Ops[i] != N->getOperand(i)) { 5147193323Sed AnyChange = true; 5148193323Sed break; 5149193323Sed } 5150193323Sed } 5151193323Sed 5152193323Sed // No operands changed, just return the input node. 5153210299Sed if (!AnyChange) return N; 5154193323Sed 5155193323Sed // See if the modified node already exists. 5156193323Sed void *InsertPos = 0; 5157193323Sed if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 5158210299Sed return Existing; 5159193323Sed 5160193323Sed // Nope it doesn't. Remove the node from its current place in the maps. 5161193323Sed if (InsertPos) 5162193323Sed if (!RemoveNodeFromCSEMaps(N)) 5163193323Sed InsertPos = 0; 5164193323Sed 5165193323Sed // Now we update the operands. 5166193323Sed for (unsigned i = 0; i != NumOps; ++i) 5167193323Sed if (N->OperandList[i] != Ops[i]) 5168193323Sed N->OperandList[i].set(Ops[i]); 5169193323Sed 5170193323Sed // If this gets put into a CSE map, add it. 5171193323Sed if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5172210299Sed return N; 5173193323Sed} 5174193323Sed 5175193323Sed/// DropOperands - Release the operands and set this node to have 5176193323Sed/// zero operands. 5177193323Sedvoid SDNode::DropOperands() { 5178193323Sed // Unlike the code in MorphNodeTo that does this, we don't need to 5179193323Sed // watch for dead nodes here. 5180193323Sed for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 5181193323Sed SDUse &Use = *I++; 5182193323Sed Use.set(SDValue()); 5183193323Sed } 5184193323Sed} 5185193323Sed 5186193323Sed/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 5187193323Sed/// machine opcode. 5188193323Sed/// 5189193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5190198090Srdivacky EVT VT) { 5191193323Sed SDVTList VTs = getVTList(VT); 5192193323Sed return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 5193193323Sed} 5194193323Sed 5195193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5196198090Srdivacky EVT VT, SDValue Op1) { 5197193323Sed SDVTList VTs = getVTList(VT); 5198193323Sed SDValue Ops[] = { Op1 }; 5199193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5200193323Sed} 5201193323Sed 5202193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5203198090Srdivacky EVT VT, SDValue Op1, 5204193323Sed SDValue Op2) { 5205193323Sed SDVTList VTs = getVTList(VT); 5206193323Sed SDValue Ops[] = { Op1, Op2 }; 5207193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5208193323Sed} 5209193323Sed 5210193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5211198090Srdivacky EVT VT, SDValue Op1, 5212193323Sed SDValue Op2, SDValue Op3) { 5213193323Sed SDVTList VTs = getVTList(VT); 5214193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5215193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5216193323Sed} 5217193323Sed 5218193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5219198090Srdivacky EVT VT, const SDValue *Ops, 5220193323Sed unsigned NumOps) { 5221193323Sed SDVTList VTs = getVTList(VT); 5222193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5223193323Sed} 5224193323Sed 5225193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5226198090Srdivacky EVT VT1, EVT VT2, const SDValue *Ops, 5227193323Sed unsigned NumOps) { 5228193323Sed SDVTList VTs = getVTList(VT1, VT2); 5229193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5230193323Sed} 5231193323Sed 5232193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5233198090Srdivacky EVT VT1, EVT VT2) { 5234193323Sed SDVTList VTs = getVTList(VT1, VT2); 5235193323Sed return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 5236193323Sed} 5237193323Sed 5238193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5239198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5240193323Sed const SDValue *Ops, unsigned NumOps) { 5241193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5242193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5243193323Sed} 5244193323Sed 5245193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5246198090Srdivacky EVT VT1, EVT VT2, EVT VT3, EVT VT4, 5247193323Sed const SDValue *Ops, unsigned NumOps) { 5248193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5249193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5250193323Sed} 5251193323Sed 5252193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5253198090Srdivacky EVT VT1, EVT VT2, 5254193323Sed SDValue Op1) { 5255193323Sed SDVTList VTs = getVTList(VT1, VT2); 5256193323Sed SDValue Ops[] = { Op1 }; 5257193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5258193323Sed} 5259193323Sed 5260193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5261198090Srdivacky EVT VT1, EVT VT2, 5262193323Sed SDValue Op1, SDValue Op2) { 5263193323Sed SDVTList VTs = getVTList(VT1, VT2); 5264193323Sed SDValue Ops[] = { Op1, Op2 }; 5265193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5266193323Sed} 5267193323Sed 5268193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5269198090Srdivacky EVT VT1, EVT VT2, 5270193323Sed SDValue Op1, SDValue Op2, 5271193323Sed SDValue Op3) { 5272193323Sed SDVTList VTs = getVTList(VT1, VT2); 5273193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5274193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5275193323Sed} 5276193323Sed 5277193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5278198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5279193323Sed SDValue Op1, SDValue Op2, 5280193323Sed SDValue Op3) { 5281193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5282193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5283193323Sed return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5284193323Sed} 5285193323Sed 5286193323SedSDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5287193323Sed SDVTList VTs, const SDValue *Ops, 5288193323Sed unsigned NumOps) { 5289204642Srdivacky N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 5290204642Srdivacky // Reset the NodeID to -1. 5291204642Srdivacky N->setNodeId(-1); 5292204642Srdivacky return N; 5293193323Sed} 5294193323Sed 5295263509Sdim/// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away 5296235633Sdim/// the line number information on the merged node since it is not possible to 5297235633Sdim/// preserve the information that operation is associated with multiple lines. 5298235633Sdim/// This will make the debugger working better at -O0, were there is a higher 5299235633Sdim/// probability having other instructions associated with that line. 5300235633Sdim/// 5301263509Sdim/// For IROrder, we keep the smaller of the two 5302263509SdimSDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { 5303235633Sdim DebugLoc NLoc = N->getDebugLoc(); 5304263509Sdim if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && 5305263509Sdim (OLoc.getDebugLoc() != NLoc)) { 5306235633Sdim N->setDebugLoc(DebugLoc()); 5307235633Sdim } 5308263509Sdim unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 5309263509Sdim N->setIROrder(Order); 5310235633Sdim return N; 5311235633Sdim} 5312235633Sdim 5313204642Srdivacky/// MorphNodeTo - This *mutates* the specified node to have the specified 5314193323Sed/// return type, opcode, and operands. 5315193323Sed/// 5316193323Sed/// Note that MorphNodeTo returns the resultant node. If there is already a 5317193323Sed/// node of the specified opcode and operands, it returns that node instead of 5318263509Sdim/// the current one. Note that the SDLoc need not be the same. 5319193323Sed/// 5320193323Sed/// Using MorphNodeTo is faster than creating a new node and swapping it in 5321193323Sed/// with ReplaceAllUsesWith both because it often avoids allocating a new 5322193323Sed/// node, and because it doesn't require CSE recalculation for any of 5323193323Sed/// the node's users. 5324193323Sed/// 5325193323SedSDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 5326193323Sed SDVTList VTs, const SDValue *Ops, 5327193323Sed unsigned NumOps) { 5328193323Sed // If an identical node already exists, use it. 5329193323Sed void *IP = 0; 5330218893Sdim if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 5331193323Sed FoldingSetNodeID ID; 5332193323Sed AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 5333201360Srdivacky if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 5334263509Sdim return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N)); 5335193323Sed } 5336193323Sed 5337193323Sed if (!RemoveNodeFromCSEMaps(N)) 5338193323Sed IP = 0; 5339193323Sed 5340193323Sed // Start the morphing. 5341193323Sed N->NodeType = Opc; 5342193323Sed N->ValueList = VTs.VTs; 5343193323Sed N->NumValues = VTs.NumVTs; 5344193323Sed 5345193323Sed // Clear the operands list, updating used nodes to remove this from their 5346193323Sed // use list. Keep track of any operands that become dead as a result. 5347193323Sed SmallPtrSet<SDNode*, 16> DeadNodeSet; 5348193323Sed for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 5349193323Sed SDUse &Use = *I++; 5350193323Sed SDNode *Used = Use.getNode(); 5351193323Sed Use.set(SDValue()); 5352193323Sed if (Used->use_empty()) 5353193323Sed DeadNodeSet.insert(Used); 5354193323Sed } 5355193323Sed 5356198090Srdivacky if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 5357198090Srdivacky // Initialize the memory references information. 5358198090Srdivacky MN->setMemRefs(0, 0); 5359198090Srdivacky // If NumOps is larger than the # of operands we can have in a 5360198090Srdivacky // MachineSDNode, reallocate the operand list. 5361198090Srdivacky if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 5362198090Srdivacky if (MN->OperandsNeedDelete) 5363198090Srdivacky delete[] MN->OperandList; 5364198090Srdivacky if (NumOps > array_lengthof(MN->LocalOperands)) 5365198090Srdivacky // We're creating a final node that will live unmorphed for the 5366198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5367198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5368198090Srdivacky MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5369205407Srdivacky Ops, NumOps); 5370198090Srdivacky else 5371198090Srdivacky MN->InitOperands(MN->LocalOperands, Ops, NumOps); 5372198090Srdivacky MN->OperandsNeedDelete = false; 5373198090Srdivacky } else 5374198090Srdivacky MN->InitOperands(MN->OperandList, Ops, NumOps); 5375198090Srdivacky } else { 5376198090Srdivacky // If NumOps is larger than the # of operands we currently have, reallocate 5377198090Srdivacky // the operand list. 5378198090Srdivacky if (NumOps > N->NumOperands) { 5379198090Srdivacky if (N->OperandsNeedDelete) 5380198090Srdivacky delete[] N->OperandList; 5381198090Srdivacky N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5382193323Sed N->OperandsNeedDelete = true; 5383198090Srdivacky } else 5384198396Srdivacky N->InitOperands(N->OperandList, Ops, NumOps); 5385193323Sed } 5386193323Sed 5387193323Sed // Delete any nodes that are still dead after adding the uses for the 5388193323Sed // new operands. 5389204642Srdivacky if (!DeadNodeSet.empty()) { 5390204642Srdivacky SmallVector<SDNode *, 16> DeadNodes; 5391204642Srdivacky for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5392204642Srdivacky E = DeadNodeSet.end(); I != E; ++I) 5393204642Srdivacky if ((*I)->use_empty()) 5394204642Srdivacky DeadNodes.push_back(*I); 5395204642Srdivacky RemoveDeadNodes(DeadNodes); 5396204642Srdivacky } 5397193323Sed 5398193323Sed if (IP) 5399193323Sed CSEMap.InsertNode(N, IP); // Memoize the new node. 5400193323Sed return N; 5401193323Sed} 5402193323Sed 5403193323Sed 5404198090Srdivacky/// getMachineNode - These are used for target selectors to create a new node 5405198090Srdivacky/// with specified return type(s), MachineInstr opcode, and operands. 5406193323Sed/// 5407198090Srdivacky/// Note that getMachineNode returns the resultant node. If there is already a 5408193323Sed/// node of the specified opcode and operands, it returns that node instead of 5409193323Sed/// the current one. 5410198090SrdivackyMachineSDNode * 5411263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) { 5412198090Srdivacky SDVTList VTs = getVTList(VT); 5413252723Sdim return getMachineNode(Opcode, dl, VTs, None); 5414193323Sed} 5415193323Sed 5416198090SrdivackyMachineSDNode * 5417263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) { 5418198090Srdivacky SDVTList VTs = getVTList(VT); 5419198090Srdivacky SDValue Ops[] = { Op1 }; 5420252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5421193323Sed} 5422193323Sed 5423198090SrdivackyMachineSDNode * 5424263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5425198090Srdivacky SDValue Op1, SDValue Op2) { 5426198090Srdivacky SDVTList VTs = getVTList(VT); 5427198090Srdivacky SDValue Ops[] = { Op1, Op2 }; 5428252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5429193323Sed} 5430193323Sed 5431198090SrdivackyMachineSDNode * 5432263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5433198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5434198090Srdivacky SDVTList VTs = getVTList(VT); 5435198090Srdivacky SDValue Ops[] = { Op1, Op2, Op3 }; 5436252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5437193323Sed} 5438193323Sed 5439198090SrdivackyMachineSDNode * 5440263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5441252723Sdim ArrayRef<SDValue> Ops) { 5442198090Srdivacky SDVTList VTs = getVTList(VT); 5443252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5444193323Sed} 5445193323Sed 5446198090SrdivackyMachineSDNode * 5447263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) { 5448193323Sed SDVTList VTs = getVTList(VT1, VT2); 5449252723Sdim return getMachineNode(Opcode, dl, VTs, None); 5450193323Sed} 5451193323Sed 5452198090SrdivackyMachineSDNode * 5453263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5454198090Srdivacky EVT VT1, EVT VT2, SDValue Op1) { 5455193323Sed SDVTList VTs = getVTList(VT1, VT2); 5456198090Srdivacky SDValue Ops[] = { Op1 }; 5457252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5458193323Sed} 5459193323Sed 5460198090SrdivackyMachineSDNode * 5461263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5462198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5463193323Sed SDVTList VTs = getVTList(VT1, VT2); 5464193323Sed SDValue Ops[] = { Op1, Op2 }; 5465252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5466193323Sed} 5467193323Sed 5468198090SrdivackyMachineSDNode * 5469263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5470198090Srdivacky EVT VT1, EVT VT2, SDValue Op1, 5471198090Srdivacky SDValue Op2, SDValue Op3) { 5472193323Sed SDVTList VTs = getVTList(VT1, VT2); 5473193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5474252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5475193323Sed} 5476193323Sed 5477198090SrdivackyMachineSDNode * 5478263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5479198090Srdivacky EVT VT1, EVT VT2, 5480252723Sdim ArrayRef<SDValue> Ops) { 5481193323Sed SDVTList VTs = getVTList(VT1, VT2); 5482252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5483193323Sed} 5484193323Sed 5485198090SrdivackyMachineSDNode * 5486263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5487198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5488198090Srdivacky SDValue Op1, SDValue Op2) { 5489193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5490193323Sed SDValue Ops[] = { Op1, Op2 }; 5491252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5492193323Sed} 5493193323Sed 5494198090SrdivackyMachineSDNode * 5495263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5496198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5497198090Srdivacky SDValue Op1, SDValue Op2, SDValue Op3) { 5498193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5499193323Sed SDValue Ops[] = { Op1, Op2, Op3 }; 5500252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5501193323Sed} 5502193323Sed 5503198090SrdivackyMachineSDNode * 5504263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5505198090Srdivacky EVT VT1, EVT VT2, EVT VT3, 5506252723Sdim ArrayRef<SDValue> Ops) { 5507193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3); 5508252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5509193323Sed} 5510193323Sed 5511198090SrdivackyMachineSDNode * 5512263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, 5513198090Srdivacky EVT VT2, EVT VT3, EVT VT4, 5514252723Sdim ArrayRef<SDValue> Ops) { 5515193323Sed SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5516252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5517193323Sed} 5518193323Sed 5519198090SrdivackyMachineSDNode * 5520263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5521252723Sdim ArrayRef<EVT> ResultTys, 5522252723Sdim ArrayRef<SDValue> Ops) { 5523198090Srdivacky SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5524252723Sdim return getMachineNode(Opcode, dl, VTs, Ops); 5525193323Sed} 5526193323Sed 5527198090SrdivackyMachineSDNode * 5528263509SdimSelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, 5529252723Sdim ArrayRef<SDValue> OpsArray) { 5530218893Sdim bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5531198090Srdivacky MachineSDNode *N; 5532218893Sdim void *IP = 0; 5533252723Sdim const SDValue *Ops = OpsArray.data(); 5534252723Sdim unsigned NumOps = OpsArray.size(); 5535198090Srdivacky 5536198090Srdivacky if (DoCSE) { 5537198090Srdivacky FoldingSetNodeID ID; 5538198090Srdivacky AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5539198090Srdivacky IP = 0; 5540235633Sdim if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5541263509Sdim return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); 5542235633Sdim } 5543198090Srdivacky } 5544198090Srdivacky 5545198090Srdivacky // Allocate a new MachineSDNode. 5546263509Sdim N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(), 5547263509Sdim DL.getDebugLoc(), VTs); 5548198090Srdivacky 5549198090Srdivacky // Initialize the operands list. 5550198090Srdivacky if (NumOps > array_lengthof(N->LocalOperands)) 5551198090Srdivacky // We're creating a final node that will live unmorphed for the 5552198090Srdivacky // remainder of the current SelectionDAG iteration, so we can allocate 5553198090Srdivacky // the operands directly out of a pool with no recycling metadata. 5554198090Srdivacky N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5555198090Srdivacky Ops, NumOps); 5556198090Srdivacky else 5557198090Srdivacky N->InitOperands(N->LocalOperands, Ops, NumOps); 5558198090Srdivacky N->OperandsNeedDelete = false; 5559198090Srdivacky 5560198090Srdivacky if (DoCSE) 5561198090Srdivacky CSEMap.InsertNode(N, IP); 5562198090Srdivacky 5563198090Srdivacky AllNodes.push_back(N); 5564198090Srdivacky#ifndef NDEBUG 5565218893Sdim VerifyMachineNode(N); 5566198090Srdivacky#endif 5567198090Srdivacky return N; 5568198090Srdivacky} 5569198090Srdivacky 5570198090Srdivacky/// getTargetExtractSubreg - A convenience function for creating 5571203954Srdivacky/// TargetOpcode::EXTRACT_SUBREG nodes. 5572198090SrdivackySDValue 5573263509SdimSelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, 5574198090Srdivacky SDValue Operand) { 5575198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5576203954Srdivacky SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5577198090Srdivacky VT, Operand, SRIdxVal); 5578198090Srdivacky return SDValue(Subreg, 0); 5579198090Srdivacky} 5580198090Srdivacky 5581198090Srdivacky/// getTargetInsertSubreg - A convenience function for creating 5582203954Srdivacky/// TargetOpcode::INSERT_SUBREG nodes. 5583198090SrdivackySDValue 5584263509SdimSelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, 5585198090Srdivacky SDValue Operand, SDValue Subreg) { 5586198090Srdivacky SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5587203954Srdivacky SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5588198090Srdivacky VT, Operand, Subreg, SRIdxVal); 5589198090Srdivacky return SDValue(Result, 0); 5590198090Srdivacky} 5591198090Srdivacky 5592193323Sed/// getNodeIfExists - Get the specified node if it's already available, or 5593193323Sed/// else return NULL. 5594193323SedSDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5595193323Sed const SDValue *Ops, unsigned NumOps) { 5596218893Sdim if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5597193323Sed FoldingSetNodeID ID; 5598193323Sed AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5599193323Sed void *IP = 0; 5600201360Srdivacky if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5601193323Sed return E; 5602193323Sed } 5603193323Sed return NULL; 5604193323Sed} 5605193323Sed 5606206083Srdivacky/// getDbgValue - Creates a SDDbgValue node. 5607206083Srdivacky/// 5608206083SrdivackySDDbgValue * 5609206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5610206083Srdivacky DebugLoc DL, unsigned O) { 5611206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5612206083Srdivacky} 5613206083Srdivacky 5614206083SrdivackySDDbgValue * 5615207618SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5616206083Srdivacky DebugLoc DL, unsigned O) { 5617206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5618206083Srdivacky} 5619206083Srdivacky 5620206083SrdivackySDDbgValue * 5621206083SrdivackySelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5622206083Srdivacky DebugLoc DL, unsigned O) { 5623206083Srdivacky return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5624206083Srdivacky} 5625206083Srdivacky 5626204792Srdivackynamespace { 5627204792Srdivacky 5628204792Srdivacky/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5629204792Srdivacky/// pointed to by a use iterator is deleted, increment the use iterator 5630204792Srdivacky/// so that it doesn't dangle. 5631204792Srdivacky/// 5632204792Srdivackyclass RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5633204792Srdivacky SDNode::use_iterator &UI; 5634204792Srdivacky SDNode::use_iterator &UE; 5635204792Srdivacky 5636204792Srdivacky virtual void NodeDeleted(SDNode *N, SDNode *E) { 5637204792Srdivacky // Increment the iterator as needed. 5638204792Srdivacky while (UI != UE && N == *UI) 5639204792Srdivacky ++UI; 5640204792Srdivacky } 5641204792Srdivacky 5642204792Srdivackypublic: 5643245431Sdim RAUWUpdateListener(SelectionDAG &d, 5644204792Srdivacky SDNode::use_iterator &ui, 5645204792Srdivacky SDNode::use_iterator &ue) 5646245431Sdim : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 5647204792Srdivacky}; 5648204792Srdivacky 5649204792Srdivacky} 5650204792Srdivacky 5651193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5652193323Sed/// This can cause recursive merging of nodes in the DAG. 5653193323Sed/// 5654193323Sed/// This version assumes From has a single result value. 5655193323Sed/// 5656245431Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 5657193323Sed SDNode *From = FromN.getNode(); 5658193323Sed assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5659193323Sed "Cannot replace with this method!"); 5660193323Sed assert(From != To.getNode() && "Cannot replace uses of with self"); 5661193323Sed 5662193323Sed // Iterate over all the existing uses of From. New uses will be added 5663193323Sed // to the beginning of the use list, which we avoid visiting. 5664193323Sed // This specifically avoids visiting uses of From that arise while the 5665193323Sed // replacement is happening, because any such uses would be the result 5666193323Sed // of CSE: If an existing node looks like From after one of its operands 5667193323Sed // is replaced by To, we don't want to replace of all its users with To 5668193323Sed // too. See PR3018 for more info. 5669193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5670245431Sdim RAUWUpdateListener Listener(*this, UI, UE); 5671193323Sed while (UI != UE) { 5672193323Sed SDNode *User = *UI; 5673193323Sed 5674193323Sed // This node is about to morph, remove its old self from the CSE maps. 5675193323Sed RemoveNodeFromCSEMaps(User); 5676193323Sed 5677193323Sed // A user can appear in a use list multiple times, and when this 5678193323Sed // happens the uses are usually next to each other in the list. 5679193323Sed // To help reduce the number of CSE recomputations, process all 5680193323Sed // the uses of this user that we can find this way. 5681193323Sed do { 5682193323Sed SDUse &Use = UI.getUse(); 5683193323Sed ++UI; 5684193323Sed Use.set(To); 5685193323Sed } while (UI != UE && *UI == User); 5686193323Sed 5687193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5688193323Sed // already exists there, recursively merge the results together. 5689245431Sdim AddModifiedNodeToCSEMaps(User); 5690193323Sed } 5691235633Sdim 5692235633Sdim // If we just RAUW'd the root, take note. 5693235633Sdim if (FromN == getRoot()) 5694235633Sdim setRoot(To); 5695193323Sed} 5696193323Sed 5697193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5698193323Sed/// This can cause recursive merging of nodes in the DAG. 5699193323Sed/// 5700193323Sed/// This version assumes that for each value of From, there is a 5701193323Sed/// corresponding value in To in the same position with the same type. 5702193323Sed/// 5703245431Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 5704193323Sed#ifndef NDEBUG 5705193323Sed for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5706193323Sed assert((!From->hasAnyUseOfValue(i) || 5707193323Sed From->getValueType(i) == To->getValueType(i)) && 5708193323Sed "Cannot use this version of ReplaceAllUsesWith!"); 5709193323Sed#endif 5710193323Sed 5711193323Sed // Handle the trivial case. 5712193323Sed if (From == To) 5713193323Sed return; 5714193323Sed 5715193323Sed // Iterate over just the existing users of From. See the comments in 5716193323Sed // the ReplaceAllUsesWith above. 5717193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5718245431Sdim RAUWUpdateListener Listener(*this, UI, UE); 5719193323Sed while (UI != UE) { 5720193323Sed SDNode *User = *UI; 5721193323Sed 5722193323Sed // This node is about to morph, remove its old self from the CSE maps. 5723193323Sed RemoveNodeFromCSEMaps(User); 5724193323Sed 5725193323Sed // A user can appear in a use list multiple times, and when this 5726193323Sed // happens the uses are usually next to each other in the list. 5727193323Sed // To help reduce the number of CSE recomputations, process all 5728193323Sed // the uses of this user that we can find this way. 5729193323Sed do { 5730193323Sed SDUse &Use = UI.getUse(); 5731193323Sed ++UI; 5732193323Sed Use.setNode(To); 5733193323Sed } while (UI != UE && *UI == User); 5734193323Sed 5735193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5736193323Sed // already exists there, recursively merge the results together. 5737245431Sdim AddModifiedNodeToCSEMaps(User); 5738193323Sed } 5739235633Sdim 5740235633Sdim // If we just RAUW'd the root, take note. 5741235633Sdim if (From == getRoot().getNode()) 5742235633Sdim setRoot(SDValue(To, getRoot().getResNo())); 5743193323Sed} 5744193323Sed 5745193323Sed/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5746193323Sed/// This can cause recursive merging of nodes in the DAG. 5747193323Sed/// 5748193323Sed/// This version can replace From with any result values. To must match the 5749193323Sed/// number and types of values returned by From. 5750245431Sdimvoid SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 5751193323Sed if (From->getNumValues() == 1) // Handle the simple case efficiently. 5752245431Sdim return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 5753193323Sed 5754193323Sed // Iterate over just the existing users of From. See the comments in 5755193323Sed // the ReplaceAllUsesWith above. 5756193323Sed SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5757245431Sdim RAUWUpdateListener Listener(*this, UI, UE); 5758193323Sed while (UI != UE) { 5759193323Sed SDNode *User = *UI; 5760193323Sed 5761193323Sed // This node is about to morph, remove its old self from the CSE maps. 5762193323Sed RemoveNodeFromCSEMaps(User); 5763193323Sed 5764193323Sed // A user can appear in a use list multiple times, and when this 5765193323Sed // happens the uses are usually next to each other in the list. 5766193323Sed // To help reduce the number of CSE recomputations, process all 5767193323Sed // the uses of this user that we can find this way. 5768193323Sed do { 5769193323Sed SDUse &Use = UI.getUse(); 5770193323Sed const SDValue &ToOp = To[Use.getResNo()]; 5771193323Sed ++UI; 5772193323Sed Use.set(ToOp); 5773193323Sed } while (UI != UE && *UI == User); 5774193323Sed 5775193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5776193323Sed // already exists there, recursively merge the results together. 5777245431Sdim AddModifiedNodeToCSEMaps(User); 5778193323Sed } 5779235633Sdim 5780235633Sdim // If we just RAUW'd the root, take note. 5781235633Sdim if (From == getRoot().getNode()) 5782235633Sdim setRoot(SDValue(To[getRoot().getResNo()])); 5783193323Sed} 5784193323Sed 5785193323Sed/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5786193323Sed/// uses of other values produced by From.getNode() alone. The Deleted 5787193323Sed/// vector is handled the same way as for ReplaceAllUsesWith. 5788245431Sdimvoid SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 5789193323Sed // Handle the really simple, really trivial case efficiently. 5790193323Sed if (From == To) return; 5791193323Sed 5792193323Sed // Handle the simple, trivial, case efficiently. 5793193323Sed if (From.getNode()->getNumValues() == 1) { 5794245431Sdim ReplaceAllUsesWith(From, To); 5795193323Sed return; 5796193323Sed } 5797193323Sed 5798193323Sed // Iterate over just the existing users of From. See the comments in 5799193323Sed // the ReplaceAllUsesWith above. 5800193323Sed SDNode::use_iterator UI = From.getNode()->use_begin(), 5801193323Sed UE = From.getNode()->use_end(); 5802245431Sdim RAUWUpdateListener Listener(*this, UI, UE); 5803193323Sed while (UI != UE) { 5804193323Sed SDNode *User = *UI; 5805193323Sed bool UserRemovedFromCSEMaps = false; 5806193323Sed 5807193323Sed // A user can appear in a use list multiple times, and when this 5808193323Sed // happens the uses are usually next to each other in the list. 5809193323Sed // To help reduce the number of CSE recomputations, process all 5810193323Sed // the uses of this user that we can find this way. 5811193323Sed do { 5812193323Sed SDUse &Use = UI.getUse(); 5813193323Sed 5814193323Sed // Skip uses of different values from the same node. 5815193323Sed if (Use.getResNo() != From.getResNo()) { 5816193323Sed ++UI; 5817193323Sed continue; 5818193323Sed } 5819193323Sed 5820193323Sed // If this node hasn't been modified yet, it's still in the CSE maps, 5821193323Sed // so remove its old self from the CSE maps. 5822193323Sed if (!UserRemovedFromCSEMaps) { 5823193323Sed RemoveNodeFromCSEMaps(User); 5824193323Sed UserRemovedFromCSEMaps = true; 5825193323Sed } 5826193323Sed 5827193323Sed ++UI; 5828193323Sed Use.set(To); 5829193323Sed } while (UI != UE && *UI == User); 5830193323Sed 5831193323Sed // We are iterating over all uses of the From node, so if a use 5832193323Sed // doesn't use the specific value, no changes are made. 5833193323Sed if (!UserRemovedFromCSEMaps) 5834193323Sed continue; 5835193323Sed 5836193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5837193323Sed // already exists there, recursively merge the results together. 5838245431Sdim AddModifiedNodeToCSEMaps(User); 5839193323Sed } 5840235633Sdim 5841235633Sdim // If we just RAUW'd the root, take note. 5842235633Sdim if (From == getRoot()) 5843235633Sdim setRoot(To); 5844193323Sed} 5845193323Sed 5846193323Sednamespace { 5847193323Sed /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5848193323Sed /// to record information about a use. 5849193323Sed struct UseMemo { 5850193323Sed SDNode *User; 5851193323Sed unsigned Index; 5852193323Sed SDUse *Use; 5853193323Sed }; 5854193323Sed 5855193323Sed /// operator< - Sort Memos by User. 5856193323Sed bool operator<(const UseMemo &L, const UseMemo &R) { 5857193323Sed return (intptr_t)L.User < (intptr_t)R.User; 5858193323Sed } 5859193323Sed} 5860193323Sed 5861193323Sed/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5862193323Sed/// uses of other values produced by From.getNode() alone. The same value 5863193323Sed/// may appear in both the From and To list. The Deleted vector is 5864193323Sed/// handled the same way as for ReplaceAllUsesWith. 5865193323Sedvoid SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5866193323Sed const SDValue *To, 5867245431Sdim unsigned Num){ 5868193323Sed // Handle the simple, trivial case efficiently. 5869193323Sed if (Num == 1) 5870245431Sdim return ReplaceAllUsesOfValueWith(*From, *To); 5871193323Sed 5872193323Sed // Read up all the uses and make records of them. This helps 5873193323Sed // processing new uses that are introduced during the 5874193323Sed // replacement process. 5875193323Sed SmallVector<UseMemo, 4> Uses; 5876193323Sed for (unsigned i = 0; i != Num; ++i) { 5877193323Sed unsigned FromResNo = From[i].getResNo(); 5878193323Sed SDNode *FromNode = From[i].getNode(); 5879193323Sed for (SDNode::use_iterator UI = FromNode->use_begin(), 5880193323Sed E = FromNode->use_end(); UI != E; ++UI) { 5881193323Sed SDUse &Use = UI.getUse(); 5882193323Sed if (Use.getResNo() == FromResNo) { 5883193323Sed UseMemo Memo = { *UI, i, &Use }; 5884193323Sed Uses.push_back(Memo); 5885193323Sed } 5886193323Sed } 5887193323Sed } 5888193323Sed 5889193323Sed // Sort the uses, so that all the uses from a given User are together. 5890193323Sed std::sort(Uses.begin(), Uses.end()); 5891193323Sed 5892193323Sed for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5893193323Sed UseIndex != UseIndexEnd; ) { 5894193323Sed // We know that this user uses some value of From. If it is the right 5895193323Sed // value, update it. 5896193323Sed SDNode *User = Uses[UseIndex].User; 5897193323Sed 5898193323Sed // This node is about to morph, remove its old self from the CSE maps. 5899193323Sed RemoveNodeFromCSEMaps(User); 5900193323Sed 5901193323Sed // The Uses array is sorted, so all the uses for a given User 5902193323Sed // are next to each other in the list. 5903193323Sed // To help reduce the number of CSE recomputations, process all 5904193323Sed // the uses of this user that we can find this way. 5905193323Sed do { 5906193323Sed unsigned i = Uses[UseIndex].Index; 5907193323Sed SDUse &Use = *Uses[UseIndex].Use; 5908193323Sed ++UseIndex; 5909193323Sed 5910193323Sed Use.set(To[i]); 5911193323Sed } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5912193323Sed 5913193323Sed // Now that we have modified User, add it back to the CSE maps. If it 5914193323Sed // already exists there, recursively merge the results together. 5915245431Sdim AddModifiedNodeToCSEMaps(User); 5916193323Sed } 5917193323Sed} 5918193323Sed 5919193323Sed/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5920193323Sed/// based on their topological order. It returns the maximum id and a vector 5921193323Sed/// of the SDNodes* in assigned order by reference. 5922193323Sedunsigned SelectionDAG::AssignTopologicalOrder() { 5923193323Sed 5924193323Sed unsigned DAGSize = 0; 5925193323Sed 5926193323Sed // SortedPos tracks the progress of the algorithm. Nodes before it are 5927193323Sed // sorted, nodes after it are unsorted. When the algorithm completes 5928193323Sed // it is at the end of the list. 5929193323Sed allnodes_iterator SortedPos = allnodes_begin(); 5930193323Sed 5931193323Sed // Visit all the nodes. Move nodes with no operands to the front of 5932193323Sed // the list immediately. Annotate nodes that do have operands with their 5933193323Sed // operand count. Before we do this, the Node Id fields of the nodes 5934193323Sed // may contain arbitrary values. After, the Node Id fields for nodes 5935193323Sed // before SortedPos will contain the topological sort index, and the 5936193323Sed // Node Id fields for nodes At SortedPos and after will contain the 5937193323Sed // count of outstanding operands. 5938193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5939193323Sed SDNode *N = I++; 5940202878Srdivacky checkForCycles(N); 5941193323Sed unsigned Degree = N->getNumOperands(); 5942193323Sed if (Degree == 0) { 5943193323Sed // A node with no uses, add it to the result array immediately. 5944193323Sed N->setNodeId(DAGSize++); 5945193323Sed allnodes_iterator Q = N; 5946193323Sed if (Q != SortedPos) 5947193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5948202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5949193323Sed ++SortedPos; 5950193323Sed } else { 5951193323Sed // Temporarily use the Node Id as scratch space for the degree count. 5952193323Sed N->setNodeId(Degree); 5953193323Sed } 5954193323Sed } 5955193323Sed 5956245431Sdim // Visit all the nodes. As we iterate, move nodes into sorted order, 5957193323Sed // such that by the time the end is reached all nodes will be sorted. 5958193323Sed for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5959193323Sed SDNode *N = I; 5960202878Srdivacky checkForCycles(N); 5961202878Srdivacky // N is in sorted position, so all its uses have one less operand 5962202878Srdivacky // that needs to be sorted. 5963193323Sed for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5964193323Sed UI != UE; ++UI) { 5965193323Sed SDNode *P = *UI; 5966193323Sed unsigned Degree = P->getNodeId(); 5967202878Srdivacky assert(Degree != 0 && "Invalid node degree"); 5968193323Sed --Degree; 5969193323Sed if (Degree == 0) { 5970193323Sed // All of P's operands are sorted, so P may sorted now. 5971193323Sed P->setNodeId(DAGSize++); 5972193323Sed if (P != SortedPos) 5973193323Sed SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5974202878Srdivacky assert(SortedPos != AllNodes.end() && "Overran node list"); 5975193323Sed ++SortedPos; 5976193323Sed } else { 5977193323Sed // Update P's outstanding operand count. 5978193323Sed P->setNodeId(Degree); 5979193323Sed } 5980193323Sed } 5981202878Srdivacky if (I == SortedPos) { 5982203954Srdivacky#ifndef NDEBUG 5983203954Srdivacky SDNode *S = ++I; 5984203954Srdivacky dbgs() << "Overran sorted position:\n"; 5985202878Srdivacky S->dumprFull(); 5986203954Srdivacky#endif 5987203954Srdivacky llvm_unreachable(0); 5988202878Srdivacky } 5989193323Sed } 5990193323Sed 5991193323Sed assert(SortedPos == AllNodes.end() && 5992193323Sed "Topological sort incomplete!"); 5993193323Sed assert(AllNodes.front().getOpcode() == ISD::EntryToken && 5994193323Sed "First node in topological sort is not the entry token!"); 5995193323Sed assert(AllNodes.front().getNodeId() == 0 && 5996193323Sed "First node in topological sort has non-zero id!"); 5997193323Sed assert(AllNodes.front().getNumOperands() == 0 && 5998193323Sed "First node in topological sort has operands!"); 5999193323Sed assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 6000193323Sed "Last node in topologic sort has unexpected id!"); 6001193323Sed assert(AllNodes.back().use_empty() && 6002193323Sed "Last node in topologic sort has users!"); 6003193323Sed assert(DAGSize == allnodes_size() && "Node count mismatch!"); 6004193323Sed return DAGSize; 6005193323Sed} 6006193323Sed 6007206083Srdivacky/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 6008206083Srdivacky/// value is produced by SD. 6009207618Srdivackyvoid SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 6010207618Srdivacky DbgInfo->add(DB, SD, isParameter); 6011206083Srdivacky if (SD) 6012206083Srdivacky SD->setHasDebugValue(true); 6013205218Srdivacky} 6014201360Srdivacky 6015218893Sdim/// TransferDbgValues - Transfer SDDbgValues. 6016218893Sdimvoid SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 6017218893Sdim if (From == To || !From.getNode()->getHasDebugValue()) 6018218893Sdim return; 6019218893Sdim SDNode *FromNode = From.getNode(); 6020218893Sdim SDNode *ToNode = To.getNode(); 6021224145Sdim ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 6022218893Sdim SmallVector<SDDbgValue *, 2> ClonedDVs; 6023224145Sdim for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 6024218893Sdim I != E; ++I) { 6025218893Sdim SDDbgValue *Dbg = *I; 6026218893Sdim if (Dbg->getKind() == SDDbgValue::SDNODE) { 6027218893Sdim SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 6028218893Sdim Dbg->getOffset(), Dbg->getDebugLoc(), 6029218893Sdim Dbg->getOrder()); 6030218893Sdim ClonedDVs.push_back(Clone); 6031218893Sdim } 6032218893Sdim } 6033263509Sdim for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(), 6034218893Sdim E = ClonedDVs.end(); I != E; ++I) 6035218893Sdim AddDbgValue(*I, ToNode, false); 6036218893Sdim} 6037218893Sdim 6038193323Sed//===----------------------------------------------------------------------===// 6039193323Sed// SDNode Class 6040193323Sed//===----------------------------------------------------------------------===// 6041193323Sed 6042193323SedHandleSDNode::~HandleSDNode() { 6043193323Sed DropOperands(); 6044193323Sed} 6045193323Sed 6046263509SdimGlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 6047263509Sdim DebugLoc DL, const GlobalValue *GA, 6048198090Srdivacky EVT VT, int64_t o, unsigned char TF) 6049263509Sdim : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 6050207618Srdivacky TheGlobal = GA; 6051193323Sed} 6052193323Sed 6053263509SdimAddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, 6054263509Sdim SDValue X, unsigned SrcAS, 6055263509Sdim unsigned DestAS) 6056263509Sdim : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), 6057263509Sdim SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 6058263509Sdim 6059263509SdimMemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6060263509Sdim EVT memvt, MachineMemOperand *mmo) 6061263509Sdim : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 6062204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6063235633Sdim MMO->isNonTemporal(), MMO->isInvariant()); 6064198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6065204642Srdivacky assert(isNonTemporal() == MMO->isNonTemporal() && 6066204642Srdivacky "Non-temporal encoding error!"); 6067198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6068193323Sed} 6069193323Sed 6070263509SdimMemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6071218893Sdim const SDValue *Ops, unsigned NumOps, EVT memvt, 6072198090Srdivacky MachineMemOperand *mmo) 6073263509Sdim : SDNode(Opc, Order, dl, VTs, Ops, NumOps), 6074198090Srdivacky MemoryVT(memvt), MMO(mmo) { 6075204642Srdivacky SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6076235633Sdim MMO->isNonTemporal(), MMO->isInvariant()); 6077198090Srdivacky assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6078198090Srdivacky assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6079193323Sed} 6080193323Sed 6081193323Sed/// Profile - Gather unique data for the node. 6082193323Sed/// 6083193323Sedvoid SDNode::Profile(FoldingSetNodeID &ID) const { 6084193323Sed AddNodeIDNode(ID, this); 6085193323Sed} 6086193323Sed 6087198090Srdivackynamespace { 6088198090Srdivacky struct EVTArray { 6089198090Srdivacky std::vector<EVT> VTs; 6090218893Sdim 6091198090Srdivacky EVTArray() { 6092198090Srdivacky VTs.reserve(MVT::LAST_VALUETYPE); 6093198090Srdivacky for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 6094198090Srdivacky VTs.push_back(MVT((MVT::SimpleValueType)i)); 6095198090Srdivacky } 6096198090Srdivacky }; 6097198090Srdivacky} 6098198090Srdivacky 6099198090Srdivackystatic ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 6100198090Srdivackystatic ManagedStatic<EVTArray> SimpleVTArray; 6101195098Sedstatic ManagedStatic<sys::SmartMutex<true> > VTMutex; 6102195098Sed 6103193323Sed/// getValueTypeList - Return a pointer to the specified value type. 6104193323Sed/// 6105198090Srdivackyconst EVT *SDNode::getValueTypeList(EVT VT) { 6106193323Sed if (VT.isExtended()) { 6107198090Srdivacky sys::SmartScopedLock<true> Lock(*VTMutex); 6108195098Sed return &(*EVTs->insert(VT).first); 6109193323Sed } else { 6110218893Sdim assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 6111208599Srdivacky "Value type out of range!"); 6112198090Srdivacky return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 6113193323Sed } 6114193323Sed} 6115193323Sed 6116193323Sed/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 6117193323Sed/// indicated value. This method ignores uses of other values defined by this 6118193323Sed/// operation. 6119193323Sedbool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 6120193323Sed assert(Value < getNumValues() && "Bad value!"); 6121193323Sed 6122193323Sed // TODO: Only iterate over uses of a given value of the node 6123193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 6124193323Sed if (UI.getUse().getResNo() == Value) { 6125193323Sed if (NUses == 0) 6126193323Sed return false; 6127193323Sed --NUses; 6128193323Sed } 6129193323Sed } 6130193323Sed 6131193323Sed // Found exactly the right number of uses? 6132193323Sed return NUses == 0; 6133193323Sed} 6134193323Sed 6135193323Sed 6136193323Sed/// hasAnyUseOfValue - Return true if there are any use of the indicated 6137193323Sed/// value. This method ignores uses of other values defined by this operation. 6138193323Sedbool SDNode::hasAnyUseOfValue(unsigned Value) const { 6139193323Sed assert(Value < getNumValues() && "Bad value!"); 6140193323Sed 6141193323Sed for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 6142193323Sed if (UI.getUse().getResNo() == Value) 6143193323Sed return true; 6144193323Sed 6145193323Sed return false; 6146193323Sed} 6147193323Sed 6148193323Sed 6149193323Sed/// isOnlyUserOf - Return true if this node is the only use of N. 6150193323Sed/// 6151193323Sedbool SDNode::isOnlyUserOf(SDNode *N) const { 6152193323Sed bool Seen = false; 6153193323Sed for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 6154193323Sed SDNode *User = *I; 6155193323Sed if (User == this) 6156193323Sed Seen = true; 6157193323Sed else 6158193323Sed return false; 6159193323Sed } 6160193323Sed 6161193323Sed return Seen; 6162193323Sed} 6163193323Sed 6164193323Sed/// isOperand - Return true if this node is an operand of N. 6165193323Sed/// 6166193323Sedbool SDValue::isOperandOf(SDNode *N) const { 6167193323Sed for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6168193323Sed if (*this == N->getOperand(i)) 6169193323Sed return true; 6170193323Sed return false; 6171193323Sed} 6172193323Sed 6173193323Sedbool SDNode::isOperandOf(SDNode *N) const { 6174193323Sed for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 6175193323Sed if (this == N->OperandList[i].getNode()) 6176193323Sed return true; 6177193323Sed return false; 6178193323Sed} 6179193323Sed 6180193323Sed/// reachesChainWithoutSideEffects - Return true if this operand (which must 6181193323Sed/// be a chain) reaches the specified operand without crossing any 6182218893Sdim/// side-effecting instructions on any chain path. In practice, this looks 6183218893Sdim/// through token factors and non-volatile loads. In order to remain efficient, 6184218893Sdim/// this only looks a couple of nodes in, it does not do an exhaustive search. 6185193323Sedbool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 6186193323Sed unsigned Depth) const { 6187193323Sed if (*this == Dest) return true; 6188193323Sed 6189193323Sed // Don't search too deeply, we just want to be able to see through 6190193323Sed // TokenFactor's etc. 6191193323Sed if (Depth == 0) return false; 6192193323Sed 6193193323Sed // If this is a token factor, all inputs to the TF happen in parallel. If any 6194218893Sdim // of the operands of the TF does not reach dest, then we cannot do the xform. 6195193323Sed if (getOpcode() == ISD::TokenFactor) { 6196193323Sed for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6197218893Sdim if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 6198218893Sdim return false; 6199218893Sdim return true; 6200193323Sed } 6201193323Sed 6202193323Sed // Loads don't have side effects, look through them. 6203193323Sed if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 6204193323Sed if (!Ld->isVolatile()) 6205193323Sed return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 6206193323Sed } 6207193323Sed return false; 6208193323Sed} 6209193323Sed 6210224145Sdim/// hasPredecessor - Return true if N is a predecessor of this node. 6211224145Sdim/// N is either an operand of this node, or can be reached by recursively 6212224145Sdim/// traversing up the operands. 6213224145Sdim/// NOTE: This is an expensive method. Use it carefully. 6214224145Sdimbool SDNode::hasPredecessor(const SDNode *N) const { 6215224145Sdim SmallPtrSet<const SDNode *, 32> Visited; 6216224145Sdim SmallVector<const SDNode *, 16> Worklist; 6217224145Sdim return hasPredecessorHelper(N, Visited, Worklist); 6218224145Sdim} 6219198892Srdivacky 6220263509Sdimbool 6221263509SdimSDNode::hasPredecessorHelper(const SDNode *N, 6222263509Sdim SmallPtrSet<const SDNode *, 32> &Visited, 6223263509Sdim SmallVectorImpl<const SDNode *> &Worklist) const { 6224224145Sdim if (Visited.empty()) { 6225224145Sdim Worklist.push_back(this); 6226224145Sdim } else { 6227224145Sdim // Take a look in the visited set. If we've already encountered this node 6228224145Sdim // we needn't search further. 6229224145Sdim if (Visited.count(N)) 6230224145Sdim return true; 6231224145Sdim } 6232224145Sdim 6233224145Sdim // Haven't visited N yet. Continue the search. 6234224145Sdim while (!Worklist.empty()) { 6235224145Sdim const SDNode *M = Worklist.pop_back_val(); 6236224145Sdim for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 6237224145Sdim SDNode *Op = M->getOperand(i).getNode(); 6238198892Srdivacky if (Visited.insert(Op)) 6239198892Srdivacky Worklist.push_back(Op); 6240224145Sdim if (Op == N) 6241224145Sdim return true; 6242198892Srdivacky } 6243224145Sdim } 6244198892Srdivacky 6245198892Srdivacky return false; 6246193323Sed} 6247193323Sed 6248193323Seduint64_t SDNode::getConstantOperandVal(unsigned Num) const { 6249193323Sed assert(Num < NumOperands && "Invalid child # of SDNode!"); 6250193323Sed return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 6251193323Sed} 6252193323Sed 6253199989SrdivackySDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6254199989Srdivacky assert(N->getNumValues() == 1 && 6255199989Srdivacky "Can't unroll a vector with multiple results!"); 6256199989Srdivacky 6257199989Srdivacky EVT VT = N->getValueType(0); 6258199989Srdivacky unsigned NE = VT.getVectorNumElements(); 6259199989Srdivacky EVT EltVT = VT.getVectorElementType(); 6260263509Sdim SDLoc dl(N); 6261199989Srdivacky 6262199989Srdivacky SmallVector<SDValue, 8> Scalars; 6263199989Srdivacky SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6264199989Srdivacky 6265199989Srdivacky // If ResNE is 0, fully unroll the vector op. 6266199989Srdivacky if (ResNE == 0) 6267199989Srdivacky ResNE = NE; 6268199989Srdivacky else if (NE > ResNE) 6269199989Srdivacky NE = ResNE; 6270199989Srdivacky 6271199989Srdivacky unsigned i; 6272199989Srdivacky for (i= 0; i != NE; ++i) { 6273207618Srdivacky for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6274199989Srdivacky SDValue Operand = N->getOperand(j); 6275199989Srdivacky EVT OperandVT = Operand.getValueType(); 6276199989Srdivacky if (OperandVT.isVector()) { 6277199989Srdivacky // A vector operand; extract a single element. 6278263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6279199989Srdivacky EVT OperandEltVT = OperandVT.getVectorElementType(); 6280199989Srdivacky Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6281199989Srdivacky OperandEltVT, 6282199989Srdivacky Operand, 6283263509Sdim getConstant(i, TLI->getVectorIdxTy())); 6284199989Srdivacky } else { 6285199989Srdivacky // A scalar operand; just use it as is. 6286199989Srdivacky Operands[j] = Operand; 6287199989Srdivacky } 6288199989Srdivacky } 6289199989Srdivacky 6290199989Srdivacky switch (N->getOpcode()) { 6291199989Srdivacky default: 6292199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6293199989Srdivacky &Operands[0], Operands.size())); 6294199989Srdivacky break; 6295226890Sdim case ISD::VSELECT: 6296226890Sdim Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6297226890Sdim &Operands[0], Operands.size())); 6298226890Sdim break; 6299199989Srdivacky case ISD::SHL: 6300199989Srdivacky case ISD::SRA: 6301199989Srdivacky case ISD::SRL: 6302199989Srdivacky case ISD::ROTL: 6303199989Srdivacky case ISD::ROTR: 6304199989Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6305263509Sdim getShiftAmountOperand(Operands[0].getValueType(), 6306263509Sdim Operands[1]))); 6307199989Srdivacky break; 6308202375Srdivacky case ISD::SIGN_EXTEND_INREG: 6309202375Srdivacky case ISD::FP_ROUND_INREG: { 6310202375Srdivacky EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6311202375Srdivacky Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6312202375Srdivacky Operands[0], 6313202375Srdivacky getValueType(ExtVT))); 6314199989Srdivacky } 6315202375Srdivacky } 6316199989Srdivacky } 6317199989Srdivacky 6318199989Srdivacky for (; i < ResNE; ++i) 6319199989Srdivacky Scalars.push_back(getUNDEF(EltVT)); 6320199989Srdivacky 6321199989Srdivacky return getNode(ISD::BUILD_VECTOR, dl, 6322199989Srdivacky EVT::getVectorVT(*getContext(), EltVT, ResNE), 6323199989Srdivacky &Scalars[0], Scalars.size()); 6324199989Srdivacky} 6325199989Srdivacky 6326200581Srdivacky 6327218893Sdim/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6328218893Sdim/// location that is 'Dist' units away from the location that the 'Base' load 6329200581Srdivacky/// is loading from. 6330218893Sdimbool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6331200581Srdivacky unsigned Bytes, int Dist) const { 6332200581Srdivacky if (LD->getChain() != Base->getChain()) 6333200581Srdivacky return false; 6334200581Srdivacky EVT VT = LD->getValueType(0); 6335200581Srdivacky if (VT.getSizeInBits() / 8 != Bytes) 6336200581Srdivacky return false; 6337200581Srdivacky 6338200581Srdivacky SDValue Loc = LD->getOperand(1); 6339200581Srdivacky SDValue BaseLoc = Base->getOperand(1); 6340200581Srdivacky if (Loc.getOpcode() == ISD::FrameIndex) { 6341200581Srdivacky if (BaseLoc.getOpcode() != ISD::FrameIndex) 6342200581Srdivacky return false; 6343200581Srdivacky const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6344200581Srdivacky int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6345200581Srdivacky int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6346200581Srdivacky int FS = MFI->getObjectSize(FI); 6347200581Srdivacky int BFS = MFI->getObjectSize(BFI); 6348200581Srdivacky if (FS != BFS || FS != (int)Bytes) return false; 6349200581Srdivacky return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6350200581Srdivacky } 6351200581Srdivacky 6352218893Sdim // Handle X+C 6353218893Sdim if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6354218893Sdim cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6355218893Sdim return true; 6356218893Sdim 6357207618Srdivacky const GlobalValue *GV1 = NULL; 6358207618Srdivacky const GlobalValue *GV2 = NULL; 6359200581Srdivacky int64_t Offset1 = 0; 6360200581Srdivacky int64_t Offset2 = 0; 6361263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6362263509Sdim bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6363263509Sdim bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6364200581Srdivacky if (isGA1 && isGA2 && GV1 == GV2) 6365200581Srdivacky return Offset1 == (Offset2 + Dist*Bytes); 6366200581Srdivacky return false; 6367200581Srdivacky} 6368200581Srdivacky 6369200581Srdivacky 6370200581Srdivacky/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6371200581Srdivacky/// it cannot be inferred. 6372200581Srdivackyunsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6373200581Srdivacky // If this is a GlobalAddress + cst, return the alignment. 6374207618Srdivacky const GlobalValue *GV; 6375200581Srdivacky int64_t GVOffset = 0; 6376263509Sdim const TargetLowering *TLI = TM.getTargetLowering(); 6377263509Sdim if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6378263509Sdim unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType()); 6379235633Sdim APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6380235633Sdim llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne, 6381263509Sdim TLI->getDataLayout()); 6382235633Sdim unsigned AlignBits = KnownZero.countTrailingOnes(); 6383235633Sdim unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6384235633Sdim if (Align) 6385235633Sdim return MinAlign(Align, GVOffset); 6386206083Srdivacky } 6387200581Srdivacky 6388200581Srdivacky // If this is a direct reference to a stack slot, use information about the 6389200581Srdivacky // stack slot's alignment. 6390200581Srdivacky int FrameIdx = 1 << 31; 6391200581Srdivacky int64_t FrameOffset = 0; 6392200581Srdivacky if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6393200581Srdivacky FrameIdx = FI->getIndex(); 6394218893Sdim } else if (isBaseWithConstantOffset(Ptr) && 6395200581Srdivacky isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6396218893Sdim // Handle FI+Cst 6397200581Srdivacky FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6398200581Srdivacky FrameOffset = Ptr.getConstantOperandVal(1); 6399200581Srdivacky } 6400200581Srdivacky 6401200581Srdivacky if (FrameIdx != (1 << 31)) { 6402200581Srdivacky const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6403200581Srdivacky unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6404200581Srdivacky FrameOffset); 6405200581Srdivacky return FIInfoAlign; 6406200581Srdivacky } 6407200581Srdivacky 6408200581Srdivacky return 0; 6409200581Srdivacky} 6410200581Srdivacky 6411263509Sdim/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 6412263509Sdim/// which is split (or expanded) into two not necessarily identical pieces. 6413263509Sdimstd::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { 6414263509Sdim // Currently all types are split in half. 6415263509Sdim EVT LoVT, HiVT; 6416263509Sdim if (!VT.isVector()) { 6417263509Sdim LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); 6418263509Sdim } else { 6419263509Sdim unsigned NumElements = VT.getVectorNumElements(); 6420263509Sdim assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 6421263509Sdim LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), 6422263509Sdim NumElements/2); 6423263509Sdim } 6424263509Sdim return std::make_pair(LoVT, HiVT); 6425263509Sdim} 6426263509Sdim 6427263509Sdim/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 6428263509Sdim/// low/high part. 6429263509Sdimstd::pair<SDValue, SDValue> 6430263509SdimSelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, 6431263509Sdim const EVT &HiVT) { 6432263509Sdim assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <= 6433263509Sdim N.getValueType().getVectorNumElements() && 6434263509Sdim "More vector elements requested than available!"); 6435263509Sdim SDValue Lo, Hi; 6436263509Sdim Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, 6437263509Sdim getConstant(0, TLI->getVectorIdxTy())); 6438263509Sdim Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, 6439263509Sdim getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy())); 6440263509Sdim return std::make_pair(Lo, Hi); 6441263509Sdim} 6442263509Sdim 6443193323Sed// getAddressSpace - Return the address space this GlobalAddress belongs to. 6444193323Sedunsigned GlobalAddressSDNode::getAddressSpace() const { 6445193323Sed return getGlobal()->getType()->getAddressSpace(); 6446193323Sed} 6447193323Sed 6448193323Sed 6449226890SdimType *ConstantPoolSDNode::getType() const { 6450193323Sed if (isMachineConstantPoolEntry()) 6451193323Sed return Val.MachineCPVal->getType(); 6452193323Sed return Val.ConstVal->getType(); 6453193323Sed} 6454193323Sed 6455193323Sedbool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6456193323Sed APInt &SplatUndef, 6457193323Sed unsigned &SplatBitSize, 6458193323Sed bool &HasAnyUndefs, 6459199481Srdivacky unsigned MinSplatBits, 6460199481Srdivacky bool isBigEndian) { 6461198090Srdivacky EVT VT = getValueType(0); 6462193323Sed assert(VT.isVector() && "Expected a vector type"); 6463193323Sed unsigned sz = VT.getSizeInBits(); 6464193323Sed if (MinSplatBits > sz) 6465193323Sed return false; 6466193323Sed 6467193323Sed SplatValue = APInt(sz, 0); 6468193323Sed SplatUndef = APInt(sz, 0); 6469193323Sed 6470193323Sed // Get the bits. Bits with undefined values (when the corresponding element 6471193323Sed // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6472193323Sed // in SplatValue. If any of the values are not constant, give up and return 6473193323Sed // false. 6474193323Sed unsigned int nOps = getNumOperands(); 6475193323Sed assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6476193323Sed unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6477199481Srdivacky 6478199481Srdivacky for (unsigned j = 0; j < nOps; ++j) { 6479199481Srdivacky unsigned i = isBigEndian ? nOps-1-j : j; 6480193323Sed SDValue OpVal = getOperand(i); 6481199481Srdivacky unsigned BitPos = j * EltBitSize; 6482193323Sed 6483193323Sed if (OpVal.getOpcode() == ISD::UNDEF) 6484199481Srdivacky SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6485193323Sed else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6486218893Sdim SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6487207618Srdivacky zextOrTrunc(sz) << BitPos; 6488193323Sed else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6489193323Sed SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6490193323Sed else 6491193323Sed return false; 6492193323Sed } 6493193323Sed 6494193323Sed // The build_vector is all constants or undefs. Find the smallest element 6495193323Sed // size that splats the vector. 6496193323Sed 6497193323Sed HasAnyUndefs = (SplatUndef != 0); 6498193323Sed while (sz > 8) { 6499193323Sed 6500193323Sed unsigned HalfSize = sz / 2; 6501218893Sdim APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6502218893Sdim APInt LowValue = SplatValue.trunc(HalfSize); 6503218893Sdim APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6504218893Sdim APInt LowUndef = SplatUndef.trunc(HalfSize); 6505193323Sed 6506193323Sed // If the two halves do not match (ignoring undef bits), stop here. 6507193323Sed if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6508193323Sed MinSplatBits > HalfSize) 6509193323Sed break; 6510193323Sed 6511193323Sed SplatValue = HighValue | LowValue; 6512193323Sed SplatUndef = HighUndef & LowUndef; 6513198090Srdivacky 6514193323Sed sz = HalfSize; 6515193323Sed } 6516193323Sed 6517193323Sed SplatBitSize = sz; 6518193323Sed return true; 6519193323Sed} 6520193323Sed 6521198090Srdivackybool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6522193323Sed // Find the first non-undef value in the shuffle mask. 6523193323Sed unsigned i, e; 6524193323Sed for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6525193323Sed /* search */; 6526193323Sed 6527193323Sed assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6528198090Srdivacky 6529193323Sed // Make sure all remaining elements are either undef or the same as the first 6530193323Sed // non-undef value. 6531193323Sed for (int Idx = Mask[i]; i != e; ++i) 6532193323Sed if (Mask[i] >= 0 && Mask[i] != Idx) 6533193323Sed return false; 6534193323Sed return true; 6535193323Sed} 6536202878Srdivacky 6537204642Srdivacky#ifdef XDEBUG 6538202878Srdivackystatic void checkForCyclesHelper(const SDNode *N, 6539204642Srdivacky SmallPtrSet<const SDNode*, 32> &Visited, 6540204642Srdivacky SmallPtrSet<const SDNode*, 32> &Checked) { 6541204642Srdivacky // If this node has already been checked, don't check it again. 6542204642Srdivacky if (Checked.count(N)) 6543204642Srdivacky return; 6544218893Sdim 6545204642Srdivacky // If a node has already been visited on this depth-first walk, reject it as 6546204642Srdivacky // a cycle. 6547204642Srdivacky if (!Visited.insert(N)) { 6548202878Srdivacky dbgs() << "Offending node:\n"; 6549202878Srdivacky N->dumprFull(); 6550204642Srdivacky errs() << "Detected cycle in SelectionDAG\n"; 6551204642Srdivacky abort(); 6552202878Srdivacky } 6553218893Sdim 6554204642Srdivacky for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6555204642Srdivacky checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6556218893Sdim 6557204642Srdivacky Checked.insert(N); 6558204642Srdivacky Visited.erase(N); 6559202878Srdivacky} 6560204642Srdivacky#endif 6561202878Srdivacky 6562202878Srdivackyvoid llvm::checkForCycles(const llvm::SDNode *N) { 6563202878Srdivacky#ifdef XDEBUG 6564263509Sdim assert(N && "Checking nonexistent SDNode"); 6565204642Srdivacky SmallPtrSet<const SDNode*, 32> visited; 6566204642Srdivacky SmallPtrSet<const SDNode*, 32> checked; 6567204642Srdivacky checkForCyclesHelper(N, visited, checked); 6568202878Srdivacky#endif 6569202878Srdivacky} 6570202878Srdivacky 6571202878Srdivackyvoid llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6572202878Srdivacky checkForCycles(DAG->getRoot().getNode()); 6573202878Srdivacky} 6574