1193323Sed//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file implements the SelectionDAG::Legalize method. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13193323Sed 14249423Sdim#include "llvm/CodeGen/SelectionDAG.h" 15249423Sdim#include "llvm/ADT/SmallPtrSet.h" 16249423Sdim#include "llvm/ADT/SmallVector.h" 17249423Sdim#include "llvm/ADT/Triple.h" 18218893Sdim#include "llvm/CodeGen/Analysis.h" 19193323Sed#include "llvm/CodeGen/MachineFunction.h" 20193323Sed#include "llvm/CodeGen/MachineJumpTableInfo.h" 21249423Sdim#include "llvm/DebugInfo.h" 22249423Sdim#include "llvm/IR/CallingConv.h" 23249423Sdim#include "llvm/IR/Constants.h" 24249423Sdim#include "llvm/IR/DataLayout.h" 25249423Sdim#include "llvm/IR/DerivedTypes.h" 26249423Sdim#include "llvm/IR/Function.h" 27249423Sdim#include "llvm/IR/LLVMContext.h" 28202375Srdivacky#include "llvm/Support/Debug.h" 29210299Sed#include "llvm/Support/ErrorHandling.h" 30193323Sed#include "llvm/Support/MathExtras.h" 31198090Srdivacky#include "llvm/Support/raw_ostream.h" 32249423Sdim#include "llvm/Target/TargetFrameLowering.h" 33249423Sdim#include "llvm/Target/TargetLowering.h" 34249423Sdim#include "llvm/Target/TargetMachine.h" 35193323Sedusing namespace llvm; 36193323Sed 37193323Sed//===----------------------------------------------------------------------===// 38193323Sed/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and 39193323Sed/// hacks on it until the target machine can handle it. This involves 40193323Sed/// eliminating value sizes the machine cannot handle (promoting small sizes to 41193323Sed/// large sizes or splitting up large values into small values) as well as 42193323Sed/// eliminating operations the machine cannot handle. 43193323Sed/// 44193323Sed/// This code also does a small amount of optimization and recognition of idioms 45193323Sed/// as part of its processing. For example, if a target does not support a 46193323Sed/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this 47193323Sed/// will attempt merge setcc and brc instructions into brcc's. 48193323Sed/// 49193323Sednamespace { 50234353Sdimclass SelectionDAGLegalize : public SelectionDAG::DAGUpdateListener { 51207618Srdivacky const TargetMachine &TM; 52207618Srdivacky const TargetLowering &TLI; 53193323Sed SelectionDAG &DAG; 54193323Sed 55234353Sdim /// LegalizePosition - The iterator for walking through the node list. 56234353Sdim SelectionDAG::allnodes_iterator LegalizePosition; 57193323Sed 58234353Sdim /// LegalizedNodes - The set of nodes which have already been legalized. 59234353Sdim SmallPtrSet<SDNode *, 16> LegalizedNodes; 60193323Sed 61263508Sdim EVT getSetCCResultType(EVT VT) const { 62263508Sdim return TLI.getSetCCResultType(*DAG.getContext(), VT); 63263508Sdim } 64263508Sdim 65234353Sdim // Libcall insertion helpers. 66226633Sdim 67193323Sedpublic: 68223017Sdim explicit SelectionDAGLegalize(SelectionDAG &DAG); 69193323Sed 70193323Sed void LegalizeDAG(); 71193323Sed 72193323Sedprivate: 73234353Sdim /// LegalizeOp - Legalizes the given operation. 74234353Sdim void LegalizeOp(SDNode *Node); 75193323Sed 76193574Sed SDValue OptimizeFloatStore(StoreSDNode *ST); 77193574Sed 78239462Sdim void LegalizeLoadOps(SDNode *Node); 79239462Sdim void LegalizeStoreOps(SDNode *Node); 80239462Sdim 81193323Sed /// PerformInsertVectorEltInMemory - Some target cannot handle a variable 82193323Sed /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 83193323Sed /// is necessary to spill the vector being inserted into to memory, perform 84193323Sed /// the insert there, and then read the result back. 85193323Sed SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, 86263508Sdim SDValue Idx, SDLoc dl); 87193323Sed SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, 88263508Sdim SDValue Idx, SDLoc dl); 89193323Sed 90193323Sed /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 91193323Sed /// performs the same shuffe in terms of order or result bytes, but on a type 92193323Sed /// whose vector element type is narrower than the original shuffle type. 93193323Sed /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 94263508Sdim SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl, 95210299Sed SDValue N1, SDValue N2, 96234353Sdim ArrayRef<int> Mask) const; 97193323Sed 98263508Sdim bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, 99263508Sdim bool &NeedInvert, SDLoc dl); 100193323Sed 101193323Sed SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); 102221345Sdim SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, 103263508Sdim unsigned NumOps, bool isSigned, SDLoc dl); 104221345Sdim 105210299Sed std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, 106210299Sed SDNode *Node, bool isSigned); 107193323Sed SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, 108193323Sed RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, 109249423Sdim RTLIB::Libcall Call_F128, 110193323Sed RTLIB::Libcall Call_PPCF128); 111199481Srdivacky SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, 112199481Srdivacky RTLIB::Libcall Call_I8, 113199481Srdivacky RTLIB::Libcall Call_I16, 114199481Srdivacky RTLIB::Libcall Call_I32, 115199481Srdivacky RTLIB::Libcall Call_I64, 116193323Sed RTLIB::Libcall Call_I128); 117221345Sdim void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 118249423Sdim void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); 119193323Sed 120263508Sdim SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, SDLoc dl); 121193323Sed SDValue ExpandBUILD_VECTOR(SDNode *Node); 122193323Sed SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); 123193323Sed void ExpandDYNAMIC_STACKALLOC(SDNode *Node, 124193323Sed SmallVectorImpl<SDValue> &Results); 125193323Sed SDValue ExpandFCOPYSIGN(SDNode *Node); 126198090Srdivacky SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT, 127263508Sdim SDLoc dl); 128198090Srdivacky SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, 129263508Sdim SDLoc dl); 130198090Srdivacky SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, 131263508Sdim SDLoc dl); 132193323Sed 133263508Sdim SDValue ExpandBSWAP(SDValue Op, SDLoc dl); 134263508Sdim SDValue ExpandBitCount(unsigned Opc, SDValue Op, SDLoc dl); 135193323Sed 136193323Sed SDValue ExpandExtractFromVectorThroughStack(SDValue Op); 137218893Sdim SDValue ExpandInsertToVectorThroughStack(SDValue Op); 138193574Sed SDValue ExpandVectorBuildThroughStack(SDNode* Node); 139193323Sed 140234353Sdim SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); 141234353Sdim 142210299Sed std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); 143210299Sed 144234353Sdim void ExpandNode(SDNode *Node); 145234353Sdim void PromoteNode(SDNode *Node); 146234353Sdim 147234353Sdim void ForgetNode(SDNode *N) { 148234353Sdim LegalizedNodes.erase(N); 149234353Sdim if (LegalizePosition == SelectionDAG::allnodes_iterator(N)) 150234353Sdim ++LegalizePosition; 151234353Sdim } 152234353Sdim 153234353Sdimpublic: 154234353Sdim // DAGUpdateListener implementation. 155234353Sdim virtual void NodeDeleted(SDNode *N, SDNode *E) { 156234353Sdim ForgetNode(N); 157234353Sdim } 158234353Sdim virtual void NodeUpdated(SDNode *N) {} 159234353Sdim 160234353Sdim // Node replacement helpers 161234353Sdim void ReplacedNode(SDNode *N) { 162234353Sdim if (N->use_empty()) { 163239462Sdim DAG.RemoveDeadNode(N); 164234353Sdim } else { 165234353Sdim ForgetNode(N); 166234353Sdim } 167234353Sdim } 168234353Sdim void ReplaceNode(SDNode *Old, SDNode *New) { 169239462Sdim DAG.ReplaceAllUsesWith(Old, New); 170234353Sdim ReplacedNode(Old); 171234353Sdim } 172234353Sdim void ReplaceNode(SDValue Old, SDValue New) { 173239462Sdim DAG.ReplaceAllUsesWith(Old, New); 174234353Sdim ReplacedNode(Old.getNode()); 175234353Sdim } 176234353Sdim void ReplaceNode(SDNode *Old, const SDValue *New) { 177239462Sdim DAG.ReplaceAllUsesWith(Old, New); 178234353Sdim ReplacedNode(Old); 179234353Sdim } 180193323Sed}; 181193323Sed} 182193323Sed 183193323Sed/// ShuffleWithNarrowerEltType - Return a vector shuffle operation which 184193323Sed/// performs the same shuffe in terms of order or result bytes, but on a type 185193323Sed/// whose vector element type is narrower than the original shuffle type. 186193323Sed/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> 187210299SedSDValue 188263508SdimSelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl, 189193323Sed SDValue N1, SDValue N2, 190234353Sdim ArrayRef<int> Mask) const { 191193323Sed unsigned NumMaskElts = VT.getVectorNumElements(); 192193323Sed unsigned NumDestElts = NVT.getVectorNumElements(); 193193323Sed unsigned NumEltsGrowth = NumDestElts / NumMaskElts; 194193323Sed 195193323Sed assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); 196193323Sed 197193323Sed if (NumEltsGrowth == 1) 198193323Sed return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]); 199210299Sed 200193323Sed SmallVector<int, 8> NewMask; 201193323Sed for (unsigned i = 0; i != NumMaskElts; ++i) { 202193323Sed int Idx = Mask[i]; 203193323Sed for (unsigned j = 0; j != NumEltsGrowth; ++j) { 204210299Sed if (Idx < 0) 205193323Sed NewMask.push_back(-1); 206193323Sed else 207193323Sed NewMask.push_back(Idx * NumEltsGrowth + j); 208193323Sed } 209193323Sed } 210193323Sed assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); 211193323Sed assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); 212193323Sed return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); 213193323Sed} 214193323Sed 215223017SdimSelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) 216239462Sdim : SelectionDAG::DAGUpdateListener(dag), 217239462Sdim TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()), 218224145Sdim DAG(dag) { 219193323Sed} 220193323Sed 221193323Sedvoid SelectionDAGLegalize::LegalizeDAG() { 222193323Sed DAG.AssignTopologicalOrder(); 223193323Sed 224234353Sdim // Visit all the nodes. We start in topological order, so that we see 225234353Sdim // nodes with their original operands intact. Legalization can produce 226234353Sdim // new nodes which may themselves need to be legalized. Iterate until all 227234353Sdim // nodes have been legalized. 228234353Sdim for (;;) { 229234353Sdim bool AnyLegalized = false; 230234353Sdim for (LegalizePosition = DAG.allnodes_end(); 231234353Sdim LegalizePosition != DAG.allnodes_begin(); ) { 232234353Sdim --LegalizePosition; 233193323Sed 234234353Sdim SDNode *N = LegalizePosition; 235234353Sdim if (LegalizedNodes.insert(N)) { 236234353Sdim AnyLegalized = true; 237234353Sdim LegalizeOp(N); 238234353Sdim } 239193323Sed } 240234353Sdim if (!AnyLegalized) 241218893Sdim break; 242193323Sed 243193323Sed } 244193323Sed 245234353Sdim // Remove dead nodes now. 246234353Sdim DAG.RemoveDeadNodes(); 247193323Sed} 248193323Sed 249193323Sed/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or 250193323Sed/// a load from the constant pool. 251234353SdimSDValue 252234353SdimSelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { 253193323Sed bool Extend = false; 254263508Sdim SDLoc dl(CFP); 255193323Sed 256193323Sed // If a FP immediate is precise when represented as a float and if the 257193323Sed // target can do an extending load from float to double, we put it into 258193323Sed // the constant pool as a float, even if it's is statically typed as a 259193323Sed // double. This shrinks FP constants and canonicalizes them for targets where 260193323Sed // an FP extending load is the same cost as a normal load (such as on the x87 261193323Sed // fp stack or PPC FP unit). 262198090Srdivacky EVT VT = CFP->getValueType(0); 263193323Sed ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); 264193323Sed if (!UseCP) { 265193323Sed assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); 266193323Sed return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), 267193323Sed (VT == MVT::f64) ? MVT::i64 : MVT::i32); 268193323Sed } 269193323Sed 270198090Srdivacky EVT OrigVT = VT; 271198090Srdivacky EVT SVT = VT; 272193323Sed while (SVT != MVT::f32) { 273198090Srdivacky SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); 274210299Sed if (ConstantFPSDNode::isValueValidForType(SVT, CFP->getValueAPF()) && 275193323Sed // Only do this if the target has a native EXTLOAD instruction from 276193323Sed // smaller type. 277193323Sed TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && 278193323Sed TLI.ShouldShrinkFPConstant(OrigVT)) { 279226633Sdim Type *SType = SVT.getTypeForEVT(*DAG.getContext()); 280193323Sed LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); 281193323Sed VT = SVT; 282193323Sed Extend = true; 283193323Sed } 284193323Sed } 285193323Sed 286193323Sed SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); 287193323Sed unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 288234353Sdim if (Extend) { 289234353Sdim SDValue Result = 290234353Sdim DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT, 291234353Sdim DAG.getEntryNode(), 292234353Sdim CPIdx, MachinePointerInfo::getConstantPool(), 293234353Sdim VT, false, false, Alignment); 294234353Sdim return Result; 295234353Sdim } 296234353Sdim SDValue Result = 297234353Sdim DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx, 298234353Sdim MachinePointerInfo::getConstantPool(), false, false, false, 299234353Sdim Alignment); 300234353Sdim return Result; 301193323Sed} 302193323Sed 303193323Sed/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. 304234353Sdimstatic void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, 305234353Sdim const TargetLowering &TLI, 306234353Sdim SelectionDAGLegalize *DAGLegalize) { 307234353Sdim assert(ST->getAddressingMode() == ISD::UNINDEXED && 308234353Sdim "unaligned indexed stores not implemented!"); 309193323Sed SDValue Chain = ST->getChain(); 310193323Sed SDValue Ptr = ST->getBasePtr(); 311193323Sed SDValue Val = ST->getValue(); 312198090Srdivacky EVT VT = Val.getValueType(); 313193323Sed int Alignment = ST->getAlignment(); 314263508Sdim unsigned AS = ST->getAddressSpace(); 315263508Sdim 316263508Sdim SDLoc dl(ST); 317193323Sed if (ST->getMemoryVT().isFloatingPoint() || 318193323Sed ST->getMemoryVT().isVector()) { 319198090Srdivacky EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 320193323Sed if (TLI.isTypeLegal(intVT)) { 321193323Sed // Expand to a bitconvert of the value to the integer type of the 322193323Sed // same size, then a (misaligned) int store. 323193323Sed // FIXME: Does not handle truncating floating point stores! 324218893Sdim SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); 325234353Sdim Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(), 326234353Sdim ST->isVolatile(), ST->isNonTemporal(), Alignment); 327234353Sdim DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 328234353Sdim return; 329223017Sdim } 330223017Sdim // Do a (aligned) store to a stack slot, then copy from the stack slot 331223017Sdim // to the final destination using (unaligned) integer loads and stores. 332223017Sdim EVT StoredVT = ST->getMemoryVT(); 333249423Sdim MVT RegVT = 334223017Sdim TLI.getRegisterType(*DAG.getContext(), 335223017Sdim EVT::getIntegerVT(*DAG.getContext(), 336223017Sdim StoredVT.getSizeInBits())); 337223017Sdim unsigned StoredBytes = StoredVT.getSizeInBits() / 8; 338223017Sdim unsigned RegBytes = RegVT.getSizeInBits() / 8; 339223017Sdim unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; 340193323Sed 341223017Sdim // Make sure the stack slot is also aligned for the register type. 342223017Sdim SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); 343193323Sed 344223017Sdim // Perform the original store, only redirected to the stack slot. 345223017Sdim SDValue Store = DAG.getTruncStore(Chain, dl, 346223017Sdim Val, StackPtr, MachinePointerInfo(), 347223017Sdim StoredVT, false, false, 0); 348263508Sdim SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy(AS)); 349223017Sdim SmallVector<SDValue, 8> Stores; 350223017Sdim unsigned Offset = 0; 351193323Sed 352223017Sdim // Do all but one copies using the full register width. 353223017Sdim for (unsigned i = 1; i < NumRegs; i++) { 354223017Sdim // Load one integer register's worth from the stack slot. 355223017Sdim SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr, 356223017Sdim MachinePointerInfo(), 357234353Sdim false, false, false, 0); 358223017Sdim // Store it to the final location. Remember the store. 359223017Sdim Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, 360223017Sdim ST->getPointerInfo().getWithOffset(Offset), 361223017Sdim ST->isVolatile(), ST->isNonTemporal(), 362223017Sdim MinAlign(ST->getAlignment(), Offset))); 363223017Sdim // Increment the pointers. 364223017Sdim Offset += RegBytes; 365223017Sdim StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 366223017Sdim Increment); 367223017Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 368223017Sdim } 369193323Sed 370223017Sdim // The last store may be partial. Do a truncating store. On big-endian 371223017Sdim // machines this requires an extending load from the stack slot to ensure 372223017Sdim // that the bits are in the right place. 373223017Sdim EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 374223017Sdim 8 * (StoredBytes - Offset)); 375193323Sed 376223017Sdim // Load from the stack slot. 377223017Sdim SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, 378223017Sdim MachinePointerInfo(), 379223017Sdim MemVT, false, false, 0); 380193323Sed 381223017Sdim Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, 382223017Sdim ST->getPointerInfo() 383223017Sdim .getWithOffset(Offset), 384223017Sdim MemVT, ST->isVolatile(), 385223017Sdim ST->isNonTemporal(), 386263508Sdim MinAlign(ST->getAlignment(), Offset), 387263508Sdim ST->getTBAAInfo())); 388223017Sdim // The order of the stores doesn't matter - say it with a TokenFactor. 389234353Sdim SDValue Result = 390234353Sdim DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 391234353Sdim Stores.size()); 392234353Sdim DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 393234353Sdim return; 394193323Sed } 395193323Sed assert(ST->getMemoryVT().isInteger() && 396193323Sed !ST->getMemoryVT().isVector() && 397193323Sed "Unaligned store of unknown type."); 398193323Sed // Get the half-size VT 399201360Srdivacky EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); 400193323Sed int NumBits = NewStoredVT.getSizeInBits(); 401193323Sed int IncrementSize = NumBits / 8; 402193323Sed 403193323Sed // Divide the stored value in two parts. 404219077Sdim SDValue ShiftAmount = DAG.getConstant(NumBits, 405219077Sdim TLI.getShiftAmountTy(Val.getValueType())); 406193323Sed SDValue Lo = Val; 407193323Sed SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); 408193323Sed 409193323Sed // Store the two parts 410193323Sed SDValue Store1, Store2; 411193323Sed Store1 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Lo:Hi, Ptr, 412218893Sdim ST->getPointerInfo(), NewStoredVT, 413203954Srdivacky ST->isVolatile(), ST->isNonTemporal(), Alignment); 414263508Sdim 415193323Sed Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 416263508Sdim DAG.getConstant(IncrementSize, TLI.getPointerTy(AS))); 417193323Sed Alignment = MinAlign(Alignment, IncrementSize); 418193323Sed Store2 = DAG.getTruncStore(Chain, dl, TLI.isLittleEndian()?Hi:Lo, Ptr, 419218893Sdim ST->getPointerInfo().getWithOffset(IncrementSize), 420203954Srdivacky NewStoredVT, ST->isVolatile(), ST->isNonTemporal(), 421263508Sdim Alignment, ST->getTBAAInfo()); 422193323Sed 423234353Sdim SDValue Result = 424234353Sdim DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); 425234353Sdim DAGLegalize->ReplaceNode(SDValue(ST, 0), Result); 426193323Sed} 427193323Sed 428193323Sed/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. 429234353Sdimstatic void 430234353SdimExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, 431234353Sdim const TargetLowering &TLI, 432234353Sdim SDValue &ValResult, SDValue &ChainResult) { 433234353Sdim assert(LD->getAddressingMode() == ISD::UNINDEXED && 434234353Sdim "unaligned indexed loads not implemented!"); 435193323Sed SDValue Chain = LD->getChain(); 436193323Sed SDValue Ptr = LD->getBasePtr(); 437198090Srdivacky EVT VT = LD->getValueType(0); 438198090Srdivacky EVT LoadedVT = LD->getMemoryVT(); 439263508Sdim SDLoc dl(LD); 440193323Sed if (VT.isFloatingPoint() || VT.isVector()) { 441198090Srdivacky EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits()); 442239462Sdim if (TLI.isTypeLegal(intVT) && TLI.isTypeLegal(LoadedVT)) { 443193323Sed // Expand to a (misaligned) integer load of the same size, 444193323Sed // then bitconvert to floating point or vector. 445263508Sdim SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, 446263508Sdim LD->getMemOperand()); 447218893Sdim SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); 448239462Sdim if (LoadedVT != VT) 449239462Sdim Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : 450239462Sdim ISD::ANY_EXTEND, dl, VT, Result); 451193323Sed 452234353Sdim ValResult = Result; 453234353Sdim ChainResult = Chain; 454234353Sdim return; 455218893Sdim } 456193323Sed 457218893Sdim // Copy the value to a (aligned) stack slot using (unaligned) integer 458218893Sdim // loads and stores, then do a (aligned) load from the stack slot. 459249423Sdim MVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT); 460218893Sdim unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8; 461218893Sdim unsigned RegBytes = RegVT.getSizeInBits() / 8; 462218893Sdim unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes; 463193323Sed 464218893Sdim // Make sure the stack slot is also aligned for the register type. 465218893Sdim SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT); 466193323Sed 467218893Sdim SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy()); 468218893Sdim SmallVector<SDValue, 8> Stores; 469218893Sdim SDValue StackPtr = StackBase; 470218893Sdim unsigned Offset = 0; 471193323Sed 472218893Sdim // Do all but one copies using the full register width. 473218893Sdim for (unsigned i = 1; i < NumRegs; i++) { 474218893Sdim // Load one integer register's worth from the original location. 475218893Sdim SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr, 476218893Sdim LD->getPointerInfo().getWithOffset(Offset), 477218893Sdim LD->isVolatile(), LD->isNonTemporal(), 478234353Sdim LD->isInvariant(), 479263508Sdim MinAlign(LD->getAlignment(), Offset), 480263508Sdim LD->getTBAAInfo()); 481193323Sed // Follow the load with a store to the stack slot. Remember the store. 482218893Sdim Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr, 483218893Sdim MachinePointerInfo(), false, false, 0)); 484218893Sdim // Increment the pointers. 485218893Sdim Offset += RegBytes; 486218893Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 487218893Sdim StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 488218893Sdim Increment); 489218893Sdim } 490193323Sed 491218893Sdim // The last copy may be partial. Do an extending load. 492218893Sdim EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), 493218893Sdim 8 * (LoadedBytes - Offset)); 494218893Sdim SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, 495218893Sdim LD->getPointerInfo().getWithOffset(Offset), 496218893Sdim MemVT, LD->isVolatile(), 497218893Sdim LD->isNonTemporal(), 498263508Sdim MinAlign(LD->getAlignment(), Offset), 499263508Sdim LD->getTBAAInfo()); 500218893Sdim // Follow the load with a store to the stack slot. Remember the store. 501218893Sdim // On big-endian machines this requires a truncating store to ensure 502218893Sdim // that the bits end up in the right place. 503218893Sdim Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr, 504218893Sdim MachinePointerInfo(), MemVT, 505218893Sdim false, false, 0)); 506193323Sed 507218893Sdim // The order of the stores doesn't matter - say it with a TokenFactor. 508218893Sdim SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Stores[0], 509218893Sdim Stores.size()); 510193323Sed 511218893Sdim // Finally, perform the original load only redirected to the stack slot. 512218893Sdim Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, 513218893Sdim MachinePointerInfo(), LoadedVT, false, false, 0); 514218893Sdim 515218893Sdim // Callers expect a MERGE_VALUES node. 516234353Sdim ValResult = Load; 517234353Sdim ChainResult = TF; 518234353Sdim return; 519193323Sed } 520193323Sed assert(LoadedVT.isInteger() && !LoadedVT.isVector() && 521193323Sed "Unaligned load of unsupported type."); 522193323Sed 523193323Sed // Compute the new VT that is half the size of the old one. This is an 524193323Sed // integer MVT. 525193323Sed unsigned NumBits = LoadedVT.getSizeInBits(); 526198090Srdivacky EVT NewLoadedVT; 527198090Srdivacky NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2); 528193323Sed NumBits >>= 1; 529193323Sed 530193323Sed unsigned Alignment = LD->getAlignment(); 531193323Sed unsigned IncrementSize = NumBits / 8; 532193323Sed ISD::LoadExtType HiExtType = LD->getExtensionType(); 533193323Sed 534193323Sed // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD. 535193323Sed if (HiExtType == ISD::NON_EXTLOAD) 536193323Sed HiExtType = ISD::ZEXTLOAD; 537193323Sed 538193323Sed // Load the value in two parts 539193323Sed SDValue Lo, Hi; 540193323Sed if (TLI.isLittleEndian()) { 541218893Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), 542218893Sdim NewLoadedVT, LD->isVolatile(), 543263508Sdim LD->isNonTemporal(), Alignment, LD->getTBAAInfo()); 544193323Sed Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 545263508Sdim DAG.getConstant(IncrementSize, Ptr.getValueType())); 546218893Sdim Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, 547218893Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 548218893Sdim NewLoadedVT, LD->isVolatile(), 549263508Sdim LD->isNonTemporal(), MinAlign(Alignment, IncrementSize), 550263508Sdim LD->getTBAAInfo()); 551193323Sed } else { 552218893Sdim Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), 553218893Sdim NewLoadedVT, LD->isVolatile(), 554263508Sdim LD->isNonTemporal(), Alignment, LD->getTBAAInfo()); 555193323Sed Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 556263508Sdim DAG.getConstant(IncrementSize, Ptr.getValueType())); 557218893Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, 558218893Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 559218893Sdim NewLoadedVT, LD->isVolatile(), 560263508Sdim LD->isNonTemporal(), MinAlign(Alignment, IncrementSize), 561263508Sdim LD->getTBAAInfo()); 562193323Sed } 563193323Sed 564193323Sed // aggregate the two parts 565219077Sdim SDValue ShiftAmount = DAG.getConstant(NumBits, 566219077Sdim TLI.getShiftAmountTy(Hi.getValueType())); 567193323Sed SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); 568193323Sed Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); 569193323Sed 570193323Sed SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 571193323Sed Hi.getValue(1)); 572193323Sed 573234353Sdim ValResult = Result; 574234353Sdim ChainResult = TF; 575193323Sed} 576193323Sed 577193323Sed/// PerformInsertVectorEltInMemory - Some target cannot handle a variable 578193323Sed/// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it 579193323Sed/// is necessary to spill the vector being inserted into to memory, perform 580193323Sed/// the insert there, and then read the result back. 581193323SedSDValue SelectionDAGLegalize:: 582193323SedPerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, 583263508Sdim SDLoc dl) { 584193323Sed SDValue Tmp1 = Vec; 585193323Sed SDValue Tmp2 = Val; 586193323Sed SDValue Tmp3 = Idx; 587193323Sed 588193323Sed // If the target doesn't support this, we have to spill the input vector 589193323Sed // to a temporary stack slot, update the element, then reload it. This is 590193323Sed // badness. We could also load the value into a vector register (either 591193323Sed // with a "move to register" or "extload into register" instruction, then 592193323Sed // permute it into place, if the idx is a constant and if the idx is 593193323Sed // supported by the target. 594198090Srdivacky EVT VT = Tmp1.getValueType(); 595198090Srdivacky EVT EltVT = VT.getVectorElementType(); 596198090Srdivacky EVT IdxVT = Tmp3.getValueType(); 597198090Srdivacky EVT PtrVT = TLI.getPointerTy(); 598193323Sed SDValue StackPtr = DAG.CreateStackTemporary(VT); 599193323Sed 600193323Sed int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 601193323Sed 602193323Sed // Store the vector. 603193323Sed SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr, 604218893Sdim MachinePointerInfo::getFixedStack(SPFI), 605203954Srdivacky false, false, 0); 606193323Sed 607193323Sed // Truncate or zero extend offset to target pointer type. 608193323Sed unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 609193323Sed Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3); 610193323Sed // Add the offset to the index. 611193323Sed unsigned EltSize = EltVT.getSizeInBits()/8; 612193323Sed Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); 613193323Sed SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); 614193323Sed // Store the scalar value. 615218893Sdim Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT, 616203954Srdivacky false, false, 0); 617193323Sed // Load the updated vector. 618193323Sed return DAG.getLoad(VT, dl, Ch, StackPtr, 619263508Sdim MachinePointerInfo::getFixedStack(SPFI), false, false, 620234353Sdim false, 0); 621193323Sed} 622193323Sed 623193323Sed 624193323SedSDValue SelectionDAGLegalize:: 625263508SdimExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, SDLoc dl) { 626193323Sed if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { 627193323Sed // SCALAR_TO_VECTOR requires that the type of the value being inserted 628193323Sed // match the element type of the vector being created, except for 629193323Sed // integers in which case the inserted value can be over width. 630198090Srdivacky EVT EltVT = Vec.getValueType().getVectorElementType(); 631193323Sed if (Val.getValueType() == EltVT || 632193323Sed (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { 633193323Sed SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, 634193323Sed Vec.getValueType(), Val); 635193323Sed 636193323Sed unsigned NumElts = Vec.getValueType().getVectorNumElements(); 637193323Sed // We generate a shuffle of InVec and ScVec, so the shuffle mask 638193323Sed // should be 0,1,2,3,4,5... with the appropriate element replaced with 639193323Sed // elt 0 of the RHS. 640193323Sed SmallVector<int, 8> ShufOps; 641193323Sed for (unsigned i = 0; i != NumElts; ++i) 642193323Sed ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); 643193323Sed 644193323Sed return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, 645193323Sed &ShufOps[0]); 646193323Sed } 647193323Sed } 648193323Sed return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); 649193323Sed} 650193323Sed 651193574SedSDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { 652193574Sed // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 653193574Sed // FIXME: We shouldn't do this for TargetConstantFP's. 654193574Sed // FIXME: move this to the DAG Combiner! Note that we can't regress due 655193574Sed // to phase ordering between legalized code and the dag combiner. This 656193574Sed // probably means that we need to integrate dag combiner and legalizer 657193574Sed // together. 658193574Sed // We generally can't do this one for long doubles. 659239462Sdim SDValue Chain = ST->getChain(); 660239462Sdim SDValue Ptr = ST->getBasePtr(); 661193574Sed unsigned Alignment = ST->getAlignment(); 662193574Sed bool isVolatile = ST->isVolatile(); 663203954Srdivacky bool isNonTemporal = ST->isNonTemporal(); 664263508Sdim const MDNode *TBAAInfo = ST->getTBAAInfo(); 665263508Sdim SDLoc dl(ST); 666193574Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { 667193574Sed if (CFP->getValueType(0) == MVT::f32 && 668224145Sdim TLI.isTypeLegal(MVT::i32)) { 669239462Sdim SDValue Con = DAG.getConstant(CFP->getValueAPF(). 670193574Sed bitcastToAPInt().zextOrTrunc(32), 671193574Sed MVT::i32); 672239462Sdim return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 673263508Sdim isVolatile, isNonTemporal, Alignment, TBAAInfo); 674218893Sdim } 675218893Sdim 676218893Sdim if (CFP->getValueType(0) == MVT::f64) { 677193574Sed // If this target supports 64-bit registers, do a single 64-bit store. 678224145Sdim if (TLI.isTypeLegal(MVT::i64)) { 679239462Sdim SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 680193574Sed zextOrTrunc(64), MVT::i64); 681239462Sdim return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), 682263508Sdim isVolatile, isNonTemporal, Alignment, TBAAInfo); 683218893Sdim } 684218893Sdim 685224145Sdim if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { 686193574Sed // Otherwise, if the target supports 32-bit registers, use 2 32-bit 687193574Sed // stores. If the target supports neither 32- nor 64-bits, this 688193574Sed // xform is certainly not worth it. 689193574Sed const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); 690218893Sdim SDValue Lo = DAG.getConstant(IntVal.trunc(32), MVT::i32); 691193574Sed SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); 692193574Sed if (TLI.isBigEndian()) std::swap(Lo, Hi); 693193574Sed 694239462Sdim Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), isVolatile, 695263508Sdim isNonTemporal, Alignment, TBAAInfo); 696239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 697263508Sdim DAG.getConstant(4, Ptr.getValueType())); 698239462Sdim Hi = DAG.getStore(Chain, dl, Hi, Ptr, 699218893Sdim ST->getPointerInfo().getWithOffset(4), 700263508Sdim isVolatile, isNonTemporal, MinAlign(Alignment, 4U), 701263508Sdim TBAAInfo); 702193574Sed 703193574Sed return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 704193574Sed } 705193574Sed } 706193574Sed } 707221345Sdim return SDValue(0, 0); 708193574Sed} 709193574Sed 710239462Sdimvoid SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { 711239462Sdim StoreSDNode *ST = cast<StoreSDNode>(Node); 712239462Sdim SDValue Chain = ST->getChain(); 713239462Sdim SDValue Ptr = ST->getBasePtr(); 714263508Sdim SDLoc dl(Node); 715239462Sdim 716239462Sdim unsigned Alignment = ST->getAlignment(); 717239462Sdim bool isVolatile = ST->isVolatile(); 718239462Sdim bool isNonTemporal = ST->isNonTemporal(); 719263508Sdim const MDNode *TBAAInfo = ST->getTBAAInfo(); 720239462Sdim 721239462Sdim if (!ST->isTruncatingStore()) { 722239462Sdim if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { 723239462Sdim ReplaceNode(ST, OptStore); 724239462Sdim return; 725239462Sdim } 726239462Sdim 727239462Sdim { 728239462Sdim SDValue Value = ST->getValue(); 729249423Sdim MVT VT = Value.getSimpleValueType(); 730239462Sdim switch (TLI.getOperationAction(ISD::STORE, VT)) { 731239462Sdim default: llvm_unreachable("This action is not supported yet!"); 732239462Sdim case TargetLowering::Legal: 733239462Sdim // If this is an unaligned store and the target doesn't support it, 734239462Sdim // expand it. 735239462Sdim if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 736239462Sdim Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 737243830Sdim unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 738239462Sdim if (ST->getAlignment() < ABIAlignment) 739239462Sdim ExpandUnalignedStore(cast<StoreSDNode>(Node), 740239462Sdim DAG, TLI, this); 741239462Sdim } 742239462Sdim break; 743239462Sdim case TargetLowering::Custom: { 744239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 745239462Sdim if (Res.getNode()) 746239462Sdim ReplaceNode(SDValue(Node, 0), Res); 747239462Sdim return; 748239462Sdim } 749239462Sdim case TargetLowering::Promote: { 750249423Sdim MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); 751249423Sdim assert(NVT.getSizeInBits() == VT.getSizeInBits() && 752249423Sdim "Can only promote stores to same size type"); 753249423Sdim Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); 754239462Sdim SDValue Result = 755239462Sdim DAG.getStore(Chain, dl, Value, Ptr, 756239462Sdim ST->getPointerInfo(), isVolatile, 757263508Sdim isNonTemporal, Alignment, TBAAInfo); 758239462Sdim ReplaceNode(SDValue(Node, 0), Result); 759239462Sdim break; 760239462Sdim } 761239462Sdim } 762239462Sdim return; 763239462Sdim } 764239462Sdim } else { 765239462Sdim SDValue Value = ST->getValue(); 766239462Sdim 767239462Sdim EVT StVT = ST->getMemoryVT(); 768239462Sdim unsigned StWidth = StVT.getSizeInBits(); 769239462Sdim 770239462Sdim if (StWidth != StVT.getStoreSizeInBits()) { 771239462Sdim // Promote to a byte-sized store with upper bits zero if not 772239462Sdim // storing an integral number of bytes. For example, promote 773239462Sdim // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) 774239462Sdim EVT NVT = EVT::getIntegerVT(*DAG.getContext(), 775239462Sdim StVT.getStoreSizeInBits()); 776239462Sdim Value = DAG.getZeroExtendInReg(Value, dl, StVT); 777239462Sdim SDValue Result = 778239462Sdim DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 779263508Sdim NVT, isVolatile, isNonTemporal, Alignment, 780263508Sdim TBAAInfo); 781239462Sdim ReplaceNode(SDValue(Node, 0), Result); 782239462Sdim } else if (StWidth & (StWidth - 1)) { 783239462Sdim // If not storing a power-of-2 number of bits, expand as two stores. 784239462Sdim assert(!StVT.isVector() && "Unsupported truncstore!"); 785239462Sdim unsigned RoundWidth = 1 << Log2_32(StWidth); 786239462Sdim assert(RoundWidth < StWidth); 787239462Sdim unsigned ExtraWidth = StWidth - RoundWidth; 788239462Sdim assert(ExtraWidth < RoundWidth); 789239462Sdim assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 790239462Sdim "Store size not an integral number of bytes!"); 791239462Sdim EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 792239462Sdim EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 793239462Sdim SDValue Lo, Hi; 794239462Sdim unsigned IncrementSize; 795239462Sdim 796239462Sdim if (TLI.isLittleEndian()) { 797239462Sdim // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) 798239462Sdim // Store the bottom RoundWidth bits. 799239462Sdim Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 800239462Sdim RoundVT, 801263508Sdim isVolatile, isNonTemporal, Alignment, 802263508Sdim TBAAInfo); 803239462Sdim 804239462Sdim // Store the remaining ExtraWidth bits. 805239462Sdim IncrementSize = RoundWidth / 8; 806239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 807263508Sdim DAG.getConstant(IncrementSize, Ptr.getValueType())); 808239462Sdim Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 809239462Sdim DAG.getConstant(RoundWidth, 810239462Sdim TLI.getShiftAmountTy(Value.getValueType()))); 811239462Sdim Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, 812239462Sdim ST->getPointerInfo().getWithOffset(IncrementSize), 813239462Sdim ExtraVT, isVolatile, isNonTemporal, 814263508Sdim MinAlign(Alignment, IncrementSize), TBAAInfo); 815239462Sdim } else { 816239462Sdim // Big endian - avoid unaligned stores. 817239462Sdim // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X 818239462Sdim // Store the top RoundWidth bits. 819239462Sdim Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, 820239462Sdim DAG.getConstant(ExtraWidth, 821239462Sdim TLI.getShiftAmountTy(Value.getValueType()))); 822239462Sdim Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), 823263508Sdim RoundVT, isVolatile, isNonTemporal, Alignment, 824263508Sdim TBAAInfo); 825239462Sdim 826239462Sdim // Store the remaining ExtraWidth bits. 827239462Sdim IncrementSize = RoundWidth / 8; 828239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 829263508Sdim DAG.getConstant(IncrementSize, Ptr.getValueType())); 830239462Sdim Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, 831239462Sdim ST->getPointerInfo().getWithOffset(IncrementSize), 832239462Sdim ExtraVT, isVolatile, isNonTemporal, 833263508Sdim MinAlign(Alignment, IncrementSize), TBAAInfo); 834239462Sdim } 835239462Sdim 836239462Sdim // The order of the stores doesn't matter. 837239462Sdim SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); 838239462Sdim ReplaceNode(SDValue(Node, 0), Result); 839239462Sdim } else { 840249423Sdim switch (TLI.getTruncStoreAction(ST->getValue().getSimpleValueType(), 841249423Sdim StVT.getSimpleVT())) { 842239462Sdim default: llvm_unreachable("This action is not supported yet!"); 843239462Sdim case TargetLowering::Legal: 844239462Sdim // If this is an unaligned store and the target doesn't support it, 845239462Sdim // expand it. 846239462Sdim if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) { 847239462Sdim Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext()); 848243830Sdim unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty); 849239462Sdim if (ST->getAlignment() < ABIAlignment) 850239462Sdim ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this); 851239462Sdim } 852239462Sdim break; 853239462Sdim case TargetLowering::Custom: { 854239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 855239462Sdim if (Res.getNode()) 856239462Sdim ReplaceNode(SDValue(Node, 0), Res); 857239462Sdim return; 858239462Sdim } 859239462Sdim case TargetLowering::Expand: 860239462Sdim assert(!StVT.isVector() && 861239462Sdim "Vector Stores are handled in LegalizeVectorOps"); 862239462Sdim 863239462Sdim // TRUNCSTORE:i16 i32 -> STORE i16 864239462Sdim assert(TLI.isTypeLegal(StVT) && 865239462Sdim "Do not know how to expand this store!"); 866239462Sdim Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); 867239462Sdim SDValue Result = 868239462Sdim DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), 869263508Sdim isVolatile, isNonTemporal, Alignment, TBAAInfo); 870239462Sdim ReplaceNode(SDValue(Node, 0), Result); 871239462Sdim break; 872239462Sdim } 873239462Sdim } 874239462Sdim } 875239462Sdim} 876239462Sdim 877239462Sdimvoid SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { 878239462Sdim LoadSDNode *LD = cast<LoadSDNode>(Node); 879239462Sdim SDValue Chain = LD->getChain(); // The chain. 880239462Sdim SDValue Ptr = LD->getBasePtr(); // The base pointer. 881239462Sdim SDValue Value; // The value returned by the load op. 882263508Sdim SDLoc dl(Node); 883239462Sdim 884239462Sdim ISD::LoadExtType ExtType = LD->getExtensionType(); 885239462Sdim if (ExtType == ISD::NON_EXTLOAD) { 886249423Sdim MVT VT = Node->getSimpleValueType(0); 887239462Sdim SDValue RVal = SDValue(Node, 0); 888239462Sdim SDValue RChain = SDValue(Node, 1); 889239462Sdim 890239462Sdim switch (TLI.getOperationAction(Node->getOpcode(), VT)) { 891239462Sdim default: llvm_unreachable("This action is not supported yet!"); 892239462Sdim case TargetLowering::Legal: 893243830Sdim // If this is an unaligned load and the target doesn't support it, 894243830Sdim // expand it. 895243830Sdim if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 896243830Sdim Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 897243830Sdim unsigned ABIAlignment = 898243830Sdim TLI.getDataLayout()->getABITypeAlignment(Ty); 899243830Sdim if (LD->getAlignment() < ABIAlignment){ 900243830Sdim ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain); 901243830Sdim } 902243830Sdim } 903243830Sdim break; 904239462Sdim case TargetLowering::Custom: { 905243830Sdim SDValue Res = TLI.LowerOperation(RVal, DAG); 906243830Sdim if (Res.getNode()) { 907243830Sdim RVal = Res; 908243830Sdim RChain = Res.getValue(1); 909243830Sdim } 910243830Sdim break; 911239462Sdim } 912239462Sdim case TargetLowering::Promote: { 913249423Sdim MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); 914249423Sdim assert(NVT.getSizeInBits() == VT.getSizeInBits() && 915249423Sdim "Can only promote loads to same size type"); 916239462Sdim 917263508Sdim SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); 918239462Sdim RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); 919239462Sdim RChain = Res.getValue(1); 920239462Sdim break; 921239462Sdim } 922239462Sdim } 923239462Sdim if (RChain.getNode() != Node) { 924239462Sdim assert(RVal.getNode() != Node && "Load must be completely replaced"); 925239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); 926239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); 927239462Sdim ReplacedNode(Node); 928239462Sdim } 929239462Sdim return; 930239462Sdim } 931239462Sdim 932239462Sdim EVT SrcVT = LD->getMemoryVT(); 933239462Sdim unsigned SrcWidth = SrcVT.getSizeInBits(); 934239462Sdim unsigned Alignment = LD->getAlignment(); 935239462Sdim bool isVolatile = LD->isVolatile(); 936239462Sdim bool isNonTemporal = LD->isNonTemporal(); 937263508Sdim const MDNode *TBAAInfo = LD->getTBAAInfo(); 938239462Sdim 939239462Sdim if (SrcWidth != SrcVT.getStoreSizeInBits() && 940239462Sdim // Some targets pretend to have an i1 loading operation, and actually 941239462Sdim // load an i8. This trick is correct for ZEXTLOAD because the top 7 942239462Sdim // bits are guaranteed to be zero; it helps the optimizers understand 943239462Sdim // that these bits are zero. It is also useful for EXTLOAD, since it 944239462Sdim // tells the optimizers that those bits are undefined. It would be 945239462Sdim // nice to have an effective generic way of getting these benefits... 946239462Sdim // Until such a way is found, don't insist on promoting i1 here. 947239462Sdim (SrcVT != MVT::i1 || 948239462Sdim TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { 949239462Sdim // Promote to a byte-sized load if not loading an integral number of 950239462Sdim // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. 951239462Sdim unsigned NewWidth = SrcVT.getStoreSizeInBits(); 952239462Sdim EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); 953239462Sdim SDValue Ch; 954239462Sdim 955239462Sdim // The extra bits are guaranteed to be zero, since we stored them that 956239462Sdim // way. A zext load from NVT thus automatically gives zext from SrcVT. 957239462Sdim 958239462Sdim ISD::LoadExtType NewExtType = 959239462Sdim ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; 960239462Sdim 961239462Sdim SDValue Result = 962239462Sdim DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), 963239462Sdim Chain, Ptr, LD->getPointerInfo(), 964263508Sdim NVT, isVolatile, isNonTemporal, Alignment, TBAAInfo); 965239462Sdim 966239462Sdim Ch = Result.getValue(1); // The chain. 967239462Sdim 968239462Sdim if (ExtType == ISD::SEXTLOAD) 969239462Sdim // Having the top bits zero doesn't help when sign extending. 970239462Sdim Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 971239462Sdim Result.getValueType(), 972239462Sdim Result, DAG.getValueType(SrcVT)); 973239462Sdim else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) 974239462Sdim // All the top bits are guaranteed to be zero - inform the optimizers. 975239462Sdim Result = DAG.getNode(ISD::AssertZext, dl, 976239462Sdim Result.getValueType(), Result, 977239462Sdim DAG.getValueType(SrcVT)); 978239462Sdim 979239462Sdim Value = Result; 980239462Sdim Chain = Ch; 981239462Sdim } else if (SrcWidth & (SrcWidth - 1)) { 982239462Sdim // If not loading a power-of-2 number of bits, expand as two loads. 983239462Sdim assert(!SrcVT.isVector() && "Unsupported extload!"); 984239462Sdim unsigned RoundWidth = 1 << Log2_32(SrcWidth); 985239462Sdim assert(RoundWidth < SrcWidth); 986239462Sdim unsigned ExtraWidth = SrcWidth - RoundWidth; 987239462Sdim assert(ExtraWidth < RoundWidth); 988239462Sdim assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && 989239462Sdim "Load size not an integral number of bytes!"); 990239462Sdim EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); 991239462Sdim EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); 992239462Sdim SDValue Lo, Hi, Ch; 993239462Sdim unsigned IncrementSize; 994239462Sdim 995239462Sdim if (TLI.isLittleEndian()) { 996239462Sdim // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) 997239462Sdim // Load the bottom RoundWidth bits. 998239462Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), 999239462Sdim Chain, Ptr, 1000239462Sdim LD->getPointerInfo(), RoundVT, isVolatile, 1001263508Sdim isNonTemporal, Alignment, TBAAInfo); 1002239462Sdim 1003239462Sdim // Load the remaining ExtraWidth bits. 1004239462Sdim IncrementSize = RoundWidth / 8; 1005239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1006263508Sdim DAG.getConstant(IncrementSize, Ptr.getValueType())); 1007239462Sdim Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 1008239462Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 1009239462Sdim ExtraVT, isVolatile, isNonTemporal, 1010263508Sdim MinAlign(Alignment, IncrementSize), TBAAInfo); 1011239462Sdim 1012239462Sdim // Build a factor node to remember that this load is independent of 1013239462Sdim // the other one. 1014239462Sdim Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1015239462Sdim Hi.getValue(1)); 1016239462Sdim 1017239462Sdim // Move the top bits to the right place. 1018239462Sdim Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1019239462Sdim DAG.getConstant(RoundWidth, 1020239462Sdim TLI.getShiftAmountTy(Hi.getValueType()))); 1021239462Sdim 1022239462Sdim // Join the hi and lo parts. 1023239462Sdim Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1024239462Sdim } else { 1025239462Sdim // Big endian - avoid unaligned loads. 1026239462Sdim // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 1027239462Sdim // Load the top RoundWidth bits. 1028239462Sdim Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, 1029239462Sdim LD->getPointerInfo(), RoundVT, isVolatile, 1030263508Sdim isNonTemporal, Alignment, TBAAInfo); 1031239462Sdim 1032239462Sdim // Load the remaining ExtraWidth bits. 1033239462Sdim IncrementSize = RoundWidth / 8; 1034239462Sdim Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 1035263508Sdim DAG.getConstant(IncrementSize, Ptr.getValueType())); 1036239462Sdim Lo = DAG.getExtLoad(ISD::ZEXTLOAD, 1037239462Sdim dl, Node->getValueType(0), Chain, Ptr, 1038239462Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 1039239462Sdim ExtraVT, isVolatile, isNonTemporal, 1040263508Sdim MinAlign(Alignment, IncrementSize), TBAAInfo); 1041239462Sdim 1042239462Sdim // Build a factor node to remember that this load is independent of 1043239462Sdim // the other one. 1044239462Sdim Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1045239462Sdim Hi.getValue(1)); 1046239462Sdim 1047239462Sdim // Move the top bits to the right place. 1048239462Sdim Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, 1049239462Sdim DAG.getConstant(ExtraWidth, 1050239462Sdim TLI.getShiftAmountTy(Hi.getValueType()))); 1051239462Sdim 1052239462Sdim // Join the hi and lo parts. 1053239462Sdim Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); 1054239462Sdim } 1055239462Sdim 1056239462Sdim Chain = Ch; 1057239462Sdim } else { 1058239462Sdim bool isCustom = false; 1059249423Sdim switch (TLI.getLoadExtAction(ExtType, SrcVT.getSimpleVT())) { 1060239462Sdim default: llvm_unreachable("This action is not supported yet!"); 1061239462Sdim case TargetLowering::Custom: 1062239462Sdim isCustom = true; 1063239462Sdim // FALLTHROUGH 1064239462Sdim case TargetLowering::Legal: { 1065239462Sdim Value = SDValue(Node, 0); 1066239462Sdim Chain = SDValue(Node, 1); 1067239462Sdim 1068239462Sdim if (isCustom) { 1069239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1070239462Sdim if (Res.getNode()) { 1071239462Sdim Value = Res; 1072239462Sdim Chain = Res.getValue(1); 1073239462Sdim } 1074239462Sdim } else { 1075239462Sdim // If this is an unaligned load and the target doesn't support it, 1076239462Sdim // expand it. 1077239462Sdim if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) { 1078239462Sdim Type *Ty = 1079239462Sdim LD->getMemoryVT().getTypeForEVT(*DAG.getContext()); 1080239462Sdim unsigned ABIAlignment = 1081243830Sdim TLI.getDataLayout()->getABITypeAlignment(Ty); 1082239462Sdim if (LD->getAlignment() < ABIAlignment){ 1083239462Sdim ExpandUnalignedLoad(cast<LoadSDNode>(Node), 1084239462Sdim DAG, TLI, Value, Chain); 1085239462Sdim } 1086239462Sdim } 1087239462Sdim } 1088239462Sdim break; 1089239462Sdim } 1090239462Sdim case TargetLowering::Expand: 1091239462Sdim if (!TLI.isLoadExtLegal(ISD::EXTLOAD, SrcVT) && TLI.isTypeLegal(SrcVT)) { 1092239462Sdim SDValue Load = DAG.getLoad(SrcVT, dl, Chain, Ptr, 1093263508Sdim LD->getMemOperand()); 1094239462Sdim unsigned ExtendOp; 1095239462Sdim switch (ExtType) { 1096239462Sdim case ISD::EXTLOAD: 1097239462Sdim ExtendOp = (SrcVT.isFloatingPoint() ? 1098239462Sdim ISD::FP_EXTEND : ISD::ANY_EXTEND); 1099239462Sdim break; 1100239462Sdim case ISD::SEXTLOAD: ExtendOp = ISD::SIGN_EXTEND; break; 1101239462Sdim case ISD::ZEXTLOAD: ExtendOp = ISD::ZERO_EXTEND; break; 1102239462Sdim default: llvm_unreachable("Unexpected extend load type!"); 1103239462Sdim } 1104239462Sdim Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); 1105239462Sdim Chain = Load.getValue(1); 1106239462Sdim break; 1107239462Sdim } 1108239462Sdim 1109239462Sdim assert(!SrcVT.isVector() && 1110239462Sdim "Vector Loads are handled in LegalizeVectorOps"); 1111239462Sdim 1112239462Sdim // FIXME: This does not work for vectors on most targets. Sign- and 1113239462Sdim // zero-extend operations are currently folded into extending loads, 1114239462Sdim // whether they are legal or not, and then we end up here without any 1115239462Sdim // support for legalizing them. 1116239462Sdim assert(ExtType != ISD::EXTLOAD && 1117239462Sdim "EXTLOAD should always be supported!"); 1118239462Sdim // Turn the unsupported load into an EXTLOAD followed by an explicit 1119239462Sdim // zero/sign extend inreg. 1120239462Sdim SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, Node->getValueType(0), 1121263508Sdim Chain, Ptr, SrcVT, 1122263508Sdim LD->getMemOperand()); 1123239462Sdim SDValue ValRes; 1124239462Sdim if (ExtType == ISD::SEXTLOAD) 1125239462Sdim ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, 1126239462Sdim Result.getValueType(), 1127239462Sdim Result, DAG.getValueType(SrcVT)); 1128239462Sdim else 1129239462Sdim ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); 1130239462Sdim Value = ValRes; 1131239462Sdim Chain = Result.getValue(1); 1132239462Sdim break; 1133239462Sdim } 1134239462Sdim } 1135239462Sdim 1136239462Sdim // Since loads produce two values, make sure to remember that we legalized 1137239462Sdim // both of them. 1138239462Sdim if (Chain.getNode() != Node) { 1139239462Sdim assert(Value.getNode() != Node && "Load must be completely replaced"); 1140239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); 1141239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 1142239462Sdim ReplacedNode(Node); 1143239462Sdim } 1144239462Sdim} 1145239462Sdim 1146224145Sdim/// LegalizeOp - Return a legal replacement for the given operation, with 1147224145Sdim/// all legal operands. 1148234353Sdimvoid SelectionDAGLegalize::LegalizeOp(SDNode *Node) { 1149234353Sdim if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. 1150234353Sdim return; 1151193323Sed 1152193323Sed for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1153224145Sdim assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == 1154224145Sdim TargetLowering::TypeLegal && 1155193323Sed "Unexpected illegal type!"); 1156193323Sed 1157193323Sed for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) 1158224145Sdim assert((TLI.getTypeAction(*DAG.getContext(), 1159224145Sdim Node->getOperand(i).getValueType()) == 1160224145Sdim TargetLowering::TypeLegal || 1161193323Sed Node->getOperand(i).getOpcode() == ISD::TargetConstant) && 1162193323Sed "Unexpected illegal type!"); 1163193323Sed 1164193323Sed // Figure out the correct action; the way to query this varies by opcode 1165218893Sdim TargetLowering::LegalizeAction Action = TargetLowering::Legal; 1166193323Sed bool SimpleFinishLegalizing = true; 1167193323Sed switch (Node->getOpcode()) { 1168193323Sed case ISD::INTRINSIC_W_CHAIN: 1169193323Sed case ISD::INTRINSIC_WO_CHAIN: 1170193323Sed case ISD::INTRINSIC_VOID: 1171193323Sed case ISD::STACKSAVE: 1172193323Sed Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1173193323Sed break; 1174234353Sdim case ISD::VAARG: 1175234353Sdim Action = TLI.getOperationAction(Node->getOpcode(), 1176234353Sdim Node->getValueType(0)); 1177234353Sdim if (Action != TargetLowering::Promote) 1178234353Sdim Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); 1179234353Sdim break; 1180193323Sed case ISD::SINT_TO_FP: 1181193323Sed case ISD::UINT_TO_FP: 1182193323Sed case ISD::EXTRACT_VECTOR_ELT: 1183193323Sed Action = TLI.getOperationAction(Node->getOpcode(), 1184193323Sed Node->getOperand(0).getValueType()); 1185193323Sed break; 1186193323Sed case ISD::FP_ROUND_INREG: 1187193323Sed case ISD::SIGN_EXTEND_INREG: { 1188198090Srdivacky EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); 1189193323Sed Action = TLI.getOperationAction(Node->getOpcode(), InnerType); 1190193323Sed break; 1191193323Sed } 1192226633Sdim case ISD::ATOMIC_STORE: { 1193226633Sdim Action = TLI.getOperationAction(Node->getOpcode(), 1194226633Sdim Node->getOperand(2).getValueType()); 1195226633Sdim break; 1196226633Sdim } 1197193323Sed case ISD::SELECT_CC: 1198193323Sed case ISD::SETCC: 1199193323Sed case ISD::BR_CC: { 1200193323Sed unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : 1201193323Sed Node->getOpcode() == ISD::SETCC ? 2 : 1; 1202193323Sed unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; 1203249423Sdim MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); 1204193323Sed ISD::CondCode CCCode = 1205193323Sed cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); 1206193323Sed Action = TLI.getCondCodeAction(CCCode, OpVT); 1207193323Sed if (Action == TargetLowering::Legal) { 1208193323Sed if (Node->getOpcode() == ISD::SELECT_CC) 1209193323Sed Action = TLI.getOperationAction(Node->getOpcode(), 1210193323Sed Node->getValueType(0)); 1211193323Sed else 1212193323Sed Action = TLI.getOperationAction(Node->getOpcode(), OpVT); 1213193323Sed } 1214193323Sed break; 1215193323Sed } 1216193323Sed case ISD::LOAD: 1217193323Sed case ISD::STORE: 1218193323Sed // FIXME: Model these properly. LOAD and STORE are complicated, and 1219193323Sed // STORE expects the unlegalized operand in some cases. 1220193323Sed SimpleFinishLegalizing = false; 1221193323Sed break; 1222193323Sed case ISD::CALLSEQ_START: 1223193323Sed case ISD::CALLSEQ_END: 1224193323Sed // FIXME: This shouldn't be necessary. These nodes have special properties 1225193323Sed // dealing with the recursive nature of legalization. Removing this 1226193323Sed // special case should be done as part of making LegalizeDAG non-recursive. 1227193323Sed SimpleFinishLegalizing = false; 1228193323Sed break; 1229193323Sed case ISD::EXTRACT_ELEMENT: 1230193323Sed case ISD::FLT_ROUNDS_: 1231193323Sed case ISD::SADDO: 1232193323Sed case ISD::SSUBO: 1233193323Sed case ISD::UADDO: 1234193323Sed case ISD::USUBO: 1235193323Sed case ISD::SMULO: 1236193323Sed case ISD::UMULO: 1237193323Sed case ISD::FPOWI: 1238193323Sed case ISD::MERGE_VALUES: 1239193323Sed case ISD::EH_RETURN: 1240193323Sed case ISD::FRAME_TO_ARGS_OFFSET: 1241210299Sed case ISD::EH_SJLJ_SETJMP: 1242210299Sed case ISD::EH_SJLJ_LONGJMP: 1243193323Sed // These operations lie about being legal: when they claim to be legal, 1244193323Sed // they should actually be expanded. 1245193323Sed Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1246193323Sed if (Action == TargetLowering::Legal) 1247193323Sed Action = TargetLowering::Expand; 1248193323Sed break; 1249226633Sdim case ISD::INIT_TRAMPOLINE: 1250226633Sdim case ISD::ADJUST_TRAMPOLINE: 1251193323Sed case ISD::FRAMEADDR: 1252193323Sed case ISD::RETURNADDR: 1253193323Sed // These operations lie about being legal: when they claim to be legal, 1254193323Sed // they should actually be custom-lowered. 1255193323Sed Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1256193323Sed if (Action == TargetLowering::Legal) 1257193323Sed Action = TargetLowering::Custom; 1258193323Sed break; 1259243830Sdim case ISD::DEBUGTRAP: 1260243830Sdim Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1261243830Sdim if (Action == TargetLowering::Expand) { 1262243830Sdim // replace ISD::DEBUGTRAP with ISD::TRAP 1263243830Sdim SDValue NewVal; 1264263508Sdim NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), 1265243830Sdim Node->getOperand(0)); 1266243830Sdim ReplaceNode(Node, NewVal.getNode()); 1267243830Sdim LegalizeOp(NewVal.getNode()); 1268243830Sdim return; 1269243830Sdim } 1270243830Sdim break; 1271243830Sdim 1272193323Sed default: 1273193323Sed if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { 1274193323Sed Action = TargetLowering::Legal; 1275193323Sed } else { 1276193323Sed Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); 1277193323Sed } 1278193323Sed break; 1279193323Sed } 1280193323Sed 1281193323Sed if (SimpleFinishLegalizing) { 1282239462Sdim SDNode *NewNode = Node; 1283193323Sed switch (Node->getOpcode()) { 1284193323Sed default: break; 1285193323Sed case ISD::SHL: 1286193323Sed case ISD::SRL: 1287193323Sed case ISD::SRA: 1288193323Sed case ISD::ROTL: 1289193323Sed case ISD::ROTR: 1290193323Sed // Legalizing shifts/rotates requires adjusting the shift amount 1291193323Sed // to the appropriate width. 1292239462Sdim if (!Node->getOperand(1).getValueType().isVector()) { 1293239462Sdim SDValue SAO = 1294239462Sdim DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1295239462Sdim Node->getOperand(1)); 1296234353Sdim HandleSDNode Handle(SAO); 1297234353Sdim LegalizeOp(SAO.getNode()); 1298239462Sdim NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1299239462Sdim Handle.getValue()); 1300234353Sdim } 1301193323Sed break; 1302198090Srdivacky case ISD::SRL_PARTS: 1303198090Srdivacky case ISD::SRA_PARTS: 1304198090Srdivacky case ISD::SHL_PARTS: 1305198090Srdivacky // Legalizing shifts/rotates requires adjusting the shift amount 1306198090Srdivacky // to the appropriate width. 1307239462Sdim if (!Node->getOperand(2).getValueType().isVector()) { 1308239462Sdim SDValue SAO = 1309239462Sdim DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(), 1310239462Sdim Node->getOperand(2)); 1311234353Sdim HandleSDNode Handle(SAO); 1312234353Sdim LegalizeOp(SAO.getNode()); 1313239462Sdim NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0), 1314239462Sdim Node->getOperand(1), 1315239462Sdim Handle.getValue()); 1316234353Sdim } 1317198090Srdivacky break; 1318193323Sed } 1319193323Sed 1320234353Sdim if (NewNode != Node) { 1321239462Sdim DAG.ReplaceAllUsesWith(Node, NewNode); 1322234353Sdim for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1323234353Sdim DAG.TransferDbgValues(SDValue(Node, i), SDValue(NewNode, i)); 1324234353Sdim ReplacedNode(Node); 1325234353Sdim Node = NewNode; 1326234353Sdim } 1327193323Sed switch (Action) { 1328193323Sed case TargetLowering::Legal: 1329234353Sdim return; 1330239462Sdim case TargetLowering::Custom: { 1331193323Sed // FIXME: The handling for custom lowering with multiple results is 1332193323Sed // a complete mess. 1333239462Sdim SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); 1334239462Sdim if (Res.getNode()) { 1335234353Sdim SmallVector<SDValue, 8> ResultVals; 1336193323Sed for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { 1337193323Sed if (e == 1) 1338239462Sdim ResultVals.push_back(Res); 1339193323Sed else 1340239462Sdim ResultVals.push_back(Res.getValue(i)); 1341193323Sed } 1342239462Sdim if (Res.getNode() != Node || Res.getResNo() != 0) { 1343239462Sdim DAG.ReplaceAllUsesWith(Node, ResultVals.data()); 1344234353Sdim for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) 1345234353Sdim DAG.TransferDbgValues(SDValue(Node, i), ResultVals[i]); 1346234353Sdim ReplacedNode(Node); 1347234353Sdim } 1348234353Sdim return; 1349193323Sed } 1350239462Sdim } 1351193323Sed // FALL THROUGH 1352193323Sed case TargetLowering::Expand: 1353234353Sdim ExpandNode(Node); 1354234353Sdim return; 1355193323Sed case TargetLowering::Promote: 1356234353Sdim PromoteNode(Node); 1357234353Sdim return; 1358193323Sed } 1359193323Sed } 1360193323Sed 1361193323Sed switch (Node->getOpcode()) { 1362193323Sed default: 1363193323Sed#ifndef NDEBUG 1364202375Srdivacky dbgs() << "NODE: "; 1365202375Srdivacky Node->dump( &DAG); 1366202375Srdivacky dbgs() << "\n"; 1367193323Sed#endif 1368234353Sdim llvm_unreachable("Do not know how to legalize this operator!"); 1369193323Sed 1370234353Sdim case ISD::CALLSEQ_START: 1371234353Sdim case ISD::CALLSEQ_END: 1372226633Sdim break; 1373193323Sed case ISD::LOAD: { 1374239462Sdim return LegalizeLoadOps(Node); 1375193323Sed } 1376193323Sed case ISD::STORE: { 1377239462Sdim return LegalizeStoreOps(Node); 1378193323Sed } 1379193323Sed } 1380193323Sed} 1381193323Sed 1382193323SedSDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { 1383193323Sed SDValue Vec = Op.getOperand(0); 1384193323Sed SDValue Idx = Op.getOperand(1); 1385263508Sdim SDLoc dl(Op); 1386193323Sed // Store the value to a temporary stack slot, then LOAD the returned part. 1387193323Sed SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1388218893Sdim SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1389218893Sdim MachinePointerInfo(), false, false, 0); 1390193323Sed 1391193323Sed // Add the offset to the index. 1392193323Sed unsigned EltSize = 1393193323Sed Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1394193323Sed Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1395193323Sed DAG.getConstant(EltSize, Idx.getValueType())); 1396193323Sed 1397263508Sdim Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy()); 1398193323Sed StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); 1399193323Sed 1400198090Srdivacky if (Op.getValueType().isVector()) 1401218893Sdim return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,MachinePointerInfo(), 1402234353Sdim false, false, false, 0); 1403218893Sdim return DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, 1404218893Sdim MachinePointerInfo(), 1405218893Sdim Vec.getValueType().getVectorElementType(), 1406218893Sdim false, false, 0); 1407218893Sdim} 1408218893Sdim 1409218893SdimSDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { 1410218893Sdim assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); 1411218893Sdim 1412218893Sdim SDValue Vec = Op.getOperand(0); 1413218893Sdim SDValue Part = Op.getOperand(1); 1414218893Sdim SDValue Idx = Op.getOperand(2); 1415263508Sdim SDLoc dl(Op); 1416218893Sdim 1417218893Sdim // Store the value to a temporary stack slot, then LOAD the returned part. 1418218893Sdim 1419218893Sdim SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); 1420218893Sdim int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1421218893Sdim MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1422218893Sdim 1423218893Sdim // First store the whole vector. 1424218893Sdim SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1425218893Sdim false, false, 0); 1426218893Sdim 1427218893Sdim // Then store the inserted part. 1428218893Sdim 1429218893Sdim // Add the offset to the index. 1430218893Sdim unsigned EltSize = 1431218893Sdim Vec.getValueType().getVectorElementType().getSizeInBits()/8; 1432218893Sdim 1433218893Sdim Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, 1434218893Sdim DAG.getConstant(EltSize, Idx.getValueType())); 1435263508Sdim Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy()); 1436218893Sdim 1437218893Sdim SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, 1438218893Sdim StackPtr); 1439218893Sdim 1440218893Sdim // Store the subvector. 1441218893Sdim Ch = DAG.getStore(DAG.getEntryNode(), dl, Part, SubStackPtr, 1442218893Sdim MachinePointerInfo(), false, false, 0); 1443218893Sdim 1444218893Sdim // Finally, load the updated vector. 1445218893Sdim return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo, 1446234353Sdim false, false, false, 0); 1447193323Sed} 1448193323Sed 1449193574SedSDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { 1450193574Sed // We can't handle this case efficiently. Allocate a sufficiently 1451193574Sed // aligned object on the stack, store each element into it, then load 1452193574Sed // the result as a vector. 1453193574Sed // Create the stack frame object. 1454198090Srdivacky EVT VT = Node->getValueType(0); 1455199989Srdivacky EVT EltVT = VT.getVectorElementType(); 1456263508Sdim SDLoc dl(Node); 1457193574Sed SDValue FIPtr = DAG.CreateStackTemporary(VT); 1458193574Sed int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); 1459218893Sdim MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(FI); 1460193574Sed 1461193574Sed // Emit a store of each element to the stack slot. 1462193574Sed SmallVector<SDValue, 8> Stores; 1463199989Srdivacky unsigned TypeByteSize = EltVT.getSizeInBits() / 8; 1464193574Sed // Store (in the right endianness) the elements to memory. 1465193574Sed for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1466193574Sed // Ignore undef elements. 1467193574Sed if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; 1468193574Sed 1469193574Sed unsigned Offset = TypeByteSize*i; 1470193574Sed 1471193574Sed SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); 1472193574Sed Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); 1473193574Sed 1474204642Srdivacky // If the destination vector element type is narrower than the source 1475204642Srdivacky // element type, only store the bits necessary. 1476204642Srdivacky if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { 1477199989Srdivacky Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, 1478218893Sdim Node->getOperand(i), Idx, 1479218893Sdim PtrInfo.getWithOffset(Offset), 1480203954Srdivacky EltVT, false, false, 0)); 1481203954Srdivacky } else 1482210299Sed Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, 1483218893Sdim Node->getOperand(i), Idx, 1484218893Sdim PtrInfo.getWithOffset(Offset), 1485203954Srdivacky false, false, 0)); 1486193574Sed } 1487193574Sed 1488193574Sed SDValue StoreChain; 1489193574Sed if (!Stores.empty()) // Not all undef elements? 1490193574Sed StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1491193574Sed &Stores[0], Stores.size()); 1492193574Sed else 1493193574Sed StoreChain = DAG.getEntryNode(); 1494193574Sed 1495193574Sed // Result is a load from the stack slot. 1496263508Sdim return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo, 1497234353Sdim false, false, false, 0); 1498193574Sed} 1499193574Sed 1500193323SedSDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) { 1501263508Sdim SDLoc dl(Node); 1502193323Sed SDValue Tmp1 = Node->getOperand(0); 1503193323Sed SDValue Tmp2 = Node->getOperand(1); 1504205218Srdivacky 1505205218Srdivacky // Get the sign bit of the RHS. First obtain a value that has the same 1506205218Srdivacky // sign as the sign bit, i.e. negative if and only if the sign bit is 1. 1507193323Sed SDValue SignBit; 1508205218Srdivacky EVT FloatVT = Tmp2.getValueType(); 1509205218Srdivacky EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits()); 1510224145Sdim if (TLI.isTypeLegal(IVT)) { 1511205218Srdivacky // Convert to an integer with the same sign bit. 1512218893Sdim SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2); 1513193323Sed } else { 1514205218Srdivacky // Store the float to memory, then load the sign part out as an integer. 1515205218Srdivacky MVT LoadTy = TLI.getPointerTy(); 1516205218Srdivacky // First create a temporary that is aligned for both the load and store. 1517205218Srdivacky SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); 1518205218Srdivacky // Then store the float to it. 1519193323Sed SDValue Ch = 1520218893Sdim DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(), 1521203954Srdivacky false, false, 0); 1522205218Srdivacky if (TLI.isBigEndian()) { 1523205218Srdivacky assert(FloatVT.isByteSized() && "Unsupported floating point type!"); 1524205218Srdivacky // Load out a legal integer with the same sign bit as the float. 1525218893Sdim SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(), 1526234353Sdim false, false, false, 0); 1527205218Srdivacky } else { // Little endian 1528205218Srdivacky SDValue LoadPtr = StackPtr; 1529205218Srdivacky // The float may be wider than the integer we are going to load. Advance 1530205218Srdivacky // the pointer so that the loaded integer will contain the sign bit. 1531205218Srdivacky unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits(); 1532205218Srdivacky unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8; 1533205218Srdivacky LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), 1534263508Sdim LoadPtr, 1535263508Sdim DAG.getConstant(ByteOffset, LoadPtr.getValueType())); 1536205218Srdivacky // Load a legal integer containing the sign bit. 1537218893Sdim SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(), 1538234353Sdim false, false, false, 0); 1539205218Srdivacky // Move the sign bit to the top bit of the loaded integer. 1540205218Srdivacky unsigned BitShift = LoadTy.getSizeInBits() - 1541205218Srdivacky (FloatVT.getSizeInBits() - 8 * ByteOffset); 1542205218Srdivacky assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?"); 1543205218Srdivacky if (BitShift) 1544205218Srdivacky SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit, 1545219077Sdim DAG.getConstant(BitShift, 1546219077Sdim TLI.getShiftAmountTy(SignBit.getValueType()))); 1547205218Srdivacky } 1548193323Sed } 1549205218Srdivacky // Now get the sign bit proper, by seeing whether the value is negative. 1550263508Sdim SignBit = DAG.getSetCC(dl, getSetCCResultType(SignBit.getValueType()), 1551205218Srdivacky SignBit, DAG.getConstant(0, SignBit.getValueType()), 1552205218Srdivacky ISD::SETLT); 1553193323Sed // Get the absolute value of the result. 1554193323Sed SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); 1555193323Sed // Select between the nabs and abs value based on the sign bit of 1556193323Sed // the input. 1557263508Sdim return DAG.getSelect(dl, AbsVal.getValueType(), SignBit, 1558263508Sdim DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal), 1559263508Sdim AbsVal); 1560193323Sed} 1561193323Sed 1562193323Sedvoid SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, 1563193323Sed SmallVectorImpl<SDValue> &Results) { 1564193323Sed unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); 1565193323Sed assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" 1566193323Sed " not tell us which reg is the stack pointer!"); 1567263508Sdim SDLoc dl(Node); 1568198090Srdivacky EVT VT = Node->getValueType(0); 1569193323Sed SDValue Tmp1 = SDValue(Node, 0); 1570193323Sed SDValue Tmp2 = SDValue(Node, 1); 1571193323Sed SDValue Tmp3 = Node->getOperand(2); 1572193323Sed SDValue Chain = Tmp1.getOperand(0); 1573193323Sed 1574193323Sed // Chain the dynamic stack allocation so that it doesn't modify the stack 1575193323Sed // pointer when other instructions are using the stack. 1576263508Sdim Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true), 1577263508Sdim SDLoc(Node)); 1578193323Sed 1579193323Sed SDValue Size = Tmp2.getOperand(1); 1580193323Sed SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); 1581193323Sed Chain = SP.getValue(1); 1582193323Sed unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); 1583218893Sdim unsigned StackAlign = TM.getFrameLowering()->getStackAlignment(); 1584263508Sdim Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value 1585193323Sed if (Align > StackAlign) 1586263508Sdim Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, 1587263508Sdim DAG.getConstant(-(uint64_t)Align, VT)); 1588193323Sed Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain 1589193323Sed 1590193323Sed Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), 1591263508Sdim DAG.getIntPtrConstant(0, true), SDValue(), 1592263508Sdim SDLoc(Node)); 1593193323Sed 1594193323Sed Results.push_back(Tmp1); 1595193323Sed Results.push_back(Tmp2); 1596193323Sed} 1597193323Sed 1598193323Sed/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and 1599263508Sdim/// condition code CC on the current target. 1600263508Sdim/// 1601263508Sdim/// If the SETCC has been legalized using AND / OR, then the legalized node 1602263508Sdim/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert 1603263508Sdim/// will be set to false. 1604263508Sdim/// 1605263508Sdim/// If the SETCC has been legalized by using getSetCCSwappedOperands(), 1606263508Sdim/// then the values of LHS and RHS will be swapped, CC will be set to the 1607263508Sdim/// new condition, and NeedInvert will be set to false. 1608263508Sdim/// 1609263508Sdim/// If the SETCC has been legalized using the inverse condcode, then LHS and 1610263508Sdim/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert 1611263508Sdim/// will be set to true. The caller must invert the result of the SETCC with 1612263508Sdim/// SelectionDAG::getNOT() or take equivalent action to swap the effect of a 1613263508Sdim/// true/false result. 1614263508Sdim/// 1615263508Sdim/// \returns true if the SetCC has been legalized, false if it hasn't. 1616263508Sdimbool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, 1617193323Sed SDValue &LHS, SDValue &RHS, 1618193323Sed SDValue &CC, 1619263508Sdim bool &NeedInvert, 1620263508Sdim SDLoc dl) { 1621249423Sdim MVT OpVT = LHS.getSimpleValueType(); 1622193323Sed ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); 1623263508Sdim NeedInvert = false; 1624193323Sed switch (TLI.getCondCodeAction(CCCode, OpVT)) { 1625234353Sdim default: llvm_unreachable("Unknown condition code action!"); 1626193323Sed case TargetLowering::Legal: 1627193323Sed // Nothing to do. 1628193323Sed break; 1629193323Sed case TargetLowering::Expand: { 1630263508Sdim ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); 1631263508Sdim if (TLI.isCondCodeLegal(InvCC, OpVT)) { 1632263508Sdim std::swap(LHS, RHS); 1633263508Sdim CC = DAG.getCondCode(InvCC); 1634263508Sdim return true; 1635263508Sdim } 1636193323Sed ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; 1637193323Sed unsigned Opc = 0; 1638193323Sed switch (CCCode) { 1639234353Sdim default: llvm_unreachable("Don't know how to expand this condition!"); 1640263508Sdim case ISD::SETO: 1641243830Sdim assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT) 1642243830Sdim == TargetLowering::Legal 1643243830Sdim && "If SETO is expanded, SETOEQ must be legal!"); 1644243830Sdim CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; 1645263508Sdim case ISD::SETUO: 1646243830Sdim assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT) 1647243830Sdim == TargetLowering::Legal 1648243830Sdim && "If SETUO is expanded, SETUNE must be legal!"); 1649243830Sdim CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; 1650243830Sdim case ISD::SETOEQ: 1651243830Sdim case ISD::SETOGT: 1652243830Sdim case ISD::SETOGE: 1653243830Sdim case ISD::SETOLT: 1654243830Sdim case ISD::SETOLE: 1655263508Sdim case ISD::SETONE: 1656263508Sdim case ISD::SETUEQ: 1657263508Sdim case ISD::SETUNE: 1658263508Sdim case ISD::SETUGT: 1659263508Sdim case ISD::SETUGE: 1660263508Sdim case ISD::SETULT: 1661243830Sdim case ISD::SETULE: 1662243830Sdim // If we are floating point, assign and break, otherwise fall through. 1663243830Sdim if (!OpVT.isInteger()) { 1664243830Sdim // We can use the 4th bit to tell if we are the unordered 1665243830Sdim // or ordered version of the opcode. 1666243830Sdim CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; 1667243830Sdim Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; 1668243830Sdim CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); 1669243830Sdim break; 1670243830Sdim } 1671243830Sdim // Fallthrough if we are unsigned integer. 1672243830Sdim case ISD::SETLE: 1673243830Sdim case ISD::SETGT: 1674243830Sdim case ISD::SETGE: 1675243830Sdim case ISD::SETLT: 1676263508Sdim // We only support using the inverted operation, which is computed above 1677263508Sdim // and not a different manner of supporting expanding these cases. 1678263508Sdim llvm_unreachable("Don't know how to expand this condition!"); 1679243830Sdim case ISD::SETNE: 1680243830Sdim case ISD::SETEQ: 1681263508Sdim // Try inverting the result of the inverse condition. 1682263508Sdim InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ; 1683263508Sdim if (TLI.isCondCodeLegal(InvCC, OpVT)) { 1684263508Sdim CC = DAG.getCondCode(InvCC); 1685263508Sdim NeedInvert = true; 1686263508Sdim return true; 1687243830Sdim } 1688263508Sdim // If inverting the condition didn't work then we have no means to expand 1689263508Sdim // the condition. 1690263508Sdim llvm_unreachable("Don't know how to expand this condition!"); 1691193323Sed } 1692263508Sdim 1693243830Sdim SDValue SetCC1, SetCC2; 1694243830Sdim if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { 1695243830Sdim // If we aren't the ordered or unorder operation, 1696243830Sdim // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). 1697243830Sdim SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); 1698243830Sdim SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); 1699243830Sdim } else { 1700243830Sdim // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) 1701243830Sdim SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); 1702243830Sdim SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); 1703243830Sdim } 1704193323Sed LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); 1705193323Sed RHS = SDValue(); 1706193323Sed CC = SDValue(); 1707263508Sdim return true; 1708193323Sed } 1709193323Sed } 1710263508Sdim return false; 1711193323Sed} 1712193323Sed 1713193323Sed/// EmitStackConvert - Emit a store/load combination to the stack. This stores 1714193323Sed/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does 1715193323Sed/// a load from the stack slot to DestVT, extending it if needed. 1716193323Sed/// The resultant code need not be legal. 1717193323SedSDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, 1718198090Srdivacky EVT SlotVT, 1719198090Srdivacky EVT DestVT, 1720263508Sdim SDLoc dl) { 1721193323Sed // Create the stack frame object. 1722193323Sed unsigned SrcAlign = 1723243830Sdim TLI.getDataLayout()->getPrefTypeAlignment(SrcOp.getValueType(). 1724198090Srdivacky getTypeForEVT(*DAG.getContext())); 1725193323Sed SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); 1726193323Sed 1727193323Sed FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); 1728193323Sed int SPFI = StackPtrFI->getIndex(); 1729218893Sdim MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI); 1730193323Sed 1731193323Sed unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); 1732193323Sed unsigned SlotSize = SlotVT.getSizeInBits(); 1733193323Sed unsigned DestSize = DestVT.getSizeInBits(); 1734226633Sdim Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); 1735243830Sdim unsigned DestAlign = TLI.getDataLayout()->getPrefTypeAlignment(DestType); 1736193323Sed 1737193323Sed // Emit a store to the stack slot. Use a truncstore if the input value is 1738193323Sed // later than DestVT. 1739193323Sed SDValue Store; 1740193323Sed 1741193323Sed if (SrcSize > SlotSize) 1742193323Sed Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1743218893Sdim PtrInfo, SlotVT, false, false, SrcAlign); 1744193323Sed else { 1745193323Sed assert(SrcSize == SlotSize && "Invalid store"); 1746193323Sed Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, 1747218893Sdim PtrInfo, false, false, SrcAlign); 1748193323Sed } 1749193323Sed 1750193323Sed // Result is a load from the stack slot. 1751193323Sed if (SlotSize == DestSize) 1752218893Sdim return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, 1753234353Sdim false, false, false, DestAlign); 1754193323Sed 1755193323Sed assert(SlotSize < DestSize && "Unknown extension!"); 1756218893Sdim return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, 1757218893Sdim PtrInfo, SlotVT, false, false, DestAlign); 1758193323Sed} 1759193323Sed 1760193323SedSDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { 1761263508Sdim SDLoc dl(Node); 1762193323Sed // Create a vector sized/aligned stack slot, store the value to element #0, 1763193323Sed // then load the whole vector back out. 1764193323Sed SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); 1765193323Sed 1766193323Sed FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); 1767193323Sed int SPFI = StackPtrFI->getIndex(); 1768193323Sed 1769193323Sed SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0), 1770193323Sed StackPtr, 1771218893Sdim MachinePointerInfo::getFixedStack(SPFI), 1772203954Srdivacky Node->getValueType(0).getVectorElementType(), 1773203954Srdivacky false, false, 0); 1774193323Sed return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr, 1775218893Sdim MachinePointerInfo::getFixedStack(SPFI), 1776234353Sdim false, false, false, 0); 1777193323Sed} 1778193323Sed 1779193323Sed 1780193323Sed/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't 1781193323Sed/// support the operation, but do support the resultant vector type. 1782193323SedSDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { 1783193323Sed unsigned NumElems = Node->getNumOperands(); 1784193630Sed SDValue Value1, Value2; 1785263508Sdim SDLoc dl(Node); 1786198090Srdivacky EVT VT = Node->getValueType(0); 1787198090Srdivacky EVT OpVT = Node->getOperand(0).getValueType(); 1788198090Srdivacky EVT EltVT = VT.getVectorElementType(); 1789193323Sed 1790193323Sed // If the only non-undef value is the low element, turn this into a 1791193323Sed // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. 1792193323Sed bool isOnlyLowElement = true; 1793193630Sed bool MoreThanTwoValues = false; 1794193323Sed bool isConstant = true; 1795193630Sed for (unsigned i = 0; i < NumElems; ++i) { 1796193323Sed SDValue V = Node->getOperand(i); 1797193630Sed if (V.getOpcode() == ISD::UNDEF) 1798193630Sed continue; 1799193630Sed if (i > 0) 1800193323Sed isOnlyLowElement = false; 1801193630Sed if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) 1802193630Sed isConstant = false; 1803193323Sed 1804193630Sed if (!Value1.getNode()) { 1805193630Sed Value1 = V; 1806193630Sed } else if (!Value2.getNode()) { 1807193630Sed if (V != Value1) 1808193630Sed Value2 = V; 1809193630Sed } else if (V != Value1 && V != Value2) { 1810193630Sed MoreThanTwoValues = true; 1811193630Sed } 1812193323Sed } 1813193323Sed 1814193630Sed if (!Value1.getNode()) 1815193630Sed return DAG.getUNDEF(VT); 1816193630Sed 1817193630Sed if (isOnlyLowElement) 1818193323Sed return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); 1819193323Sed 1820193323Sed // If all elements are constants, create a load from the constant pool. 1821193323Sed if (isConstant) { 1822234353Sdim SmallVector<Constant*, 16> CV; 1823193323Sed for (unsigned i = 0, e = NumElems; i != e; ++i) { 1824193323Sed if (ConstantFPSDNode *V = 1825193323Sed dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { 1826193323Sed CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); 1827193323Sed } else if (ConstantSDNode *V = 1828193323Sed dyn_cast<ConstantSDNode>(Node->getOperand(i))) { 1829199481Srdivacky if (OpVT==EltVT) 1830199481Srdivacky CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); 1831199481Srdivacky else { 1832199481Srdivacky // If OpVT and EltVT don't match, EltVT is not legal and the 1833199481Srdivacky // element values have been promoted/truncated earlier. Undo this; 1834199481Srdivacky // we don't want a v16i8 to become a v16i32 for example. 1835199481Srdivacky const ConstantInt *CI = V->getConstantIntValue(); 1836199481Srdivacky CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), 1837199481Srdivacky CI->getZExtValue())); 1838199481Srdivacky } 1839193323Sed } else { 1840193323Sed assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); 1841226633Sdim Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); 1842193323Sed CV.push_back(UndefValue::get(OpNTy)); 1843193323Sed } 1844193323Sed } 1845193323Sed Constant *CP = ConstantVector::get(CV); 1846193323Sed SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); 1847193323Sed unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 1848193323Sed return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx, 1849218893Sdim MachinePointerInfo::getConstantPool(), 1850234353Sdim false, false, false, Alignment); 1851193323Sed } 1852193323Sed 1853193630Sed if (!MoreThanTwoValues) { 1854193630Sed SmallVector<int, 8> ShuffleVec(NumElems, -1); 1855193630Sed for (unsigned i = 0; i < NumElems; ++i) { 1856193630Sed SDValue V = Node->getOperand(i); 1857193630Sed if (V.getOpcode() == ISD::UNDEF) 1858193630Sed continue; 1859193630Sed ShuffleVec[i] = V == Value1 ? 0 : NumElems; 1860193630Sed } 1861193630Sed if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { 1862193323Sed // Get the splatted value into the low element of a vector register. 1863193630Sed SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); 1864193630Sed SDValue Vec2; 1865193630Sed if (Value2.getNode()) 1866193630Sed Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); 1867193630Sed else 1868193630Sed Vec2 = DAG.getUNDEF(VT); 1869193323Sed 1870193323Sed // Return shuffle(LowValVec, undef, <0,0,0,0>) 1871193630Sed return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data()); 1872193323Sed } 1873193323Sed } 1874193323Sed 1875193574Sed // Otherwise, we can't handle this case efficiently. 1876193574Sed return ExpandVectorBuildThroughStack(Node); 1877193323Sed} 1878193323Sed 1879193323Sed// ExpandLibCall - Expand a node into a call to a libcall. If the result value 1880193323Sed// does not fit into a register, return the lo part and set the hi part to the 1881193323Sed// by-reg argument. If it does fit into a single register, return the result 1882193323Sed// and leave the Hi part unset. 1883193323SedSDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, 1884193323Sed bool isSigned) { 1885193323Sed TargetLowering::ArgListTy Args; 1886193323Sed TargetLowering::ArgListEntry Entry; 1887193323Sed for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 1888198090Srdivacky EVT ArgVT = Node->getOperand(i).getValueType(); 1889226633Sdim Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1890193323Sed Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 1891193323Sed Entry.isSExt = isSigned; 1892193323Sed Entry.isZExt = !isSigned; 1893193323Sed Args.push_back(Entry); 1894193323Sed } 1895193323Sed SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1896193323Sed TLI.getPointerTy()); 1897193323Sed 1898226633Sdim Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1899218893Sdim 1900234353Sdim // By default, the input chain to this libcall is the entry node of the 1901234353Sdim // function. If the libcall is going to be emitted as a tail call then 1902234353Sdim // TLI.isUsedByReturnOnly will change it to the right chain if the return 1903234353Sdim // node which is being folded has a non-entry input chain. 1904234353Sdim SDValue InChain = DAG.getEntryNode(); 1905234353Sdim 1906218893Sdim // isTailCall may be true since the callee does not reference caller stack 1907218893Sdim // frame. Check if it's in the right position. 1908234353Sdim SDValue TCChain = InChain; 1909249423Sdim bool isTailCall = TLI.isInTailCallPosition(DAG, Node, TCChain); 1910234353Sdim if (isTailCall) 1911234353Sdim InChain = TCChain; 1912234353Sdim 1913239462Sdim TargetLowering:: 1914239462Sdim CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 1915218893Sdim 0, TLI.getLibcallCallingConv(LC), isTailCall, 1916234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1917263508Sdim Callee, Args, DAG, SDLoc(Node)); 1918239462Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 1919193323Sed 1920239462Sdim 1921218893Sdim if (!CallInfo.second.getNode()) 1922218893Sdim // It's a tailcall, return the chain (which is the DAG root). 1923218893Sdim return DAG.getRoot(); 1924218893Sdim 1925221345Sdim return CallInfo.first; 1926221345Sdim} 1927221345Sdim 1928223017Sdim/// ExpandLibCall - Generate a libcall taking the given operands as arguments 1929221345Sdim/// and returning a result of type RetVT. 1930221345SdimSDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, 1931221345Sdim const SDValue *Ops, unsigned NumOps, 1932263508Sdim bool isSigned, SDLoc dl) { 1933221345Sdim TargetLowering::ArgListTy Args; 1934221345Sdim Args.reserve(NumOps); 1935223017Sdim 1936221345Sdim TargetLowering::ArgListEntry Entry; 1937221345Sdim for (unsigned i = 0; i != NumOps; ++i) { 1938221345Sdim Entry.Node = Ops[i]; 1939221345Sdim Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); 1940221345Sdim Entry.isSExt = isSigned; 1941221345Sdim Entry.isZExt = !isSigned; 1942221345Sdim Args.push_back(Entry); 1943221345Sdim } 1944221345Sdim SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1945221345Sdim TLI.getPointerTy()); 1946223017Sdim 1947226633Sdim Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 1948239462Sdim TargetLowering:: 1949239462Sdim CallLoweringInfo CLI(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false, 1950239462Sdim false, 0, TLI.getLibcallCallingConv(LC), 1951239462Sdim /*isTailCall=*/false, 1952234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1953221345Sdim Callee, Args, DAG, dl); 1954239462Sdim std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI); 1955223017Sdim 1956193323Sed return CallInfo.first; 1957193323Sed} 1958193323Sed 1959210299Sed// ExpandChainLibCall - Expand a node into a call to a libcall. Similar to 1960210299Sed// ExpandLibCall except that the first operand is the in-chain. 1961210299Sedstd::pair<SDValue, SDValue> 1962210299SedSelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, 1963210299Sed SDNode *Node, 1964210299Sed bool isSigned) { 1965210299Sed SDValue InChain = Node->getOperand(0); 1966210299Sed 1967210299Sed TargetLowering::ArgListTy Args; 1968210299Sed TargetLowering::ArgListEntry Entry; 1969210299Sed for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { 1970210299Sed EVT ArgVT = Node->getOperand(i).getValueType(); 1971226633Sdim Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 1972210299Sed Entry.Node = Node->getOperand(i); 1973210299Sed Entry.Ty = ArgTy; 1974210299Sed Entry.isSExt = isSigned; 1975210299Sed Entry.isZExt = !isSigned; 1976210299Sed Args.push_back(Entry); 1977210299Sed } 1978210299Sed SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 1979210299Sed TLI.getPointerTy()); 1980210299Sed 1981226633Sdim Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); 1982239462Sdim TargetLowering:: 1983239462Sdim CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 1984218893Sdim 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 1985234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 1986263508Sdim Callee, Args, DAG, SDLoc(Node)); 1987239462Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 1988210299Sed 1989210299Sed return CallInfo; 1990210299Sed} 1991210299Sed 1992193323SedSDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, 1993193323Sed RTLIB::Libcall Call_F32, 1994193323Sed RTLIB::Libcall Call_F64, 1995193323Sed RTLIB::Libcall Call_F80, 1996249423Sdim RTLIB::Libcall Call_F128, 1997193323Sed RTLIB::Libcall Call_PPCF128) { 1998193323Sed RTLIB::Libcall LC; 1999263508Sdim switch (Node->getSimpleValueType(0).SimpleTy) { 2000234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2001193323Sed case MVT::f32: LC = Call_F32; break; 2002193323Sed case MVT::f64: LC = Call_F64; break; 2003193323Sed case MVT::f80: LC = Call_F80; break; 2004249423Sdim case MVT::f128: LC = Call_F128; break; 2005193323Sed case MVT::ppcf128: LC = Call_PPCF128; break; 2006193323Sed } 2007193323Sed return ExpandLibCall(LC, Node, false); 2008193323Sed} 2009193323Sed 2010193323SedSDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, 2011199481Srdivacky RTLIB::Libcall Call_I8, 2012193323Sed RTLIB::Libcall Call_I16, 2013193323Sed RTLIB::Libcall Call_I32, 2014193323Sed RTLIB::Libcall Call_I64, 2015193323Sed RTLIB::Libcall Call_I128) { 2016193323Sed RTLIB::Libcall LC; 2017263508Sdim switch (Node->getSimpleValueType(0).SimpleTy) { 2018234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2019199481Srdivacky case MVT::i8: LC = Call_I8; break; 2020199481Srdivacky case MVT::i16: LC = Call_I16; break; 2021199481Srdivacky case MVT::i32: LC = Call_I32; break; 2022199481Srdivacky case MVT::i64: LC = Call_I64; break; 2023193323Sed case MVT::i128: LC = Call_I128; break; 2024193323Sed } 2025193323Sed return ExpandLibCall(LC, Node, isSigned); 2026193323Sed} 2027193323Sed 2028221345Sdim/// isDivRemLibcallAvailable - Return true if divmod libcall is available. 2029221345Sdimstatic bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, 2030221345Sdim const TargetLowering &TLI) { 2031221345Sdim RTLIB::Libcall LC; 2032263508Sdim switch (Node->getSimpleValueType(0).SimpleTy) { 2033234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2034221345Sdim case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2035221345Sdim case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2036221345Sdim case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2037221345Sdim case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2038221345Sdim case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2039221345Sdim } 2040221345Sdim 2041221345Sdim return TLI.getLibcallName(LC) != 0; 2042221345Sdim} 2043221345Sdim 2044239462Sdim/// useDivRem - Only issue divrem libcall if both quotient and remainder are 2045221345Sdim/// needed. 2046239462Sdimstatic bool useDivRem(SDNode *Node, bool isSigned, bool isDIV) { 2047239462Sdim // The other use might have been replaced with a divrem already. 2048239462Sdim unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 2049221345Sdim unsigned OtherOpcode = 0; 2050221345Sdim if (isSigned) 2051221345Sdim OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV; 2052221345Sdim else 2053221345Sdim OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV; 2054221345Sdim 2055221345Sdim SDValue Op0 = Node->getOperand(0); 2056221345Sdim SDValue Op1 = Node->getOperand(1); 2057221345Sdim for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2058221345Sdim UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2059221345Sdim SDNode *User = *UI; 2060221345Sdim if (User == Node) 2061221345Sdim continue; 2062239462Sdim if ((User->getOpcode() == OtherOpcode || User->getOpcode() == DivRemOpc) && 2063221345Sdim User->getOperand(0) == Op0 && 2064221345Sdim User->getOperand(1) == Op1) 2065221345Sdim return true; 2066221345Sdim } 2067221345Sdim return false; 2068221345Sdim} 2069221345Sdim 2070221345Sdim/// ExpandDivRemLibCall - Issue libcalls to __{u}divmod to compute div / rem 2071221345Sdim/// pairs. 2072221345Sdimvoid 2073221345SdimSelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, 2074221345Sdim SmallVectorImpl<SDValue> &Results) { 2075221345Sdim unsigned Opcode = Node->getOpcode(); 2076221345Sdim bool isSigned = Opcode == ISD::SDIVREM; 2077221345Sdim 2078221345Sdim RTLIB::Libcall LC; 2079263508Sdim switch (Node->getSimpleValueType(0).SimpleTy) { 2080234353Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2081221345Sdim case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; 2082221345Sdim case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; 2083221345Sdim case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; 2084221345Sdim case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; 2085221345Sdim case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; 2086221345Sdim } 2087221345Sdim 2088221345Sdim // The input chain to this libcall is the entry node of the function. 2089221345Sdim // Legalizing the call will automatically add the previous call to the 2090221345Sdim // dependence. 2091221345Sdim SDValue InChain = DAG.getEntryNode(); 2092221345Sdim 2093221345Sdim EVT RetVT = Node->getValueType(0); 2094226633Sdim Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2095221345Sdim 2096221345Sdim TargetLowering::ArgListTy Args; 2097221345Sdim TargetLowering::ArgListEntry Entry; 2098221345Sdim for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { 2099221345Sdim EVT ArgVT = Node->getOperand(i).getValueType(); 2100226633Sdim Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); 2101221345Sdim Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy; 2102221345Sdim Entry.isSExt = isSigned; 2103221345Sdim Entry.isZExt = !isSigned; 2104221345Sdim Args.push_back(Entry); 2105221345Sdim } 2106221345Sdim 2107221345Sdim // Also pass the return address of the remainder. 2108221345Sdim SDValue FIPtr = DAG.CreateStackTemporary(RetVT); 2109221345Sdim Entry.Node = FIPtr; 2110221345Sdim Entry.Ty = RetTy->getPointerTo(); 2111221345Sdim Entry.isSExt = isSigned; 2112221345Sdim Entry.isZExt = !isSigned; 2113221345Sdim Args.push_back(Entry); 2114221345Sdim 2115221345Sdim SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2116221345Sdim TLI.getPointerTy()); 2117221345Sdim 2118263508Sdim SDLoc dl(Node); 2119239462Sdim TargetLowering:: 2120239462Sdim CallLoweringInfo CLI(InChain, RetTy, isSigned, !isSigned, false, false, 2121221345Sdim 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 2122234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2123234353Sdim Callee, Args, DAG, dl); 2124239462Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2125221345Sdim 2126221345Sdim // Remainder is loaded back from the stack frame. 2127234353Sdim SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, 2128234353Sdim MachinePointerInfo(), false, false, false, 0); 2129221345Sdim Results.push_back(CallInfo.first); 2130221345Sdim Results.push_back(Rem); 2131221345Sdim} 2132221345Sdim 2133249423Sdim/// isSinCosLibcallAvailable - Return true if sincos libcall is available. 2134249423Sdimstatic bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { 2135249423Sdim RTLIB::Libcall LC; 2136263508Sdim switch (Node->getSimpleValueType(0).SimpleTy) { 2137249423Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2138249423Sdim case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2139249423Sdim case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2140249423Sdim case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2141249423Sdim case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2142249423Sdim case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2143249423Sdim } 2144249423Sdim return TLI.getLibcallName(LC) != 0; 2145249423Sdim} 2146249423Sdim 2147249423Sdim/// canCombineSinCosLibcall - Return true if sincos libcall is available and 2148249423Sdim/// can be used to combine sin and cos. 2149249423Sdimstatic bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI, 2150249423Sdim const TargetMachine &TM) { 2151249423Sdim if (!isSinCosLibcallAvailable(Node, TLI)) 2152249423Sdim return false; 2153249423Sdim // GNU sin/cos functions set errno while sincos does not. Therefore 2154249423Sdim // combining sin and cos is only safe if unsafe-fpmath is enabled. 2155249423Sdim bool isGNU = Triple(TM.getTargetTriple()).getEnvironment() == Triple::GNU; 2156249423Sdim if (isGNU && !TM.Options.UnsafeFPMath) 2157249423Sdim return false; 2158249423Sdim return true; 2159249423Sdim} 2160249423Sdim 2161249423Sdim/// useSinCos - Only issue sincos libcall if both sin and cos are 2162249423Sdim/// needed. 2163249423Sdimstatic bool useSinCos(SDNode *Node) { 2164249423Sdim unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN 2165249423Sdim ? ISD::FCOS : ISD::FSIN; 2166263508Sdim 2167249423Sdim SDValue Op0 = Node->getOperand(0); 2168249423Sdim for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), 2169249423Sdim UE = Op0.getNode()->use_end(); UI != UE; ++UI) { 2170249423Sdim SDNode *User = *UI; 2171249423Sdim if (User == Node) 2172249423Sdim continue; 2173249423Sdim // The other user might have been turned into sincos already. 2174249423Sdim if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) 2175249423Sdim return true; 2176249423Sdim } 2177249423Sdim return false; 2178249423Sdim} 2179249423Sdim 2180249423Sdim/// ExpandSinCosLibCall - Issue libcalls to sincos to compute sin / cos 2181249423Sdim/// pairs. 2182249423Sdimvoid 2183249423SdimSelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, 2184249423Sdim SmallVectorImpl<SDValue> &Results) { 2185249423Sdim RTLIB::Libcall LC; 2186263508Sdim switch (Node->getSimpleValueType(0).SimpleTy) { 2187249423Sdim default: llvm_unreachable("Unexpected request for libcall!"); 2188249423Sdim case MVT::f32: LC = RTLIB::SINCOS_F32; break; 2189249423Sdim case MVT::f64: LC = RTLIB::SINCOS_F64; break; 2190249423Sdim case MVT::f80: LC = RTLIB::SINCOS_F80; break; 2191249423Sdim case MVT::f128: LC = RTLIB::SINCOS_F128; break; 2192249423Sdim case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; 2193249423Sdim } 2194263508Sdim 2195249423Sdim // The input chain to this libcall is the entry node of the function. 2196249423Sdim // Legalizing the call will automatically add the previous call to the 2197249423Sdim // dependence. 2198249423Sdim SDValue InChain = DAG.getEntryNode(); 2199263508Sdim 2200249423Sdim EVT RetVT = Node->getValueType(0); 2201249423Sdim Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); 2202263508Sdim 2203249423Sdim TargetLowering::ArgListTy Args; 2204249423Sdim TargetLowering::ArgListEntry Entry; 2205263508Sdim 2206249423Sdim // Pass the argument. 2207249423Sdim Entry.Node = Node->getOperand(0); 2208249423Sdim Entry.Ty = RetTy; 2209249423Sdim Entry.isSExt = false; 2210249423Sdim Entry.isZExt = false; 2211249423Sdim Args.push_back(Entry); 2212263508Sdim 2213249423Sdim // Pass the return address of sin. 2214249423Sdim SDValue SinPtr = DAG.CreateStackTemporary(RetVT); 2215249423Sdim Entry.Node = SinPtr; 2216249423Sdim Entry.Ty = RetTy->getPointerTo(); 2217249423Sdim Entry.isSExt = false; 2218249423Sdim Entry.isZExt = false; 2219249423Sdim Args.push_back(Entry); 2220263508Sdim 2221249423Sdim // Also pass the return address of the cos. 2222249423Sdim SDValue CosPtr = DAG.CreateStackTemporary(RetVT); 2223249423Sdim Entry.Node = CosPtr; 2224249423Sdim Entry.Ty = RetTy->getPointerTo(); 2225249423Sdim Entry.isSExt = false; 2226249423Sdim Entry.isZExt = false; 2227249423Sdim Args.push_back(Entry); 2228263508Sdim 2229249423Sdim SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), 2230249423Sdim TLI.getPointerTy()); 2231263508Sdim 2232263508Sdim SDLoc dl(Node); 2233249423Sdim TargetLowering:: 2234249423Sdim CallLoweringInfo CLI(InChain, Type::getVoidTy(*DAG.getContext()), 2235249423Sdim false, false, false, false, 2236249423Sdim 0, TLI.getLibcallCallingConv(LC), /*isTailCall=*/false, 2237249423Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2238249423Sdim Callee, Args, DAG, dl); 2239249423Sdim std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); 2240249423Sdim 2241249423Sdim Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, 2242249423Sdim MachinePointerInfo(), false, false, false, 0)); 2243249423Sdim Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, 2244249423Sdim MachinePointerInfo(), false, false, false, 0)); 2245249423Sdim} 2246249423Sdim 2247193323Sed/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a 2248193323Sed/// INT_TO_FP operation of the specified operand when the target requests that 2249193323Sed/// we expand it. At this point, we know that the result and operand types are 2250193323Sed/// legal for the target. 2251193323SedSDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, 2252193323Sed SDValue Op0, 2253198090Srdivacky EVT DestVT, 2254263508Sdim SDLoc dl) { 2255243830Sdim if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) { 2256193323Sed // simple 32-bit [signed|unsigned] integer to float/double expansion 2257193323Sed 2258193323Sed // Get the stack frame index of a 8 byte buffer. 2259193323Sed SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); 2260193323Sed 2261193323Sed // word offset constant for Hi/Lo address computation 2262263508Sdim SDValue WordOff = DAG.getConstant(sizeof(int), StackSlot.getValueType()); 2263193323Sed // set up Hi and Lo (into buffer) address based on endian 2264193323Sed SDValue Hi = StackSlot; 2265263508Sdim SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), 2266263508Sdim StackSlot, WordOff); 2267193323Sed if (TLI.isLittleEndian()) 2268193323Sed std::swap(Hi, Lo); 2269193323Sed 2270193323Sed // if signed map to unsigned space 2271193323Sed SDValue Op0Mapped; 2272193323Sed if (isSigned) { 2273193323Sed // constant used to invert sign bit (signed to unsigned mapping) 2274193323Sed SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); 2275193323Sed Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); 2276193323Sed } else { 2277193323Sed Op0Mapped = Op0; 2278193323Sed } 2279193323Sed // store the lo of the constructed double - based on integer input 2280193323Sed SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, 2281218893Sdim Op0Mapped, Lo, MachinePointerInfo(), 2282203954Srdivacky false, false, 0); 2283193323Sed // initial hi portion of constructed double 2284193323Sed SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); 2285193323Sed // store the hi of the constructed double - biased exponent 2286218893Sdim SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi, 2287218893Sdim MachinePointerInfo(), 2288218893Sdim false, false, 0); 2289193323Sed // load the constructed double 2290218893Sdim SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot, 2291234353Sdim MachinePointerInfo(), false, false, false, 0); 2292193323Sed // FP constant to bias correct the final result 2293193323Sed SDValue Bias = DAG.getConstantFP(isSigned ? 2294193323Sed BitsToDouble(0x4330000080000000ULL) : 2295193323Sed BitsToDouble(0x4330000000000000ULL), 2296193323Sed MVT::f64); 2297193323Sed // subtract the bias 2298193323Sed SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); 2299193323Sed // final result 2300193323Sed SDValue Result; 2301193323Sed // handle final rounding 2302193323Sed if (DestVT == MVT::f64) { 2303193323Sed // do nothing 2304193323Sed Result = Sub; 2305193323Sed } else if (DestVT.bitsLT(MVT::f64)) { 2306193323Sed Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, 2307193323Sed DAG.getIntPtrConstant(0)); 2308193323Sed } else if (DestVT.bitsGT(MVT::f64)) { 2309193323Sed Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); 2310193323Sed } 2311193323Sed return Result; 2312193323Sed } 2313193323Sed assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); 2314208599Srdivacky // Code below here assumes !isSigned without checking again. 2315204792Srdivacky 2316204792Srdivacky // Implementation of unsigned i64 to f64 following the algorithm in 2317204792Srdivacky // __floatundidf in compiler_rt. This implementation has the advantage 2318204792Srdivacky // of performing rounding correctly, both in the default rounding mode 2319204792Srdivacky // and in all alternate rounding modes. 2320204792Srdivacky // TODO: Generalize this for use with other types. 2321204792Srdivacky if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) { 2322204792Srdivacky SDValue TwoP52 = 2323204792Srdivacky DAG.getConstant(UINT64_C(0x4330000000000000), MVT::i64); 2324204792Srdivacky SDValue TwoP84PlusTwoP52 = 2325204792Srdivacky DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), MVT::f64); 2326204792Srdivacky SDValue TwoP84 = 2327204792Srdivacky DAG.getConstant(UINT64_C(0x4530000000000000), MVT::i64); 2328204792Srdivacky 2329204792Srdivacky SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32); 2330204792Srdivacky SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, 2331204792Srdivacky DAG.getConstant(32, MVT::i64)); 2332204792Srdivacky SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); 2333204792Srdivacky SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); 2334218893Sdim SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); 2335218893Sdim SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); 2336210299Sed SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, 2337210299Sed TwoP84PlusTwoP52); 2338204792Srdivacky return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); 2339204792Srdivacky } 2340204792Srdivacky 2341218893Sdim // Implementation of unsigned i64 to f32. 2342208599Srdivacky // TODO: Generalize this for use with other types. 2343208599Srdivacky if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) { 2344218893Sdim // For unsigned conversions, convert them to signed conversions using the 2345218893Sdim // algorithm from the x86_64 __floatundidf in compiler_rt. 2346218893Sdim if (!isSigned) { 2347218893Sdim SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); 2348218893Sdim 2349219077Sdim SDValue ShiftConst = 2350219077Sdim DAG.getConstant(1, TLI.getShiftAmountTy(Op0.getValueType())); 2351218893Sdim SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); 2352218893Sdim SDValue AndConst = DAG.getConstant(1, MVT::i64); 2353218893Sdim SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); 2354218893Sdim SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); 2355218893Sdim 2356218893Sdim SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); 2357218893Sdim SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); 2358218893Sdim 2359218893Sdim // TODO: This really should be implemented using a branch rather than a 2360218893Sdim // select. We happen to get lucky and machinesink does the right 2361218893Sdim // thing most of the time. This would be a good candidate for a 2362218893Sdim //pseudo-op, or, even better, for whole-function isel. 2363263508Sdim SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 2364218893Sdim Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT); 2365263508Sdim return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast); 2366218893Sdim } 2367218893Sdim 2368218893Sdim // Otherwise, implement the fully general conversion. 2369208599Srdivacky 2370210299Sed SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2371208599Srdivacky DAG.getConstant(UINT64_C(0xfffffffffffff800), MVT::i64)); 2372208599Srdivacky SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, 2373208599Srdivacky DAG.getConstant(UINT64_C(0x800), MVT::i64)); 2374210299Sed SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, 2375208599Srdivacky DAG.getConstant(UINT64_C(0x7ff), MVT::i64)); 2376263508Sdim SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 2377208599Srdivacky And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); 2378263508Sdim SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0); 2379263508Sdim SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), 2380208599Srdivacky Op0, DAG.getConstant(UINT64_C(0x0020000000000000), MVT::i64), 2381218893Sdim ISD::SETUGE); 2382263508Sdim SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0); 2383219077Sdim EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType()); 2384208599Srdivacky 2385208599Srdivacky SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, 2386208599Srdivacky DAG.getConstant(32, SHVT)); 2387208599Srdivacky SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); 2388208599Srdivacky SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); 2389208599Srdivacky SDValue TwoP32 = 2390208599Srdivacky DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), MVT::f64); 2391208599Srdivacky SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); 2392208599Srdivacky SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); 2393208599Srdivacky SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); 2394208599Srdivacky SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); 2395208599Srdivacky return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, 2396208599Srdivacky DAG.getIntPtrConstant(0)); 2397208599Srdivacky } 2398208599Srdivacky 2399193323Sed SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); 2400193323Sed 2401263508Sdim SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()), 2402193323Sed Op0, DAG.getConstant(0, Op0.getValueType()), 2403193323Sed ISD::SETLT); 2404193323Sed SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); 2405263508Sdim SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), 2406193323Sed SignSet, Four, Zero); 2407193323Sed 2408193323Sed // If the sign bit of the integer is set, the large number will be treated 2409193323Sed // as a negative number. To counteract this, the dynamic code adds an 2410193323Sed // offset depending on the data type. 2411193323Sed uint64_t FF; 2412263508Sdim switch (Op0.getSimpleValueType().SimpleTy) { 2413234353Sdim default: llvm_unreachable("Unsupported integer type!"); 2414193323Sed case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) 2415193323Sed case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) 2416193323Sed case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) 2417193323Sed case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) 2418193323Sed } 2419193323Sed if (TLI.isLittleEndian()) FF <<= 32; 2420198090Srdivacky Constant *FudgeFactor = ConstantInt::get( 2421198090Srdivacky Type::getInt64Ty(*DAG.getContext()), FF); 2422193323Sed 2423193323Sed SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); 2424193323Sed unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 2425263508Sdim CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); 2426193323Sed Alignment = std::min(Alignment, 4u); 2427193323Sed SDValue FudgeInReg; 2428193323Sed if (DestVT == MVT::f32) 2429193323Sed FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, 2430218893Sdim MachinePointerInfo::getConstantPool(), 2431234353Sdim false, false, false, Alignment); 2432193323Sed else { 2433234353Sdim SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, 2434234353Sdim DAG.getEntryNode(), CPIdx, 2435234353Sdim MachinePointerInfo::getConstantPool(), 2436234353Sdim MVT::f32, false, false, Alignment); 2437234353Sdim HandleSDNode Handle(Load); 2438234353Sdim LegalizeOp(Load.getNode()); 2439234353Sdim FudgeInReg = Handle.getValue(); 2440193323Sed } 2441193323Sed 2442193323Sed return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); 2443193323Sed} 2444193323Sed 2445193323Sed/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a 2446193323Sed/// *INT_TO_FP operation of the specified operand when the target requests that 2447193323Sed/// we promote it. At this point, we know that the result and operand types are 2448193323Sed/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP 2449193323Sed/// operation that takes a larger input. 2450193323SedSDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, 2451198090Srdivacky EVT DestVT, 2452193323Sed bool isSigned, 2453263508Sdim SDLoc dl) { 2454193323Sed // First step, figure out the appropriate *INT_TO_FP operation to use. 2455198090Srdivacky EVT NewInTy = LegalOp.getValueType(); 2456193323Sed 2457193323Sed unsigned OpToUse = 0; 2458193323Sed 2459193323Sed // Scan for the appropriate larger type to use. 2460193323Sed while (1) { 2461198090Srdivacky NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); 2462193323Sed assert(NewInTy.isInteger() && "Ran out of possibilities!"); 2463193323Sed 2464193323Sed // If the target supports SINT_TO_FP of this type, use it. 2465193323Sed if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { 2466193323Sed OpToUse = ISD::SINT_TO_FP; 2467193323Sed break; 2468193323Sed } 2469193323Sed if (isSigned) continue; 2470193323Sed 2471193323Sed // If the target supports UINT_TO_FP of this type, use it. 2472193323Sed if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { 2473193323Sed OpToUse = ISD::UINT_TO_FP; 2474193323Sed break; 2475193323Sed } 2476193323Sed 2477193323Sed // Otherwise, try a larger type. 2478193323Sed } 2479193323Sed 2480193323Sed // Okay, we found the operation and type to use. Zero extend our input to the 2481193323Sed // desired type then run the operation on it. 2482193323Sed return DAG.getNode(OpToUse, dl, DestVT, 2483193323Sed DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 2484193323Sed dl, NewInTy, LegalOp)); 2485193323Sed} 2486193323Sed 2487193323Sed/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a 2488193323Sed/// FP_TO_*INT operation of the specified operand when the target requests that 2489193323Sed/// we promote it. At this point, we know that the result and operand types are 2490193323Sed/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT 2491193323Sed/// operation that returns a larger result. 2492193323SedSDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, 2493198090Srdivacky EVT DestVT, 2494193323Sed bool isSigned, 2495263508Sdim SDLoc dl) { 2496193323Sed // First step, figure out the appropriate FP_TO*INT operation to use. 2497198090Srdivacky EVT NewOutTy = DestVT; 2498193323Sed 2499193323Sed unsigned OpToUse = 0; 2500193323Sed 2501193323Sed // Scan for the appropriate larger type to use. 2502193323Sed while (1) { 2503198090Srdivacky NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); 2504193323Sed assert(NewOutTy.isInteger() && "Ran out of possibilities!"); 2505193323Sed 2506193323Sed if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { 2507193323Sed OpToUse = ISD::FP_TO_SINT; 2508193323Sed break; 2509193323Sed } 2510193323Sed 2511193323Sed if (TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { 2512193323Sed OpToUse = ISD::FP_TO_UINT; 2513193323Sed break; 2514193323Sed } 2515193323Sed 2516193323Sed // Otherwise, try a larger type. 2517193323Sed } 2518193323Sed 2519193323Sed 2520193323Sed // Okay, we found the operation and type to use. 2521193323Sed SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); 2522193323Sed 2523193323Sed // Truncate the result of the extended FP_TO_*INT operation to the desired 2524193323Sed // size. 2525193323Sed return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); 2526193323Sed} 2527193323Sed 2528193323Sed/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. 2529193323Sed/// 2530263508SdimSDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, SDLoc dl) { 2531198090Srdivacky EVT VT = Op.getValueType(); 2532219077Sdim EVT SHVT = TLI.getShiftAmountTy(VT); 2533193323Sed SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; 2534198090Srdivacky switch (VT.getSimpleVT().SimpleTy) { 2535234353Sdim default: llvm_unreachable("Unhandled Expand type in BSWAP!"); 2536193323Sed case MVT::i16: 2537193323Sed Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2538193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2539193323Sed return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 2540193323Sed case MVT::i32: 2541193323Sed Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2542193323Sed Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2543193323Sed Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2544193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2545193323Sed Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); 2546193323Sed Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); 2547193323Sed Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2548193323Sed Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2549193323Sed return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2550193323Sed case MVT::i64: 2551193323Sed Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2552193323Sed Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2553193323Sed Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2554193323Sed Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2555193323Sed Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); 2556193323Sed Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); 2557193323Sed Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); 2558193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); 2559193323Sed Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); 2560193323Sed Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); 2561193323Sed Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); 2562193323Sed Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); 2563193323Sed Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); 2564193323Sed Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); 2565193323Sed Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); 2566193323Sed Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); 2567193323Sed Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); 2568193323Sed Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); 2569193323Sed Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); 2570193323Sed Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); 2571193323Sed return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); 2572193323Sed } 2573193323Sed} 2574193323Sed 2575193323Sed/// ExpandBitCount - Expand the specified bitcount instruction into operations. 2576193323Sed/// 2577193323SedSDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, 2578263508Sdim SDLoc dl) { 2579193323Sed switch (Opc) { 2580234353Sdim default: llvm_unreachable("Cannot expand this yet!"); 2581193323Sed case ISD::CTPOP: { 2582198090Srdivacky EVT VT = Op.getValueType(); 2583219077Sdim EVT ShVT = TLI.getShiftAmountTy(VT); 2584218893Sdim unsigned Len = VT.getSizeInBits(); 2585218893Sdim 2586218893Sdim assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 && 2587218893Sdim "CTPOP not implemented for this type."); 2588218893Sdim 2589218893Sdim // This is the "best" algorithm from 2590218893Sdim // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 2591218893Sdim 2592249423Sdim SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), VT); 2593249423Sdim SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), VT); 2594249423Sdim SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), VT); 2595249423Sdim SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), VT); 2596218893Sdim 2597218893Sdim // v = v - ((v >> 1) & 0x55555555...) 2598218893Sdim Op = DAG.getNode(ISD::SUB, dl, VT, Op, 2599218893Sdim DAG.getNode(ISD::AND, dl, VT, 2600218893Sdim DAG.getNode(ISD::SRL, dl, VT, Op, 2601218893Sdim DAG.getConstant(1, ShVT)), 2602218893Sdim Mask55)); 2603218893Sdim // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...) 2604218893Sdim Op = DAG.getNode(ISD::ADD, dl, VT, 2605218893Sdim DAG.getNode(ISD::AND, dl, VT, Op, Mask33), 2606218893Sdim DAG.getNode(ISD::AND, dl, VT, 2607218893Sdim DAG.getNode(ISD::SRL, dl, VT, Op, 2608218893Sdim DAG.getConstant(2, ShVT)), 2609218893Sdim Mask33)); 2610218893Sdim // v = (v + (v >> 4)) & 0x0F0F0F0F... 2611218893Sdim Op = DAG.getNode(ISD::AND, dl, VT, 2612218893Sdim DAG.getNode(ISD::ADD, dl, VT, Op, 2613218893Sdim DAG.getNode(ISD::SRL, dl, VT, Op, 2614218893Sdim DAG.getConstant(4, ShVT))), 2615218893Sdim Mask0F); 2616218893Sdim // v = (v * 0x01010101...) >> (Len - 8) 2617218893Sdim Op = DAG.getNode(ISD::SRL, dl, VT, 2618218893Sdim DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), 2619218893Sdim DAG.getConstant(Len - 8, ShVT)); 2620219077Sdim 2621193323Sed return Op; 2622193323Sed } 2623234353Sdim case ISD::CTLZ_ZERO_UNDEF: 2624234353Sdim // This trivially expands to CTLZ. 2625234353Sdim return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); 2626193323Sed case ISD::CTLZ: { 2627193323Sed // for now, we do this: 2628193323Sed // x = x | (x >> 1); 2629193323Sed // x = x | (x >> 2); 2630193323Sed // ... 2631193323Sed // x = x | (x >>16); 2632193323Sed // x = x | (x >>32); // for 64-bit input 2633193323Sed // return popcount(~x); 2634193323Sed // 2635193323Sed // but see also: http://www.hackersdelight.org/HDcode/nlz.cc 2636198090Srdivacky EVT VT = Op.getValueType(); 2637219077Sdim EVT ShVT = TLI.getShiftAmountTy(VT); 2638193323Sed unsigned len = VT.getSizeInBits(); 2639193323Sed for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { 2640193323Sed SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); 2641193323Sed Op = DAG.getNode(ISD::OR, dl, VT, Op, 2642193323Sed DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); 2643193323Sed } 2644193323Sed Op = DAG.getNOT(dl, Op, VT); 2645193323Sed return DAG.getNode(ISD::CTPOP, dl, VT, Op); 2646193323Sed } 2647234353Sdim case ISD::CTTZ_ZERO_UNDEF: 2648234353Sdim // This trivially expands to CTTZ. 2649234353Sdim return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); 2650193323Sed case ISD::CTTZ: { 2651193323Sed // for now, we use: { return popcount(~x & (x - 1)); } 2652193323Sed // unless the target has ctlz but not ctpop, in which case we use: 2653193323Sed // { return 32 - nlz(~x & (x-1)); } 2654193323Sed // see also http://www.hackersdelight.org/HDcode/ntz.cc 2655198090Srdivacky EVT VT = Op.getValueType(); 2656193323Sed SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, 2657193323Sed DAG.getNOT(dl, Op, VT), 2658193323Sed DAG.getNode(ISD::SUB, dl, VT, Op, 2659193323Sed DAG.getConstant(1, VT))); 2660193323Sed // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. 2661193323Sed if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && 2662193323Sed TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) 2663193323Sed return DAG.getNode(ISD::SUB, dl, VT, 2664193323Sed DAG.getConstant(VT.getSizeInBits(), VT), 2665193323Sed DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); 2666193323Sed return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); 2667193323Sed } 2668193323Sed } 2669193323Sed} 2670193323Sed 2671210299Sedstd::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) { 2672210299Sed unsigned Opc = Node->getOpcode(); 2673210299Sed MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); 2674210299Sed RTLIB::Libcall LC; 2675210299Sed 2676210299Sed switch (Opc) { 2677210299Sed default: 2678210299Sed llvm_unreachable("Unhandled atomic intrinsic Expand!"); 2679210299Sed case ISD::ATOMIC_SWAP: 2680210299Sed switch (VT.SimpleTy) { 2681210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2682210299Sed case MVT::i8: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_1; break; 2683210299Sed case MVT::i16: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_2; break; 2684210299Sed case MVT::i32: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_4; break; 2685210299Sed case MVT::i64: LC = RTLIB::SYNC_LOCK_TEST_AND_SET_8; break; 2686263508Sdim case MVT::i128:LC = RTLIB::SYNC_LOCK_TEST_AND_SET_16;break; 2687210299Sed } 2688210299Sed break; 2689210299Sed case ISD::ATOMIC_CMP_SWAP: 2690210299Sed switch (VT.SimpleTy) { 2691210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2692210299Sed case MVT::i8: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1; break; 2693210299Sed case MVT::i16: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2; break; 2694210299Sed case MVT::i32: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_4; break; 2695210299Sed case MVT::i64: LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_8; break; 2696263508Sdim case MVT::i128:LC = RTLIB::SYNC_VAL_COMPARE_AND_SWAP_16;break; 2697210299Sed } 2698210299Sed break; 2699210299Sed case ISD::ATOMIC_LOAD_ADD: 2700210299Sed switch (VT.SimpleTy) { 2701210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2702210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_ADD_1; break; 2703210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_ADD_2; break; 2704210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_ADD_4; break; 2705210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_ADD_8; break; 2706263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_ADD_16;break; 2707210299Sed } 2708210299Sed break; 2709210299Sed case ISD::ATOMIC_LOAD_SUB: 2710210299Sed switch (VT.SimpleTy) { 2711210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2712210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_SUB_1; break; 2713210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_SUB_2; break; 2714210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_SUB_4; break; 2715210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_SUB_8; break; 2716263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_SUB_16;break; 2717210299Sed } 2718210299Sed break; 2719210299Sed case ISD::ATOMIC_LOAD_AND: 2720210299Sed switch (VT.SimpleTy) { 2721210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2722210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_AND_1; break; 2723210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_AND_2; break; 2724210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_AND_4; break; 2725210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_AND_8; break; 2726263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_AND_16;break; 2727210299Sed } 2728210299Sed break; 2729210299Sed case ISD::ATOMIC_LOAD_OR: 2730210299Sed switch (VT.SimpleTy) { 2731210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2732210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_OR_1; break; 2733210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_OR_2; break; 2734210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_OR_4; break; 2735210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_OR_8; break; 2736263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_OR_16;break; 2737210299Sed } 2738210299Sed break; 2739210299Sed case ISD::ATOMIC_LOAD_XOR: 2740210299Sed switch (VT.SimpleTy) { 2741210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2742210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_XOR_1; break; 2743210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_XOR_2; break; 2744210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_XOR_4; break; 2745210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_XOR_8; break; 2746263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_XOR_16;break; 2747210299Sed } 2748210299Sed break; 2749210299Sed case ISD::ATOMIC_LOAD_NAND: 2750210299Sed switch (VT.SimpleTy) { 2751210299Sed default: llvm_unreachable("Unexpected value type for atomic!"); 2752210299Sed case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_NAND_1; break; 2753210299Sed case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_NAND_2; break; 2754210299Sed case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_NAND_4; break; 2755210299Sed case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_NAND_8; break; 2756263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_NAND_16;break; 2757210299Sed } 2758210299Sed break; 2759263508Sdim case ISD::ATOMIC_LOAD_MAX: 2760263508Sdim switch (VT.SimpleTy) { 2761263508Sdim default: llvm_unreachable("Unexpected value type for atomic!"); 2762263508Sdim case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MAX_1; break; 2763263508Sdim case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MAX_2; break; 2764263508Sdim case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MAX_4; break; 2765263508Sdim case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MAX_8; break; 2766263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MAX_16;break; 2767263508Sdim } 2768263508Sdim break; 2769263508Sdim case ISD::ATOMIC_LOAD_UMAX: 2770263508Sdim switch (VT.SimpleTy) { 2771263508Sdim default: llvm_unreachable("Unexpected value type for atomic!"); 2772263508Sdim case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMAX_1; break; 2773263508Sdim case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMAX_2; break; 2774263508Sdim case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMAX_4; break; 2775263508Sdim case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMAX_8; break; 2776263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMAX_16;break; 2777263508Sdim } 2778263508Sdim break; 2779263508Sdim case ISD::ATOMIC_LOAD_MIN: 2780263508Sdim switch (VT.SimpleTy) { 2781263508Sdim default: llvm_unreachable("Unexpected value type for atomic!"); 2782263508Sdim case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_MIN_1; break; 2783263508Sdim case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_MIN_2; break; 2784263508Sdim case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_MIN_4; break; 2785263508Sdim case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_MIN_8; break; 2786263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_MIN_16;break; 2787263508Sdim } 2788263508Sdim break; 2789263508Sdim case ISD::ATOMIC_LOAD_UMIN: 2790263508Sdim switch (VT.SimpleTy) { 2791263508Sdim default: llvm_unreachable("Unexpected value type for atomic!"); 2792263508Sdim case MVT::i8: LC = RTLIB::SYNC_FETCH_AND_UMIN_1; break; 2793263508Sdim case MVT::i16: LC = RTLIB::SYNC_FETCH_AND_UMIN_2; break; 2794263508Sdim case MVT::i32: LC = RTLIB::SYNC_FETCH_AND_UMIN_4; break; 2795263508Sdim case MVT::i64: LC = RTLIB::SYNC_FETCH_AND_UMIN_8; break; 2796263508Sdim case MVT::i128:LC = RTLIB::SYNC_FETCH_AND_UMIN_16;break; 2797263508Sdim } 2798263508Sdim break; 2799210299Sed } 2800210299Sed 2801210299Sed return ExpandChainLibCall(LC, Node, false); 2802210299Sed} 2803210299Sed 2804234353Sdimvoid SelectionDAGLegalize::ExpandNode(SDNode *Node) { 2805234353Sdim SmallVector<SDValue, 8> Results; 2806263508Sdim SDLoc dl(Node); 2807193323Sed SDValue Tmp1, Tmp2, Tmp3, Tmp4; 2808263508Sdim bool NeedInvert; 2809193323Sed switch (Node->getOpcode()) { 2810193323Sed case ISD::CTPOP: 2811193323Sed case ISD::CTLZ: 2812234353Sdim case ISD::CTLZ_ZERO_UNDEF: 2813193323Sed case ISD::CTTZ: 2814234353Sdim case ISD::CTTZ_ZERO_UNDEF: 2815193323Sed Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); 2816193323Sed Results.push_back(Tmp1); 2817193323Sed break; 2818193323Sed case ISD::BSWAP: 2819193323Sed Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); 2820193323Sed break; 2821193323Sed case ISD::FRAMEADDR: 2822193323Sed case ISD::RETURNADDR: 2823193323Sed case ISD::FRAME_TO_ARGS_OFFSET: 2824193323Sed Results.push_back(DAG.getConstant(0, Node->getValueType(0))); 2825193323Sed break; 2826193323Sed case ISD::FLT_ROUNDS_: 2827193323Sed Results.push_back(DAG.getConstant(1, Node->getValueType(0))); 2828193323Sed break; 2829193323Sed case ISD::EH_RETURN: 2830193323Sed case ISD::EH_LABEL: 2831193323Sed case ISD::PREFETCH: 2832193323Sed case ISD::VAEND: 2833210299Sed case ISD::EH_SJLJ_LONGJMP: 2834218893Sdim // If the target didn't expand these, there's nothing to do, so just 2835218893Sdim // preserve the chain and be done. 2836193323Sed Results.push_back(Node->getOperand(0)); 2837193323Sed break; 2838210299Sed case ISD::EH_SJLJ_SETJMP: 2839218893Sdim // If the target didn't expand this, just return 'zero' and preserve the 2840218893Sdim // chain. 2841210299Sed Results.push_back(DAG.getConstant(0, MVT::i32)); 2842210299Sed Results.push_back(Node->getOperand(0)); 2843210299Sed break; 2844251662Sdim case ISD::ATOMIC_FENCE: { 2845210299Sed // If the target didn't lower this, lower it to '__sync_synchronize()' call 2846226633Sdim // FIXME: handle "fence singlethread" more efficiently. 2847210299Sed TargetLowering::ArgListTy Args; 2848239462Sdim TargetLowering:: 2849239462Sdim CallLoweringInfo CLI(Node->getOperand(0), 2850239462Sdim Type::getVoidTy(*DAG.getContext()), 2851218893Sdim false, false, false, false, 0, CallingConv::C, 2852218893Sdim /*isTailCall=*/false, 2853234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2854210299Sed DAG.getExternalSymbol("__sync_synchronize", 2855210299Sed TLI.getPointerTy()), 2856210299Sed Args, DAG, dl); 2857239462Sdim std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 2858239462Sdim 2859210299Sed Results.push_back(CallResult.second); 2860210299Sed break; 2861210299Sed } 2862226633Sdim case ISD::ATOMIC_LOAD: { 2863226633Sdim // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. 2864226633Sdim SDValue Zero = DAG.getConstant(0, Node->getValueType(0)); 2865226633Sdim SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, 2866226633Sdim cast<AtomicSDNode>(Node)->getMemoryVT(), 2867226633Sdim Node->getOperand(0), 2868226633Sdim Node->getOperand(1), Zero, Zero, 2869226633Sdim cast<AtomicSDNode>(Node)->getMemOperand(), 2870226633Sdim cast<AtomicSDNode>(Node)->getOrdering(), 2871226633Sdim cast<AtomicSDNode>(Node)->getSynchScope()); 2872226633Sdim Results.push_back(Swap.getValue(0)); 2873226633Sdim Results.push_back(Swap.getValue(1)); 2874226633Sdim break; 2875226633Sdim } 2876226633Sdim case ISD::ATOMIC_STORE: { 2877226633Sdim // There is no libcall for atomic store; fake it with ATOMIC_SWAP. 2878226633Sdim SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, 2879226633Sdim cast<AtomicSDNode>(Node)->getMemoryVT(), 2880226633Sdim Node->getOperand(0), 2881226633Sdim Node->getOperand(1), Node->getOperand(2), 2882226633Sdim cast<AtomicSDNode>(Node)->getMemOperand(), 2883226633Sdim cast<AtomicSDNode>(Node)->getOrdering(), 2884226633Sdim cast<AtomicSDNode>(Node)->getSynchScope()); 2885226633Sdim Results.push_back(Swap.getValue(1)); 2886226633Sdim break; 2887226633Sdim } 2888210299Sed // By default, atomic intrinsics are marked Legal and lowered. Targets 2889210299Sed // which don't support them directly, however, may want libcalls, in which 2890210299Sed // case they mark them Expand, and we get here. 2891210299Sed case ISD::ATOMIC_SWAP: 2892210299Sed case ISD::ATOMIC_LOAD_ADD: 2893210299Sed case ISD::ATOMIC_LOAD_SUB: 2894210299Sed case ISD::ATOMIC_LOAD_AND: 2895210299Sed case ISD::ATOMIC_LOAD_OR: 2896210299Sed case ISD::ATOMIC_LOAD_XOR: 2897210299Sed case ISD::ATOMIC_LOAD_NAND: 2898210299Sed case ISD::ATOMIC_LOAD_MIN: 2899210299Sed case ISD::ATOMIC_LOAD_MAX: 2900210299Sed case ISD::ATOMIC_LOAD_UMIN: 2901210299Sed case ISD::ATOMIC_LOAD_UMAX: 2902210299Sed case ISD::ATOMIC_CMP_SWAP: { 2903210299Sed std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node); 2904210299Sed Results.push_back(Tmp.first); 2905210299Sed Results.push_back(Tmp.second); 2906210299Sed break; 2907210299Sed } 2908193323Sed case ISD::DYNAMIC_STACKALLOC: 2909193323Sed ExpandDYNAMIC_STACKALLOC(Node, Results); 2910193323Sed break; 2911193323Sed case ISD::MERGE_VALUES: 2912193323Sed for (unsigned i = 0; i < Node->getNumValues(); i++) 2913193323Sed Results.push_back(Node->getOperand(i)); 2914193323Sed break; 2915193323Sed case ISD::UNDEF: { 2916198090Srdivacky EVT VT = Node->getValueType(0); 2917193323Sed if (VT.isInteger()) 2918193323Sed Results.push_back(DAG.getConstant(0, VT)); 2919207618Srdivacky else { 2920207618Srdivacky assert(VT.isFloatingPoint() && "Unknown value type!"); 2921193323Sed Results.push_back(DAG.getConstantFP(0, VT)); 2922207618Srdivacky } 2923193323Sed break; 2924193323Sed } 2925193323Sed case ISD::TRAP: { 2926193323Sed // If this operation is not supported, lower it to 'abort()' call 2927193323Sed TargetLowering::ArgListTy Args; 2928239462Sdim TargetLowering:: 2929239462Sdim CallLoweringInfo CLI(Node->getOperand(0), 2930239462Sdim Type::getVoidTy(*DAG.getContext()), 2931218893Sdim false, false, false, false, 0, CallingConv::C, 2932218893Sdim /*isTailCall=*/false, 2933234353Sdim /*doesNotReturn=*/false, /*isReturnValueUsed=*/true, 2934193323Sed DAG.getExternalSymbol("abort", TLI.getPointerTy()), 2935204642Srdivacky Args, DAG, dl); 2936239462Sdim std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); 2937239462Sdim 2938193323Sed Results.push_back(CallResult.second); 2939193323Sed break; 2940193323Sed } 2941193323Sed case ISD::FP_ROUND: 2942218893Sdim case ISD::BITCAST: 2943193323Sed Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), 2944193323Sed Node->getValueType(0), dl); 2945193323Sed Results.push_back(Tmp1); 2946193323Sed break; 2947193323Sed case ISD::FP_EXTEND: 2948193323Sed Tmp1 = EmitStackConvert(Node->getOperand(0), 2949193323Sed Node->getOperand(0).getValueType(), 2950193323Sed Node->getValueType(0), dl); 2951193323Sed Results.push_back(Tmp1); 2952193323Sed break; 2953193323Sed case ISD::SIGN_EXTEND_INREG: { 2954193323Sed // NOTE: we could fall back on load/store here too for targets without 2955193323Sed // SAR. However, it is doubtful that any exist. 2956198090Srdivacky EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2957200581Srdivacky EVT VT = Node->getValueType(0); 2958219077Sdim EVT ShiftAmountTy = TLI.getShiftAmountTy(VT); 2959202375Srdivacky if (VT.isVector()) 2960200581Srdivacky ShiftAmountTy = VT; 2961202375Srdivacky unsigned BitsDiff = VT.getScalarType().getSizeInBits() - 2962202375Srdivacky ExtraVT.getScalarType().getSizeInBits(); 2963200581Srdivacky SDValue ShiftCst = DAG.getConstant(BitsDiff, ShiftAmountTy); 2964193323Sed Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), 2965193323Sed Node->getOperand(0), ShiftCst); 2966193323Sed Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); 2967193323Sed Results.push_back(Tmp1); 2968193323Sed break; 2969193323Sed } 2970193323Sed case ISD::FP_ROUND_INREG: { 2971193323Sed // The only way we can lower this is to turn it into a TRUNCSTORE, 2972221345Sdim // EXTLOAD pair, targeting a temporary location (a stack slot). 2973193323Sed 2974193323Sed // NOTE: there is a choice here between constantly creating new stack 2975193323Sed // slots and always reusing the same one. We currently always create 2976193323Sed // new ones, as reuse may inhibit scheduling. 2977198090Srdivacky EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 2978193323Sed Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, 2979193323Sed Node->getValueType(0), dl); 2980193323Sed Results.push_back(Tmp1); 2981193323Sed break; 2982193323Sed } 2983193323Sed case ISD::SINT_TO_FP: 2984193323Sed case ISD::UINT_TO_FP: 2985193323Sed Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, 2986193323Sed Node->getOperand(0), Node->getValueType(0), dl); 2987193323Sed Results.push_back(Tmp1); 2988193323Sed break; 2989193323Sed case ISD::FP_TO_UINT: { 2990193323Sed SDValue True, False; 2991198090Srdivacky EVT VT = Node->getOperand(0).getValueType(); 2992198090Srdivacky EVT NVT = Node->getValueType(0); 2993249423Sdim APFloat apf(DAG.EVTToAPFloatSemantics(VT), 2994249423Sdim APInt::getNullValue(VT.getSizeInBits())); 2995193323Sed APInt x = APInt::getSignBit(NVT.getSizeInBits()); 2996193323Sed (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); 2997193323Sed Tmp1 = DAG.getConstantFP(apf, VT); 2998263508Sdim Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT), 2999193323Sed Node->getOperand(0), 3000193323Sed Tmp1, ISD::SETLT); 3001193323Sed True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); 3002193323Sed False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, 3003193323Sed DAG.getNode(ISD::FSUB, dl, VT, 3004193323Sed Node->getOperand(0), Tmp1)); 3005193323Sed False = DAG.getNode(ISD::XOR, dl, NVT, False, 3006193323Sed DAG.getConstant(x, NVT)); 3007263508Sdim Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False); 3008193323Sed Results.push_back(Tmp1); 3009193323Sed break; 3010193323Sed } 3011193323Sed case ISD::VAARG: { 3012193323Sed const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3013198090Srdivacky EVT VT = Node->getValueType(0); 3014193323Sed Tmp1 = Node->getOperand(0); 3015193323Sed Tmp2 = Node->getOperand(1); 3016210299Sed unsigned Align = Node->getConstantOperandVal(3); 3017210299Sed 3018218893Sdim SDValue VAListLoad = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, 3019263508Sdim MachinePointerInfo(V), 3020234353Sdim false, false, false, 0); 3021210299Sed SDValue VAList = VAListLoad; 3022210299Sed 3023210299Sed if (Align > TLI.getMinStackArgumentAlignment()) { 3024210299Sed assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); 3025210299Sed 3026263508Sdim VAList = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 3027210299Sed DAG.getConstant(Align - 1, 3028263508Sdim VAList.getValueType())); 3029210299Sed 3030263508Sdim VAList = DAG.getNode(ISD::AND, dl, VAList.getValueType(), VAList, 3031218893Sdim DAG.getConstant(-(int64_t)Align, 3032263508Sdim VAList.getValueType())); 3033210299Sed } 3034210299Sed 3035193323Sed // Increment the pointer, VAList, to the next vaarg 3036263508Sdim Tmp3 = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, 3037243830Sdim DAG.getConstant(TLI.getDataLayout()-> 3038207618Srdivacky getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())), 3039263508Sdim VAList.getValueType())); 3040193323Sed // Store the incremented VAList to the legalized pointer 3041218893Sdim Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2, 3042218893Sdim MachinePointerInfo(V), false, false, 0); 3043193323Sed // Load the actual argument out of the pointer VAList 3044218893Sdim Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(), 3045234353Sdim false, false, false, 0)); 3046193323Sed Results.push_back(Results[0].getValue(1)); 3047193323Sed break; 3048193323Sed } 3049193323Sed case ISD::VACOPY: { 3050193323Sed // This defaults to loading a pointer from the input and storing it to the 3051193323Sed // output, returning the chain. 3052193323Sed const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); 3053193323Sed const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); 3054193323Sed Tmp1 = DAG.getLoad(TLI.getPointerTy(), dl, Node->getOperand(0), 3055218893Sdim Node->getOperand(2), MachinePointerInfo(VS), 3056234353Sdim false, false, false, 0); 3057218893Sdim Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), 3058218893Sdim MachinePointerInfo(VD), false, false, 0); 3059193323Sed Results.push_back(Tmp1); 3060193323Sed break; 3061193323Sed } 3062193323Sed case ISD::EXTRACT_VECTOR_ELT: 3063193323Sed if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) 3064193323Sed // This must be an access of the only element. Return it. 3065218893Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), 3066193323Sed Node->getOperand(0)); 3067193323Sed else 3068193323Sed Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); 3069193323Sed Results.push_back(Tmp1); 3070193323Sed break; 3071193323Sed case ISD::EXTRACT_SUBVECTOR: 3072193323Sed Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); 3073193323Sed break; 3074218893Sdim case ISD::INSERT_SUBVECTOR: 3075218893Sdim Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); 3076218893Sdim break; 3077193323Sed case ISD::CONCAT_VECTORS: { 3078193574Sed Results.push_back(ExpandVectorBuildThroughStack(Node)); 3079193323Sed break; 3080193323Sed } 3081193323Sed case ISD::SCALAR_TO_VECTOR: 3082193323Sed Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); 3083193323Sed break; 3084193323Sed case ISD::INSERT_VECTOR_ELT: 3085193323Sed Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), 3086193323Sed Node->getOperand(1), 3087193323Sed Node->getOperand(2), dl)); 3088193323Sed break; 3089193323Sed case ISD::VECTOR_SHUFFLE: { 3090234353Sdim SmallVector<int, 32> NewMask; 3091234353Sdim ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 3092193323Sed 3093198090Srdivacky EVT VT = Node->getValueType(0); 3094198090Srdivacky EVT EltVT = VT.getVectorElementType(); 3095234353Sdim SDValue Op0 = Node->getOperand(0); 3096234353Sdim SDValue Op1 = Node->getOperand(1); 3097234353Sdim if (!TLI.isTypeLegal(EltVT)) { 3098234353Sdim 3099234353Sdim EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); 3100234353Sdim 3101234353Sdim // BUILD_VECTOR operands are allowed to be wider than the element type. 3102234353Sdim // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept it 3103234353Sdim if (NewEltVT.bitsLT(EltVT)) { 3104234353Sdim 3105234353Sdim // Convert shuffle node. 3106234353Sdim // If original node was v4i64 and the new EltVT is i32, 3107234353Sdim // cast operands to v8i32 and re-build the mask. 3108234353Sdim 3109234353Sdim // Calculate new VT, the size of the new VT should be equal to original. 3110263508Sdim EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltVT, 3111234353Sdim VT.getSizeInBits()/NewEltVT.getSizeInBits()); 3112234353Sdim assert(NewVT.bitsEq(VT)); 3113234353Sdim 3114234353Sdim // cast operands to new VT 3115234353Sdim Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); 3116234353Sdim Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); 3117234353Sdim 3118234353Sdim // Convert the shuffle mask 3119234353Sdim unsigned int factor = NewVT.getVectorNumElements()/VT.getVectorNumElements(); 3120234353Sdim 3121234353Sdim // EltVT gets smaller 3122234353Sdim assert(factor > 0); 3123234353Sdim 3124234353Sdim for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { 3125234353Sdim if (Mask[i] < 0) { 3126234353Sdim for (unsigned fi = 0; fi < factor; ++fi) 3127234353Sdim NewMask.push_back(Mask[i]); 3128234353Sdim } 3129234353Sdim else { 3130234353Sdim for (unsigned fi = 0; fi < factor; ++fi) 3131234353Sdim NewMask.push_back(Mask[i]*factor+fi); 3132234353Sdim } 3133234353Sdim } 3134234353Sdim Mask = NewMask; 3135234353Sdim VT = NewVT; 3136234353Sdim } 3137234353Sdim EltVT = NewEltVT; 3138234353Sdim } 3139193323Sed unsigned NumElems = VT.getVectorNumElements(); 3140234353Sdim SmallVector<SDValue, 16> Ops; 3141193323Sed for (unsigned i = 0; i != NumElems; ++i) { 3142193323Sed if (Mask[i] < 0) { 3143193323Sed Ops.push_back(DAG.getUNDEF(EltVT)); 3144193323Sed continue; 3145193323Sed } 3146193323Sed unsigned Idx = Mask[i]; 3147193323Sed if (Idx < NumElems) 3148193323Sed Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 3149234353Sdim Op0, 3150263508Sdim DAG.getConstant(Idx, TLI.getVectorIdxTy()))); 3151193323Sed else 3152193323Sed Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 3153234353Sdim Op1, 3154263508Sdim DAG.getConstant(Idx - NumElems, 3155263508Sdim TLI.getVectorIdxTy()))); 3156193323Sed } 3157234353Sdim 3158193323Sed Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); 3159234353Sdim // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. 3160234353Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); 3161193323Sed Results.push_back(Tmp1); 3162193323Sed break; 3163193323Sed } 3164193323Sed case ISD::EXTRACT_ELEMENT: { 3165198090Srdivacky EVT OpTy = Node->getOperand(0).getValueType(); 3166193323Sed if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { 3167193323Sed // 1 -> Hi 3168193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), 3169193323Sed DAG.getConstant(OpTy.getSizeInBits()/2, 3170219077Sdim TLI.getShiftAmountTy(Node->getOperand(0).getValueType()))); 3171193323Sed Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); 3172193323Sed } else { 3173193323Sed // 0 -> Lo 3174193323Sed Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), 3175193323Sed Node->getOperand(0)); 3176193323Sed } 3177193323Sed Results.push_back(Tmp1); 3178193323Sed break; 3179193323Sed } 3180193323Sed case ISD::STACKSAVE: 3181193323Sed // Expand to CopyFromReg if the target set 3182193323Sed // StackPointerRegisterToSaveRestore. 3183193323Sed if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3184193323Sed Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, 3185193323Sed Node->getValueType(0))); 3186193323Sed Results.push_back(Results[0].getValue(1)); 3187193323Sed } else { 3188193323Sed Results.push_back(DAG.getUNDEF(Node->getValueType(0))); 3189193323Sed Results.push_back(Node->getOperand(0)); 3190193323Sed } 3191193323Sed break; 3192193323Sed case ISD::STACKRESTORE: 3193193323Sed // Expand to CopyToReg if the target set 3194193323Sed // StackPointerRegisterToSaveRestore. 3195193323Sed if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { 3196193323Sed Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, 3197193323Sed Node->getOperand(1))); 3198193323Sed } else { 3199193323Sed Results.push_back(Node->getOperand(0)); 3200193323Sed } 3201193323Sed break; 3202193323Sed case ISD::FCOPYSIGN: 3203193323Sed Results.push_back(ExpandFCOPYSIGN(Node)); 3204193323Sed break; 3205193323Sed case ISD::FNEG: 3206193323Sed // Expand Y = FNEG(X) -> Y = SUB -0.0, X 3207193323Sed Tmp1 = DAG.getConstantFP(-0.0, Node->getValueType(0)); 3208193323Sed Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, 3209193323Sed Node->getOperand(0)); 3210193323Sed Results.push_back(Tmp1); 3211193323Sed break; 3212193323Sed case ISD::FABS: { 3213193323Sed // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X). 3214198090Srdivacky EVT VT = Node->getValueType(0); 3215193323Sed Tmp1 = Node->getOperand(0); 3216193323Sed Tmp2 = DAG.getConstantFP(0.0, VT); 3217263508Sdim Tmp2 = DAG.getSetCC(dl, getSetCCResultType(Tmp1.getValueType()), 3218193323Sed Tmp1, Tmp2, ISD::SETUGT); 3219193323Sed Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); 3220263508Sdim Tmp1 = DAG.getSelect(dl, VT, Tmp2, Tmp1, Tmp3); 3221193323Sed Results.push_back(Tmp1); 3222193323Sed break; 3223193323Sed } 3224193323Sed case ISD::FSQRT: 3225193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, 3226249423Sdim RTLIB::SQRT_F80, RTLIB::SQRT_F128, 3227249423Sdim RTLIB::SQRT_PPCF128)); 3228193323Sed break; 3229193323Sed case ISD::FSIN: 3230249423Sdim case ISD::FCOS: { 3231249423Sdim EVT VT = Node->getValueType(0); 3232249423Sdim bool isSIN = Node->getOpcode() == ISD::FSIN; 3233249423Sdim // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / 3234249423Sdim // fcos which share the same operand and both are used. 3235249423Sdim if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || 3236249423Sdim canCombineSinCosLibcall(Node, TLI, TM)) 3237249423Sdim && useSinCos(Node)) { 3238249423Sdim SDVTList VTs = DAG.getVTList(VT, VT); 3239249423Sdim Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); 3240249423Sdim if (!isSIN) 3241249423Sdim Tmp1 = Tmp1.getValue(1); 3242249423Sdim Results.push_back(Tmp1); 3243249423Sdim } else if (isSIN) { 3244249423Sdim Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, 3245249423Sdim RTLIB::SIN_F80, RTLIB::SIN_F128, 3246249423Sdim RTLIB::SIN_PPCF128)); 3247249423Sdim } else { 3248249423Sdim Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, 3249249423Sdim RTLIB::COS_F80, RTLIB::COS_F128, 3250249423Sdim RTLIB::COS_PPCF128)); 3251249423Sdim } 3252193323Sed break; 3253249423Sdim } 3254249423Sdim case ISD::FSINCOS: 3255249423Sdim // Expand into sincos libcall. 3256249423Sdim ExpandSinCosLibCall(Node, Results); 3257193323Sed break; 3258193323Sed case ISD::FLOG: 3259193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, 3260249423Sdim RTLIB::LOG_F80, RTLIB::LOG_F128, 3261249423Sdim RTLIB::LOG_PPCF128)); 3262193323Sed break; 3263193323Sed case ISD::FLOG2: 3264193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, 3265249423Sdim RTLIB::LOG2_F80, RTLIB::LOG2_F128, 3266249423Sdim RTLIB::LOG2_PPCF128)); 3267193323Sed break; 3268193323Sed case ISD::FLOG10: 3269193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, 3270249423Sdim RTLIB::LOG10_F80, RTLIB::LOG10_F128, 3271249423Sdim RTLIB::LOG10_PPCF128)); 3272193323Sed break; 3273193323Sed case ISD::FEXP: 3274193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, 3275249423Sdim RTLIB::EXP_F80, RTLIB::EXP_F128, 3276249423Sdim RTLIB::EXP_PPCF128)); 3277193323Sed break; 3278193323Sed case ISD::FEXP2: 3279193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, 3280249423Sdim RTLIB::EXP2_F80, RTLIB::EXP2_F128, 3281249423Sdim RTLIB::EXP2_PPCF128)); 3282193323Sed break; 3283193323Sed case ISD::FTRUNC: 3284193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 3285249423Sdim RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 3286249423Sdim RTLIB::TRUNC_PPCF128)); 3287193323Sed break; 3288193323Sed case ISD::FFLOOR: 3289193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 3290249423Sdim RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 3291249423Sdim RTLIB::FLOOR_PPCF128)); 3292193323Sed break; 3293193323Sed case ISD::FCEIL: 3294193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, 3295249423Sdim RTLIB::CEIL_F80, RTLIB::CEIL_F128, 3296249423Sdim RTLIB::CEIL_PPCF128)); 3297193323Sed break; 3298193323Sed case ISD::FRINT: 3299193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, 3300249423Sdim RTLIB::RINT_F80, RTLIB::RINT_F128, 3301249423Sdim RTLIB::RINT_PPCF128)); 3302193323Sed break; 3303193323Sed case ISD::FNEARBYINT: 3304193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, 3305193323Sed RTLIB::NEARBYINT_F64, 3306193323Sed RTLIB::NEARBYINT_F80, 3307249423Sdim RTLIB::NEARBYINT_F128, 3308193323Sed RTLIB::NEARBYINT_PPCF128)); 3309193323Sed break; 3310263508Sdim case ISD::FROUND: 3311263508Sdim Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32, 3312263508Sdim RTLIB::ROUND_F64, 3313263508Sdim RTLIB::ROUND_F80, 3314263508Sdim RTLIB::ROUND_F128, 3315263508Sdim RTLIB::ROUND_PPCF128)); 3316263508Sdim break; 3317193323Sed case ISD::FPOWI: 3318193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, 3319249423Sdim RTLIB::POWI_F80, RTLIB::POWI_F128, 3320249423Sdim RTLIB::POWI_PPCF128)); 3321193323Sed break; 3322193323Sed case ISD::FPOW: 3323193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, 3324249423Sdim RTLIB::POW_F80, RTLIB::POW_F128, 3325249423Sdim RTLIB::POW_PPCF128)); 3326193323Sed break; 3327193323Sed case ISD::FDIV: 3328193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, 3329249423Sdim RTLIB::DIV_F80, RTLIB::DIV_F128, 3330249423Sdim RTLIB::DIV_PPCF128)); 3331193323Sed break; 3332193323Sed case ISD::FREM: 3333193323Sed Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, 3334249423Sdim RTLIB::REM_F80, RTLIB::REM_F128, 3335249423Sdim RTLIB::REM_PPCF128)); 3336193323Sed break; 3337224145Sdim case ISD::FMA: 3338224145Sdim Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, 3339249423Sdim RTLIB::FMA_F80, RTLIB::FMA_F128, 3340249423Sdim RTLIB::FMA_PPCF128)); 3341224145Sdim break; 3342205218Srdivacky case ISD::FP16_TO_FP32: 3343205218Srdivacky Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); 3344205218Srdivacky break; 3345205218Srdivacky case ISD::FP32_TO_FP16: 3346205218Srdivacky Results.push_back(ExpandLibCall(RTLIB::FPROUND_F32_F16, Node, false)); 3347205218Srdivacky break; 3348193323Sed case ISD::ConstantFP: { 3349193323Sed ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); 3350193323Sed // Check to see if this FP immediate is already legal. 3351193323Sed // If this is a legal constant, turn it into a TargetConstantFP node. 3352234353Sdim if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) 3353234353Sdim Results.push_back(ExpandConstantFP(CFP, true)); 3354193323Sed break; 3355193323Sed } 3356234353Sdim case ISD::FSUB: { 3357234353Sdim EVT VT = Node->getValueType(0); 3358234353Sdim assert(TLI.isOperationLegalOrCustom(ISD::FADD, VT) && 3359234353Sdim TLI.isOperationLegalOrCustom(ISD::FNEG, VT) && 3360234353Sdim "Don't know how to expand this FP subtraction!"); 3361234353Sdim Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); 3362234353Sdim Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1); 3363234353Sdim Results.push_back(Tmp1); 3364234353Sdim break; 3365234353Sdim } 3366193323Sed case ISD::SUB: { 3367198090Srdivacky EVT VT = Node->getValueType(0); 3368193323Sed assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && 3369193323Sed TLI.isOperationLegalOrCustom(ISD::XOR, VT) && 3370193323Sed "Don't know how to expand this subtraction!"); 3371193323Sed Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), 3372193323Sed DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT)); 3373239462Sdim Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT)); 3374193323Sed Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); 3375193323Sed break; 3376193323Sed } 3377193323Sed case ISD::UREM: 3378193323Sed case ISD::SREM: { 3379198090Srdivacky EVT VT = Node->getValueType(0); 3380193323Sed bool isSigned = Node->getOpcode() == ISD::SREM; 3381193323Sed unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; 3382193323Sed unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3383193323Sed Tmp2 = Node->getOperand(0); 3384193323Sed Tmp3 = Node->getOperand(1); 3385221345Sdim if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3386221345Sdim (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3387243830Sdim // If div is legal, it's better to do the normal expansion 3388243830Sdim !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) && 3389239462Sdim useDivRem(Node, isSigned, false))) { 3390249423Sdim SDVTList VTs = DAG.getVTList(VT, VT); 3391193323Sed Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); 3392193323Sed } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { 3393193323Sed // X % Y -> X-X/Y*Y 3394193323Sed Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); 3395193323Sed Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); 3396193323Sed Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); 3397221345Sdim } else if (isSigned) 3398199481Srdivacky Tmp1 = ExpandIntLibCall(Node, true, 3399199481Srdivacky RTLIB::SREM_I8, 3400199481Srdivacky RTLIB::SREM_I16, RTLIB::SREM_I32, 3401193323Sed RTLIB::SREM_I64, RTLIB::SREM_I128); 3402221345Sdim else 3403199481Srdivacky Tmp1 = ExpandIntLibCall(Node, false, 3404199481Srdivacky RTLIB::UREM_I8, 3405199481Srdivacky RTLIB::UREM_I16, RTLIB::UREM_I32, 3406193323Sed RTLIB::UREM_I64, RTLIB::UREM_I128); 3407193323Sed Results.push_back(Tmp1); 3408193323Sed break; 3409193323Sed } 3410193323Sed case ISD::UDIV: 3411193323Sed case ISD::SDIV: { 3412193323Sed bool isSigned = Node->getOpcode() == ISD::SDIV; 3413193323Sed unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; 3414198090Srdivacky EVT VT = Node->getValueType(0); 3415193323Sed SDVTList VTs = DAG.getVTList(VT, VT); 3416221345Sdim if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) || 3417221345Sdim (isDivRemLibcallAvailable(Node, isSigned, TLI) && 3418239462Sdim useDivRem(Node, isSigned, true))) 3419193323Sed Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), 3420193323Sed Node->getOperand(1)); 3421193323Sed else if (isSigned) 3422199481Srdivacky Tmp1 = ExpandIntLibCall(Node, true, 3423199481Srdivacky RTLIB::SDIV_I8, 3424199481Srdivacky RTLIB::SDIV_I16, RTLIB::SDIV_I32, 3425193323Sed RTLIB::SDIV_I64, RTLIB::SDIV_I128); 3426193323Sed else 3427199481Srdivacky Tmp1 = ExpandIntLibCall(Node, false, 3428199481Srdivacky RTLIB::UDIV_I8, 3429199481Srdivacky RTLIB::UDIV_I16, RTLIB::UDIV_I32, 3430193323Sed RTLIB::UDIV_I64, RTLIB::UDIV_I128); 3431193323Sed Results.push_back(Tmp1); 3432193323Sed break; 3433193323Sed } 3434193323Sed case ISD::MULHU: 3435193323Sed case ISD::MULHS: { 3436193323Sed unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : 3437193323Sed ISD::SMUL_LOHI; 3438198090Srdivacky EVT VT = Node->getValueType(0); 3439193323Sed SDVTList VTs = DAG.getVTList(VT, VT); 3440193323Sed assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) && 3441193323Sed "If this wasn't legal, it shouldn't have been created!"); 3442193323Sed Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), 3443193323Sed Node->getOperand(1)); 3444193323Sed Results.push_back(Tmp1.getValue(1)); 3445193323Sed break; 3446193323Sed } 3447221345Sdim case ISD::SDIVREM: 3448221345Sdim case ISD::UDIVREM: 3449221345Sdim // Expand into divrem libcall 3450221345Sdim ExpandDivRemLibCall(Node, Results); 3451221345Sdim break; 3452193323Sed case ISD::MUL: { 3453198090Srdivacky EVT VT = Node->getValueType(0); 3454193323Sed SDVTList VTs = DAG.getVTList(VT, VT); 3455193323Sed // See if multiply or divide can be lowered using two-result operations. 3456193323Sed // We just need the low half of the multiply; try both the signed 3457193323Sed // and unsigned forms. If the target supports both SMUL_LOHI and 3458193323Sed // UMUL_LOHI, form a preference by checking which forms of plain 3459193323Sed // MULH it supports. 3460193323Sed bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); 3461193323Sed bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); 3462193323Sed bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); 3463193323Sed bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); 3464193323Sed unsigned OpToUse = 0; 3465193323Sed if (HasSMUL_LOHI && !HasMULHS) { 3466193323Sed OpToUse = ISD::SMUL_LOHI; 3467193323Sed } else if (HasUMUL_LOHI && !HasMULHU) { 3468193323Sed OpToUse = ISD::UMUL_LOHI; 3469193323Sed } else if (HasSMUL_LOHI) { 3470193323Sed OpToUse = ISD::SMUL_LOHI; 3471193323Sed } else if (HasUMUL_LOHI) { 3472193323Sed OpToUse = ISD::UMUL_LOHI; 3473193323Sed } 3474193323Sed if (OpToUse) { 3475193323Sed Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), 3476193323Sed Node->getOperand(1))); 3477193323Sed break; 3478193323Sed } 3479199481Srdivacky Tmp1 = ExpandIntLibCall(Node, false, 3480199481Srdivacky RTLIB::MUL_I8, 3481199481Srdivacky RTLIB::MUL_I16, RTLIB::MUL_I32, 3482193323Sed RTLIB::MUL_I64, RTLIB::MUL_I128); 3483193323Sed Results.push_back(Tmp1); 3484193323Sed break; 3485193323Sed } 3486193323Sed case ISD::SADDO: 3487193323Sed case ISD::SSUBO: { 3488193323Sed SDValue LHS = Node->getOperand(0); 3489193323Sed SDValue RHS = Node->getOperand(1); 3490193323Sed SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? 3491193323Sed ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3492193323Sed LHS, RHS); 3493193323Sed Results.push_back(Sum); 3494198090Srdivacky EVT OType = Node->getValueType(1); 3495193323Sed 3496193323Sed SDValue Zero = DAG.getConstant(0, LHS.getValueType()); 3497193323Sed 3498193323Sed // LHSSign -> LHS >= 0 3499193323Sed // RHSSign -> RHS >= 0 3500193323Sed // SumSign -> Sum >= 0 3501193323Sed // 3502193323Sed // Add: 3503193323Sed // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) 3504193323Sed // Sub: 3505193323Sed // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) 3506193323Sed // 3507193323Sed SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); 3508193323Sed SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); 3509193323Sed SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, 3510193323Sed Node->getOpcode() == ISD::SADDO ? 3511193323Sed ISD::SETEQ : ISD::SETNE); 3512193323Sed 3513193323Sed SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); 3514193323Sed SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); 3515193323Sed 3516193323Sed SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); 3517193323Sed Results.push_back(Cmp); 3518193323Sed break; 3519193323Sed } 3520193323Sed case ISD::UADDO: 3521193323Sed case ISD::USUBO: { 3522193323Sed SDValue LHS = Node->getOperand(0); 3523193323Sed SDValue RHS = Node->getOperand(1); 3524193323Sed SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? 3525193323Sed ISD::ADD : ISD::SUB, dl, LHS.getValueType(), 3526193323Sed LHS, RHS); 3527193323Sed Results.push_back(Sum); 3528193323Sed Results.push_back(DAG.getSetCC(dl, Node->getValueType(1), Sum, LHS, 3529193323Sed Node->getOpcode () == ISD::UADDO ? 3530193323Sed ISD::SETULT : ISD::SETUGT)); 3531193323Sed break; 3532193323Sed } 3533194612Sed case ISD::UMULO: 3534194612Sed case ISD::SMULO: { 3535198090Srdivacky EVT VT = Node->getValueType(0); 3536221345Sdim EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); 3537194612Sed SDValue LHS = Node->getOperand(0); 3538194612Sed SDValue RHS = Node->getOperand(1); 3539194612Sed SDValue BottomHalf; 3540194612Sed SDValue TopHalf; 3541201360Srdivacky static const unsigned Ops[2][3] = 3542194612Sed { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, 3543194612Sed { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; 3544194612Sed bool isSigned = Node->getOpcode() == ISD::SMULO; 3545194612Sed if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) { 3546194612Sed BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); 3547194612Sed TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); 3548194612Sed } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) { 3549194612Sed BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, 3550194612Sed RHS); 3551194612Sed TopHalf = BottomHalf.getValue(1); 3552218893Sdim } else if (TLI.isTypeLegal(EVT::getIntegerVT(*DAG.getContext(), 3553218893Sdim VT.getSizeInBits() * 2))) { 3554194612Sed LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); 3555194612Sed RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); 3556194612Sed Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); 3557194612Sed BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3558194612Sed DAG.getIntPtrConstant(0)); 3559194612Sed TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, 3560194612Sed DAG.getIntPtrConstant(1)); 3561218893Sdim } else { 3562218893Sdim // We can fall back to a libcall with an illegal type for the MUL if we 3563218893Sdim // have a libcall big enough. 3564218893Sdim // Also, we can fall back to a division in some cases, but that's a big 3565218893Sdim // performance hit in the general case. 3566218893Sdim RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 3567218893Sdim if (WideVT == MVT::i16) 3568218893Sdim LC = RTLIB::MUL_I16; 3569218893Sdim else if (WideVT == MVT::i32) 3570218893Sdim LC = RTLIB::MUL_I32; 3571218893Sdim else if (WideVT == MVT::i64) 3572218893Sdim LC = RTLIB::MUL_I64; 3573218893Sdim else if (WideVT == MVT::i128) 3574218893Sdim LC = RTLIB::MUL_I128; 3575218893Sdim assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); 3576223017Sdim 3577223017Sdim // The high part is obtained by SRA'ing all but one of the bits of low 3578221345Sdim // part. 3579221345Sdim unsigned LoSize = VT.getSizeInBits(); 3580221345Sdim SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS, 3581221345Sdim DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3582221345Sdim SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS, 3583221345Sdim DAG.getConstant(LoSize-1, TLI.getPointerTy())); 3584219077Sdim 3585221345Sdim // Here we're passing the 2 arguments explicitly as 4 arguments that are 3586221345Sdim // pre-lowered to the correct types. This all depends upon WideVT not 3587221345Sdim // being a legal type for the architecture and thus has to be split to 3588221345Sdim // two arguments. 3589221345Sdim SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; 3590221345Sdim SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); 3591221345Sdim BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3592221345Sdim DAG.getIntPtrConstant(0)); 3593221345Sdim TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, 3594221345Sdim DAG.getIntPtrConstant(1)); 3595234353Sdim // Ret is a node with an illegal type. Because such things are not 3596234353Sdim // generally permitted during this phase of legalization, delete the 3597234353Sdim // node. The above EXTRACT_ELEMENT nodes should have been folded. 3598234353Sdim DAG.DeleteNode(Ret.getNode()); 3599194612Sed } 3600223017Sdim 3601194612Sed if (isSigned) { 3602219077Sdim Tmp1 = DAG.getConstant(VT.getSizeInBits() - 1, 3603219077Sdim TLI.getShiftAmountTy(BottomHalf.getValueType())); 3604194612Sed Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); 3605263508Sdim TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1, 3606194612Sed ISD::SETNE); 3607194612Sed } else { 3608263508Sdim TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, 3609194612Sed DAG.getConstant(0, VT), ISD::SETNE); 3610194612Sed } 3611194612Sed Results.push_back(BottomHalf); 3612194612Sed Results.push_back(TopHalf); 3613194612Sed break; 3614194612Sed } 3615193323Sed case ISD::BUILD_PAIR: { 3616198090Srdivacky EVT PairTy = Node->getValueType(0); 3617193323Sed Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); 3618193323Sed Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); 3619193323Sed Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, 3620193323Sed DAG.getConstant(PairTy.getSizeInBits()/2, 3621219077Sdim TLI.getShiftAmountTy(PairTy))); 3622193323Sed Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); 3623193323Sed break; 3624193323Sed } 3625193323Sed case ISD::SELECT: 3626193323Sed Tmp1 = Node->getOperand(0); 3627193323Sed Tmp2 = Node->getOperand(1); 3628193323Sed Tmp3 = Node->getOperand(2); 3629193323Sed if (Tmp1.getOpcode() == ISD::SETCC) { 3630193323Sed Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), 3631193323Sed Tmp2, Tmp3, 3632193323Sed cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); 3633193323Sed } else { 3634193323Sed Tmp1 = DAG.getSelectCC(dl, Tmp1, 3635193323Sed DAG.getConstant(0, Tmp1.getValueType()), 3636193323Sed Tmp2, Tmp3, ISD::SETNE); 3637193323Sed } 3638193323Sed Results.push_back(Tmp1); 3639193323Sed break; 3640193323Sed case ISD::BR_JT: { 3641193323Sed SDValue Chain = Node->getOperand(0); 3642193323Sed SDValue Table = Node->getOperand(1); 3643193323Sed SDValue Index = Node->getOperand(2); 3644193323Sed 3645198090Srdivacky EVT PTy = TLI.getPointerTy(); 3646203954Srdivacky 3647243830Sdim const DataLayout &TD = *TLI.getDataLayout(); 3648203954Srdivacky unsigned EntrySize = 3649203954Srdivacky DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); 3650210299Sed 3651263508Sdim Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), 3652263508Sdim Index, DAG.getConstant(EntrySize, Index.getValueType())); 3653263508Sdim SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), 3654263508Sdim Index, Table); 3655193323Sed 3656198090Srdivacky EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); 3657218893Sdim SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, 3658218893Sdim MachinePointerInfo::getJumpTable(), MemVT, 3659203954Srdivacky false, false, 0); 3660193323Sed Addr = LD; 3661207618Srdivacky if (TM.getRelocationModel() == Reloc::PIC_) { 3662193323Sed // For PIC, the sequence is: 3663193323Sed // BRIND(load(Jumptable + index) + RelocBase) 3664193323Sed // RelocBase can be JumpTable, GOT or some sort of global base. 3665193323Sed Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, 3666193323Sed TLI.getPICJumpTableRelocBase(Table, DAG)); 3667193323Sed } 3668193323Sed Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); 3669193323Sed Results.push_back(Tmp1); 3670193323Sed break; 3671193323Sed } 3672193323Sed case ISD::BRCOND: 3673193323Sed // Expand brcond's setcc into its constituent parts and create a BR_CC 3674193323Sed // Node. 3675193323Sed Tmp1 = Node->getOperand(0); 3676193323Sed Tmp2 = Node->getOperand(1); 3677193323Sed if (Tmp2.getOpcode() == ISD::SETCC) { 3678193323Sed Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, 3679193323Sed Tmp1, Tmp2.getOperand(2), 3680193323Sed Tmp2.getOperand(0), Tmp2.getOperand(1), 3681193323Sed Node->getOperand(2)); 3682193323Sed } else { 3683223017Sdim // We test only the i1 bit. Skip the AND if UNDEF. 3684223017Sdim Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 : 3685223017Sdim DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, 3686223017Sdim DAG.getConstant(1, Tmp2.getValueType())); 3687193323Sed Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, 3688223017Sdim DAG.getCondCode(ISD::SETNE), Tmp3, 3689223017Sdim DAG.getConstant(0, Tmp3.getValueType()), 3690193323Sed Node->getOperand(2)); 3691193323Sed } 3692193323Sed Results.push_back(Tmp1); 3693193323Sed break; 3694193323Sed case ISD::SETCC: { 3695193323Sed Tmp1 = Node->getOperand(0); 3696193323Sed Tmp2 = Node->getOperand(1); 3697193323Sed Tmp3 = Node->getOperand(2); 3698263508Sdim bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, 3699263508Sdim Tmp3, NeedInvert, dl); 3700193323Sed 3701263508Sdim if (Legalized) { 3702263508Sdim // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 3703263508Sdim // condition code, create a new SETCC node. 3704263508Sdim if (Tmp3.getNode()) 3705263508Sdim Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 3706263508Sdim Tmp1, Tmp2, Tmp3); 3707263508Sdim 3708263508Sdim // If we expanded the SETCC by inverting the condition code, then wrap 3709263508Sdim // the existing SETCC in a NOT to restore the intended condition. 3710263508Sdim if (NeedInvert) 3711263508Sdim Tmp1 = DAG.getNOT(dl, Tmp1, Tmp1->getValueType(0)); 3712263508Sdim 3713193323Sed Results.push_back(Tmp1); 3714193323Sed break; 3715193323Sed } 3716193323Sed 3717193323Sed // Otherwise, SETCC for the given comparison type must be completely 3718193323Sed // illegal; expand it into a SELECT_CC. 3719198090Srdivacky EVT VT = Node->getValueType(0); 3720249423Sdim int TrueValue; 3721249423Sdim switch (TLI.getBooleanContents(VT.isVector())) { 3722249423Sdim case TargetLowering::ZeroOrOneBooleanContent: 3723249423Sdim case TargetLowering::UndefinedBooleanContent: 3724249423Sdim TrueValue = 1; 3725249423Sdim break; 3726249423Sdim case TargetLowering::ZeroOrNegativeOneBooleanContent: 3727249423Sdim TrueValue = -1; 3728249423Sdim break; 3729249423Sdim } 3730193323Sed Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, 3731249423Sdim DAG.getConstant(TrueValue, VT), DAG.getConstant(0, VT), 3732249423Sdim Tmp3); 3733193323Sed Results.push_back(Tmp1); 3734193323Sed break; 3735193323Sed } 3736193323Sed case ISD::SELECT_CC: { 3737193323Sed Tmp1 = Node->getOperand(0); // LHS 3738193323Sed Tmp2 = Node->getOperand(1); // RHS 3739193323Sed Tmp3 = Node->getOperand(2); // True 3740193323Sed Tmp4 = Node->getOperand(3); // False 3741193323Sed SDValue CC = Node->getOperand(4); 3742193323Sed 3743263508Sdim bool Legalized = false; 3744263508Sdim // Try to legalize by inverting the condition. This is for targets that 3745263508Sdim // might support an ordered version of a condition, but not the unordered 3746263508Sdim // version (or vice versa). 3747263508Sdim ISD::CondCode InvCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), 3748263508Sdim Tmp1.getValueType().isInteger()); 3749263508Sdim if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) { 3750263508Sdim // Use the new condition code and swap true and false 3751263508Sdim Legalized = true; 3752263508Sdim Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); 3753263508Sdim } else { 3754263508Sdim // If The inverse is not legal, then try to swap the arguments using 3755263508Sdim // the inverse condition code. 3756263508Sdim ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); 3757263508Sdim if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) { 3758263508Sdim // The swapped inverse condition is legal, so swap true and false, 3759263508Sdim // lhs and rhs. 3760263508Sdim Legalized = true; 3761263508Sdim Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); 3762263508Sdim } 3763263508Sdim } 3764193323Sed 3765263508Sdim if (!Legalized) { 3766263508Sdim Legalized = LegalizeSetCCCondCode( 3767263508Sdim getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert, 3768263508Sdim dl); 3769263508Sdim 3770263508Sdim assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); 3771263508Sdim 3772263508Sdim // If we expanded the SETCC by inverting the condition code, then swap 3773263508Sdim // the True/False operands to match. 3774263508Sdim if (NeedInvert) 3775263508Sdim std::swap(Tmp3, Tmp4); 3776263508Sdim 3777263508Sdim // If we expanded the SETCC by swapping LHS and RHS, or by inverting the 3778263508Sdim // condition code, create a new SELECT_CC node. 3779263508Sdim if (CC.getNode()) { 3780263508Sdim Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), 3781263508Sdim Tmp1, Tmp2, Tmp3, Tmp4, CC); 3782263508Sdim } else { 3783263508Sdim Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); 3784263508Sdim CC = DAG.getCondCode(ISD::SETNE); 3785263508Sdim Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2, 3786263508Sdim Tmp3, Tmp4, CC); 3787263508Sdim } 3788263508Sdim } 3789193323Sed Results.push_back(Tmp1); 3790193323Sed break; 3791193323Sed } 3792193323Sed case ISD::BR_CC: { 3793193323Sed Tmp1 = Node->getOperand(0); // Chain 3794193323Sed Tmp2 = Node->getOperand(2); // LHS 3795193323Sed Tmp3 = Node->getOperand(3); // RHS 3796193323Sed Tmp4 = Node->getOperand(1); // CC 3797193323Sed 3798263508Sdim bool Legalized = LegalizeSetCCCondCode(getSetCCResultType( 3799263508Sdim Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl); 3800263508Sdim (void)Legalized; 3801263508Sdim assert(Legalized && "Can't legalize BR_CC with legal condition!"); 3802193323Sed 3803263508Sdim // If we expanded the SETCC by inverting the condition code, then wrap 3804263508Sdim // the existing SETCC in a NOT to restore the intended condition. 3805263508Sdim if (NeedInvert) 3806263508Sdim Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0)); 3807263508Sdim 3808263508Sdim // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC 3809263508Sdim // node. 3810263508Sdim if (Tmp4.getNode()) { 3811263508Sdim Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, 3812263508Sdim Tmp4, Tmp2, Tmp3, Node->getOperand(4)); 3813263508Sdim } else { 3814263508Sdim Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); 3815263508Sdim Tmp4 = DAG.getCondCode(ISD::SETNE); 3816263508Sdim Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, Tmp2, 3817263508Sdim Tmp3, Node->getOperand(4)); 3818263508Sdim } 3819193323Sed Results.push_back(Tmp1); 3820193323Sed break; 3821193323Sed } 3822234353Sdim case ISD::BUILD_VECTOR: 3823234353Sdim Results.push_back(ExpandBUILD_VECTOR(Node)); 3824234353Sdim break; 3825234353Sdim case ISD::SRA: 3826234353Sdim case ISD::SRL: 3827234353Sdim case ISD::SHL: { 3828234353Sdim // Scalarize vector SRA/SRL/SHL. 3829234353Sdim EVT VT = Node->getValueType(0); 3830234353Sdim assert(VT.isVector() && "Unable to legalize non-vector shift"); 3831234353Sdim assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); 3832234353Sdim unsigned NumElem = VT.getVectorNumElements(); 3833234353Sdim 3834234353Sdim SmallVector<SDValue, 8> Scalars; 3835234353Sdim for (unsigned Idx = 0; Idx < NumElem; Idx++) { 3836234353Sdim SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3837234353Sdim VT.getScalarType(), 3838263508Sdim Node->getOperand(0), DAG.getConstant(Idx, 3839263508Sdim TLI.getVectorIdxTy())); 3840234353Sdim SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, 3841234353Sdim VT.getScalarType(), 3842263508Sdim Node->getOperand(1), DAG.getConstant(Idx, 3843263508Sdim TLI.getVectorIdxTy())); 3844234353Sdim Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, 3845234353Sdim VT.getScalarType(), Ex, Sh)); 3846234353Sdim } 3847234353Sdim SDValue Result = 3848234353Sdim DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), 3849234353Sdim &Scalars[0], Scalars.size()); 3850234353Sdim ReplaceNode(SDValue(Node, 0), Result); 3851234353Sdim break; 3852234353Sdim } 3853193323Sed case ISD::GLOBAL_OFFSET_TABLE: 3854193323Sed case ISD::GlobalAddress: 3855193323Sed case ISD::GlobalTLSAddress: 3856193323Sed case ISD::ExternalSymbol: 3857193323Sed case ISD::ConstantPool: 3858193323Sed case ISD::JumpTable: 3859193323Sed case ISD::INTRINSIC_W_CHAIN: 3860193323Sed case ISD::INTRINSIC_WO_CHAIN: 3861193323Sed case ISD::INTRINSIC_VOID: 3862193323Sed // FIXME: Custom lowering for these operations shouldn't return null! 3863193323Sed break; 3864193323Sed } 3865234353Sdim 3866234353Sdim // Replace the original node with the legalized result. 3867234353Sdim if (!Results.empty()) 3868234353Sdim ReplaceNode(Node, Results.data()); 3869193323Sed} 3870234353Sdim 3871234353Sdimvoid SelectionDAGLegalize::PromoteNode(SDNode *Node) { 3872234353Sdim SmallVector<SDValue, 8> Results; 3873249423Sdim MVT OVT = Node->getSimpleValueType(0); 3874193323Sed if (Node->getOpcode() == ISD::UINT_TO_FP || 3875198090Srdivacky Node->getOpcode() == ISD::SINT_TO_FP || 3876198090Srdivacky Node->getOpcode() == ISD::SETCC) { 3877249423Sdim OVT = Node->getOperand(0).getSimpleValueType(); 3878193323Sed } 3879249423Sdim MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); 3880263508Sdim SDLoc dl(Node); 3881193323Sed SDValue Tmp1, Tmp2, Tmp3; 3882193323Sed switch (Node->getOpcode()) { 3883193323Sed case ISD::CTTZ: 3884234353Sdim case ISD::CTTZ_ZERO_UNDEF: 3885193323Sed case ISD::CTLZ: 3886234353Sdim case ISD::CTLZ_ZERO_UNDEF: 3887193323Sed case ISD::CTPOP: 3888193323Sed // Zero extend the argument. 3889193323Sed Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3890234353Sdim // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is 3891234353Sdim // already the correct result. 3892198090Srdivacky Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 3893193323Sed if (Node->getOpcode() == ISD::CTTZ) { 3894234353Sdim // FIXME: This should set a bit in the zero extended value instead. 3895263508Sdim Tmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), 3896193323Sed Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), 3897193323Sed ISD::SETEQ); 3898263508Sdim Tmp1 = DAG.getSelect(dl, NVT, Tmp2, 3899263508Sdim DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); 3900234353Sdim } else if (Node->getOpcode() == ISD::CTLZ || 3901234353Sdim Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { 3902193323Sed // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) 3903193323Sed Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, 3904193323Sed DAG.getConstant(NVT.getSizeInBits() - 3905193323Sed OVT.getSizeInBits(), NVT)); 3906193323Sed } 3907198090Srdivacky Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); 3908193323Sed break; 3909193323Sed case ISD::BSWAP: { 3910193323Sed unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); 3911201360Srdivacky Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); 3912193323Sed Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); 3913193323Sed Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, 3914219077Sdim DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT))); 3915193323Sed Results.push_back(Tmp1); 3916193323Sed break; 3917193323Sed } 3918193323Sed case ISD::FP_TO_UINT: 3919193323Sed case ISD::FP_TO_SINT: 3920193323Sed Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), 3921193323Sed Node->getOpcode() == ISD::FP_TO_SINT, dl); 3922193323Sed Results.push_back(Tmp1); 3923193323Sed break; 3924193323Sed case ISD::UINT_TO_FP: 3925193323Sed case ISD::SINT_TO_FP: 3926193323Sed Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), 3927193323Sed Node->getOpcode() == ISD::SINT_TO_FP, dl); 3928193323Sed Results.push_back(Tmp1); 3929193323Sed break; 3930234353Sdim case ISD::VAARG: { 3931234353Sdim SDValue Chain = Node->getOperand(0); // Get the chain. 3932234353Sdim SDValue Ptr = Node->getOperand(1); // Get the pointer. 3933234353Sdim 3934234353Sdim unsigned TruncOp; 3935234353Sdim if (OVT.isVector()) { 3936234353Sdim TruncOp = ISD::BITCAST; 3937234353Sdim } else { 3938234353Sdim assert(OVT.isInteger() 3939234353Sdim && "VAARG promotion is supported only for vectors or integer types"); 3940234353Sdim TruncOp = ISD::TRUNCATE; 3941234353Sdim } 3942234353Sdim 3943234353Sdim // Perform the larger operation, then convert back 3944234353Sdim Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), 3945234353Sdim Node->getConstantOperandVal(3)); 3946234353Sdim Chain = Tmp1.getValue(1); 3947234353Sdim 3948234353Sdim Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); 3949234353Sdim 3950234353Sdim // Modified the chain result - switch anything that used the old chain to 3951234353Sdim // use the new one. 3952234353Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); 3953234353Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); 3954234353Sdim ReplacedNode(Node); 3955234353Sdim break; 3956234353Sdim } 3957193323Sed case ISD::AND: 3958193323Sed case ISD::OR: 3959198090Srdivacky case ISD::XOR: { 3960198090Srdivacky unsigned ExtOp, TruncOp; 3961198090Srdivacky if (OVT.isVector()) { 3962218893Sdim ExtOp = ISD::BITCAST; 3963218893Sdim TruncOp = ISD::BITCAST; 3964207618Srdivacky } else { 3965207618Srdivacky assert(OVT.isInteger() && "Cannot promote logic operation"); 3966198090Srdivacky ExtOp = ISD::ANY_EXTEND; 3967198090Srdivacky TruncOp = ISD::TRUNCATE; 3968198090Srdivacky } 3969198090Srdivacky // Promote each of the values to the new type. 3970198090Srdivacky Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 3971198090Srdivacky Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3972198090Srdivacky // Perform the larger operation, then convert back 3973193323Sed Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 3974198090Srdivacky Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); 3975193323Sed break; 3976198090Srdivacky } 3977198090Srdivacky case ISD::SELECT: { 3978193323Sed unsigned ExtOp, TruncOp; 3979193323Sed if (Node->getValueType(0).isVector()) { 3980218893Sdim ExtOp = ISD::BITCAST; 3981218893Sdim TruncOp = ISD::BITCAST; 3982193323Sed } else if (Node->getValueType(0).isInteger()) { 3983193323Sed ExtOp = ISD::ANY_EXTEND; 3984193323Sed TruncOp = ISD::TRUNCATE; 3985193323Sed } else { 3986193323Sed ExtOp = ISD::FP_EXTEND; 3987193323Sed TruncOp = ISD::FP_ROUND; 3988193323Sed } 3989193323Sed Tmp1 = Node->getOperand(0); 3990193323Sed // Promote each of the values to the new type. 3991193323Sed Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 3992193323Sed Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); 3993193323Sed // Perform the larger operation, then round down. 3994263508Sdim Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); 3995193323Sed if (TruncOp != ISD::FP_ROUND) 3996193323Sed Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); 3997193323Sed else 3998193323Sed Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, 3999193323Sed DAG.getIntPtrConstant(0)); 4000193323Sed Results.push_back(Tmp1); 4001193323Sed break; 4002198090Srdivacky } 4003193323Sed case ISD::VECTOR_SHUFFLE: { 4004234353Sdim ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); 4005193323Sed 4006193323Sed // Cast the two input vectors. 4007218893Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); 4008218893Sdim Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); 4009193323Sed 4010193323Sed // Convert the shuffle mask to the right # elements. 4011193323Sed Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); 4012218893Sdim Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); 4013193323Sed Results.push_back(Tmp1); 4014193323Sed break; 4015193323Sed } 4016193323Sed case ISD::SETCC: { 4017198090Srdivacky unsigned ExtOp = ISD::FP_EXTEND; 4018198090Srdivacky if (NVT.isInteger()) { 4019198090Srdivacky ISD::CondCode CCCode = 4020198090Srdivacky cast<CondCodeSDNode>(Node->getOperand(2))->get(); 4021198090Srdivacky ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 4022193323Sed } 4023198090Srdivacky Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); 4024198090Srdivacky Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); 4025193323Sed Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), 4026193323Sed Tmp1, Tmp2, Node->getOperand(2))); 4027193323Sed break; 4028193323Sed } 4029234353Sdim case ISD::FDIV: 4030234353Sdim case ISD::FREM: 4031234353Sdim case ISD::FPOW: { 4032234353Sdim Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4033234353Sdim Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); 4034234353Sdim Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); 4035234353Sdim Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4036234353Sdim Tmp3, DAG.getIntPtrConstant(0))); 4037234353Sdim break; 4038193323Sed } 4039234353Sdim case ISD::FLOG2: 4040234353Sdim case ISD::FEXP2: 4041234353Sdim case ISD::FLOG: 4042234353Sdim case ISD::FEXP: { 4043234353Sdim Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); 4044234353Sdim Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); 4045234353Sdim Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, 4046234353Sdim Tmp2, DAG.getIntPtrConstant(0))); 4047234353Sdim break; 4048234353Sdim } 4049234353Sdim } 4050234353Sdim 4051234353Sdim // Replace the original node with the legalized result. 4052234353Sdim if (!Results.empty()) 4053234353Sdim ReplaceNode(Node, Results.data()); 4054193323Sed} 4055193323Sed 4056193323Sed// SelectionDAG::Legalize - This is the entry point for the file. 4057193323Sed// 4058223017Sdimvoid SelectionDAG::Legalize() { 4059193323Sed /// run - This is the main entry point to this class. 4060193323Sed /// 4061223017Sdim SelectionDAGLegalize(*this).LegalizeDAG(); 4062193323Sed} 4063