1239310Sdim//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// 2239310Sdim// 3239310Sdim// The LLVM Compiler Infrastructure 4239310Sdim// 5239310Sdim// This file is distributed under the University of Illinois Open Source 6239310Sdim// License. See LICENSE.TXT for details. 7239310Sdim// 8239310Sdim//===----------------------------------------------------------------------===// 9239310Sdim// 10239310Sdim// This file defines an instruction selector for the NVPTX target. 11239310Sdim// 12239310Sdim//===----------------------------------------------------------------------===// 13239310Sdim 14239310Sdim#include "NVPTXISelDAGToDAG.h" 15249423Sdim#include "llvm/IR/GlobalValue.h" 16249423Sdim#include "llvm/IR/Instructions.h" 17249423Sdim#include "llvm/Support/CommandLine.h" 18239310Sdim#include "llvm/Support/Debug.h" 19239310Sdim#include "llvm/Support/ErrorHandling.h" 20249423Sdim#include "llvm/Support/raw_ostream.h" 21239310Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 22239310Sdim 23239310Sdim#undef DEBUG_TYPE 24239310Sdim#define DEBUG_TYPE "nvptx-isel" 25239310Sdim 26239310Sdimusing namespace llvm; 27239310Sdim 28249423Sdimstatic cl::opt<bool> UseFMADInstruction( 29249423Sdim "nvptx-mad-enable", cl::ZeroOrMore, 30249423Sdim cl::desc("NVPTX Specific: Enable generating FMAD instructions"), 31249423Sdim cl::init(false)); 32239310Sdim 33239310Sdimstatic cl::opt<int> 34249423SdimFMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, 35239310Sdim cl::desc("NVPTX Specific: FMA contraction (0: don't do it" 36249423Sdim " 1: do it 2: do it aggressively"), 37249423Sdim cl::init(2)); 38239310Sdim 39249423Sdimstatic cl::opt<int> UsePrecDivF32( 40249423Sdim "nvptx-prec-divf32", cl::ZeroOrMore, 41249423Sdim cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 42249423Sdim " IEEE Compliant F32 div.rnd if avaiable."), 43249423Sdim cl::init(2)); 44239310Sdim 45251662Sdimstatic cl::opt<bool> 46251662SdimUsePrecSqrtF32("nvptx-prec-sqrtf32", 47251662Sdim cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 48251662Sdim cl::init(true)); 49251662Sdim 50239310Sdim/// createNVPTXISelDag - This pass converts a legalized DAG into a 51239310Sdim/// NVPTX-specific DAG, ready for instruction scheduling. 52239310SdimFunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, 53239310Sdim llvm::CodeGenOpt::Level OptLevel) { 54239310Sdim return new NVPTXDAGToDAGISel(TM, OptLevel); 55239310Sdim} 56239310Sdim 57239310SdimNVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 58239310Sdim CodeGenOpt::Level OptLevel) 59249423Sdim : SelectionDAGISel(tm, OptLevel), 60249423Sdim Subtarget(tm.getSubtarget<NVPTXSubtarget>()) { 61239310Sdim // Always do fma.f32 fpcontract if the target supports the instruction. 62239310Sdim // Always do fma.f64 fpcontract if the target supports the instruction. 63239310Sdim // Do mad.f32 is nvptx-mad-enable is specified and the target does not 64239310Sdim // support fma.f32. 65239310Sdim 66239310Sdim doFMADF32 = (OptLevel > 0) && UseFMADInstruction && !Subtarget.hasFMAF32(); 67249423Sdim doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1); 68249423Sdim doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1); 69249423Sdim doFMAF32AGG = 70249423Sdim (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2); 71249423Sdim doFMAF64AGG = 72249423Sdim (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2); 73239310Sdim 74239310Sdim allowFMA = (FMAContractLevel >= 1) || UseFMADInstruction; 75239310Sdim 76239310Sdim UseF32FTZ = false; 77239310Sdim 78239310Sdim doMulWide = (OptLevel > 0); 79239310Sdim 80239310Sdim // Decide how to translate f32 div 81239310Sdim do_DIVF32_PREC = UsePrecDivF32; 82251662Sdim // Decide how to translate f32 sqrt 83251662Sdim do_SQRTF32_PREC = UsePrecSqrtF32; 84239310Sdim // sm less than sm_20 does not support div.rnd. Use div.full. 85239310Sdim if (do_DIVF32_PREC == 2 && !Subtarget.reqPTX20()) 86239310Sdim do_DIVF32_PREC = 1; 87239310Sdim 88239310Sdim} 89239310Sdim 90239310Sdim/// Select - Select instructions not customized! Used for 91239310Sdim/// expanded, promoted and normal instructions. 92249423SdimSDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 93239310Sdim 94255804Sdim if (N->isMachineOpcode()) { 95255804Sdim N->setNodeId(-1); 96249423Sdim return NULL; // Already selected. 97255804Sdim } 98239310Sdim 99239310Sdim SDNode *ResNode = NULL; 100239310Sdim switch (N->getOpcode()) { 101239310Sdim case ISD::LOAD: 102239310Sdim ResNode = SelectLoad(N); 103239310Sdim break; 104239310Sdim case ISD::STORE: 105239310Sdim ResNode = SelectStore(N); 106239310Sdim break; 107249423Sdim case NVPTXISD::LoadV2: 108249423Sdim case NVPTXISD::LoadV4: 109249423Sdim ResNode = SelectLoadVector(N); 110249423Sdim break; 111249423Sdim case NVPTXISD::LDGV2: 112249423Sdim case NVPTXISD::LDGV4: 113249423Sdim case NVPTXISD::LDUV2: 114249423Sdim case NVPTXISD::LDUV4: 115249423Sdim ResNode = SelectLDGLDUVector(N); 116249423Sdim break; 117249423Sdim case NVPTXISD::StoreV2: 118249423Sdim case NVPTXISD::StoreV4: 119249423Sdim ResNode = SelectStoreVector(N); 120249423Sdim break; 121249423Sdim default: 122249423Sdim break; 123239310Sdim } 124239310Sdim if (ResNode) 125239310Sdim return ResNode; 126239310Sdim return SelectCode(N); 127239310Sdim} 128239310Sdim 129249423Sdimstatic unsigned int getCodeAddrSpace(MemSDNode *N, 130249423Sdim const NVPTXSubtarget &Subtarget) { 131239310Sdim const Value *Src = N->getSrcValue(); 132239310Sdim if (!Src) 133239310Sdim return NVPTX::PTXLdStInstCode::LOCAL; 134239310Sdim 135239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { 136239310Sdim switch (PT->getAddressSpace()) { 137249423Sdim case llvm::ADDRESS_SPACE_LOCAL: 138249423Sdim return NVPTX::PTXLdStInstCode::LOCAL; 139249423Sdim case llvm::ADDRESS_SPACE_GLOBAL: 140249423Sdim return NVPTX::PTXLdStInstCode::GLOBAL; 141249423Sdim case llvm::ADDRESS_SPACE_SHARED: 142249423Sdim return NVPTX::PTXLdStInstCode::SHARED; 143239310Sdim case llvm::ADDRESS_SPACE_CONST_NOT_GEN: 144239310Sdim return NVPTX::PTXLdStInstCode::CONSTANT; 145249423Sdim case llvm::ADDRESS_SPACE_GENERIC: 146249423Sdim return NVPTX::PTXLdStInstCode::GENERIC; 147249423Sdim case llvm::ADDRESS_SPACE_PARAM: 148249423Sdim return NVPTX::PTXLdStInstCode::PARAM; 149239310Sdim case llvm::ADDRESS_SPACE_CONST: 150239310Sdim // If the arch supports generic address space, translate it to GLOBAL 151239310Sdim // for correctness. 152239310Sdim // If the arch does not support generic address space, then the arch 153239310Sdim // does not really support ADDRESS_SPACE_CONST, translate it to 154239310Sdim // to CONSTANT for better performance. 155239310Sdim if (Subtarget.hasGenericLdSt()) 156239310Sdim return NVPTX::PTXLdStInstCode::GLOBAL; 157239310Sdim else 158239310Sdim return NVPTX::PTXLdStInstCode::CONSTANT; 159249423Sdim default: 160249423Sdim break; 161239310Sdim } 162239310Sdim } 163239310Sdim return NVPTX::PTXLdStInstCode::LOCAL; 164239310Sdim} 165239310Sdim 166249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 167239310Sdim DebugLoc dl = N->getDebugLoc(); 168239310Sdim LoadSDNode *LD = cast<LoadSDNode>(N); 169239310Sdim EVT LoadedVT = LD->getMemoryVT(); 170249423Sdim SDNode *NVPTXLD = NULL; 171239310Sdim 172239310Sdim // do not support pre/post inc/dec 173239310Sdim if (LD->isIndexed()) 174239310Sdim return NULL; 175239310Sdim 176239310Sdim if (!LoadedVT.isSimple()) 177239310Sdim return NULL; 178239310Sdim 179239310Sdim // Address Space Setting 180239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget); 181239310Sdim 182239310Sdim // Volatile Setting 183239310Sdim // - .volatile is only availalble for .global and .shared 184239310Sdim bool isVolatile = LD->isVolatile(); 185239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 186239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 187239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 188239310Sdim isVolatile = false; 189239310Sdim 190239310Sdim // Vector Setting 191239310Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 192239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 193239310Sdim if (SimpleVT.isVector()) { 194239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 195239310Sdim if (num == 2) 196239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 197239310Sdim else if (num == 4) 198239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 199239310Sdim else 200239310Sdim return NULL; 201239310Sdim } 202239310Sdim 203239310Sdim // Type Setting: fromType + fromTypeWidth 204239310Sdim // 205239310Sdim // Sign : ISD::SEXTLOAD 206239310Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 207239310Sdim // type is integer 208239310Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 209239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 210249423Sdim unsigned fromTypeWidth = ScalarVT.getSizeInBits(); 211239310Sdim unsigned int fromType; 212239310Sdim if ((LD->getExtensionType() == ISD::SEXTLOAD)) 213239310Sdim fromType = NVPTX::PTXLdStInstCode::Signed; 214239310Sdim else if (ScalarVT.isFloatingPoint()) 215239310Sdim fromType = NVPTX::PTXLdStInstCode::Float; 216239310Sdim else 217239310Sdim fromType = NVPTX::PTXLdStInstCode::Unsigned; 218239310Sdim 219239310Sdim // Create the machine instruction DAG 220239310Sdim SDValue Chain = N->getOperand(0); 221239310Sdim SDValue N1 = N->getOperand(1); 222239310Sdim SDValue Addr; 223239310Sdim SDValue Offset, Base; 224239310Sdim unsigned Opcode; 225239310Sdim MVT::SimpleValueType TargetVT = LD->getValueType(0).getSimpleVT().SimpleTy; 226239310Sdim 227239310Sdim if (SelectDirectAddr(N1, Addr)) { 228239310Sdim switch (TargetVT) { 229249423Sdim case MVT::i8: 230249423Sdim Opcode = NVPTX::LD_i8_avar; 231249423Sdim break; 232249423Sdim case MVT::i16: 233249423Sdim Opcode = NVPTX::LD_i16_avar; 234249423Sdim break; 235249423Sdim case MVT::i32: 236249423Sdim Opcode = NVPTX::LD_i32_avar; 237249423Sdim break; 238249423Sdim case MVT::i64: 239249423Sdim Opcode = NVPTX::LD_i64_avar; 240249423Sdim break; 241249423Sdim case MVT::f32: 242249423Sdim Opcode = NVPTX::LD_f32_avar; 243249423Sdim break; 244249423Sdim case MVT::f64: 245249423Sdim Opcode = NVPTX::LD_f64_avar; 246249423Sdim break; 247249423Sdim default: 248249423Sdim return NULL; 249239310Sdim } 250249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 251249423Sdim getI32Imm(vecType), getI32Imm(fromType), 252249423Sdim getI32Imm(fromTypeWidth), Addr, Chain }; 253251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 254249423Sdim } else if (Subtarget.is64Bit() 255249423Sdim ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) 256249423Sdim : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { 257239310Sdim switch (TargetVT) { 258249423Sdim case MVT::i8: 259249423Sdim Opcode = NVPTX::LD_i8_asi; 260249423Sdim break; 261249423Sdim case MVT::i16: 262249423Sdim Opcode = NVPTX::LD_i16_asi; 263249423Sdim break; 264249423Sdim case MVT::i32: 265249423Sdim Opcode = NVPTX::LD_i32_asi; 266249423Sdim break; 267249423Sdim case MVT::i64: 268249423Sdim Opcode = NVPTX::LD_i64_asi; 269249423Sdim break; 270249423Sdim case MVT::f32: 271249423Sdim Opcode = NVPTX::LD_f32_asi; 272249423Sdim break; 273249423Sdim case MVT::f64: 274249423Sdim Opcode = NVPTX::LD_f64_asi; 275249423Sdim break; 276249423Sdim default: 277249423Sdim return NULL; 278239310Sdim } 279249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 280249423Sdim getI32Imm(vecType), getI32Imm(fromType), 281249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 282251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 283249423Sdim } else if (Subtarget.is64Bit() 284249423Sdim ? SelectADDRri64(N1.getNode(), N1, Base, Offset) 285249423Sdim : SelectADDRri(N1.getNode(), N1, Base, Offset)) { 286249423Sdim if (Subtarget.is64Bit()) { 287249423Sdim switch (TargetVT) { 288249423Sdim case MVT::i8: 289249423Sdim Opcode = NVPTX::LD_i8_ari_64; 290249423Sdim break; 291249423Sdim case MVT::i16: 292249423Sdim Opcode = NVPTX::LD_i16_ari_64; 293249423Sdim break; 294249423Sdim case MVT::i32: 295249423Sdim Opcode = NVPTX::LD_i32_ari_64; 296249423Sdim break; 297249423Sdim case MVT::i64: 298249423Sdim Opcode = NVPTX::LD_i64_ari_64; 299249423Sdim break; 300249423Sdim case MVT::f32: 301249423Sdim Opcode = NVPTX::LD_f32_ari_64; 302249423Sdim break; 303249423Sdim case MVT::f64: 304249423Sdim Opcode = NVPTX::LD_f64_ari_64; 305249423Sdim break; 306249423Sdim default: 307249423Sdim return NULL; 308249423Sdim } 309249423Sdim } else { 310249423Sdim switch (TargetVT) { 311249423Sdim case MVT::i8: 312249423Sdim Opcode = NVPTX::LD_i8_ari; 313249423Sdim break; 314249423Sdim case MVT::i16: 315249423Sdim Opcode = NVPTX::LD_i16_ari; 316249423Sdim break; 317249423Sdim case MVT::i32: 318249423Sdim Opcode = NVPTX::LD_i32_ari; 319249423Sdim break; 320249423Sdim case MVT::i64: 321249423Sdim Opcode = NVPTX::LD_i64_ari; 322249423Sdim break; 323249423Sdim case MVT::f32: 324249423Sdim Opcode = NVPTX::LD_f32_ari; 325249423Sdim break; 326249423Sdim case MVT::f64: 327249423Sdim Opcode = NVPTX::LD_f64_ari; 328249423Sdim break; 329249423Sdim default: 330249423Sdim return NULL; 331249423Sdim } 332239310Sdim } 333249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 334249423Sdim getI32Imm(vecType), getI32Imm(fromType), 335249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 336251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 337249423Sdim } else { 338249423Sdim if (Subtarget.is64Bit()) { 339249423Sdim switch (TargetVT) { 340249423Sdim case MVT::i8: 341249423Sdim Opcode = NVPTX::LD_i8_areg_64; 342249423Sdim break; 343249423Sdim case MVT::i16: 344249423Sdim Opcode = NVPTX::LD_i16_areg_64; 345249423Sdim break; 346249423Sdim case MVT::i32: 347249423Sdim Opcode = NVPTX::LD_i32_areg_64; 348249423Sdim break; 349249423Sdim case MVT::i64: 350249423Sdim Opcode = NVPTX::LD_i64_areg_64; 351249423Sdim break; 352249423Sdim case MVT::f32: 353249423Sdim Opcode = NVPTX::LD_f32_areg_64; 354249423Sdim break; 355249423Sdim case MVT::f64: 356249423Sdim Opcode = NVPTX::LD_f64_areg_64; 357249423Sdim break; 358249423Sdim default: 359249423Sdim return NULL; 360249423Sdim } 361249423Sdim } else { 362249423Sdim switch (TargetVT) { 363249423Sdim case MVT::i8: 364249423Sdim Opcode = NVPTX::LD_i8_areg; 365249423Sdim break; 366249423Sdim case MVT::i16: 367249423Sdim Opcode = NVPTX::LD_i16_areg; 368249423Sdim break; 369249423Sdim case MVT::i32: 370249423Sdim Opcode = NVPTX::LD_i32_areg; 371249423Sdim break; 372249423Sdim case MVT::i64: 373249423Sdim Opcode = NVPTX::LD_i64_areg; 374249423Sdim break; 375249423Sdim case MVT::f32: 376249423Sdim Opcode = NVPTX::LD_f32_areg; 377249423Sdim break; 378249423Sdim case MVT::f64: 379249423Sdim Opcode = NVPTX::LD_f64_areg; 380249423Sdim break; 381249423Sdim default: 382249423Sdim return NULL; 383249423Sdim } 384239310Sdim } 385249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 386249423Sdim getI32Imm(vecType), getI32Imm(fromType), 387249423Sdim getI32Imm(fromTypeWidth), N1, Chain }; 388251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 389239310Sdim } 390239310Sdim 391239310Sdim if (NVPTXLD != NULL) { 392239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 393239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 394239310Sdim cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 395239310Sdim } 396239310Sdim 397239310Sdim return NVPTXLD; 398239310Sdim} 399239310Sdim 400249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { 401249423Sdim 402249423Sdim SDValue Chain = N->getOperand(0); 403249423Sdim SDValue Op1 = N->getOperand(1); 404249423Sdim SDValue Addr, Offset, Base; 405249423Sdim unsigned Opcode; 406249423Sdim DebugLoc DL = N->getDebugLoc(); 407249423Sdim SDNode *LD; 408249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 409249423Sdim EVT LoadedVT = MemSD->getMemoryVT(); 410249423Sdim 411249423Sdim if (!LoadedVT.isSimple()) 412249423Sdim return NULL; 413249423Sdim 414249423Sdim // Address Space Setting 415249423Sdim unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 416249423Sdim 417249423Sdim // Volatile Setting 418249423Sdim // - .volatile is only availalble for .global and .shared 419249423Sdim bool IsVolatile = MemSD->isVolatile(); 420249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 421249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 422249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 423249423Sdim IsVolatile = false; 424249423Sdim 425249423Sdim // Vector Setting 426249423Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 427249423Sdim 428249423Sdim // Type Setting: fromType + fromTypeWidth 429249423Sdim // 430249423Sdim // Sign : ISD::SEXTLOAD 431249423Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 432249423Sdim // type is integer 433249423Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 434249423Sdim MVT ScalarVT = SimpleVT.getScalarType(); 435249423Sdim unsigned FromTypeWidth = ScalarVT.getSizeInBits(); 436249423Sdim unsigned int FromType; 437249423Sdim // The last operand holds the original LoadSDNode::getExtensionType() value 438249423Sdim unsigned ExtensionType = cast<ConstantSDNode>( 439249423Sdim N->getOperand(N->getNumOperands() - 1))->getZExtValue(); 440249423Sdim if (ExtensionType == ISD::SEXTLOAD) 441249423Sdim FromType = NVPTX::PTXLdStInstCode::Signed; 442249423Sdim else if (ScalarVT.isFloatingPoint()) 443249423Sdim FromType = NVPTX::PTXLdStInstCode::Float; 444249423Sdim else 445249423Sdim FromType = NVPTX::PTXLdStInstCode::Unsigned; 446249423Sdim 447249423Sdim unsigned VecType; 448249423Sdim 449249423Sdim switch (N->getOpcode()) { 450249423Sdim case NVPTXISD::LoadV2: 451249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 452249423Sdim break; 453249423Sdim case NVPTXISD::LoadV4: 454249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 455249423Sdim break; 456249423Sdim default: 457249423Sdim return NULL; 458249423Sdim } 459249423Sdim 460249423Sdim EVT EltVT = N->getValueType(0); 461249423Sdim 462249423Sdim if (SelectDirectAddr(Op1, Addr)) { 463249423Sdim switch (N->getOpcode()) { 464249423Sdim default: 465249423Sdim return NULL; 466249423Sdim case NVPTXISD::LoadV2: 467249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 468249423Sdim default: 469249423Sdim return NULL; 470249423Sdim case MVT::i8: 471249423Sdim Opcode = NVPTX::LDV_i8_v2_avar; 472249423Sdim break; 473249423Sdim case MVT::i16: 474249423Sdim Opcode = NVPTX::LDV_i16_v2_avar; 475249423Sdim break; 476249423Sdim case MVT::i32: 477249423Sdim Opcode = NVPTX::LDV_i32_v2_avar; 478249423Sdim break; 479249423Sdim case MVT::i64: 480249423Sdim Opcode = NVPTX::LDV_i64_v2_avar; 481249423Sdim break; 482249423Sdim case MVT::f32: 483249423Sdim Opcode = NVPTX::LDV_f32_v2_avar; 484249423Sdim break; 485249423Sdim case MVT::f64: 486249423Sdim Opcode = NVPTX::LDV_f64_v2_avar; 487249423Sdim break; 488249423Sdim } 489249423Sdim break; 490249423Sdim case NVPTXISD::LoadV4: 491249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 492249423Sdim default: 493249423Sdim return NULL; 494249423Sdim case MVT::i8: 495249423Sdim Opcode = NVPTX::LDV_i8_v4_avar; 496249423Sdim break; 497249423Sdim case MVT::i16: 498249423Sdim Opcode = NVPTX::LDV_i16_v4_avar; 499249423Sdim break; 500249423Sdim case MVT::i32: 501249423Sdim Opcode = NVPTX::LDV_i32_v4_avar; 502249423Sdim break; 503249423Sdim case MVT::f32: 504249423Sdim Opcode = NVPTX::LDV_f32_v4_avar; 505249423Sdim break; 506249423Sdim } 507249423Sdim break; 508249423Sdim } 509249423Sdim 510249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 511249423Sdim getI32Imm(VecType), getI32Imm(FromType), 512249423Sdim getI32Imm(FromTypeWidth), Addr, Chain }; 513251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 514249423Sdim } else if (Subtarget.is64Bit() 515249423Sdim ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 516249423Sdim : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 517249423Sdim switch (N->getOpcode()) { 518249423Sdim default: 519249423Sdim return NULL; 520249423Sdim case NVPTXISD::LoadV2: 521249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 522249423Sdim default: 523249423Sdim return NULL; 524249423Sdim case MVT::i8: 525249423Sdim Opcode = NVPTX::LDV_i8_v2_asi; 526249423Sdim break; 527249423Sdim case MVT::i16: 528249423Sdim Opcode = NVPTX::LDV_i16_v2_asi; 529249423Sdim break; 530249423Sdim case MVT::i32: 531249423Sdim Opcode = NVPTX::LDV_i32_v2_asi; 532249423Sdim break; 533249423Sdim case MVT::i64: 534249423Sdim Opcode = NVPTX::LDV_i64_v2_asi; 535249423Sdim break; 536249423Sdim case MVT::f32: 537249423Sdim Opcode = NVPTX::LDV_f32_v2_asi; 538249423Sdim break; 539249423Sdim case MVT::f64: 540249423Sdim Opcode = NVPTX::LDV_f64_v2_asi; 541249423Sdim break; 542249423Sdim } 543249423Sdim break; 544249423Sdim case NVPTXISD::LoadV4: 545249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 546249423Sdim default: 547249423Sdim return NULL; 548249423Sdim case MVT::i8: 549249423Sdim Opcode = NVPTX::LDV_i8_v4_asi; 550249423Sdim break; 551249423Sdim case MVT::i16: 552249423Sdim Opcode = NVPTX::LDV_i16_v4_asi; 553249423Sdim break; 554249423Sdim case MVT::i32: 555249423Sdim Opcode = NVPTX::LDV_i32_v4_asi; 556249423Sdim break; 557249423Sdim case MVT::f32: 558249423Sdim Opcode = NVPTX::LDV_f32_v4_asi; 559249423Sdim break; 560249423Sdim } 561249423Sdim break; 562249423Sdim } 563249423Sdim 564249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 565249423Sdim getI32Imm(VecType), getI32Imm(FromType), 566249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 567251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 568249423Sdim } else if (Subtarget.is64Bit() 569249423Sdim ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 570249423Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 571249423Sdim if (Subtarget.is64Bit()) { 572249423Sdim switch (N->getOpcode()) { 573249423Sdim default: 574249423Sdim return NULL; 575249423Sdim case NVPTXISD::LoadV2: 576249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 577249423Sdim default: 578249423Sdim return NULL; 579249423Sdim case MVT::i8: 580249423Sdim Opcode = NVPTX::LDV_i8_v2_ari_64; 581249423Sdim break; 582249423Sdim case MVT::i16: 583249423Sdim Opcode = NVPTX::LDV_i16_v2_ari_64; 584249423Sdim break; 585249423Sdim case MVT::i32: 586249423Sdim Opcode = NVPTX::LDV_i32_v2_ari_64; 587249423Sdim break; 588249423Sdim case MVT::i64: 589249423Sdim Opcode = NVPTX::LDV_i64_v2_ari_64; 590249423Sdim break; 591249423Sdim case MVT::f32: 592249423Sdim Opcode = NVPTX::LDV_f32_v2_ari_64; 593249423Sdim break; 594249423Sdim case MVT::f64: 595249423Sdim Opcode = NVPTX::LDV_f64_v2_ari_64; 596249423Sdim break; 597249423Sdim } 598249423Sdim break; 599249423Sdim case NVPTXISD::LoadV4: 600249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 601249423Sdim default: 602249423Sdim return NULL; 603249423Sdim case MVT::i8: 604249423Sdim Opcode = NVPTX::LDV_i8_v4_ari_64; 605249423Sdim break; 606249423Sdim case MVT::i16: 607249423Sdim Opcode = NVPTX::LDV_i16_v4_ari_64; 608249423Sdim break; 609249423Sdim case MVT::i32: 610249423Sdim Opcode = NVPTX::LDV_i32_v4_ari_64; 611249423Sdim break; 612249423Sdim case MVT::f32: 613249423Sdim Opcode = NVPTX::LDV_f32_v4_ari_64; 614249423Sdim break; 615249423Sdim } 616249423Sdim break; 617249423Sdim } 618249423Sdim } else { 619249423Sdim switch (N->getOpcode()) { 620249423Sdim default: 621249423Sdim return NULL; 622249423Sdim case NVPTXISD::LoadV2: 623249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 624249423Sdim default: 625249423Sdim return NULL; 626249423Sdim case MVT::i8: 627249423Sdim Opcode = NVPTX::LDV_i8_v2_ari; 628249423Sdim break; 629249423Sdim case MVT::i16: 630249423Sdim Opcode = NVPTX::LDV_i16_v2_ari; 631249423Sdim break; 632249423Sdim case MVT::i32: 633249423Sdim Opcode = NVPTX::LDV_i32_v2_ari; 634249423Sdim break; 635249423Sdim case MVT::i64: 636249423Sdim Opcode = NVPTX::LDV_i64_v2_ari; 637249423Sdim break; 638249423Sdim case MVT::f32: 639249423Sdim Opcode = NVPTX::LDV_f32_v2_ari; 640249423Sdim break; 641249423Sdim case MVT::f64: 642249423Sdim Opcode = NVPTX::LDV_f64_v2_ari; 643249423Sdim break; 644249423Sdim } 645249423Sdim break; 646249423Sdim case NVPTXISD::LoadV4: 647249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 648249423Sdim default: 649249423Sdim return NULL; 650249423Sdim case MVT::i8: 651249423Sdim Opcode = NVPTX::LDV_i8_v4_ari; 652249423Sdim break; 653249423Sdim case MVT::i16: 654249423Sdim Opcode = NVPTX::LDV_i16_v4_ari; 655249423Sdim break; 656249423Sdim case MVT::i32: 657249423Sdim Opcode = NVPTX::LDV_i32_v4_ari; 658249423Sdim break; 659249423Sdim case MVT::f32: 660249423Sdim Opcode = NVPTX::LDV_f32_v4_ari; 661249423Sdim break; 662249423Sdim } 663249423Sdim break; 664249423Sdim } 665249423Sdim } 666249423Sdim 667249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 668249423Sdim getI32Imm(VecType), getI32Imm(FromType), 669249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 670249423Sdim 671251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 672249423Sdim } else { 673249423Sdim if (Subtarget.is64Bit()) { 674249423Sdim switch (N->getOpcode()) { 675249423Sdim default: 676249423Sdim return NULL; 677249423Sdim case NVPTXISD::LoadV2: 678249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 679249423Sdim default: 680249423Sdim return NULL; 681249423Sdim case MVT::i8: 682249423Sdim Opcode = NVPTX::LDV_i8_v2_areg_64; 683249423Sdim break; 684249423Sdim case MVT::i16: 685249423Sdim Opcode = NVPTX::LDV_i16_v2_areg_64; 686249423Sdim break; 687249423Sdim case MVT::i32: 688249423Sdim Opcode = NVPTX::LDV_i32_v2_areg_64; 689249423Sdim break; 690249423Sdim case MVT::i64: 691249423Sdim Opcode = NVPTX::LDV_i64_v2_areg_64; 692249423Sdim break; 693249423Sdim case MVT::f32: 694249423Sdim Opcode = NVPTX::LDV_f32_v2_areg_64; 695249423Sdim break; 696249423Sdim case MVT::f64: 697249423Sdim Opcode = NVPTX::LDV_f64_v2_areg_64; 698249423Sdim break; 699249423Sdim } 700249423Sdim break; 701249423Sdim case NVPTXISD::LoadV4: 702249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 703249423Sdim default: 704249423Sdim return NULL; 705249423Sdim case MVT::i8: 706249423Sdim Opcode = NVPTX::LDV_i8_v4_areg_64; 707249423Sdim break; 708249423Sdim case MVT::i16: 709249423Sdim Opcode = NVPTX::LDV_i16_v4_areg_64; 710249423Sdim break; 711249423Sdim case MVT::i32: 712249423Sdim Opcode = NVPTX::LDV_i32_v4_areg_64; 713249423Sdim break; 714249423Sdim case MVT::f32: 715249423Sdim Opcode = NVPTX::LDV_f32_v4_areg_64; 716249423Sdim break; 717249423Sdim } 718249423Sdim break; 719249423Sdim } 720249423Sdim } else { 721249423Sdim switch (N->getOpcode()) { 722249423Sdim default: 723249423Sdim return NULL; 724249423Sdim case NVPTXISD::LoadV2: 725249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 726249423Sdim default: 727249423Sdim return NULL; 728249423Sdim case MVT::i8: 729249423Sdim Opcode = NVPTX::LDV_i8_v2_areg; 730249423Sdim break; 731249423Sdim case MVT::i16: 732249423Sdim Opcode = NVPTX::LDV_i16_v2_areg; 733249423Sdim break; 734249423Sdim case MVT::i32: 735249423Sdim Opcode = NVPTX::LDV_i32_v2_areg; 736249423Sdim break; 737249423Sdim case MVT::i64: 738249423Sdim Opcode = NVPTX::LDV_i64_v2_areg; 739249423Sdim break; 740249423Sdim case MVT::f32: 741249423Sdim Opcode = NVPTX::LDV_f32_v2_areg; 742249423Sdim break; 743249423Sdim case MVT::f64: 744249423Sdim Opcode = NVPTX::LDV_f64_v2_areg; 745249423Sdim break; 746249423Sdim } 747249423Sdim break; 748249423Sdim case NVPTXISD::LoadV4: 749249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 750249423Sdim default: 751249423Sdim return NULL; 752249423Sdim case MVT::i8: 753249423Sdim Opcode = NVPTX::LDV_i8_v4_areg; 754249423Sdim break; 755249423Sdim case MVT::i16: 756249423Sdim Opcode = NVPTX::LDV_i16_v4_areg; 757249423Sdim break; 758249423Sdim case MVT::i32: 759249423Sdim Opcode = NVPTX::LDV_i32_v4_areg; 760249423Sdim break; 761249423Sdim case MVT::f32: 762249423Sdim Opcode = NVPTX::LDV_f32_v4_areg; 763249423Sdim break; 764249423Sdim } 765249423Sdim break; 766249423Sdim } 767249423Sdim } 768249423Sdim 769249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 770249423Sdim getI32Imm(VecType), getI32Imm(FromType), 771249423Sdim getI32Imm(FromTypeWidth), Op1, Chain }; 772251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 773249423Sdim } 774249423Sdim 775249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 776249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 777249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 778249423Sdim 779249423Sdim return LD; 780249423Sdim} 781249423Sdim 782249423SdimSDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) { 783249423Sdim 784249423Sdim SDValue Chain = N->getOperand(0); 785249423Sdim SDValue Op1 = N->getOperand(1); 786249423Sdim unsigned Opcode; 787249423Sdim DebugLoc DL = N->getDebugLoc(); 788249423Sdim SDNode *LD; 789249423Sdim 790249423Sdim EVT RetVT = N->getValueType(0); 791249423Sdim 792249423Sdim // Select opcode 793249423Sdim if (Subtarget.is64Bit()) { 794249423Sdim switch (N->getOpcode()) { 795249423Sdim default: 796249423Sdim return NULL; 797249423Sdim case NVPTXISD::LDGV2: 798249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 799249423Sdim default: 800249423Sdim return NULL; 801249423Sdim case MVT::i8: 802249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_64; 803249423Sdim break; 804249423Sdim case MVT::i16: 805249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_64; 806249423Sdim break; 807249423Sdim case MVT::i32: 808249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_64; 809249423Sdim break; 810249423Sdim case MVT::i64: 811249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_64; 812249423Sdim break; 813249423Sdim case MVT::f32: 814249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_64; 815249423Sdim break; 816249423Sdim case MVT::f64: 817249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_64; 818249423Sdim break; 819249423Sdim } 820249423Sdim break; 821249423Sdim case NVPTXISD::LDGV4: 822249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 823249423Sdim default: 824249423Sdim return NULL; 825249423Sdim case MVT::i8: 826249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_64; 827249423Sdim break; 828249423Sdim case MVT::i16: 829249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_64; 830249423Sdim break; 831249423Sdim case MVT::i32: 832249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_64; 833249423Sdim break; 834249423Sdim case MVT::f32: 835249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_64; 836249423Sdim break; 837249423Sdim } 838249423Sdim break; 839249423Sdim case NVPTXISD::LDUV2: 840249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 841249423Sdim default: 842249423Sdim return NULL; 843249423Sdim case MVT::i8: 844249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_64; 845249423Sdim break; 846249423Sdim case MVT::i16: 847249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_64; 848249423Sdim break; 849249423Sdim case MVT::i32: 850249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_64; 851249423Sdim break; 852249423Sdim case MVT::i64: 853249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_64; 854249423Sdim break; 855249423Sdim case MVT::f32: 856249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_64; 857249423Sdim break; 858249423Sdim case MVT::f64: 859249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_64; 860249423Sdim break; 861249423Sdim } 862249423Sdim break; 863249423Sdim case NVPTXISD::LDUV4: 864249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 865249423Sdim default: 866249423Sdim return NULL; 867249423Sdim case MVT::i8: 868249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_64; 869249423Sdim break; 870249423Sdim case MVT::i16: 871249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_64; 872249423Sdim break; 873249423Sdim case MVT::i32: 874249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_64; 875249423Sdim break; 876249423Sdim case MVT::f32: 877249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_64; 878249423Sdim break; 879249423Sdim } 880249423Sdim break; 881249423Sdim } 882249423Sdim } else { 883249423Sdim switch (N->getOpcode()) { 884249423Sdim default: 885249423Sdim return NULL; 886249423Sdim case NVPTXISD::LDGV2: 887249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 888249423Sdim default: 889249423Sdim return NULL; 890249423Sdim case MVT::i8: 891249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_32; 892249423Sdim break; 893249423Sdim case MVT::i16: 894249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_32; 895249423Sdim break; 896249423Sdim case MVT::i32: 897249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_32; 898249423Sdim break; 899249423Sdim case MVT::i64: 900249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_32; 901249423Sdim break; 902249423Sdim case MVT::f32: 903249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_32; 904249423Sdim break; 905249423Sdim case MVT::f64: 906249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_32; 907249423Sdim break; 908249423Sdim } 909249423Sdim break; 910249423Sdim case NVPTXISD::LDGV4: 911249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 912249423Sdim default: 913249423Sdim return NULL; 914249423Sdim case MVT::i8: 915249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_32; 916249423Sdim break; 917249423Sdim case MVT::i16: 918249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_32; 919249423Sdim break; 920249423Sdim case MVT::i32: 921249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_32; 922249423Sdim break; 923249423Sdim case MVT::f32: 924249423Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_32; 925249423Sdim break; 926249423Sdim } 927249423Sdim break; 928249423Sdim case NVPTXISD::LDUV2: 929249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 930249423Sdim default: 931249423Sdim return NULL; 932249423Sdim case MVT::i8: 933249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_32; 934249423Sdim break; 935249423Sdim case MVT::i16: 936249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_32; 937249423Sdim break; 938249423Sdim case MVT::i32: 939249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_32; 940249423Sdim break; 941249423Sdim case MVT::i64: 942249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_32; 943249423Sdim break; 944249423Sdim case MVT::f32: 945249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_32; 946249423Sdim break; 947249423Sdim case MVT::f64: 948249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_32; 949249423Sdim break; 950249423Sdim } 951249423Sdim break; 952249423Sdim case NVPTXISD::LDUV4: 953249423Sdim switch (RetVT.getSimpleVT().SimpleTy) { 954249423Sdim default: 955249423Sdim return NULL; 956249423Sdim case MVT::i8: 957249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_32; 958249423Sdim break; 959249423Sdim case MVT::i16: 960249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_32; 961249423Sdim break; 962249423Sdim case MVT::i32: 963249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_32; 964249423Sdim break; 965249423Sdim case MVT::f32: 966249423Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_32; 967249423Sdim break; 968249423Sdim } 969249423Sdim break; 970249423Sdim } 971249423Sdim } 972249423Sdim 973249423Sdim SDValue Ops[] = { Op1, Chain }; 974251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 975249423Sdim 976249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 977249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 978249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 979249423Sdim 980249423Sdim return LD; 981249423Sdim} 982249423Sdim 983249423SdimSDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 984239310Sdim DebugLoc dl = N->getDebugLoc(); 985239310Sdim StoreSDNode *ST = cast<StoreSDNode>(N); 986239310Sdim EVT StoreVT = ST->getMemoryVT(); 987239310Sdim SDNode *NVPTXST = NULL; 988239310Sdim 989239310Sdim // do not support pre/post inc/dec 990239310Sdim if (ST->isIndexed()) 991239310Sdim return NULL; 992239310Sdim 993239310Sdim if (!StoreVT.isSimple()) 994239310Sdim return NULL; 995239310Sdim 996239310Sdim // Address Space Setting 997239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget); 998239310Sdim 999239310Sdim // Volatile Setting 1000239310Sdim // - .volatile is only availalble for .global and .shared 1001239310Sdim bool isVolatile = ST->isVolatile(); 1002239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1003239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1004239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1005239310Sdim isVolatile = false; 1006239310Sdim 1007239310Sdim // Vector Setting 1008239310Sdim MVT SimpleVT = StoreVT.getSimpleVT(); 1009239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 1010239310Sdim if (SimpleVT.isVector()) { 1011239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 1012239310Sdim if (num == 2) 1013239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 1014239310Sdim else if (num == 4) 1015239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 1016239310Sdim else 1017239310Sdim return NULL; 1018239310Sdim } 1019239310Sdim 1020239310Sdim // Type Setting: toType + toTypeWidth 1021239310Sdim // - for integer type, always use 'u' 1022239310Sdim // 1023239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 1024249423Sdim unsigned toTypeWidth = ScalarVT.getSizeInBits(); 1025239310Sdim unsigned int toType; 1026239310Sdim if (ScalarVT.isFloatingPoint()) 1027239310Sdim toType = NVPTX::PTXLdStInstCode::Float; 1028239310Sdim else 1029239310Sdim toType = NVPTX::PTXLdStInstCode::Unsigned; 1030239310Sdim 1031239310Sdim // Create the machine instruction DAG 1032239310Sdim SDValue Chain = N->getOperand(0); 1033239310Sdim SDValue N1 = N->getOperand(1); 1034239310Sdim SDValue N2 = N->getOperand(2); 1035239310Sdim SDValue Addr; 1036239310Sdim SDValue Offset, Base; 1037239310Sdim unsigned Opcode; 1038239310Sdim MVT::SimpleValueType SourceVT = 1039239310Sdim N1.getNode()->getValueType(0).getSimpleVT().SimpleTy; 1040239310Sdim 1041239310Sdim if (SelectDirectAddr(N2, Addr)) { 1042239310Sdim switch (SourceVT) { 1043249423Sdim case MVT::i8: 1044249423Sdim Opcode = NVPTX::ST_i8_avar; 1045249423Sdim break; 1046249423Sdim case MVT::i16: 1047249423Sdim Opcode = NVPTX::ST_i16_avar; 1048249423Sdim break; 1049249423Sdim case MVT::i32: 1050249423Sdim Opcode = NVPTX::ST_i32_avar; 1051249423Sdim break; 1052249423Sdim case MVT::i64: 1053249423Sdim Opcode = NVPTX::ST_i64_avar; 1054249423Sdim break; 1055249423Sdim case MVT::f32: 1056249423Sdim Opcode = NVPTX::ST_f32_avar; 1057249423Sdim break; 1058249423Sdim case MVT::f64: 1059249423Sdim Opcode = NVPTX::ST_f64_avar; 1060249423Sdim break; 1061249423Sdim default: 1062249423Sdim return NULL; 1063239310Sdim } 1064249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1065249423Sdim getI32Imm(vecType), getI32Imm(toType), 1066249423Sdim getI32Imm(toTypeWidth), Addr, Chain }; 1067251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1068249423Sdim } else if (Subtarget.is64Bit() 1069249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1070249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1071239310Sdim switch (SourceVT) { 1072249423Sdim case MVT::i8: 1073249423Sdim Opcode = NVPTX::ST_i8_asi; 1074249423Sdim break; 1075249423Sdim case MVT::i16: 1076249423Sdim Opcode = NVPTX::ST_i16_asi; 1077249423Sdim break; 1078249423Sdim case MVT::i32: 1079249423Sdim Opcode = NVPTX::ST_i32_asi; 1080249423Sdim break; 1081249423Sdim case MVT::i64: 1082249423Sdim Opcode = NVPTX::ST_i64_asi; 1083249423Sdim break; 1084249423Sdim case MVT::f32: 1085249423Sdim Opcode = NVPTX::ST_f32_asi; 1086249423Sdim break; 1087249423Sdim case MVT::f64: 1088249423Sdim Opcode = NVPTX::ST_f64_asi; 1089249423Sdim break; 1090249423Sdim default: 1091249423Sdim return NULL; 1092239310Sdim } 1093249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1094249423Sdim getI32Imm(vecType), getI32Imm(toType), 1095249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 1096251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1097249423Sdim } else if (Subtarget.is64Bit() 1098249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1099249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1100249423Sdim if (Subtarget.is64Bit()) { 1101249423Sdim switch (SourceVT) { 1102249423Sdim case MVT::i8: 1103249423Sdim Opcode = NVPTX::ST_i8_ari_64; 1104249423Sdim break; 1105249423Sdim case MVT::i16: 1106249423Sdim Opcode = NVPTX::ST_i16_ari_64; 1107249423Sdim break; 1108249423Sdim case MVT::i32: 1109249423Sdim Opcode = NVPTX::ST_i32_ari_64; 1110249423Sdim break; 1111249423Sdim case MVT::i64: 1112249423Sdim Opcode = NVPTX::ST_i64_ari_64; 1113249423Sdim break; 1114249423Sdim case MVT::f32: 1115249423Sdim Opcode = NVPTX::ST_f32_ari_64; 1116249423Sdim break; 1117249423Sdim case MVT::f64: 1118249423Sdim Opcode = NVPTX::ST_f64_ari_64; 1119249423Sdim break; 1120249423Sdim default: 1121249423Sdim return NULL; 1122249423Sdim } 1123249423Sdim } else { 1124249423Sdim switch (SourceVT) { 1125249423Sdim case MVT::i8: 1126249423Sdim Opcode = NVPTX::ST_i8_ari; 1127249423Sdim break; 1128249423Sdim case MVT::i16: 1129249423Sdim Opcode = NVPTX::ST_i16_ari; 1130249423Sdim break; 1131249423Sdim case MVT::i32: 1132249423Sdim Opcode = NVPTX::ST_i32_ari; 1133249423Sdim break; 1134249423Sdim case MVT::i64: 1135249423Sdim Opcode = NVPTX::ST_i64_ari; 1136249423Sdim break; 1137249423Sdim case MVT::f32: 1138249423Sdim Opcode = NVPTX::ST_f32_ari; 1139249423Sdim break; 1140249423Sdim case MVT::f64: 1141249423Sdim Opcode = NVPTX::ST_f64_ari; 1142249423Sdim break; 1143249423Sdim default: 1144249423Sdim return NULL; 1145249423Sdim } 1146239310Sdim } 1147249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1148249423Sdim getI32Imm(vecType), getI32Imm(toType), 1149249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 1150251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1151239310Sdim } else { 1152249423Sdim if (Subtarget.is64Bit()) { 1153249423Sdim switch (SourceVT) { 1154249423Sdim case MVT::i8: 1155249423Sdim Opcode = NVPTX::ST_i8_areg_64; 1156249423Sdim break; 1157249423Sdim case MVT::i16: 1158249423Sdim Opcode = NVPTX::ST_i16_areg_64; 1159249423Sdim break; 1160249423Sdim case MVT::i32: 1161249423Sdim Opcode = NVPTX::ST_i32_areg_64; 1162249423Sdim break; 1163249423Sdim case MVT::i64: 1164249423Sdim Opcode = NVPTX::ST_i64_areg_64; 1165249423Sdim break; 1166249423Sdim case MVT::f32: 1167249423Sdim Opcode = NVPTX::ST_f32_areg_64; 1168249423Sdim break; 1169249423Sdim case MVT::f64: 1170249423Sdim Opcode = NVPTX::ST_f64_areg_64; 1171249423Sdim break; 1172249423Sdim default: 1173249423Sdim return NULL; 1174249423Sdim } 1175249423Sdim } else { 1176249423Sdim switch (SourceVT) { 1177249423Sdim case MVT::i8: 1178249423Sdim Opcode = NVPTX::ST_i8_areg; 1179249423Sdim break; 1180249423Sdim case MVT::i16: 1181249423Sdim Opcode = NVPTX::ST_i16_areg; 1182249423Sdim break; 1183249423Sdim case MVT::i32: 1184249423Sdim Opcode = NVPTX::ST_i32_areg; 1185249423Sdim break; 1186249423Sdim case MVT::i64: 1187249423Sdim Opcode = NVPTX::ST_i64_areg; 1188249423Sdim break; 1189249423Sdim case MVT::f32: 1190249423Sdim Opcode = NVPTX::ST_f32_areg; 1191249423Sdim break; 1192249423Sdim case MVT::f64: 1193249423Sdim Opcode = NVPTX::ST_f64_areg; 1194249423Sdim break; 1195249423Sdim default: 1196249423Sdim return NULL; 1197249423Sdim } 1198239310Sdim } 1199249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1200249423Sdim getI32Imm(vecType), getI32Imm(toType), 1201249423Sdim getI32Imm(toTypeWidth), N2, Chain }; 1202251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1203239310Sdim } 1204239310Sdim 1205239310Sdim if (NVPTXST != NULL) { 1206239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1207239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1208239310Sdim cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1209239310Sdim } 1210239310Sdim 1211239310Sdim return NVPTXST; 1212239310Sdim} 1213239310Sdim 1214249423SdimSDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { 1215249423Sdim SDValue Chain = N->getOperand(0); 1216249423Sdim SDValue Op1 = N->getOperand(1); 1217249423Sdim SDValue Addr, Offset, Base; 1218249423Sdim unsigned Opcode; 1219249423Sdim DebugLoc DL = N->getDebugLoc(); 1220249423Sdim SDNode *ST; 1221249423Sdim EVT EltVT = Op1.getValueType(); 1222249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 1223249423Sdim EVT StoreVT = MemSD->getMemoryVT(); 1224249423Sdim 1225249423Sdim // Address Space Setting 1226249423Sdim unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 1227249423Sdim 1228249423Sdim if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { 1229249423Sdim report_fatal_error("Cannot store to pointer that points to constant " 1230249423Sdim "memory space"); 1231249423Sdim } 1232249423Sdim 1233249423Sdim // Volatile Setting 1234249423Sdim // - .volatile is only availalble for .global and .shared 1235249423Sdim bool IsVolatile = MemSD->isVolatile(); 1236249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1237249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1238249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1239249423Sdim IsVolatile = false; 1240249423Sdim 1241249423Sdim // Type Setting: toType + toTypeWidth 1242249423Sdim // - for integer type, always use 'u' 1243249423Sdim assert(StoreVT.isSimple() && "Store value is not simple"); 1244249423Sdim MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); 1245249423Sdim unsigned ToTypeWidth = ScalarVT.getSizeInBits(); 1246249423Sdim unsigned ToType; 1247249423Sdim if (ScalarVT.isFloatingPoint()) 1248249423Sdim ToType = NVPTX::PTXLdStInstCode::Float; 1249249423Sdim else 1250249423Sdim ToType = NVPTX::PTXLdStInstCode::Unsigned; 1251249423Sdim 1252249423Sdim SmallVector<SDValue, 12> StOps; 1253249423Sdim SDValue N2; 1254249423Sdim unsigned VecType; 1255249423Sdim 1256249423Sdim switch (N->getOpcode()) { 1257249423Sdim case NVPTXISD::StoreV2: 1258249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 1259249423Sdim StOps.push_back(N->getOperand(1)); 1260249423Sdim StOps.push_back(N->getOperand(2)); 1261249423Sdim N2 = N->getOperand(3); 1262249423Sdim break; 1263249423Sdim case NVPTXISD::StoreV4: 1264249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 1265249423Sdim StOps.push_back(N->getOperand(1)); 1266249423Sdim StOps.push_back(N->getOperand(2)); 1267249423Sdim StOps.push_back(N->getOperand(3)); 1268249423Sdim StOps.push_back(N->getOperand(4)); 1269249423Sdim N2 = N->getOperand(5); 1270249423Sdim break; 1271249423Sdim default: 1272249423Sdim return NULL; 1273249423Sdim } 1274249423Sdim 1275249423Sdim StOps.push_back(getI32Imm(IsVolatile)); 1276249423Sdim StOps.push_back(getI32Imm(CodeAddrSpace)); 1277249423Sdim StOps.push_back(getI32Imm(VecType)); 1278249423Sdim StOps.push_back(getI32Imm(ToType)); 1279249423Sdim StOps.push_back(getI32Imm(ToTypeWidth)); 1280249423Sdim 1281249423Sdim if (SelectDirectAddr(N2, Addr)) { 1282249423Sdim switch (N->getOpcode()) { 1283249423Sdim default: 1284249423Sdim return NULL; 1285249423Sdim case NVPTXISD::StoreV2: 1286249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1287249423Sdim default: 1288249423Sdim return NULL; 1289249423Sdim case MVT::i8: 1290249423Sdim Opcode = NVPTX::STV_i8_v2_avar; 1291249423Sdim break; 1292249423Sdim case MVT::i16: 1293249423Sdim Opcode = NVPTX::STV_i16_v2_avar; 1294249423Sdim break; 1295249423Sdim case MVT::i32: 1296249423Sdim Opcode = NVPTX::STV_i32_v2_avar; 1297249423Sdim break; 1298249423Sdim case MVT::i64: 1299249423Sdim Opcode = NVPTX::STV_i64_v2_avar; 1300249423Sdim break; 1301249423Sdim case MVT::f32: 1302249423Sdim Opcode = NVPTX::STV_f32_v2_avar; 1303249423Sdim break; 1304249423Sdim case MVT::f64: 1305249423Sdim Opcode = NVPTX::STV_f64_v2_avar; 1306249423Sdim break; 1307249423Sdim } 1308249423Sdim break; 1309249423Sdim case NVPTXISD::StoreV4: 1310249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1311249423Sdim default: 1312249423Sdim return NULL; 1313249423Sdim case MVT::i8: 1314249423Sdim Opcode = NVPTX::STV_i8_v4_avar; 1315249423Sdim break; 1316249423Sdim case MVT::i16: 1317249423Sdim Opcode = NVPTX::STV_i16_v4_avar; 1318249423Sdim break; 1319249423Sdim case MVT::i32: 1320249423Sdim Opcode = NVPTX::STV_i32_v4_avar; 1321249423Sdim break; 1322249423Sdim case MVT::f32: 1323249423Sdim Opcode = NVPTX::STV_f32_v4_avar; 1324249423Sdim break; 1325249423Sdim } 1326249423Sdim break; 1327249423Sdim } 1328249423Sdim StOps.push_back(Addr); 1329249423Sdim } else if (Subtarget.is64Bit() 1330249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1331249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1332249423Sdim switch (N->getOpcode()) { 1333249423Sdim default: 1334249423Sdim return NULL; 1335249423Sdim case NVPTXISD::StoreV2: 1336249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1337249423Sdim default: 1338249423Sdim return NULL; 1339249423Sdim case MVT::i8: 1340249423Sdim Opcode = NVPTX::STV_i8_v2_asi; 1341249423Sdim break; 1342249423Sdim case MVT::i16: 1343249423Sdim Opcode = NVPTX::STV_i16_v2_asi; 1344249423Sdim break; 1345249423Sdim case MVT::i32: 1346249423Sdim Opcode = NVPTX::STV_i32_v2_asi; 1347249423Sdim break; 1348249423Sdim case MVT::i64: 1349249423Sdim Opcode = NVPTX::STV_i64_v2_asi; 1350249423Sdim break; 1351249423Sdim case MVT::f32: 1352249423Sdim Opcode = NVPTX::STV_f32_v2_asi; 1353249423Sdim break; 1354249423Sdim case MVT::f64: 1355249423Sdim Opcode = NVPTX::STV_f64_v2_asi; 1356249423Sdim break; 1357249423Sdim } 1358249423Sdim break; 1359249423Sdim case NVPTXISD::StoreV4: 1360249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1361249423Sdim default: 1362249423Sdim return NULL; 1363249423Sdim case MVT::i8: 1364249423Sdim Opcode = NVPTX::STV_i8_v4_asi; 1365249423Sdim break; 1366249423Sdim case MVT::i16: 1367249423Sdim Opcode = NVPTX::STV_i16_v4_asi; 1368249423Sdim break; 1369249423Sdim case MVT::i32: 1370249423Sdim Opcode = NVPTX::STV_i32_v4_asi; 1371249423Sdim break; 1372249423Sdim case MVT::f32: 1373249423Sdim Opcode = NVPTX::STV_f32_v4_asi; 1374249423Sdim break; 1375249423Sdim } 1376249423Sdim break; 1377249423Sdim } 1378249423Sdim StOps.push_back(Base); 1379249423Sdim StOps.push_back(Offset); 1380249423Sdim } else if (Subtarget.is64Bit() 1381249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1382249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1383249423Sdim if (Subtarget.is64Bit()) { 1384249423Sdim switch (N->getOpcode()) { 1385249423Sdim default: 1386249423Sdim return NULL; 1387249423Sdim case NVPTXISD::StoreV2: 1388249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1389249423Sdim default: 1390249423Sdim return NULL; 1391249423Sdim case MVT::i8: 1392249423Sdim Opcode = NVPTX::STV_i8_v2_ari_64; 1393249423Sdim break; 1394249423Sdim case MVT::i16: 1395249423Sdim Opcode = NVPTX::STV_i16_v2_ari_64; 1396249423Sdim break; 1397249423Sdim case MVT::i32: 1398249423Sdim Opcode = NVPTX::STV_i32_v2_ari_64; 1399249423Sdim break; 1400249423Sdim case MVT::i64: 1401249423Sdim Opcode = NVPTX::STV_i64_v2_ari_64; 1402249423Sdim break; 1403249423Sdim case MVT::f32: 1404249423Sdim Opcode = NVPTX::STV_f32_v2_ari_64; 1405249423Sdim break; 1406249423Sdim case MVT::f64: 1407249423Sdim Opcode = NVPTX::STV_f64_v2_ari_64; 1408249423Sdim break; 1409249423Sdim } 1410249423Sdim break; 1411249423Sdim case NVPTXISD::StoreV4: 1412249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1413249423Sdim default: 1414249423Sdim return NULL; 1415249423Sdim case MVT::i8: 1416249423Sdim Opcode = NVPTX::STV_i8_v4_ari_64; 1417249423Sdim break; 1418249423Sdim case MVT::i16: 1419249423Sdim Opcode = NVPTX::STV_i16_v4_ari_64; 1420249423Sdim break; 1421249423Sdim case MVT::i32: 1422249423Sdim Opcode = NVPTX::STV_i32_v4_ari_64; 1423249423Sdim break; 1424249423Sdim case MVT::f32: 1425249423Sdim Opcode = NVPTX::STV_f32_v4_ari_64; 1426249423Sdim break; 1427249423Sdim } 1428249423Sdim break; 1429249423Sdim } 1430249423Sdim } else { 1431249423Sdim switch (N->getOpcode()) { 1432249423Sdim default: 1433249423Sdim return NULL; 1434249423Sdim case NVPTXISD::StoreV2: 1435249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1436249423Sdim default: 1437249423Sdim return NULL; 1438249423Sdim case MVT::i8: 1439249423Sdim Opcode = NVPTX::STV_i8_v2_ari; 1440249423Sdim break; 1441249423Sdim case MVT::i16: 1442249423Sdim Opcode = NVPTX::STV_i16_v2_ari; 1443249423Sdim break; 1444249423Sdim case MVT::i32: 1445249423Sdim Opcode = NVPTX::STV_i32_v2_ari; 1446249423Sdim break; 1447249423Sdim case MVT::i64: 1448249423Sdim Opcode = NVPTX::STV_i64_v2_ari; 1449249423Sdim break; 1450249423Sdim case MVT::f32: 1451249423Sdim Opcode = NVPTX::STV_f32_v2_ari; 1452249423Sdim break; 1453249423Sdim case MVT::f64: 1454249423Sdim Opcode = NVPTX::STV_f64_v2_ari; 1455249423Sdim break; 1456249423Sdim } 1457249423Sdim break; 1458249423Sdim case NVPTXISD::StoreV4: 1459249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1460249423Sdim default: 1461249423Sdim return NULL; 1462249423Sdim case MVT::i8: 1463249423Sdim Opcode = NVPTX::STV_i8_v4_ari; 1464249423Sdim break; 1465249423Sdim case MVT::i16: 1466249423Sdim Opcode = NVPTX::STV_i16_v4_ari; 1467249423Sdim break; 1468249423Sdim case MVT::i32: 1469249423Sdim Opcode = NVPTX::STV_i32_v4_ari; 1470249423Sdim break; 1471249423Sdim case MVT::f32: 1472249423Sdim Opcode = NVPTX::STV_f32_v4_ari; 1473249423Sdim break; 1474249423Sdim } 1475249423Sdim break; 1476249423Sdim } 1477249423Sdim } 1478249423Sdim StOps.push_back(Base); 1479249423Sdim StOps.push_back(Offset); 1480249423Sdim } else { 1481249423Sdim if (Subtarget.is64Bit()) { 1482249423Sdim switch (N->getOpcode()) { 1483249423Sdim default: 1484249423Sdim return NULL; 1485249423Sdim case NVPTXISD::StoreV2: 1486249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1487249423Sdim default: 1488249423Sdim return NULL; 1489249423Sdim case MVT::i8: 1490249423Sdim Opcode = NVPTX::STV_i8_v2_areg_64; 1491249423Sdim break; 1492249423Sdim case MVT::i16: 1493249423Sdim Opcode = NVPTX::STV_i16_v2_areg_64; 1494249423Sdim break; 1495249423Sdim case MVT::i32: 1496249423Sdim Opcode = NVPTX::STV_i32_v2_areg_64; 1497249423Sdim break; 1498249423Sdim case MVT::i64: 1499249423Sdim Opcode = NVPTX::STV_i64_v2_areg_64; 1500249423Sdim break; 1501249423Sdim case MVT::f32: 1502249423Sdim Opcode = NVPTX::STV_f32_v2_areg_64; 1503249423Sdim break; 1504249423Sdim case MVT::f64: 1505249423Sdim Opcode = NVPTX::STV_f64_v2_areg_64; 1506249423Sdim break; 1507249423Sdim } 1508249423Sdim break; 1509249423Sdim case NVPTXISD::StoreV4: 1510249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1511249423Sdim default: 1512249423Sdim return NULL; 1513249423Sdim case MVT::i8: 1514249423Sdim Opcode = NVPTX::STV_i8_v4_areg_64; 1515249423Sdim break; 1516249423Sdim case MVT::i16: 1517249423Sdim Opcode = NVPTX::STV_i16_v4_areg_64; 1518249423Sdim break; 1519249423Sdim case MVT::i32: 1520249423Sdim Opcode = NVPTX::STV_i32_v4_areg_64; 1521249423Sdim break; 1522249423Sdim case MVT::f32: 1523249423Sdim Opcode = NVPTX::STV_f32_v4_areg_64; 1524249423Sdim break; 1525249423Sdim } 1526249423Sdim break; 1527249423Sdim } 1528249423Sdim } else { 1529249423Sdim switch (N->getOpcode()) { 1530249423Sdim default: 1531249423Sdim return NULL; 1532249423Sdim case NVPTXISD::StoreV2: 1533249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1534249423Sdim default: 1535249423Sdim return NULL; 1536249423Sdim case MVT::i8: 1537249423Sdim Opcode = NVPTX::STV_i8_v2_areg; 1538249423Sdim break; 1539249423Sdim case MVT::i16: 1540249423Sdim Opcode = NVPTX::STV_i16_v2_areg; 1541249423Sdim break; 1542249423Sdim case MVT::i32: 1543249423Sdim Opcode = NVPTX::STV_i32_v2_areg; 1544249423Sdim break; 1545249423Sdim case MVT::i64: 1546249423Sdim Opcode = NVPTX::STV_i64_v2_areg; 1547249423Sdim break; 1548249423Sdim case MVT::f32: 1549249423Sdim Opcode = NVPTX::STV_f32_v2_areg; 1550249423Sdim break; 1551249423Sdim case MVT::f64: 1552249423Sdim Opcode = NVPTX::STV_f64_v2_areg; 1553249423Sdim break; 1554249423Sdim } 1555249423Sdim break; 1556249423Sdim case NVPTXISD::StoreV4: 1557249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1558249423Sdim default: 1559249423Sdim return NULL; 1560249423Sdim case MVT::i8: 1561249423Sdim Opcode = NVPTX::STV_i8_v4_areg; 1562249423Sdim break; 1563249423Sdim case MVT::i16: 1564249423Sdim Opcode = NVPTX::STV_i16_v4_areg; 1565249423Sdim break; 1566249423Sdim case MVT::i32: 1567249423Sdim Opcode = NVPTX::STV_i32_v4_areg; 1568249423Sdim break; 1569249423Sdim case MVT::f32: 1570249423Sdim Opcode = NVPTX::STV_f32_v4_areg; 1571249423Sdim break; 1572249423Sdim } 1573249423Sdim break; 1574249423Sdim } 1575249423Sdim } 1576249423Sdim StOps.push_back(N2); 1577249423Sdim } 1578249423Sdim 1579249423Sdim StOps.push_back(Chain); 1580249423Sdim 1581251662Sdim ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); 1582249423Sdim 1583249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1584249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1585249423Sdim cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1586249423Sdim 1587249423Sdim return ST; 1588249423Sdim} 1589249423Sdim 1590239310Sdim// SelectDirectAddr - Match a direct address for DAG. 1591239310Sdim// A direct address could be a globaladdress or externalsymbol. 1592239310Sdimbool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { 1593239310Sdim // Return true if TGA or ES. 1594249423Sdim if (N.getOpcode() == ISD::TargetGlobalAddress || 1595249423Sdim N.getOpcode() == ISD::TargetExternalSymbol) { 1596239310Sdim Address = N; 1597239310Sdim return true; 1598239310Sdim } 1599239310Sdim if (N.getOpcode() == NVPTXISD::Wrapper) { 1600239310Sdim Address = N.getOperand(0); 1601239310Sdim return true; 1602239310Sdim } 1603239310Sdim if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 1604239310Sdim unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); 1605239310Sdim if (IID == Intrinsic::nvvm_ptr_gen_to_param) 1606239310Sdim if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) 1607239310Sdim return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); 1608239310Sdim } 1609239310Sdim return false; 1610239310Sdim} 1611239310Sdim 1612239310Sdim// symbol+offset 1613249423Sdimbool NVPTXDAGToDAGISel::SelectADDRsi_imp( 1614249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 1615239310Sdim if (Addr.getOpcode() == ISD::ADD) { 1616239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 1617249423Sdim SDValue base = Addr.getOperand(0); 1618239310Sdim if (SelectDirectAddr(base, Base)) { 1619239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 1620239310Sdim return true; 1621239310Sdim } 1622239310Sdim } 1623239310Sdim } 1624239310Sdim return false; 1625239310Sdim} 1626239310Sdim 1627239310Sdim// symbol+offset 1628239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, 1629239310Sdim SDValue &Base, SDValue &Offset) { 1630239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); 1631239310Sdim} 1632239310Sdim 1633239310Sdim// symbol+offset 1634239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, 1635239310Sdim SDValue &Base, SDValue &Offset) { 1636239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); 1637239310Sdim} 1638239310Sdim 1639239310Sdim// register+offset 1640249423Sdimbool NVPTXDAGToDAGISel::SelectADDRri_imp( 1641249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 1642239310Sdim if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 1643239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 1644239310Sdim Offset = CurDAG->getTargetConstant(0, mvt); 1645239310Sdim return true; 1646239310Sdim } 1647239310Sdim if (Addr.getOpcode() == ISD::TargetExternalSymbol || 1648239310Sdim Addr.getOpcode() == ISD::TargetGlobalAddress) 1649249423Sdim return false; // direct calls. 1650239310Sdim 1651239310Sdim if (Addr.getOpcode() == ISD::ADD) { 1652239310Sdim if (SelectDirectAddr(Addr.getOperand(0), Addr)) { 1653239310Sdim return false; 1654239310Sdim } 1655239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 1656239310Sdim if (FrameIndexSDNode *FIN = 1657249423Sdim dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) 1658239310Sdim // Constant offset from frame ref. 1659239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 1660239310Sdim else 1661239310Sdim Base = Addr.getOperand(0); 1662239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 1663239310Sdim return true; 1664239310Sdim } 1665239310Sdim } 1666239310Sdim return false; 1667239310Sdim} 1668239310Sdim 1669239310Sdim// register+offset 1670239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, 1671239310Sdim SDValue &Base, SDValue &Offset) { 1672239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); 1673239310Sdim} 1674239310Sdim 1675239310Sdim// register+offset 1676239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, 1677239310Sdim SDValue &Base, SDValue &Offset) { 1678239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 1679239310Sdim} 1680239310Sdim 1681239310Sdimbool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 1682239310Sdim unsigned int spN) const { 1683239310Sdim const Value *Src = NULL; 1684239310Sdim // Even though MemIntrinsicSDNode is a subclas of MemSDNode, 1685239310Sdim // the classof() for MemSDNode does not include MemIntrinsicSDNode 1686239310Sdim // (See SelectionDAGNodes.h). So we need to check for both. 1687239310Sdim if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 1688239310Sdim Src = mN->getSrcValue(); 1689249423Sdim } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) { 1690239310Sdim Src = mN->getSrcValue(); 1691239310Sdim } 1692239310Sdim if (!Src) 1693239310Sdim return false; 1694239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) 1695239310Sdim return (PT->getAddressSpace() == spN); 1696239310Sdim return false; 1697239310Sdim} 1698239310Sdim 1699239310Sdim/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 1700239310Sdim/// inline asm expressions. 1701249423Sdimbool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( 1702249423Sdim const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) { 1703239310Sdim SDValue Op0, Op1; 1704239310Sdim switch (ConstraintCode) { 1705249423Sdim default: 1706249423Sdim return true; 1707249423Sdim case 'm': // memory 1708239310Sdim if (SelectDirectAddr(Op, Op0)) { 1709239310Sdim OutOps.push_back(Op0); 1710239310Sdim OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32)); 1711239310Sdim return false; 1712239310Sdim } 1713239310Sdim if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { 1714239310Sdim OutOps.push_back(Op0); 1715239310Sdim OutOps.push_back(Op1); 1716239310Sdim return false; 1717239310Sdim } 1718239310Sdim break; 1719239310Sdim } 1720239310Sdim return true; 1721239310Sdim} 1722239310Sdim 1723239310Sdim// Return true if N is a undef or a constant. 1724239310Sdim// If N was undef, return a (i8imm 0) in Retval 1725239310Sdim// If N was imm, convert it to i8imm and return in Retval 1726239310Sdim// Note: The convert to i8imm is required, otherwise the 1727239310Sdim// pattern matcher inserts a bunch of IMOVi8rr to convert 1728239310Sdim// the imm to i8imm, and this causes instruction selection 1729239310Sdim// to fail. 1730249423Sdimbool NVPTXDAGToDAGISel::UndefOrImm(SDValue Op, SDValue N, SDValue &Retval) { 1731249423Sdim if (!(N.getOpcode() == ISD::UNDEF) && !(N.getOpcode() == ISD::Constant)) 1732239310Sdim return false; 1733239310Sdim 1734239310Sdim if (N.getOpcode() == ISD::UNDEF) 1735239310Sdim Retval = CurDAG->getTargetConstant(0, MVT::i8); 1736239310Sdim else { 1737239310Sdim ConstantSDNode *cn = cast<ConstantSDNode>(N.getNode()); 1738239310Sdim unsigned retval = cn->getZExtValue(); 1739239310Sdim Retval = CurDAG->getTargetConstant(retval, MVT::i8); 1740239310Sdim } 1741239310Sdim return true; 1742239310Sdim} 1743