1239310Sdim//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// 2239310Sdim// 3239310Sdim// The LLVM Compiler Infrastructure 4239310Sdim// 5239310Sdim// This file is distributed under the University of Illinois Open Source 6239310Sdim// License. See LICENSE.TXT for details. 7239310Sdim// 8239310Sdim//===----------------------------------------------------------------------===// 9239310Sdim// 10239310Sdim// This file defines an instruction selector for the NVPTX target. 11239310Sdim// 12239310Sdim//===----------------------------------------------------------------------===// 13239310Sdim 14239310Sdim#include "NVPTXISelDAGToDAG.h" 15249423Sdim#include "llvm/IR/GlobalValue.h" 16249423Sdim#include "llvm/IR/Instructions.h" 17249423Sdim#include "llvm/Support/CommandLine.h" 18239310Sdim#include "llvm/Support/Debug.h" 19239310Sdim#include "llvm/Support/ErrorHandling.h" 20249423Sdim#include "llvm/Support/raw_ostream.h" 21239310Sdim#include "llvm/Target/TargetIntrinsicInfo.h" 22239310Sdim 23239310Sdim#undef DEBUG_TYPE 24239310Sdim#define DEBUG_TYPE "nvptx-isel" 25239310Sdim 26239310Sdimusing namespace llvm; 27239310Sdim 28239310Sdimstatic cl::opt<int> 29263508SdimFMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden, 30239310Sdim cl::desc("NVPTX Specific: FMA contraction (0: don't do it" 31249423Sdim " 1: do it 2: do it aggressively"), 32249423Sdim cl::init(2)); 33239310Sdim 34249423Sdimstatic cl::opt<int> UsePrecDivF32( 35263508Sdim "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, 36249423Sdim cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 37249423Sdim " IEEE Compliant F32 div.rnd if avaiable."), 38249423Sdim cl::init(2)); 39239310Sdim 40251662Sdimstatic cl::opt<bool> 41263508SdimUsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, 42251662Sdim cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 43251662Sdim cl::init(true)); 44251662Sdim 45263508Sdimstatic cl::opt<bool> 46263508SdimFtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden, 47263508Sdim cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), 48263508Sdim cl::init(false)); 49263508Sdim 50263508Sdim 51239310Sdim/// createNVPTXISelDag - This pass converts a legalized DAG into a 52239310Sdim/// NVPTX-specific DAG, ready for instruction scheduling. 53239310SdimFunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, 54239310Sdim llvm::CodeGenOpt::Level OptLevel) { 55239310Sdim return new NVPTXDAGToDAGISel(TM, OptLevel); 56239310Sdim} 57239310Sdim 58239310SdimNVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 59239310Sdim CodeGenOpt::Level OptLevel) 60249423Sdim : SelectionDAGISel(tm, OptLevel), 61249423Sdim Subtarget(tm.getSubtarget<NVPTXSubtarget>()) { 62239310Sdim 63249423Sdim doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1); 64249423Sdim doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1); 65249423Sdim doFMAF32AGG = 66249423Sdim (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2); 67249423Sdim doFMAF64AGG = 68249423Sdim (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2); 69239310Sdim 70263508Sdim allowFMA = (FMAContractLevel >= 1); 71239310Sdim 72239310Sdim doMulWide = (OptLevel > 0); 73263508Sdim} 74239310Sdim 75263508Sdimint NVPTXDAGToDAGISel::getDivF32Level() const { 76263508Sdim if (UsePrecDivF32.getNumOccurrences() > 0) { 77263508Sdim // If nvptx-prec-div32=N is used on the command-line, always honor it 78263508Sdim return UsePrecDivF32; 79263508Sdim } else { 80263508Sdim // Otherwise, use div.approx if fast math is enabled 81263508Sdim if (TM.Options.UnsafeFPMath) 82263508Sdim return 0; 83263508Sdim else 84263508Sdim return 2; 85263508Sdim } 86263508Sdim} 87239310Sdim 88263508Sdimbool NVPTXDAGToDAGISel::usePrecSqrtF32() const { 89263508Sdim if (UsePrecSqrtF32.getNumOccurrences() > 0) { 90263508Sdim // If nvptx-prec-sqrtf32 is used on the command-line, always honor it 91263508Sdim return UsePrecSqrtF32; 92263508Sdim } else { 93263508Sdim // Otherwise, use sqrt.approx if fast math is enabled 94263508Sdim if (TM.Options.UnsafeFPMath) 95263508Sdim return false; 96263508Sdim else 97263508Sdim return true; 98263508Sdim } 99239310Sdim} 100239310Sdim 101263508Sdimbool NVPTXDAGToDAGISel::useF32FTZ() const { 102263508Sdim if (FtzEnabled.getNumOccurrences() > 0) { 103263508Sdim // If nvptx-f32ftz is used on the command-line, always honor it 104263508Sdim return FtzEnabled; 105263508Sdim } else { 106263508Sdim const Function *F = MF->getFunction(); 107263508Sdim // Otherwise, check for an nvptx-f32ftz attribute on the function 108263508Sdim if (F->hasFnAttribute("nvptx-f32ftz")) 109263508Sdim return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex, 110263508Sdim "nvptx-f32ftz") 111263508Sdim .getValueAsString() == "true"); 112263508Sdim else 113263508Sdim return false; 114263508Sdim } 115263508Sdim} 116263508Sdim 117239310Sdim/// Select - Select instructions not customized! Used for 118239310Sdim/// expanded, promoted and normal instructions. 119249423SdimSDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 120239310Sdim 121255804Sdim if (N->isMachineOpcode()) { 122255804Sdim N->setNodeId(-1); 123249423Sdim return NULL; // Already selected. 124255804Sdim } 125239310Sdim 126239310Sdim SDNode *ResNode = NULL; 127239310Sdim switch (N->getOpcode()) { 128239310Sdim case ISD::LOAD: 129239310Sdim ResNode = SelectLoad(N); 130239310Sdim break; 131239310Sdim case ISD::STORE: 132239310Sdim ResNode = SelectStore(N); 133239310Sdim break; 134249423Sdim case NVPTXISD::LoadV2: 135249423Sdim case NVPTXISD::LoadV4: 136249423Sdim ResNode = SelectLoadVector(N); 137249423Sdim break; 138249423Sdim case NVPTXISD::LDGV2: 139249423Sdim case NVPTXISD::LDGV4: 140249423Sdim case NVPTXISD::LDUV2: 141249423Sdim case NVPTXISD::LDUV4: 142249423Sdim ResNode = SelectLDGLDUVector(N); 143249423Sdim break; 144249423Sdim case NVPTXISD::StoreV2: 145249423Sdim case NVPTXISD::StoreV4: 146249423Sdim ResNode = SelectStoreVector(N); 147249423Sdim break; 148263508Sdim case NVPTXISD::LoadParam: 149263508Sdim case NVPTXISD::LoadParamV2: 150263508Sdim case NVPTXISD::LoadParamV4: 151263508Sdim ResNode = SelectLoadParam(N); 152263508Sdim break; 153263508Sdim case NVPTXISD::StoreRetval: 154263508Sdim case NVPTXISD::StoreRetvalV2: 155263508Sdim case NVPTXISD::StoreRetvalV4: 156263508Sdim ResNode = SelectStoreRetval(N); 157263508Sdim break; 158263508Sdim case NVPTXISD::StoreParam: 159263508Sdim case NVPTXISD::StoreParamV2: 160263508Sdim case NVPTXISD::StoreParamV4: 161263508Sdim case NVPTXISD::StoreParamS32: 162263508Sdim case NVPTXISD::StoreParamU32: 163263508Sdim ResNode = SelectStoreParam(N); 164263508Sdim break; 165249423Sdim default: 166249423Sdim break; 167239310Sdim } 168239310Sdim if (ResNode) 169239310Sdim return ResNode; 170239310Sdim return SelectCode(N); 171239310Sdim} 172239310Sdim 173249423Sdimstatic unsigned int getCodeAddrSpace(MemSDNode *N, 174249423Sdim const NVPTXSubtarget &Subtarget) { 175239310Sdim const Value *Src = N->getSrcValue(); 176263508Sdim 177239310Sdim if (!Src) 178263508Sdim return NVPTX::PTXLdStInstCode::GENERIC; 179239310Sdim 180239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { 181239310Sdim switch (PT->getAddressSpace()) { 182263508Sdim case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL; 183263508Sdim case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL; 184263508Sdim case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED; 185263508Sdim case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC; 186263508Sdim case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM; 187263508Sdim case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT; 188263508Sdim default: break; 189239310Sdim } 190239310Sdim } 191263508Sdim return NVPTX::PTXLdStInstCode::GENERIC; 192239310Sdim} 193239310Sdim 194249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 195263508Sdim SDLoc dl(N); 196239310Sdim LoadSDNode *LD = cast<LoadSDNode>(N); 197239310Sdim EVT LoadedVT = LD->getMemoryVT(); 198249423Sdim SDNode *NVPTXLD = NULL; 199239310Sdim 200239310Sdim // do not support pre/post inc/dec 201239310Sdim if (LD->isIndexed()) 202239310Sdim return NULL; 203239310Sdim 204239310Sdim if (!LoadedVT.isSimple()) 205239310Sdim return NULL; 206239310Sdim 207239310Sdim // Address Space Setting 208239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget); 209239310Sdim 210239310Sdim // Volatile Setting 211239310Sdim // - .volatile is only availalble for .global and .shared 212239310Sdim bool isVolatile = LD->isVolatile(); 213239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 214239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 215239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 216239310Sdim isVolatile = false; 217239310Sdim 218239310Sdim // Vector Setting 219239310Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 220239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 221239310Sdim if (SimpleVT.isVector()) { 222239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 223239310Sdim if (num == 2) 224239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 225239310Sdim else if (num == 4) 226239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 227239310Sdim else 228239310Sdim return NULL; 229239310Sdim } 230239310Sdim 231239310Sdim // Type Setting: fromType + fromTypeWidth 232239310Sdim // 233239310Sdim // Sign : ISD::SEXTLOAD 234239310Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 235239310Sdim // type is integer 236239310Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 237239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 238263508Sdim // Read at least 8 bits (predicates are stored as 8-bit values) 239263508Sdim unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 240239310Sdim unsigned int fromType; 241239310Sdim if ((LD->getExtensionType() == ISD::SEXTLOAD)) 242239310Sdim fromType = NVPTX::PTXLdStInstCode::Signed; 243239310Sdim else if (ScalarVT.isFloatingPoint()) 244239310Sdim fromType = NVPTX::PTXLdStInstCode::Float; 245239310Sdim else 246239310Sdim fromType = NVPTX::PTXLdStInstCode::Unsigned; 247239310Sdim 248239310Sdim // Create the machine instruction DAG 249239310Sdim SDValue Chain = N->getOperand(0); 250239310Sdim SDValue N1 = N->getOperand(1); 251239310Sdim SDValue Addr; 252239310Sdim SDValue Offset, Base; 253239310Sdim unsigned Opcode; 254263508Sdim MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy; 255239310Sdim 256239310Sdim if (SelectDirectAddr(N1, Addr)) { 257239310Sdim switch (TargetVT) { 258249423Sdim case MVT::i8: 259249423Sdim Opcode = NVPTX::LD_i8_avar; 260249423Sdim break; 261249423Sdim case MVT::i16: 262249423Sdim Opcode = NVPTX::LD_i16_avar; 263249423Sdim break; 264249423Sdim case MVT::i32: 265249423Sdim Opcode = NVPTX::LD_i32_avar; 266249423Sdim break; 267249423Sdim case MVT::i64: 268249423Sdim Opcode = NVPTX::LD_i64_avar; 269249423Sdim break; 270249423Sdim case MVT::f32: 271249423Sdim Opcode = NVPTX::LD_f32_avar; 272249423Sdim break; 273249423Sdim case MVT::f64: 274249423Sdim Opcode = NVPTX::LD_f64_avar; 275249423Sdim break; 276249423Sdim default: 277249423Sdim return NULL; 278239310Sdim } 279249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 280249423Sdim getI32Imm(vecType), getI32Imm(fromType), 281249423Sdim getI32Imm(fromTypeWidth), Addr, Chain }; 282251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 283249423Sdim } else if (Subtarget.is64Bit() 284249423Sdim ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) 285249423Sdim : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { 286239310Sdim switch (TargetVT) { 287249423Sdim case MVT::i8: 288249423Sdim Opcode = NVPTX::LD_i8_asi; 289249423Sdim break; 290249423Sdim case MVT::i16: 291249423Sdim Opcode = NVPTX::LD_i16_asi; 292249423Sdim break; 293249423Sdim case MVT::i32: 294249423Sdim Opcode = NVPTX::LD_i32_asi; 295249423Sdim break; 296249423Sdim case MVT::i64: 297249423Sdim Opcode = NVPTX::LD_i64_asi; 298249423Sdim break; 299249423Sdim case MVT::f32: 300249423Sdim Opcode = NVPTX::LD_f32_asi; 301249423Sdim break; 302249423Sdim case MVT::f64: 303249423Sdim Opcode = NVPTX::LD_f64_asi; 304249423Sdim break; 305249423Sdim default: 306249423Sdim return NULL; 307239310Sdim } 308249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 309249423Sdim getI32Imm(vecType), getI32Imm(fromType), 310249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 311251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 312249423Sdim } else if (Subtarget.is64Bit() 313249423Sdim ? SelectADDRri64(N1.getNode(), N1, Base, Offset) 314249423Sdim : SelectADDRri(N1.getNode(), N1, Base, Offset)) { 315249423Sdim if (Subtarget.is64Bit()) { 316249423Sdim switch (TargetVT) { 317249423Sdim case MVT::i8: 318249423Sdim Opcode = NVPTX::LD_i8_ari_64; 319249423Sdim break; 320249423Sdim case MVT::i16: 321249423Sdim Opcode = NVPTX::LD_i16_ari_64; 322249423Sdim break; 323249423Sdim case MVT::i32: 324249423Sdim Opcode = NVPTX::LD_i32_ari_64; 325249423Sdim break; 326249423Sdim case MVT::i64: 327249423Sdim Opcode = NVPTX::LD_i64_ari_64; 328249423Sdim break; 329249423Sdim case MVT::f32: 330249423Sdim Opcode = NVPTX::LD_f32_ari_64; 331249423Sdim break; 332249423Sdim case MVT::f64: 333249423Sdim Opcode = NVPTX::LD_f64_ari_64; 334249423Sdim break; 335249423Sdim default: 336249423Sdim return NULL; 337249423Sdim } 338249423Sdim } else { 339249423Sdim switch (TargetVT) { 340249423Sdim case MVT::i8: 341249423Sdim Opcode = NVPTX::LD_i8_ari; 342249423Sdim break; 343249423Sdim case MVT::i16: 344249423Sdim Opcode = NVPTX::LD_i16_ari; 345249423Sdim break; 346249423Sdim case MVT::i32: 347249423Sdim Opcode = NVPTX::LD_i32_ari; 348249423Sdim break; 349249423Sdim case MVT::i64: 350249423Sdim Opcode = NVPTX::LD_i64_ari; 351249423Sdim break; 352249423Sdim case MVT::f32: 353249423Sdim Opcode = NVPTX::LD_f32_ari; 354249423Sdim break; 355249423Sdim case MVT::f64: 356249423Sdim Opcode = NVPTX::LD_f64_ari; 357249423Sdim break; 358249423Sdim default: 359249423Sdim return NULL; 360249423Sdim } 361239310Sdim } 362249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 363249423Sdim getI32Imm(vecType), getI32Imm(fromType), 364249423Sdim getI32Imm(fromTypeWidth), Base, Offset, Chain }; 365251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 366249423Sdim } else { 367249423Sdim if (Subtarget.is64Bit()) { 368249423Sdim switch (TargetVT) { 369249423Sdim case MVT::i8: 370249423Sdim Opcode = NVPTX::LD_i8_areg_64; 371249423Sdim break; 372249423Sdim case MVT::i16: 373249423Sdim Opcode = NVPTX::LD_i16_areg_64; 374249423Sdim break; 375249423Sdim case MVT::i32: 376249423Sdim Opcode = NVPTX::LD_i32_areg_64; 377249423Sdim break; 378249423Sdim case MVT::i64: 379249423Sdim Opcode = NVPTX::LD_i64_areg_64; 380249423Sdim break; 381249423Sdim case MVT::f32: 382249423Sdim Opcode = NVPTX::LD_f32_areg_64; 383249423Sdim break; 384249423Sdim case MVT::f64: 385249423Sdim Opcode = NVPTX::LD_f64_areg_64; 386249423Sdim break; 387249423Sdim default: 388249423Sdim return NULL; 389249423Sdim } 390249423Sdim } else { 391249423Sdim switch (TargetVT) { 392249423Sdim case MVT::i8: 393249423Sdim Opcode = NVPTX::LD_i8_areg; 394249423Sdim break; 395249423Sdim case MVT::i16: 396249423Sdim Opcode = NVPTX::LD_i16_areg; 397249423Sdim break; 398249423Sdim case MVT::i32: 399249423Sdim Opcode = NVPTX::LD_i32_areg; 400249423Sdim break; 401249423Sdim case MVT::i64: 402249423Sdim Opcode = NVPTX::LD_i64_areg; 403249423Sdim break; 404249423Sdim case MVT::f32: 405249423Sdim Opcode = NVPTX::LD_f32_areg; 406249423Sdim break; 407249423Sdim case MVT::f64: 408249423Sdim Opcode = NVPTX::LD_f64_areg; 409249423Sdim break; 410249423Sdim default: 411249423Sdim return NULL; 412249423Sdim } 413239310Sdim } 414249423Sdim SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 415249423Sdim getI32Imm(vecType), getI32Imm(fromType), 416249423Sdim getI32Imm(fromTypeWidth), N1, Chain }; 417251662Sdim NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 418239310Sdim } 419239310Sdim 420239310Sdim if (NVPTXLD != NULL) { 421239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 422239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 423239310Sdim cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 424239310Sdim } 425239310Sdim 426239310Sdim return NVPTXLD; 427239310Sdim} 428239310Sdim 429249423SdimSDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { 430249423Sdim 431249423Sdim SDValue Chain = N->getOperand(0); 432249423Sdim SDValue Op1 = N->getOperand(1); 433249423Sdim SDValue Addr, Offset, Base; 434249423Sdim unsigned Opcode; 435263508Sdim SDLoc DL(N); 436249423Sdim SDNode *LD; 437249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 438249423Sdim EVT LoadedVT = MemSD->getMemoryVT(); 439249423Sdim 440249423Sdim if (!LoadedVT.isSimple()) 441249423Sdim return NULL; 442249423Sdim 443249423Sdim // Address Space Setting 444249423Sdim unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 445249423Sdim 446249423Sdim // Volatile Setting 447249423Sdim // - .volatile is only availalble for .global and .shared 448249423Sdim bool IsVolatile = MemSD->isVolatile(); 449249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 450249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 451249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 452249423Sdim IsVolatile = false; 453249423Sdim 454249423Sdim // Vector Setting 455249423Sdim MVT SimpleVT = LoadedVT.getSimpleVT(); 456249423Sdim 457249423Sdim // Type Setting: fromType + fromTypeWidth 458249423Sdim // 459249423Sdim // Sign : ISD::SEXTLOAD 460249423Sdim // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 461249423Sdim // type is integer 462249423Sdim // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 463249423Sdim MVT ScalarVT = SimpleVT.getScalarType(); 464263508Sdim // Read at least 8 bits (predicates are stored as 8-bit values) 465263508Sdim unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 466249423Sdim unsigned int FromType; 467249423Sdim // The last operand holds the original LoadSDNode::getExtensionType() value 468249423Sdim unsigned ExtensionType = cast<ConstantSDNode>( 469249423Sdim N->getOperand(N->getNumOperands() - 1))->getZExtValue(); 470249423Sdim if (ExtensionType == ISD::SEXTLOAD) 471249423Sdim FromType = NVPTX::PTXLdStInstCode::Signed; 472249423Sdim else if (ScalarVT.isFloatingPoint()) 473249423Sdim FromType = NVPTX::PTXLdStInstCode::Float; 474249423Sdim else 475249423Sdim FromType = NVPTX::PTXLdStInstCode::Unsigned; 476249423Sdim 477249423Sdim unsigned VecType; 478249423Sdim 479249423Sdim switch (N->getOpcode()) { 480249423Sdim case NVPTXISD::LoadV2: 481249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 482249423Sdim break; 483249423Sdim case NVPTXISD::LoadV4: 484249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 485249423Sdim break; 486249423Sdim default: 487249423Sdim return NULL; 488249423Sdim } 489249423Sdim 490249423Sdim EVT EltVT = N->getValueType(0); 491249423Sdim 492249423Sdim if (SelectDirectAddr(Op1, Addr)) { 493249423Sdim switch (N->getOpcode()) { 494249423Sdim default: 495249423Sdim return NULL; 496249423Sdim case NVPTXISD::LoadV2: 497249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 498249423Sdim default: 499249423Sdim return NULL; 500249423Sdim case MVT::i8: 501249423Sdim Opcode = NVPTX::LDV_i8_v2_avar; 502249423Sdim break; 503249423Sdim case MVT::i16: 504249423Sdim Opcode = NVPTX::LDV_i16_v2_avar; 505249423Sdim break; 506249423Sdim case MVT::i32: 507249423Sdim Opcode = NVPTX::LDV_i32_v2_avar; 508249423Sdim break; 509249423Sdim case MVT::i64: 510249423Sdim Opcode = NVPTX::LDV_i64_v2_avar; 511249423Sdim break; 512249423Sdim case MVT::f32: 513249423Sdim Opcode = NVPTX::LDV_f32_v2_avar; 514249423Sdim break; 515249423Sdim case MVT::f64: 516249423Sdim Opcode = NVPTX::LDV_f64_v2_avar; 517249423Sdim break; 518249423Sdim } 519249423Sdim break; 520249423Sdim case NVPTXISD::LoadV4: 521249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 522249423Sdim default: 523249423Sdim return NULL; 524249423Sdim case MVT::i8: 525249423Sdim Opcode = NVPTX::LDV_i8_v4_avar; 526249423Sdim break; 527249423Sdim case MVT::i16: 528249423Sdim Opcode = NVPTX::LDV_i16_v4_avar; 529249423Sdim break; 530249423Sdim case MVT::i32: 531249423Sdim Opcode = NVPTX::LDV_i32_v4_avar; 532249423Sdim break; 533249423Sdim case MVT::f32: 534249423Sdim Opcode = NVPTX::LDV_f32_v4_avar; 535249423Sdim break; 536249423Sdim } 537249423Sdim break; 538249423Sdim } 539249423Sdim 540249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 541249423Sdim getI32Imm(VecType), getI32Imm(FromType), 542249423Sdim getI32Imm(FromTypeWidth), Addr, Chain }; 543251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 544249423Sdim } else if (Subtarget.is64Bit() 545249423Sdim ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 546249423Sdim : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 547249423Sdim switch (N->getOpcode()) { 548249423Sdim default: 549249423Sdim return NULL; 550249423Sdim case NVPTXISD::LoadV2: 551249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 552249423Sdim default: 553249423Sdim return NULL; 554249423Sdim case MVT::i8: 555249423Sdim Opcode = NVPTX::LDV_i8_v2_asi; 556249423Sdim break; 557249423Sdim case MVT::i16: 558249423Sdim Opcode = NVPTX::LDV_i16_v2_asi; 559249423Sdim break; 560249423Sdim case MVT::i32: 561249423Sdim Opcode = NVPTX::LDV_i32_v2_asi; 562249423Sdim break; 563249423Sdim case MVT::i64: 564249423Sdim Opcode = NVPTX::LDV_i64_v2_asi; 565249423Sdim break; 566249423Sdim case MVT::f32: 567249423Sdim Opcode = NVPTX::LDV_f32_v2_asi; 568249423Sdim break; 569249423Sdim case MVT::f64: 570249423Sdim Opcode = NVPTX::LDV_f64_v2_asi; 571249423Sdim break; 572249423Sdim } 573249423Sdim break; 574249423Sdim case NVPTXISD::LoadV4: 575249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 576249423Sdim default: 577249423Sdim return NULL; 578249423Sdim case MVT::i8: 579249423Sdim Opcode = NVPTX::LDV_i8_v4_asi; 580249423Sdim break; 581249423Sdim case MVT::i16: 582249423Sdim Opcode = NVPTX::LDV_i16_v4_asi; 583249423Sdim break; 584249423Sdim case MVT::i32: 585249423Sdim Opcode = NVPTX::LDV_i32_v4_asi; 586249423Sdim break; 587249423Sdim case MVT::f32: 588249423Sdim Opcode = NVPTX::LDV_f32_v4_asi; 589249423Sdim break; 590249423Sdim } 591249423Sdim break; 592249423Sdim } 593249423Sdim 594249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 595249423Sdim getI32Imm(VecType), getI32Imm(FromType), 596249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 597251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 598249423Sdim } else if (Subtarget.is64Bit() 599249423Sdim ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 600249423Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 601249423Sdim if (Subtarget.is64Bit()) { 602249423Sdim switch (N->getOpcode()) { 603249423Sdim default: 604249423Sdim return NULL; 605249423Sdim case NVPTXISD::LoadV2: 606249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 607249423Sdim default: 608249423Sdim return NULL; 609249423Sdim case MVT::i8: 610249423Sdim Opcode = NVPTX::LDV_i8_v2_ari_64; 611249423Sdim break; 612249423Sdim case MVT::i16: 613249423Sdim Opcode = NVPTX::LDV_i16_v2_ari_64; 614249423Sdim break; 615249423Sdim case MVT::i32: 616249423Sdim Opcode = NVPTX::LDV_i32_v2_ari_64; 617249423Sdim break; 618249423Sdim case MVT::i64: 619249423Sdim Opcode = NVPTX::LDV_i64_v2_ari_64; 620249423Sdim break; 621249423Sdim case MVT::f32: 622249423Sdim Opcode = NVPTX::LDV_f32_v2_ari_64; 623249423Sdim break; 624249423Sdim case MVT::f64: 625249423Sdim Opcode = NVPTX::LDV_f64_v2_ari_64; 626249423Sdim break; 627249423Sdim } 628249423Sdim break; 629249423Sdim case NVPTXISD::LoadV4: 630249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 631249423Sdim default: 632249423Sdim return NULL; 633249423Sdim case MVT::i8: 634249423Sdim Opcode = NVPTX::LDV_i8_v4_ari_64; 635249423Sdim break; 636249423Sdim case MVT::i16: 637249423Sdim Opcode = NVPTX::LDV_i16_v4_ari_64; 638249423Sdim break; 639249423Sdim case MVT::i32: 640249423Sdim Opcode = NVPTX::LDV_i32_v4_ari_64; 641249423Sdim break; 642249423Sdim case MVT::f32: 643249423Sdim Opcode = NVPTX::LDV_f32_v4_ari_64; 644249423Sdim break; 645249423Sdim } 646249423Sdim break; 647249423Sdim } 648249423Sdim } else { 649249423Sdim switch (N->getOpcode()) { 650249423Sdim default: 651249423Sdim return NULL; 652249423Sdim case NVPTXISD::LoadV2: 653249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 654249423Sdim default: 655249423Sdim return NULL; 656249423Sdim case MVT::i8: 657249423Sdim Opcode = NVPTX::LDV_i8_v2_ari; 658249423Sdim break; 659249423Sdim case MVT::i16: 660249423Sdim Opcode = NVPTX::LDV_i16_v2_ari; 661249423Sdim break; 662249423Sdim case MVT::i32: 663249423Sdim Opcode = NVPTX::LDV_i32_v2_ari; 664249423Sdim break; 665249423Sdim case MVT::i64: 666249423Sdim Opcode = NVPTX::LDV_i64_v2_ari; 667249423Sdim break; 668249423Sdim case MVT::f32: 669249423Sdim Opcode = NVPTX::LDV_f32_v2_ari; 670249423Sdim break; 671249423Sdim case MVT::f64: 672249423Sdim Opcode = NVPTX::LDV_f64_v2_ari; 673249423Sdim break; 674249423Sdim } 675249423Sdim break; 676249423Sdim case NVPTXISD::LoadV4: 677249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 678249423Sdim default: 679249423Sdim return NULL; 680249423Sdim case MVT::i8: 681249423Sdim Opcode = NVPTX::LDV_i8_v4_ari; 682249423Sdim break; 683249423Sdim case MVT::i16: 684249423Sdim Opcode = NVPTX::LDV_i16_v4_ari; 685249423Sdim break; 686249423Sdim case MVT::i32: 687249423Sdim Opcode = NVPTX::LDV_i32_v4_ari; 688249423Sdim break; 689249423Sdim case MVT::f32: 690249423Sdim Opcode = NVPTX::LDV_f32_v4_ari; 691249423Sdim break; 692249423Sdim } 693249423Sdim break; 694249423Sdim } 695249423Sdim } 696249423Sdim 697249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 698249423Sdim getI32Imm(VecType), getI32Imm(FromType), 699249423Sdim getI32Imm(FromTypeWidth), Base, Offset, Chain }; 700249423Sdim 701251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 702249423Sdim } else { 703249423Sdim if (Subtarget.is64Bit()) { 704249423Sdim switch (N->getOpcode()) { 705249423Sdim default: 706249423Sdim return NULL; 707249423Sdim case NVPTXISD::LoadV2: 708249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 709249423Sdim default: 710249423Sdim return NULL; 711249423Sdim case MVT::i8: 712249423Sdim Opcode = NVPTX::LDV_i8_v2_areg_64; 713249423Sdim break; 714249423Sdim case MVT::i16: 715249423Sdim Opcode = NVPTX::LDV_i16_v2_areg_64; 716249423Sdim break; 717249423Sdim case MVT::i32: 718249423Sdim Opcode = NVPTX::LDV_i32_v2_areg_64; 719249423Sdim break; 720249423Sdim case MVT::i64: 721249423Sdim Opcode = NVPTX::LDV_i64_v2_areg_64; 722249423Sdim break; 723249423Sdim case MVT::f32: 724249423Sdim Opcode = NVPTX::LDV_f32_v2_areg_64; 725249423Sdim break; 726249423Sdim case MVT::f64: 727249423Sdim Opcode = NVPTX::LDV_f64_v2_areg_64; 728249423Sdim break; 729249423Sdim } 730249423Sdim break; 731249423Sdim case NVPTXISD::LoadV4: 732249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 733249423Sdim default: 734249423Sdim return NULL; 735249423Sdim case MVT::i8: 736249423Sdim Opcode = NVPTX::LDV_i8_v4_areg_64; 737249423Sdim break; 738249423Sdim case MVT::i16: 739249423Sdim Opcode = NVPTX::LDV_i16_v4_areg_64; 740249423Sdim break; 741249423Sdim case MVT::i32: 742249423Sdim Opcode = NVPTX::LDV_i32_v4_areg_64; 743249423Sdim break; 744249423Sdim case MVT::f32: 745249423Sdim Opcode = NVPTX::LDV_f32_v4_areg_64; 746249423Sdim break; 747249423Sdim } 748249423Sdim break; 749249423Sdim } 750249423Sdim } else { 751249423Sdim switch (N->getOpcode()) { 752249423Sdim default: 753249423Sdim return NULL; 754249423Sdim case NVPTXISD::LoadV2: 755249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 756249423Sdim default: 757249423Sdim return NULL; 758249423Sdim case MVT::i8: 759249423Sdim Opcode = NVPTX::LDV_i8_v2_areg; 760249423Sdim break; 761249423Sdim case MVT::i16: 762249423Sdim Opcode = NVPTX::LDV_i16_v2_areg; 763249423Sdim break; 764249423Sdim case MVT::i32: 765249423Sdim Opcode = NVPTX::LDV_i32_v2_areg; 766249423Sdim break; 767249423Sdim case MVT::i64: 768249423Sdim Opcode = NVPTX::LDV_i64_v2_areg; 769249423Sdim break; 770249423Sdim case MVT::f32: 771249423Sdim Opcode = NVPTX::LDV_f32_v2_areg; 772249423Sdim break; 773249423Sdim case MVT::f64: 774249423Sdim Opcode = NVPTX::LDV_f64_v2_areg; 775249423Sdim break; 776249423Sdim } 777249423Sdim break; 778249423Sdim case NVPTXISD::LoadV4: 779249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 780249423Sdim default: 781249423Sdim return NULL; 782249423Sdim case MVT::i8: 783249423Sdim Opcode = NVPTX::LDV_i8_v4_areg; 784249423Sdim break; 785249423Sdim case MVT::i16: 786249423Sdim Opcode = NVPTX::LDV_i16_v4_areg; 787249423Sdim break; 788249423Sdim case MVT::i32: 789249423Sdim Opcode = NVPTX::LDV_i32_v4_areg; 790249423Sdim break; 791249423Sdim case MVT::f32: 792249423Sdim Opcode = NVPTX::LDV_f32_v4_areg; 793249423Sdim break; 794249423Sdim } 795249423Sdim break; 796249423Sdim } 797249423Sdim } 798249423Sdim 799249423Sdim SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 800249423Sdim getI32Imm(VecType), getI32Imm(FromType), 801249423Sdim getI32Imm(FromTypeWidth), Op1, Chain }; 802251662Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 803249423Sdim } 804249423Sdim 805249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 806249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 807249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 808249423Sdim 809249423Sdim return LD; 810249423Sdim} 811249423Sdim 812249423SdimSDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) { 813249423Sdim 814249423Sdim SDValue Chain = N->getOperand(0); 815249423Sdim SDValue Op1 = N->getOperand(1); 816249423Sdim unsigned Opcode; 817263508Sdim SDLoc DL(N); 818249423Sdim SDNode *LD; 819263508Sdim MemSDNode *Mem = cast<MemSDNode>(N); 820263508Sdim SDValue Base, Offset, Addr; 821249423Sdim 822263508Sdim EVT EltVT = Mem->getMemoryVT().getVectorElementType(); 823249423Sdim 824263508Sdim if (SelectDirectAddr(Op1, Addr)) { 825249423Sdim switch (N->getOpcode()) { 826249423Sdim default: 827249423Sdim return NULL; 828249423Sdim case NVPTXISD::LDGV2: 829263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 830249423Sdim default: 831249423Sdim return NULL; 832249423Sdim case MVT::i8: 833263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; 834249423Sdim break; 835249423Sdim case MVT::i16: 836263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; 837249423Sdim break; 838249423Sdim case MVT::i32: 839263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar; 840249423Sdim break; 841249423Sdim case MVT::i64: 842263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar; 843249423Sdim break; 844249423Sdim case MVT::f32: 845263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar; 846249423Sdim break; 847249423Sdim case MVT::f64: 848263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar; 849249423Sdim break; 850249423Sdim } 851249423Sdim break; 852263508Sdim case NVPTXISD::LDUV2: 853263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 854249423Sdim default: 855249423Sdim return NULL; 856249423Sdim case MVT::i8: 857263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; 858249423Sdim break; 859249423Sdim case MVT::i16: 860263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; 861249423Sdim break; 862249423Sdim case MVT::i32: 863263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar; 864249423Sdim break; 865263508Sdim case MVT::i64: 866263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar; 867263508Sdim break; 868249423Sdim case MVT::f32: 869263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar; 870249423Sdim break; 871263508Sdim case MVT::f64: 872263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar; 873263508Sdim break; 874249423Sdim } 875249423Sdim break; 876263508Sdim case NVPTXISD::LDGV4: 877263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 878249423Sdim default: 879249423Sdim return NULL; 880249423Sdim case MVT::i8: 881263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; 882249423Sdim break; 883249423Sdim case MVT::i16: 884263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar; 885249423Sdim break; 886249423Sdim case MVT::i32: 887263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar; 888249423Sdim break; 889249423Sdim case MVT::f32: 890263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar; 891249423Sdim break; 892249423Sdim } 893249423Sdim break; 894249423Sdim case NVPTXISD::LDUV4: 895263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 896249423Sdim default: 897249423Sdim return NULL; 898249423Sdim case MVT::i8: 899263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; 900249423Sdim break; 901249423Sdim case MVT::i16: 902263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar; 903249423Sdim break; 904249423Sdim case MVT::i32: 905263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar; 906249423Sdim break; 907249423Sdim case MVT::f32: 908263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar; 909249423Sdim break; 910249423Sdim } 911249423Sdim break; 912249423Sdim } 913263508Sdim 914263508Sdim SDValue Ops[] = { Addr, Chain }; 915263508Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 916263508Sdim ArrayRef<SDValue>(Ops, 2)); 917263508Sdim } else if (Subtarget.is64Bit() 918263508Sdim ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 919263508Sdim : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 920263508Sdim if (Subtarget.is64Bit()) { 921263508Sdim switch (N->getOpcode()) { 922249423Sdim default: 923249423Sdim return NULL; 924263508Sdim case NVPTXISD::LDGV2: 925263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 926263508Sdim default: 927263508Sdim return NULL; 928263508Sdim case MVT::i8: 929263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; 930263508Sdim break; 931263508Sdim case MVT::i16: 932263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; 933263508Sdim break; 934263508Sdim case MVT::i32: 935263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64; 936263508Sdim break; 937263508Sdim case MVT::i64: 938263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64; 939263508Sdim break; 940263508Sdim case MVT::f32: 941263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64; 942263508Sdim break; 943263508Sdim case MVT::f64: 944263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64; 945263508Sdim break; 946263508Sdim } 947249423Sdim break; 948263508Sdim case NVPTXISD::LDUV2: 949263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 950263508Sdim default: 951263508Sdim return NULL; 952263508Sdim case MVT::i8: 953263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; 954263508Sdim break; 955263508Sdim case MVT::i16: 956263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; 957263508Sdim break; 958263508Sdim case MVT::i32: 959263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64; 960263508Sdim break; 961263508Sdim case MVT::i64: 962263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64; 963263508Sdim break; 964263508Sdim case MVT::f32: 965263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64; 966263508Sdim break; 967263508Sdim case MVT::f64: 968263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64; 969263508Sdim break; 970263508Sdim } 971249423Sdim break; 972263508Sdim case NVPTXISD::LDGV4: 973263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 974263508Sdim default: 975263508Sdim return NULL; 976263508Sdim case MVT::i8: 977263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; 978263508Sdim break; 979263508Sdim case MVT::i16: 980263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64; 981263508Sdim break; 982263508Sdim case MVT::i32: 983263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64; 984263508Sdim break; 985263508Sdim case MVT::f32: 986263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64; 987263508Sdim break; 988263508Sdim } 989249423Sdim break; 990263508Sdim case NVPTXISD::LDUV4: 991263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 992263508Sdim default: 993263508Sdim return NULL; 994263508Sdim case MVT::i8: 995263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; 996263508Sdim break; 997263508Sdim case MVT::i16: 998263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64; 999263508Sdim break; 1000263508Sdim case MVT::i32: 1001263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64; 1002263508Sdim break; 1003263508Sdim case MVT::f32: 1004263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64; 1005263508Sdim break; 1006263508Sdim } 1007249423Sdim break; 1008249423Sdim } 1009263508Sdim } else { 1010263508Sdim switch (N->getOpcode()) { 1011249423Sdim default: 1012249423Sdim return NULL; 1013263508Sdim case NVPTXISD::LDGV2: 1014263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1015263508Sdim default: 1016263508Sdim return NULL; 1017263508Sdim case MVT::i8: 1018263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; 1019263508Sdim break; 1020263508Sdim case MVT::i16: 1021263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; 1022263508Sdim break; 1023263508Sdim case MVT::i32: 1024263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32; 1025263508Sdim break; 1026263508Sdim case MVT::i64: 1027263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32; 1028263508Sdim break; 1029263508Sdim case MVT::f32: 1030263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32; 1031263508Sdim break; 1032263508Sdim case MVT::f64: 1033263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32; 1034263508Sdim break; 1035263508Sdim } 1036249423Sdim break; 1037263508Sdim case NVPTXISD::LDUV2: 1038263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1039263508Sdim default: 1040263508Sdim return NULL; 1041263508Sdim case MVT::i8: 1042263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; 1043263508Sdim break; 1044263508Sdim case MVT::i16: 1045263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; 1046263508Sdim break; 1047263508Sdim case MVT::i32: 1048263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32; 1049263508Sdim break; 1050263508Sdim case MVT::i64: 1051263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32; 1052263508Sdim break; 1053263508Sdim case MVT::f32: 1054263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32; 1055263508Sdim break; 1056263508Sdim case MVT::f64: 1057263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32; 1058263508Sdim break; 1059263508Sdim } 1060249423Sdim break; 1061263508Sdim case NVPTXISD::LDGV4: 1062263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1063263508Sdim default: 1064263508Sdim return NULL; 1065263508Sdim case MVT::i8: 1066263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; 1067263508Sdim break; 1068263508Sdim case MVT::i16: 1069263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32; 1070263508Sdim break; 1071263508Sdim case MVT::i32: 1072263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32; 1073263508Sdim break; 1074263508Sdim case MVT::f32: 1075263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32; 1076263508Sdim break; 1077263508Sdim } 1078249423Sdim break; 1079263508Sdim case NVPTXISD::LDUV4: 1080263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1081263508Sdim default: 1082263508Sdim return NULL; 1083263508Sdim case MVT::i8: 1084263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; 1085263508Sdim break; 1086263508Sdim case MVT::i16: 1087263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32; 1088263508Sdim break; 1089263508Sdim case MVT::i32: 1090263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32; 1091263508Sdim break; 1092263508Sdim case MVT::f32: 1093263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32; 1094263508Sdim break; 1095263508Sdim } 1096249423Sdim break; 1097249423Sdim } 1098263508Sdim } 1099263508Sdim 1100263508Sdim SDValue Ops[] = { Base, Offset, Chain }; 1101263508Sdim 1102263508Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 1103263508Sdim ArrayRef<SDValue>(Ops, 3)); 1104263508Sdim } else { 1105263508Sdim if (Subtarget.is64Bit()) { 1106263508Sdim switch (N->getOpcode()) { 1107249423Sdim default: 1108249423Sdim return NULL; 1109263508Sdim case NVPTXISD::LDGV2: 1110263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1111263508Sdim default: 1112263508Sdim return NULL; 1113263508Sdim case MVT::i8: 1114263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; 1115263508Sdim break; 1116263508Sdim case MVT::i16: 1117263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; 1118263508Sdim break; 1119263508Sdim case MVT::i32: 1120263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64; 1121263508Sdim break; 1122263508Sdim case MVT::i64: 1123263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64; 1124263508Sdim break; 1125263508Sdim case MVT::f32: 1126263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64; 1127263508Sdim break; 1128263508Sdim case MVT::f64: 1129263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64; 1130263508Sdim break; 1131263508Sdim } 1132249423Sdim break; 1133263508Sdim case NVPTXISD::LDUV2: 1134263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1135263508Sdim default: 1136263508Sdim return NULL; 1137263508Sdim case MVT::i8: 1138263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; 1139263508Sdim break; 1140263508Sdim case MVT::i16: 1141263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; 1142263508Sdim break; 1143263508Sdim case MVT::i32: 1144263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64; 1145263508Sdim break; 1146263508Sdim case MVT::i64: 1147263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64; 1148263508Sdim break; 1149263508Sdim case MVT::f32: 1150263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64; 1151263508Sdim break; 1152263508Sdim case MVT::f64: 1153263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64; 1154263508Sdim break; 1155263508Sdim } 1156249423Sdim break; 1157263508Sdim case NVPTXISD::LDGV4: 1158263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1159263508Sdim default: 1160263508Sdim return NULL; 1161263508Sdim case MVT::i8: 1162263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; 1163263508Sdim break; 1164263508Sdim case MVT::i16: 1165263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64; 1166263508Sdim break; 1167263508Sdim case MVT::i32: 1168263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64; 1169263508Sdim break; 1170263508Sdim case MVT::f32: 1171263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64; 1172263508Sdim break; 1173263508Sdim } 1174249423Sdim break; 1175263508Sdim case NVPTXISD::LDUV4: 1176263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1177263508Sdim default: 1178263508Sdim return NULL; 1179263508Sdim case MVT::i8: 1180263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; 1181263508Sdim break; 1182263508Sdim case MVT::i16: 1183263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64; 1184263508Sdim break; 1185263508Sdim case MVT::i32: 1186263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64; 1187263508Sdim break; 1188263508Sdim case MVT::f32: 1189263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64; 1190263508Sdim break; 1191263508Sdim } 1192249423Sdim break; 1193249423Sdim } 1194263508Sdim } else { 1195263508Sdim switch (N->getOpcode()) { 1196249423Sdim default: 1197249423Sdim return NULL; 1198263508Sdim case NVPTXISD::LDGV2: 1199263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1200263508Sdim default: 1201263508Sdim return NULL; 1202263508Sdim case MVT::i8: 1203263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; 1204263508Sdim break; 1205263508Sdim case MVT::i16: 1206263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; 1207263508Sdim break; 1208263508Sdim case MVT::i32: 1209263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32; 1210263508Sdim break; 1211263508Sdim case MVT::i64: 1212263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32; 1213263508Sdim break; 1214263508Sdim case MVT::f32: 1215263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32; 1216263508Sdim break; 1217263508Sdim case MVT::f64: 1218263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32; 1219263508Sdim break; 1220263508Sdim } 1221249423Sdim break; 1222263508Sdim case NVPTXISD::LDUV2: 1223263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1224263508Sdim default: 1225263508Sdim return NULL; 1226263508Sdim case MVT::i8: 1227263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; 1228263508Sdim break; 1229263508Sdim case MVT::i16: 1230263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; 1231263508Sdim break; 1232263508Sdim case MVT::i32: 1233263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32; 1234263508Sdim break; 1235263508Sdim case MVT::i64: 1236263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32; 1237263508Sdim break; 1238263508Sdim case MVT::f32: 1239263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32; 1240263508Sdim break; 1241263508Sdim case MVT::f64: 1242263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32; 1243263508Sdim break; 1244263508Sdim } 1245249423Sdim break; 1246263508Sdim case NVPTXISD::LDGV4: 1247263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1248263508Sdim default: 1249263508Sdim return NULL; 1250263508Sdim case MVT::i8: 1251263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; 1252263508Sdim break; 1253263508Sdim case MVT::i16: 1254263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32; 1255263508Sdim break; 1256263508Sdim case MVT::i32: 1257263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32; 1258263508Sdim break; 1259263508Sdim case MVT::f32: 1260263508Sdim Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32; 1261263508Sdim break; 1262263508Sdim } 1263249423Sdim break; 1264263508Sdim case NVPTXISD::LDUV4: 1265263508Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1266263508Sdim default: 1267263508Sdim return NULL; 1268263508Sdim case MVT::i8: 1269263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; 1270263508Sdim break; 1271263508Sdim case MVT::i16: 1272263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32; 1273263508Sdim break; 1274263508Sdim case MVT::i32: 1275263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32; 1276263508Sdim break; 1277263508Sdim case MVT::f32: 1278263508Sdim Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32; 1279263508Sdim break; 1280263508Sdim } 1281249423Sdim break; 1282249423Sdim } 1283249423Sdim } 1284263508Sdim 1285263508Sdim SDValue Ops[] = { Op1, Chain }; 1286263508Sdim LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 1287263508Sdim ArrayRef<SDValue>(Ops, 2)); 1288249423Sdim } 1289249423Sdim 1290249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1291249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1292249423Sdim cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1293249423Sdim 1294249423Sdim return LD; 1295249423Sdim} 1296249423Sdim 1297249423SdimSDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 1298263508Sdim SDLoc dl(N); 1299239310Sdim StoreSDNode *ST = cast<StoreSDNode>(N); 1300239310Sdim EVT StoreVT = ST->getMemoryVT(); 1301239310Sdim SDNode *NVPTXST = NULL; 1302239310Sdim 1303239310Sdim // do not support pre/post inc/dec 1304239310Sdim if (ST->isIndexed()) 1305239310Sdim return NULL; 1306239310Sdim 1307239310Sdim if (!StoreVT.isSimple()) 1308239310Sdim return NULL; 1309239310Sdim 1310239310Sdim // Address Space Setting 1311239310Sdim unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget); 1312239310Sdim 1313239310Sdim // Volatile Setting 1314239310Sdim // - .volatile is only availalble for .global and .shared 1315239310Sdim bool isVolatile = ST->isVolatile(); 1316239310Sdim if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1317239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1318239310Sdim codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1319239310Sdim isVolatile = false; 1320239310Sdim 1321239310Sdim // Vector Setting 1322239310Sdim MVT SimpleVT = StoreVT.getSimpleVT(); 1323239310Sdim unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 1324239310Sdim if (SimpleVT.isVector()) { 1325239310Sdim unsigned num = SimpleVT.getVectorNumElements(); 1326239310Sdim if (num == 2) 1327239310Sdim vecType = NVPTX::PTXLdStInstCode::V2; 1328239310Sdim else if (num == 4) 1329239310Sdim vecType = NVPTX::PTXLdStInstCode::V4; 1330239310Sdim else 1331239310Sdim return NULL; 1332239310Sdim } 1333239310Sdim 1334239310Sdim // Type Setting: toType + toTypeWidth 1335239310Sdim // - for integer type, always use 'u' 1336239310Sdim // 1337239310Sdim MVT ScalarVT = SimpleVT.getScalarType(); 1338249423Sdim unsigned toTypeWidth = ScalarVT.getSizeInBits(); 1339239310Sdim unsigned int toType; 1340239310Sdim if (ScalarVT.isFloatingPoint()) 1341239310Sdim toType = NVPTX::PTXLdStInstCode::Float; 1342239310Sdim else 1343239310Sdim toType = NVPTX::PTXLdStInstCode::Unsigned; 1344239310Sdim 1345239310Sdim // Create the machine instruction DAG 1346239310Sdim SDValue Chain = N->getOperand(0); 1347239310Sdim SDValue N1 = N->getOperand(1); 1348239310Sdim SDValue N2 = N->getOperand(2); 1349239310Sdim SDValue Addr; 1350239310Sdim SDValue Offset, Base; 1351239310Sdim unsigned Opcode; 1352263508Sdim MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy; 1353239310Sdim 1354239310Sdim if (SelectDirectAddr(N2, Addr)) { 1355239310Sdim switch (SourceVT) { 1356249423Sdim case MVT::i8: 1357249423Sdim Opcode = NVPTX::ST_i8_avar; 1358249423Sdim break; 1359249423Sdim case MVT::i16: 1360249423Sdim Opcode = NVPTX::ST_i16_avar; 1361249423Sdim break; 1362249423Sdim case MVT::i32: 1363249423Sdim Opcode = NVPTX::ST_i32_avar; 1364249423Sdim break; 1365249423Sdim case MVT::i64: 1366249423Sdim Opcode = NVPTX::ST_i64_avar; 1367249423Sdim break; 1368249423Sdim case MVT::f32: 1369249423Sdim Opcode = NVPTX::ST_f32_avar; 1370249423Sdim break; 1371249423Sdim case MVT::f64: 1372249423Sdim Opcode = NVPTX::ST_f64_avar; 1373249423Sdim break; 1374249423Sdim default: 1375249423Sdim return NULL; 1376239310Sdim } 1377249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1378249423Sdim getI32Imm(vecType), getI32Imm(toType), 1379249423Sdim getI32Imm(toTypeWidth), Addr, Chain }; 1380251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1381249423Sdim } else if (Subtarget.is64Bit() 1382249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1383249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1384239310Sdim switch (SourceVT) { 1385249423Sdim case MVT::i8: 1386249423Sdim Opcode = NVPTX::ST_i8_asi; 1387249423Sdim break; 1388249423Sdim case MVT::i16: 1389249423Sdim Opcode = NVPTX::ST_i16_asi; 1390249423Sdim break; 1391249423Sdim case MVT::i32: 1392249423Sdim Opcode = NVPTX::ST_i32_asi; 1393249423Sdim break; 1394249423Sdim case MVT::i64: 1395249423Sdim Opcode = NVPTX::ST_i64_asi; 1396249423Sdim break; 1397249423Sdim case MVT::f32: 1398249423Sdim Opcode = NVPTX::ST_f32_asi; 1399249423Sdim break; 1400249423Sdim case MVT::f64: 1401249423Sdim Opcode = NVPTX::ST_f64_asi; 1402249423Sdim break; 1403249423Sdim default: 1404249423Sdim return NULL; 1405239310Sdim } 1406249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1407249423Sdim getI32Imm(vecType), getI32Imm(toType), 1408249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 1409251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1410249423Sdim } else if (Subtarget.is64Bit() 1411249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1412249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1413249423Sdim if (Subtarget.is64Bit()) { 1414249423Sdim switch (SourceVT) { 1415249423Sdim case MVT::i8: 1416249423Sdim Opcode = NVPTX::ST_i8_ari_64; 1417249423Sdim break; 1418249423Sdim case MVT::i16: 1419249423Sdim Opcode = NVPTX::ST_i16_ari_64; 1420249423Sdim break; 1421249423Sdim case MVT::i32: 1422249423Sdim Opcode = NVPTX::ST_i32_ari_64; 1423249423Sdim break; 1424249423Sdim case MVT::i64: 1425249423Sdim Opcode = NVPTX::ST_i64_ari_64; 1426249423Sdim break; 1427249423Sdim case MVT::f32: 1428249423Sdim Opcode = NVPTX::ST_f32_ari_64; 1429249423Sdim break; 1430249423Sdim case MVT::f64: 1431249423Sdim Opcode = NVPTX::ST_f64_ari_64; 1432249423Sdim break; 1433249423Sdim default: 1434249423Sdim return NULL; 1435249423Sdim } 1436249423Sdim } else { 1437249423Sdim switch (SourceVT) { 1438249423Sdim case MVT::i8: 1439249423Sdim Opcode = NVPTX::ST_i8_ari; 1440249423Sdim break; 1441249423Sdim case MVT::i16: 1442249423Sdim Opcode = NVPTX::ST_i16_ari; 1443249423Sdim break; 1444249423Sdim case MVT::i32: 1445249423Sdim Opcode = NVPTX::ST_i32_ari; 1446249423Sdim break; 1447249423Sdim case MVT::i64: 1448249423Sdim Opcode = NVPTX::ST_i64_ari; 1449249423Sdim break; 1450249423Sdim case MVT::f32: 1451249423Sdim Opcode = NVPTX::ST_f32_ari; 1452249423Sdim break; 1453249423Sdim case MVT::f64: 1454249423Sdim Opcode = NVPTX::ST_f64_ari; 1455249423Sdim break; 1456249423Sdim default: 1457249423Sdim return NULL; 1458249423Sdim } 1459239310Sdim } 1460249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1461249423Sdim getI32Imm(vecType), getI32Imm(toType), 1462249423Sdim getI32Imm(toTypeWidth), Base, Offset, Chain }; 1463251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1464239310Sdim } else { 1465249423Sdim if (Subtarget.is64Bit()) { 1466249423Sdim switch (SourceVT) { 1467249423Sdim case MVT::i8: 1468249423Sdim Opcode = NVPTX::ST_i8_areg_64; 1469249423Sdim break; 1470249423Sdim case MVT::i16: 1471249423Sdim Opcode = NVPTX::ST_i16_areg_64; 1472249423Sdim break; 1473249423Sdim case MVT::i32: 1474249423Sdim Opcode = NVPTX::ST_i32_areg_64; 1475249423Sdim break; 1476249423Sdim case MVT::i64: 1477249423Sdim Opcode = NVPTX::ST_i64_areg_64; 1478249423Sdim break; 1479249423Sdim case MVT::f32: 1480249423Sdim Opcode = NVPTX::ST_f32_areg_64; 1481249423Sdim break; 1482249423Sdim case MVT::f64: 1483249423Sdim Opcode = NVPTX::ST_f64_areg_64; 1484249423Sdim break; 1485249423Sdim default: 1486249423Sdim return NULL; 1487249423Sdim } 1488249423Sdim } else { 1489249423Sdim switch (SourceVT) { 1490249423Sdim case MVT::i8: 1491249423Sdim Opcode = NVPTX::ST_i8_areg; 1492249423Sdim break; 1493249423Sdim case MVT::i16: 1494249423Sdim Opcode = NVPTX::ST_i16_areg; 1495249423Sdim break; 1496249423Sdim case MVT::i32: 1497249423Sdim Opcode = NVPTX::ST_i32_areg; 1498249423Sdim break; 1499249423Sdim case MVT::i64: 1500249423Sdim Opcode = NVPTX::ST_i64_areg; 1501249423Sdim break; 1502249423Sdim case MVT::f32: 1503249423Sdim Opcode = NVPTX::ST_f32_areg; 1504249423Sdim break; 1505249423Sdim case MVT::f64: 1506249423Sdim Opcode = NVPTX::ST_f64_areg; 1507249423Sdim break; 1508249423Sdim default: 1509249423Sdim return NULL; 1510249423Sdim } 1511239310Sdim } 1512249423Sdim SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1513249423Sdim getI32Imm(vecType), getI32Imm(toType), 1514249423Sdim getI32Imm(toTypeWidth), N2, Chain }; 1515251662Sdim NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1516239310Sdim } 1517239310Sdim 1518239310Sdim if (NVPTXST != NULL) { 1519239310Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1520239310Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1521239310Sdim cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1522239310Sdim } 1523239310Sdim 1524239310Sdim return NVPTXST; 1525239310Sdim} 1526239310Sdim 1527249423SdimSDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { 1528249423Sdim SDValue Chain = N->getOperand(0); 1529249423Sdim SDValue Op1 = N->getOperand(1); 1530249423Sdim SDValue Addr, Offset, Base; 1531249423Sdim unsigned Opcode; 1532263508Sdim SDLoc DL(N); 1533249423Sdim SDNode *ST; 1534249423Sdim EVT EltVT = Op1.getValueType(); 1535249423Sdim MemSDNode *MemSD = cast<MemSDNode>(N); 1536249423Sdim EVT StoreVT = MemSD->getMemoryVT(); 1537249423Sdim 1538249423Sdim // Address Space Setting 1539249423Sdim unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 1540249423Sdim 1541249423Sdim if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { 1542249423Sdim report_fatal_error("Cannot store to pointer that points to constant " 1543249423Sdim "memory space"); 1544249423Sdim } 1545249423Sdim 1546249423Sdim // Volatile Setting 1547249423Sdim // - .volatile is only availalble for .global and .shared 1548249423Sdim bool IsVolatile = MemSD->isVolatile(); 1549249423Sdim if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1550249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1551249423Sdim CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1552249423Sdim IsVolatile = false; 1553249423Sdim 1554249423Sdim // Type Setting: toType + toTypeWidth 1555249423Sdim // - for integer type, always use 'u' 1556249423Sdim assert(StoreVT.isSimple() && "Store value is not simple"); 1557249423Sdim MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); 1558249423Sdim unsigned ToTypeWidth = ScalarVT.getSizeInBits(); 1559249423Sdim unsigned ToType; 1560249423Sdim if (ScalarVT.isFloatingPoint()) 1561249423Sdim ToType = NVPTX::PTXLdStInstCode::Float; 1562249423Sdim else 1563249423Sdim ToType = NVPTX::PTXLdStInstCode::Unsigned; 1564249423Sdim 1565249423Sdim SmallVector<SDValue, 12> StOps; 1566249423Sdim SDValue N2; 1567249423Sdim unsigned VecType; 1568249423Sdim 1569249423Sdim switch (N->getOpcode()) { 1570249423Sdim case NVPTXISD::StoreV2: 1571249423Sdim VecType = NVPTX::PTXLdStInstCode::V2; 1572249423Sdim StOps.push_back(N->getOperand(1)); 1573249423Sdim StOps.push_back(N->getOperand(2)); 1574249423Sdim N2 = N->getOperand(3); 1575249423Sdim break; 1576249423Sdim case NVPTXISD::StoreV4: 1577249423Sdim VecType = NVPTX::PTXLdStInstCode::V4; 1578249423Sdim StOps.push_back(N->getOperand(1)); 1579249423Sdim StOps.push_back(N->getOperand(2)); 1580249423Sdim StOps.push_back(N->getOperand(3)); 1581249423Sdim StOps.push_back(N->getOperand(4)); 1582249423Sdim N2 = N->getOperand(5); 1583249423Sdim break; 1584249423Sdim default: 1585249423Sdim return NULL; 1586249423Sdim } 1587249423Sdim 1588249423Sdim StOps.push_back(getI32Imm(IsVolatile)); 1589249423Sdim StOps.push_back(getI32Imm(CodeAddrSpace)); 1590249423Sdim StOps.push_back(getI32Imm(VecType)); 1591249423Sdim StOps.push_back(getI32Imm(ToType)); 1592249423Sdim StOps.push_back(getI32Imm(ToTypeWidth)); 1593249423Sdim 1594249423Sdim if (SelectDirectAddr(N2, Addr)) { 1595249423Sdim switch (N->getOpcode()) { 1596249423Sdim default: 1597249423Sdim return NULL; 1598249423Sdim case NVPTXISD::StoreV2: 1599249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1600249423Sdim default: 1601249423Sdim return NULL; 1602249423Sdim case MVT::i8: 1603249423Sdim Opcode = NVPTX::STV_i8_v2_avar; 1604249423Sdim break; 1605249423Sdim case MVT::i16: 1606249423Sdim Opcode = NVPTX::STV_i16_v2_avar; 1607249423Sdim break; 1608249423Sdim case MVT::i32: 1609249423Sdim Opcode = NVPTX::STV_i32_v2_avar; 1610249423Sdim break; 1611249423Sdim case MVT::i64: 1612249423Sdim Opcode = NVPTX::STV_i64_v2_avar; 1613249423Sdim break; 1614249423Sdim case MVT::f32: 1615249423Sdim Opcode = NVPTX::STV_f32_v2_avar; 1616249423Sdim break; 1617249423Sdim case MVT::f64: 1618249423Sdim Opcode = NVPTX::STV_f64_v2_avar; 1619249423Sdim break; 1620249423Sdim } 1621249423Sdim break; 1622249423Sdim case NVPTXISD::StoreV4: 1623249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1624249423Sdim default: 1625249423Sdim return NULL; 1626249423Sdim case MVT::i8: 1627249423Sdim Opcode = NVPTX::STV_i8_v4_avar; 1628249423Sdim break; 1629249423Sdim case MVT::i16: 1630249423Sdim Opcode = NVPTX::STV_i16_v4_avar; 1631249423Sdim break; 1632249423Sdim case MVT::i32: 1633249423Sdim Opcode = NVPTX::STV_i32_v4_avar; 1634249423Sdim break; 1635249423Sdim case MVT::f32: 1636249423Sdim Opcode = NVPTX::STV_f32_v4_avar; 1637249423Sdim break; 1638249423Sdim } 1639249423Sdim break; 1640249423Sdim } 1641249423Sdim StOps.push_back(Addr); 1642249423Sdim } else if (Subtarget.is64Bit() 1643249423Sdim ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1644249423Sdim : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1645249423Sdim switch (N->getOpcode()) { 1646249423Sdim default: 1647249423Sdim return NULL; 1648249423Sdim case NVPTXISD::StoreV2: 1649249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1650249423Sdim default: 1651249423Sdim return NULL; 1652249423Sdim case MVT::i8: 1653249423Sdim Opcode = NVPTX::STV_i8_v2_asi; 1654249423Sdim break; 1655249423Sdim case MVT::i16: 1656249423Sdim Opcode = NVPTX::STV_i16_v2_asi; 1657249423Sdim break; 1658249423Sdim case MVT::i32: 1659249423Sdim Opcode = NVPTX::STV_i32_v2_asi; 1660249423Sdim break; 1661249423Sdim case MVT::i64: 1662249423Sdim Opcode = NVPTX::STV_i64_v2_asi; 1663249423Sdim break; 1664249423Sdim case MVT::f32: 1665249423Sdim Opcode = NVPTX::STV_f32_v2_asi; 1666249423Sdim break; 1667249423Sdim case MVT::f64: 1668249423Sdim Opcode = NVPTX::STV_f64_v2_asi; 1669249423Sdim break; 1670249423Sdim } 1671249423Sdim break; 1672249423Sdim case NVPTXISD::StoreV4: 1673249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1674249423Sdim default: 1675249423Sdim return NULL; 1676249423Sdim case MVT::i8: 1677249423Sdim Opcode = NVPTX::STV_i8_v4_asi; 1678249423Sdim break; 1679249423Sdim case MVT::i16: 1680249423Sdim Opcode = NVPTX::STV_i16_v4_asi; 1681249423Sdim break; 1682249423Sdim case MVT::i32: 1683249423Sdim Opcode = NVPTX::STV_i32_v4_asi; 1684249423Sdim break; 1685249423Sdim case MVT::f32: 1686249423Sdim Opcode = NVPTX::STV_f32_v4_asi; 1687249423Sdim break; 1688249423Sdim } 1689249423Sdim break; 1690249423Sdim } 1691249423Sdim StOps.push_back(Base); 1692249423Sdim StOps.push_back(Offset); 1693249423Sdim } else if (Subtarget.is64Bit() 1694249423Sdim ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1695249423Sdim : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1696249423Sdim if (Subtarget.is64Bit()) { 1697249423Sdim switch (N->getOpcode()) { 1698249423Sdim default: 1699249423Sdim return NULL; 1700249423Sdim case NVPTXISD::StoreV2: 1701249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1702249423Sdim default: 1703249423Sdim return NULL; 1704249423Sdim case MVT::i8: 1705249423Sdim Opcode = NVPTX::STV_i8_v2_ari_64; 1706249423Sdim break; 1707249423Sdim case MVT::i16: 1708249423Sdim Opcode = NVPTX::STV_i16_v2_ari_64; 1709249423Sdim break; 1710249423Sdim case MVT::i32: 1711249423Sdim Opcode = NVPTX::STV_i32_v2_ari_64; 1712249423Sdim break; 1713249423Sdim case MVT::i64: 1714249423Sdim Opcode = NVPTX::STV_i64_v2_ari_64; 1715249423Sdim break; 1716249423Sdim case MVT::f32: 1717249423Sdim Opcode = NVPTX::STV_f32_v2_ari_64; 1718249423Sdim break; 1719249423Sdim case MVT::f64: 1720249423Sdim Opcode = NVPTX::STV_f64_v2_ari_64; 1721249423Sdim break; 1722249423Sdim } 1723249423Sdim break; 1724249423Sdim case NVPTXISD::StoreV4: 1725249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1726249423Sdim default: 1727249423Sdim return NULL; 1728249423Sdim case MVT::i8: 1729249423Sdim Opcode = NVPTX::STV_i8_v4_ari_64; 1730249423Sdim break; 1731249423Sdim case MVT::i16: 1732249423Sdim Opcode = NVPTX::STV_i16_v4_ari_64; 1733249423Sdim break; 1734249423Sdim case MVT::i32: 1735249423Sdim Opcode = NVPTX::STV_i32_v4_ari_64; 1736249423Sdim break; 1737249423Sdim case MVT::f32: 1738249423Sdim Opcode = NVPTX::STV_f32_v4_ari_64; 1739249423Sdim break; 1740249423Sdim } 1741249423Sdim break; 1742249423Sdim } 1743249423Sdim } else { 1744249423Sdim switch (N->getOpcode()) { 1745249423Sdim default: 1746249423Sdim return NULL; 1747249423Sdim case NVPTXISD::StoreV2: 1748249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1749249423Sdim default: 1750249423Sdim return NULL; 1751249423Sdim case MVT::i8: 1752249423Sdim Opcode = NVPTX::STV_i8_v2_ari; 1753249423Sdim break; 1754249423Sdim case MVT::i16: 1755249423Sdim Opcode = NVPTX::STV_i16_v2_ari; 1756249423Sdim break; 1757249423Sdim case MVT::i32: 1758249423Sdim Opcode = NVPTX::STV_i32_v2_ari; 1759249423Sdim break; 1760249423Sdim case MVT::i64: 1761249423Sdim Opcode = NVPTX::STV_i64_v2_ari; 1762249423Sdim break; 1763249423Sdim case MVT::f32: 1764249423Sdim Opcode = NVPTX::STV_f32_v2_ari; 1765249423Sdim break; 1766249423Sdim case MVT::f64: 1767249423Sdim Opcode = NVPTX::STV_f64_v2_ari; 1768249423Sdim break; 1769249423Sdim } 1770249423Sdim break; 1771249423Sdim case NVPTXISD::StoreV4: 1772249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1773249423Sdim default: 1774249423Sdim return NULL; 1775249423Sdim case MVT::i8: 1776249423Sdim Opcode = NVPTX::STV_i8_v4_ari; 1777249423Sdim break; 1778249423Sdim case MVT::i16: 1779249423Sdim Opcode = NVPTX::STV_i16_v4_ari; 1780249423Sdim break; 1781249423Sdim case MVT::i32: 1782249423Sdim Opcode = NVPTX::STV_i32_v4_ari; 1783249423Sdim break; 1784249423Sdim case MVT::f32: 1785249423Sdim Opcode = NVPTX::STV_f32_v4_ari; 1786249423Sdim break; 1787249423Sdim } 1788249423Sdim break; 1789249423Sdim } 1790249423Sdim } 1791249423Sdim StOps.push_back(Base); 1792249423Sdim StOps.push_back(Offset); 1793249423Sdim } else { 1794249423Sdim if (Subtarget.is64Bit()) { 1795249423Sdim switch (N->getOpcode()) { 1796249423Sdim default: 1797249423Sdim return NULL; 1798249423Sdim case NVPTXISD::StoreV2: 1799249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1800249423Sdim default: 1801249423Sdim return NULL; 1802249423Sdim case MVT::i8: 1803249423Sdim Opcode = NVPTX::STV_i8_v2_areg_64; 1804249423Sdim break; 1805249423Sdim case MVT::i16: 1806249423Sdim Opcode = NVPTX::STV_i16_v2_areg_64; 1807249423Sdim break; 1808249423Sdim case MVT::i32: 1809249423Sdim Opcode = NVPTX::STV_i32_v2_areg_64; 1810249423Sdim break; 1811249423Sdim case MVT::i64: 1812249423Sdim Opcode = NVPTX::STV_i64_v2_areg_64; 1813249423Sdim break; 1814249423Sdim case MVT::f32: 1815249423Sdim Opcode = NVPTX::STV_f32_v2_areg_64; 1816249423Sdim break; 1817249423Sdim case MVT::f64: 1818249423Sdim Opcode = NVPTX::STV_f64_v2_areg_64; 1819249423Sdim break; 1820249423Sdim } 1821249423Sdim break; 1822249423Sdim case NVPTXISD::StoreV4: 1823249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1824249423Sdim default: 1825249423Sdim return NULL; 1826249423Sdim case MVT::i8: 1827249423Sdim Opcode = NVPTX::STV_i8_v4_areg_64; 1828249423Sdim break; 1829249423Sdim case MVT::i16: 1830249423Sdim Opcode = NVPTX::STV_i16_v4_areg_64; 1831249423Sdim break; 1832249423Sdim case MVT::i32: 1833249423Sdim Opcode = NVPTX::STV_i32_v4_areg_64; 1834249423Sdim break; 1835249423Sdim case MVT::f32: 1836249423Sdim Opcode = NVPTX::STV_f32_v4_areg_64; 1837249423Sdim break; 1838249423Sdim } 1839249423Sdim break; 1840249423Sdim } 1841249423Sdim } else { 1842249423Sdim switch (N->getOpcode()) { 1843249423Sdim default: 1844249423Sdim return NULL; 1845249423Sdim case NVPTXISD::StoreV2: 1846249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1847249423Sdim default: 1848249423Sdim return NULL; 1849249423Sdim case MVT::i8: 1850249423Sdim Opcode = NVPTX::STV_i8_v2_areg; 1851249423Sdim break; 1852249423Sdim case MVT::i16: 1853249423Sdim Opcode = NVPTX::STV_i16_v2_areg; 1854249423Sdim break; 1855249423Sdim case MVT::i32: 1856249423Sdim Opcode = NVPTX::STV_i32_v2_areg; 1857249423Sdim break; 1858249423Sdim case MVT::i64: 1859249423Sdim Opcode = NVPTX::STV_i64_v2_areg; 1860249423Sdim break; 1861249423Sdim case MVT::f32: 1862249423Sdim Opcode = NVPTX::STV_f32_v2_areg; 1863249423Sdim break; 1864249423Sdim case MVT::f64: 1865249423Sdim Opcode = NVPTX::STV_f64_v2_areg; 1866249423Sdim break; 1867249423Sdim } 1868249423Sdim break; 1869249423Sdim case NVPTXISD::StoreV4: 1870249423Sdim switch (EltVT.getSimpleVT().SimpleTy) { 1871249423Sdim default: 1872249423Sdim return NULL; 1873249423Sdim case MVT::i8: 1874249423Sdim Opcode = NVPTX::STV_i8_v4_areg; 1875249423Sdim break; 1876249423Sdim case MVT::i16: 1877249423Sdim Opcode = NVPTX::STV_i16_v4_areg; 1878249423Sdim break; 1879249423Sdim case MVT::i32: 1880249423Sdim Opcode = NVPTX::STV_i32_v4_areg; 1881249423Sdim break; 1882249423Sdim case MVT::f32: 1883249423Sdim Opcode = NVPTX::STV_f32_v4_areg; 1884249423Sdim break; 1885249423Sdim } 1886249423Sdim break; 1887249423Sdim } 1888249423Sdim } 1889249423Sdim StOps.push_back(N2); 1890249423Sdim } 1891249423Sdim 1892249423Sdim StOps.push_back(Chain); 1893249423Sdim 1894251662Sdim ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); 1895249423Sdim 1896249423Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1897249423Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1898249423Sdim cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1899249423Sdim 1900249423Sdim return ST; 1901249423Sdim} 1902249423Sdim 1903263508SdimSDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) { 1904263508Sdim SDValue Chain = Node->getOperand(0); 1905263508Sdim SDValue Offset = Node->getOperand(2); 1906263508Sdim SDValue Flag = Node->getOperand(3); 1907263508Sdim SDLoc DL(Node); 1908263508Sdim MemSDNode *Mem = cast<MemSDNode>(Node); 1909263508Sdim 1910263508Sdim unsigned VecSize; 1911263508Sdim switch (Node->getOpcode()) { 1912263508Sdim default: 1913263508Sdim return NULL; 1914263508Sdim case NVPTXISD::LoadParam: 1915263508Sdim VecSize = 1; 1916263508Sdim break; 1917263508Sdim case NVPTXISD::LoadParamV2: 1918263508Sdim VecSize = 2; 1919263508Sdim break; 1920263508Sdim case NVPTXISD::LoadParamV4: 1921263508Sdim VecSize = 4; 1922263508Sdim break; 1923263508Sdim } 1924263508Sdim 1925263508Sdim EVT EltVT = Node->getValueType(0); 1926263508Sdim EVT MemVT = Mem->getMemoryVT(); 1927263508Sdim 1928263508Sdim unsigned Opc = 0; 1929263508Sdim 1930263508Sdim switch (VecSize) { 1931263508Sdim default: 1932263508Sdim return NULL; 1933263508Sdim case 1: 1934263508Sdim switch (MemVT.getSimpleVT().SimpleTy) { 1935263508Sdim default: 1936263508Sdim return NULL; 1937263508Sdim case MVT::i1: 1938263508Sdim Opc = NVPTX::LoadParamMemI8; 1939263508Sdim break; 1940263508Sdim case MVT::i8: 1941263508Sdim Opc = NVPTX::LoadParamMemI8; 1942263508Sdim break; 1943263508Sdim case MVT::i16: 1944263508Sdim Opc = NVPTX::LoadParamMemI16; 1945263508Sdim break; 1946263508Sdim case MVT::i32: 1947263508Sdim Opc = NVPTX::LoadParamMemI32; 1948263508Sdim break; 1949263508Sdim case MVT::i64: 1950263508Sdim Opc = NVPTX::LoadParamMemI64; 1951263508Sdim break; 1952263508Sdim case MVT::f32: 1953263508Sdim Opc = NVPTX::LoadParamMemF32; 1954263508Sdim break; 1955263508Sdim case MVT::f64: 1956263508Sdim Opc = NVPTX::LoadParamMemF64; 1957263508Sdim break; 1958263508Sdim } 1959263508Sdim break; 1960263508Sdim case 2: 1961263508Sdim switch (MemVT.getSimpleVT().SimpleTy) { 1962263508Sdim default: 1963263508Sdim return NULL; 1964263508Sdim case MVT::i1: 1965263508Sdim Opc = NVPTX::LoadParamMemV2I8; 1966263508Sdim break; 1967263508Sdim case MVT::i8: 1968263508Sdim Opc = NVPTX::LoadParamMemV2I8; 1969263508Sdim break; 1970263508Sdim case MVT::i16: 1971263508Sdim Opc = NVPTX::LoadParamMemV2I16; 1972263508Sdim break; 1973263508Sdim case MVT::i32: 1974263508Sdim Opc = NVPTX::LoadParamMemV2I32; 1975263508Sdim break; 1976263508Sdim case MVT::i64: 1977263508Sdim Opc = NVPTX::LoadParamMemV2I64; 1978263508Sdim break; 1979263508Sdim case MVT::f32: 1980263508Sdim Opc = NVPTX::LoadParamMemV2F32; 1981263508Sdim break; 1982263508Sdim case MVT::f64: 1983263508Sdim Opc = NVPTX::LoadParamMemV2F64; 1984263508Sdim break; 1985263508Sdim } 1986263508Sdim break; 1987263508Sdim case 4: 1988263508Sdim switch (MemVT.getSimpleVT().SimpleTy) { 1989263508Sdim default: 1990263508Sdim return NULL; 1991263508Sdim case MVT::i1: 1992263508Sdim Opc = NVPTX::LoadParamMemV4I8; 1993263508Sdim break; 1994263508Sdim case MVT::i8: 1995263508Sdim Opc = NVPTX::LoadParamMemV4I8; 1996263508Sdim break; 1997263508Sdim case MVT::i16: 1998263508Sdim Opc = NVPTX::LoadParamMemV4I16; 1999263508Sdim break; 2000263508Sdim case MVT::i32: 2001263508Sdim Opc = NVPTX::LoadParamMemV4I32; 2002263508Sdim break; 2003263508Sdim case MVT::f32: 2004263508Sdim Opc = NVPTX::LoadParamMemV4F32; 2005263508Sdim break; 2006263508Sdim } 2007263508Sdim break; 2008263508Sdim } 2009263508Sdim 2010263508Sdim SDVTList VTs; 2011263508Sdim if (VecSize == 1) { 2012263508Sdim VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); 2013263508Sdim } else if (VecSize == 2) { 2014263508Sdim VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); 2015263508Sdim } else { 2016263508Sdim EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; 2017263508Sdim VTs = CurDAG->getVTList(&EVTs[0], 5); 2018263508Sdim } 2019263508Sdim 2020263508Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2021263508Sdim 2022263508Sdim SmallVector<SDValue, 2> Ops; 2023263508Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2024263508Sdim Ops.push_back(Chain); 2025263508Sdim Ops.push_back(Flag); 2026263508Sdim 2027263508Sdim SDNode *Ret = 2028263508Sdim CurDAG->getMachineNode(Opc, DL, VTs, Ops); 2029263508Sdim return Ret; 2030263508Sdim} 2031263508Sdim 2032263508SdimSDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) { 2033263508Sdim SDLoc DL(N); 2034263508Sdim SDValue Chain = N->getOperand(0); 2035263508Sdim SDValue Offset = N->getOperand(1); 2036263508Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2037263508Sdim MemSDNode *Mem = cast<MemSDNode>(N); 2038263508Sdim 2039263508Sdim // How many elements do we have? 2040263508Sdim unsigned NumElts = 1; 2041263508Sdim switch (N->getOpcode()) { 2042263508Sdim default: 2043263508Sdim return NULL; 2044263508Sdim case NVPTXISD::StoreRetval: 2045263508Sdim NumElts = 1; 2046263508Sdim break; 2047263508Sdim case NVPTXISD::StoreRetvalV2: 2048263508Sdim NumElts = 2; 2049263508Sdim break; 2050263508Sdim case NVPTXISD::StoreRetvalV4: 2051263508Sdim NumElts = 4; 2052263508Sdim break; 2053263508Sdim } 2054263508Sdim 2055263508Sdim // Build vector of operands 2056263508Sdim SmallVector<SDValue, 6> Ops; 2057263508Sdim for (unsigned i = 0; i < NumElts; ++i) 2058263508Sdim Ops.push_back(N->getOperand(i + 2)); 2059263508Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2060263508Sdim Ops.push_back(Chain); 2061263508Sdim 2062263508Sdim // Determine target opcode 2063263508Sdim // If we have an i1, use an 8-bit store. The lowering code in 2064263508Sdim // NVPTXISelLowering will have already emitted an upcast. 2065263508Sdim unsigned Opcode = 0; 2066263508Sdim switch (NumElts) { 2067263508Sdim default: 2068263508Sdim return NULL; 2069263508Sdim case 1: 2070263508Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2071263508Sdim default: 2072263508Sdim return NULL; 2073263508Sdim case MVT::i1: 2074263508Sdim Opcode = NVPTX::StoreRetvalI8; 2075263508Sdim break; 2076263508Sdim case MVT::i8: 2077263508Sdim Opcode = NVPTX::StoreRetvalI8; 2078263508Sdim break; 2079263508Sdim case MVT::i16: 2080263508Sdim Opcode = NVPTX::StoreRetvalI16; 2081263508Sdim break; 2082263508Sdim case MVT::i32: 2083263508Sdim Opcode = NVPTX::StoreRetvalI32; 2084263508Sdim break; 2085263508Sdim case MVT::i64: 2086263508Sdim Opcode = NVPTX::StoreRetvalI64; 2087263508Sdim break; 2088263508Sdim case MVT::f32: 2089263508Sdim Opcode = NVPTX::StoreRetvalF32; 2090263508Sdim break; 2091263508Sdim case MVT::f64: 2092263508Sdim Opcode = NVPTX::StoreRetvalF64; 2093263508Sdim break; 2094263508Sdim } 2095263508Sdim break; 2096263508Sdim case 2: 2097263508Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2098263508Sdim default: 2099263508Sdim return NULL; 2100263508Sdim case MVT::i1: 2101263508Sdim Opcode = NVPTX::StoreRetvalV2I8; 2102263508Sdim break; 2103263508Sdim case MVT::i8: 2104263508Sdim Opcode = NVPTX::StoreRetvalV2I8; 2105263508Sdim break; 2106263508Sdim case MVT::i16: 2107263508Sdim Opcode = NVPTX::StoreRetvalV2I16; 2108263508Sdim break; 2109263508Sdim case MVT::i32: 2110263508Sdim Opcode = NVPTX::StoreRetvalV2I32; 2111263508Sdim break; 2112263508Sdim case MVT::i64: 2113263508Sdim Opcode = NVPTX::StoreRetvalV2I64; 2114263508Sdim break; 2115263508Sdim case MVT::f32: 2116263508Sdim Opcode = NVPTX::StoreRetvalV2F32; 2117263508Sdim break; 2118263508Sdim case MVT::f64: 2119263508Sdim Opcode = NVPTX::StoreRetvalV2F64; 2120263508Sdim break; 2121263508Sdim } 2122263508Sdim break; 2123263508Sdim case 4: 2124263508Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2125263508Sdim default: 2126263508Sdim return NULL; 2127263508Sdim case MVT::i1: 2128263508Sdim Opcode = NVPTX::StoreRetvalV4I8; 2129263508Sdim break; 2130263508Sdim case MVT::i8: 2131263508Sdim Opcode = NVPTX::StoreRetvalV4I8; 2132263508Sdim break; 2133263508Sdim case MVT::i16: 2134263508Sdim Opcode = NVPTX::StoreRetvalV4I16; 2135263508Sdim break; 2136263508Sdim case MVT::i32: 2137263508Sdim Opcode = NVPTX::StoreRetvalV4I32; 2138263508Sdim break; 2139263508Sdim case MVT::f32: 2140263508Sdim Opcode = NVPTX::StoreRetvalV4F32; 2141263508Sdim break; 2142263508Sdim } 2143263508Sdim break; 2144263508Sdim } 2145263508Sdim 2146263508Sdim SDNode *Ret = 2147263508Sdim CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 2148263508Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2149263508Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2150263508Sdim cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2151263508Sdim 2152263508Sdim return Ret; 2153263508Sdim} 2154263508Sdim 2155263508SdimSDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) { 2156263508Sdim SDLoc DL(N); 2157263508Sdim SDValue Chain = N->getOperand(0); 2158263508Sdim SDValue Param = N->getOperand(1); 2159263508Sdim unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue(); 2160263508Sdim SDValue Offset = N->getOperand(2); 2161263508Sdim unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2162263508Sdim MemSDNode *Mem = cast<MemSDNode>(N); 2163263508Sdim SDValue Flag = N->getOperand(N->getNumOperands() - 1); 2164263508Sdim 2165263508Sdim // How many elements do we have? 2166263508Sdim unsigned NumElts = 1; 2167263508Sdim switch (N->getOpcode()) { 2168263508Sdim default: 2169263508Sdim return NULL; 2170263508Sdim case NVPTXISD::StoreParamU32: 2171263508Sdim case NVPTXISD::StoreParamS32: 2172263508Sdim case NVPTXISD::StoreParam: 2173263508Sdim NumElts = 1; 2174263508Sdim break; 2175263508Sdim case NVPTXISD::StoreParamV2: 2176263508Sdim NumElts = 2; 2177263508Sdim break; 2178263508Sdim case NVPTXISD::StoreParamV4: 2179263508Sdim NumElts = 4; 2180263508Sdim break; 2181263508Sdim } 2182263508Sdim 2183263508Sdim // Build vector of operands 2184263508Sdim SmallVector<SDValue, 8> Ops; 2185263508Sdim for (unsigned i = 0; i < NumElts; ++i) 2186263508Sdim Ops.push_back(N->getOperand(i + 3)); 2187263508Sdim Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32)); 2188263508Sdim Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2189263508Sdim Ops.push_back(Chain); 2190263508Sdim Ops.push_back(Flag); 2191263508Sdim 2192263508Sdim // Determine target opcode 2193263508Sdim // If we have an i1, use an 8-bit store. The lowering code in 2194263508Sdim // NVPTXISelLowering will have already emitted an upcast. 2195263508Sdim unsigned Opcode = 0; 2196263508Sdim switch (N->getOpcode()) { 2197263508Sdim default: 2198263508Sdim switch (NumElts) { 2199263508Sdim default: 2200263508Sdim return NULL; 2201263508Sdim case 1: 2202263508Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2203263508Sdim default: 2204263508Sdim return NULL; 2205263508Sdim case MVT::i1: 2206263508Sdim Opcode = NVPTX::StoreParamI8; 2207263508Sdim break; 2208263508Sdim case MVT::i8: 2209263508Sdim Opcode = NVPTX::StoreParamI8; 2210263508Sdim break; 2211263508Sdim case MVT::i16: 2212263508Sdim Opcode = NVPTX::StoreParamI16; 2213263508Sdim break; 2214263508Sdim case MVT::i32: 2215263508Sdim Opcode = NVPTX::StoreParamI32; 2216263508Sdim break; 2217263508Sdim case MVT::i64: 2218263508Sdim Opcode = NVPTX::StoreParamI64; 2219263508Sdim break; 2220263508Sdim case MVT::f32: 2221263508Sdim Opcode = NVPTX::StoreParamF32; 2222263508Sdim break; 2223263508Sdim case MVT::f64: 2224263508Sdim Opcode = NVPTX::StoreParamF64; 2225263508Sdim break; 2226263508Sdim } 2227263508Sdim break; 2228263508Sdim case 2: 2229263508Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2230263508Sdim default: 2231263508Sdim return NULL; 2232263508Sdim case MVT::i1: 2233263508Sdim Opcode = NVPTX::StoreParamV2I8; 2234263508Sdim break; 2235263508Sdim case MVT::i8: 2236263508Sdim Opcode = NVPTX::StoreParamV2I8; 2237263508Sdim break; 2238263508Sdim case MVT::i16: 2239263508Sdim Opcode = NVPTX::StoreParamV2I16; 2240263508Sdim break; 2241263508Sdim case MVT::i32: 2242263508Sdim Opcode = NVPTX::StoreParamV2I32; 2243263508Sdim break; 2244263508Sdim case MVT::i64: 2245263508Sdim Opcode = NVPTX::StoreParamV2I64; 2246263508Sdim break; 2247263508Sdim case MVT::f32: 2248263508Sdim Opcode = NVPTX::StoreParamV2F32; 2249263508Sdim break; 2250263508Sdim case MVT::f64: 2251263508Sdim Opcode = NVPTX::StoreParamV2F64; 2252263508Sdim break; 2253263508Sdim } 2254263508Sdim break; 2255263508Sdim case 4: 2256263508Sdim switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2257263508Sdim default: 2258263508Sdim return NULL; 2259263508Sdim case MVT::i1: 2260263508Sdim Opcode = NVPTX::StoreParamV4I8; 2261263508Sdim break; 2262263508Sdim case MVT::i8: 2263263508Sdim Opcode = NVPTX::StoreParamV4I8; 2264263508Sdim break; 2265263508Sdim case MVT::i16: 2266263508Sdim Opcode = NVPTX::StoreParamV4I16; 2267263508Sdim break; 2268263508Sdim case MVT::i32: 2269263508Sdim Opcode = NVPTX::StoreParamV4I32; 2270263508Sdim break; 2271263508Sdim case MVT::f32: 2272263508Sdim Opcode = NVPTX::StoreParamV4F32; 2273263508Sdim break; 2274263508Sdim } 2275263508Sdim break; 2276263508Sdim } 2277263508Sdim break; 2278263508Sdim // Special case: if we have a sign-extend/zero-extend node, insert the 2279263508Sdim // conversion instruction first, and use that as the value operand to 2280263508Sdim // the selected StoreParam node. 2281263508Sdim case NVPTXISD::StoreParamU32: { 2282263508Sdim Opcode = NVPTX::StoreParamI32; 2283263508Sdim SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2284263508Sdim MVT::i32); 2285263508Sdim SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL, 2286263508Sdim MVT::i32, Ops[0], CvtNone); 2287263508Sdim Ops[0] = SDValue(Cvt, 0); 2288263508Sdim break; 2289263508Sdim } 2290263508Sdim case NVPTXISD::StoreParamS32: { 2291263508Sdim Opcode = NVPTX::StoreParamI32; 2292263508Sdim SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2293263508Sdim MVT::i32); 2294263508Sdim SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL, 2295263508Sdim MVT::i32, Ops[0], CvtNone); 2296263508Sdim Ops[0] = SDValue(Cvt, 0); 2297263508Sdim break; 2298263508Sdim } 2299263508Sdim } 2300263508Sdim 2301263508Sdim SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 2302263508Sdim SDNode *Ret = 2303263508Sdim CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); 2304263508Sdim MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2305263508Sdim MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2306263508Sdim cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2307263508Sdim 2308263508Sdim return Ret; 2309263508Sdim} 2310263508Sdim 2311239310Sdim// SelectDirectAddr - Match a direct address for DAG. 2312239310Sdim// A direct address could be a globaladdress or externalsymbol. 2313239310Sdimbool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { 2314239310Sdim // Return true if TGA or ES. 2315249423Sdim if (N.getOpcode() == ISD::TargetGlobalAddress || 2316249423Sdim N.getOpcode() == ISD::TargetExternalSymbol) { 2317239310Sdim Address = N; 2318239310Sdim return true; 2319239310Sdim } 2320239310Sdim if (N.getOpcode() == NVPTXISD::Wrapper) { 2321239310Sdim Address = N.getOperand(0); 2322239310Sdim return true; 2323239310Sdim } 2324239310Sdim if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 2325239310Sdim unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); 2326239310Sdim if (IID == Intrinsic::nvvm_ptr_gen_to_param) 2327239310Sdim if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) 2328239310Sdim return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); 2329239310Sdim } 2330239310Sdim return false; 2331239310Sdim} 2332239310Sdim 2333239310Sdim// symbol+offset 2334249423Sdimbool NVPTXDAGToDAGISel::SelectADDRsi_imp( 2335249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 2336239310Sdim if (Addr.getOpcode() == ISD::ADD) { 2337239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 2338249423Sdim SDValue base = Addr.getOperand(0); 2339239310Sdim if (SelectDirectAddr(base, Base)) { 2340239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 2341239310Sdim return true; 2342239310Sdim } 2343239310Sdim } 2344239310Sdim } 2345239310Sdim return false; 2346239310Sdim} 2347239310Sdim 2348239310Sdim// symbol+offset 2349239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, 2350239310Sdim SDValue &Base, SDValue &Offset) { 2351239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); 2352239310Sdim} 2353239310Sdim 2354239310Sdim// symbol+offset 2355239310Sdimbool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, 2356239310Sdim SDValue &Base, SDValue &Offset) { 2357239310Sdim return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); 2358239310Sdim} 2359239310Sdim 2360239310Sdim// register+offset 2361249423Sdimbool NVPTXDAGToDAGISel::SelectADDRri_imp( 2362249423Sdim SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 2363239310Sdim if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 2364239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 2365239310Sdim Offset = CurDAG->getTargetConstant(0, mvt); 2366239310Sdim return true; 2367239310Sdim } 2368239310Sdim if (Addr.getOpcode() == ISD::TargetExternalSymbol || 2369239310Sdim Addr.getOpcode() == ISD::TargetGlobalAddress) 2370249423Sdim return false; // direct calls. 2371239310Sdim 2372239310Sdim if (Addr.getOpcode() == ISD::ADD) { 2373239310Sdim if (SelectDirectAddr(Addr.getOperand(0), Addr)) { 2374239310Sdim return false; 2375239310Sdim } 2376239310Sdim if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 2377239310Sdim if (FrameIndexSDNode *FIN = 2378249423Sdim dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) 2379239310Sdim // Constant offset from frame ref. 2380239310Sdim Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 2381239310Sdim else 2382239310Sdim Base = Addr.getOperand(0); 2383239310Sdim Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 2384239310Sdim return true; 2385239310Sdim } 2386239310Sdim } 2387239310Sdim return false; 2388239310Sdim} 2389239310Sdim 2390239310Sdim// register+offset 2391239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, 2392239310Sdim SDValue &Base, SDValue &Offset) { 2393239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); 2394239310Sdim} 2395239310Sdim 2396239310Sdim// register+offset 2397239310Sdimbool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, 2398239310Sdim SDValue &Base, SDValue &Offset) { 2399239310Sdim return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 2400239310Sdim} 2401239310Sdim 2402239310Sdimbool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 2403239310Sdim unsigned int spN) const { 2404239310Sdim const Value *Src = NULL; 2405239310Sdim // Even though MemIntrinsicSDNode is a subclas of MemSDNode, 2406239310Sdim // the classof() for MemSDNode does not include MemIntrinsicSDNode 2407239310Sdim // (See SelectionDAGNodes.h). So we need to check for both. 2408239310Sdim if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 2409239310Sdim Src = mN->getSrcValue(); 2410249423Sdim } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) { 2411239310Sdim Src = mN->getSrcValue(); 2412239310Sdim } 2413239310Sdim if (!Src) 2414239310Sdim return false; 2415239310Sdim if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) 2416239310Sdim return (PT->getAddressSpace() == spN); 2417239310Sdim return false; 2418239310Sdim} 2419239310Sdim 2420239310Sdim/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 2421239310Sdim/// inline asm expressions. 2422249423Sdimbool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( 2423249423Sdim const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) { 2424239310Sdim SDValue Op0, Op1; 2425239310Sdim switch (ConstraintCode) { 2426249423Sdim default: 2427249423Sdim return true; 2428249423Sdim case 'm': // memory 2429239310Sdim if (SelectDirectAddr(Op, Op0)) { 2430239310Sdim OutOps.push_back(Op0); 2431239310Sdim OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32)); 2432239310Sdim return false; 2433239310Sdim } 2434239310Sdim if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { 2435239310Sdim OutOps.push_back(Op0); 2436239310Sdim OutOps.push_back(Op1); 2437239310Sdim return false; 2438239310Sdim } 2439239310Sdim break; 2440239310Sdim } 2441239310Sdim return true; 2442239310Sdim} 2443239310Sdim 2444239310Sdim// Return true if N is a undef or a constant. 2445239310Sdim// If N was undef, return a (i8imm 0) in Retval 2446239310Sdim// If N was imm, convert it to i8imm and return in Retval 2447239310Sdim// Note: The convert to i8imm is required, otherwise the 2448239310Sdim// pattern matcher inserts a bunch of IMOVi8rr to convert 2449239310Sdim// the imm to i8imm, and this causes instruction selection 2450239310Sdim// to fail. 2451249423Sdimbool NVPTXDAGToDAGISel::UndefOrImm(SDValue Op, SDValue N, SDValue &Retval) { 2452249423Sdim if (!(N.getOpcode() == ISD::UNDEF) && !(N.getOpcode() == ISD::Constant)) 2453239310Sdim return false; 2454239310Sdim 2455239310Sdim if (N.getOpcode() == ISD::UNDEF) 2456239310Sdim Retval = CurDAG->getTargetConstant(0, MVT::i8); 2457239310Sdim else { 2458239310Sdim ConstantSDNode *cn = cast<ConstantSDNode>(N.getNode()); 2459239310Sdim unsigned retval = cn->getZExtValue(); 2460239310Sdim Retval = CurDAG->getTargetConstant(retval, MVT::i8); 2461239310Sdim } 2462239310Sdim return true; 2463239310Sdim} 2464