1193323Sed//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file performs vector type splitting and scalarization for LegalizeTypes. 11193323Sed// Scalarization is the act of changing a computation in an illegal one-element 12193323Sed// vector type to be a computation in its scalar element type. For example, 13193323Sed// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 14193323Sed// as a base case when scalarizing vector arithmetic like <4 x f32>, which 15193323Sed// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 16193323Sed// types. 17193323Sed// Splitting is the act of changing a computation in an invalid vector type to 18198090Srdivacky// be a computation in two vectors of half the size. For example, implementing 19198090Srdivacky// <128 x f32> operations in terms of two <64 x f32> operations. 20193323Sed// 21193323Sed//===----------------------------------------------------------------------===// 22193323Sed 23193323Sed#include "LegalizeTypes.h" 24249423Sdim#include "llvm/IR/DataLayout.h" 25198090Srdivacky#include "llvm/Support/ErrorHandling.h" 26198090Srdivacky#include "llvm/Support/raw_ostream.h" 27193323Sedusing namespace llvm; 28193323Sed 29193323Sed//===----------------------------------------------------------------------===// 30193323Sed// Result Vector Scalarization: <1 x ty> -> ty. 31193323Sed//===----------------------------------------------------------------------===// 32193323Sed 33193323Sedvoid DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 34202375Srdivacky DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; 35198090Srdivacky N->dump(&DAG); 36202375Srdivacky dbgs() << "\n"); 37193323Sed SDValue R = SDValue(); 38193323Sed 39193323Sed switch (N->getOpcode()) { 40193323Sed default: 41193323Sed#ifndef NDEBUG 42202375Srdivacky dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 43198090Srdivacky N->dump(&DAG); 44202375Srdivacky dbgs() << "\n"; 45193323Sed#endif 46226633Sdim report_fatal_error("Do not know how to scalarize the result of this " 47226633Sdim "operator!\n"); 48193323Sed 49226633Sdim case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 50218893Sdim case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 51239462Sdim case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 52193323Sed case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break; 53193323Sed case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 54221345Sdim case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 55202375Srdivacky case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; 56193323Sed case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 57193323Sed case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 58193323Sed case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 59193323Sed case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 60202375Srdivacky case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 61234353Sdim case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 62193323Sed case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 63193323Sed case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 64198090Srdivacky case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 65193323Sed case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 66193323Sed case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 67221345Sdim case ISD::ANY_EXTEND: 68193323Sed case ISD::CTLZ: 69193323Sed case ISD::CTPOP: 70193323Sed case ISD::CTTZ: 71193323Sed case ISD::FABS: 72221345Sdim case ISD::FCEIL: 73193323Sed case ISD::FCOS: 74221345Sdim case ISD::FEXP: 75221345Sdim case ISD::FEXP2: 76221345Sdim case ISD::FFLOOR: 77221345Sdim case ISD::FLOG: 78221345Sdim case ISD::FLOG10: 79221345Sdim case ISD::FLOG2: 80221345Sdim case ISD::FNEARBYINT: 81193323Sed case ISD::FNEG: 82221345Sdim case ISD::FP_EXTEND: 83193323Sed case ISD::FP_TO_SINT: 84193323Sed case ISD::FP_TO_UINT: 85221345Sdim case ISD::FRINT: 86193323Sed case ISD::FSIN: 87193323Sed case ISD::FSQRT: 88193323Sed case ISD::FTRUNC: 89221345Sdim case ISD::SIGN_EXTEND: 90193323Sed case ISD::SINT_TO_FP: 91193323Sed case ISD::TRUNCATE: 92221345Sdim case ISD::UINT_TO_FP: 93198090Srdivacky case ISD::ZERO_EXTEND: 94198090Srdivacky R = ScalarizeVecRes_UnaryOp(N); 95198090Srdivacky break; 96193323Sed 97193323Sed case ISD::ADD: 98193323Sed case ISD::AND: 99193323Sed case ISD::FADD: 100193323Sed case ISD::FDIV: 101193323Sed case ISD::FMUL: 102193323Sed case ISD::FPOW: 103193323Sed case ISD::FREM: 104193323Sed case ISD::FSUB: 105193323Sed case ISD::MUL: 106193323Sed case ISD::OR: 107193323Sed case ISD::SDIV: 108193323Sed case ISD::SREM: 109193323Sed case ISD::SUB: 110193323Sed case ISD::UDIV: 111193323Sed case ISD::UREM: 112198090Srdivacky case ISD::XOR: 113193323Sed case ISD::SHL: 114193323Sed case ISD::SRA: 115198090Srdivacky case ISD::SRL: 116198090Srdivacky R = ScalarizeVecRes_BinOp(N); 117198090Srdivacky break; 118239462Sdim case ISD::FMA: 119239462Sdim R = ScalarizeVecRes_TernaryOp(N); 120239462Sdim break; 121193323Sed } 122193323Sed 123193323Sed // If R is null, the sub-method took care of registering the result. 124193323Sed if (R.getNode()) 125193323Sed SetScalarizedVector(SDValue(N, ResNo), R); 126193323Sed} 127193323Sed 128193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 129193323Sed SDValue LHS = GetScalarizedVector(N->getOperand(0)); 130193323Sed SDValue RHS = GetScalarizedVector(N->getOperand(1)); 131193323Sed return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 132193323Sed LHS.getValueType(), LHS, RHS); 133193323Sed} 134193323Sed 135239462SdimSDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 136239462Sdim SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 137239462Sdim SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 138239462Sdim SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 139239462Sdim return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 140239462Sdim Op0.getValueType(), Op0, Op1, Op2); 141239462Sdim} 142239462Sdim 143226633SdimSDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 144226633Sdim unsigned ResNo) { 145226633Sdim SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 146226633Sdim return GetScalarizedVector(Op); 147226633Sdim} 148226633Sdim 149218893SdimSDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 150198090Srdivacky EVT NewVT = N->getValueType(0).getVectorElementType(); 151218893Sdim return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), 152193323Sed NewVT, N->getOperand(0)); 153193323Sed} 154193323Sed 155239462SdimSDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 156239462Sdim EVT EltVT = N->getValueType(0).getVectorElementType(); 157239462Sdim SDValue InOp = N->getOperand(0); 158239462Sdim // The BUILD_VECTOR operands may be of wider element types and 159239462Sdim // we may need to truncate them back to the requested return type. 160239462Sdim if (EltVT.isInteger()) 161239462Sdim return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 162239462Sdim return InOp; 163239462Sdim} 164239462Sdim 165193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) { 166198090Srdivacky EVT NewVT = N->getValueType(0).getVectorElementType(); 167193323Sed SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 168193323Sed return DAG.getConvertRndSat(NewVT, N->getDebugLoc(), 169193323Sed Op0, DAG.getValueType(NewVT), 170193323Sed DAG.getValueType(Op0.getValueType()), 171193323Sed N->getOperand(3), 172193323Sed N->getOperand(4), 173193323Sed cast<CvtRndSatSDNode>(N)->getCvtCode()); 174193323Sed} 175193323Sed 176193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 177193323Sed return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 178193323Sed N->getValueType(0).getVectorElementType(), 179193323Sed N->getOperand(0), N->getOperand(1)); 180193323Sed} 181193323Sed 182221345SdimSDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 183221345Sdim EVT NewVT = N->getValueType(0).getVectorElementType(); 184221345Sdim SDValue Op = GetScalarizedVector(N->getOperand(0)); 185221345Sdim return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), 186221345Sdim NewVT, Op, N->getOperand(1)); 187221345Sdim} 188221345Sdim 189193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 190193323Sed SDValue Op = GetScalarizedVector(N->getOperand(0)); 191193323Sed return DAG.getNode(ISD::FPOWI, N->getDebugLoc(), 192193323Sed Op.getValueType(), Op, N->getOperand(1)); 193193323Sed} 194193323Sed 195193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 196193323Sed // The value to insert may have a wider type than the vector element type, 197193323Sed // so be sure to truncate it to the element type if necessary. 198193323Sed SDValue Op = N->getOperand(1); 199198090Srdivacky EVT EltVT = N->getValueType(0).getVectorElementType(); 200193323Sed if (Op.getValueType() != EltVT) 201193323Sed // FIXME: Can this happen for floating point types? 202193323Sed Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op); 203193323Sed return Op; 204193323Sed} 205193323Sed 206193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 207193323Sed assert(N->isUnindexed() && "Indexed vector load?"); 208193323Sed 209210299Sed SDValue Result = DAG.getLoad(ISD::UNINDEXED, 210193323Sed N->getExtensionType(), 211193323Sed N->getValueType(0).getVectorElementType(), 212210299Sed N->getDebugLoc(), 213193323Sed N->getChain(), N->getBasePtr(), 214193323Sed DAG.getUNDEF(N->getBasePtr().getValueType()), 215218893Sdim N->getPointerInfo(), 216193323Sed N->getMemoryVT().getVectorElementType(), 217203954Srdivacky N->isVolatile(), N->isNonTemporal(), 218234353Sdim N->isInvariant(), N->getOriginalAlignment()); 219193323Sed 220193323Sed // Legalized the chain result - switch anything that used the old chain to 221193323Sed // use the new one. 222193323Sed ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 223193323Sed return Result; 224193323Sed} 225193323Sed 226193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 227193323Sed // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 228198090Srdivacky EVT DestVT = N->getValueType(0).getVectorElementType(); 229193323Sed SDValue Op = GetScalarizedVector(N->getOperand(0)); 230193323Sed return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op); 231193323Sed} 232193323Sed 233202375SrdivackySDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 234202375Srdivacky EVT EltVT = N->getValueType(0).getVectorElementType(); 235202375Srdivacky EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 236202375Srdivacky SDValue LHS = GetScalarizedVector(N->getOperand(0)); 237202375Srdivacky return DAG.getNode(N->getOpcode(), N->getDebugLoc(), EltVT, 238202375Srdivacky LHS, DAG.getValueType(ExtVT)); 239202375Srdivacky} 240202375Srdivacky 241193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 242193323Sed // If the operand is wider than the vector element type then it is implicitly 243193323Sed // truncated. Make that explicit here. 244198090Srdivacky EVT EltVT = N->getValueType(0).getVectorElementType(); 245193323Sed SDValue InOp = N->getOperand(0); 246193323Sed if (InOp.getValueType() != EltVT) 247193323Sed return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp); 248193323Sed return InOp; 249193323Sed} 250193323Sed 251234353SdimSDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 252234353Sdim SDValue Cond = GetScalarizedVector(N->getOperand(0)); 253234353Sdim SDValue LHS = GetScalarizedVector(N->getOperand(1)); 254234353Sdim TargetLowering::BooleanContent ScalarBool = TLI.getBooleanContents(false); 255234353Sdim TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true); 256234353Sdim if (ScalarBool != VecBool) { 257234353Sdim EVT CondVT = Cond.getValueType(); 258234353Sdim switch (ScalarBool) { 259234353Sdim case TargetLowering::UndefinedBooleanContent: 260234353Sdim break; 261234353Sdim case TargetLowering::ZeroOrOneBooleanContent: 262234353Sdim assert(VecBool == TargetLowering::UndefinedBooleanContent || 263234353Sdim VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 264234353Sdim // Vector read from all ones, scalar expects a single 1 so mask. 265234353Sdim Cond = DAG.getNode(ISD::AND, N->getDebugLoc(), CondVT, 266234353Sdim Cond, DAG.getConstant(1, CondVT)); 267234353Sdim break; 268234353Sdim case TargetLowering::ZeroOrNegativeOneBooleanContent: 269234353Sdim assert(VecBool == TargetLowering::UndefinedBooleanContent || 270234353Sdim VecBool == TargetLowering::ZeroOrOneBooleanContent); 271234353Sdim // Vector reads from a one, scalar from all ones so sign extend. 272234353Sdim Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, N->getDebugLoc(), CondVT, 273234353Sdim Cond, DAG.getValueType(MVT::i1)); 274234353Sdim break; 275234353Sdim } 276234353Sdim } 277234353Sdim return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 278234353Sdim LHS.getValueType(), Cond, LHS, 279234353Sdim GetScalarizedVector(N->getOperand(2))); 280234353Sdim} 281234353Sdim 282193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 283193323Sed SDValue LHS = GetScalarizedVector(N->getOperand(1)); 284193323Sed return DAG.getNode(ISD::SELECT, N->getDebugLoc(), 285193323Sed LHS.getValueType(), N->getOperand(0), LHS, 286193323Sed GetScalarizedVector(N->getOperand(2))); 287193323Sed} 288193323Sed 289193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 290193323Sed SDValue LHS = GetScalarizedVector(N->getOperand(2)); 291193323Sed return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(), 292193323Sed N->getOperand(0), N->getOperand(1), 293193323Sed LHS, GetScalarizedVector(N->getOperand(3)), 294193323Sed N->getOperand(4)); 295193323Sed} 296193323Sed 297198090SrdivackySDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 298226633Sdim assert(N->getValueType(0).isVector() == 299226633Sdim N->getOperand(0).getValueType().isVector() && 300226633Sdim "Scalar/Vector type mismatch"); 301226633Sdim 302226633Sdim if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N); 303226633Sdim 304198090Srdivacky SDValue LHS = GetScalarizedVector(N->getOperand(0)); 305198090Srdivacky SDValue RHS = GetScalarizedVector(N->getOperand(1)); 306198090Srdivacky DebugLoc DL = N->getDebugLoc(); 307198090Srdivacky 308198090Srdivacky // Turn it into a scalar SETCC. 309198090Srdivacky return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2)); 310198090Srdivacky} 311198090Srdivacky 312193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 313193323Sed return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 314193323Sed} 315193323Sed 316193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 317193323Sed // Figure out if the scalar is the LHS or RHS and return it. 318193323Sed SDValue Arg = N->getOperand(2).getOperand(0); 319193323Sed if (Arg.getOpcode() == ISD::UNDEF) 320193323Sed return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 321193323Sed unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 322193323Sed return GetScalarizedVector(N->getOperand(Op)); 323193323Sed} 324193323Sed 325193323SedSDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) { 326226633Sdim assert(N->getValueType(0).isVector() && 327226633Sdim N->getOperand(0).getValueType().isVector() && 328226633Sdim "Operand types must be vectors"); 329226633Sdim 330193323Sed SDValue LHS = GetScalarizedVector(N->getOperand(0)); 331193323Sed SDValue RHS = GetScalarizedVector(N->getOperand(1)); 332198090Srdivacky EVT NVT = N->getValueType(0).getVectorElementType(); 333198090Srdivacky DebugLoc DL = N->getDebugLoc(); 334193323Sed 335193323Sed // Turn it into a scalar SETCC. 336226633Sdim SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 337226633Sdim N->getOperand(2)); 338226633Sdim // Vectors may have a different boolean contents to scalars. Promote the 339226633Sdim // value appropriately. 340226633Sdim ISD::NodeType ExtendCode = 341226633Sdim TargetLowering::getExtendForContent(TLI.getBooleanContents(true)); 342226633Sdim return DAG.getNode(ExtendCode, DL, NVT, Res); 343193323Sed} 344193323Sed 345193323Sed 346193323Sed//===----------------------------------------------------------------------===// 347193323Sed// Operand Vector Scalarization <1 x ty> -> ty. 348193323Sed//===----------------------------------------------------------------------===// 349193323Sed 350193323Sedbool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 351202375Srdivacky DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; 352198090Srdivacky N->dump(&DAG); 353202375Srdivacky dbgs() << "\n"); 354193323Sed SDValue Res = SDValue(); 355193323Sed 356193323Sed if (Res.getNode() == 0) { 357193323Sed switch (N->getOpcode()) { 358193323Sed default: 359193323Sed#ifndef NDEBUG 360202375Srdivacky dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 361198090Srdivacky N->dump(&DAG); 362202375Srdivacky dbgs() << "\n"; 363193323Sed#endif 364198090Srdivacky llvm_unreachable("Do not know how to scalarize this operator's operand!"); 365218893Sdim case ISD::BITCAST: 366218893Sdim Res = ScalarizeVecOp_BITCAST(N); 367198090Srdivacky break; 368249423Sdim case ISD::ANY_EXTEND: 369249423Sdim case ISD::ZERO_EXTEND: 370249423Sdim case ISD::SIGN_EXTEND: 371249423Sdim Res = ScalarizeVecOp_EXTEND(N); 372249423Sdim break; 373193323Sed case ISD::CONCAT_VECTORS: 374198090Srdivacky Res = ScalarizeVecOp_CONCAT_VECTORS(N); 375198090Srdivacky break; 376193323Sed case ISD::EXTRACT_VECTOR_ELT: 377198090Srdivacky Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 378198090Srdivacky break; 379193323Sed case ISD::STORE: 380198090Srdivacky Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 381198090Srdivacky break; 382193323Sed } 383193323Sed } 384193323Sed 385193323Sed // If the result is null, the sub-method took care of registering results etc. 386193323Sed if (!Res.getNode()) return false; 387193323Sed 388193323Sed // If the result is N, the sub-method updated N in place. Tell the legalizer 389193323Sed // core about this. 390193323Sed if (Res.getNode() == N) 391193323Sed return true; 392193323Sed 393193323Sed assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 394193323Sed "Invalid operand expansion"); 395193323Sed 396193323Sed ReplaceValueWith(SDValue(N, 0), Res); 397193323Sed return false; 398193323Sed} 399193323Sed 400218893Sdim/// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs 401193323Sed/// to be scalarized, it must be <1 x ty>. Convert the element instead. 402218893SdimSDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 403193323Sed SDValue Elt = GetScalarizedVector(N->getOperand(0)); 404218893Sdim return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), 405193323Sed N->getValueType(0), Elt); 406193323Sed} 407193323Sed 408249423Sdim/// ScalarizeVecOp_EXTEND - If the value to extend is a vector that needs 409249423Sdim/// to be scalarized, it must be <1 x ty>. Extend the element instead. 410249423SdimSDValue DAGTypeLegalizer::ScalarizeVecOp_EXTEND(SDNode *N) { 411249423Sdim assert(N->getValueType(0).getVectorNumElements() == 1 && 412249423Sdim "Unexected vector type!"); 413249423Sdim SDValue Elt = GetScalarizedVector(N->getOperand(0)); 414249423Sdim SmallVector<SDValue, 1> Ops(1); 415249423Sdim Ops[0] = DAG.getNode(N->getOpcode(), N->getDebugLoc(), 416249423Sdim N->getValueType(0).getScalarType(), Elt); 417249423Sdim // Revectorize the result so the types line up with what the uses of this 418249423Sdim // expression expect. 419249423Sdim return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 420249423Sdim &Ops[0], 1); 421249423Sdim} 422249423Sdim 423193323Sed/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one - 424193323Sed/// use a BUILD_VECTOR instead. 425193323SedSDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 426193323Sed SmallVector<SDValue, 8> Ops(N->getNumOperands()); 427193323Sed for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 428193323Sed Ops[i] = GetScalarizedVector(N->getOperand(i)); 429193323Sed return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0), 430193323Sed &Ops[0], Ops.size()); 431193323Sed} 432193323Sed 433193323Sed/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to 434193323Sed/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the 435193323Sed/// index. 436193323SedSDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 437198090Srdivacky SDValue Res = GetScalarizedVector(N->getOperand(0)); 438198090Srdivacky if (Res.getValueType() != N->getValueType(0)) 439198090Srdivacky Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0), 440198090Srdivacky Res); 441198090Srdivacky return Res; 442193323Sed} 443193323Sed 444193323Sed/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be 445193323Sed/// scalarized, it must be <1 x ty>. Just store the element. 446193323SedSDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 447193323Sed assert(N->isUnindexed() && "Indexed store of one-element vector?"); 448193323Sed assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 449193323Sed DebugLoc dl = N->getDebugLoc(); 450193323Sed 451193323Sed if (N->isTruncatingStore()) 452193323Sed return DAG.getTruncStore(N->getChain(), dl, 453193323Sed GetScalarizedVector(N->getOperand(1)), 454218893Sdim N->getBasePtr(), N->getPointerInfo(), 455193323Sed N->getMemoryVT().getVectorElementType(), 456203954Srdivacky N->isVolatile(), N->isNonTemporal(), 457203954Srdivacky N->getAlignment()); 458193323Sed 459193323Sed return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 460218893Sdim N->getBasePtr(), N->getPointerInfo(), 461203954Srdivacky N->isVolatile(), N->isNonTemporal(), 462203954Srdivacky N->getOriginalAlignment()); 463193323Sed} 464193323Sed 465193323Sed 466193323Sed//===----------------------------------------------------------------------===// 467193323Sed// Result Vector Splitting 468193323Sed//===----------------------------------------------------------------------===// 469193323Sed 470193323Sed/// SplitVectorResult - This method is called when the specified result of the 471193323Sed/// specified node is found to need vector splitting. At this point, the node 472193323Sed/// may also have invalid operands or may have other results that need 473193323Sed/// legalization, we just know that (at least) one result needs vector 474193323Sed/// splitting. 475193323Sedvoid DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 476202375Srdivacky DEBUG(dbgs() << "Split node result: "; 477198090Srdivacky N->dump(&DAG); 478202375Srdivacky dbgs() << "\n"); 479193323Sed SDValue Lo, Hi; 480239462Sdim 481234353Sdim // See if the target wants to custom expand this node. 482234353Sdim if (CustomLowerNode(N, N->getValueType(ResNo), true)) 483234353Sdim return; 484193323Sed 485193323Sed switch (N->getOpcode()) { 486193323Sed default: 487193323Sed#ifndef NDEBUG 488202375Srdivacky dbgs() << "SplitVectorResult #" << ResNo << ": "; 489198090Srdivacky N->dump(&DAG); 490202375Srdivacky dbgs() << "\n"; 491193323Sed#endif 492239462Sdim report_fatal_error("Do not know how to split the result of this " 493239462Sdim "operator!\n"); 494193323Sed 495226633Sdim case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 496226633Sdim case ISD::VSELECT: 497193323Sed case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 498193323Sed case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 499193323Sed case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 500218893Sdim case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 501193323Sed case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 502193323Sed case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 503193323Sed case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 504202375Srdivacky case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 505193323Sed case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 506193323Sed case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 507193323Sed case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 508202375Srdivacky case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 509198090Srdivacky case ISD::LOAD: 510198090Srdivacky SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 511198090Srdivacky break; 512198090Srdivacky case ISD::SETCC: 513198090Srdivacky SplitVecRes_SETCC(N, Lo, Hi); 514198090Srdivacky break; 515193323Sed case ISD::VECTOR_SHUFFLE: 516198090Srdivacky SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 517198090Srdivacky break; 518193323Sed 519221345Sdim case ISD::ANY_EXTEND: 520221345Sdim case ISD::CONVERT_RNDSAT: 521193323Sed case ISD::CTLZ: 522234353Sdim case ISD::CTTZ: 523234353Sdim case ISD::CTLZ_ZERO_UNDEF: 524234353Sdim case ISD::CTTZ_ZERO_UNDEF: 525193323Sed case ISD::CTPOP: 526193323Sed case ISD::FABS: 527221345Sdim case ISD::FCEIL: 528193323Sed case ISD::FCOS: 529221345Sdim case ISD::FEXP: 530221345Sdim case ISD::FEXP2: 531193323Sed case ISD::FFLOOR: 532221345Sdim case ISD::FLOG: 533221345Sdim case ISD::FLOG10: 534221345Sdim case ISD::FLOG2: 535193323Sed case ISD::FNEARBYINT: 536221345Sdim case ISD::FNEG: 537221345Sdim case ISD::FP_EXTEND: 538221345Sdim case ISD::FP_ROUND: 539193323Sed case ISD::FP_TO_SINT: 540193323Sed case ISD::FP_TO_UINT: 541221345Sdim case ISD::FRINT: 542221345Sdim case ISD::FSIN: 543221345Sdim case ISD::FSQRT: 544221345Sdim case ISD::FTRUNC: 545221345Sdim case ISD::SIGN_EXTEND: 546193323Sed case ISD::SINT_TO_FP: 547221345Sdim case ISD::TRUNCATE: 548198090Srdivacky case ISD::UINT_TO_FP: 549198090Srdivacky case ISD::ZERO_EXTEND: 550198090Srdivacky SplitVecRes_UnaryOp(N, Lo, Hi); 551198090Srdivacky break; 552193323Sed 553193323Sed case ISD::ADD: 554193323Sed case ISD::SUB: 555193323Sed case ISD::MUL: 556193323Sed case ISD::FADD: 557193323Sed case ISD::FSUB: 558193323Sed case ISD::FMUL: 559193323Sed case ISD::SDIV: 560193323Sed case ISD::UDIV: 561193323Sed case ISD::FDIV: 562193323Sed case ISD::FPOW: 563193323Sed case ISD::AND: 564193323Sed case ISD::OR: 565193323Sed case ISD::XOR: 566193323Sed case ISD::SHL: 567193323Sed case ISD::SRA: 568193323Sed case ISD::SRL: 569193323Sed case ISD::UREM: 570193323Sed case ISD::SREM: 571198090Srdivacky case ISD::FREM: 572198090Srdivacky SplitVecRes_BinOp(N, Lo, Hi); 573198090Srdivacky break; 574239462Sdim case ISD::FMA: 575239462Sdim SplitVecRes_TernaryOp(N, Lo, Hi); 576239462Sdim break; 577193323Sed } 578193323Sed 579193323Sed // If Lo/Hi is null, the sub-method took care of registering results etc. 580193323Sed if (Lo.getNode()) 581193323Sed SetSplitVector(SDValue(N, ResNo), Lo, Hi); 582193323Sed} 583193323Sed 584193323Sedvoid DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 585193323Sed SDValue &Hi) { 586193323Sed SDValue LHSLo, LHSHi; 587193323Sed GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 588193323Sed SDValue RHSLo, RHSHi; 589193323Sed GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 590193323Sed DebugLoc dl = N->getDebugLoc(); 591193323Sed 592193323Sed Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo); 593193323Sed Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi); 594193323Sed} 595193323Sed 596239462Sdimvoid DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 597239462Sdim SDValue &Hi) { 598239462Sdim SDValue Op0Lo, Op0Hi; 599239462Sdim GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 600239462Sdim SDValue Op1Lo, Op1Hi; 601239462Sdim GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 602239462Sdim SDValue Op2Lo, Op2Hi; 603239462Sdim GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 604239462Sdim DebugLoc dl = N->getDebugLoc(); 605239462Sdim 606239462Sdim Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 607239462Sdim Op0Lo, Op1Lo, Op2Lo); 608239462Sdim Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 609239462Sdim Op0Hi, Op1Hi, Op2Hi); 610239462Sdim} 611239462Sdim 612218893Sdimvoid DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 613218893Sdim SDValue &Hi) { 614193323Sed // We know the result is a vector. The input may be either a vector or a 615193323Sed // scalar value. 616198090Srdivacky EVT LoVT, HiVT; 617193323Sed GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 618193323Sed DebugLoc dl = N->getDebugLoc(); 619193323Sed 620193323Sed SDValue InOp = N->getOperand(0); 621198090Srdivacky EVT InVT = InOp.getValueType(); 622193323Sed 623193323Sed // Handle some special cases efficiently. 624193323Sed switch (getTypeAction(InVT)) { 625223017Sdim case TargetLowering::TypeLegal: 626223017Sdim case TargetLowering::TypePromoteInteger: 627223017Sdim case TargetLowering::TypeSoftenFloat: 628223017Sdim case TargetLowering::TypeScalarizeVector: 629226633Sdim case TargetLowering::TypeWidenVector: 630193323Sed break; 631223017Sdim case TargetLowering::TypeExpandInteger: 632223017Sdim case TargetLowering::TypeExpandFloat: 633193323Sed // A scalar to vector conversion, where the scalar needs expansion. 634193323Sed // If the vector is being split in two then we can just convert the 635193323Sed // expanded pieces. 636193323Sed if (LoVT == HiVT) { 637193323Sed GetExpandedOp(InOp, Lo, Hi); 638193323Sed if (TLI.isBigEndian()) 639193323Sed std::swap(Lo, Hi); 640218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 641218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 642193323Sed return; 643193323Sed } 644193323Sed break; 645223017Sdim case TargetLowering::TypeSplitVector: 646193323Sed // If the input is a vector that needs to be split, convert each split 647193323Sed // piece of the input now. 648193323Sed GetSplitVector(InOp, Lo, Hi); 649218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 650218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 651193323Sed return; 652193323Sed } 653193323Sed 654193323Sed // In the general case, convert the input to an integer and split it by hand. 655198090Srdivacky EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 656198090Srdivacky EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 657193323Sed if (TLI.isBigEndian()) 658193323Sed std::swap(LoIntVT, HiIntVT); 659193323Sed 660193323Sed SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 661193323Sed 662193323Sed if (TLI.isBigEndian()) 663193323Sed std::swap(Lo, Hi); 664218893Sdim Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 665218893Sdim Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 666193323Sed} 667193323Sed 668193323Sedvoid DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 669193323Sed SDValue &Hi) { 670198090Srdivacky EVT LoVT, HiVT; 671193323Sed DebugLoc dl = N->getDebugLoc(); 672193323Sed GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 673193323Sed unsigned LoNumElts = LoVT.getVectorNumElements(); 674193323Sed SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 675193323Sed Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size()); 676193323Sed 677193323Sed SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 678193323Sed Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size()); 679193323Sed} 680193323Sed 681193323Sedvoid DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 682193323Sed SDValue &Hi) { 683193323Sed assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 684193323Sed DebugLoc dl = N->getDebugLoc(); 685193323Sed unsigned NumSubvectors = N->getNumOperands() / 2; 686193323Sed if (NumSubvectors == 1) { 687193323Sed Lo = N->getOperand(0); 688193323Sed Hi = N->getOperand(1); 689193323Sed return; 690193323Sed } 691193323Sed 692198090Srdivacky EVT LoVT, HiVT; 693193323Sed GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 694193323Sed 695193323Sed SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 696193323Sed Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size()); 697193323Sed 698193323Sed SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 699193323Sed Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size()); 700193323Sed} 701193323Sed 702193323Sedvoid DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 703193323Sed SDValue &Hi) { 704193323Sed SDValue Vec = N->getOperand(0); 705193323Sed SDValue Idx = N->getOperand(1); 706193323Sed DebugLoc dl = N->getDebugLoc(); 707193323Sed 708198090Srdivacky EVT LoVT, HiVT; 709193323Sed GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 710193323Sed 711193323Sed Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 712218893Sdim uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 713218893Sdim Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 714218893Sdim DAG.getIntPtrConstant(IdxVal + LoVT.getVectorNumElements())); 715193323Sed} 716193323Sed 717193323Sedvoid DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 718193323Sed SDValue &Hi) { 719193323Sed DebugLoc dl = N->getDebugLoc(); 720193323Sed GetSplitVector(N->getOperand(0), Lo, Hi); 721193323Sed Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 722193323Sed Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 723193323Sed} 724193323Sed 725202375Srdivackyvoid DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 726202375Srdivacky SDValue &Hi) { 727202375Srdivacky SDValue LHSLo, LHSHi; 728202375Srdivacky GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 729202375Srdivacky DebugLoc dl = N->getDebugLoc(); 730202375Srdivacky 731202375Srdivacky EVT LoVT, HiVT; 732202375Srdivacky GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT(), LoVT, HiVT); 733202375Srdivacky 734202375Srdivacky Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 735202375Srdivacky DAG.getValueType(LoVT)); 736202375Srdivacky Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 737202375Srdivacky DAG.getValueType(HiVT)); 738202375Srdivacky} 739202375Srdivacky 740193323Sedvoid DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 741193323Sed SDValue &Hi) { 742193323Sed SDValue Vec = N->getOperand(0); 743193323Sed SDValue Elt = N->getOperand(1); 744193323Sed SDValue Idx = N->getOperand(2); 745193323Sed DebugLoc dl = N->getDebugLoc(); 746193323Sed GetSplitVector(Vec, Lo, Hi); 747193323Sed 748193323Sed if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 749193323Sed unsigned IdxVal = CIdx->getZExtValue(); 750193323Sed unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 751193323Sed if (IdxVal < LoNumElts) 752193323Sed Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 753193323Sed Lo.getValueType(), Lo, Elt, Idx); 754193323Sed else 755193323Sed Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 756193323Sed DAG.getIntPtrConstant(IdxVal - LoNumElts)); 757193323Sed return; 758193323Sed } 759193323Sed 760193323Sed // Spill the vector to the stack. 761198090Srdivacky EVT VecVT = Vec.getValueType(); 762198090Srdivacky EVT EltVT = VecVT.getVectorElementType(); 763193323Sed SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 764218893Sdim SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 765218893Sdim MachinePointerInfo(), false, false, 0); 766193323Sed 767193323Sed // Store the new element. This may be larger than the vector element type, 768193323Sed // so use a truncating store. 769193323Sed SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 770226633Sdim Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 771193323Sed unsigned Alignment = 772243830Sdim TLI.getDataLayout()->getPrefTypeAlignment(VecType); 773218893Sdim Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT, 774203954Srdivacky false, false, 0); 775193323Sed 776193323Sed // Load the Lo part from the stack slot. 777218893Sdim Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 778234353Sdim false, false, false, 0); 779193323Sed 780193323Sed // Increment the pointer to the other part. 781193323Sed unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8; 782193323Sed StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 783193323Sed DAG.getIntPtrConstant(IncrementSize)); 784193323Sed 785193323Sed // Load the Hi part from the stack slot. 786218893Sdim Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 787234353Sdim false, false, false, MinAlign(Alignment, IncrementSize)); 788193323Sed} 789193323Sed 790193323Sedvoid DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 791193323Sed SDValue &Hi) { 792198090Srdivacky EVT LoVT, HiVT; 793193323Sed DebugLoc dl = N->getDebugLoc(); 794193323Sed GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 795193323Sed Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 796193323Sed Hi = DAG.getUNDEF(HiVT); 797193323Sed} 798193323Sed 799193323Sedvoid DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 800193323Sed SDValue &Hi) { 801193323Sed assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 802198090Srdivacky EVT LoVT, HiVT; 803193323Sed DebugLoc dl = LD->getDebugLoc(); 804193323Sed GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT); 805193323Sed 806193323Sed ISD::LoadExtType ExtType = LD->getExtensionType(); 807193323Sed SDValue Ch = LD->getChain(); 808193323Sed SDValue Ptr = LD->getBasePtr(); 809193323Sed SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 810198090Srdivacky EVT MemoryVT = LD->getMemoryVT(); 811198090Srdivacky unsigned Alignment = LD->getOriginalAlignment(); 812193323Sed bool isVolatile = LD->isVolatile(); 813203954Srdivacky bool isNonTemporal = LD->isNonTemporal(); 814234353Sdim bool isInvariant = LD->isInvariant(); 815193323Sed 816198090Srdivacky EVT LoMemVT, HiMemVT; 817193323Sed GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 818193323Sed 819210299Sed Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 820218893Sdim LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal, 821234353Sdim isInvariant, Alignment); 822193323Sed 823193323Sed unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 824193323Sed Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, 825193323Sed DAG.getIntPtrConstant(IncrementSize)); 826210299Sed Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 827218893Sdim LD->getPointerInfo().getWithOffset(IncrementSize), 828234353Sdim HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment); 829193323Sed 830193323Sed // Build a factor node to remember that this load is independent of the 831193323Sed // other one. 832193323Sed Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 833193323Sed Hi.getValue(1)); 834193323Sed 835193323Sed // Legalized the chain result - switch anything that used the old chain to 836193323Sed // use the new one. 837193323Sed ReplaceValueWith(SDValue(LD, 1), Ch); 838193323Sed} 839193323Sed 840198090Srdivackyvoid DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 841226633Sdim assert(N->getValueType(0).isVector() && 842226633Sdim N->getOperand(0).getValueType().isVector() && 843226633Sdim "Operand types must be vectors"); 844226633Sdim 845198090Srdivacky EVT LoVT, HiVT; 846198090Srdivacky DebugLoc DL = N->getDebugLoc(); 847198090Srdivacky GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 848198090Srdivacky 849198090Srdivacky // Split the input. 850198090Srdivacky EVT InVT = N->getOperand(0).getValueType(); 851198090Srdivacky SDValue LL, LH, RL, RH; 852198090Srdivacky EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 853198090Srdivacky LoVT.getVectorNumElements()); 854198090Srdivacky LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 855198090Srdivacky DAG.getIntPtrConstant(0)); 856198090Srdivacky LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0), 857198090Srdivacky DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 858198090Srdivacky 859198090Srdivacky RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 860198090Srdivacky DAG.getIntPtrConstant(0)); 861198090Srdivacky RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1), 862198090Srdivacky DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 863198090Srdivacky 864198090Srdivacky Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 865198090Srdivacky Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 866198090Srdivacky} 867198090Srdivacky 868193323Sedvoid DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 869193323Sed SDValue &Hi) { 870193323Sed // Get the dest types - they may not match the input types, e.g. int_to_fp. 871198090Srdivacky EVT LoVT, HiVT; 872193323Sed DebugLoc dl = N->getDebugLoc(); 873193323Sed GetSplitDestVTs(N->getValueType(0), LoVT, HiVT); 874193323Sed 875234353Sdim // If the input also splits, handle it directly for a compile time speedup. 876234353Sdim // Otherwise split it by hand. 877198090Srdivacky EVT InVT = N->getOperand(0).getValueType(); 878234353Sdim if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 879234353Sdim GetSplitVector(N->getOperand(0), Lo, Hi); 880234353Sdim } else { 881198090Srdivacky EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(), 882193323Sed LoVT.getVectorNumElements()); 883193323Sed Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 884193323Sed DAG.getIntPtrConstant(0)); 885193323Sed Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0), 886193323Sed DAG.getIntPtrConstant(InNVT.getVectorNumElements())); 887193323Sed } 888193323Sed 889221345Sdim if (N->getOpcode() == ISD::FP_ROUND) { 890221345Sdim Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 891221345Sdim Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 892221345Sdim } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) { 893221345Sdim SDValue DTyOpLo = DAG.getValueType(LoVT); 894221345Sdim SDValue DTyOpHi = DAG.getValueType(HiVT); 895221345Sdim SDValue STyOpLo = DAG.getValueType(Lo.getValueType()); 896221345Sdim SDValue STyOpHi = DAG.getValueType(Hi.getValueType()); 897221345Sdim SDValue RndOp = N->getOperand(3); 898221345Sdim SDValue SatOp = N->getOperand(4); 899221345Sdim ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 900221345Sdim Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp, 901221345Sdim CvtCode); 902221345Sdim Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp, 903221345Sdim CvtCode); 904221345Sdim } else { 905221345Sdim Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 906221345Sdim Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 907221345Sdim } 908193323Sed} 909193323Sed 910193323Sedvoid DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 911193323Sed SDValue &Lo, SDValue &Hi) { 912193323Sed // The low and high parts of the original input give four input vectors. 913193323Sed SDValue Inputs[4]; 914193323Sed DebugLoc dl = N->getDebugLoc(); 915193323Sed GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 916193323Sed GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 917198090Srdivacky EVT NewVT = Inputs[0].getValueType(); 918193323Sed unsigned NewElts = NewVT.getVectorNumElements(); 919193323Sed 920193323Sed // If Lo or Hi uses elements from at most two of the four input vectors, then 921193323Sed // express it as a vector shuffle of those two inputs. Otherwise extract the 922193323Sed // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 923193323Sed SmallVector<int, 16> Ops; 924193323Sed for (unsigned High = 0; High < 2; ++High) { 925193323Sed SDValue &Output = High ? Hi : Lo; 926193323Sed 927193323Sed // Build a shuffle mask for the output, discovering on the fly which 928193323Sed // input vectors to use as shuffle operands (recorded in InputUsed). 929193323Sed // If building a suitable shuffle vector proves too hard, then bail 930193323Sed // out with useBuildVector set. 931193323Sed unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 932193323Sed unsigned FirstMaskIdx = High * NewElts; 933193323Sed bool useBuildVector = false; 934193323Sed for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 935193323Sed // The mask element. This indexes into the input. 936193323Sed int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 937193323Sed 938193323Sed // The input vector this mask element indexes into. 939193323Sed unsigned Input = (unsigned)Idx / NewElts; 940193323Sed 941193323Sed if (Input >= array_lengthof(Inputs)) { 942193323Sed // The mask element does not index into any input vector. 943193323Sed Ops.push_back(-1); 944193323Sed continue; 945193323Sed } 946193323Sed 947193323Sed // Turn the index into an offset from the start of the input vector. 948193323Sed Idx -= Input * NewElts; 949193323Sed 950193323Sed // Find or create a shuffle vector operand to hold this input. 951193323Sed unsigned OpNo; 952193323Sed for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 953193323Sed if (InputUsed[OpNo] == Input) { 954193323Sed // This input vector is already an operand. 955193323Sed break; 956193323Sed } else if (InputUsed[OpNo] == -1U) { 957193323Sed // Create a new operand for this input vector. 958193323Sed InputUsed[OpNo] = Input; 959193323Sed break; 960193323Sed } 961193323Sed } 962193323Sed 963193323Sed if (OpNo >= array_lengthof(InputUsed)) { 964193323Sed // More than two input vectors used! Give up on trying to create a 965193323Sed // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 966193323Sed useBuildVector = true; 967193323Sed break; 968193323Sed } 969193323Sed 970193323Sed // Add the mask index for the new shuffle vector. 971193323Sed Ops.push_back(Idx + OpNo * NewElts); 972193323Sed } 973193323Sed 974193323Sed if (useBuildVector) { 975198090Srdivacky EVT EltVT = NewVT.getVectorElementType(); 976193323Sed SmallVector<SDValue, 16> SVOps; 977193323Sed 978193323Sed // Extract the input elements by hand. 979193323Sed for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 980193323Sed // The mask element. This indexes into the input. 981193323Sed int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 982193323Sed 983193323Sed // The input vector this mask element indexes into. 984193323Sed unsigned Input = (unsigned)Idx / NewElts; 985193323Sed 986193323Sed if (Input >= array_lengthof(Inputs)) { 987193323Sed // The mask element is "undef" or indexes off the end of the input. 988193323Sed SVOps.push_back(DAG.getUNDEF(EltVT)); 989193323Sed continue; 990193323Sed } 991193323Sed 992193323Sed // Turn the index into an offset from the start of the input vector. 993193323Sed Idx -= Input * NewElts; 994193323Sed 995193323Sed // Extract the vector element by hand. 996193323Sed SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 997193323Sed Inputs[Input], DAG.getIntPtrConstant(Idx))); 998193323Sed } 999193323Sed 1000193323Sed // Construct the Lo/Hi output using a BUILD_VECTOR. 1001193323Sed Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size()); 1002193323Sed } else if (InputUsed[0] == -1U) { 1003193323Sed // No input vectors were used! The result is undefined. 1004193323Sed Output = DAG.getUNDEF(NewVT); 1005193323Sed } else { 1006193323Sed SDValue Op0 = Inputs[InputUsed[0]]; 1007193323Sed // If only one input was used, use an undefined vector for the other. 1008193323Sed SDValue Op1 = InputUsed[1] == -1U ? 1009193323Sed DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 1010193323Sed // At least one input vector was used. Create a new shuffle vector. 1011193323Sed Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]); 1012193323Sed } 1013193323Sed 1014193323Sed Ops.clear(); 1015193323Sed } 1016193323Sed} 1017193323Sed 1018193323Sed 1019193323Sed//===----------------------------------------------------------------------===// 1020193323Sed// Operand Vector Splitting 1021193323Sed//===----------------------------------------------------------------------===// 1022193323Sed 1023193323Sed/// SplitVectorOperand - This method is called when the specified operand of the 1024193323Sed/// specified node is found to need vector splitting. At this point, all of the 1025193323Sed/// result types of the node are known to be legal, but other operands of the 1026193323Sed/// node may need legalization as well as the specified one. 1027193323Sedbool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1028202375Srdivacky DEBUG(dbgs() << "Split node operand: "; 1029198090Srdivacky N->dump(&DAG); 1030202375Srdivacky dbgs() << "\n"); 1031193323Sed SDValue Res = SDValue(); 1032193323Sed 1033193323Sed if (Res.getNode() == 0) { 1034193323Sed switch (N->getOpcode()) { 1035193323Sed default: 1036193323Sed#ifndef NDEBUG 1037202375Srdivacky dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1038198090Srdivacky N->dump(&DAG); 1039202375Srdivacky dbgs() << "\n"; 1040193323Sed#endif 1041239462Sdim report_fatal_error("Do not know how to split this operator's " 1042239462Sdim "operand!\n"); 1043239462Sdim 1044226633Sdim case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1045218893Sdim case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1046193323Sed case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1047193323Sed case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1048212904Sdim case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1049251662Sdim case ISD::TRUNCATE: Res = SplitVecOp_TRUNCATE(N); break; 1050218893Sdim case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1051198090Srdivacky case ISD::STORE: 1052198090Srdivacky Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1053198090Srdivacky break; 1054249423Sdim case ISD::VSELECT: 1055249423Sdim Res = SplitVecOp_VSELECT(N, OpNo); 1056249423Sdim break; 1057193323Sed case ISD::CTTZ: 1058193323Sed case ISD::CTLZ: 1059193323Sed case ISD::CTPOP: 1060221345Sdim case ISD::FP_EXTEND: 1061193323Sed case ISD::FP_TO_SINT: 1062193323Sed case ISD::FP_TO_UINT: 1063193323Sed case ISD::SINT_TO_FP: 1064198090Srdivacky case ISD::UINT_TO_FP: 1065218893Sdim case ISD::FTRUNC: 1066198090Srdivacky case ISD::SIGN_EXTEND: 1067198090Srdivacky case ISD::ZERO_EXTEND: 1068198090Srdivacky case ISD::ANY_EXTEND: 1069198090Srdivacky Res = SplitVecOp_UnaryOp(N); 1070198090Srdivacky break; 1071193323Sed } 1072193323Sed } 1073193323Sed 1074193323Sed // If the result is null, the sub-method took care of registering results etc. 1075193323Sed if (!Res.getNode()) return false; 1076193323Sed 1077193323Sed // If the result is N, the sub-method updated N in place. Tell the legalizer 1078193323Sed // core about this. 1079193323Sed if (Res.getNode() == N) 1080193323Sed return true; 1081193323Sed 1082193323Sed assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1083193323Sed "Invalid operand expansion"); 1084193323Sed 1085193323Sed ReplaceValueWith(SDValue(N, 0), Res); 1086193323Sed return false; 1087193323Sed} 1088193323Sed 1089249423SdimSDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 1090249423Sdim // The only possibility for an illegal operand is the mask, since result type 1091249423Sdim // legalization would have handled this node already otherwise. 1092249423Sdim assert(OpNo == 0 && "Illegal operand must be mask"); 1093249423Sdim 1094249423Sdim SDValue Mask = N->getOperand(0); 1095249423Sdim SDValue Src0 = N->getOperand(1); 1096249423Sdim SDValue Src1 = N->getOperand(2); 1097249423Sdim DebugLoc DL = N->getDebugLoc(); 1098249423Sdim EVT MaskVT = Mask.getValueType(); 1099249423Sdim assert(MaskVT.isVector() && "VSELECT without a vector mask?"); 1100249423Sdim 1101249423Sdim SDValue Lo, Hi; 1102249423Sdim GetSplitVector(N->getOperand(0), Lo, Hi); 1103249423Sdim assert(Lo.getValueType() == Hi.getValueType() && 1104249423Sdim "Lo and Hi have differing types");; 1105249423Sdim 1106249423Sdim unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 1107249423Sdim unsigned HiNumElts = Hi.getValueType().getVectorNumElements(); 1108249423Sdim assert(LoNumElts == HiNumElts && "Asymmetric vector split?"); 1109249423Sdim 1110249423Sdim LLVMContext &Ctx = *DAG.getContext(); 1111249423Sdim SDValue Zero = DAG.getIntPtrConstant(0); 1112249423Sdim SDValue LoElts = DAG.getIntPtrConstant(LoNumElts); 1113249423Sdim EVT Src0VT = Src0.getValueType(); 1114249423Sdim EVT Src0EltTy = Src0VT.getVectorElementType(); 1115249423Sdim EVT MaskEltTy = MaskVT.getVectorElementType(); 1116249423Sdim 1117249423Sdim EVT LoOpVT = EVT::getVectorVT(Ctx, Src0EltTy, LoNumElts); 1118249423Sdim EVT LoMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, LoNumElts); 1119249423Sdim EVT HiOpVT = EVT::getVectorVT(Ctx, Src0EltTy, HiNumElts); 1120249423Sdim EVT HiMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, HiNumElts); 1121249423Sdim 1122249423Sdim SDValue LoOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src0, Zero); 1123249423Sdim SDValue LoOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src1, Zero); 1124249423Sdim 1125249423Sdim SDValue HiOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src0, LoElts); 1126249423Sdim SDValue HiOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src1, LoElts); 1127249423Sdim 1128249423Sdim SDValue LoMask = 1129249423Sdim DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoMaskVT, Mask, Zero); 1130249423Sdim SDValue HiMask = 1131249423Sdim DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiMaskVT, Mask, LoElts); 1132249423Sdim 1133249423Sdim SDValue LoSelect = 1134249423Sdim DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 1135249423Sdim SDValue HiSelect = 1136249423Sdim DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 1137249423Sdim 1138249423Sdim return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 1139249423Sdim} 1140249423Sdim 1141193323SedSDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 1142193323Sed // The result has a legal vector type, but the input needs splitting. 1143198090Srdivacky EVT ResVT = N->getValueType(0); 1144193323Sed SDValue Lo, Hi; 1145193323Sed DebugLoc dl = N->getDebugLoc(); 1146193323Sed GetSplitVector(N->getOperand(0), Lo, Hi); 1147198090Srdivacky EVT InVT = Lo.getValueType(); 1148193323Sed 1149198090Srdivacky EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1150193323Sed InVT.getVectorNumElements()); 1151193323Sed 1152193323Sed Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 1153193323Sed Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 1154193323Sed 1155193323Sed return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 1156193323Sed} 1157193323Sed 1158218893SdimSDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 1159218893Sdim // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 1160193323Sed // end up being split all the way down to individual components. Convert the 1161193323Sed // split pieces into integers and reassemble. 1162193323Sed SDValue Lo, Hi; 1163193323Sed GetSplitVector(N->getOperand(0), Lo, Hi); 1164193323Sed Lo = BitConvertToInteger(Lo); 1165193323Sed Hi = BitConvertToInteger(Hi); 1166193323Sed 1167193323Sed if (TLI.isBigEndian()) 1168193323Sed std::swap(Lo, Hi); 1169193323Sed 1170218893Sdim return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), N->getValueType(0), 1171193323Sed JoinIntegers(Lo, Hi)); 1172193323Sed} 1173193323Sed 1174193323SedSDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 1175218893Sdim // We know that the extracted result type is legal. 1176198090Srdivacky EVT SubVT = N->getValueType(0); 1177193323Sed SDValue Idx = N->getOperand(1); 1178193323Sed DebugLoc dl = N->getDebugLoc(); 1179193323Sed SDValue Lo, Hi; 1180193323Sed GetSplitVector(N->getOperand(0), Lo, Hi); 1181193323Sed 1182193323Sed uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1183193323Sed uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1184193323Sed 1185193323Sed if (IdxVal < LoElts) { 1186193323Sed assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 1187193323Sed "Extracted subvector crosses vector split!"); 1188193323Sed return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 1189193323Sed } else { 1190193323Sed return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 1191193323Sed DAG.getConstant(IdxVal - LoElts, Idx.getValueType())); 1192193323Sed } 1193193323Sed} 1194193323Sed 1195193323SedSDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 1196193323Sed SDValue Vec = N->getOperand(0); 1197193323Sed SDValue Idx = N->getOperand(1); 1198198090Srdivacky EVT VecVT = Vec.getValueType(); 1199193323Sed 1200193323Sed if (isa<ConstantSDNode>(Idx)) { 1201193323Sed uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1202193323Sed assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); 1203193323Sed 1204193323Sed SDValue Lo, Hi; 1205193323Sed GetSplitVector(Vec, Lo, Hi); 1206193323Sed 1207193323Sed uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 1208193323Sed 1209193323Sed if (IdxVal < LoElts) 1210210299Sed return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 1211210299Sed return SDValue(DAG.UpdateNodeOperands(N, Hi, 1212198090Srdivacky DAG.getConstant(IdxVal - LoElts, 1213212904Sdim Idx.getValueType())), 0); 1214193323Sed } 1215193323Sed 1216193323Sed // Store the vector to the stack. 1217198090Srdivacky EVT EltVT = VecVT.getVectorElementType(); 1218193323Sed DebugLoc dl = N->getDebugLoc(); 1219193323Sed SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1220218893Sdim SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, 1221218893Sdim MachinePointerInfo(), false, false, 0); 1222193323Sed 1223193323Sed // Load back the required element. 1224193323Sed StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx); 1225218893Sdim return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 1226218893Sdim MachinePointerInfo(), EltVT, false, false, 0); 1227193323Sed} 1228193323Sed 1229193323SedSDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 1230193323Sed assert(N->isUnindexed() && "Indexed store of vector?"); 1231193323Sed assert(OpNo == 1 && "Can only split the stored value"); 1232212904Sdim DebugLoc DL = N->getDebugLoc(); 1233193323Sed 1234193323Sed bool isTruncating = N->isTruncatingStore(); 1235193323Sed SDValue Ch = N->getChain(); 1236193323Sed SDValue Ptr = N->getBasePtr(); 1237198090Srdivacky EVT MemoryVT = N->getMemoryVT(); 1238198090Srdivacky unsigned Alignment = N->getOriginalAlignment(); 1239193323Sed bool isVol = N->isVolatile(); 1240203954Srdivacky bool isNT = N->isNonTemporal(); 1241193323Sed SDValue Lo, Hi; 1242193323Sed GetSplitVector(N->getOperand(1), Lo, Hi); 1243193323Sed 1244198090Srdivacky EVT LoMemVT, HiMemVT; 1245193323Sed GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); 1246193323Sed 1247193323Sed unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1248193323Sed 1249193323Sed if (isTruncating) 1250218893Sdim Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1251203954Srdivacky LoMemVT, isVol, isNT, Alignment); 1252193323Sed else 1253218893Sdim Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), 1254203954Srdivacky isVol, isNT, Alignment); 1255193323Sed 1256193323Sed // Increment the pointer to the other half. 1257212904Sdim Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, 1258193323Sed DAG.getIntPtrConstant(IncrementSize)); 1259193323Sed 1260193323Sed if (isTruncating) 1261218893Sdim Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 1262218893Sdim N->getPointerInfo().getWithOffset(IncrementSize), 1263203954Srdivacky HiMemVT, isVol, isNT, Alignment); 1264193323Sed else 1265218893Sdim Hi = DAG.getStore(Ch, DL, Hi, Ptr, 1266218893Sdim N->getPointerInfo().getWithOffset(IncrementSize), 1267203954Srdivacky isVol, isNT, Alignment); 1268193323Sed 1269212904Sdim return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 1270193323Sed} 1271193323Sed 1272212904SdimSDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 1273212904Sdim DebugLoc DL = N->getDebugLoc(); 1274218893Sdim 1275251662Sdim // The input operands all must have the same type, and we know the result 1276251662Sdim // type is valid. Convert this to a buildvector which extracts all the 1277212904Sdim // input elements. 1278212904Sdim // TODO: If the input elements are power-two vectors, we could convert this to 1279212904Sdim // a new CONCAT_VECTORS node with elements that are half-wide. 1280212904Sdim SmallVector<SDValue, 32> Elts; 1281212904Sdim EVT EltVT = N->getValueType(0).getVectorElementType(); 1282212904Sdim for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) { 1283212904Sdim SDValue Op = N->getOperand(op); 1284212904Sdim for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 1285212904Sdim i != e; ++i) { 1286212904Sdim Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, 1287212904Sdim Op, DAG.getIntPtrConstant(i))); 1288193323Sed 1289212904Sdim } 1290212904Sdim } 1291218893Sdim 1292212904Sdim return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0), 1293212904Sdim &Elts[0], Elts.size()); 1294212904Sdim} 1295212904Sdim 1296251662SdimSDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) { 1297251662Sdim // The result type is legal, but the input type is illegal. If splitting 1298251662Sdim // ends up with the result type of each half still being legal, just 1299251662Sdim // do that. If, however, that would result in an illegal result type, 1300251662Sdim // we can try to get more clever with power-two vectors. Specifically, 1301251662Sdim // split the input type, but also widen the result element size, then 1302251662Sdim // concatenate the halves and truncate again. For example, consider a target 1303251662Sdim // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 1304251662Sdim // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 1305251662Sdim // %inlo = v4i32 extract_subvector %in, 0 1306251662Sdim // %inhi = v4i32 extract_subvector %in, 4 1307251662Sdim // %lo16 = v4i16 trunc v4i32 %inlo 1308251662Sdim // %hi16 = v4i16 trunc v4i32 %inhi 1309251662Sdim // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 1310251662Sdim // %res = v8i8 trunc v8i16 %in16 1311251662Sdim // 1312251662Sdim // Without this transform, the original truncate would end up being 1313251662Sdim // scalarized, which is pretty much always a last resort. 1314251662Sdim SDValue InVec = N->getOperand(0); 1315251662Sdim EVT InVT = InVec->getValueType(0); 1316251662Sdim EVT OutVT = N->getValueType(0); 1317251662Sdim unsigned NumElements = OutVT.getVectorNumElements(); 1318251662Sdim // Widening should have already made sure this is a power-two vector 1319251662Sdim // if we're trying to split it at all. assert() that's true, just in case. 1320251662Sdim assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 1321251662Sdim 1322251662Sdim unsigned InElementSize = InVT.getVectorElementType().getSizeInBits(); 1323251662Sdim unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits(); 1324251662Sdim 1325251662Sdim // If the input elements are only 1/2 the width of the result elements, 1326251662Sdim // just use the normal splitting. Our trick only work if there's room 1327251662Sdim // to split more than once. 1328251662Sdim if (InElementSize <= OutElementSize * 2) 1329251662Sdim return SplitVecOp_UnaryOp(N); 1330251662Sdim DebugLoc DL = N->getDebugLoc(); 1331251662Sdim 1332251662Sdim // Extract the halves of the input via extract_subvector. 1333251662Sdim EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), 1334251662Sdim InVT.getVectorElementType(), NumElements/2); 1335251662Sdim SDValue InLoVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, InVec, 1336251662Sdim DAG.getIntPtrConstant(0)); 1337251662Sdim SDValue InHiVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, InVec, 1338251662Sdim DAG.getIntPtrConstant(NumElements/2)); 1339251662Sdim // Truncate them to 1/2 the element size. 1340251662Sdim EVT HalfElementVT = EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 1341251662Sdim EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 1342251662Sdim NumElements/2); 1343251662Sdim SDValue HalfLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InLoVec); 1344251662Sdim SDValue HalfHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InHiVec); 1345251662Sdim // Concatenate them to get the full intermediate truncation result. 1346251662Sdim EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 1347251662Sdim SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 1348251662Sdim HalfHi); 1349251662Sdim // Now finish up by truncating all the way down to the original result 1350251662Sdim // type. This should normally be something that ends up being legal directly, 1351251662Sdim // but in theory if a target has very wide vectors and an annoyingly 1352251662Sdim // restricted set of legal types, this split can chain to build things up. 1353251662Sdim return DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 1354251662Sdim} 1355251662Sdim 1356226633SdimSDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 1357226633Sdim assert(N->getValueType(0).isVector() && 1358226633Sdim N->getOperand(0).getValueType().isVector() && 1359226633Sdim "Operand types must be vectors"); 1360226633Sdim // The result has a legal vector type, but the input needs splitting. 1361226633Sdim SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 1362226633Sdim DebugLoc DL = N->getDebugLoc(); 1363226633Sdim GetSplitVector(N->getOperand(0), Lo0, Hi0); 1364226633Sdim GetSplitVector(N->getOperand(1), Lo1, Hi1); 1365226633Sdim unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 1366226633Sdim EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 1367226633Sdim EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 1368226633Sdim 1369226633Sdim LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 1370226633Sdim HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 1371226633Sdim SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 1372226633Sdim return PromoteTargetBoolean(Con, N->getValueType(0)); 1373226633Sdim} 1374226633Sdim 1375226633Sdim 1376218893SdimSDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 1377218893Sdim // The result has a legal vector type, but the input needs splitting. 1378218893Sdim EVT ResVT = N->getValueType(0); 1379218893Sdim SDValue Lo, Hi; 1380218893Sdim DebugLoc DL = N->getDebugLoc(); 1381218893Sdim GetSplitVector(N->getOperand(0), Lo, Hi); 1382218893Sdim EVT InVT = Lo.getValueType(); 1383239462Sdim 1384218893Sdim EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 1385218893Sdim InVT.getVectorNumElements()); 1386239462Sdim 1387218893Sdim Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 1388218893Sdim Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 1389239462Sdim 1390218893Sdim return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 1391239462Sdim} 1392212904Sdim 1393218893Sdim 1394218893Sdim 1395193323Sed//===----------------------------------------------------------------------===// 1396193323Sed// Result Vector Widening 1397193323Sed//===----------------------------------------------------------------------===// 1398193323Sed 1399193323Sedvoid DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 1400202375Srdivacky DEBUG(dbgs() << "Widen node result " << ResNo << ": "; 1401198090Srdivacky N->dump(&DAG); 1402202375Srdivacky dbgs() << "\n"); 1403199989Srdivacky 1404199989Srdivacky // See if the target wants to custom widen this node. 1405199989Srdivacky if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 1406199989Srdivacky return; 1407199989Srdivacky 1408193323Sed SDValue Res = SDValue(); 1409193323Sed switch (N->getOpcode()) { 1410193323Sed default: 1411193323Sed#ifndef NDEBUG 1412202375Srdivacky dbgs() << "WidenVectorResult #" << ResNo << ": "; 1413198090Srdivacky N->dump(&DAG); 1414202375Srdivacky dbgs() << "\n"; 1415193323Sed#endif 1416198090Srdivacky llvm_unreachable("Do not know how to widen the result of this operator!"); 1417193323Sed 1418226633Sdim case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 1419218893Sdim case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 1420193323Sed case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 1421193323Sed case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 1422193323Sed case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break; 1423193323Sed case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 1424202375Srdivacky case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break; 1425193323Sed case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 1426193323Sed case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 1427193323Sed case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 1428202375Srdivacky case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 1429234353Sdim case ISD::VSELECT: 1430193323Sed case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 1431193323Sed case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 1432203954Srdivacky case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 1433193323Sed case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 1434198090Srdivacky case ISD::VECTOR_SHUFFLE: 1435198090Srdivacky Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 1436198090Srdivacky break; 1437193323Sed case ISD::ADD: 1438193323Sed case ISD::AND: 1439193323Sed case ISD::BSWAP: 1440193323Sed case ISD::FADD: 1441193323Sed case ISD::FCOPYSIGN: 1442193323Sed case ISD::FDIV: 1443193323Sed case ISD::FMUL: 1444193323Sed case ISD::FPOW: 1445193323Sed case ISD::FREM: 1446193323Sed case ISD::FSUB: 1447193323Sed case ISD::MUL: 1448193323Sed case ISD::MULHS: 1449193323Sed case ISD::MULHU: 1450193323Sed case ISD::OR: 1451193323Sed case ISD::SDIV: 1452193323Sed case ISD::SREM: 1453193323Sed case ISD::UDIV: 1454193323Sed case ISD::UREM: 1455193323Sed case ISD::SUB: 1456198090Srdivacky case ISD::XOR: 1457198090Srdivacky Res = WidenVecRes_Binary(N); 1458198090Srdivacky break; 1459193323Sed 1460210299Sed case ISD::FPOWI: 1461210299Sed Res = WidenVecRes_POWI(N); 1462210299Sed break; 1463210299Sed 1464193323Sed case ISD::SHL: 1465193323Sed case ISD::SRA: 1466198090Srdivacky case ISD::SRL: 1467198090Srdivacky Res = WidenVecRes_Shift(N); 1468198090Srdivacky break; 1469193323Sed 1470221345Sdim case ISD::ANY_EXTEND: 1471221345Sdim case ISD::FP_EXTEND: 1472193323Sed case ISD::FP_ROUND: 1473193323Sed case ISD::FP_TO_SINT: 1474193323Sed case ISD::FP_TO_UINT: 1475221345Sdim case ISD::SIGN_EXTEND: 1476193323Sed case ISD::SINT_TO_FP: 1477221345Sdim case ISD::TRUNCATE: 1478198090Srdivacky case ISD::UINT_TO_FP: 1479193323Sed case ISD::ZERO_EXTEND: 1480198090Srdivacky Res = WidenVecRes_Convert(N); 1481198090Srdivacky break; 1482193323Sed 1483193323Sed case ISD::CTLZ: 1484193323Sed case ISD::CTPOP: 1485193323Sed case ISD::CTTZ: 1486193323Sed case ISD::FABS: 1487221345Sdim case ISD::FCEIL: 1488193323Sed case ISD::FCOS: 1489210299Sed case ISD::FEXP: 1490210299Sed case ISD::FEXP2: 1491221345Sdim case ISD::FFLOOR: 1492210299Sed case ISD::FLOG: 1493221345Sdim case ISD::FLOG10: 1494210299Sed case ISD::FLOG2: 1495221345Sdim case ISD::FNEARBYINT: 1496221345Sdim case ISD::FNEG: 1497221345Sdim case ISD::FRINT: 1498221345Sdim case ISD::FSIN: 1499221345Sdim case ISD::FSQRT: 1500221345Sdim case ISD::FTRUNC: 1501198090Srdivacky Res = WidenVecRes_Unary(N); 1502198090Srdivacky break; 1503243830Sdim case ISD::FMA: 1504243830Sdim Res = WidenVecRes_Ternary(N); 1505243830Sdim break; 1506193323Sed } 1507193323Sed 1508193323Sed // If Res is null, the sub-method took care of registering the result. 1509193323Sed if (Res.getNode()) 1510193323Sed SetWidenedVector(SDValue(N, ResNo), Res); 1511193323Sed} 1512193323Sed 1513243830SdimSDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 1514243830Sdim // Ternary op widening. 1515243830Sdim DebugLoc dl = N->getDebugLoc(); 1516243830Sdim EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1517243830Sdim SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1518243830Sdim SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1519243830Sdim SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 1520243830Sdim return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 1521243830Sdim} 1522243830Sdim 1523193323SedSDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 1524193323Sed // Binary op widening. 1525203954Srdivacky unsigned Opcode = N->getOpcode(); 1526203954Srdivacky DebugLoc dl = N->getDebugLoc(); 1527198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1528203954Srdivacky EVT WidenEltVT = WidenVT.getVectorElementType(); 1529203954Srdivacky EVT VT = WidenVT; 1530203954Srdivacky unsigned NumElts = VT.getVectorNumElements(); 1531218893Sdim while (!TLI.isTypeLegal(VT) && NumElts != 1) { 1532212904Sdim NumElts = NumElts / 2; 1533212904Sdim VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1534203954Srdivacky } 1535203954Srdivacky 1536203954Srdivacky if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 1537203954Srdivacky // Operation doesn't trap so just widen as normal. 1538203954Srdivacky SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1539203954Srdivacky SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1540203954Srdivacky return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2); 1541212904Sdim } 1542218893Sdim 1543212904Sdim // No legal vector version so unroll the vector operation and then widen. 1544212904Sdim if (NumElts == 1) 1545203954Srdivacky return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 1546218893Sdim 1547212904Sdim // Since the operation can trap, apply operation on the original vector. 1548212904Sdim EVT MaxVT = VT; 1549212904Sdim SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 1550212904Sdim SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 1551212904Sdim unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 1552203954Srdivacky 1553212904Sdim SmallVector<SDValue, 16> ConcatOps(CurNumElts); 1554212904Sdim unsigned ConcatEnd = 0; // Current ConcatOps index. 1555212904Sdim int Idx = 0; // Current Idx into input vectors. 1556210299Sed 1557218893Sdim // NumElts := greatest legal vector size (at most WidenVT) 1558212904Sdim // while (orig. vector has unhandled elements) { 1559212904Sdim // take munches of size NumElts from the beginning and add to ConcatOps 1560212904Sdim // NumElts := next smaller supported vector size or 1 1561212904Sdim // } 1562212904Sdim while (CurNumElts != 0) { 1563212904Sdim while (CurNumElts >= NumElts) { 1564212904Sdim SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 1565212904Sdim DAG.getIntPtrConstant(Idx)); 1566212904Sdim SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 1567212904Sdim DAG.getIntPtrConstant(Idx)); 1568212904Sdim ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2); 1569212904Sdim Idx += NumElts; 1570212904Sdim CurNumElts -= NumElts; 1571212904Sdim } 1572212904Sdim do { 1573212904Sdim NumElts = NumElts / 2; 1574212904Sdim VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 1575218893Sdim } while (!TLI.isTypeLegal(VT) && NumElts != 1); 1576203954Srdivacky 1577212904Sdim if (NumElts == 1) { 1578212904Sdim for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 1579218893Sdim SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1580212904Sdim InOp1, DAG.getIntPtrConstant(Idx)); 1581218893Sdim SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 1582212904Sdim InOp2, DAG.getIntPtrConstant(Idx)); 1583212904Sdim ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 1584212904Sdim EOp1, EOp2); 1585203954Srdivacky } 1586212904Sdim CurNumElts = 0; 1587203954Srdivacky } 1588212904Sdim } 1589203954Srdivacky 1590212904Sdim // Check to see if we have a single operation with the widen type. 1591212904Sdim if (ConcatEnd == 1) { 1592212904Sdim VT = ConcatOps[0].getValueType(); 1593212904Sdim if (VT == WidenVT) 1594212904Sdim return ConcatOps[0]; 1595212904Sdim } 1596203954Srdivacky 1597212904Sdim // while (Some element of ConcatOps is not of type MaxVT) { 1598212904Sdim // From the end of ConcatOps, collect elements of the same type and put 1599212904Sdim // them into an op of the next larger supported type 1600212904Sdim // } 1601212904Sdim while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 1602212904Sdim Idx = ConcatEnd - 1; 1603212904Sdim VT = ConcatOps[Idx--].getValueType(); 1604212904Sdim while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 1605212904Sdim Idx--; 1606210299Sed 1607212904Sdim int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 1608212904Sdim EVT NextVT; 1609212904Sdim do { 1610212904Sdim NextSize *= 2; 1611212904Sdim NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 1612218893Sdim } while (!TLI.isTypeLegal(NextVT)); 1613210299Sed 1614212904Sdim if (!VT.isVector()) { 1615212904Sdim // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 1616212904Sdim SDValue VecOp = DAG.getUNDEF(NextVT); 1617212904Sdim unsigned NumToInsert = ConcatEnd - Idx - 1; 1618212904Sdim for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 1619212904Sdim VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 1620212904Sdim ConcatOps[OpIdx], DAG.getIntPtrConstant(i)); 1621203954Srdivacky } 1622212904Sdim ConcatOps[Idx+1] = VecOp; 1623212904Sdim ConcatEnd = Idx + 2; 1624212904Sdim } else { 1625212904Sdim // Vector type, create a CONCAT_VECTORS of type NextVT 1626212904Sdim SDValue undefVec = DAG.getUNDEF(VT); 1627212904Sdim unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 1628212904Sdim SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 1629212904Sdim unsigned RealVals = ConcatEnd - Idx - 1; 1630212904Sdim unsigned SubConcatEnd = 0; 1631212904Sdim unsigned SubConcatIdx = Idx + 1; 1632212904Sdim while (SubConcatEnd < RealVals) 1633212904Sdim SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 1634212904Sdim while (SubConcatEnd < OpsToConcat) 1635212904Sdim SubConcatOps[SubConcatEnd++] = undefVec; 1636212904Sdim ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1637212904Sdim NextVT, &SubConcatOps[0], 1638212904Sdim OpsToConcat); 1639212904Sdim ConcatEnd = SubConcatIdx + 1; 1640203954Srdivacky } 1641212904Sdim } 1642210299Sed 1643212904Sdim // Check to see if we have a single operation with the widen type. 1644212904Sdim if (ConcatEnd == 1) { 1645212904Sdim VT = ConcatOps[0].getValueType(); 1646212904Sdim if (VT == WidenVT) 1647212904Sdim return ConcatOps[0]; 1648203954Srdivacky } 1649218893Sdim 1650212904Sdim // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 1651212904Sdim unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 1652212904Sdim if (NumOps != ConcatEnd ) { 1653212904Sdim SDValue UndefVal = DAG.getUNDEF(MaxVT); 1654212904Sdim for (unsigned j = ConcatEnd; j < NumOps; ++j) 1655212904Sdim ConcatOps[j] = UndefVal; 1656212904Sdim } 1657212904Sdim return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], NumOps); 1658193323Sed} 1659193323Sed 1660193323SedSDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 1661193323Sed SDValue InOp = N->getOperand(0); 1662218893Sdim DebugLoc DL = N->getDebugLoc(); 1663193323Sed 1664198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1665193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1666193323Sed 1667198090Srdivacky EVT InVT = InOp.getValueType(); 1668198090Srdivacky EVT InEltVT = InVT.getVectorElementType(); 1669198090Srdivacky EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1670193323Sed 1671193323Sed unsigned Opcode = N->getOpcode(); 1672193323Sed unsigned InVTNumElts = InVT.getVectorNumElements(); 1673193323Sed 1674223017Sdim if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1675193323Sed InOp = GetWidenedVector(N->getOperand(0)); 1676193323Sed InVT = InOp.getValueType(); 1677193323Sed InVTNumElts = InVT.getVectorNumElements(); 1678218893Sdim if (InVTNumElts == WidenNumElts) { 1679218893Sdim if (N->getNumOperands() == 1) 1680218893Sdim return DAG.getNode(Opcode, DL, WidenVT, InOp); 1681218893Sdim return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1)); 1682218893Sdim } 1683193323Sed } 1684193323Sed 1685218893Sdim if (TLI.isTypeLegal(InWidenVT)) { 1686193323Sed // Because the result and the input are different vector types, widening 1687193323Sed // the result could create a legal type but widening the input might make 1688193323Sed // it an illegal type that might lead to repeatedly splitting the input 1689193323Sed // and then widening it. To avoid this, we widen the input only if 1690193323Sed // it results in a legal type. 1691193323Sed if (WidenNumElts % InVTNumElts == 0) { 1692193323Sed // Widen the input and call convert on the widened input vector. 1693193323Sed unsigned NumConcat = WidenNumElts/InVTNumElts; 1694193323Sed SmallVector<SDValue, 16> Ops(NumConcat); 1695193323Sed Ops[0] = InOp; 1696193323Sed SDValue UndefVal = DAG.getUNDEF(InVT); 1697193323Sed for (unsigned i = 1; i != NumConcat; ++i) 1698193323Sed Ops[i] = UndefVal; 1699218893Sdim SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, 1700218893Sdim &Ops[0], NumConcat); 1701218893Sdim if (N->getNumOperands() == 1) 1702218893Sdim return DAG.getNode(Opcode, DL, WidenVT, InVec); 1703218893Sdim return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1)); 1704193323Sed } 1705193323Sed 1706193323Sed if (InVTNumElts % WidenNumElts == 0) { 1707218893Sdim SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, 1708218893Sdim InOp, DAG.getIntPtrConstant(0)); 1709193323Sed // Extract the input and convert the shorten input vector. 1710218893Sdim if (N->getNumOperands() == 1) 1711218893Sdim return DAG.getNode(Opcode, DL, WidenVT, InVal); 1712218893Sdim return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1)); 1713193323Sed } 1714193323Sed } 1715193323Sed 1716193323Sed // Otherwise unroll into some nasty scalar code and rebuild the vector. 1717193323Sed SmallVector<SDValue, 16> Ops(WidenNumElts); 1718198090Srdivacky EVT EltVT = WidenVT.getVectorElementType(); 1719193323Sed unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 1720193323Sed unsigned i; 1721218893Sdim for (i=0; i < MinElts; ++i) { 1722218893Sdim SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 1723218893Sdim DAG.getIntPtrConstant(i)); 1724218893Sdim if (N->getNumOperands() == 1) 1725218893Sdim Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 1726218893Sdim else 1727218893Sdim Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1)); 1728218893Sdim } 1729193323Sed 1730193323Sed SDValue UndefVal = DAG.getUNDEF(EltVT); 1731193323Sed for (; i < WidenNumElts; ++i) 1732193323Sed Ops[i] = UndefVal; 1733193323Sed 1734218893Sdim return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, &Ops[0], WidenNumElts); 1735193323Sed} 1736193323Sed 1737210299SedSDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 1738210299Sed EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1739210299Sed SDValue InOp = GetWidenedVector(N->getOperand(0)); 1740210299Sed SDValue ShOp = N->getOperand(1); 1741210299Sed return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1742210299Sed} 1743210299Sed 1744193323SedSDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 1745198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1746193323Sed SDValue InOp = GetWidenedVector(N->getOperand(0)); 1747193323Sed SDValue ShOp = N->getOperand(1); 1748193323Sed 1749198090Srdivacky EVT ShVT = ShOp.getValueType(); 1750223017Sdim if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 1751193323Sed ShOp = GetWidenedVector(ShOp); 1752193323Sed ShVT = ShOp.getValueType(); 1753193323Sed } 1754207618Srdivacky EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 1755207618Srdivacky ShVT.getVectorElementType(), 1756193323Sed WidenVT.getVectorNumElements()); 1757193323Sed if (ShVT != ShWidenVT) 1758193323Sed ShOp = ModifyToType(ShOp, ShWidenVT); 1759193323Sed 1760193323Sed return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp); 1761193323Sed} 1762193323Sed 1763193323SedSDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 1764193323Sed // Unary op widening. 1765198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1766193323Sed SDValue InOp = GetWidenedVector(N->getOperand(0)); 1767193323Sed return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp); 1768193323Sed} 1769193323Sed 1770202375SrdivackySDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 1771202375Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1772202375Srdivacky EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 1773202375Srdivacky cast<VTSDNode>(N->getOperand(1))->getVT() 1774202375Srdivacky .getVectorElementType(), 1775202375Srdivacky WidenVT.getVectorNumElements()); 1776202375Srdivacky SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 1777202375Srdivacky return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 1778202375Srdivacky WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 1779202375Srdivacky} 1780202375Srdivacky 1781226633SdimSDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 1782226633Sdim SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 1783226633Sdim return GetWidenedVector(WidenVec); 1784226633Sdim} 1785226633Sdim 1786218893SdimSDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 1787193323Sed SDValue InOp = N->getOperand(0); 1788198090Srdivacky EVT InVT = InOp.getValueType(); 1789198090Srdivacky EVT VT = N->getValueType(0); 1790198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1791193323Sed DebugLoc dl = N->getDebugLoc(); 1792193323Sed 1793193323Sed switch (getTypeAction(InVT)) { 1794223017Sdim case TargetLowering::TypeLegal: 1795193323Sed break; 1796223017Sdim case TargetLowering::TypePromoteInteger: 1797234353Sdim // If the incoming type is a vector that is being promoted, then 1798234353Sdim // we know that the elements are arranged differently and that we 1799234353Sdim // must perform the conversion using a stack slot. 1800234353Sdim if (InVT.isVector()) 1801234353Sdim break; 1802234353Sdim 1803193323Sed // If the InOp is promoted to the same size, convert it. Otherwise, 1804193323Sed // fall out of the switch and widen the promoted input. 1805193323Sed InOp = GetPromotedInteger(InOp); 1806193323Sed InVT = InOp.getValueType(); 1807193323Sed if (WidenVT.bitsEq(InVT)) 1808218893Sdim return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1809193323Sed break; 1810223017Sdim case TargetLowering::TypeSoftenFloat: 1811223017Sdim case TargetLowering::TypeExpandInteger: 1812223017Sdim case TargetLowering::TypeExpandFloat: 1813223017Sdim case TargetLowering::TypeScalarizeVector: 1814223017Sdim case TargetLowering::TypeSplitVector: 1815193323Sed break; 1816223017Sdim case TargetLowering::TypeWidenVector: 1817193323Sed // If the InOp is widened to the same size, convert it. Otherwise, fall 1818193323Sed // out of the switch and widen the widened input. 1819193323Sed InOp = GetWidenedVector(InOp); 1820193323Sed InVT = InOp.getValueType(); 1821193323Sed if (WidenVT.bitsEq(InVT)) 1822193323Sed // The input widens to the same size. Convert to the widen value. 1823218893Sdim return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 1824193323Sed break; 1825193323Sed } 1826193323Sed 1827193323Sed unsigned WidenSize = WidenVT.getSizeInBits(); 1828193323Sed unsigned InSize = InVT.getSizeInBits(); 1829218893Sdim // x86mmx is not an acceptable vector element type, so don't try. 1830218893Sdim if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 1831193323Sed // Determine new input vector type. The new input vector type will use 1832193323Sed // the same element type (if its a vector) or use the input type as a 1833193323Sed // vector. It is the same size as the type to widen to. 1834198090Srdivacky EVT NewInVT; 1835193323Sed unsigned NewNumElts = WidenSize / InSize; 1836193323Sed if (InVT.isVector()) { 1837198090Srdivacky EVT InEltVT = InVT.getVectorElementType(); 1838212904Sdim NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 1839212904Sdim WidenSize / InEltVT.getSizeInBits()); 1840193323Sed } else { 1841198090Srdivacky NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 1842193323Sed } 1843193323Sed 1844218893Sdim if (TLI.isTypeLegal(NewInVT)) { 1845193323Sed // Because the result and the input are different vector types, widening 1846193323Sed // the result could create a legal type but widening the input might make 1847193323Sed // it an illegal type that might lead to repeatedly splitting the input 1848193323Sed // and then widening it. To avoid this, we widen the input only if 1849193323Sed // it results in a legal type. 1850193323Sed SmallVector<SDValue, 16> Ops(NewNumElts); 1851193323Sed SDValue UndefVal = DAG.getUNDEF(InVT); 1852193323Sed Ops[0] = InOp; 1853193323Sed for (unsigned i = 1; i < NewNumElts; ++i) 1854193323Sed Ops[i] = UndefVal; 1855193323Sed 1856193323Sed SDValue NewVec; 1857193323Sed if (InVT.isVector()) 1858193323Sed NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, 1859193323Sed NewInVT, &Ops[0], NewNumElts); 1860193323Sed else 1861193323Sed NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, 1862193323Sed NewInVT, &Ops[0], NewNumElts); 1863218893Sdim return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 1864193323Sed } 1865193323Sed } 1866193323Sed 1867198090Srdivacky return CreateStackStoreLoad(InOp, WidenVT); 1868193323Sed} 1869193323Sed 1870193323SedSDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 1871193323Sed DebugLoc dl = N->getDebugLoc(); 1872193323Sed // Build a vector with undefined for the new nodes. 1873198090Srdivacky EVT VT = N->getValueType(0); 1874198090Srdivacky EVT EltVT = VT.getVectorElementType(); 1875193323Sed unsigned NumElts = VT.getVectorNumElements(); 1876193323Sed 1877198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1878193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1879193323Sed 1880193323Sed SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 1881193323Sed NewOps.reserve(WidenNumElts); 1882193323Sed for (unsigned i = NumElts; i < WidenNumElts; ++i) 1883193323Sed NewOps.push_back(DAG.getUNDEF(EltVT)); 1884193323Sed 1885193323Sed return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size()); 1886193323Sed} 1887193323Sed 1888193323SedSDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 1889198090Srdivacky EVT InVT = N->getOperand(0).getValueType(); 1890198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1891193323Sed DebugLoc dl = N->getDebugLoc(); 1892193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1893226633Sdim unsigned NumInElts = InVT.getVectorNumElements(); 1894193323Sed unsigned NumOperands = N->getNumOperands(); 1895193323Sed 1896193323Sed bool InputWidened = false; // Indicates we need to widen the input. 1897223017Sdim if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 1898193323Sed if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 1899193323Sed // Add undef vectors to widen to correct length. 1900193323Sed unsigned NumConcat = WidenVT.getVectorNumElements() / 1901193323Sed InVT.getVectorNumElements(); 1902193323Sed SDValue UndefVal = DAG.getUNDEF(InVT); 1903193323Sed SmallVector<SDValue, 16> Ops(NumConcat); 1904193323Sed for (unsigned i=0; i < NumOperands; ++i) 1905193323Sed Ops[i] = N->getOperand(i); 1906193323Sed for (unsigned i = NumOperands; i != NumConcat; ++i) 1907193323Sed Ops[i] = UndefVal; 1908193323Sed return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat); 1909193323Sed } 1910193323Sed } else { 1911193323Sed InputWidened = true; 1912198090Srdivacky if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 1913193323Sed // The inputs and the result are widen to the same value. 1914193323Sed unsigned i; 1915193323Sed for (i=1; i < NumOperands; ++i) 1916193323Sed if (N->getOperand(i).getOpcode() != ISD::UNDEF) 1917193323Sed break; 1918193323Sed 1919226633Sdim if (i == NumOperands) 1920193323Sed // Everything but the first operand is an UNDEF so just return the 1921193323Sed // widened first operand. 1922193323Sed return GetWidenedVector(N->getOperand(0)); 1923193323Sed 1924193323Sed if (NumOperands == 2) { 1925193323Sed // Replace concat of two operands with a shuffle. 1926226633Sdim SmallVector<int, 16> MaskOps(WidenNumElts, -1); 1927226633Sdim for (unsigned i = 0; i < NumInElts; ++i) { 1928193323Sed MaskOps[i] = i; 1929226633Sdim MaskOps[i + NumInElts] = i + WidenNumElts; 1930193323Sed } 1931198090Srdivacky return DAG.getVectorShuffle(WidenVT, dl, 1932193323Sed GetWidenedVector(N->getOperand(0)), 1933193323Sed GetWidenedVector(N->getOperand(1)), 1934193323Sed &MaskOps[0]); 1935193323Sed } 1936193323Sed } 1937193323Sed } 1938193323Sed 1939193323Sed // Fall back to use extracts and build vector. 1940198090Srdivacky EVT EltVT = WidenVT.getVectorElementType(); 1941193323Sed SmallVector<SDValue, 16> Ops(WidenNumElts); 1942193323Sed unsigned Idx = 0; 1943193323Sed for (unsigned i=0; i < NumOperands; ++i) { 1944193323Sed SDValue InOp = N->getOperand(i); 1945193323Sed if (InputWidened) 1946193323Sed InOp = GetWidenedVector(InOp); 1947193323Sed for (unsigned j=0; j < NumInElts; ++j) 1948239462Sdim Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 1949239462Sdim DAG.getIntPtrConstant(j)); 1950193323Sed } 1951193323Sed SDValue UndefVal = DAG.getUNDEF(EltVT); 1952193323Sed for (; Idx < WidenNumElts; ++Idx) 1953193323Sed Ops[Idx] = UndefVal; 1954193323Sed return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 1955193323Sed} 1956193323Sed 1957193323SedSDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) { 1958193323Sed DebugLoc dl = N->getDebugLoc(); 1959193323Sed SDValue InOp = N->getOperand(0); 1960193323Sed SDValue RndOp = N->getOperand(3); 1961193323Sed SDValue SatOp = N->getOperand(4); 1962193323Sed 1963212904Sdim EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1964193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 1965193323Sed 1966198090Srdivacky EVT InVT = InOp.getValueType(); 1967198090Srdivacky EVT InEltVT = InVT.getVectorElementType(); 1968198090Srdivacky EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 1969193323Sed 1970193323Sed SDValue DTyOp = DAG.getValueType(WidenVT); 1971193323Sed SDValue STyOp = DAG.getValueType(InWidenVT); 1972193323Sed ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode(); 1973193323Sed 1974193323Sed unsigned InVTNumElts = InVT.getVectorNumElements(); 1975223017Sdim if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 1976193323Sed InOp = GetWidenedVector(InOp); 1977193323Sed InVT = InOp.getValueType(); 1978193323Sed InVTNumElts = InVT.getVectorNumElements(); 1979193323Sed if (InVTNumElts == WidenNumElts) 1980193323Sed return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 1981193323Sed SatOp, CvtCode); 1982193323Sed } 1983193323Sed 1984218893Sdim if (TLI.isTypeLegal(InWidenVT)) { 1985193323Sed // Because the result and the input are different vector types, widening 1986193323Sed // the result could create a legal type but widening the input might make 1987193323Sed // it an illegal type that might lead to repeatedly splitting the input 1988193323Sed // and then widening it. To avoid this, we widen the input only if 1989193323Sed // it results in a legal type. 1990193323Sed if (WidenNumElts % InVTNumElts == 0) { 1991193323Sed // Widen the input and call convert on the widened input vector. 1992193323Sed unsigned NumConcat = WidenNumElts/InVTNumElts; 1993193323Sed SmallVector<SDValue, 16> Ops(NumConcat); 1994193323Sed Ops[0] = InOp; 1995193323Sed SDValue UndefVal = DAG.getUNDEF(InVT); 1996212904Sdim for (unsigned i = 1; i != NumConcat; ++i) 1997193323Sed Ops[i] = UndefVal; 1998212904Sdim 1999193323Sed InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat); 2000193323Sed return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2001193323Sed SatOp, CvtCode); 2002193323Sed } 2003193323Sed 2004193323Sed if (InVTNumElts % WidenNumElts == 0) { 2005193323Sed // Extract the input and convert the shorten input vector. 2006193323Sed InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp, 2007193323Sed DAG.getIntPtrConstant(0)); 2008193323Sed return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp, 2009239462Sdim SatOp, CvtCode); 2010193323Sed } 2011193323Sed } 2012193323Sed 2013193323Sed // Otherwise unroll into some nasty scalar code and rebuild the vector. 2014193323Sed SmallVector<SDValue, 16> Ops(WidenNumElts); 2015198090Srdivacky EVT EltVT = WidenVT.getVectorElementType(); 2016193323Sed DTyOp = DAG.getValueType(EltVT); 2017193323Sed STyOp = DAG.getValueType(InEltVT); 2018193323Sed 2019193323Sed unsigned MinElts = std::min(InVTNumElts, WidenNumElts); 2020193323Sed unsigned i; 2021193323Sed for (i=0; i < MinElts; ++i) { 2022193323Sed SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2023193323Sed DAG.getIntPtrConstant(i)); 2024193323Sed Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp, 2025239462Sdim SatOp, CvtCode); 2026193323Sed } 2027193323Sed 2028193323Sed SDValue UndefVal = DAG.getUNDEF(EltVT); 2029193323Sed for (; i < WidenNumElts; ++i) 2030193323Sed Ops[i] = UndefVal; 2031193323Sed 2032193323Sed return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 2033193323Sed} 2034193323Sed 2035193323SedSDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 2036198090Srdivacky EVT VT = N->getValueType(0); 2037198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2038193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2039193323Sed SDValue InOp = N->getOperand(0); 2040193323Sed SDValue Idx = N->getOperand(1); 2041193323Sed DebugLoc dl = N->getDebugLoc(); 2042193323Sed 2043223017Sdim if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2044193323Sed InOp = GetWidenedVector(InOp); 2045193323Sed 2046198090Srdivacky EVT InVT = InOp.getValueType(); 2047193323Sed 2048218893Sdim // Check if we can just return the input vector after widening. 2049218893Sdim uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2050218893Sdim if (IdxVal == 0 && InVT == WidenVT) 2051218893Sdim return InOp; 2052193323Sed 2053218893Sdim // Check if we can extract from the vector. 2054218893Sdim unsigned InNumElts = InVT.getVectorNumElements(); 2055218893Sdim if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 2056218893Sdim return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 2057193323Sed 2058193323Sed // We could try widening the input to the right length but for now, extract 2059193323Sed // the original elements, fill the rest with undefs and build a vector. 2060193323Sed SmallVector<SDValue, 16> Ops(WidenNumElts); 2061198090Srdivacky EVT EltVT = VT.getVectorElementType(); 2062193323Sed unsigned NumElts = VT.getVectorNumElements(); 2063193323Sed unsigned i; 2064218893Sdim for (i=0; i < NumElts; ++i) 2065218893Sdim Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2066218893Sdim DAG.getIntPtrConstant(IdxVal+i)); 2067193323Sed 2068193323Sed SDValue UndefVal = DAG.getUNDEF(EltVT); 2069193323Sed for (; i < WidenNumElts; ++i) 2070193323Sed Ops[i] = UndefVal; 2071193323Sed return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts); 2072193323Sed} 2073193323Sed 2074193323SedSDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 2075193323Sed SDValue InOp = GetWidenedVector(N->getOperand(0)); 2076193323Sed return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(), 2077193323Sed InOp.getValueType(), InOp, 2078193323Sed N->getOperand(1), N->getOperand(2)); 2079193323Sed} 2080193323Sed 2081193323SedSDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 2082193323Sed LoadSDNode *LD = cast<LoadSDNode>(N); 2083193323Sed ISD::LoadExtType ExtType = LD->getExtensionType(); 2084193323Sed 2085193323Sed SDValue Result; 2086193323Sed SmallVector<SDValue, 16> LdChain; // Chain for the series of load 2087203954Srdivacky if (ExtType != ISD::NON_EXTLOAD) 2088203954Srdivacky Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 2089203954Srdivacky else 2090203954Srdivacky Result = GenWidenVectorLoads(LdChain, LD); 2091193323Sed 2092203954Srdivacky // If we generate a single load, we can use that for the chain. Otherwise, 2093203954Srdivacky // build a factor node to remember the multiple loads are independent and 2094203954Srdivacky // chain to that. 2095203954Srdivacky SDValue NewChain; 2096203954Srdivacky if (LdChain.size() == 1) 2097203954Srdivacky NewChain = LdChain[0]; 2098203954Srdivacky else 2099203954Srdivacky NewChain = DAG.getNode(ISD::TokenFactor, LD->getDebugLoc(), MVT::Other, 2100203954Srdivacky &LdChain[0], LdChain.size()); 2101193323Sed 2102193323Sed // Modified the chain - switch anything that used the old chain to use 2103193323Sed // the new one. 2104198090Srdivacky ReplaceValueWith(SDValue(N, 1), NewChain); 2105193323Sed 2106193323Sed return Result; 2107193323Sed} 2108193323Sed 2109193323SedSDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 2110198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2111193323Sed return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(), 2112193323Sed WidenVT, N->getOperand(0)); 2113193323Sed} 2114193323Sed 2115193323SedSDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 2116198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2117193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2118193323Sed 2119193323Sed SDValue Cond1 = N->getOperand(0); 2120198090Srdivacky EVT CondVT = Cond1.getValueType(); 2121193323Sed if (CondVT.isVector()) { 2122198090Srdivacky EVT CondEltVT = CondVT.getVectorElementType(); 2123207618Srdivacky EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 2124207618Srdivacky CondEltVT, WidenNumElts); 2125223017Sdim if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 2126193323Sed Cond1 = GetWidenedVector(Cond1); 2127193323Sed 2128193323Sed if (Cond1.getValueType() != CondWidenVT) 2129239462Sdim Cond1 = ModifyToType(Cond1, CondWidenVT); 2130193323Sed } 2131193323Sed 2132193323Sed SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2133193323Sed SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 2134193323Sed assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 2135234353Sdim return DAG.getNode(N->getOpcode(), N->getDebugLoc(), 2136193323Sed WidenVT, Cond1, InOp1, InOp2); 2137193323Sed} 2138193323Sed 2139193323SedSDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 2140193323Sed SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 2141193323Sed SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 2142193323Sed return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), 2143193323Sed InOp1.getValueType(), N->getOperand(0), 2144193323Sed N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 2145193323Sed} 2146193323Sed 2147203954SrdivackySDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 2148226633Sdim assert(N->getValueType(0).isVector() == 2149226633Sdim N->getOperand(0).getValueType().isVector() && 2150226633Sdim "Scalar/Vector type mismatch"); 2151226633Sdim if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N); 2152226633Sdim 2153203954Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2154203954Srdivacky SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2155203954Srdivacky SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2156203954Srdivacky return DAG.getNode(ISD::SETCC, N->getDebugLoc(), WidenVT, 2157203954Srdivacky InOp1, InOp2, N->getOperand(2)); 2158203954Srdivacky} 2159203954Srdivacky 2160193323SedSDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 2161198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2162193323Sed return DAG.getUNDEF(WidenVT); 2163193323Sed} 2164193323Sed 2165193323SedSDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 2166198090Srdivacky EVT VT = N->getValueType(0); 2167193323Sed DebugLoc dl = N->getDebugLoc(); 2168193323Sed 2169198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2170193323Sed unsigned NumElts = VT.getVectorNumElements(); 2171193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2172193323Sed 2173193323Sed SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2174193323Sed SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2175193323Sed 2176193323Sed // Adjust mask based on new input vector length. 2177193323Sed SmallVector<int, 16> NewMask; 2178193323Sed for (unsigned i = 0; i != NumElts; ++i) { 2179193323Sed int Idx = N->getMaskElt(i); 2180193323Sed if (Idx < (int)NumElts) 2181193323Sed NewMask.push_back(Idx); 2182193323Sed else 2183193323Sed NewMask.push_back(Idx - NumElts + WidenNumElts); 2184193323Sed } 2185193323Sed for (unsigned i = NumElts; i != WidenNumElts; ++i) 2186193323Sed NewMask.push_back(-1); 2187193323Sed return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]); 2188193323Sed} 2189193323Sed 2190193323SedSDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) { 2191226633Sdim assert(N->getValueType(0).isVector() && 2192226633Sdim N->getOperand(0).getValueType().isVector() && 2193226633Sdim "Operands must be vectors"); 2194198090Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2195193323Sed unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2196193323Sed 2197193323Sed SDValue InOp1 = N->getOperand(0); 2198198090Srdivacky EVT InVT = InOp1.getValueType(); 2199193323Sed assert(InVT.isVector() && "can not widen non vector type"); 2200207618Srdivacky EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 2201207618Srdivacky InVT.getVectorElementType(), WidenNumElts); 2202193323Sed InOp1 = GetWidenedVector(InOp1); 2203193323Sed SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2204193323Sed 2205193323Sed // Assume that the input and output will be widen appropriately. If not, 2206193323Sed // we will have to unroll it at some point. 2207193323Sed assert(InOp1.getValueType() == WidenInVT && 2208193323Sed InOp2.getValueType() == WidenInVT && 2209193323Sed "Input not widened to expected type!"); 2210226633Sdim (void)WidenInVT; 2211226633Sdim return DAG.getNode(ISD::SETCC, N->getDebugLoc(), 2212193323Sed WidenVT, InOp1, InOp2, N->getOperand(2)); 2213193323Sed} 2214193323Sed 2215193323Sed 2216193323Sed//===----------------------------------------------------------------------===// 2217193323Sed// Widen Vector Operand 2218193323Sed//===----------------------------------------------------------------------===// 2219243830Sdimbool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 2220243830Sdim DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; 2221198090Srdivacky N->dump(&DAG); 2222202375Srdivacky dbgs() << "\n"); 2223193323Sed SDValue Res = SDValue(); 2224193323Sed 2225243830Sdim // See if the target wants to custom widen this node. 2226243830Sdim if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2227243830Sdim return false; 2228243830Sdim 2229193323Sed switch (N->getOpcode()) { 2230193323Sed default: 2231193323Sed#ifndef NDEBUG 2232243830Sdim dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 2233198090Srdivacky N->dump(&DAG); 2234202375Srdivacky dbgs() << "\n"; 2235193323Sed#endif 2236198090Srdivacky llvm_unreachable("Do not know how to widen this operator's operand!"); 2237193323Sed 2238218893Sdim case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 2239193323Sed case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 2240198396Srdivacky case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 2241193323Sed case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 2242193323Sed case ISD::STORE: Res = WidenVecOp_STORE(N); break; 2243234353Sdim case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 2244193323Sed 2245221345Sdim case ISD::FP_EXTEND: 2246193323Sed case ISD::FP_TO_SINT: 2247193323Sed case ISD::FP_TO_UINT: 2248193323Sed case ISD::SINT_TO_FP: 2249198090Srdivacky case ISD::UINT_TO_FP: 2250193323Sed case ISD::TRUNCATE: 2251198090Srdivacky case ISD::SIGN_EXTEND: 2252198090Srdivacky case ISD::ZERO_EXTEND: 2253198090Srdivacky case ISD::ANY_EXTEND: 2254198090Srdivacky Res = WidenVecOp_Convert(N); 2255198090Srdivacky break; 2256193323Sed } 2257193323Sed 2258193323Sed // If Res is null, the sub-method took care of registering the result. 2259193323Sed if (!Res.getNode()) return false; 2260193323Sed 2261193323Sed // If the result is N, the sub-method updated N in place. Tell the legalizer 2262193323Sed // core about this. 2263193323Sed if (Res.getNode() == N) 2264193323Sed return true; 2265193323Sed 2266193323Sed 2267193323Sed assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2268193323Sed "Invalid operand expansion"); 2269193323Sed 2270193323Sed ReplaceValueWith(SDValue(N, 0), Res); 2271193323Sed return false; 2272193323Sed} 2273193323Sed 2274193323SedSDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 2275193323Sed // Since the result is legal and the input is illegal, it is unlikely 2276193323Sed // that we can fix the input to a legal type so unroll the convert 2277193323Sed // into some scalar code and create a nasty build vector. 2278198090Srdivacky EVT VT = N->getValueType(0); 2279198090Srdivacky EVT EltVT = VT.getVectorElementType(); 2280193323Sed DebugLoc dl = N->getDebugLoc(); 2281193323Sed unsigned NumElts = VT.getVectorNumElements(); 2282193323Sed SDValue InOp = N->getOperand(0); 2283223017Sdim if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2284193323Sed InOp = GetWidenedVector(InOp); 2285198090Srdivacky EVT InVT = InOp.getValueType(); 2286198090Srdivacky EVT InEltVT = InVT.getVectorElementType(); 2287193323Sed 2288193323Sed unsigned Opcode = N->getOpcode(); 2289193323Sed SmallVector<SDValue, 16> Ops(NumElts); 2290193323Sed for (unsigned i=0; i < NumElts; ++i) 2291193323Sed Ops[i] = DAG.getNode(Opcode, dl, EltVT, 2292193323Sed DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 2293193323Sed DAG.getIntPtrConstant(i))); 2294193323Sed 2295193323Sed return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 2296193323Sed} 2297193323Sed 2298218893SdimSDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 2299198090Srdivacky EVT VT = N->getValueType(0); 2300193323Sed SDValue InOp = GetWidenedVector(N->getOperand(0)); 2301198090Srdivacky EVT InWidenVT = InOp.getValueType(); 2302193323Sed DebugLoc dl = N->getDebugLoc(); 2303193323Sed 2304193323Sed // Check if we can convert between two legal vector types and extract. 2305193323Sed unsigned InWidenSize = InWidenVT.getSizeInBits(); 2306193323Sed unsigned Size = VT.getSizeInBits(); 2307218893Sdim // x86mmx is not an acceptable vector element type, so don't try. 2308218893Sdim if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 2309193323Sed unsigned NewNumElts = InWidenSize / Size; 2310198090Srdivacky EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 2311218893Sdim if (TLI.isTypeLegal(NewVT)) { 2312218893Sdim SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 2313193323Sed return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 2314193323Sed DAG.getIntPtrConstant(0)); 2315193323Sed } 2316193323Sed } 2317193323Sed 2318198090Srdivacky return CreateStackStoreLoad(InOp, VT); 2319193323Sed} 2320193323Sed 2321193323SedSDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 2322193323Sed // If the input vector is not legal, it is likely that we will not find a 2323193323Sed // legal vector of the same size. Replace the concatenate vector with a 2324193323Sed // nasty build vector. 2325198090Srdivacky EVT VT = N->getValueType(0); 2326198090Srdivacky EVT EltVT = VT.getVectorElementType(); 2327193323Sed DebugLoc dl = N->getDebugLoc(); 2328193323Sed unsigned NumElts = VT.getVectorNumElements(); 2329193323Sed SmallVector<SDValue, 16> Ops(NumElts); 2330193323Sed 2331198090Srdivacky EVT InVT = N->getOperand(0).getValueType(); 2332193323Sed unsigned NumInElts = InVT.getVectorNumElements(); 2333193323Sed 2334193323Sed unsigned Idx = 0; 2335193323Sed unsigned NumOperands = N->getNumOperands(); 2336193323Sed for (unsigned i=0; i < NumOperands; ++i) { 2337193323Sed SDValue InOp = N->getOperand(i); 2338223017Sdim if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 2339193323Sed InOp = GetWidenedVector(InOp); 2340193323Sed for (unsigned j=0; j < NumInElts; ++j) 2341193323Sed Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2342193323Sed DAG.getIntPtrConstant(j)); 2343193323Sed } 2344193323Sed return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts); 2345193323Sed} 2346193323Sed 2347198396SrdivackySDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2348198396Srdivacky SDValue InOp = GetWidenedVector(N->getOperand(0)); 2349198396Srdivacky return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), 2350198396Srdivacky N->getValueType(0), InOp, N->getOperand(1)); 2351198396Srdivacky} 2352198396Srdivacky 2353193323SedSDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2354193323Sed SDValue InOp = GetWidenedVector(N->getOperand(0)); 2355193323Sed return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(), 2356198090Srdivacky N->getValueType(0), InOp, N->getOperand(1)); 2357193323Sed} 2358193323Sed 2359193323SedSDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 2360193323Sed // We have to widen the value but we want only to store the original 2361193323Sed // vector type. 2362193323Sed StoreSDNode *ST = cast<StoreSDNode>(N); 2363193323Sed 2364203954Srdivacky SmallVector<SDValue, 16> StChain; 2365203954Srdivacky if (ST->isTruncatingStore()) 2366203954Srdivacky GenWidenVectorTruncStores(StChain, ST); 2367203954Srdivacky else 2368203954Srdivacky GenWidenVectorStores(StChain, ST); 2369193323Sed 2370193323Sed if (StChain.size() == 1) 2371193323Sed return StChain[0]; 2372193323Sed else 2373203954Srdivacky return DAG.getNode(ISD::TokenFactor, ST->getDebugLoc(), 2374193323Sed MVT::Other,&StChain[0],StChain.size()); 2375193323Sed} 2376193323Sed 2377234353SdimSDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 2378234353Sdim SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 2379234353Sdim SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 2380234353Sdim DebugLoc dl = N->getDebugLoc(); 2381234353Sdim 2382234353Sdim // WARNING: In this code we widen the compare instruction with garbage. 2383234353Sdim // This garbage may contain denormal floats which may be slow. Is this a real 2384234353Sdim // concern ? Should we zero the unused lanes if this is a float compare ? 2385234353Sdim 2386234353Sdim // Get a new SETCC node to compare the newly widened operands. 2387234353Sdim // Only some of the compared elements are legal. 2388234353Sdim EVT SVT = TLI.getSetCCResultType(InOp0.getValueType()); 2389234353Sdim SDValue WideSETCC = DAG.getNode(ISD::SETCC, N->getDebugLoc(), 2390234353Sdim SVT, InOp0, InOp1, N->getOperand(2)); 2391234353Sdim 2392234353Sdim // Extract the needed results from the result vector. 2393234353Sdim EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 2394234353Sdim SVT.getVectorElementType(), 2395234353Sdim N->getValueType(0).getVectorNumElements()); 2396234353Sdim SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, 2397234353Sdim ResVT, WideSETCC, DAG.getIntPtrConstant(0)); 2398234353Sdim 2399239462Sdim return PromoteTargetBoolean(CC, N->getValueType(0)); 2400234353Sdim} 2401234353Sdim 2402234353Sdim 2403193323Sed//===----------------------------------------------------------------------===// 2404193323Sed// Vector Widening Utilities 2405193323Sed//===----------------------------------------------------------------------===// 2406193323Sed 2407203954Srdivacky// Utility function to find the type to chop up a widen vector for load/store 2408203954Srdivacky// TLI: Target lowering used to determine legal types. 2409203954Srdivacky// Width: Width left need to load/store. 2410203954Srdivacky// WidenVT: The widen vector type to load to/store from 2411203954Srdivacky// Align: If 0, don't allow use of a wider type 2412203954Srdivacky// WidenEx: If Align is not 0, the amount additional we can load/store from. 2413193323Sed 2414203954Srdivackystatic EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 2415203954Srdivacky unsigned Width, EVT WidenVT, 2416203954Srdivacky unsigned Align = 0, unsigned WidenEx = 0) { 2417203954Srdivacky EVT WidenEltVT = WidenVT.getVectorElementType(); 2418203954Srdivacky unsigned WidenWidth = WidenVT.getSizeInBits(); 2419203954Srdivacky unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 2420203954Srdivacky unsigned AlignInBits = Align*8; 2421203954Srdivacky 2422203954Srdivacky // If we have one element to load/store, return it. 2423203954Srdivacky EVT RetVT = WidenEltVT; 2424203954Srdivacky if (Width == WidenEltWidth) 2425203954Srdivacky return RetVT; 2426203954Srdivacky 2427218893Sdim // See if there is larger legal integer than the element type to load/store 2428203954Srdivacky unsigned VT; 2429203954Srdivacky for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 2430203954Srdivacky VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 2431203954Srdivacky EVT MemVT((MVT::SimpleValueType) VT); 2432203954Srdivacky unsigned MemVTWidth = MemVT.getSizeInBits(); 2433203954Srdivacky if (MemVT.getSizeInBits() <= WidenEltWidth) 2434203954Srdivacky break; 2435218893Sdim if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 && 2436224145Sdim isPowerOf2_32(WidenWidth / MemVTWidth) && 2437203954Srdivacky (MemVTWidth <= Width || 2438203954Srdivacky (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2439203954Srdivacky RetVT = MemVT; 2440203954Srdivacky break; 2441203954Srdivacky } 2442193323Sed } 2443203954Srdivacky 2444203954Srdivacky // See if there is a larger vector type to load/store that has the same vector 2445203954Srdivacky // element type and is evenly divisible with the WidenVT. 2446203954Srdivacky for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 2447203954Srdivacky VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 2448203954Srdivacky EVT MemVT = (MVT::SimpleValueType) VT; 2449203954Srdivacky unsigned MemVTWidth = MemVT.getSizeInBits(); 2450218893Sdim if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && 2451203954Srdivacky (WidenWidth % MemVTWidth) == 0 && 2452224145Sdim isPowerOf2_32(WidenWidth / MemVTWidth) && 2453203954Srdivacky (MemVTWidth <= Width || 2454203954Srdivacky (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 2455203954Srdivacky if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 2456203954Srdivacky return MemVT; 2457203954Srdivacky } 2458203954Srdivacky } 2459203954Srdivacky 2460203954Srdivacky return RetVT; 2461193323Sed} 2462193323Sed 2463203954Srdivacky// Builds a vector type from scalar loads 2464203954Srdivacky// VecTy: Resulting Vector type 2465203954Srdivacky// LDOps: Load operators to build a vector type 2466203954Srdivacky// [Start,End) the list of loads to use. 2467203954Srdivackystatic SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 2468203954Srdivacky SmallVector<SDValue, 16>& LdOps, 2469203954Srdivacky unsigned Start, unsigned End) { 2470203954Srdivacky DebugLoc dl = LdOps[Start].getDebugLoc(); 2471203954Srdivacky EVT LdTy = LdOps[Start].getValueType(); 2472203954Srdivacky unsigned Width = VecTy.getSizeInBits(); 2473203954Srdivacky unsigned NumElts = Width / LdTy.getSizeInBits(); 2474203954Srdivacky EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 2475203954Srdivacky 2476203954Srdivacky unsigned Idx = 1; 2477203954Srdivacky SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 2478203954Srdivacky 2479203954Srdivacky for (unsigned i = Start + 1; i != End; ++i) { 2480203954Srdivacky EVT NewLdTy = LdOps[i].getValueType(); 2481203954Srdivacky if (NewLdTy != LdTy) { 2482203954Srdivacky NumElts = Width / NewLdTy.getSizeInBits(); 2483203954Srdivacky NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 2484218893Sdim VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 2485203954Srdivacky // Readjust position and vector position based on new load type 2486203954Srdivacky Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 2487203954Srdivacky LdTy = NewLdTy; 2488203954Srdivacky } 2489203954Srdivacky VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 2490203954Srdivacky DAG.getIntPtrConstant(Idx++)); 2491203954Srdivacky } 2492218893Sdim return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 2493203954Srdivacky} 2494203954Srdivacky 2495218893SdimSDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16> &LdChain, 2496218893Sdim LoadSDNode *LD) { 2497193323Sed // The strategy assumes that we can efficiently load powers of two widths. 2498203954Srdivacky // The routines chops the vector into the largest vector loads with the same 2499203954Srdivacky // element type or scalar loads and then recombines it to the widen vector 2500203954Srdivacky // type. 2501207618Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2502203954Srdivacky unsigned WidenWidth = WidenVT.getSizeInBits(); 2503203954Srdivacky EVT LdVT = LD->getMemoryVT(); 2504203954Srdivacky DebugLoc dl = LD->getDebugLoc(); 2505203954Srdivacky assert(LdVT.isVector() && WidenVT.isVector()); 2506203954Srdivacky assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 2507193323Sed 2508203954Srdivacky // Load information 2509203954Srdivacky SDValue Chain = LD->getChain(); 2510203954Srdivacky SDValue BasePtr = LD->getBasePtr(); 2511203954Srdivacky unsigned Align = LD->getAlignment(); 2512203954Srdivacky bool isVolatile = LD->isVolatile(); 2513203954Srdivacky bool isNonTemporal = LD->isNonTemporal(); 2514234353Sdim bool isInvariant = LD->isInvariant(); 2515193323Sed 2516203954Srdivacky int LdWidth = LdVT.getSizeInBits(); 2517203954Srdivacky int WidthDiff = WidenWidth - LdWidth; // Difference 2518203954Srdivacky unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads 2519203954Srdivacky 2520193323Sed // Find the vector type that can load from. 2521203954Srdivacky EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2522203954Srdivacky int NewVTWidth = NewVT.getSizeInBits(); 2523218893Sdim SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 2524234353Sdim isVolatile, isNonTemporal, isInvariant, Align); 2525193323Sed LdChain.push_back(LdOp.getValue(1)); 2526193323Sed 2527193323Sed // Check if we can load the element with one instruction 2528203954Srdivacky if (LdWidth <= NewVTWidth) { 2529212904Sdim if (!NewVT.isVector()) { 2530205407Srdivacky unsigned NumElts = WidenWidth / NewVTWidth; 2531203954Srdivacky EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2532203954Srdivacky SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 2533218893Sdim return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 2534203954Srdivacky } 2535212904Sdim if (NewVT == WidenVT) 2536212904Sdim return LdOp; 2537212904Sdim 2538212904Sdim assert(WidenWidth % NewVTWidth == 0); 2539212904Sdim unsigned NumConcat = WidenWidth / NewVTWidth; 2540212904Sdim SmallVector<SDValue, 16> ConcatOps(NumConcat); 2541212904Sdim SDValue UndefVal = DAG.getUNDEF(NewVT); 2542212904Sdim ConcatOps[0] = LdOp; 2543212904Sdim for (unsigned i = 1; i != NumConcat; ++i) 2544212904Sdim ConcatOps[i] = UndefVal; 2545212904Sdim return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], 2546212904Sdim NumConcat); 2547193323Sed } 2548193323Sed 2549203954Srdivacky // Load vector by using multiple loads from largest vector to scalar 2550203954Srdivacky SmallVector<SDValue, 16> LdOps; 2551203954Srdivacky LdOps.push_back(LdOp); 2552203954Srdivacky 2553203954Srdivacky LdWidth -= NewVTWidth; 2554193323Sed unsigned Offset = 0; 2555193323Sed 2556193323Sed while (LdWidth > 0) { 2557203954Srdivacky unsigned Increment = NewVTWidth / 8; 2558193323Sed Offset += Increment; 2559193323Sed BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2560193323Sed DAG.getIntPtrConstant(Increment)); 2561193323Sed 2562234353Sdim SDValue L; 2563203954Srdivacky if (LdWidth < NewVTWidth) { 2564203954Srdivacky // Our current type we are using is too large, find a better size 2565203954Srdivacky NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 2566203954Srdivacky NewVTWidth = NewVT.getSizeInBits(); 2567234353Sdim L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2568239462Sdim LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2569239462Sdim isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2570234353Sdim LdChain.push_back(L.getValue(1)); 2571234353Sdim if (L->getValueType(0).isVector()) { 2572234353Sdim SmallVector<SDValue, 16> Loads; 2573234353Sdim Loads.push_back(L); 2574234353Sdim unsigned size = L->getValueSizeInBits(0); 2575234353Sdim while (size < LdOp->getValueSizeInBits(0)) { 2576234353Sdim Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 2577234353Sdim size += L->getValueSizeInBits(0); 2578234353Sdim } 2579234353Sdim L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), 2580234353Sdim &Loads[0], Loads.size()); 2581234353Sdim } 2582234353Sdim } else { 2583234353Sdim L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 2584234353Sdim LD->getPointerInfo().getWithOffset(Offset), isVolatile, 2585234353Sdim isNonTemporal, isInvariant, MinAlign(Align, Increment)); 2586234353Sdim LdChain.push_back(L.getValue(1)); 2587234353Sdim } 2588193323Sed 2589234353Sdim LdOps.push_back(L); 2590234353Sdim 2591234353Sdim 2592203954Srdivacky LdWidth -= NewVTWidth; 2593193323Sed } 2594193323Sed 2595203954Srdivacky // Build the vector from the loads operations 2596203954Srdivacky unsigned End = LdOps.size(); 2597212904Sdim if (!LdOps[0].getValueType().isVector()) 2598212904Sdim // All the loads are scalar loads. 2599212904Sdim return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 2600218893Sdim 2601212904Sdim // If the load contains vectors, build the vector using concat vector. 2602212904Sdim // All of the vectors used to loads are power of 2 and the scalars load 2603212904Sdim // can be combined to make a power of 2 vector. 2604212904Sdim SmallVector<SDValue, 16> ConcatOps(End); 2605212904Sdim int i = End - 1; 2606212904Sdim int Idx = End; 2607212904Sdim EVT LdTy = LdOps[i].getValueType(); 2608212904Sdim // First combine the scalar loads to a vector 2609212904Sdim if (!LdTy.isVector()) { 2610212904Sdim for (--i; i >= 0; --i) { 2611212904Sdim LdTy = LdOps[i].getValueType(); 2612212904Sdim if (LdTy.isVector()) 2613212904Sdim break; 2614203954Srdivacky } 2615212904Sdim ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End); 2616212904Sdim } 2617212904Sdim ConcatOps[--Idx] = LdOps[i]; 2618212904Sdim for (--i; i >= 0; --i) { 2619212904Sdim EVT NewLdTy = LdOps[i].getValueType(); 2620212904Sdim if (NewLdTy != LdTy) { 2621212904Sdim // Create a larger vector 2622212904Sdim ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 2623212904Sdim &ConcatOps[Idx], End - Idx); 2624212904Sdim Idx = End - 1; 2625212904Sdim LdTy = NewLdTy; 2626212904Sdim } 2627203954Srdivacky ConcatOps[--Idx] = LdOps[i]; 2628212904Sdim } 2629203954Srdivacky 2630212904Sdim if (WidenWidth == LdTy.getSizeInBits()*(End - Idx)) 2631212904Sdim return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2632212904Sdim &ConcatOps[Idx], End - Idx); 2633212904Sdim 2634212904Sdim // We need to fill the rest with undefs to build the vector 2635212904Sdim unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 2636212904Sdim SmallVector<SDValue, 16> WidenOps(NumOps); 2637212904Sdim SDValue UndefVal = DAG.getUNDEF(LdTy); 2638212904Sdim { 2639212904Sdim unsigned i = 0; 2640212904Sdim for (; i != End-Idx; ++i) 2641212904Sdim WidenOps[i] = ConcatOps[Idx+i]; 2642212904Sdim for (; i != NumOps; ++i) 2643212904Sdim WidenOps[i] = UndefVal; 2644212904Sdim } 2645212904Sdim return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &WidenOps[0],NumOps); 2646193323Sed} 2647193323Sed 2648203954SrdivackySDValue 2649203954SrdivackyDAGTypeLegalizer::GenWidenVectorExtLoads(SmallVector<SDValue, 16>& LdChain, 2650203954Srdivacky LoadSDNode * LD, 2651203954Srdivacky ISD::LoadExtType ExtType) { 2652203954Srdivacky // For extension loads, it may not be more efficient to chop up the vector 2653203954Srdivacky // and then extended it. Instead, we unroll the load and build a new vector. 2654203954Srdivacky EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 2655203954Srdivacky EVT LdVT = LD->getMemoryVT(); 2656203954Srdivacky DebugLoc dl = LD->getDebugLoc(); 2657203954Srdivacky assert(LdVT.isVector() && WidenVT.isVector()); 2658193323Sed 2659203954Srdivacky // Load information 2660203954Srdivacky SDValue Chain = LD->getChain(); 2661203954Srdivacky SDValue BasePtr = LD->getBasePtr(); 2662203954Srdivacky unsigned Align = LD->getAlignment(); 2663203954Srdivacky bool isVolatile = LD->isVolatile(); 2664203954Srdivacky bool isNonTemporal = LD->isNonTemporal(); 2665193323Sed 2666203954Srdivacky EVT EltVT = WidenVT.getVectorElementType(); 2667203954Srdivacky EVT LdEltVT = LdVT.getVectorElementType(); 2668203954Srdivacky unsigned NumElts = LdVT.getVectorNumElements(); 2669193323Sed 2670203954Srdivacky // Load each element and widen 2671203954Srdivacky unsigned WidenNumElts = WidenVT.getVectorNumElements(); 2672203954Srdivacky SmallVector<SDValue, 16> Ops(WidenNumElts); 2673203954Srdivacky unsigned Increment = LdEltVT.getSizeInBits() / 8; 2674218893Sdim Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, 2675218893Sdim LD->getPointerInfo(), 2676203954Srdivacky LdEltVT, isVolatile, isNonTemporal, Align); 2677203954Srdivacky LdChain.push_back(Ops[0].getValue(1)); 2678203954Srdivacky unsigned i = 0, Offset = Increment; 2679203954Srdivacky for (i=1; i < NumElts; ++i, Offset += Increment) { 2680203954Srdivacky SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2681203954Srdivacky BasePtr, DAG.getIntPtrConstant(Offset)); 2682218893Sdim Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 2683218893Sdim LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 2684218893Sdim isVolatile, isNonTemporal, Align); 2685203954Srdivacky LdChain.push_back(Ops[i].getValue(1)); 2686193323Sed } 2687193323Sed 2688203954Srdivacky // Fill the rest with undefs 2689203954Srdivacky SDValue UndefVal = DAG.getUNDEF(EltVT); 2690203954Srdivacky for (; i != WidenNumElts; ++i) 2691203954Srdivacky Ops[i] = UndefVal; 2692193323Sed 2693203954Srdivacky return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size()); 2694203954Srdivacky} 2695193323Sed 2696193323Sed 2697203954Srdivackyvoid DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, 2698203954Srdivacky StoreSDNode *ST) { 2699203954Srdivacky // The strategy assumes that we can efficiently store powers of two widths. 2700203954Srdivacky // The routines chops the vector into the largest vector stores with the same 2701203954Srdivacky // element type or scalar stores. 2702203954Srdivacky SDValue Chain = ST->getChain(); 2703203954Srdivacky SDValue BasePtr = ST->getBasePtr(); 2704203954Srdivacky unsigned Align = ST->getAlignment(); 2705203954Srdivacky bool isVolatile = ST->isVolatile(); 2706203954Srdivacky bool isNonTemporal = ST->isNonTemporal(); 2707203954Srdivacky SDValue ValOp = GetWidenedVector(ST->getValue()); 2708203954Srdivacky DebugLoc dl = ST->getDebugLoc(); 2709203954Srdivacky 2710203954Srdivacky EVT StVT = ST->getMemoryVT(); 2711203954Srdivacky unsigned StWidth = StVT.getSizeInBits(); 2712203954Srdivacky EVT ValVT = ValOp.getValueType(); 2713203954Srdivacky unsigned ValWidth = ValVT.getSizeInBits(); 2714203954Srdivacky EVT ValEltVT = ValVT.getVectorElementType(); 2715203954Srdivacky unsigned ValEltWidth = ValEltVT.getSizeInBits(); 2716203954Srdivacky assert(StVT.getVectorElementType() == ValEltVT); 2717203954Srdivacky 2718203954Srdivacky int Idx = 0; // current index to store 2719203954Srdivacky unsigned Offset = 0; // offset from base to store 2720203954Srdivacky while (StWidth != 0) { 2721203954Srdivacky // Find the largest vector type we can store with 2722203954Srdivacky EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 2723203954Srdivacky unsigned NewVTWidth = NewVT.getSizeInBits(); 2724203954Srdivacky unsigned Increment = NewVTWidth / 8; 2725203954Srdivacky if (NewVT.isVector()) { 2726203954Srdivacky unsigned NumVTElts = NewVT.getVectorNumElements(); 2727203954Srdivacky do { 2728203954Srdivacky SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 2729203954Srdivacky DAG.getIntPtrConstant(Idx)); 2730218893Sdim StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2731218893Sdim ST->getPointerInfo().getWithOffset(Offset), 2732218893Sdim isVolatile, isNonTemporal, 2733203954Srdivacky MinAlign(Align, Offset))); 2734203954Srdivacky StWidth -= NewVTWidth; 2735203954Srdivacky Offset += Increment; 2736203954Srdivacky Idx += NumVTElts; 2737203954Srdivacky BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2738203954Srdivacky DAG.getIntPtrConstant(Increment)); 2739203954Srdivacky } while (StWidth != 0 && StWidth >= NewVTWidth); 2740203954Srdivacky } else { 2741203954Srdivacky // Cast the vector to the scalar type we can store 2742203954Srdivacky unsigned NumElts = ValWidth / NewVTWidth; 2743203954Srdivacky EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 2744218893Sdim SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 2745203954Srdivacky // Readjust index position based on new vector type 2746203954Srdivacky Idx = Idx * ValEltWidth / NewVTWidth; 2747203954Srdivacky do { 2748203954Srdivacky SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 2749193323Sed DAG.getIntPtrConstant(Idx++)); 2750218893Sdim StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, 2751218893Sdim ST->getPointerInfo().getWithOffset(Offset), 2752218893Sdim isVolatile, isNonTemporal, 2753218893Sdim MinAlign(Align, Offset))); 2754203954Srdivacky StWidth -= NewVTWidth; 2755203954Srdivacky Offset += Increment; 2756203954Srdivacky BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 2757203954Srdivacky DAG.getIntPtrConstant(Increment)); 2758239462Sdim } while (StWidth != 0 && StWidth >= NewVTWidth); 2759203954Srdivacky // Restore index back to be relative to the original widen element type 2760203954Srdivacky Idx = Idx * NewVTWidth / ValEltWidth; 2761203954Srdivacky } 2762193323Sed } 2763193323Sed} 2764193323Sed 2765203954Srdivackyvoid 2766203954SrdivackyDAGTypeLegalizer::GenWidenVectorTruncStores(SmallVector<SDValue, 16>& StChain, 2767203954Srdivacky StoreSDNode *ST) { 2768203954Srdivacky // For extension loads, it may not be more efficient to truncate the vector 2769203954Srdivacky // and then store it. Instead, we extract each element and then store it. 2770203954Srdivacky SDValue Chain = ST->getChain(); 2771203954Srdivacky SDValue BasePtr = ST->getBasePtr(); 2772203954Srdivacky unsigned Align = ST->getAlignment(); 2773203954Srdivacky bool isVolatile = ST->isVolatile(); 2774203954Srdivacky bool isNonTemporal = ST->isNonTemporal(); 2775203954Srdivacky SDValue ValOp = GetWidenedVector(ST->getValue()); 2776203954Srdivacky DebugLoc dl = ST->getDebugLoc(); 2777218893Sdim 2778203954Srdivacky EVT StVT = ST->getMemoryVT(); 2779203954Srdivacky EVT ValVT = ValOp.getValueType(); 2780203954Srdivacky 2781203954Srdivacky // It must be true that we the widen vector type is bigger than where 2782203954Srdivacky // we need to store. 2783203954Srdivacky assert(StVT.isVector() && ValOp.getValueType().isVector()); 2784203954Srdivacky assert(StVT.bitsLT(ValOp.getValueType())); 2785203954Srdivacky 2786203954Srdivacky // For truncating stores, we can not play the tricks of chopping legal 2787203954Srdivacky // vector types and bit cast it to the right type. Instead, we unroll 2788203954Srdivacky // the store. 2789203954Srdivacky EVT StEltVT = StVT.getVectorElementType(); 2790203954Srdivacky EVT ValEltVT = ValVT.getVectorElementType(); 2791203954Srdivacky unsigned Increment = ValEltVT.getSizeInBits() / 8; 2792203954Srdivacky unsigned NumElts = StVT.getVectorNumElements(); 2793203954Srdivacky SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2794203954Srdivacky DAG.getIntPtrConstant(0)); 2795218893Sdim StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 2796218893Sdim ST->getPointerInfo(), StEltVT, 2797203954Srdivacky isVolatile, isNonTemporal, Align)); 2798203954Srdivacky unsigned Offset = Increment; 2799203954Srdivacky for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 2800203954Srdivacky SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 2801203954Srdivacky BasePtr, DAG.getIntPtrConstant(Offset)); 2802203954Srdivacky SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 2803203954Srdivacky DAG.getIntPtrConstant(0)); 2804218893Sdim StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, 2805218893Sdim ST->getPointerInfo().getWithOffset(Offset), 2806218893Sdim StEltVT, isVolatile, isNonTemporal, 2807203954Srdivacky MinAlign(Align, Offset))); 2808203954Srdivacky } 2809203954Srdivacky} 2810203954Srdivacky 2811193323Sed/// Modifies a vector input (widen or narrows) to a vector of NVT. The 2812193323Sed/// input vector must have the same element type as NVT. 2813198090SrdivackySDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) { 2814193323Sed // Note that InOp might have been widened so it might already have 2815193323Sed // the right width or it might need be narrowed. 2816198090Srdivacky EVT InVT = InOp.getValueType(); 2817193323Sed assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 2818193323Sed "input and widen element type must match"); 2819193323Sed DebugLoc dl = InOp.getDebugLoc(); 2820193323Sed 2821193323Sed // Check if InOp already has the right width. 2822193323Sed if (InVT == NVT) 2823193323Sed return InOp; 2824193323Sed 2825193323Sed unsigned InNumElts = InVT.getVectorNumElements(); 2826193323Sed unsigned WidenNumElts = NVT.getVectorNumElements(); 2827193323Sed if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 2828193323Sed unsigned NumConcat = WidenNumElts / InNumElts; 2829193323Sed SmallVector<SDValue, 16> Ops(NumConcat); 2830193323Sed SDValue UndefVal = DAG.getUNDEF(InVT); 2831193323Sed Ops[0] = InOp; 2832193323Sed for (unsigned i = 1; i != NumConcat; ++i) 2833193323Sed Ops[i] = UndefVal; 2834193323Sed 2835193323Sed return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat); 2836193323Sed } 2837193323Sed 2838193323Sed if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 2839193323Sed return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 2840193323Sed DAG.getIntPtrConstant(0)); 2841193323Sed 2842193323Sed // Fall back to extract and build. 2843193323Sed SmallVector<SDValue, 16> Ops(WidenNumElts); 2844198090Srdivacky EVT EltVT = NVT.getVectorElementType(); 2845193323Sed unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 2846193323Sed unsigned Idx; 2847193323Sed for (Idx = 0; Idx < MinNumElts; ++Idx) 2848193323Sed Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 2849193323Sed DAG.getIntPtrConstant(Idx)); 2850193323Sed 2851193323Sed SDValue UndefVal = DAG.getUNDEF(EltVT); 2852193323Sed for ( ; Idx < WidenNumElts; ++Idx) 2853193323Sed Ops[Idx] = UndefVal; 2854193323Sed return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts); 2855193323Sed} 2856