LegalizeVectorTypes.cpp revision 360784
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file performs vector type splitting and scalarization for LegalizeTypes. 10// Scalarization is the act of changing a computation in an illegal one-element 11// vector type to be a computation in its scalar element type. For example, 12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 13// as a base case when scalarizing vector arithmetic like <4 x f32>, which 14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 15// types. 16// Splitting is the act of changing a computation in an invalid vector type to 17// be a computation in two vectors of half the size. For example, implementing 18// <128 x f32> operations in terms of two <64 x f32> operations. 19// 20//===----------------------------------------------------------------------===// 21 22#include "LegalizeTypes.h" 23#include "llvm/IR/DataLayout.h" 24#include "llvm/Support/ErrorHandling.h" 25#include "llvm/Support/raw_ostream.h" 26#include "llvm/Support/TypeSize.h" 27using namespace llvm; 28 29#define DEBUG_TYPE "legalize-types" 30 31//===----------------------------------------------------------------------===// 32// Result Vector Scalarization: <1 x ty> -> ty. 33//===----------------------------------------------------------------------===// 34 35void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 36 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 37 dbgs() << "\n"); 38 SDValue R = SDValue(); 39 40 switch (N->getOpcode()) { 41 default: 42#ifndef NDEBUG 43 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 44 N->dump(&DAG); 45 dbgs() << "\n"; 46#endif 47 report_fatal_error("Do not know how to scalarize the result of this " 48 "operator!\n"); 49 50 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 51 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 52 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 53 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 54 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 55 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 56 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 57 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 58 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 59 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 60 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 61 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 62 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 63 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 64 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 65 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 66 case ISD::ANY_EXTEND_VECTOR_INREG: 67 case ISD::SIGN_EXTEND_VECTOR_INREG: 68 case ISD::ZERO_EXTEND_VECTOR_INREG: 69 R = ScalarizeVecRes_VecInregOp(N); 70 break; 71 case ISD::ABS: 72 case ISD::ANY_EXTEND: 73 case ISD::BITREVERSE: 74 case ISD::BSWAP: 75 case ISD::CTLZ: 76 case ISD::CTLZ_ZERO_UNDEF: 77 case ISD::CTPOP: 78 case ISD::CTTZ: 79 case ISD::CTTZ_ZERO_UNDEF: 80 case ISD::FABS: 81 case ISD::FCEIL: 82 case ISD::FCOS: 83 case ISD::FEXP: 84 case ISD::FEXP2: 85 case ISD::FFLOOR: 86 case ISD::FLOG: 87 case ISD::FLOG10: 88 case ISD::FLOG2: 89 case ISD::FNEARBYINT: 90 case ISD::FNEG: 91 case ISD::FP_EXTEND: 92 case ISD::FP_TO_SINT: 93 case ISD::FP_TO_UINT: 94 case ISD::FRINT: 95 case ISD::FROUND: 96 case ISD::FSIN: 97 case ISD::FSQRT: 98 case ISD::FTRUNC: 99 case ISD::SIGN_EXTEND: 100 case ISD::SINT_TO_FP: 101 case ISD::TRUNCATE: 102 case ISD::UINT_TO_FP: 103 case ISD::ZERO_EXTEND: 104 case ISD::FCANONICALIZE: 105 R = ScalarizeVecRes_UnaryOp(N); 106 break; 107 108 case ISD::ADD: 109 case ISD::AND: 110 case ISD::FADD: 111 case ISD::FCOPYSIGN: 112 case ISD::FDIV: 113 case ISD::FMUL: 114 case ISD::FMINNUM: 115 case ISD::FMAXNUM: 116 case ISD::FMINNUM_IEEE: 117 case ISD::FMAXNUM_IEEE: 118 case ISD::FMINIMUM: 119 case ISD::FMAXIMUM: 120 case ISD::SMIN: 121 case ISD::SMAX: 122 case ISD::UMIN: 123 case ISD::UMAX: 124 125 case ISD::SADDSAT: 126 case ISD::UADDSAT: 127 case ISD::SSUBSAT: 128 case ISD::USUBSAT: 129 130 case ISD::FPOW: 131 case ISD::FREM: 132 case ISD::FSUB: 133 case ISD::MUL: 134 case ISD::OR: 135 case ISD::SDIV: 136 case ISD::SREM: 137 case ISD::SUB: 138 case ISD::UDIV: 139 case ISD::UREM: 140 case ISD::XOR: 141 case ISD::SHL: 142 case ISD::SRA: 143 case ISD::SRL: 144 R = ScalarizeVecRes_BinOp(N); 145 break; 146 case ISD::FMA: 147 R = ScalarizeVecRes_TernaryOp(N); 148 break; 149 150#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 151 case ISD::STRICT_##DAGN: 152#include "llvm/IR/ConstrainedOps.def" 153 R = ScalarizeVecRes_StrictFPOp(N); 154 break; 155 156 case ISD::UADDO: 157 case ISD::SADDO: 158 case ISD::USUBO: 159 case ISD::SSUBO: 160 case ISD::UMULO: 161 case ISD::SMULO: 162 R = ScalarizeVecRes_OverflowOp(N, ResNo); 163 break; 164 case ISD::SMULFIX: 165 case ISD::SMULFIXSAT: 166 case ISD::UMULFIX: 167 case ISD::UMULFIXSAT: 168 case ISD::SDIVFIX: 169 case ISD::UDIVFIX: 170 R = ScalarizeVecRes_FIX(N); 171 break; 172 } 173 174 // If R is null, the sub-method took care of registering the result. 175 if (R.getNode()) 176 SetScalarizedVector(SDValue(N, ResNo), R); 177} 178 179SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 180 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 181 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 182 return DAG.getNode(N->getOpcode(), SDLoc(N), 183 LHS.getValueType(), LHS, RHS, N->getFlags()); 184} 185 186SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 187 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 188 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 189 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 190 return DAG.getNode(N->getOpcode(), SDLoc(N), 191 Op0.getValueType(), Op0, Op1, Op2); 192} 193 194SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) { 195 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 196 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 197 SDValue Op2 = N->getOperand(2); 198 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1, 199 Op2); 200} 201 202SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) { 203 EVT VT = N->getValueType(0).getVectorElementType(); 204 unsigned NumOpers = N->getNumOperands(); 205 SDValue Chain = N->getOperand(0); 206 EVT ValueVTs[] = {VT, MVT::Other}; 207 SDLoc dl(N); 208 209 SmallVector<SDValue, 4> Opers(NumOpers); 210 211 // The Chain is the first operand. 212 Opers[0] = Chain; 213 214 // Now process the remaining operands. 215 for (unsigned i = 1; i < NumOpers; ++i) { 216 SDValue Oper = N->getOperand(i); 217 218 if (Oper.getValueType().isVector()) 219 Oper = GetScalarizedVector(Oper); 220 221 Opers[i] = Oper; 222 } 223 224 SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers); 225 226 // Legalize the chain result - switch anything that used the old chain to 227 // use the new one. 228 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 229 return Result; 230} 231 232SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, 233 unsigned ResNo) { 234 SDLoc DL(N); 235 EVT ResVT = N->getValueType(0); 236 EVT OvVT = N->getValueType(1); 237 238 SDValue ScalarLHS, ScalarRHS; 239 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) { 240 ScalarLHS = GetScalarizedVector(N->getOperand(0)); 241 ScalarRHS = GetScalarizedVector(N->getOperand(1)); 242 } else { 243 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS; 244 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS); 245 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS); 246 ScalarLHS = ElemsLHS[0]; 247 ScalarRHS = ElemsRHS[0]; 248 } 249 250 SDVTList ScalarVTs = DAG.getVTList( 251 ResVT.getVectorElementType(), OvVT.getVectorElementType()); 252 SDNode *ScalarNode = DAG.getNode( 253 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode(); 254 255 // Replace the other vector result not being explicitly scalarized here. 256 unsigned OtherNo = 1 - ResNo; 257 EVT OtherVT = N->getValueType(OtherNo); 258 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) { 259 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo)); 260 } else { 261 SDValue OtherVal = DAG.getNode( 262 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo)); 263 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 264 } 265 266 return SDValue(ScalarNode, ResNo); 267} 268 269SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 270 unsigned ResNo) { 271 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 272 return GetScalarizedVector(Op); 273} 274 275SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 276 SDValue Op = N->getOperand(0); 277 if (Op.getValueType().isVector() 278 && Op.getValueType().getVectorNumElements() == 1 279 && !isSimpleLegalType(Op.getValueType())) 280 Op = GetScalarizedVector(Op); 281 EVT NewVT = N->getValueType(0).getVectorElementType(); 282 return DAG.getNode(ISD::BITCAST, SDLoc(N), 283 NewVT, Op); 284} 285 286SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 287 EVT EltVT = N->getValueType(0).getVectorElementType(); 288 SDValue InOp = N->getOperand(0); 289 // The BUILD_VECTOR operands may be of wider element types and 290 // we may need to truncate them back to the requested return type. 291 if (EltVT.isInteger()) 292 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 293 return InOp; 294} 295 296SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 297 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 298 N->getValueType(0).getVectorElementType(), 299 N->getOperand(0), N->getOperand(1)); 300} 301 302SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 303 EVT NewVT = N->getValueType(0).getVectorElementType(); 304 SDValue Op = GetScalarizedVector(N->getOperand(0)); 305 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 306 NewVT, Op, N->getOperand(1)); 307} 308 309SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 310 SDValue Op = GetScalarizedVector(N->getOperand(0)); 311 return DAG.getNode(ISD::FPOWI, SDLoc(N), 312 Op.getValueType(), Op, N->getOperand(1)); 313} 314 315SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 316 // The value to insert may have a wider type than the vector element type, 317 // so be sure to truncate it to the element type if necessary. 318 SDValue Op = N->getOperand(1); 319 EVT EltVT = N->getValueType(0).getVectorElementType(); 320 if (Op.getValueType() != EltVT) 321 // FIXME: Can this happen for floating point types? 322 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 323 return Op; 324} 325 326SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 327 assert(N->isUnindexed() && "Indexed vector load?"); 328 329 SDValue Result = DAG.getLoad( 330 ISD::UNINDEXED, N->getExtensionType(), 331 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(), 332 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()), 333 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(), 334 N->getOriginalAlignment(), N->getMemOperand()->getFlags(), 335 N->getAAInfo()); 336 337 // Legalize the chain result - switch anything that used the old chain to 338 // use the new one. 339 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 340 return Result; 341} 342 343SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 344 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 345 EVT DestVT = N->getValueType(0).getVectorElementType(); 346 SDValue Op = N->getOperand(0); 347 EVT OpVT = Op.getValueType(); 348 SDLoc DL(N); 349 // The result needs scalarizing, but it's not a given that the source does. 350 // This is a workaround for targets where it's impossible to scalarize the 351 // result of a conversion, because the source type is legal. 352 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32} 353 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is 354 // legal and was not scalarized. 355 // See the similar logic in ScalarizeVecRes_SETCC 356 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 357 Op = GetScalarizedVector(Op); 358 } else { 359 EVT VT = OpVT.getVectorElementType(); 360 Op = DAG.getNode( 361 ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 362 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 363 } 364 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op); 365} 366 367SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 368 EVT EltVT = N->getValueType(0).getVectorElementType(); 369 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 370 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 371 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 372 LHS, DAG.getValueType(ExtVT)); 373} 374 375SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) { 376 SDLoc DL(N); 377 SDValue Op = N->getOperand(0); 378 379 EVT OpVT = Op.getValueType(); 380 EVT OpEltVT = OpVT.getVectorElementType(); 381 EVT EltVT = N->getValueType(0).getVectorElementType(); 382 383 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 384 Op = GetScalarizedVector(Op); 385 } else { 386 Op = DAG.getNode( 387 ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op, 388 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 389 } 390 391 switch (N->getOpcode()) { 392 case ISD::ANY_EXTEND_VECTOR_INREG: 393 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op); 394 case ISD::SIGN_EXTEND_VECTOR_INREG: 395 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op); 396 case ISD::ZERO_EXTEND_VECTOR_INREG: 397 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op); 398 } 399 400 llvm_unreachable("Illegal extend_vector_inreg opcode"); 401} 402 403SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 404 // If the operand is wider than the vector element type then it is implicitly 405 // truncated. Make that explicit here. 406 EVT EltVT = N->getValueType(0).getVectorElementType(); 407 SDValue InOp = N->getOperand(0); 408 if (InOp.getValueType() != EltVT) 409 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 410 return InOp; 411} 412 413SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 414 SDValue Cond = N->getOperand(0); 415 EVT OpVT = Cond.getValueType(); 416 SDLoc DL(N); 417 // The vselect result and true/value operands needs scalarizing, but it's 418 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal. 419 // See the similar logic in ScalarizeVecRes_SETCC 420 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 421 Cond = GetScalarizedVector(Cond); 422 } else { 423 EVT VT = OpVT.getVectorElementType(); 424 Cond = DAG.getNode( 425 ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond, 426 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 427 } 428 429 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 430 TargetLowering::BooleanContent ScalarBool = 431 TLI.getBooleanContents(false, false); 432 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 433 434 // If integer and float booleans have different contents then we can't 435 // reliably optimize in all cases. There is a full explanation for this in 436 // DAGCombiner::visitSELECT() where the same issue affects folding 437 // (select C, 0, 1) to (xor C, 1). 438 if (TLI.getBooleanContents(false, false) != 439 TLI.getBooleanContents(false, true)) { 440 // At least try the common case where the boolean is generated by a 441 // comparison. 442 if (Cond->getOpcode() == ISD::SETCC) { 443 EVT OpVT = Cond->getOperand(0).getValueType(); 444 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 445 VecBool = TLI.getBooleanContents(OpVT); 446 } else 447 ScalarBool = TargetLowering::UndefinedBooleanContent; 448 } 449 450 EVT CondVT = Cond.getValueType(); 451 if (ScalarBool != VecBool) { 452 switch (ScalarBool) { 453 case TargetLowering::UndefinedBooleanContent: 454 break; 455 case TargetLowering::ZeroOrOneBooleanContent: 456 assert(VecBool == TargetLowering::UndefinedBooleanContent || 457 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 458 // Vector read from all ones, scalar expects a single 1 so mask. 459 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 460 Cond, DAG.getConstant(1, SDLoc(N), CondVT)); 461 break; 462 case TargetLowering::ZeroOrNegativeOneBooleanContent: 463 assert(VecBool == TargetLowering::UndefinedBooleanContent || 464 VecBool == TargetLowering::ZeroOrOneBooleanContent); 465 // Vector reads from a one, scalar from all ones so sign extend. 466 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 467 Cond, DAG.getValueType(MVT::i1)); 468 break; 469 } 470 } 471 472 // Truncate the condition if needed 473 auto BoolVT = getSetCCResultType(CondVT); 474 if (BoolVT.bitsLT(CondVT)) 475 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond); 476 477 return DAG.getSelect(SDLoc(N), 478 LHS.getValueType(), Cond, LHS, 479 GetScalarizedVector(N->getOperand(2))); 480} 481 482SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 483 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 484 return DAG.getSelect(SDLoc(N), 485 LHS.getValueType(), N->getOperand(0), LHS, 486 GetScalarizedVector(N->getOperand(2))); 487} 488 489SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 490 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 491 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 492 N->getOperand(0), N->getOperand(1), 493 LHS, GetScalarizedVector(N->getOperand(3)), 494 N->getOperand(4)); 495} 496 497SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 498 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 499} 500 501SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 502 // Figure out if the scalar is the LHS or RHS and return it. 503 SDValue Arg = N->getOperand(2).getOperand(0); 504 if (Arg.isUndef()) 505 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 506 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 507 return GetScalarizedVector(N->getOperand(Op)); 508} 509 510SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 511 assert(N->getValueType(0).isVector() && 512 N->getOperand(0).getValueType().isVector() && 513 "Operand types must be vectors"); 514 SDValue LHS = N->getOperand(0); 515 SDValue RHS = N->getOperand(1); 516 EVT OpVT = LHS.getValueType(); 517 EVT NVT = N->getValueType(0).getVectorElementType(); 518 SDLoc DL(N); 519 520 // The result needs scalarizing, but it's not a given that the source does. 521 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 522 LHS = GetScalarizedVector(LHS); 523 RHS = GetScalarizedVector(RHS); 524 } else { 525 EVT VT = OpVT.getVectorElementType(); 526 LHS = DAG.getNode( 527 ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 528 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 529 RHS = DAG.getNode( 530 ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 531 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 532 } 533 534 // Turn it into a scalar SETCC. 535 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 536 N->getOperand(2)); 537 // Vectors may have a different boolean contents to scalars. Promote the 538 // value appropriately. 539 ISD::NodeType ExtendCode = 540 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 541 return DAG.getNode(ExtendCode, DL, NVT, Res); 542} 543 544 545//===----------------------------------------------------------------------===// 546// Operand Vector Scalarization <1 x ty> -> ty. 547//===----------------------------------------------------------------------===// 548 549bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 550 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 551 dbgs() << "\n"); 552 SDValue Res = SDValue(); 553 554 if (!Res.getNode()) { 555 switch (N->getOpcode()) { 556 default: 557#ifndef NDEBUG 558 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 559 N->dump(&DAG); 560 dbgs() << "\n"; 561#endif 562 report_fatal_error("Do not know how to scalarize this operator's " 563 "operand!\n"); 564 case ISD::BITCAST: 565 Res = ScalarizeVecOp_BITCAST(N); 566 break; 567 case ISD::ANY_EXTEND: 568 case ISD::ZERO_EXTEND: 569 case ISD::SIGN_EXTEND: 570 case ISD::TRUNCATE: 571 case ISD::FP_TO_SINT: 572 case ISD::FP_TO_UINT: 573 case ISD::SINT_TO_FP: 574 case ISD::UINT_TO_FP: 575 Res = ScalarizeVecOp_UnaryOp(N); 576 break; 577 case ISD::STRICT_SINT_TO_FP: 578 case ISD::STRICT_UINT_TO_FP: 579 case ISD::STRICT_FP_TO_SINT: 580 case ISD::STRICT_FP_TO_UINT: 581 Res = ScalarizeVecOp_UnaryOp_StrictFP(N); 582 break; 583 case ISD::CONCAT_VECTORS: 584 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 585 break; 586 case ISD::EXTRACT_VECTOR_ELT: 587 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 588 break; 589 case ISD::VSELECT: 590 Res = ScalarizeVecOp_VSELECT(N); 591 break; 592 case ISD::SETCC: 593 Res = ScalarizeVecOp_VSETCC(N); 594 break; 595 case ISD::STORE: 596 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 597 break; 598 case ISD::STRICT_FP_ROUND: 599 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo); 600 break; 601 case ISD::FP_ROUND: 602 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 603 break; 604 case ISD::VECREDUCE_FADD: 605 case ISD::VECREDUCE_FMUL: 606 case ISD::VECREDUCE_ADD: 607 case ISD::VECREDUCE_MUL: 608 case ISD::VECREDUCE_AND: 609 case ISD::VECREDUCE_OR: 610 case ISD::VECREDUCE_XOR: 611 case ISD::VECREDUCE_SMAX: 612 case ISD::VECREDUCE_SMIN: 613 case ISD::VECREDUCE_UMAX: 614 case ISD::VECREDUCE_UMIN: 615 case ISD::VECREDUCE_FMAX: 616 case ISD::VECREDUCE_FMIN: 617 Res = ScalarizeVecOp_VECREDUCE(N); 618 break; 619 } 620 } 621 622 // If the result is null, the sub-method took care of registering results etc. 623 if (!Res.getNode()) return false; 624 625 // If the result is N, the sub-method updated N in place. Tell the legalizer 626 // core about this. 627 if (Res.getNode() == N) 628 return true; 629 630 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 631 "Invalid operand expansion"); 632 633 ReplaceValueWith(SDValue(N, 0), Res); 634 return false; 635} 636 637/// If the value to convert is a vector that needs to be scalarized, it must be 638/// <1 x ty>. Convert the element instead. 639SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 640 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 641 return DAG.getNode(ISD::BITCAST, SDLoc(N), 642 N->getValueType(0), Elt); 643} 644 645/// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 646/// Do the operation on the element instead. 647SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 648 assert(N->getValueType(0).getVectorNumElements() == 1 && 649 "Unexpected vector type!"); 650 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 651 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 652 N->getValueType(0).getScalarType(), Elt); 653 // Revectorize the result so the types line up with what the uses of this 654 // expression expect. 655 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op); 656} 657 658/// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 659/// Do the strict FP operation on the element instead. 660SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) { 661 assert(N->getValueType(0).getVectorNumElements() == 1 && 662 "Unexpected vector type!"); 663 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 664 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 665 { N->getValueType(0).getScalarType(), MVT::Other }, 666 { N->getOperand(0), Elt }); 667 // Legalize the chain result - switch anything that used the old chain to 668 // use the new one. 669 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 670 // Revectorize the result so the types line up with what the uses of this 671 // expression expect. 672 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 673 674 // Do our own replacement and return SDValue() to tell the caller that we 675 // handled all replacements since caller can only handle a single result. 676 ReplaceValueWith(SDValue(N, 0), Res); 677 return SDValue(); 678} 679 680/// The vectors to concatenate have length one - use a BUILD_VECTOR instead. 681SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 682 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 683 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 684 Ops[i] = GetScalarizedVector(N->getOperand(i)); 685 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops); 686} 687 688/// If the input is a vector that needs to be scalarized, it must be <1 x ty>, 689/// so just return the element, ignoring the index. 690SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 691 EVT VT = N->getValueType(0); 692 SDValue Res = GetScalarizedVector(N->getOperand(0)); 693 if (Res.getValueType() != VT) 694 Res = VT.isFloatingPoint() 695 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res) 696 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res); 697 return Res; 698} 699 700/// If the input condition is a vector that needs to be scalarized, it must be 701/// <1 x i1>, so just convert to a normal ISD::SELECT 702/// (still with vector output type since that was acceptable if we got here). 703SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 704 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 705 EVT VT = N->getValueType(0); 706 707 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 708 N->getOperand(2)); 709} 710 711/// If the operand is a vector that needs to be scalarized then the 712/// result must be v1i1, so just convert to a scalar SETCC and wrap 713/// with a scalar_to_vector since the res type is legal if we got here 714SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) { 715 assert(N->getValueType(0).isVector() && 716 N->getOperand(0).getValueType().isVector() && 717 "Operand types must be vectors"); 718 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type"); 719 720 EVT VT = N->getValueType(0); 721 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 722 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 723 724 EVT OpVT = N->getOperand(0).getValueType(); 725 EVT NVT = VT.getVectorElementType(); 726 SDLoc DL(N); 727 // Turn it into a scalar SETCC. 728 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 729 N->getOperand(2)); 730 731 // Vectors may have a different boolean contents to scalars. Promote the 732 // value appropriately. 733 ISD::NodeType ExtendCode = 734 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 735 736 Res = DAG.getNode(ExtendCode, DL, NVT, Res); 737 738 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res); 739} 740 741/// If the value to store is a vector that needs to be scalarized, it must be 742/// <1 x ty>. Just store the element. 743SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 744 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 745 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 746 SDLoc dl(N); 747 748 if (N->isTruncatingStore()) 749 return DAG.getTruncStore( 750 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 751 N->getBasePtr(), N->getPointerInfo(), 752 N->getMemoryVT().getVectorElementType(), N->getAlignment(), 753 N->getMemOperand()->getFlags(), N->getAAInfo()); 754 755 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 756 N->getBasePtr(), N->getPointerInfo(), 757 N->getOriginalAlignment(), N->getMemOperand()->getFlags(), 758 N->getAAInfo()); 759} 760 761/// If the value to round is a vector that needs to be scalarized, it must be 762/// <1 x ty>. Convert the element instead. 763SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 764 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 765 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 766 N->getValueType(0).getVectorElementType(), Elt, 767 N->getOperand(1)); 768 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 769} 770 771SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, 772 unsigned OpNo) { 773 assert(OpNo == 1 && "Wrong operand for scalarization!"); 774 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 775 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 776 { N->getValueType(0).getVectorElementType(), 777 MVT::Other }, 778 { N->getOperand(0), Elt, N->getOperand(2) }); 779 // Legalize the chain result - switch anything that used the old chain to 780 // use the new one. 781 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 782 783 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 784 785 // Do our own replacement and return SDValue() to tell the caller that we 786 // handled all replacements since caller can only handle a single result. 787 ReplaceValueWith(SDValue(N, 0), Res); 788 return SDValue(); 789} 790 791SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) { 792 SDValue Res = GetScalarizedVector(N->getOperand(0)); 793 // Result type may be wider than element type. 794 if (Res.getValueType() != N->getValueType(0)) 795 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res); 796 return Res; 797} 798 799//===----------------------------------------------------------------------===// 800// Result Vector Splitting 801//===----------------------------------------------------------------------===// 802 803/// This method is called when the specified result of the specified node is 804/// found to need vector splitting. At this point, the node may also have 805/// invalid operands or may have other results that need legalization, we just 806/// know that (at least) one result needs vector splitting. 807void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 808 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n"); 809 SDValue Lo, Hi; 810 811 // See if the target wants to custom expand this node. 812 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 813 return; 814 815 switch (N->getOpcode()) { 816 default: 817#ifndef NDEBUG 818 dbgs() << "SplitVectorResult #" << ResNo << ": "; 819 N->dump(&DAG); 820 dbgs() << "\n"; 821#endif 822 report_fatal_error("Do not know how to split the result of this " 823 "operator!\n"); 824 825 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 826 case ISD::VSELECT: 827 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 828 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 829 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 830 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 831 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 832 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 833 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 834 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 835 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 836 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break; 837 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 838 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break; 839 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 840 case ISD::LOAD: 841 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 842 break; 843 case ISD::MLOAD: 844 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi); 845 break; 846 case ISD::MGATHER: 847 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi); 848 break; 849 case ISD::SETCC: 850 SplitVecRes_SETCC(N, Lo, Hi); 851 break; 852 case ISD::VECTOR_SHUFFLE: 853 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 854 break; 855 case ISD::VAARG: 856 SplitVecRes_VAARG(N, Lo, Hi); 857 break; 858 859 case ISD::ANY_EXTEND_VECTOR_INREG: 860 case ISD::SIGN_EXTEND_VECTOR_INREG: 861 case ISD::ZERO_EXTEND_VECTOR_INREG: 862 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 863 break; 864 865 case ISD::ABS: 866 case ISD::BITREVERSE: 867 case ISD::BSWAP: 868 case ISD::CTLZ: 869 case ISD::CTTZ: 870 case ISD::CTLZ_ZERO_UNDEF: 871 case ISD::CTTZ_ZERO_UNDEF: 872 case ISD::CTPOP: 873 case ISD::FABS: 874 case ISD::FCEIL: 875 case ISD::FCOS: 876 case ISD::FEXP: 877 case ISD::FEXP2: 878 case ISD::FFLOOR: 879 case ISD::FLOG: 880 case ISD::FLOG10: 881 case ISD::FLOG2: 882 case ISD::FNEARBYINT: 883 case ISD::FNEG: 884 case ISD::FP_EXTEND: 885 case ISD::FP_ROUND: 886 case ISD::FP_TO_SINT: 887 case ISD::FP_TO_UINT: 888 case ISD::FRINT: 889 case ISD::FROUND: 890 case ISD::FSIN: 891 case ISD::FSQRT: 892 case ISD::FTRUNC: 893 case ISD::SINT_TO_FP: 894 case ISD::TRUNCATE: 895 case ISD::UINT_TO_FP: 896 case ISD::FCANONICALIZE: 897 SplitVecRes_UnaryOp(N, Lo, Hi); 898 break; 899 900 case ISD::ANY_EXTEND: 901 case ISD::SIGN_EXTEND: 902 case ISD::ZERO_EXTEND: 903 SplitVecRes_ExtendOp(N, Lo, Hi); 904 break; 905 906 case ISD::ADD: 907 case ISD::SUB: 908 case ISD::MUL: 909 case ISD::MULHS: 910 case ISD::MULHU: 911 case ISD::FADD: 912 case ISD::FSUB: 913 case ISD::FMUL: 914 case ISD::FMINNUM: 915 case ISD::FMAXNUM: 916 case ISD::FMINIMUM: 917 case ISD::FMAXIMUM: 918 case ISD::SDIV: 919 case ISD::UDIV: 920 case ISD::FDIV: 921 case ISD::FPOW: 922 case ISD::AND: 923 case ISD::OR: 924 case ISD::XOR: 925 case ISD::SHL: 926 case ISD::SRA: 927 case ISD::SRL: 928 case ISD::UREM: 929 case ISD::SREM: 930 case ISD::FREM: 931 case ISD::SMIN: 932 case ISD::SMAX: 933 case ISD::UMIN: 934 case ISD::UMAX: 935 case ISD::SADDSAT: 936 case ISD::UADDSAT: 937 case ISD::SSUBSAT: 938 case ISD::USUBSAT: 939 SplitVecRes_BinOp(N, Lo, Hi); 940 break; 941 case ISD::FMA: 942 SplitVecRes_TernaryOp(N, Lo, Hi); 943 break; 944 945#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 946 case ISD::STRICT_##DAGN: 947#include "llvm/IR/ConstrainedOps.def" 948 SplitVecRes_StrictFPOp(N, Lo, Hi); 949 break; 950 951 case ISD::UADDO: 952 case ISD::SADDO: 953 case ISD::USUBO: 954 case ISD::SSUBO: 955 case ISD::UMULO: 956 case ISD::SMULO: 957 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi); 958 break; 959 case ISD::SMULFIX: 960 case ISD::SMULFIXSAT: 961 case ISD::UMULFIX: 962 case ISD::UMULFIXSAT: 963 case ISD::SDIVFIX: 964 case ISD::UDIVFIX: 965 SplitVecRes_FIX(N, Lo, Hi); 966 break; 967 } 968 969 // If Lo/Hi is null, the sub-method took care of registering results etc. 970 if (Lo.getNode()) 971 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 972} 973 974void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 975 SDValue &Hi) { 976 SDValue LHSLo, LHSHi; 977 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 978 SDValue RHSLo, RHSHi; 979 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 980 SDLoc dl(N); 981 982 const SDNodeFlags Flags = N->getFlags(); 983 unsigned Opcode = N->getOpcode(); 984 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags); 985 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags); 986} 987 988void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 989 SDValue &Hi) { 990 SDValue Op0Lo, Op0Hi; 991 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 992 SDValue Op1Lo, Op1Hi; 993 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 994 SDValue Op2Lo, Op2Hi; 995 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 996 SDLoc dl(N); 997 998 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), 999 Op0Lo, Op1Lo, Op2Lo); 1000 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), 1001 Op0Hi, Op1Hi, Op2Hi); 1002} 1003 1004void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) { 1005 SDValue LHSLo, LHSHi; 1006 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1007 SDValue RHSLo, RHSHi; 1008 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 1009 SDLoc dl(N); 1010 SDValue Op2 = N->getOperand(2); 1011 1012 unsigned Opcode = N->getOpcode(); 1013 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2); 1014 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2); 1015} 1016 1017void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 1018 SDValue &Hi) { 1019 // We know the result is a vector. The input may be either a vector or a 1020 // scalar value. 1021 EVT LoVT, HiVT; 1022 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1023 SDLoc dl(N); 1024 1025 SDValue InOp = N->getOperand(0); 1026 EVT InVT = InOp.getValueType(); 1027 1028 // Handle some special cases efficiently. 1029 switch (getTypeAction(InVT)) { 1030 case TargetLowering::TypeLegal: 1031 case TargetLowering::TypePromoteInteger: 1032 case TargetLowering::TypePromoteFloat: 1033 case TargetLowering::TypeSoftenFloat: 1034 case TargetLowering::TypeScalarizeVector: 1035 case TargetLowering::TypeWidenVector: 1036 break; 1037 case TargetLowering::TypeExpandInteger: 1038 case TargetLowering::TypeExpandFloat: 1039 // A scalar to vector conversion, where the scalar needs expansion. 1040 // If the vector is being split in two then we can just convert the 1041 // expanded pieces. 1042 if (LoVT == HiVT) { 1043 GetExpandedOp(InOp, Lo, Hi); 1044 if (DAG.getDataLayout().isBigEndian()) 1045 std::swap(Lo, Hi); 1046 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1047 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1048 return; 1049 } 1050 break; 1051 case TargetLowering::TypeSplitVector: 1052 // If the input is a vector that needs to be split, convert each split 1053 // piece of the input now. 1054 GetSplitVector(InOp, Lo, Hi); 1055 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1056 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1057 return; 1058 } 1059 1060 // In the general case, convert the input to an integer and split it by hand. 1061 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 1062 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 1063 if (DAG.getDataLayout().isBigEndian()) 1064 std::swap(LoIntVT, HiIntVT); 1065 1066 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 1067 1068 if (DAG.getDataLayout().isBigEndian()) 1069 std::swap(Lo, Hi); 1070 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1071 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1072} 1073 1074void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 1075 SDValue &Hi) { 1076 EVT LoVT, HiVT; 1077 SDLoc dl(N); 1078 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1079 unsigned LoNumElts = LoVT.getVectorNumElements(); 1080 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 1081 Lo = DAG.getBuildVector(LoVT, dl, LoOps); 1082 1083 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 1084 Hi = DAG.getBuildVector(HiVT, dl, HiOps); 1085} 1086 1087void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 1088 SDValue &Hi) { 1089 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 1090 SDLoc dl(N); 1091 unsigned NumSubvectors = N->getNumOperands() / 2; 1092 if (NumSubvectors == 1) { 1093 Lo = N->getOperand(0); 1094 Hi = N->getOperand(1); 1095 return; 1096 } 1097 1098 EVT LoVT, HiVT; 1099 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1100 1101 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 1102 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 1103 1104 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 1105 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 1106} 1107 1108void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 1109 SDValue &Hi) { 1110 SDValue Vec = N->getOperand(0); 1111 SDValue Idx = N->getOperand(1); 1112 SDLoc dl(N); 1113 1114 EVT LoVT, HiVT; 1115 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1116 1117 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 1118 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1119 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 1120 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl, 1121 TLI.getVectorIdxTy(DAG.getDataLayout()))); 1122} 1123 1124void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 1125 SDValue &Hi) { 1126 SDValue Vec = N->getOperand(0); 1127 SDValue SubVec = N->getOperand(1); 1128 SDValue Idx = N->getOperand(2); 1129 SDLoc dl(N); 1130 GetSplitVector(Vec, Lo, Hi); 1131 1132 EVT VecVT = Vec.getValueType(); 1133 unsigned VecElems = VecVT.getVectorNumElements(); 1134 unsigned SubElems = SubVec.getValueType().getVectorNumElements(); 1135 1136 // If we know the index is 0, and we know the subvector doesn't cross the 1137 // boundary between the halves, we can avoid spilling the vector, and insert 1138 // into the lower half of the split vector directly. 1139 // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever 1140 // the index is constant and there is no boundary crossing. But those cases 1141 // don't seem to get hit in practice. 1142 if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) { 1143 unsigned IdxVal = ConstIdx->getZExtValue(); 1144 if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) { 1145 EVT LoVT, HiVT; 1146 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1147 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx); 1148 return; 1149 } 1150 } 1151 1152 // Spill the vector to the stack. 1153 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1154 SDValue Store = 1155 DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo()); 1156 1157 // Store the new subvector into the specified index. 1158 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1159 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 1160 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType); 1161 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo()); 1162 1163 // Load the Lo part from the stack slot. 1164 Lo = 1165 DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo()); 1166 1167 // Increment the pointer to the other part. 1168 unsigned IncrementSize = Lo.getValueSizeInBits() / 8; 1169 StackPtr = DAG.getMemBasePlusOffset(StackPtr, IncrementSize, dl); 1170 1171 // Load the Hi part from the stack slot. 1172 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(), 1173 MinAlign(Alignment, IncrementSize)); 1174} 1175 1176void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 1177 SDValue &Hi) { 1178 SDLoc dl(N); 1179 GetSplitVector(N->getOperand(0), Lo, Hi); 1180 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 1181 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 1182} 1183 1184void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, 1185 SDValue &Hi) { 1186 SDValue LHSLo, LHSHi; 1187 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1188 SDLoc DL(N); 1189 1190 SDValue RHSLo, RHSHi; 1191 SDValue RHS = N->getOperand(1); 1192 EVT RHSVT = RHS.getValueType(); 1193 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector) 1194 GetSplitVector(RHS, RHSLo, RHSHi); 1195 else 1196 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS)); 1197 1198 1199 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo); 1200 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi); 1201} 1202 1203void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 1204 SDValue &Hi) { 1205 SDValue LHSLo, LHSHi; 1206 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1207 SDLoc dl(N); 1208 1209 EVT LoVT, HiVT; 1210 std::tie(LoVT, HiVT) = 1211 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 1212 1213 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 1214 DAG.getValueType(LoVT)); 1215 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 1216 DAG.getValueType(HiVT)); 1217} 1218 1219void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, 1220 SDValue &Hi) { 1221 unsigned Opcode = N->getOpcode(); 1222 SDValue N0 = N->getOperand(0); 1223 1224 SDLoc dl(N); 1225 SDValue InLo, InHi; 1226 1227 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector) 1228 GetSplitVector(N0, InLo, InHi); 1229 else 1230 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0); 1231 1232 EVT InLoVT = InLo.getValueType(); 1233 unsigned InNumElements = InLoVT.getVectorNumElements(); 1234 1235 EVT OutLoVT, OutHiVT; 1236 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1237 unsigned OutNumElements = OutLoVT.getVectorNumElements(); 1238 assert((2 * OutNumElements) <= InNumElements && 1239 "Illegal extend vector in reg split"); 1240 1241 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the 1242 // input vector (i.e. we only use InLo): 1243 // OutLo will extend the first OutNumElements from InLo. 1244 // OutHi will extend the next OutNumElements from InLo. 1245 1246 // Shuffle the elements from InLo for OutHi into the bottom elements to 1247 // create a 'fake' InHi. 1248 SmallVector<int, 8> SplitHi(InNumElements, -1); 1249 for (unsigned i = 0; i != OutNumElements; ++i) 1250 SplitHi[i] = i + OutNumElements; 1251 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi); 1252 1253 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo); 1254 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi); 1255} 1256 1257void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, 1258 SDValue &Hi) { 1259 unsigned NumOps = N->getNumOperands(); 1260 SDValue Chain = N->getOperand(0); 1261 EVT LoVT, HiVT; 1262 SDLoc dl(N); 1263 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1264 1265 SmallVector<SDValue, 4> OpsLo(NumOps); 1266 SmallVector<SDValue, 4> OpsHi(NumOps); 1267 1268 // The Chain is the first operand. 1269 OpsLo[0] = Chain; 1270 OpsHi[0] = Chain; 1271 1272 // Now process the remaining operands. 1273 for (unsigned i = 1; i < NumOps; ++i) { 1274 SDValue Op = N->getOperand(i); 1275 SDValue OpLo = Op; 1276 SDValue OpHi = Op; 1277 1278 EVT InVT = Op.getValueType(); 1279 if (InVT.isVector()) { 1280 // If the input also splits, handle it directly for a 1281 // compile time speedup. Otherwise split it by hand. 1282 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1283 GetSplitVector(Op, OpLo, OpHi); 1284 else 1285 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i); 1286 } 1287 1288 OpsLo[i] = OpLo; 1289 OpsHi[i] = OpHi; 1290 } 1291 1292 EVT LoValueVTs[] = {LoVT, MVT::Other}; 1293 EVT HiValueVTs[] = {HiVT, MVT::Other}; 1294 Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo); 1295 Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi); 1296 1297 // Build a factor node to remember that this Op is independent of the 1298 // other one. 1299 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1300 Lo.getValue(1), Hi.getValue(1)); 1301 1302 // Legalize the chain result - switch anything that used the old chain to 1303 // use the new one. 1304 ReplaceValueWith(SDValue(N, 1), Chain); 1305} 1306 1307SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { 1308 SDValue Chain = N->getOperand(0); 1309 EVT VT = N->getValueType(0); 1310 unsigned NE = VT.getVectorNumElements(); 1311 EVT EltVT = VT.getVectorElementType(); 1312 SDLoc dl(N); 1313 1314 SmallVector<SDValue, 8> Scalars; 1315 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 1316 1317 // If ResNE is 0, fully unroll the vector op. 1318 if (ResNE == 0) 1319 ResNE = NE; 1320 else if (NE > ResNE) 1321 NE = ResNE; 1322 1323 //The results of each unrolled operation, including the chain. 1324 EVT ChainVTs[] = {EltVT, MVT::Other}; 1325 SmallVector<SDValue, 8> Chains; 1326 1327 unsigned i; 1328 for (i = 0; i != NE; ++i) { 1329 Operands[0] = Chain; 1330 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) { 1331 SDValue Operand = N->getOperand(j); 1332 EVT OperandVT = Operand.getValueType(); 1333 if (OperandVT.isVector()) { 1334 EVT OperandEltVT = OperandVT.getVectorElementType(); 1335 Operands[j] = 1336 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand, 1337 DAG.getConstant(i, dl, TLI.getVectorIdxTy( 1338 DAG.getDataLayout()))); 1339 } else { 1340 Operands[j] = Operand; 1341 } 1342 } 1343 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); 1344 Scalar.getNode()->setFlags(N->getFlags()); 1345 1346 //Add in the scalar as well as its chain value to the 1347 //result vectors. 1348 Scalars.push_back(Scalar); 1349 Chains.push_back(Scalar.getValue(1)); 1350 } 1351 1352 for (; i < ResNE; ++i) 1353 Scalars.push_back(DAG.getUNDEF(EltVT)); 1354 1355 // Build a new factor node to connect the chain back together. 1356 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 1357 ReplaceValueWith(SDValue(N, 1), Chain); 1358 1359 // Create a new BUILD_VECTOR node 1360 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE); 1361 return DAG.getBuildVector(VecVT, dl, Scalars); 1362} 1363 1364void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, 1365 SDValue &Lo, SDValue &Hi) { 1366 SDLoc dl(N); 1367 EVT ResVT = N->getValueType(0); 1368 EVT OvVT = N->getValueType(1); 1369 EVT LoResVT, HiResVT, LoOvVT, HiOvVT; 1370 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT); 1371 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT); 1372 1373 SDValue LoLHS, HiLHS, LoRHS, HiRHS; 1374 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) { 1375 GetSplitVector(N->getOperand(0), LoLHS, HiLHS); 1376 GetSplitVector(N->getOperand(1), LoRHS, HiRHS); 1377 } else { 1378 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0); 1379 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1); 1380 } 1381 1382 unsigned Opcode = N->getOpcode(); 1383 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT); 1384 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT); 1385 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode(); 1386 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode(); 1387 1388 Lo = SDValue(LoNode, ResNo); 1389 Hi = SDValue(HiNode, ResNo); 1390 1391 // Replace the other vector result not being explicitly split here. 1392 unsigned OtherNo = 1 - ResNo; 1393 EVT OtherVT = N->getValueType(OtherNo); 1394 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) { 1395 SetSplitVector(SDValue(N, OtherNo), 1396 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1397 } else { 1398 SDValue OtherVal = DAG.getNode( 1399 ISD::CONCAT_VECTORS, dl, OtherVT, 1400 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1401 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 1402 } 1403} 1404 1405void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 1406 SDValue &Hi) { 1407 SDValue Vec = N->getOperand(0); 1408 SDValue Elt = N->getOperand(1); 1409 SDValue Idx = N->getOperand(2); 1410 SDLoc dl(N); 1411 GetSplitVector(Vec, Lo, Hi); 1412 1413 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 1414 unsigned IdxVal = CIdx->getZExtValue(); 1415 unsigned LoNumElts = Lo.getValueType().getVectorNumElements(); 1416 if (IdxVal < LoNumElts) 1417 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 1418 Lo.getValueType(), Lo, Elt, Idx); 1419 else 1420 Hi = 1421 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 1422 DAG.getConstant(IdxVal - LoNumElts, dl, 1423 TLI.getVectorIdxTy(DAG.getDataLayout()))); 1424 return; 1425 } 1426 1427 // See if the target wants to custom expand this node. 1428 if (CustomLowerNode(N, N->getValueType(0), true)) 1429 return; 1430 1431 // Make the vector elements byte-addressable if they aren't already. 1432 EVT VecVT = Vec.getValueType(); 1433 EVT EltVT = VecVT.getVectorElementType(); 1434 if (VecVT.getScalarSizeInBits() < 8) { 1435 EltVT = MVT::i8; 1436 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 1437 VecVT.getVectorNumElements()); 1438 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 1439 // Extend the element type to match if needed. 1440 if (EltVT.bitsGT(Elt.getValueType())) 1441 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt); 1442 } 1443 1444 // Spill the vector to the stack. 1445 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 1446 auto &MF = DAG.getMachineFunction(); 1447 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1448 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 1449 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); 1450 1451 // Store the new element. This may be larger than the vector element type, 1452 // so use a truncating store. 1453 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1454 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext()); 1455 unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType); 1456 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, 1457 MachinePointerInfo::getUnknownStack(MF), EltVT); 1458 1459 EVT LoVT, HiVT; 1460 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT); 1461 1462 // Load the Lo part from the stack slot. 1463 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo); 1464 1465 // Increment the pointer to the other part. 1466 unsigned IncrementSize = LoVT.getSizeInBits() / 8; 1467 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, 1468 DAG.getConstant(IncrementSize, dl, 1469 StackPtr.getValueType())); 1470 1471 // Load the Hi part from the stack slot. 1472 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, 1473 PtrInfo.getWithOffset(IncrementSize), 1474 MinAlign(Alignment, IncrementSize)); 1475 1476 // If we adjusted the original type, we need to truncate the results. 1477 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1478 if (LoVT != Lo.getValueType()) 1479 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo); 1480 if (HiVT != Hi.getValueType()) 1481 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi); 1482} 1483 1484void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, 1485 SDValue &Hi) { 1486 EVT LoVT, HiVT; 1487 SDLoc dl(N); 1488 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1489 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0)); 1490 Hi = DAG.getUNDEF(HiVT); 1491} 1492 1493void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 1494 SDValue &Hi) { 1495 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 1496 EVT LoVT, HiVT; 1497 SDLoc dl(LD); 1498 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 1499 1500 ISD::LoadExtType ExtType = LD->getExtensionType(); 1501 SDValue Ch = LD->getChain(); 1502 SDValue Ptr = LD->getBasePtr(); 1503 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 1504 EVT MemoryVT = LD->getMemoryVT(); 1505 unsigned Alignment = LD->getOriginalAlignment(); 1506 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 1507 AAMDNodes AAInfo = LD->getAAInfo(); 1508 1509 EVT LoMemVT, HiMemVT; 1510 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1511 1512 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 1513 LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo); 1514 1515 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 1516 Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize); 1517 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, 1518 LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT, 1519 Alignment, MMOFlags, AAInfo); 1520 1521 // Build a factor node to remember that this load is independent of the 1522 // other one. 1523 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1524 Hi.getValue(1)); 1525 1526 // Legalize the chain result - switch anything that used the old chain to 1527 // use the new one. 1528 ReplaceValueWith(SDValue(LD, 1), Ch); 1529} 1530 1531void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, 1532 SDValue &Lo, SDValue &Hi) { 1533 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!"); 1534 EVT LoVT, HiVT; 1535 SDLoc dl(MLD); 1536 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0)); 1537 1538 SDValue Ch = MLD->getChain(); 1539 SDValue Ptr = MLD->getBasePtr(); 1540 SDValue Offset = MLD->getOffset(); 1541 assert(Offset.isUndef() && "Unexpected indexed masked load offset"); 1542 SDValue Mask = MLD->getMask(); 1543 SDValue PassThru = MLD->getPassThru(); 1544 unsigned Alignment = MLD->getOriginalAlignment(); 1545 ISD::LoadExtType ExtType = MLD->getExtensionType(); 1546 1547 // Split Mask operand 1548 SDValue MaskLo, MaskHi; 1549 if (Mask.getOpcode() == ISD::SETCC) { 1550 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 1551 } else { 1552 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1553 GetSplitVector(Mask, MaskLo, MaskHi); 1554 else 1555 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1556 } 1557 1558 EVT MemoryVT = MLD->getMemoryVT(); 1559 EVT LoMemVT, HiMemVT; 1560 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1561 1562 SDValue PassThruLo, PassThruHi; 1563 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 1564 GetSplitVector(PassThru, PassThruLo, PassThruHi); 1565 else 1566 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 1567 1568 MachineMemOperand *MMO = DAG.getMachineFunction(). 1569 getMachineMemOperand(MLD->getPointerInfo(), 1570 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), 1571 Alignment, MLD->getAAInfo(), MLD->getRanges()); 1572 1573 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT, 1574 MMO, MLD->getAddressingMode(), ExtType, 1575 MLD->isExpandingLoad()); 1576 1577 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG, 1578 MLD->isExpandingLoad()); 1579 unsigned HiOffset = LoMemVT.getStoreSize(); 1580 1581 MMO = DAG.getMachineFunction().getMachineMemOperand( 1582 MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad, 1583 HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(), 1584 MLD->getRanges()); 1585 1586 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi, HiMemVT, 1587 MMO, MLD->getAddressingMode(), ExtType, 1588 MLD->isExpandingLoad()); 1589 1590 // Build a factor node to remember that this load is independent of the 1591 // other one. 1592 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1593 Hi.getValue(1)); 1594 1595 // Legalize the chain result - switch anything that used the old chain to 1596 // use the new one. 1597 ReplaceValueWith(SDValue(MLD, 1), Ch); 1598 1599} 1600 1601void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, 1602 SDValue &Lo, SDValue &Hi) { 1603 EVT LoVT, HiVT; 1604 SDLoc dl(MGT); 1605 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0)); 1606 1607 SDValue Ch = MGT->getChain(); 1608 SDValue Ptr = MGT->getBasePtr(); 1609 SDValue Mask = MGT->getMask(); 1610 SDValue PassThru = MGT->getPassThru(); 1611 SDValue Index = MGT->getIndex(); 1612 SDValue Scale = MGT->getScale(); 1613 unsigned Alignment = MGT->getOriginalAlignment(); 1614 1615 // Split Mask operand 1616 SDValue MaskLo, MaskHi; 1617 if (Mask.getOpcode() == ISD::SETCC) { 1618 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 1619 } else { 1620 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1621 GetSplitVector(Mask, MaskLo, MaskHi); 1622 else 1623 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1624 } 1625 1626 EVT MemoryVT = MGT->getMemoryVT(); 1627 EVT LoMemVT, HiMemVT; 1628 // Split MemoryVT 1629 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1630 1631 SDValue PassThruLo, PassThruHi; 1632 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 1633 GetSplitVector(PassThru, PassThruLo, PassThruHi); 1634 else 1635 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 1636 1637 SDValue IndexHi, IndexLo; 1638 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 1639 GetSplitVector(Index, IndexLo, IndexHi); 1640 else 1641 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl); 1642 1643 MachineMemOperand *MMO = DAG.getMachineFunction(). 1644 getMachineMemOperand(MGT->getPointerInfo(), 1645 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), 1646 Alignment, MGT->getAAInfo(), MGT->getRanges()); 1647 1648 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale}; 1649 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo, 1650 MMO, MGT->getIndexType()); 1651 1652 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale}; 1653 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi, 1654 MMO, MGT->getIndexType()); 1655 1656 // Build a factor node to remember that this load is independent of the 1657 // other one. 1658 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1659 Hi.getValue(1)); 1660 1661 // Legalize the chain result - switch anything that used the old chain to 1662 // use the new one. 1663 ReplaceValueWith(SDValue(MGT, 1), Ch); 1664} 1665 1666 1667void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 1668 assert(N->getValueType(0).isVector() && 1669 N->getOperand(0).getValueType().isVector() && 1670 "Operand types must be vectors"); 1671 1672 EVT LoVT, HiVT; 1673 SDLoc DL(N); 1674 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1675 1676 // If the input also splits, handle it directly. Otherwise split it by hand. 1677 SDValue LL, LH, RL, RH; 1678 if (getTypeAction(N->getOperand(0).getValueType()) == 1679 TargetLowering::TypeSplitVector) 1680 GetSplitVector(N->getOperand(0), LL, LH); 1681 else 1682 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 1683 1684 if (getTypeAction(N->getOperand(1).getValueType()) == 1685 TargetLowering::TypeSplitVector) 1686 GetSplitVector(N->getOperand(1), RL, RH); 1687 else 1688 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 1689 1690 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 1691 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 1692} 1693 1694void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 1695 SDValue &Hi) { 1696 // Get the dest types - they may not match the input types, e.g. int_to_fp. 1697 EVT LoVT, HiVT; 1698 SDLoc dl(N); 1699 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1700 1701 // If the input also splits, handle it directly for a compile time speedup. 1702 // Otherwise split it by hand. 1703 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1704 EVT InVT = N->getOperand(OpNo).getValueType(); 1705 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1706 GetSplitVector(N->getOperand(OpNo), Lo, Hi); 1707 else 1708 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo); 1709 1710 if (N->getOpcode() == ISD::FP_ROUND) { 1711 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); 1712 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); 1713 } else { 1714 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1715 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1716 } 1717} 1718 1719void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 1720 SDValue &Hi) { 1721 SDLoc dl(N); 1722 EVT SrcVT = N->getOperand(0).getValueType(); 1723 EVT DestVT = N->getValueType(0); 1724 EVT LoVT, HiVT; 1725 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 1726 1727 // We can do better than a generic split operation if the extend is doing 1728 // more than just doubling the width of the elements and the following are 1729 // true: 1730 // - The number of vector elements is even, 1731 // - the source type is legal, 1732 // - the type of a split source is illegal, 1733 // - the type of an extended (by doubling element size) source is legal, and 1734 // - the type of that extended source when split is legal. 1735 // 1736 // This won't necessarily completely legalize the operation, but it will 1737 // more effectively move in the right direction and prevent falling down 1738 // to scalarization in many cases due to the input vector being split too 1739 // far. 1740 unsigned NumElements = SrcVT.getVectorNumElements(); 1741 if ((NumElements & 1) == 0 && 1742 SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) { 1743 LLVMContext &Ctx = *DAG.getContext(); 1744 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx); 1745 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx); 1746 1747 EVT SplitLoVT, SplitHiVT; 1748 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 1749 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 1750 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 1751 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:"; 1752 N->dump(&DAG); dbgs() << "\n"); 1753 // Extend the source vector by one step. 1754 SDValue NewSrc = 1755 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 1756 // Get the low and high halves of the new, extended one step, vector. 1757 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 1758 // Extend those vector halves the rest of the way. 1759 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1760 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1761 return; 1762 } 1763 } 1764 // Fall back to the generic unary operator splitting otherwise. 1765 SplitVecRes_UnaryOp(N, Lo, Hi); 1766} 1767 1768void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 1769 SDValue &Lo, SDValue &Hi) { 1770 // The low and high parts of the original input give four input vectors. 1771 SDValue Inputs[4]; 1772 SDLoc dl(N); 1773 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 1774 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 1775 EVT NewVT = Inputs[0].getValueType(); 1776 unsigned NewElts = NewVT.getVectorNumElements(); 1777 1778 // If Lo or Hi uses elements from at most two of the four input vectors, then 1779 // express it as a vector shuffle of those two inputs. Otherwise extract the 1780 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 1781 SmallVector<int, 16> Ops; 1782 for (unsigned High = 0; High < 2; ++High) { 1783 SDValue &Output = High ? Hi : Lo; 1784 1785 // Build a shuffle mask for the output, discovering on the fly which 1786 // input vectors to use as shuffle operands (recorded in InputUsed). 1787 // If building a suitable shuffle vector proves too hard, then bail 1788 // out with useBuildVector set. 1789 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 1790 unsigned FirstMaskIdx = High * NewElts; 1791 bool useBuildVector = false; 1792 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1793 // The mask element. This indexes into the input. 1794 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1795 1796 // The input vector this mask element indexes into. 1797 unsigned Input = (unsigned)Idx / NewElts; 1798 1799 if (Input >= array_lengthof(Inputs)) { 1800 // The mask element does not index into any input vector. 1801 Ops.push_back(-1); 1802 continue; 1803 } 1804 1805 // Turn the index into an offset from the start of the input vector. 1806 Idx -= Input * NewElts; 1807 1808 // Find or create a shuffle vector operand to hold this input. 1809 unsigned OpNo; 1810 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 1811 if (InputUsed[OpNo] == Input) { 1812 // This input vector is already an operand. 1813 break; 1814 } else if (InputUsed[OpNo] == -1U) { 1815 // Create a new operand for this input vector. 1816 InputUsed[OpNo] = Input; 1817 break; 1818 } 1819 } 1820 1821 if (OpNo >= array_lengthof(InputUsed)) { 1822 // More than two input vectors used! Give up on trying to create a 1823 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 1824 useBuildVector = true; 1825 break; 1826 } 1827 1828 // Add the mask index for the new shuffle vector. 1829 Ops.push_back(Idx + OpNo * NewElts); 1830 } 1831 1832 if (useBuildVector) { 1833 EVT EltVT = NewVT.getVectorElementType(); 1834 SmallVector<SDValue, 16> SVOps; 1835 1836 // Extract the input elements by hand. 1837 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 1838 // The mask element. This indexes into the input. 1839 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 1840 1841 // The input vector this mask element indexes into. 1842 unsigned Input = (unsigned)Idx / NewElts; 1843 1844 if (Input >= array_lengthof(Inputs)) { 1845 // The mask element is "undef" or indexes off the end of the input. 1846 SVOps.push_back(DAG.getUNDEF(EltVT)); 1847 continue; 1848 } 1849 1850 // Turn the index into an offset from the start of the input vector. 1851 Idx -= Input * NewElts; 1852 1853 // Extract the vector element by hand. 1854 SVOps.push_back(DAG.getNode( 1855 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input], 1856 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); 1857 } 1858 1859 // Construct the Lo/Hi output using a BUILD_VECTOR. 1860 Output = DAG.getBuildVector(NewVT, dl, SVOps); 1861 } else if (InputUsed[0] == -1U) { 1862 // No input vectors were used! The result is undefined. 1863 Output = DAG.getUNDEF(NewVT); 1864 } else { 1865 SDValue Op0 = Inputs[InputUsed[0]]; 1866 // If only one input was used, use an undefined vector for the other. 1867 SDValue Op1 = InputUsed[1] == -1U ? 1868 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 1869 // At least one input vector was used. Create a new shuffle vector. 1870 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops); 1871 } 1872 1873 Ops.clear(); 1874 } 1875} 1876 1877void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 1878 EVT OVT = N->getValueType(0); 1879 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext()); 1880 SDValue Chain = N->getOperand(0); 1881 SDValue Ptr = N->getOperand(1); 1882 SDValue SV = N->getOperand(2); 1883 SDLoc dl(N); 1884 1885 const unsigned Alignment = DAG.getDataLayout().getABITypeAlignment( 1886 NVT.getTypeForEVT(*DAG.getContext())); 1887 1888 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment); 1889 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment); 1890 Chain = Hi.getValue(1); 1891 1892 // Modified the chain - switch anything that used the old chain to use 1893 // the new one. 1894 ReplaceValueWith(SDValue(N, 1), Chain); 1895} 1896 1897 1898//===----------------------------------------------------------------------===// 1899// Operand Vector Splitting 1900//===----------------------------------------------------------------------===// 1901 1902/// This method is called when the specified operand of the specified node is 1903/// found to need vector splitting. At this point, all of the result types of 1904/// the node are known to be legal, but other operands of the node may need 1905/// legalization as well as the specified one. 1906bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 1907 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n"); 1908 SDValue Res = SDValue(); 1909 1910 // See if the target wants to custom split this node. 1911 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 1912 return false; 1913 1914 if (!Res.getNode()) { 1915 switch (N->getOpcode()) { 1916 default: 1917#ifndef NDEBUG 1918 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 1919 N->dump(&DAG); 1920 dbgs() << "\n"; 1921#endif 1922 report_fatal_error("Do not know how to split this operator's " 1923 "operand!\n"); 1924 1925 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 1926 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 1927 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 1928 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 1929 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 1930 case ISD::TRUNCATE: 1931 Res = SplitVecOp_TruncateHelper(N); 1932 break; 1933 case ISD::STRICT_FP_ROUND: 1934 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 1935 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break; 1936 case ISD::STORE: 1937 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 1938 break; 1939 case ISD::MSTORE: 1940 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo); 1941 break; 1942 case ISD::MSCATTER: 1943 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo); 1944 break; 1945 case ISD::MGATHER: 1946 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo); 1947 break; 1948 case ISD::VSELECT: 1949 Res = SplitVecOp_VSELECT(N, OpNo); 1950 break; 1951 case ISD::STRICT_SINT_TO_FP: 1952 case ISD::STRICT_UINT_TO_FP: 1953 case ISD::SINT_TO_FP: 1954 case ISD::UINT_TO_FP: 1955 if (N->getValueType(0).bitsLT( 1956 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType())) 1957 Res = SplitVecOp_TruncateHelper(N); 1958 else 1959 Res = SplitVecOp_UnaryOp(N); 1960 break; 1961 case ISD::FP_TO_SINT: 1962 case ISD::FP_TO_UINT: 1963 case ISD::STRICT_FP_TO_SINT: 1964 case ISD::STRICT_FP_TO_UINT: 1965 case ISD::CTTZ: 1966 case ISD::CTLZ: 1967 case ISD::CTPOP: 1968 case ISD::STRICT_FP_EXTEND: 1969 case ISD::FP_EXTEND: 1970 case ISD::SIGN_EXTEND: 1971 case ISD::ZERO_EXTEND: 1972 case ISD::ANY_EXTEND: 1973 case ISD::FTRUNC: 1974 case ISD::FCANONICALIZE: 1975 Res = SplitVecOp_UnaryOp(N); 1976 break; 1977 1978 case ISD::ANY_EXTEND_VECTOR_INREG: 1979 case ISD::SIGN_EXTEND_VECTOR_INREG: 1980 case ISD::ZERO_EXTEND_VECTOR_INREG: 1981 Res = SplitVecOp_ExtVecInRegOp(N); 1982 break; 1983 1984 case ISD::VECREDUCE_FADD: 1985 case ISD::VECREDUCE_FMUL: 1986 case ISD::VECREDUCE_ADD: 1987 case ISD::VECREDUCE_MUL: 1988 case ISD::VECREDUCE_AND: 1989 case ISD::VECREDUCE_OR: 1990 case ISD::VECREDUCE_XOR: 1991 case ISD::VECREDUCE_SMAX: 1992 case ISD::VECREDUCE_SMIN: 1993 case ISD::VECREDUCE_UMAX: 1994 case ISD::VECREDUCE_UMIN: 1995 case ISD::VECREDUCE_FMAX: 1996 case ISD::VECREDUCE_FMIN: 1997 Res = SplitVecOp_VECREDUCE(N, OpNo); 1998 break; 1999 } 2000 } 2001 2002 // If the result is null, the sub-method took care of registering results etc. 2003 if (!Res.getNode()) return false; 2004 2005 // If the result is N, the sub-method updated N in place. Tell the legalizer 2006 // core about this. 2007 if (Res.getNode() == N) 2008 return true; 2009 2010 if (N->isStrictFPOpcode()) 2011 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 2012 "Invalid operand expansion"); 2013 else 2014 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2015 "Invalid operand expansion"); 2016 2017 ReplaceValueWith(SDValue(N, 0), Res); 2018 return false; 2019} 2020 2021SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 2022 // The only possibility for an illegal operand is the mask, since result type 2023 // legalization would have handled this node already otherwise. 2024 assert(OpNo == 0 && "Illegal operand must be mask"); 2025 2026 SDValue Mask = N->getOperand(0); 2027 SDValue Src0 = N->getOperand(1); 2028 SDValue Src1 = N->getOperand(2); 2029 EVT Src0VT = Src0.getValueType(); 2030 SDLoc DL(N); 2031 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 2032 2033 SDValue Lo, Hi; 2034 GetSplitVector(N->getOperand(0), Lo, Hi); 2035 assert(Lo.getValueType() == Hi.getValueType() && 2036 "Lo and Hi have differing types"); 2037 2038 EVT LoOpVT, HiOpVT; 2039 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 2040 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 2041 2042 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 2043 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 2044 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 2045 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 2046 2047 SDValue LoSelect = 2048 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 2049 SDValue HiSelect = 2050 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 2051 2052 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 2053} 2054 2055SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) { 2056 EVT ResVT = N->getValueType(0); 2057 SDValue Lo, Hi; 2058 SDLoc dl(N); 2059 2060 SDValue VecOp = N->getOperand(OpNo); 2061 EVT VecVT = VecOp.getValueType(); 2062 assert(VecVT.isVector() && "Can only split reduce vector operand"); 2063 GetSplitVector(VecOp, Lo, Hi); 2064 EVT LoOpVT, HiOpVT; 2065 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT); 2066 2067 bool NoNaN = N->getFlags().hasNoNaNs(); 2068 unsigned CombineOpc = 0; 2069 switch (N->getOpcode()) { 2070 case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break; 2071 case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break; 2072 case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break; 2073 case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break; 2074 case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break; 2075 case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break; 2076 case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break; 2077 case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break; 2078 case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break; 2079 case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break; 2080 case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break; 2081 case ISD::VECREDUCE_FMAX: 2082 CombineOpc = NoNaN ? ISD::FMAXNUM : ISD::FMAXIMUM; 2083 break; 2084 case ISD::VECREDUCE_FMIN: 2085 CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM; 2086 break; 2087 default: 2088 llvm_unreachable("Unexpected reduce ISD node"); 2089 } 2090 2091 // Use the appropriate scalar instruction on the split subvectors before 2092 // reducing the now partially reduced smaller vector. 2093 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags()); 2094 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags()); 2095} 2096 2097SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 2098 // The result has a legal vector type, but the input needs splitting. 2099 EVT ResVT = N->getValueType(0); 2100 SDValue Lo, Hi; 2101 SDLoc dl(N); 2102 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 2103 EVT InVT = Lo.getValueType(); 2104 2105 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 2106 InVT.getVectorNumElements()); 2107 2108 if (N->isStrictFPOpcode()) { 2109 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 2110 { N->getOperand(0), Lo }); 2111 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 2112 { N->getOperand(0), Hi }); 2113 2114 // Build a factor node to remember that this operation is independent 2115 // of the other one. 2116 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2117 Hi.getValue(1)); 2118 2119 // Legalize the chain result - switch anything that used the old chain to 2120 // use the new one. 2121 ReplaceValueWith(SDValue(N, 1), Ch); 2122 } else { 2123 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 2124 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 2125 } 2126 2127 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 2128} 2129 2130SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 2131 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 2132 // end up being split all the way down to individual components. Convert the 2133 // split pieces into integers and reassemble. 2134 SDValue Lo, Hi; 2135 GetSplitVector(N->getOperand(0), Lo, Hi); 2136 Lo = BitConvertToInteger(Lo); 2137 Hi = BitConvertToInteger(Hi); 2138 2139 if (DAG.getDataLayout().isBigEndian()) 2140 std::swap(Lo, Hi); 2141 2142 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 2143 JoinIntegers(Lo, Hi)); 2144} 2145 2146SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2147 // We know that the extracted result type is legal. 2148 EVT SubVT = N->getValueType(0); 2149 SDValue Idx = N->getOperand(1); 2150 SDLoc dl(N); 2151 SDValue Lo, Hi; 2152 GetSplitVector(N->getOperand(0), Lo, Hi); 2153 2154 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2155 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2156 2157 if (IdxVal < LoElts) { 2158 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts && 2159 "Extracted subvector crosses vector split!"); 2160 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 2161 } else { 2162 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 2163 DAG.getConstant(IdxVal - LoElts, dl, 2164 Idx.getValueType())); 2165 } 2166} 2167 2168SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2169 SDValue Vec = N->getOperand(0); 2170 SDValue Idx = N->getOperand(1); 2171 EVT VecVT = Vec.getValueType(); 2172 2173 if (isa<ConstantSDNode>(Idx)) { 2174 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2175 2176 SDValue Lo, Hi; 2177 GetSplitVector(Vec, Lo, Hi); 2178 2179 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2180 2181 if (IdxVal < LoElts) 2182 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 2183 return SDValue(DAG.UpdateNodeOperands(N, Hi, 2184 DAG.getConstant(IdxVal - LoElts, SDLoc(N), 2185 Idx.getValueType())), 0); 2186 } 2187 2188 // See if the target wants to custom expand this node. 2189 if (CustomLowerNode(N, N->getValueType(0), true)) 2190 return SDValue(); 2191 2192 // Make the vector elements byte-addressable if they aren't already. 2193 SDLoc dl(N); 2194 EVT EltVT = VecVT.getVectorElementType(); 2195 if (VecVT.getScalarSizeInBits() < 8) { 2196 EltVT = MVT::i8; 2197 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 2198 VecVT.getVectorNumElements()); 2199 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 2200 } 2201 2202 // Store the vector to the stack. 2203 SDValue StackPtr = DAG.CreateStackTemporary(VecVT); 2204 auto &MF = DAG.getMachineFunction(); 2205 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 2206 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 2207 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); 2208 2209 // Load back the required element. 2210 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 2211 2212 // FIXME: This is to handle i1 vectors with elements promoted to i8. 2213 // i1 vector handling needs general improvement. 2214 if (N->getValueType(0).bitsLT(EltVT)) { 2215 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr, 2216 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); 2217 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0)); 2218 } 2219 2220 return DAG.getExtLoad( 2221 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 2222 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT); 2223} 2224 2225SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) { 2226 SDValue Lo, Hi; 2227 2228 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so 2229 // splitting the result has the same effect as splitting the input operand. 2230 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 2231 2232 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi); 2233} 2234 2235SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, 2236 unsigned OpNo) { 2237 EVT LoVT, HiVT; 2238 SDLoc dl(MGT); 2239 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0)); 2240 2241 SDValue Ch = MGT->getChain(); 2242 SDValue Ptr = MGT->getBasePtr(); 2243 SDValue Index = MGT->getIndex(); 2244 SDValue Scale = MGT->getScale(); 2245 SDValue Mask = MGT->getMask(); 2246 SDValue PassThru = MGT->getPassThru(); 2247 unsigned Alignment = MGT->getOriginalAlignment(); 2248 2249 SDValue MaskLo, MaskHi; 2250 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2251 // Split Mask operand 2252 GetSplitVector(Mask, MaskLo, MaskHi); 2253 else 2254 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 2255 2256 EVT MemoryVT = MGT->getMemoryVT(); 2257 EVT LoMemVT, HiMemVT; 2258 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2259 2260 SDValue PassThruLo, PassThruHi; 2261 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 2262 GetSplitVector(PassThru, PassThruLo, PassThruHi); 2263 else 2264 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 2265 2266 SDValue IndexHi, IndexLo; 2267 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 2268 GetSplitVector(Index, IndexLo, IndexHi); 2269 else 2270 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl); 2271 2272 MachineMemOperand *MMO = DAG.getMachineFunction(). 2273 getMachineMemOperand(MGT->getPointerInfo(), 2274 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(), 2275 Alignment, MGT->getAAInfo(), MGT->getRanges()); 2276 2277 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale}; 2278 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, 2279 OpsLo, MMO, MGT->getIndexType()); 2280 2281 MMO = DAG.getMachineFunction(). 2282 getMachineMemOperand(MGT->getPointerInfo(), 2283 MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), 2284 Alignment, MGT->getAAInfo(), 2285 MGT->getRanges()); 2286 2287 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale}; 2288 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, 2289 OpsHi, MMO, MGT->getIndexType()); 2290 2291 // Build a factor node to remember that this load is independent of the 2292 // other one. 2293 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2294 Hi.getValue(1)); 2295 2296 // Legalize the chain result - switch anything that used the old chain to 2297 // use the new one. 2298 ReplaceValueWith(SDValue(MGT, 1), Ch); 2299 2300 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo, 2301 Hi); 2302 ReplaceValueWith(SDValue(MGT, 0), Res); 2303 return SDValue(); 2304} 2305 2306SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, 2307 unsigned OpNo) { 2308 assert(N->isUnindexed() && "Indexed masked store of vector?"); 2309 SDValue Ch = N->getChain(); 2310 SDValue Ptr = N->getBasePtr(); 2311 SDValue Offset = N->getOffset(); 2312 assert(Offset.isUndef() && "Unexpected indexed masked store offset"); 2313 SDValue Mask = N->getMask(); 2314 SDValue Data = N->getValue(); 2315 EVT MemoryVT = N->getMemoryVT(); 2316 unsigned Alignment = N->getOriginalAlignment(); 2317 SDLoc DL(N); 2318 2319 EVT LoMemVT, HiMemVT; 2320 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2321 2322 SDValue DataLo, DataHi; 2323 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 2324 // Split Data operand 2325 GetSplitVector(Data, DataLo, DataHi); 2326 else 2327 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 2328 2329 // Split Mask operand 2330 SDValue MaskLo, MaskHi; 2331 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 2332 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 2333 } else { 2334 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2335 GetSplitVector(Mask, MaskLo, MaskHi); 2336 else 2337 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 2338 } 2339 2340 SDValue Lo, Hi; 2341 MachineMemOperand *MMO = DAG.getMachineFunction(). 2342 getMachineMemOperand(N->getPointerInfo(), 2343 MachineMemOperand::MOStore, LoMemVT.getStoreSize(), 2344 Alignment, N->getAAInfo(), N->getRanges()); 2345 2346 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO, 2347 N->getAddressingMode(), N->isTruncatingStore(), 2348 N->isCompressingStore()); 2349 2350 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG, 2351 N->isCompressingStore()); 2352 unsigned HiOffset = LoMemVT.getStoreSize(); 2353 2354 MMO = DAG.getMachineFunction().getMachineMemOperand( 2355 N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore, 2356 HiMemVT.getStoreSize(), Alignment, N->getAAInfo(), 2357 N->getRanges()); 2358 2359 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO, 2360 N->getAddressingMode(), N->isTruncatingStore(), 2361 N->isCompressingStore()); 2362 2363 // Build a factor node to remember that this store is independent of the 2364 // other one. 2365 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 2366} 2367 2368SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N, 2369 unsigned OpNo) { 2370 SDValue Ch = N->getChain(); 2371 SDValue Ptr = N->getBasePtr(); 2372 SDValue Mask = N->getMask(); 2373 SDValue Index = N->getIndex(); 2374 SDValue Scale = N->getScale(); 2375 SDValue Data = N->getValue(); 2376 EVT MemoryVT = N->getMemoryVT(); 2377 unsigned Alignment = N->getOriginalAlignment(); 2378 SDLoc DL(N); 2379 2380 // Split all operands 2381 EVT LoMemVT, HiMemVT; 2382 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2383 2384 SDValue DataLo, DataHi; 2385 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 2386 // Split Data operand 2387 GetSplitVector(Data, DataLo, DataHi); 2388 else 2389 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 2390 2391 // Split Mask operand 2392 SDValue MaskLo, MaskHi; 2393 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 2394 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 2395 } else { 2396 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2397 GetSplitVector(Mask, MaskLo, MaskHi); 2398 else 2399 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 2400 } 2401 2402 SDValue IndexHi, IndexLo; 2403 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 2404 GetSplitVector(Index, IndexLo, IndexHi); 2405 else 2406 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL); 2407 2408 SDValue Lo; 2409 MachineMemOperand *MMO = DAG.getMachineFunction(). 2410 getMachineMemOperand(N->getPointerInfo(), 2411 MachineMemOperand::MOStore, LoMemVT.getStoreSize(), 2412 Alignment, N->getAAInfo(), N->getRanges()); 2413 2414 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale}; 2415 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(), 2416 DL, OpsLo, MMO, N->getIndexType()); 2417 2418 MMO = DAG.getMachineFunction(). 2419 getMachineMemOperand(N->getPointerInfo(), 2420 MachineMemOperand::MOStore, HiMemVT.getStoreSize(), 2421 Alignment, N->getAAInfo(), N->getRanges()); 2422 2423 // The order of the Scatter operation after split is well defined. The "Hi" 2424 // part comes after the "Lo". So these two operations should be chained one 2425 // after another. 2426 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale}; 2427 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(), 2428 DL, OpsHi, MMO, N->getIndexType()); 2429} 2430 2431SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 2432 assert(N->isUnindexed() && "Indexed store of vector?"); 2433 assert(OpNo == 1 && "Can only split the stored value"); 2434 SDLoc DL(N); 2435 2436 bool isTruncating = N->isTruncatingStore(); 2437 SDValue Ch = N->getChain(); 2438 SDValue Ptr = N->getBasePtr(); 2439 EVT MemoryVT = N->getMemoryVT(); 2440 unsigned Alignment = N->getOriginalAlignment(); 2441 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 2442 AAMDNodes AAInfo = N->getAAInfo(); 2443 SDValue Lo, Hi; 2444 GetSplitVector(N->getOperand(1), Lo, Hi); 2445 2446 EVT LoMemVT, HiMemVT; 2447 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2448 2449 // Scalarize if the split halves are not byte-sized. 2450 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) 2451 return TLI.scalarizeVectorStore(N, DAG); 2452 2453 unsigned IncrementSize = LoMemVT.getSizeInBits()/8; 2454 2455 if (isTruncating) 2456 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT, 2457 Alignment, MMOFlags, AAInfo); 2458 else 2459 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 2460 AAInfo); 2461 2462 // Increment the pointer to the other half. 2463 Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize); 2464 2465 if (isTruncating) 2466 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, 2467 N->getPointerInfo().getWithOffset(IncrementSize), 2468 HiMemVT, Alignment, MMOFlags, AAInfo); 2469 else 2470 Hi = DAG.getStore(Ch, DL, Hi, Ptr, 2471 N->getPointerInfo().getWithOffset(IncrementSize), 2472 Alignment, MMOFlags, AAInfo); 2473 2474 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 2475} 2476 2477SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 2478 SDLoc DL(N); 2479 2480 // The input operands all must have the same type, and we know the result 2481 // type is valid. Convert this to a buildvector which extracts all the 2482 // input elements. 2483 // TODO: If the input elements are power-two vectors, we could convert this to 2484 // a new CONCAT_VECTORS node with elements that are half-wide. 2485 SmallVector<SDValue, 32> Elts; 2486 EVT EltVT = N->getValueType(0).getVectorElementType(); 2487 for (const SDValue &Op : N->op_values()) { 2488 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 2489 i != e; ++i) { 2490 Elts.push_back(DAG.getNode( 2491 ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op, 2492 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())))); 2493 } 2494 } 2495 2496 return DAG.getBuildVector(N->getValueType(0), DL, Elts); 2497} 2498 2499SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { 2500 // The result type is legal, but the input type is illegal. If splitting 2501 // ends up with the result type of each half still being legal, just 2502 // do that. If, however, that would result in an illegal result type, 2503 // we can try to get more clever with power-two vectors. Specifically, 2504 // split the input type, but also widen the result element size, then 2505 // concatenate the halves and truncate again. For example, consider a target 2506 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 2507 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 2508 // %inlo = v4i32 extract_subvector %in, 0 2509 // %inhi = v4i32 extract_subvector %in, 4 2510 // %lo16 = v4i16 trunc v4i32 %inlo 2511 // %hi16 = v4i16 trunc v4i32 %inhi 2512 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 2513 // %res = v8i8 trunc v8i16 %in16 2514 // 2515 // Without this transform, the original truncate would end up being 2516 // scalarized, which is pretty much always a last resort. 2517 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 2518 SDValue InVec = N->getOperand(OpNo); 2519 EVT InVT = InVec->getValueType(0); 2520 EVT OutVT = N->getValueType(0); 2521 unsigned NumElements = OutVT.getVectorNumElements(); 2522 bool IsFloat = OutVT.isFloatingPoint(); 2523 2524 // Widening should have already made sure this is a power-two vector 2525 // if we're trying to split it at all. assert() that's true, just in case. 2526 assert(!(NumElements & 1) && "Splitting vector, but not in half!"); 2527 2528 unsigned InElementSize = InVT.getScalarSizeInBits(); 2529 unsigned OutElementSize = OutVT.getScalarSizeInBits(); 2530 2531 // Determine the split output VT. If its legal we can just split dirctly. 2532 EVT LoOutVT, HiOutVT; 2533 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT); 2534 assert(LoOutVT == HiOutVT && "Unequal split?"); 2535 2536 // If the input elements are only 1/2 the width of the result elements, 2537 // just use the normal splitting. Our trick only work if there's room 2538 // to split more than once. 2539 if (isTypeLegal(LoOutVT) || 2540 InElementSize <= OutElementSize * 2) 2541 return SplitVecOp_UnaryOp(N); 2542 SDLoc DL(N); 2543 2544 // Don't touch if this will be scalarized. 2545 EVT FinalVT = InVT; 2546 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 2547 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext()); 2548 2549 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector) 2550 return SplitVecOp_UnaryOp(N); 2551 2552 // Get the split input vector. 2553 SDValue InLoVec, InHiVec; 2554 GetSplitVector(InVec, InLoVec, InHiVec); 2555 2556 // Truncate them to 1/2 the element size. 2557 EVT HalfElementVT = IsFloat ? 2558 EVT::getFloatingPointVT(InElementSize/2) : 2559 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 2560 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 2561 NumElements/2); 2562 2563 SDValue HalfLo; 2564 SDValue HalfHi; 2565 SDValue Chain; 2566 if (N->isStrictFPOpcode()) { 2567 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 2568 {N->getOperand(0), HalfLo}); 2569 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 2570 {N->getOperand(0), HalfHi}); 2571 // Legalize the chain result - switch anything that used the old chain to 2572 // use the new one. 2573 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1), 2574 HalfHi.getValue(1)); 2575 } else { 2576 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec); 2577 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec); 2578 } 2579 // Concatenate them to get the full intermediate truncation result. 2580 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 2581 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 2582 HalfHi); 2583 // Now finish up by truncating all the way down to the original result 2584 // type. This should normally be something that ends up being legal directly, 2585 // but in theory if a target has very wide vectors and an annoyingly 2586 // restricted set of legal types, this split can chain to build things up. 2587 2588 if (N->isStrictFPOpcode()) { 2589 SDValue Res = DAG.getNode( 2590 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other}, 2591 {Chain, InterVec, 2592 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))}); 2593 // Relink the chain 2594 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 2595 return Res; 2596 } 2597 2598 return IsFloat 2599 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec, 2600 DAG.getTargetConstant( 2601 0, DL, TLI.getPointerTy(DAG.getDataLayout()))) 2602 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 2603} 2604 2605SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 2606 assert(N->getValueType(0).isVector() && 2607 N->getOperand(0).getValueType().isVector() && 2608 "Operand types must be vectors"); 2609 // The result has a legal vector type, but the input needs splitting. 2610 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 2611 SDLoc DL(N); 2612 GetSplitVector(N->getOperand(0), Lo0, Hi0); 2613 GetSplitVector(N->getOperand(1), Lo1, Hi1); 2614 unsigned PartElements = Lo0.getValueType().getVectorNumElements(); 2615 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements); 2616 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements); 2617 2618 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 2619 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 2620 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 2621 2622 EVT OpVT = N->getOperand(0).getValueType(); 2623 ISD::NodeType ExtendCode = 2624 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 2625 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con); 2626} 2627 2628 2629SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 2630 // The result has a legal vector type, but the input needs splitting. 2631 EVT ResVT = N->getValueType(0); 2632 SDValue Lo, Hi; 2633 SDLoc DL(N); 2634 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 2635 EVT InVT = Lo.getValueType(); 2636 2637 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 2638 InVT.getVectorNumElements()); 2639 2640 if (N->isStrictFPOpcode()) { 2641 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 2642 { N->getOperand(0), Lo, N->getOperand(2) }); 2643 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 2644 { N->getOperand(0), Hi, N->getOperand(2) }); 2645 // Legalize the chain result - switch anything that used the old chain to 2646 // use the new one. 2647 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 2648 Lo.getValue(1), Hi.getValue(1)); 2649 ReplaceValueWith(SDValue(N, 1), NewChain); 2650 } else { 2651 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 2652 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 2653 } 2654 2655 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 2656} 2657 2658SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) { 2659 // The result (and the first input) has a legal vector type, but the second 2660 // input needs splitting. 2661 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements()); 2662} 2663 2664 2665//===----------------------------------------------------------------------===// 2666// Result Vector Widening 2667//===----------------------------------------------------------------------===// 2668 2669void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 2670 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG); 2671 dbgs() << "\n"); 2672 2673 // See if the target wants to custom widen this node. 2674 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 2675 return; 2676 2677 SDValue Res = SDValue(); 2678 switch (N->getOpcode()) { 2679 default: 2680#ifndef NDEBUG 2681 dbgs() << "WidenVectorResult #" << ResNo << ": "; 2682 N->dump(&DAG); 2683 dbgs() << "\n"; 2684#endif 2685 llvm_unreachable("Do not know how to widen the result of this operator!"); 2686 2687 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 2688 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 2689 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 2690 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 2691 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 2692 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 2693 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 2694 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break; 2695 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 2696 case ISD::VSELECT: 2697 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 2698 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 2699 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 2700 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 2701 case ISD::VECTOR_SHUFFLE: 2702 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 2703 break; 2704 case ISD::MLOAD: 2705 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N)); 2706 break; 2707 case ISD::MGATHER: 2708 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N)); 2709 break; 2710 2711 case ISD::ADD: 2712 case ISD::AND: 2713 case ISD::MUL: 2714 case ISD::MULHS: 2715 case ISD::MULHU: 2716 case ISD::OR: 2717 case ISD::SUB: 2718 case ISD::XOR: 2719 case ISD::FMINNUM: 2720 case ISD::FMAXNUM: 2721 case ISD::FMINIMUM: 2722 case ISD::FMAXIMUM: 2723 case ISD::SMIN: 2724 case ISD::SMAX: 2725 case ISD::UMIN: 2726 case ISD::UMAX: 2727 case ISD::UADDSAT: 2728 case ISD::SADDSAT: 2729 case ISD::USUBSAT: 2730 case ISD::SSUBSAT: 2731 Res = WidenVecRes_Binary(N); 2732 break; 2733 2734 case ISD::FADD: 2735 case ISD::FMUL: 2736 case ISD::FPOW: 2737 case ISD::FSUB: 2738 case ISD::FDIV: 2739 case ISD::FREM: 2740 case ISD::SDIV: 2741 case ISD::UDIV: 2742 case ISD::SREM: 2743 case ISD::UREM: 2744 Res = WidenVecRes_BinaryCanTrap(N); 2745 break; 2746 2747 case ISD::SMULFIX: 2748 case ISD::SMULFIXSAT: 2749 case ISD::UMULFIX: 2750 case ISD::UMULFIXSAT: 2751 // These are binary operations, but with an extra operand that shouldn't 2752 // be widened (the scale). 2753 Res = WidenVecRes_BinaryWithExtraScalarOp(N); 2754 break; 2755 2756#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 2757 case ISD::STRICT_##DAGN: 2758#include "llvm/IR/ConstrainedOps.def" 2759 Res = WidenVecRes_StrictFP(N); 2760 break; 2761 2762 case ISD::UADDO: 2763 case ISD::SADDO: 2764 case ISD::USUBO: 2765 case ISD::SSUBO: 2766 case ISD::UMULO: 2767 case ISD::SMULO: 2768 Res = WidenVecRes_OverflowOp(N, ResNo); 2769 break; 2770 2771 case ISD::FCOPYSIGN: 2772 Res = WidenVecRes_FCOPYSIGN(N); 2773 break; 2774 2775 case ISD::FPOWI: 2776 Res = WidenVecRes_POWI(N); 2777 break; 2778 2779 case ISD::SHL: 2780 case ISD::SRA: 2781 case ISD::SRL: 2782 Res = WidenVecRes_Shift(N); 2783 break; 2784 2785 case ISD::ANY_EXTEND_VECTOR_INREG: 2786 case ISD::SIGN_EXTEND_VECTOR_INREG: 2787 case ISD::ZERO_EXTEND_VECTOR_INREG: 2788 Res = WidenVecRes_EXTEND_VECTOR_INREG(N); 2789 break; 2790 2791 case ISD::ANY_EXTEND: 2792 case ISD::FP_EXTEND: 2793 case ISD::FP_ROUND: 2794 case ISD::FP_TO_SINT: 2795 case ISD::FP_TO_UINT: 2796 case ISD::SIGN_EXTEND: 2797 case ISD::SINT_TO_FP: 2798 case ISD::TRUNCATE: 2799 case ISD::UINT_TO_FP: 2800 case ISD::ZERO_EXTEND: 2801 Res = WidenVecRes_Convert(N); 2802 break; 2803 2804 case ISD::FABS: 2805 case ISD::FCEIL: 2806 case ISD::FCOS: 2807 case ISD::FEXP: 2808 case ISD::FEXP2: 2809 case ISD::FFLOOR: 2810 case ISD::FLOG: 2811 case ISD::FLOG10: 2812 case ISD::FLOG2: 2813 case ISD::FNEARBYINT: 2814 case ISD::FRINT: 2815 case ISD::FROUND: 2816 case ISD::FSIN: 2817 case ISD::FSQRT: 2818 case ISD::FTRUNC: { 2819 // We're going to widen this vector op to a legal type by padding with undef 2820 // elements. If the wide vector op is eventually going to be expanded to 2821 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary 2822 // libcalls on the undef elements. 2823 EVT VT = N->getValueType(0); 2824 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2825 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) && 2826 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) { 2827 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements()); 2828 break; 2829 } 2830 } 2831 // If the target has custom/legal support for the scalar FP intrinsic ops 2832 // (they are probably not destined to become libcalls), then widen those like 2833 // any other unary ops. 2834 LLVM_FALLTHROUGH; 2835 2836 case ISD::ABS: 2837 case ISD::BITREVERSE: 2838 case ISD::BSWAP: 2839 case ISD::CTLZ: 2840 case ISD::CTLZ_ZERO_UNDEF: 2841 case ISD::CTPOP: 2842 case ISD::CTTZ: 2843 case ISD::CTTZ_ZERO_UNDEF: 2844 case ISD::FNEG: 2845 case ISD::FCANONICALIZE: 2846 Res = WidenVecRes_Unary(N); 2847 break; 2848 case ISD::FMA: 2849 Res = WidenVecRes_Ternary(N); 2850 break; 2851 } 2852 2853 // If Res is null, the sub-method took care of registering the result. 2854 if (Res.getNode()) 2855 SetWidenedVector(SDValue(N, ResNo), Res); 2856} 2857 2858SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 2859 // Ternary op widening. 2860 SDLoc dl(N); 2861 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2862 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2863 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2864 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 2865 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 2866} 2867 2868SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 2869 // Binary op widening. 2870 SDLoc dl(N); 2871 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2872 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2873 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2874 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags()); 2875} 2876 2877SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) { 2878 // Binary op widening, but with an extra operand that shouldn't be widened. 2879 SDLoc dl(N); 2880 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2881 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2882 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2883 SDValue InOp3 = N->getOperand(2); 2884 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3, 2885 N->getFlags()); 2886} 2887 2888// Given a vector of operations that have been broken up to widen, see 2889// if we can collect them together into the next widest legal VT. This 2890// implementation is trap-safe. 2891static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, 2892 SmallVectorImpl<SDValue> &ConcatOps, 2893 unsigned ConcatEnd, EVT VT, EVT MaxVT, 2894 EVT WidenVT) { 2895 // Check to see if we have a single operation with the widen type. 2896 if (ConcatEnd == 1) { 2897 VT = ConcatOps[0].getValueType(); 2898 if (VT == WidenVT) 2899 return ConcatOps[0]; 2900 } 2901 2902 SDLoc dl(ConcatOps[0]); 2903 EVT WidenEltVT = WidenVT.getVectorElementType(); 2904 2905 // while (Some element of ConcatOps is not of type MaxVT) { 2906 // From the end of ConcatOps, collect elements of the same type and put 2907 // them into an op of the next larger supported type 2908 // } 2909 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 2910 int Idx = ConcatEnd - 1; 2911 VT = ConcatOps[Idx--].getValueType(); 2912 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 2913 Idx--; 2914 2915 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 2916 EVT NextVT; 2917 do { 2918 NextSize *= 2; 2919 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 2920 } while (!TLI.isTypeLegal(NextVT)); 2921 2922 if (!VT.isVector()) { 2923 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 2924 SDValue VecOp = DAG.getUNDEF(NextVT); 2925 unsigned NumToInsert = ConcatEnd - Idx - 1; 2926 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 2927 VecOp = DAG.getNode( 2928 ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx], 2929 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 2930 } 2931 ConcatOps[Idx+1] = VecOp; 2932 ConcatEnd = Idx + 2; 2933 } else { 2934 // Vector type, create a CONCAT_VECTORS of type NextVT 2935 SDValue undefVec = DAG.getUNDEF(VT); 2936 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 2937 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 2938 unsigned RealVals = ConcatEnd - Idx - 1; 2939 unsigned SubConcatEnd = 0; 2940 unsigned SubConcatIdx = Idx + 1; 2941 while (SubConcatEnd < RealVals) 2942 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 2943 while (SubConcatEnd < OpsToConcat) 2944 SubConcatOps[SubConcatEnd++] = undefVec; 2945 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 2946 NextVT, SubConcatOps); 2947 ConcatEnd = SubConcatIdx + 1; 2948 } 2949 } 2950 2951 // Check to see if we have a single operation with the widen type. 2952 if (ConcatEnd == 1) { 2953 VT = ConcatOps[0].getValueType(); 2954 if (VT == WidenVT) 2955 return ConcatOps[0]; 2956 } 2957 2958 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 2959 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 2960 if (NumOps != ConcatEnd ) { 2961 SDValue UndefVal = DAG.getUNDEF(MaxVT); 2962 for (unsigned j = ConcatEnd; j < NumOps; ++j) 2963 ConcatOps[j] = UndefVal; 2964 } 2965 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 2966 makeArrayRef(ConcatOps.data(), NumOps)); 2967} 2968 2969SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 2970 // Binary op widening for operations that can trap. 2971 unsigned Opcode = N->getOpcode(); 2972 SDLoc dl(N); 2973 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2974 EVT WidenEltVT = WidenVT.getVectorElementType(); 2975 EVT VT = WidenVT; 2976 unsigned NumElts = VT.getVectorNumElements(); 2977 const SDNodeFlags Flags = N->getFlags(); 2978 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 2979 NumElts = NumElts / 2; 2980 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 2981 } 2982 2983 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 2984 // Operation doesn't trap so just widen as normal. 2985 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2986 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2987 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags); 2988 } 2989 2990 // No legal vector version so unroll the vector operation and then widen. 2991 if (NumElts == 1) 2992 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 2993 2994 // Since the operation can trap, apply operation on the original vector. 2995 EVT MaxVT = VT; 2996 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 2997 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 2998 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 2999 3000 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 3001 unsigned ConcatEnd = 0; // Current ConcatOps index. 3002 int Idx = 0; // Current Idx into input vectors. 3003 3004 // NumElts := greatest legal vector size (at most WidenVT) 3005 // while (orig. vector has unhandled elements) { 3006 // take munches of size NumElts from the beginning and add to ConcatOps 3007 // NumElts := next smaller supported vector size or 1 3008 // } 3009 while (CurNumElts != 0) { 3010 while (CurNumElts >= NumElts) { 3011 SDValue EOp1 = DAG.getNode( 3012 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 3013 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3014 SDValue EOp2 = DAG.getNode( 3015 ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 3016 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3017 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags); 3018 Idx += NumElts; 3019 CurNumElts -= NumElts; 3020 } 3021 do { 3022 NumElts = NumElts / 2; 3023 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 3024 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 3025 3026 if (NumElts == 1) { 3027 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 3028 SDValue EOp1 = DAG.getNode( 3029 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1, 3030 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3031 SDValue EOp2 = DAG.getNode( 3032 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2, 3033 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3034 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 3035 EOp1, EOp2, Flags); 3036 } 3037 CurNumElts = 0; 3038 } 3039 } 3040 3041 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 3042} 3043 3044SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { 3045 switch (N->getOpcode()) { 3046 case ISD::STRICT_FSETCC: 3047 case ISD::STRICT_FSETCCS: 3048 return WidenVecRes_STRICT_FSETCC(N); 3049 case ISD::STRICT_FP_EXTEND: 3050 case ISD::STRICT_FP_ROUND: 3051 case ISD::STRICT_FP_TO_SINT: 3052 case ISD::STRICT_FP_TO_UINT: 3053 case ISD::STRICT_SINT_TO_FP: 3054 case ISD::STRICT_UINT_TO_FP: 3055 return WidenVecRes_Convert_StrictFP(N); 3056 default: 3057 break; 3058 } 3059 3060 // StrictFP op widening for operations that can trap. 3061 unsigned NumOpers = N->getNumOperands(); 3062 unsigned Opcode = N->getOpcode(); 3063 SDLoc dl(N); 3064 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3065 EVT WidenEltVT = WidenVT.getVectorElementType(); 3066 EVT VT = WidenVT; 3067 unsigned NumElts = VT.getVectorNumElements(); 3068 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 3069 NumElts = NumElts / 2; 3070 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 3071 } 3072 3073 // No legal vector version so unroll the vector operation and then widen. 3074 if (NumElts == 1) 3075 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements()); 3076 3077 // Since the operation can trap, apply operation on the original vector. 3078 EVT MaxVT = VT; 3079 SmallVector<SDValue, 4> InOps; 3080 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 3081 3082 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 3083 SmallVector<SDValue, 16> Chains; 3084 unsigned ConcatEnd = 0; // Current ConcatOps index. 3085 int Idx = 0; // Current Idx into input vectors. 3086 3087 // The Chain is the first operand. 3088 InOps.push_back(N->getOperand(0)); 3089 3090 // Now process the remaining operands. 3091 for (unsigned i = 1; i < NumOpers; ++i) { 3092 SDValue Oper = N->getOperand(i); 3093 3094 if (Oper.getValueType().isVector()) { 3095 assert(Oper.getValueType() == N->getValueType(0) && 3096 "Invalid operand type to widen!"); 3097 Oper = GetWidenedVector(Oper); 3098 } 3099 3100 InOps.push_back(Oper); 3101 } 3102 3103 // NumElts := greatest legal vector size (at most WidenVT) 3104 // while (orig. vector has unhandled elements) { 3105 // take munches of size NumElts from the beginning and add to ConcatOps 3106 // NumElts := next smaller supported vector size or 1 3107 // } 3108 while (CurNumElts != 0) { 3109 while (CurNumElts >= NumElts) { 3110 SmallVector<SDValue, 4> EOps; 3111 3112 for (unsigned i = 0; i < NumOpers; ++i) { 3113 SDValue Op = InOps[i]; 3114 3115 if (Op.getValueType().isVector()) 3116 Op = DAG.getNode( 3117 ISD::EXTRACT_SUBVECTOR, dl, VT, Op, 3118 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3119 3120 EOps.push_back(Op); 3121 } 3122 3123 EVT OperVT[] = {VT, MVT::Other}; 3124 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps); 3125 ConcatOps[ConcatEnd++] = Oper; 3126 Chains.push_back(Oper.getValue(1)); 3127 Idx += NumElts; 3128 CurNumElts -= NumElts; 3129 } 3130 do { 3131 NumElts = NumElts / 2; 3132 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 3133 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 3134 3135 if (NumElts == 1) { 3136 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 3137 SmallVector<SDValue, 4> EOps; 3138 3139 for (unsigned i = 0; i < NumOpers; ++i) { 3140 SDValue Op = InOps[i]; 3141 3142 if (Op.getValueType().isVector()) 3143 Op = DAG.getNode( 3144 ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op, 3145 DAG.getConstant(Idx, dl, 3146 TLI.getVectorIdxTy(DAG.getDataLayout()))); 3147 3148 EOps.push_back(Op); 3149 } 3150 3151 EVT WidenVT[] = {WidenEltVT, MVT::Other}; 3152 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps); 3153 ConcatOps[ConcatEnd++] = Oper; 3154 Chains.push_back(Oper.getValue(1)); 3155 } 3156 CurNumElts = 0; 3157 } 3158 } 3159 3160 // Build a factor node to remember all the Ops that have been created. 3161 SDValue NewChain; 3162 if (Chains.size() == 1) 3163 NewChain = Chains[0]; 3164 else 3165 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 3166 ReplaceValueWith(SDValue(N, 1), NewChain); 3167 3168 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 3169} 3170 3171SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) { 3172 SDLoc DL(N); 3173 EVT ResVT = N->getValueType(0); 3174 EVT OvVT = N->getValueType(1); 3175 EVT WideResVT, WideOvVT; 3176 SDValue WideLHS, WideRHS; 3177 3178 // TODO: This might result in a widen/split loop. 3179 if (ResNo == 0) { 3180 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT); 3181 WideOvVT = EVT::getVectorVT( 3182 *DAG.getContext(), OvVT.getVectorElementType(), 3183 WideResVT.getVectorNumElements()); 3184 3185 WideLHS = GetWidenedVector(N->getOperand(0)); 3186 WideRHS = GetWidenedVector(N->getOperand(1)); 3187 } else { 3188 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT); 3189 WideResVT = EVT::getVectorVT( 3190 *DAG.getContext(), ResVT.getVectorElementType(), 3191 WideOvVT.getVectorNumElements()); 3192 3193 SDValue Zero = DAG.getConstant( 3194 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())); 3195 WideLHS = DAG.getNode( 3196 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 3197 N->getOperand(0), Zero); 3198 WideRHS = DAG.getNode( 3199 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 3200 N->getOperand(1), Zero); 3201 } 3202 3203 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT); 3204 SDNode *WideNode = DAG.getNode( 3205 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode(); 3206 3207 // Replace the other vector result not being explicitly widened here. 3208 unsigned OtherNo = 1 - ResNo; 3209 EVT OtherVT = N->getValueType(OtherNo); 3210 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) { 3211 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo)); 3212 } else { 3213 SDValue Zero = DAG.getConstant( 3214 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())); 3215 SDValue OtherVal = DAG.getNode( 3216 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero); 3217 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 3218 } 3219 3220 return SDValue(WideNode, ResNo); 3221} 3222 3223SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 3224 SDValue InOp = N->getOperand(0); 3225 SDLoc DL(N); 3226 3227 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3228 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3229 3230 EVT InVT = InOp.getValueType(); 3231 EVT InEltVT = InVT.getVectorElementType(); 3232 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts); 3233 3234 unsigned Opcode = N->getOpcode(); 3235 unsigned InVTNumElts = InVT.getVectorNumElements(); 3236 const SDNodeFlags Flags = N->getFlags(); 3237 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 3238 InOp = GetWidenedVector(N->getOperand(0)); 3239 InVT = InOp.getValueType(); 3240 InVTNumElts = InVT.getVectorNumElements(); 3241 if (InVTNumElts == WidenNumElts) { 3242 if (N->getNumOperands() == 1) 3243 return DAG.getNode(Opcode, DL, WidenVT, InOp); 3244 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags); 3245 } 3246 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) { 3247 // If both input and result vector types are of same width, extend 3248 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which 3249 // accepts fewer elements in the result than in the input. 3250 if (Opcode == ISD::ANY_EXTEND) 3251 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 3252 if (Opcode == ISD::SIGN_EXTEND) 3253 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 3254 if (Opcode == ISD::ZERO_EXTEND) 3255 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 3256 } 3257 } 3258 3259 if (TLI.isTypeLegal(InWidenVT)) { 3260 // Because the result and the input are different vector types, widening 3261 // the result could create a legal type but widening the input might make 3262 // it an illegal type that might lead to repeatedly splitting the input 3263 // and then widening it. To avoid this, we widen the input only if 3264 // it results in a legal type. 3265 if (WidenNumElts % InVTNumElts == 0) { 3266 // Widen the input and call convert on the widened input vector. 3267 unsigned NumConcat = WidenNumElts/InVTNumElts; 3268 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT)); 3269 Ops[0] = InOp; 3270 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 3271 if (N->getNumOperands() == 1) 3272 return DAG.getNode(Opcode, DL, WidenVT, InVec); 3273 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags); 3274 } 3275 3276 if (InVTNumElts % WidenNumElts == 0) { 3277 SDValue InVal = DAG.getNode( 3278 ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp, 3279 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3280 // Extract the input and convert the shorten input vector. 3281 if (N->getNumOperands() == 1) 3282 return DAG.getNode(Opcode, DL, WidenVT, InVal); 3283 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags); 3284 } 3285 } 3286 3287 // Otherwise unroll into some nasty scalar code and rebuild the vector. 3288 EVT EltVT = WidenVT.getVectorElementType(); 3289 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT)); 3290 // Use the original element count so we don't do more scalar opts than 3291 // necessary. 3292 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 3293 for (unsigned i=0; i < MinElts; ++i) { 3294 SDValue Val = DAG.getNode( 3295 ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 3296 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3297 if (N->getNumOperands() == 1) 3298 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 3299 else 3300 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags); 3301 } 3302 3303 return DAG.getBuildVector(WidenVT, DL, Ops); 3304} 3305 3306SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { 3307 SDValue InOp = N->getOperand(1); 3308 SDLoc DL(N); 3309 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 3310 3311 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3312 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3313 SmallVector<EVT, 2> WidenVTs = { WidenVT, MVT::Other }; 3314 3315 EVT InVT = InOp.getValueType(); 3316 EVT InEltVT = InVT.getVectorElementType(); 3317 3318 unsigned Opcode = N->getOpcode(); 3319 3320 // FIXME: Optimizations need to be implemented here. 3321 3322 // Otherwise unroll into some nasty scalar code and rebuild the vector. 3323 EVT EltVT = WidenVT.getVectorElementType(); 3324 SmallVector<EVT, 2> EltVTs = { EltVT, MVT::Other }; 3325 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT)); 3326 SmallVector<SDValue, 32> OpChains; 3327 // Use the original element count so we don't do more scalar opts than 3328 // necessary. 3329 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 3330 for (unsigned i=0; i < MinElts; ++i) { 3331 NewOps[1] = DAG.getNode( 3332 ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 3333 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3334 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps); 3335 OpChains.push_back(Ops[i].getValue(1)); 3336 } 3337 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains); 3338 ReplaceValueWith(SDValue(N, 1), NewChain); 3339 3340 return DAG.getBuildVector(WidenVT, DL, Ops); 3341} 3342 3343SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { 3344 unsigned Opcode = N->getOpcode(); 3345 SDValue InOp = N->getOperand(0); 3346 SDLoc DL(N); 3347 3348 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3349 EVT WidenSVT = WidenVT.getVectorElementType(); 3350 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3351 3352 EVT InVT = InOp.getValueType(); 3353 EVT InSVT = InVT.getVectorElementType(); 3354 unsigned InVTNumElts = InVT.getVectorNumElements(); 3355 3356 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 3357 InOp = GetWidenedVector(InOp); 3358 InVT = InOp.getValueType(); 3359 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) { 3360 switch (Opcode) { 3361 case ISD::ANY_EXTEND_VECTOR_INREG: 3362 case ISD::SIGN_EXTEND_VECTOR_INREG: 3363 case ISD::ZERO_EXTEND_VECTOR_INREG: 3364 return DAG.getNode(Opcode, DL, WidenVT, InOp); 3365 } 3366 } 3367 } 3368 3369 // Unroll, extend the scalars and rebuild the vector. 3370 SmallVector<SDValue, 16> Ops; 3371 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) { 3372 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp, 3373 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3374 switch (Opcode) { 3375 case ISD::ANY_EXTEND_VECTOR_INREG: 3376 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val); 3377 break; 3378 case ISD::SIGN_EXTEND_VECTOR_INREG: 3379 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val); 3380 break; 3381 case ISD::ZERO_EXTEND_VECTOR_INREG: 3382 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val); 3383 break; 3384 default: 3385 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected"); 3386 } 3387 Ops.push_back(Val); 3388 } 3389 3390 while (Ops.size() != WidenNumElts) 3391 Ops.push_back(DAG.getUNDEF(WidenSVT)); 3392 3393 return DAG.getBuildVector(WidenVT, DL, Ops); 3394} 3395 3396SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) { 3397 // If this is an FCOPYSIGN with same input types, we can treat it as a 3398 // normal (can trap) binary op. 3399 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType()) 3400 return WidenVecRes_BinaryCanTrap(N); 3401 3402 // If the types are different, fall back to unrolling. 3403 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3404 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 3405} 3406 3407SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 3408 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3409 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3410 SDValue ShOp = N->getOperand(1); 3411 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 3412} 3413 3414SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) { 3415 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3416 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3417 SDValue ShOp = N->getOperand(1); 3418 3419 EVT ShVT = ShOp.getValueType(); 3420 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) { 3421 ShOp = GetWidenedVector(ShOp); 3422 ShVT = ShOp.getValueType(); 3423 } 3424 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(), 3425 ShVT.getVectorElementType(), 3426 WidenVT.getVectorNumElements()); 3427 if (ShVT != ShWidenVT) 3428 ShOp = ModifyToType(ShOp, ShWidenVT); 3429 3430 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 3431} 3432 3433SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 3434 // Unary op widening. 3435 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3436 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3437 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 3438} 3439 3440SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 3441 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3442 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 3443 cast<VTSDNode>(N->getOperand(1))->getVT() 3444 .getVectorElementType(), 3445 WidenVT.getVectorNumElements()); 3446 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 3447 return DAG.getNode(N->getOpcode(), SDLoc(N), 3448 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 3449} 3450 3451SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 3452 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 3453 return GetWidenedVector(WidenVec); 3454} 3455 3456SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 3457 SDValue InOp = N->getOperand(0); 3458 EVT InVT = InOp.getValueType(); 3459 EVT VT = N->getValueType(0); 3460 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3461 SDLoc dl(N); 3462 3463 switch (getTypeAction(InVT)) { 3464 case TargetLowering::TypeLegal: 3465 break; 3466 case TargetLowering::TypePromoteInteger: { 3467 // If the incoming type is a vector that is being promoted, then 3468 // we know that the elements are arranged differently and that we 3469 // must perform the conversion using a stack slot. 3470 if (InVT.isVector()) 3471 break; 3472 3473 // If the InOp is promoted to the same size, convert it. Otherwise, 3474 // fall out of the switch and widen the promoted input. 3475 SDValue NInOp = GetPromotedInteger(InOp); 3476 EVT NInVT = NInOp.getValueType(); 3477 if (WidenVT.bitsEq(NInVT)) { 3478 // For big endian targets we need to shift the input integer or the 3479 // interesting bits will end up at the wrong place. 3480 if (DAG.getDataLayout().isBigEndian()) { 3481 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 3482 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout()); 3483 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!"); 3484 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp, 3485 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); 3486 } 3487 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp); 3488 } 3489 InOp = NInOp; 3490 InVT = NInVT; 3491 break; 3492 } 3493 case TargetLowering::TypeSoftenFloat: 3494 case TargetLowering::TypePromoteFloat: 3495 case TargetLowering::TypeExpandInteger: 3496 case TargetLowering::TypeExpandFloat: 3497 case TargetLowering::TypeScalarizeVector: 3498 case TargetLowering::TypeSplitVector: 3499 break; 3500 case TargetLowering::TypeWidenVector: 3501 // If the InOp is widened to the same size, convert it. Otherwise, fall 3502 // out of the switch and widen the widened input. 3503 InOp = GetWidenedVector(InOp); 3504 InVT = InOp.getValueType(); 3505 if (WidenVT.bitsEq(InVT)) 3506 // The input widens to the same size. Convert to the widen value. 3507 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 3508 break; 3509 } 3510 3511 unsigned WidenSize = WidenVT.getSizeInBits(); 3512 unsigned InSize = InVT.getSizeInBits(); 3513 // x86mmx is not an acceptable vector element type, so don't try. 3514 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 3515 // Determine new input vector type. The new input vector type will use 3516 // the same element type (if its a vector) or use the input type as a 3517 // vector. It is the same size as the type to widen to. 3518 EVT NewInVT; 3519 unsigned NewNumElts = WidenSize / InSize; 3520 if (InVT.isVector()) { 3521 EVT InEltVT = InVT.getVectorElementType(); 3522 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 3523 WidenSize / InEltVT.getSizeInBits()); 3524 } else { 3525 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 3526 } 3527 3528 if (TLI.isTypeLegal(NewInVT)) { 3529 SDValue NewVec; 3530 if (InVT.isVector()) { 3531 // Because the result and the input are different vector types, widening 3532 // the result could create a legal type but widening the input might make 3533 // it an illegal type that might lead to repeatedly splitting the input 3534 // and then widening it. To avoid this, we widen the input only if 3535 // it results in a legal type. 3536 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT)); 3537 Ops[0] = InOp; 3538 3539 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 3540 } else { 3541 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp); 3542 } 3543 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 3544 } 3545 } 3546 3547 return CreateStackStoreLoad(InOp, WidenVT); 3548} 3549 3550SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 3551 SDLoc dl(N); 3552 // Build a vector with undefined for the new nodes. 3553 EVT VT = N->getValueType(0); 3554 3555 // Integer BUILD_VECTOR operands may be larger than the node's vector element 3556 // type. The UNDEFs need to have the same type as the existing operands. 3557 EVT EltVT = N->getOperand(0).getValueType(); 3558 unsigned NumElts = VT.getVectorNumElements(); 3559 3560 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3561 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3562 3563 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 3564 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 3565 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 3566 3567 return DAG.getBuildVector(WidenVT, dl, NewOps); 3568} 3569 3570SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 3571 EVT InVT = N->getOperand(0).getValueType(); 3572 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3573 SDLoc dl(N); 3574 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3575 unsigned NumInElts = InVT.getVectorNumElements(); 3576 unsigned NumOperands = N->getNumOperands(); 3577 3578 bool InputWidened = false; // Indicates we need to widen the input. 3579 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 3580 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) { 3581 // Add undef vectors to widen to correct length. 3582 unsigned NumConcat = WidenVT.getVectorNumElements() / 3583 InVT.getVectorNumElements(); 3584 SDValue UndefVal = DAG.getUNDEF(InVT); 3585 SmallVector<SDValue, 16> Ops(NumConcat); 3586 for (unsigned i=0; i < NumOperands; ++i) 3587 Ops[i] = N->getOperand(i); 3588 for (unsigned i = NumOperands; i != NumConcat; ++i) 3589 Ops[i] = UndefVal; 3590 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 3591 } 3592 } else { 3593 InputWidened = true; 3594 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 3595 // The inputs and the result are widen to the same value. 3596 unsigned i; 3597 for (i=1; i < NumOperands; ++i) 3598 if (!N->getOperand(i).isUndef()) 3599 break; 3600 3601 if (i == NumOperands) 3602 // Everything but the first operand is an UNDEF so just return the 3603 // widened first operand. 3604 return GetWidenedVector(N->getOperand(0)); 3605 3606 if (NumOperands == 2) { 3607 // Replace concat of two operands with a shuffle. 3608 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 3609 for (unsigned i = 0; i < NumInElts; ++i) { 3610 MaskOps[i] = i; 3611 MaskOps[i + NumInElts] = i + WidenNumElts; 3612 } 3613 return DAG.getVectorShuffle(WidenVT, dl, 3614 GetWidenedVector(N->getOperand(0)), 3615 GetWidenedVector(N->getOperand(1)), 3616 MaskOps); 3617 } 3618 } 3619 } 3620 3621 // Fall back to use extracts and build vector. 3622 EVT EltVT = WidenVT.getVectorElementType(); 3623 SmallVector<SDValue, 16> Ops(WidenNumElts); 3624 unsigned Idx = 0; 3625 for (unsigned i=0; i < NumOperands; ++i) { 3626 SDValue InOp = N->getOperand(i); 3627 if (InputWidened) 3628 InOp = GetWidenedVector(InOp); 3629 for (unsigned j=0; j < NumInElts; ++j) 3630 Ops[Idx++] = DAG.getNode( 3631 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3632 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 3633 } 3634 SDValue UndefVal = DAG.getUNDEF(EltVT); 3635 for (; Idx < WidenNumElts; ++Idx) 3636 Ops[Idx] = UndefVal; 3637 return DAG.getBuildVector(WidenVT, dl, Ops); 3638} 3639 3640SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 3641 EVT VT = N->getValueType(0); 3642 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3643 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3644 SDValue InOp = N->getOperand(0); 3645 SDValue Idx = N->getOperand(1); 3646 SDLoc dl(N); 3647 3648 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 3649 InOp = GetWidenedVector(InOp); 3650 3651 EVT InVT = InOp.getValueType(); 3652 3653 // Check if we can just return the input vector after widening. 3654 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 3655 if (IdxVal == 0 && InVT == WidenVT) 3656 return InOp; 3657 3658 // Check if we can extract from the vector. 3659 unsigned InNumElts = InVT.getVectorNumElements(); 3660 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 3661 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 3662 3663 // We could try widening the input to the right length but for now, extract 3664 // the original elements, fill the rest with undefs and build a vector. 3665 SmallVector<SDValue, 16> Ops(WidenNumElts); 3666 EVT EltVT = VT.getVectorElementType(); 3667 unsigned NumElts = VT.getVectorNumElements(); 3668 unsigned i; 3669 for (i=0; i < NumElts; ++i) 3670 Ops[i] = 3671 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3672 DAG.getConstant(IdxVal + i, dl, 3673 TLI.getVectorIdxTy(DAG.getDataLayout()))); 3674 3675 SDValue UndefVal = DAG.getUNDEF(EltVT); 3676 for (; i < WidenNumElts; ++i) 3677 Ops[i] = UndefVal; 3678 return DAG.getBuildVector(WidenVT, dl, Ops); 3679} 3680 3681SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 3682 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3683 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 3684 InOp.getValueType(), InOp, 3685 N->getOperand(1), N->getOperand(2)); 3686} 3687 3688SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 3689 LoadSDNode *LD = cast<LoadSDNode>(N); 3690 ISD::LoadExtType ExtType = LD->getExtensionType(); 3691 3692 SDValue Result; 3693 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 3694 if (ExtType != ISD::NON_EXTLOAD) 3695 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 3696 else 3697 Result = GenWidenVectorLoads(LdChain, LD); 3698 3699 // If we generate a single load, we can use that for the chain. Otherwise, 3700 // build a factor node to remember the multiple loads are independent and 3701 // chain to that. 3702 SDValue NewChain; 3703 if (LdChain.size() == 1) 3704 NewChain = LdChain[0]; 3705 else 3706 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 3707 3708 // Modified the chain - switch anything that used the old chain to use 3709 // the new one. 3710 ReplaceValueWith(SDValue(N, 1), NewChain); 3711 3712 return Result; 3713} 3714 3715SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { 3716 3717 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0)); 3718 SDValue Mask = N->getMask(); 3719 EVT MaskVT = Mask.getValueType(); 3720 SDValue PassThru = GetWidenedVector(N->getPassThru()); 3721 ISD::LoadExtType ExtType = N->getExtensionType(); 3722 SDLoc dl(N); 3723 3724 // The mask should be widened as well 3725 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 3726 MaskVT.getVectorElementType(), 3727 WidenVT.getVectorNumElements()); 3728 Mask = ModifyToType(Mask, WideMaskVT, true); 3729 3730 SDValue Res = DAG.getMaskedLoad( 3731 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, 3732 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), 3733 ExtType, N->isExpandingLoad()); 3734 // Legalize the chain result - switch anything that used the old chain to 3735 // use the new one. 3736 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3737 return Res; 3738} 3739 3740SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) { 3741 3742 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3743 SDValue Mask = N->getMask(); 3744 EVT MaskVT = Mask.getValueType(); 3745 SDValue PassThru = GetWidenedVector(N->getPassThru()); 3746 SDValue Scale = N->getScale(); 3747 unsigned NumElts = WideVT.getVectorNumElements(); 3748 SDLoc dl(N); 3749 3750 // The mask should be widened as well 3751 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 3752 MaskVT.getVectorElementType(), 3753 WideVT.getVectorNumElements()); 3754 Mask = ModifyToType(Mask, WideMaskVT, true); 3755 3756 // Widen the Index operand 3757 SDValue Index = N->getIndex(); 3758 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 3759 Index.getValueType().getScalarType(), 3760 NumElts); 3761 Index = ModifyToType(Index, WideIndexVT); 3762 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index, 3763 Scale }; 3764 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), 3765 N->getMemoryVT(), dl, Ops, 3766 N->getMemOperand(), N->getIndexType()); 3767 3768 // Legalize the chain result - switch anything that used the old chain to 3769 // use the new one. 3770 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3771 return Res; 3772} 3773 3774SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) { 3775 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3776 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), 3777 WidenVT, N->getOperand(0)); 3778} 3779 3780// Return true is this is a SETCC node or a strict version of it. 3781static inline bool isSETCCOp(unsigned Opcode) { 3782 switch (Opcode) { 3783 case ISD::SETCC: 3784 case ISD::STRICT_FSETCC: 3785 case ISD::STRICT_FSETCCS: 3786 return true; 3787 } 3788 return false; 3789} 3790 3791// Return true if this is a node that could have two SETCCs as operands. 3792static inline bool isLogicalMaskOp(unsigned Opcode) { 3793 switch (Opcode) { 3794 case ISD::AND: 3795 case ISD::OR: 3796 case ISD::XOR: 3797 return true; 3798 } 3799 return false; 3800} 3801 3802// If N is a SETCC or a strict variant of it, return the type 3803// of the compare operands. 3804static inline EVT getSETCCOperandType(SDValue N) { 3805 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 3806 return N->getOperand(OpNo).getValueType(); 3807} 3808 3809// This is used just for the assert in convertMask(). Check that this either 3810// a SETCC or a previously handled SETCC by convertMask(). 3811#ifndef NDEBUG 3812static inline bool isSETCCorConvertedSETCC(SDValue N) { 3813 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) 3814 N = N.getOperand(0); 3815 else if (N.getOpcode() == ISD::CONCAT_VECTORS) { 3816 for (unsigned i = 1; i < N->getNumOperands(); ++i) 3817 if (!N->getOperand(i)->isUndef()) 3818 return false; 3819 N = N.getOperand(0); 3820 } 3821 3822 if (N.getOpcode() == ISD::TRUNCATE) 3823 N = N.getOperand(0); 3824 else if (N.getOpcode() == ISD::SIGN_EXTEND) 3825 N = N.getOperand(0); 3826 3827 if (isLogicalMaskOp(N.getOpcode())) 3828 return isSETCCorConvertedSETCC(N.getOperand(0)) && 3829 isSETCCorConvertedSETCC(N.getOperand(1)); 3830 3831 return (isSETCCOp(N.getOpcode()) || 3832 ISD::isBuildVectorOfConstantSDNodes(N.getNode())); 3833} 3834#endif 3835 3836// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT 3837// to ToMaskVT if needed with vector extension or truncation. 3838SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT, 3839 EVT ToMaskVT) { 3840 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled. 3841 // FIXME: This code seems to be too restrictive, we might consider 3842 // generalizing it or dropping it. 3843 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument."); 3844 3845 // Make a new Mask node, with a legal result VT. 3846 SDValue Mask; 3847 SmallVector<SDValue, 4> Ops; 3848 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i) 3849 Ops.push_back(InMask->getOperand(i)); 3850 if (InMask->isStrictFPOpcode()) { 3851 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), 3852 { MaskVT, MVT::Other }, Ops); 3853 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1)); 3854 } 3855 else 3856 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops); 3857 3858 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign 3859 // extend or truncate is needed. 3860 LLVMContext &Ctx = *DAG.getContext(); 3861 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits(); 3862 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits(); 3863 if (MaskScalarBits < ToMaskScalBits) { 3864 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 3865 MaskVT.getVectorNumElements()); 3866 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask); 3867 } else if (MaskScalarBits > ToMaskScalBits) { 3868 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 3869 MaskVT.getVectorNumElements()); 3870 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask); 3871 } 3872 3873 assert(Mask->getValueType(0).getScalarSizeInBits() == 3874 ToMaskVT.getScalarSizeInBits() && 3875 "Mask should have the right element size by now."); 3876 3877 // Adjust Mask to the right number of elements. 3878 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements(); 3879 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) { 3880 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); 3881 SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy); 3882 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask, 3883 ZeroIdx); 3884 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) { 3885 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls); 3886 EVT SubVT = Mask->getValueType(0); 3887 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT)); 3888 SubOps[0] = Mask; 3889 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps); 3890 } 3891 3892 assert((Mask->getValueType(0) == ToMaskVT) && 3893 "A mask of ToMaskVT should have been produced by now."); 3894 3895 return Mask; 3896} 3897 3898// This method tries to handle VSELECT and its mask by legalizing operands 3899// (which may require widening) and if needed adjusting the mask vector type 3900// to match that of the VSELECT. Without it, many cases end up with 3901// scalarization of the SETCC, with many unnecessary instructions. 3902SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) { 3903 LLVMContext &Ctx = *DAG.getContext(); 3904 SDValue Cond = N->getOperand(0); 3905 3906 if (N->getOpcode() != ISD::VSELECT) 3907 return SDValue(); 3908 3909 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode())) 3910 return SDValue(); 3911 3912 // If this is a splitted VSELECT that was previously already handled, do 3913 // nothing. 3914 EVT CondVT = Cond->getValueType(0); 3915 if (CondVT.getScalarSizeInBits() != 1) 3916 return SDValue(); 3917 3918 EVT VSelVT = N->getValueType(0); 3919 // Only handle vector types which are a power of 2. 3920 if (!isPowerOf2_64(VSelVT.getSizeInBits())) 3921 return SDValue(); 3922 3923 // Don't touch if this will be scalarized. 3924 EVT FinalVT = VSelVT; 3925 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 3926 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx); 3927 3928 if (FinalVT.getVectorNumElements() == 1) 3929 return SDValue(); 3930 3931 // If there is support for an i1 vector mask, don't touch. 3932 if (isSETCCOp(Cond.getOpcode())) { 3933 EVT SetCCOpVT = getSETCCOperandType(Cond); 3934 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal) 3935 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT); 3936 EVT SetCCResVT = getSetCCResultType(SetCCOpVT); 3937 if (SetCCResVT.getScalarSizeInBits() == 1) 3938 return SDValue(); 3939 } else if (CondVT.getScalarType() == MVT::i1) { 3940 // If there is support for an i1 vector mask (or only scalar i1 conditions), 3941 // don't touch. 3942 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal) 3943 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT); 3944 3945 if (CondVT.getScalarType() == MVT::i1) 3946 return SDValue(); 3947 } 3948 3949 // Get the VT and operands for VSELECT, and widen if needed. 3950 SDValue VSelOp1 = N->getOperand(1); 3951 SDValue VSelOp2 = N->getOperand(2); 3952 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) { 3953 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT); 3954 VSelOp1 = GetWidenedVector(VSelOp1); 3955 VSelOp2 = GetWidenedVector(VSelOp2); 3956 } 3957 3958 // The mask of the VSELECT should have integer elements. 3959 EVT ToMaskVT = VSelVT; 3960 if (!ToMaskVT.getScalarType().isInteger()) 3961 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger(); 3962 3963 SDValue Mask; 3964 if (isSETCCOp(Cond->getOpcode())) { 3965 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond)); 3966 Mask = convertMask(Cond, MaskVT, ToMaskVT); 3967 } else if (isLogicalMaskOp(Cond->getOpcode()) && 3968 isSETCCOp(Cond->getOperand(0).getOpcode()) && 3969 isSETCCOp(Cond->getOperand(1).getOpcode())) { 3970 // Cond is (AND/OR/XOR (SETCC, SETCC)) 3971 SDValue SETCC0 = Cond->getOperand(0); 3972 SDValue SETCC1 = Cond->getOperand(1); 3973 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0)); 3974 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1)); 3975 unsigned ScalarBits0 = VT0.getScalarSizeInBits(); 3976 unsigned ScalarBits1 = VT1.getScalarSizeInBits(); 3977 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits(); 3978 EVT MaskVT; 3979 // If the two SETCCs have different VTs, either extend/truncate one of 3980 // them to the other "towards" ToMaskVT, or truncate one and extend the 3981 // other to ToMaskVT. 3982 if (ScalarBits0 != ScalarBits1) { 3983 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1); 3984 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0); 3985 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits()) 3986 MaskVT = WideVT; 3987 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits()) 3988 MaskVT = NarrowVT; 3989 else 3990 MaskVT = ToMaskVT; 3991 } else 3992 // If the two SETCCs have the same VT, don't change it. 3993 MaskVT = VT0; 3994 3995 // Make new SETCCs and logical nodes. 3996 SETCC0 = convertMask(SETCC0, VT0, MaskVT); 3997 SETCC1 = convertMask(SETCC1, VT1, MaskVT); 3998 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1); 3999 4000 // Convert the logical op for VSELECT if needed. 4001 Mask = convertMask(Cond, MaskVT, ToMaskVT); 4002 } else 4003 return SDValue(); 4004 4005 return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2); 4006} 4007 4008SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 4009 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4010 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4011 4012 SDValue Cond1 = N->getOperand(0); 4013 EVT CondVT = Cond1.getValueType(); 4014 if (CondVT.isVector()) { 4015 if (SDValue Res = WidenVSELECTAndMask(N)) 4016 return Res; 4017 4018 EVT CondEltVT = CondVT.getVectorElementType(); 4019 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 4020 CondEltVT, WidenNumElts); 4021 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 4022 Cond1 = GetWidenedVector(Cond1); 4023 4024 // If we have to split the condition there is no point in widening the 4025 // select. This would result in an cycle of widening the select -> 4026 // widening the condition operand -> splitting the condition operand -> 4027 // splitting the select -> widening the select. Instead split this select 4028 // further and widen the resulting type. 4029 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 4030 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 4031 SDValue Res = ModifyToType(SplitSelect, WidenVT); 4032 return Res; 4033 } 4034 4035 if (Cond1.getValueType() != CondWidenVT) 4036 Cond1 = ModifyToType(Cond1, CondWidenVT); 4037 } 4038 4039 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 4040 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 4041 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 4042 return DAG.getNode(N->getOpcode(), SDLoc(N), 4043 WidenVT, Cond1, InOp1, InOp2); 4044} 4045 4046SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 4047 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 4048 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 4049 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 4050 InOp1.getValueType(), N->getOperand(0), 4051 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 4052} 4053 4054SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 4055 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4056 return DAG.getUNDEF(WidenVT); 4057} 4058 4059SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 4060 EVT VT = N->getValueType(0); 4061 SDLoc dl(N); 4062 4063 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4064 unsigned NumElts = VT.getVectorNumElements(); 4065 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4066 4067 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4068 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4069 4070 // Adjust mask based on new input vector length. 4071 SmallVector<int, 16> NewMask; 4072 for (unsigned i = 0; i != NumElts; ++i) { 4073 int Idx = N->getMaskElt(i); 4074 if (Idx < (int)NumElts) 4075 NewMask.push_back(Idx); 4076 else 4077 NewMask.push_back(Idx - NumElts + WidenNumElts); 4078 } 4079 for (unsigned i = NumElts; i != WidenNumElts; ++i) 4080 NewMask.push_back(-1); 4081 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask); 4082} 4083 4084SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 4085 assert(N->getValueType(0).isVector() && 4086 N->getOperand(0).getValueType().isVector() && 4087 "Operands must be vectors"); 4088 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4089 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4090 4091 SDValue InOp1 = N->getOperand(0); 4092 EVT InVT = InOp1.getValueType(); 4093 assert(InVT.isVector() && "can not widen non-vector type"); 4094 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 4095 InVT.getVectorElementType(), WidenNumElts); 4096 4097 // The input and output types often differ here, and it could be that while 4098 // we'd prefer to widen the result type, the input operands have been split. 4099 // In this case, we also need to split the result of this node as well. 4100 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 4101 SDValue SplitVSetCC = SplitVecOp_VSETCC(N); 4102 SDValue Res = ModifyToType(SplitVSetCC, WidenVT); 4103 return Res; 4104 } 4105 4106 // If the inputs also widen, handle them directly. Otherwise widen by hand. 4107 SDValue InOp2 = N->getOperand(1); 4108 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 4109 InOp1 = GetWidenedVector(InOp1); 4110 InOp2 = GetWidenedVector(InOp2); 4111 } else { 4112 InOp1 = DAG.WidenVector(InOp1, SDLoc(N)); 4113 InOp2 = DAG.WidenVector(InOp2, SDLoc(N)); 4114 } 4115 4116 // Assume that the input and output will be widen appropriately. If not, 4117 // we will have to unroll it at some point. 4118 assert(InOp1.getValueType() == WidenInVT && 4119 InOp2.getValueType() == WidenInVT && 4120 "Input not widened to expected type!"); 4121 (void)WidenInVT; 4122 return DAG.getNode(ISD::SETCC, SDLoc(N), 4123 WidenVT, InOp1, InOp2, N->getOperand(2)); 4124} 4125 4126SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) { 4127 assert(N->getValueType(0).isVector() && 4128 N->getOperand(1).getValueType().isVector() && 4129 "Operands must be vectors"); 4130 EVT VT = N->getValueType(0); 4131 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4132 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4133 unsigned NumElts = VT.getVectorNumElements(); 4134 EVT EltVT = VT.getVectorElementType(); 4135 4136 SDLoc dl(N); 4137 SDValue Chain = N->getOperand(0); 4138 SDValue LHS = N->getOperand(1); 4139 SDValue RHS = N->getOperand(2); 4140 SDValue CC = N->getOperand(3); 4141 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 4142 4143 // Fully unroll and reassemble. 4144 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT)); 4145 SmallVector<SDValue, 8> Chains(NumElts); 4146 for (unsigned i = 0; i != NumElts; ++i) { 4147 SDValue LHSElem = DAG.getNode( 4148 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 4149 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4150 SDValue RHSElem = DAG.getNode( 4151 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 4152 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4153 4154 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 4155 {Chain, LHSElem, RHSElem, CC}); 4156 Chains[i] = Scalars[i].getValue(1); 4157 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 4158 DAG.getBoolConstant(true, dl, EltVT, VT), 4159 DAG.getBoolConstant(false, dl, EltVT, VT)); 4160 } 4161 4162 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 4163 ReplaceValueWith(SDValue(N, 1), NewChain); 4164 4165 return DAG.getBuildVector(WidenVT, dl, Scalars); 4166} 4167 4168//===----------------------------------------------------------------------===// 4169// Widen Vector Operand 4170//===----------------------------------------------------------------------===// 4171bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 4172 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG); 4173 dbgs() << "\n"); 4174 SDValue Res = SDValue(); 4175 4176 // See if the target wants to custom widen this node. 4177 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 4178 return false; 4179 4180 switch (N->getOpcode()) { 4181 default: 4182#ifndef NDEBUG 4183 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 4184 N->dump(&DAG); 4185 dbgs() << "\n"; 4186#endif 4187 llvm_unreachable("Do not know how to widen this operator's operand!"); 4188 4189 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 4190 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 4191 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 4192 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 4193 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 4194 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; 4195 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; 4196 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; 4197 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 4198 case ISD::STRICT_FSETCC: 4199 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break; 4200 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break; 4201 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break; 4202 4203 case ISD::ANY_EXTEND: 4204 case ISD::SIGN_EXTEND: 4205 case ISD::ZERO_EXTEND: 4206 Res = WidenVecOp_EXTEND(N); 4207 break; 4208 4209 case ISD::FP_EXTEND: 4210 case ISD::STRICT_FP_EXTEND: 4211 case ISD::FP_ROUND: 4212 case ISD::STRICT_FP_ROUND: 4213 case ISD::FP_TO_SINT: 4214 case ISD::STRICT_FP_TO_SINT: 4215 case ISD::FP_TO_UINT: 4216 case ISD::STRICT_FP_TO_UINT: 4217 case ISD::SINT_TO_FP: 4218 case ISD::STRICT_SINT_TO_FP: 4219 case ISD::UINT_TO_FP: 4220 case ISD::STRICT_UINT_TO_FP: 4221 case ISD::TRUNCATE: 4222 Res = WidenVecOp_Convert(N); 4223 break; 4224 4225 case ISD::VECREDUCE_FADD: 4226 case ISD::VECREDUCE_FMUL: 4227 case ISD::VECREDUCE_ADD: 4228 case ISD::VECREDUCE_MUL: 4229 case ISD::VECREDUCE_AND: 4230 case ISD::VECREDUCE_OR: 4231 case ISD::VECREDUCE_XOR: 4232 case ISD::VECREDUCE_SMAX: 4233 case ISD::VECREDUCE_SMIN: 4234 case ISD::VECREDUCE_UMAX: 4235 case ISD::VECREDUCE_UMIN: 4236 case ISD::VECREDUCE_FMAX: 4237 case ISD::VECREDUCE_FMIN: 4238 Res = WidenVecOp_VECREDUCE(N); 4239 break; 4240 } 4241 4242 // If Res is null, the sub-method took care of registering the result. 4243 if (!Res.getNode()) return false; 4244 4245 // If the result is N, the sub-method updated N in place. Tell the legalizer 4246 // core about this. 4247 if (Res.getNode() == N) 4248 return true; 4249 4250 4251 if (N->isStrictFPOpcode()) 4252 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 4253 "Invalid operand expansion"); 4254 else 4255 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 4256 "Invalid operand expansion"); 4257 4258 ReplaceValueWith(SDValue(N, 0), Res); 4259 return false; 4260} 4261 4262SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 4263 SDLoc DL(N); 4264 EVT VT = N->getValueType(0); 4265 4266 SDValue InOp = N->getOperand(0); 4267 assert(getTypeAction(InOp.getValueType()) == 4268 TargetLowering::TypeWidenVector && 4269 "Unexpected type action"); 4270 InOp = GetWidenedVector(InOp); 4271 assert(VT.getVectorNumElements() < 4272 InOp.getValueType().getVectorNumElements() && 4273 "Input wasn't widened!"); 4274 4275 // We may need to further widen the operand until it has the same total 4276 // vector size as the result. 4277 EVT InVT = InOp.getValueType(); 4278 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 4279 EVT InEltVT = InVT.getVectorElementType(); 4280 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) { 4281 EVT FixedVT = (MVT::SimpleValueType)i; 4282 EVT FixedEltVT = FixedVT.getVectorElementType(); 4283 if (TLI.isTypeLegal(FixedVT) && 4284 FixedVT.getSizeInBits() == VT.getSizeInBits() && 4285 FixedEltVT == InEltVT) { 4286 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 4287 "Not enough elements in the fixed type for the operand!"); 4288 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 4289 "We can't have the same type as we started with!"); 4290 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 4291 InOp = DAG.getNode( 4292 ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp, 4293 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4294 else 4295 InOp = DAG.getNode( 4296 ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 4297 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4298 break; 4299 } 4300 } 4301 InVT = InOp.getValueType(); 4302 if (InVT.getSizeInBits() != VT.getSizeInBits()) 4303 // We couldn't find a legal vector type that was a widening of the input 4304 // and could be extended in-register to the result type, so we have to 4305 // scalarize. 4306 return WidenVecOp_Convert(N); 4307 } 4308 4309 // Use special DAG nodes to represent the operation of extending the 4310 // low lanes. 4311 switch (N->getOpcode()) { 4312 default: 4313 llvm_unreachable("Extend legalization on extend operation!"); 4314 case ISD::ANY_EXTEND: 4315 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp); 4316 case ISD::SIGN_EXTEND: 4317 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp); 4318 case ISD::ZERO_EXTEND: 4319 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp); 4320 } 4321} 4322 4323SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) { 4324 // The result (and first input) is legal, but the second input is illegal. 4325 // We can't do much to fix that, so just unroll and let the extracts off of 4326 // the second input be widened as needed later. 4327 return DAG.UnrollVectorOp(N); 4328} 4329 4330SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 4331 // Since the result is legal and the input is illegal. 4332 EVT VT = N->getValueType(0); 4333 EVT EltVT = VT.getVectorElementType(); 4334 SDLoc dl(N); 4335 unsigned NumElts = VT.getVectorNumElements(); 4336 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 4337 assert(getTypeAction(InOp.getValueType()) == 4338 TargetLowering::TypeWidenVector && 4339 "Unexpected type action"); 4340 InOp = GetWidenedVector(InOp); 4341 EVT InVT = InOp.getValueType(); 4342 unsigned Opcode = N->getOpcode(); 4343 4344 // See if a widened result type would be legal, if so widen the node. 4345 // FIXME: This isn't safe for StrictFP. Other optimization here is needed. 4346 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 4347 InVT.getVectorNumElements()); 4348 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) { 4349 SDValue Res; 4350 if (N->isStrictFPOpcode()) { 4351 if (Opcode == ISD::STRICT_FP_ROUND) 4352 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 4353 { N->getOperand(0), InOp, N->getOperand(2) }); 4354 else 4355 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 4356 { N->getOperand(0), InOp }); 4357 // Legalize the chain result - switch anything that used the old chain to 4358 // use the new one. 4359 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4360 } else { 4361 if (Opcode == ISD::FP_ROUND) 4362 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1)); 4363 else 4364 Res = DAG.getNode(Opcode, dl, WideVT, InOp); 4365 } 4366 return DAG.getNode( 4367 ISD::EXTRACT_SUBVECTOR, dl, VT, Res, 4368 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4369 } 4370 4371 EVT InEltVT = InVT.getVectorElementType(); 4372 4373 // Unroll the convert into some scalar code and create a nasty build vector. 4374 SmallVector<SDValue, 16> Ops(NumElts); 4375 if (N->isStrictFPOpcode()) { 4376 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 4377 SmallVector<SDValue, 32> OpChains; 4378 for (unsigned i=0; i < NumElts; ++i) { 4379 NewOps[1] = DAG.getNode( 4380 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 4381 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4382 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps); 4383 OpChains.push_back(Ops[i].getValue(1)); 4384 } 4385 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); 4386 ReplaceValueWith(SDValue(N, 1), NewChain); 4387 } else { 4388 for (unsigned i = 0; i < NumElts; ++i) 4389 Ops[i] = DAG.getNode( 4390 Opcode, dl, EltVT, 4391 DAG.getNode( 4392 ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 4393 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); 4394 } 4395 4396 return DAG.getBuildVector(VT, dl, Ops); 4397} 4398 4399SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 4400 EVT VT = N->getValueType(0); 4401 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4402 EVT InWidenVT = InOp.getValueType(); 4403 SDLoc dl(N); 4404 4405 // Check if we can convert between two legal vector types and extract. 4406 unsigned InWidenSize = InWidenVT.getSizeInBits(); 4407 unsigned Size = VT.getSizeInBits(); 4408 // x86mmx is not an acceptable vector element type, so don't try. 4409 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 4410 unsigned NewNumElts = InWidenSize / Size; 4411 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 4412 if (TLI.isTypeLegal(NewVT)) { 4413 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 4414 return DAG.getNode( 4415 ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 4416 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4417 } 4418 } 4419 4420 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened 4421 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not, 4422 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids 4423 // having to copy via memory. 4424 if (VT.isVector()) { 4425 EVT EltVT = VT.getVectorElementType(); 4426 unsigned EltSize = EltVT.getSizeInBits(); 4427 if (InWidenSize % EltSize == 0) { 4428 unsigned NewNumElts = InWidenSize / EltSize; 4429 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts); 4430 if (TLI.isTypeLegal(NewVT)) { 4431 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 4432 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp, 4433 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4434 } 4435 } 4436 } 4437 4438 return CreateStackStoreLoad(InOp, VT); 4439} 4440 4441SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 4442 EVT VT = N->getValueType(0); 4443 EVT EltVT = VT.getVectorElementType(); 4444 EVT InVT = N->getOperand(0).getValueType(); 4445 SDLoc dl(N); 4446 4447 // If the widen width for this operand is the same as the width of the concat 4448 // and all but the first operand is undef, just use the widened operand. 4449 unsigned NumOperands = N->getNumOperands(); 4450 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 4451 unsigned i; 4452 for (i = 1; i < NumOperands; ++i) 4453 if (!N->getOperand(i).isUndef()) 4454 break; 4455 4456 if (i == NumOperands) 4457 return GetWidenedVector(N->getOperand(0)); 4458 } 4459 4460 // Otherwise, fall back to a nasty build vector. 4461 unsigned NumElts = VT.getVectorNumElements(); 4462 SmallVector<SDValue, 16> Ops(NumElts); 4463 4464 unsigned NumInElts = InVT.getVectorNumElements(); 4465 4466 unsigned Idx = 0; 4467 for (unsigned i=0; i < NumOperands; ++i) { 4468 SDValue InOp = N->getOperand(i); 4469 assert(getTypeAction(InOp.getValueType()) == 4470 TargetLowering::TypeWidenVector && 4471 "Unexpected type action"); 4472 InOp = GetWidenedVector(InOp); 4473 for (unsigned j=0; j < NumInElts; ++j) 4474 Ops[Idx++] = DAG.getNode( 4475 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 4476 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4477 } 4478 return DAG.getBuildVector(VT, dl, Ops); 4479} 4480 4481SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 4482 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4483 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 4484 N->getValueType(0), InOp, N->getOperand(1)); 4485} 4486 4487SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 4488 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4489 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 4490 N->getValueType(0), InOp, N->getOperand(1)); 4491} 4492 4493SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 4494 // We have to widen the value, but we want only to store the original 4495 // vector type. 4496 StoreSDNode *ST = cast<StoreSDNode>(N); 4497 4498 if (!ST->getMemoryVT().getScalarType().isByteSized()) 4499 return TLI.scalarizeVectorStore(ST, DAG); 4500 4501 SmallVector<SDValue, 16> StChain; 4502 if (ST->isTruncatingStore()) 4503 GenWidenVectorTruncStores(StChain, ST); 4504 else 4505 GenWidenVectorStores(StChain, ST); 4506 4507 if (StChain.size() == 1) 4508 return StChain[0]; 4509 else 4510 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 4511} 4512 4513SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { 4514 assert((OpNo == 1 || OpNo == 3) && 4515 "Can widen only data or mask operand of mstore"); 4516 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 4517 SDValue Mask = MST->getMask(); 4518 EVT MaskVT = Mask.getValueType(); 4519 SDValue StVal = MST->getValue(); 4520 SDLoc dl(N); 4521 4522 if (OpNo == 1) { 4523 // Widen the value. 4524 StVal = GetWidenedVector(StVal); 4525 4526 // The mask should be widened as well. 4527 EVT WideVT = StVal.getValueType(); 4528 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4529 MaskVT.getVectorElementType(), 4530 WideVT.getVectorNumElements()); 4531 Mask = ModifyToType(Mask, WideMaskVT, true); 4532 } else { 4533 // Widen the mask. 4534 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT); 4535 Mask = ModifyToType(Mask, WideMaskVT, true); 4536 4537 EVT ValueVT = StVal.getValueType(); 4538 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), 4539 ValueVT.getVectorElementType(), 4540 WideMaskVT.getVectorNumElements()); 4541 StVal = ModifyToType(StVal, WideVT); 4542 } 4543 4544 assert(Mask.getValueType().getVectorNumElements() == 4545 StVal.getValueType().getVectorNumElements() && 4546 "Mask and data vectors should have the same number of elements"); 4547 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(), 4548 MST->getOffset(), Mask, MST->getMemoryVT(), 4549 MST->getMemOperand(), MST->getAddressingMode(), 4550 false, MST->isCompressingStore()); 4551} 4552 4553SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) { 4554 assert(OpNo == 4 && "Can widen only the index of mgather"); 4555 auto *MG = cast<MaskedGatherSDNode>(N); 4556 SDValue DataOp = MG->getPassThru(); 4557 SDValue Mask = MG->getMask(); 4558 SDValue Scale = MG->getScale(); 4559 4560 // Just widen the index. It's allowed to have extra elements. 4561 SDValue Index = GetWidenedVector(MG->getIndex()); 4562 4563 SDLoc dl(N); 4564 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index, 4565 Scale}; 4566 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops, 4567 MG->getMemOperand(), MG->getIndexType()); 4568 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4569 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 4570 return SDValue(); 4571} 4572 4573SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { 4574 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N); 4575 SDValue DataOp = MSC->getValue(); 4576 SDValue Mask = MSC->getMask(); 4577 SDValue Index = MSC->getIndex(); 4578 SDValue Scale = MSC->getScale(); 4579 4580 if (OpNo == 1) { 4581 DataOp = GetWidenedVector(DataOp); 4582 unsigned NumElts = DataOp.getValueType().getVectorNumElements(); 4583 4584 // Widen index. 4585 EVT IndexVT = Index.getValueType(); 4586 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 4587 IndexVT.getVectorElementType(), NumElts); 4588 Index = ModifyToType(Index, WideIndexVT); 4589 4590 // The mask should be widened as well. 4591 EVT MaskVT = Mask.getValueType(); 4592 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4593 MaskVT.getVectorElementType(), NumElts); 4594 Mask = ModifyToType(Mask, WideMaskVT, true); 4595 } else if (OpNo == 4) { 4596 // Just widen the index. It's allowed to have extra elements. 4597 Index = GetWidenedVector(Index); 4598 } else 4599 llvm_unreachable("Can't widen this operand of mscatter"); 4600 4601 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index, 4602 Scale}; 4603 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), 4604 MSC->getMemoryVT(), SDLoc(N), Ops, 4605 MSC->getMemOperand(), MSC->getIndexType()); 4606} 4607 4608SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 4609 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 4610 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 4611 SDLoc dl(N); 4612 EVT VT = N->getValueType(0); 4613 4614 // WARNING: In this code we widen the compare instruction with garbage. 4615 // This garbage may contain denormal floats which may be slow. Is this a real 4616 // concern ? Should we zero the unused lanes if this is a float compare ? 4617 4618 // Get a new SETCC node to compare the newly widened operands. 4619 // Only some of the compared elements are legal. 4620 EVT SVT = getSetCCResultType(InOp0.getValueType()); 4621 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC. 4622 if (VT.getScalarType() == MVT::i1) 4623 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 4624 SVT.getVectorNumElements()); 4625 4626 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 4627 SVT, InOp0, InOp1, N->getOperand(2)); 4628 4629 // Extract the needed results from the result vector. 4630 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 4631 SVT.getVectorElementType(), 4632 VT.getVectorNumElements()); 4633 SDValue CC = DAG.getNode( 4634 ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC, 4635 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4636 4637 EVT OpVT = N->getOperand(0).getValueType(); 4638 ISD::NodeType ExtendCode = 4639 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 4640 return DAG.getNode(ExtendCode, dl, VT, CC); 4641} 4642 4643SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) { 4644 SDValue Chain = N->getOperand(0); 4645 SDValue LHS = GetWidenedVector(N->getOperand(1)); 4646 SDValue RHS = GetWidenedVector(N->getOperand(2)); 4647 SDValue CC = N->getOperand(3); 4648 SDLoc dl(N); 4649 4650 EVT VT = N->getValueType(0); 4651 EVT EltVT = VT.getVectorElementType(); 4652 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 4653 unsigned NumElts = VT.getVectorNumElements(); 4654 4655 // Unroll into a build vector. 4656 SmallVector<SDValue, 8> Scalars(NumElts); 4657 SmallVector<SDValue, 8> Chains(NumElts); 4658 4659 for (unsigned i = 0; i != NumElts; ++i) { 4660 SDValue LHSElem = DAG.getNode( 4661 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 4662 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4663 SDValue RHSElem = DAG.getNode( 4664 ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 4665 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4666 4667 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 4668 {Chain, LHSElem, RHSElem, CC}); 4669 Chains[i] = Scalars[i].getValue(1); 4670 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 4671 DAG.getBoolConstant(true, dl, EltVT, VT), 4672 DAG.getBoolConstant(false, dl, EltVT, VT)); 4673 } 4674 4675 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 4676 ReplaceValueWith(SDValue(N, 1), NewChain); 4677 4678 return DAG.getBuildVector(VT, dl, Scalars); 4679} 4680 4681SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { 4682 SDLoc dl(N); 4683 SDValue Op = GetWidenedVector(N->getOperand(0)); 4684 EVT OrigVT = N->getOperand(0).getValueType(); 4685 EVT WideVT = Op.getValueType(); 4686 EVT ElemVT = OrigVT.getVectorElementType(); 4687 4688 SDValue NeutralElem; 4689 switch (N->getOpcode()) { 4690 case ISD::VECREDUCE_ADD: 4691 case ISD::VECREDUCE_OR: 4692 case ISD::VECREDUCE_XOR: 4693 case ISD::VECREDUCE_UMAX: 4694 NeutralElem = DAG.getConstant(0, dl, ElemVT); 4695 break; 4696 case ISD::VECREDUCE_MUL: 4697 NeutralElem = DAG.getConstant(1, dl, ElemVT); 4698 break; 4699 case ISD::VECREDUCE_AND: 4700 case ISD::VECREDUCE_UMIN: 4701 NeutralElem = DAG.getAllOnesConstant(dl, ElemVT); 4702 break; 4703 case ISD::VECREDUCE_SMAX: 4704 NeutralElem = DAG.getConstant( 4705 APInt::getSignedMinValue(ElemVT.getSizeInBits()), dl, ElemVT); 4706 break; 4707 case ISD::VECREDUCE_SMIN: 4708 NeutralElem = DAG.getConstant( 4709 APInt::getSignedMaxValue(ElemVT.getSizeInBits()), dl, ElemVT); 4710 break; 4711 case ISD::VECREDUCE_FADD: 4712 NeutralElem = DAG.getConstantFP(0.0, dl, ElemVT); 4713 break; 4714 case ISD::VECREDUCE_FMUL: 4715 NeutralElem = DAG.getConstantFP(1.0, dl, ElemVT); 4716 break; 4717 case ISD::VECREDUCE_FMAX: 4718 NeutralElem = DAG.getConstantFP( 4719 -std::numeric_limits<double>::infinity(), dl, ElemVT); 4720 break; 4721 case ISD::VECREDUCE_FMIN: 4722 NeutralElem = DAG.getConstantFP( 4723 std::numeric_limits<double>::infinity(), dl, ElemVT); 4724 break; 4725 } 4726 4727 // Pad the vector with the neutral element. 4728 unsigned OrigElts = OrigVT.getVectorNumElements(); 4729 unsigned WideElts = WideVT.getVectorNumElements(); 4730 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) 4731 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, 4732 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4733 4734 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags()); 4735} 4736 4737SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { 4738 // This only gets called in the case that the left and right inputs and 4739 // result are of a legal odd vector type, and the condition is illegal i1 of 4740 // the same odd width that needs widening. 4741 EVT VT = N->getValueType(0); 4742 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT)); 4743 4744 SDValue Cond = GetWidenedVector(N->getOperand(0)); 4745 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N)); 4746 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N)); 4747 SDLoc DL(N); 4748 4749 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond, 4750 LeftIn, RightIn); 4751 return DAG.getNode( 4752 ISD::EXTRACT_SUBVECTOR, DL, VT, Select, 4753 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4754} 4755 4756//===----------------------------------------------------------------------===// 4757// Vector Widening Utilities 4758//===----------------------------------------------------------------------===// 4759 4760// Utility function to find the type to chop up a widen vector for load/store 4761// TLI: Target lowering used to determine legal types. 4762// Width: Width left need to load/store. 4763// WidenVT: The widen vector type to load to/store from 4764// Align: If 0, don't allow use of a wider type 4765// WidenEx: If Align is not 0, the amount additional we can load/store from. 4766 4767static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 4768 unsigned Width, EVT WidenVT, 4769 unsigned Align = 0, unsigned WidenEx = 0) { 4770 EVT WidenEltVT = WidenVT.getVectorElementType(); 4771 const bool Scalable = WidenVT.isScalableVector(); 4772 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize(); 4773 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 4774 unsigned AlignInBits = Align*8; 4775 4776 // If we have one element to load/store, return it. 4777 EVT RetVT = WidenEltVT; 4778 if (Width == WidenEltWidth) 4779 return RetVT; 4780 4781 // See if there is larger legal integer than the element type to load/store. 4782 unsigned VT; 4783 // Don't bother looking for an integer type if the vector is scalable, skip 4784 // to vector types. 4785 if (!Scalable) { 4786 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 4787 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 4788 EVT MemVT((MVT::SimpleValueType) VT); 4789 unsigned MemVTWidth = MemVT.getSizeInBits(); 4790 if (MemVT.getSizeInBits() <= WidenEltWidth) 4791 break; 4792 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 4793 if ((Action == TargetLowering::TypeLegal || 4794 Action == TargetLowering::TypePromoteInteger) && 4795 (WidenWidth % MemVTWidth) == 0 && 4796 isPowerOf2_32(WidenWidth / MemVTWidth) && 4797 (MemVTWidth <= Width || 4798 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 4799 if (MemVTWidth == WidenWidth) 4800 return MemVT; 4801 RetVT = MemVT; 4802 break; 4803 } 4804 } 4805 } 4806 4807 // See if there is a larger vector type to load/store that has the same vector 4808 // element type and is evenly divisible with the WidenVT. 4809 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 4810 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 4811 EVT MemVT = (MVT::SimpleValueType) VT; 4812 // Skip vector MVTs which don't match the scalable property of WidenVT. 4813 if (Scalable != MemVT.isScalableVector()) 4814 continue; 4815 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize(); 4816 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 4817 if ((Action == TargetLowering::TypeLegal || 4818 Action == TargetLowering::TypePromoteInteger) && 4819 WidenEltVT == MemVT.getVectorElementType() && 4820 (WidenWidth % MemVTWidth) == 0 && 4821 isPowerOf2_32(WidenWidth / MemVTWidth) && 4822 (MemVTWidth <= Width || 4823 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 4824 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT) 4825 return MemVT; 4826 } 4827 } 4828 4829 return RetVT; 4830} 4831 4832// Builds a vector type from scalar loads 4833// VecTy: Resulting Vector type 4834// LDOps: Load operators to build a vector type 4835// [Start,End) the list of loads to use. 4836static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 4837 SmallVectorImpl<SDValue> &LdOps, 4838 unsigned Start, unsigned End) { 4839 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4840 SDLoc dl(LdOps[Start]); 4841 EVT LdTy = LdOps[Start].getValueType(); 4842 unsigned Width = VecTy.getSizeInBits(); 4843 unsigned NumElts = Width / LdTy.getSizeInBits(); 4844 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 4845 4846 unsigned Idx = 1; 4847 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 4848 4849 for (unsigned i = Start + 1; i != End; ++i) { 4850 EVT NewLdTy = LdOps[i].getValueType(); 4851 if (NewLdTy != LdTy) { 4852 NumElts = Width / NewLdTy.getSizeInBits(); 4853 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 4854 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 4855 // Readjust position and vector position based on new load type. 4856 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 4857 LdTy = NewLdTy; 4858 } 4859 VecOp = DAG.getNode( 4860 ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 4861 DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4862 } 4863 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 4864} 4865 4866SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 4867 LoadSDNode *LD) { 4868 // The strategy assumes that we can efficiently load power-of-two widths. 4869 // The routine chops the vector into the largest vector loads with the same 4870 // element type or scalar loads and then recombines it to the widen vector 4871 // type. 4872 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 4873 unsigned WidenWidth = WidenVT.getSizeInBits(); 4874 EVT LdVT = LD->getMemoryVT(); 4875 SDLoc dl(LD); 4876 assert(LdVT.isVector() && WidenVT.isVector()); 4877 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 4878 4879 // Load information 4880 SDValue Chain = LD->getChain(); 4881 SDValue BasePtr = LD->getBasePtr(); 4882 unsigned Align = LD->getAlignment(); 4883 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 4884 AAMDNodes AAInfo = LD->getAAInfo(); 4885 4886 int LdWidth = LdVT.getSizeInBits(); 4887 int WidthDiff = WidenWidth - LdWidth; 4888 unsigned LdAlign = (!LD->isSimple()) ? 0 : Align; // Allow wider loads. 4889 4890 // Find the vector type that can load from. 4891 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 4892 int NewVTWidth = NewVT.getSizeInBits(); 4893 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 4894 Align, MMOFlags, AAInfo); 4895 LdChain.push_back(LdOp.getValue(1)); 4896 4897 // Check if we can load the element with one instruction. 4898 if (LdWidth <= NewVTWidth) { 4899 if (!NewVT.isVector()) { 4900 unsigned NumElts = WidenWidth / NewVTWidth; 4901 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 4902 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 4903 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 4904 } 4905 if (NewVT == WidenVT) 4906 return LdOp; 4907 4908 assert(WidenWidth % NewVTWidth == 0); 4909 unsigned NumConcat = WidenWidth / NewVTWidth; 4910 SmallVector<SDValue, 16> ConcatOps(NumConcat); 4911 SDValue UndefVal = DAG.getUNDEF(NewVT); 4912 ConcatOps[0] = LdOp; 4913 for (unsigned i = 1; i != NumConcat; ++i) 4914 ConcatOps[i] = UndefVal; 4915 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 4916 } 4917 4918 // Load vector by using multiple loads from largest vector to scalar. 4919 SmallVector<SDValue, 16> LdOps; 4920 LdOps.push_back(LdOp); 4921 4922 LdWidth -= NewVTWidth; 4923 unsigned Offset = 0; 4924 4925 while (LdWidth > 0) { 4926 unsigned Increment = NewVTWidth / 8; 4927 Offset += Increment; 4928 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment); 4929 4930 SDValue L; 4931 if (LdWidth < NewVTWidth) { 4932 // The current type we are using is too large. Find a better size. 4933 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff); 4934 NewVTWidth = NewVT.getSizeInBits(); 4935 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 4936 LD->getPointerInfo().getWithOffset(Offset), 4937 MinAlign(Align, Increment), MMOFlags, AAInfo); 4938 LdChain.push_back(L.getValue(1)); 4939 if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) { 4940 // Later code assumes the vector loads produced will be mergeable, so we 4941 // must pad the final entry up to the previous width. Scalars are 4942 // combined separately. 4943 SmallVector<SDValue, 16> Loads; 4944 Loads.push_back(L); 4945 unsigned size = L->getValueSizeInBits(0); 4946 while (size < LdOp->getValueSizeInBits(0)) { 4947 Loads.push_back(DAG.getUNDEF(L->getValueType(0))); 4948 size += L->getValueSizeInBits(0); 4949 } 4950 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads); 4951 } 4952 } else { 4953 L = DAG.getLoad(NewVT, dl, Chain, BasePtr, 4954 LD->getPointerInfo().getWithOffset(Offset), 4955 MinAlign(Align, Increment), MMOFlags, AAInfo); 4956 LdChain.push_back(L.getValue(1)); 4957 } 4958 4959 LdOps.push_back(L); 4960 LdOp = L; 4961 4962 LdWidth -= NewVTWidth; 4963 } 4964 4965 // Build the vector from the load operations. 4966 unsigned End = LdOps.size(); 4967 if (!LdOps[0].getValueType().isVector()) 4968 // All the loads are scalar loads. 4969 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 4970 4971 // If the load contains vectors, build the vector using concat vector. 4972 // All of the vectors used to load are power-of-2, and the scalar loads can be 4973 // combined to make a power-of-2 vector. 4974 SmallVector<SDValue, 16> ConcatOps(End); 4975 int i = End - 1; 4976 int Idx = End; 4977 EVT LdTy = LdOps[i].getValueType(); 4978 // First, combine the scalar loads to a vector. 4979 if (!LdTy.isVector()) { 4980 for (--i; i >= 0; --i) { 4981 LdTy = LdOps[i].getValueType(); 4982 if (LdTy.isVector()) 4983 break; 4984 } 4985 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End); 4986 } 4987 ConcatOps[--Idx] = LdOps[i]; 4988 for (--i; i >= 0; --i) { 4989 EVT NewLdTy = LdOps[i].getValueType(); 4990 if (NewLdTy != LdTy) { 4991 // Create a larger vector. 4992 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 4993 makeArrayRef(&ConcatOps[Idx], End - Idx)); 4994 Idx = End - 1; 4995 LdTy = NewLdTy; 4996 } 4997 ConcatOps[--Idx] = LdOps[i]; 4998 } 4999 5000 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx)) 5001 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 5002 makeArrayRef(&ConcatOps[Idx], End - Idx)); 5003 5004 // We need to fill the rest with undefs to build the vector. 5005 unsigned NumOps = WidenWidth / LdTy.getSizeInBits(); 5006 SmallVector<SDValue, 16> WidenOps(NumOps); 5007 SDValue UndefVal = DAG.getUNDEF(LdTy); 5008 { 5009 unsigned i = 0; 5010 for (; i != End-Idx; ++i) 5011 WidenOps[i] = ConcatOps[Idx+i]; 5012 for (; i != NumOps; ++i) 5013 WidenOps[i] = UndefVal; 5014 } 5015 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 5016} 5017 5018SDValue 5019DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 5020 LoadSDNode *LD, 5021 ISD::LoadExtType ExtType) { 5022 // For extension loads, it may not be more efficient to chop up the vector 5023 // and then extend it. Instead, we unroll the load and build a new vector. 5024 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 5025 EVT LdVT = LD->getMemoryVT(); 5026 SDLoc dl(LD); 5027 assert(LdVT.isVector() && WidenVT.isVector()); 5028 5029 // Load information 5030 SDValue Chain = LD->getChain(); 5031 SDValue BasePtr = LD->getBasePtr(); 5032 unsigned Align = LD->getAlignment(); 5033 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 5034 AAMDNodes AAInfo = LD->getAAInfo(); 5035 5036 EVT EltVT = WidenVT.getVectorElementType(); 5037 EVT LdEltVT = LdVT.getVectorElementType(); 5038 unsigned NumElts = LdVT.getVectorNumElements(); 5039 5040 // Load each element and widen. 5041 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5042 SmallVector<SDValue, 16> Ops(WidenNumElts); 5043 unsigned Increment = LdEltVT.getSizeInBits() / 8; 5044 Ops[0] = 5045 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(), 5046 LdEltVT, Align, MMOFlags, AAInfo); 5047 LdChain.push_back(Ops[0].getValue(1)); 5048 unsigned i = 0, Offset = Increment; 5049 for (i=1; i < NumElts; ++i, Offset += Increment) { 5050 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset); 5051 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 5052 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 5053 Align, MMOFlags, AAInfo); 5054 LdChain.push_back(Ops[i].getValue(1)); 5055 } 5056 5057 // Fill the rest with undefs. 5058 SDValue UndefVal = DAG.getUNDEF(EltVT); 5059 for (; i != WidenNumElts; ++i) 5060 Ops[i] = UndefVal; 5061 5062 return DAG.getBuildVector(WidenVT, dl, Ops); 5063} 5064 5065void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 5066 StoreSDNode *ST) { 5067 // The strategy assumes that we can efficiently store power-of-two widths. 5068 // The routine chops the vector into the largest vector stores with the same 5069 // element type or scalar stores. 5070 SDValue Chain = ST->getChain(); 5071 SDValue BasePtr = ST->getBasePtr(); 5072 unsigned Align = ST->getAlignment(); 5073 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 5074 AAMDNodes AAInfo = ST->getAAInfo(); 5075 SDValue ValOp = GetWidenedVector(ST->getValue()); 5076 SDLoc dl(ST); 5077 5078 EVT StVT = ST->getMemoryVT(); 5079 unsigned StWidth = StVT.getSizeInBits(); 5080 EVT ValVT = ValOp.getValueType(); 5081 unsigned ValWidth = ValVT.getSizeInBits(); 5082 EVT ValEltVT = ValVT.getVectorElementType(); 5083 unsigned ValEltWidth = ValEltVT.getSizeInBits(); 5084 assert(StVT.getVectorElementType() == ValEltVT); 5085 5086 int Idx = 0; // current index to store 5087 unsigned Offset = 0; // offset from base to store 5088 while (StWidth != 0) { 5089 // Find the largest vector type we can store with. 5090 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT); 5091 unsigned NewVTWidth = NewVT.getSizeInBits(); 5092 unsigned Increment = NewVTWidth / 8; 5093 if (NewVT.isVector()) { 5094 unsigned NumVTElts = NewVT.getVectorNumElements(); 5095 do { 5096 SDValue EOp = DAG.getNode( 5097 ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 5098 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 5099 StChain.push_back(DAG.getStore( 5100 Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset), 5101 MinAlign(Align, Offset), MMOFlags, AAInfo)); 5102 StWidth -= NewVTWidth; 5103 Offset += Increment; 5104 Idx += NumVTElts; 5105 5106 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment); 5107 } while (StWidth != 0 && StWidth >= NewVTWidth); 5108 } else { 5109 // Cast the vector to the scalar type we can store. 5110 unsigned NumElts = ValWidth / NewVTWidth; 5111 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 5112 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 5113 // Readjust index position based on new vector type. 5114 Idx = Idx * ValEltWidth / NewVTWidth; 5115 do { 5116 SDValue EOp = DAG.getNode( 5117 ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 5118 DAG.getConstant(Idx++, dl, 5119 TLI.getVectorIdxTy(DAG.getDataLayout()))); 5120 StChain.push_back(DAG.getStore( 5121 Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset), 5122 MinAlign(Align, Offset), MMOFlags, AAInfo)); 5123 StWidth -= NewVTWidth; 5124 Offset += Increment; 5125 BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment); 5126 } while (StWidth != 0 && StWidth >= NewVTWidth); 5127 // Restore index back to be relative to the original widen element type. 5128 Idx = Idx * NewVTWidth / ValEltWidth; 5129 } 5130 } 5131} 5132 5133void 5134DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain, 5135 StoreSDNode *ST) { 5136 // For extension loads, it may not be more efficient to truncate the vector 5137 // and then store it. Instead, we extract each element and then store it. 5138 SDValue Chain = ST->getChain(); 5139 SDValue BasePtr = ST->getBasePtr(); 5140 unsigned Align = ST->getAlignment(); 5141 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 5142 AAMDNodes AAInfo = ST->getAAInfo(); 5143 SDValue ValOp = GetWidenedVector(ST->getValue()); 5144 SDLoc dl(ST); 5145 5146 EVT StVT = ST->getMemoryVT(); 5147 EVT ValVT = ValOp.getValueType(); 5148 5149 // It must be true that the wide vector type is bigger than where we need to 5150 // store. 5151 assert(StVT.isVector() && ValOp.getValueType().isVector()); 5152 assert(StVT.bitsLT(ValOp.getValueType())); 5153 5154 // For truncating stores, we can not play the tricks of chopping legal vector 5155 // types and bitcast it to the right type. Instead, we unroll the store. 5156 EVT StEltVT = StVT.getVectorElementType(); 5157 EVT ValEltVT = ValVT.getVectorElementType(); 5158 unsigned Increment = ValEltVT.getSizeInBits() / 8; 5159 unsigned NumElts = StVT.getVectorNumElements(); 5160 SDValue EOp = DAG.getNode( 5161 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 5162 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 5163 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, 5164 ST->getPointerInfo(), StEltVT, Align, 5165 MMOFlags, AAInfo)); 5166 unsigned Offset = Increment; 5167 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) { 5168 SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset); 5169 SDValue EOp = DAG.getNode( 5170 ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp, 5171 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 5172 StChain.push_back(DAG.getTruncStore( 5173 Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset), 5174 StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo)); 5175 } 5176} 5177 5178/// Modifies a vector input (widen or narrows) to a vector of NVT. The 5179/// input vector must have the same element type as NVT. 5180/// FillWithZeroes specifies that the vector should be widened with zeroes. 5181SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT, 5182 bool FillWithZeroes) { 5183 // Note that InOp might have been widened so it might already have 5184 // the right width or it might need be narrowed. 5185 EVT InVT = InOp.getValueType(); 5186 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 5187 "input and widen element type must match"); 5188 SDLoc dl(InOp); 5189 5190 // Check if InOp already has the right width. 5191 if (InVT == NVT) 5192 return InOp; 5193 5194 unsigned InNumElts = InVT.getVectorNumElements(); 5195 unsigned WidenNumElts = NVT.getVectorNumElements(); 5196 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 5197 unsigned NumConcat = WidenNumElts / InNumElts; 5198 SmallVector<SDValue, 16> Ops(NumConcat); 5199 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) : 5200 DAG.getUNDEF(InVT); 5201 Ops[0] = InOp; 5202 for (unsigned i = 1; i != NumConcat; ++i) 5203 Ops[i] = FillVal; 5204 5205 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 5206 } 5207 5208 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 5209 return DAG.getNode( 5210 ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 5211 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 5212 5213 // Fall back to extract and build. 5214 SmallVector<SDValue, 16> Ops(WidenNumElts); 5215 EVT EltVT = NVT.getVectorElementType(); 5216 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 5217 unsigned Idx; 5218 for (Idx = 0; Idx < MinNumElts; ++Idx) 5219 Ops[Idx] = DAG.getNode( 5220 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 5221 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 5222 5223 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : 5224 DAG.getUNDEF(EltVT); 5225 for ( ; Idx < WidenNumElts; ++Idx) 5226 Ops[Idx] = FillVal; 5227 return DAG.getBuildVector(NVT, dl, Ops); 5228} 5229