PPCISelLowering.cpp revision 360784
1//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file implements the PPCISelLowering class. 10// 11//===----------------------------------------------------------------------===// 12 13#include "PPCISelLowering.h" 14#include "MCTargetDesc/PPCPredicates.h" 15#include "PPC.h" 16#include "PPCCCState.h" 17#include "PPCCallingConv.h" 18#include "PPCFrameLowering.h" 19#include "PPCInstrInfo.h" 20#include "PPCMachineFunctionInfo.h" 21#include "PPCPerfectShuffle.h" 22#include "PPCRegisterInfo.h" 23#include "PPCSubtarget.h" 24#include "PPCTargetMachine.h" 25#include "llvm/ADT/APFloat.h" 26#include "llvm/ADT/APInt.h" 27#include "llvm/ADT/ArrayRef.h" 28#include "llvm/ADT/DenseMap.h" 29#include "llvm/ADT/None.h" 30#include "llvm/ADT/STLExtras.h" 31#include "llvm/ADT/SmallPtrSet.h" 32#include "llvm/ADT/SmallSet.h" 33#include "llvm/ADT/SmallVector.h" 34#include "llvm/ADT/Statistic.h" 35#include "llvm/ADT/StringRef.h" 36#include "llvm/ADT/StringSwitch.h" 37#include "llvm/CodeGen/CallingConvLower.h" 38#include "llvm/CodeGen/ISDOpcodes.h" 39#include "llvm/CodeGen/MachineBasicBlock.h" 40#include "llvm/CodeGen/MachineFrameInfo.h" 41#include "llvm/CodeGen/MachineFunction.h" 42#include "llvm/CodeGen/MachineInstr.h" 43#include "llvm/CodeGen/MachineInstrBuilder.h" 44#include "llvm/CodeGen/MachineJumpTableInfo.h" 45#include "llvm/CodeGen/MachineLoopInfo.h" 46#include "llvm/CodeGen/MachineMemOperand.h" 47#include "llvm/CodeGen/MachineModuleInfo.h" 48#include "llvm/CodeGen/MachineOperand.h" 49#include "llvm/CodeGen/MachineRegisterInfo.h" 50#include "llvm/CodeGen/RuntimeLibcalls.h" 51#include "llvm/CodeGen/SelectionDAG.h" 52#include "llvm/CodeGen/SelectionDAGNodes.h" 53#include "llvm/CodeGen/TargetInstrInfo.h" 54#include "llvm/CodeGen/TargetLowering.h" 55#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 56#include "llvm/CodeGen/TargetRegisterInfo.h" 57#include "llvm/CodeGen/ValueTypes.h" 58#include "llvm/IR/CallSite.h" 59#include "llvm/IR/CallingConv.h" 60#include "llvm/IR/Constant.h" 61#include "llvm/IR/Constants.h" 62#include "llvm/IR/DataLayout.h" 63#include "llvm/IR/DebugLoc.h" 64#include "llvm/IR/DerivedTypes.h" 65#include "llvm/IR/Function.h" 66#include "llvm/IR/GlobalValue.h" 67#include "llvm/IR/IRBuilder.h" 68#include "llvm/IR/Instructions.h" 69#include "llvm/IR/Intrinsics.h" 70#include "llvm/IR/IntrinsicsPowerPC.h" 71#include "llvm/IR/Module.h" 72#include "llvm/IR/Type.h" 73#include "llvm/IR/Use.h" 74#include "llvm/IR/Value.h" 75#include "llvm/MC/MCContext.h" 76#include "llvm/MC/MCExpr.h" 77#include "llvm/MC/MCRegisterInfo.h" 78#include "llvm/MC/MCSymbolXCOFF.h" 79#include "llvm/Support/AtomicOrdering.h" 80#include "llvm/Support/BranchProbability.h" 81#include "llvm/Support/Casting.h" 82#include "llvm/Support/CodeGen.h" 83#include "llvm/Support/CommandLine.h" 84#include "llvm/Support/Compiler.h" 85#include "llvm/Support/Debug.h" 86#include "llvm/Support/ErrorHandling.h" 87#include "llvm/Support/Format.h" 88#include "llvm/Support/KnownBits.h" 89#include "llvm/Support/MachineValueType.h" 90#include "llvm/Support/MathExtras.h" 91#include "llvm/Support/raw_ostream.h" 92#include "llvm/Target/TargetMachine.h" 93#include "llvm/Target/TargetOptions.h" 94#include <algorithm> 95#include <cassert> 96#include <cstdint> 97#include <iterator> 98#include <list> 99#include <utility> 100#include <vector> 101 102using namespace llvm; 103 104#define DEBUG_TYPE "ppc-lowering" 105 106static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc", 107cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden); 108 109static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref", 110cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden); 111 112static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned", 113cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden); 114 115static cl::opt<bool> DisableSCO("disable-ppc-sco", 116cl::desc("disable sibling call optimization on ppc"), cl::Hidden); 117 118static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32", 119cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden); 120 121static cl::opt<bool> EnableQuadPrecision("enable-ppc-quad-precision", 122cl::desc("enable quad precision float support on ppc"), cl::Hidden); 123 124static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables", 125cl::desc("use absolute jump tables on ppc"), cl::Hidden); 126 127STATISTIC(NumTailCalls, "Number of tail calls"); 128STATISTIC(NumSiblingCalls, "Number of sibling calls"); 129 130static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int); 131 132static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl); 133 134// FIXME: Remove this once the bug has been fixed! 135extern cl::opt<bool> ANDIGlueBug; 136 137PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, 138 const PPCSubtarget &STI) 139 : TargetLowering(TM), Subtarget(STI) { 140 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all 141 // arguments are at least 4/8 bytes aligned. 142 bool isPPC64 = Subtarget.isPPC64(); 143 setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4)); 144 145 // Set up the register classes. 146 addRegisterClass(MVT::i32, &PPC::GPRCRegClass); 147 if (!useSoftFloat()) { 148 if (hasSPE()) { 149 addRegisterClass(MVT::f32, &PPC::GPRCRegClass); 150 addRegisterClass(MVT::f64, &PPC::SPERCRegClass); 151 } else { 152 addRegisterClass(MVT::f32, &PPC::F4RCRegClass); 153 addRegisterClass(MVT::f64, &PPC::F8RCRegClass); 154 } 155 } 156 157 // Match BITREVERSE to customized fast code sequence in the td file. 158 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); 159 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); 160 161 // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended. 162 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 163 164 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD. 165 for (MVT VT : MVT::integer_valuetypes()) { 166 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 167 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); 168 } 169 170 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 171 172 // PowerPC has pre-inc load and store's. 173 setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal); 174 setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal); 175 setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal); 176 setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal); 177 setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal); 178 setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal); 179 setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal); 180 setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal); 181 setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal); 182 setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal); 183 if (!Subtarget.hasSPE()) { 184 setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal); 185 setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal); 186 setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal); 187 setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal); 188 } 189 190 // PowerPC uses ADDC/ADDE/SUBC/SUBE to propagate carry. 191 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; 192 for (MVT VT : ScalarIntVTs) { 193 setOperationAction(ISD::ADDC, VT, Legal); 194 setOperationAction(ISD::ADDE, VT, Legal); 195 setOperationAction(ISD::SUBC, VT, Legal); 196 setOperationAction(ISD::SUBE, VT, Legal); 197 } 198 199 if (Subtarget.useCRBits()) { 200 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 201 202 if (isPPC64 || Subtarget.hasFPCVT()) { 203 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); 204 AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, 205 isPPC64 ? MVT::i64 : MVT::i32); 206 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); 207 AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, 208 isPPC64 ? MVT::i64 : MVT::i32); 209 } else { 210 setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); 211 setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); 212 } 213 214 // PowerPC does not support direct load/store of condition registers. 215 setOperationAction(ISD::LOAD, MVT::i1, Custom); 216 setOperationAction(ISD::STORE, MVT::i1, Custom); 217 218 // FIXME: Remove this once the ANDI glue bug is fixed: 219 if (ANDIGlueBug) 220 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); 221 222 for (MVT VT : MVT::integer_valuetypes()) { 223 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 224 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 225 setTruncStoreAction(VT, MVT::i1, Expand); 226 } 227 228 addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass); 229 } 230 231 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on 232 // PPC (the libcall is not available). 233 setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom); 234 setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom); 235 236 // We do not currently implement these libm ops for PowerPC. 237 setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); 238 setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand); 239 setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand); 240 setOperationAction(ISD::FRINT, MVT::ppcf128, Expand); 241 setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand); 242 setOperationAction(ISD::FREM, MVT::ppcf128, Expand); 243 244 // PowerPC has no SREM/UREM instructions unless we are on P9 245 // On P9 we may use a hardware instruction to compute the remainder. 246 // The instructions are not legalized directly because in the cases where the 247 // result of both the remainder and the division is required it is more 248 // efficient to compute the remainder from the result of the division rather 249 // than use the remainder instruction. 250 if (Subtarget.isISA3_0()) { 251 setOperationAction(ISD::SREM, MVT::i32, Custom); 252 setOperationAction(ISD::UREM, MVT::i32, Custom); 253 setOperationAction(ISD::SREM, MVT::i64, Custom); 254 setOperationAction(ISD::UREM, MVT::i64, Custom); 255 } else { 256 setOperationAction(ISD::SREM, MVT::i32, Expand); 257 setOperationAction(ISD::UREM, MVT::i32, Expand); 258 setOperationAction(ISD::SREM, MVT::i64, Expand); 259 setOperationAction(ISD::UREM, MVT::i64, Expand); 260 } 261 262 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM. 263 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 264 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 265 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); 266 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 267 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); 268 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); 269 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); 270 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); 271 272 // We don't support sin/cos/sqrt/fmod/pow 273 setOperationAction(ISD::FSIN , MVT::f64, Expand); 274 setOperationAction(ISD::FCOS , MVT::f64, Expand); 275 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); 276 setOperationAction(ISD::FREM , MVT::f64, Expand); 277 setOperationAction(ISD::FPOW , MVT::f64, Expand); 278 setOperationAction(ISD::FSIN , MVT::f32, Expand); 279 setOperationAction(ISD::FCOS , MVT::f32, Expand); 280 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); 281 setOperationAction(ISD::FREM , MVT::f32, Expand); 282 setOperationAction(ISD::FPOW , MVT::f32, Expand); 283 if (Subtarget.hasSPE()) { 284 setOperationAction(ISD::FMA , MVT::f64, Expand); 285 setOperationAction(ISD::FMA , MVT::f32, Expand); 286 } else { 287 setOperationAction(ISD::FMA , MVT::f64, Legal); 288 setOperationAction(ISD::FMA , MVT::f32, Legal); 289 } 290 291 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); 292 293 // If we're enabling GP optimizations, use hardware square root 294 if (!Subtarget.hasFSQRT() && 295 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() && 296 Subtarget.hasFRE())) 297 setOperationAction(ISD::FSQRT, MVT::f64, Expand); 298 299 if (!Subtarget.hasFSQRT() && 300 !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() && 301 Subtarget.hasFRES())) 302 setOperationAction(ISD::FSQRT, MVT::f32, Expand); 303 304 if (Subtarget.hasFCPSGN()) { 305 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal); 306 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal); 307 } else { 308 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 309 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 310 } 311 312 if (Subtarget.hasFPRND()) { 313 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); 314 setOperationAction(ISD::FCEIL, MVT::f64, Legal); 315 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); 316 setOperationAction(ISD::FROUND, MVT::f64, Legal); 317 318 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 319 setOperationAction(ISD::FCEIL, MVT::f32, Legal); 320 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); 321 setOperationAction(ISD::FROUND, MVT::f32, Legal); 322 } 323 324 // PowerPC does not have BSWAP, but we can use vector BSWAP instruction xxbrd 325 // to speed up scalar BSWAP64. 326 // CTPOP or CTTZ were introduced in P8/P9 respectively 327 setOperationAction(ISD::BSWAP, MVT::i32 , Expand); 328 if (Subtarget.hasP9Vector()) 329 setOperationAction(ISD::BSWAP, MVT::i64 , Custom); 330 else 331 setOperationAction(ISD::BSWAP, MVT::i64 , Expand); 332 if (Subtarget.isISA3_0()) { 333 setOperationAction(ISD::CTTZ , MVT::i32 , Legal); 334 setOperationAction(ISD::CTTZ , MVT::i64 , Legal); 335 } else { 336 setOperationAction(ISD::CTTZ , MVT::i32 , Expand); 337 setOperationAction(ISD::CTTZ , MVT::i64 , Expand); 338 } 339 340 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) { 341 setOperationAction(ISD::CTPOP, MVT::i32 , Legal); 342 setOperationAction(ISD::CTPOP, MVT::i64 , Legal); 343 } else { 344 setOperationAction(ISD::CTPOP, MVT::i32 , Expand); 345 setOperationAction(ISD::CTPOP, MVT::i64 , Expand); 346 } 347 348 // PowerPC does not have ROTR 349 setOperationAction(ISD::ROTR, MVT::i32 , Expand); 350 setOperationAction(ISD::ROTR, MVT::i64 , Expand); 351 352 if (!Subtarget.useCRBits()) { 353 // PowerPC does not have Select 354 setOperationAction(ISD::SELECT, MVT::i32, Expand); 355 setOperationAction(ISD::SELECT, MVT::i64, Expand); 356 setOperationAction(ISD::SELECT, MVT::f32, Expand); 357 setOperationAction(ISD::SELECT, MVT::f64, Expand); 358 } 359 360 // PowerPC wants to turn select_cc of FP into fsel when possible. 361 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 362 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); 363 364 // PowerPC wants to optimize integer setcc a bit 365 if (!Subtarget.useCRBits()) 366 setOperationAction(ISD::SETCC, MVT::i32, Custom); 367 368 // PowerPC does not have BRCOND which requires SetCC 369 if (!Subtarget.useCRBits()) 370 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 371 372 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 373 374 if (Subtarget.hasSPE()) { 375 // SPE has built-in conversions 376 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); 377 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); 378 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); 379 } else { 380 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. 381 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 382 383 // PowerPC does not have [U|S]INT_TO_FP 384 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); 385 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); 386 } 387 388 if (Subtarget.hasDirectMove() && isPPC64) { 389 setOperationAction(ISD::BITCAST, MVT::f32, Legal); 390 setOperationAction(ISD::BITCAST, MVT::i32, Legal); 391 setOperationAction(ISD::BITCAST, MVT::i64, Legal); 392 setOperationAction(ISD::BITCAST, MVT::f64, Legal); 393 if (TM.Options.UnsafeFPMath) { 394 setOperationAction(ISD::LRINT, MVT::f64, Legal); 395 setOperationAction(ISD::LRINT, MVT::f32, Legal); 396 setOperationAction(ISD::LLRINT, MVT::f64, Legal); 397 setOperationAction(ISD::LLRINT, MVT::f32, Legal); 398 setOperationAction(ISD::LROUND, MVT::f64, Legal); 399 setOperationAction(ISD::LROUND, MVT::f32, Legal); 400 setOperationAction(ISD::LLROUND, MVT::f64, Legal); 401 setOperationAction(ISD::LLROUND, MVT::f32, Legal); 402 } 403 } else { 404 setOperationAction(ISD::BITCAST, MVT::f32, Expand); 405 setOperationAction(ISD::BITCAST, MVT::i32, Expand); 406 setOperationAction(ISD::BITCAST, MVT::i64, Expand); 407 setOperationAction(ISD::BITCAST, MVT::f64, Expand); 408 } 409 410 // We cannot sextinreg(i1). Expand to shifts. 411 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 412 413 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support 414 // SjLj exception handling but a light-weight setjmp/longjmp replacement to 415 // support continuation, user-level threading, and etc.. As a result, no 416 // other SjLj exception interfaces are implemented and please don't build 417 // your own exception handling based on them. 418 // LLVM/Clang supports zero-cost DWARF exception handling. 419 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); 420 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); 421 422 // We want to legalize GlobalAddress and ConstantPool nodes into the 423 // appropriate instructions to materialize the address. 424 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 425 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); 426 setOperationAction(ISD::BlockAddress, MVT::i32, Custom); 427 setOperationAction(ISD::ConstantPool, MVT::i32, Custom); 428 setOperationAction(ISD::JumpTable, MVT::i32, Custom); 429 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 430 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); 431 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); 432 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); 433 setOperationAction(ISD::JumpTable, MVT::i64, Custom); 434 435 // TRAP is legal. 436 setOperationAction(ISD::TRAP, MVT::Other, Legal); 437 438 // TRAMPOLINE is custom lowered. 439 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); 440 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); 441 442 // VASTART needs to be custom lowered to use the VarArgsFrameIndex 443 setOperationAction(ISD::VASTART , MVT::Other, Custom); 444 445 if (Subtarget.is64BitELFABI()) { 446 // VAARG always uses double-word chunks, so promote anything smaller. 447 setOperationAction(ISD::VAARG, MVT::i1, Promote); 448 AddPromotedToType(ISD::VAARG, MVT::i1, MVT::i64); 449 setOperationAction(ISD::VAARG, MVT::i8, Promote); 450 AddPromotedToType(ISD::VAARG, MVT::i8, MVT::i64); 451 setOperationAction(ISD::VAARG, MVT::i16, Promote); 452 AddPromotedToType(ISD::VAARG, MVT::i16, MVT::i64); 453 setOperationAction(ISD::VAARG, MVT::i32, Promote); 454 AddPromotedToType(ISD::VAARG, MVT::i32, MVT::i64); 455 setOperationAction(ISD::VAARG, MVT::Other, Expand); 456 } else if (Subtarget.is32BitELFABI()) { 457 // VAARG is custom lowered with the 32-bit SVR4 ABI. 458 setOperationAction(ISD::VAARG, MVT::Other, Custom); 459 setOperationAction(ISD::VAARG, MVT::i64, Custom); 460 } else 461 setOperationAction(ISD::VAARG, MVT::Other, Expand); 462 463 // VACOPY is custom lowered with the 32-bit SVR4 ABI. 464 if (Subtarget.is32BitELFABI()) 465 setOperationAction(ISD::VACOPY , MVT::Other, Custom); 466 else 467 setOperationAction(ISD::VACOPY , MVT::Other, Expand); 468 469 // Use the default implementation. 470 setOperationAction(ISD::VAEND , MVT::Other, Expand); 471 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand); 472 setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom); 473 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom); 474 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom); 475 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom); 476 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom); 477 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); 478 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); 479 480 // We want to custom lower some of our intrinsics. 481 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 482 483 // To handle counter-based loop conditions. 484 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom); 485 486 setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); 487 setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); 488 setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom); 489 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 490 491 // Comparisons that require checking two conditions. 492 if (Subtarget.hasSPE()) { 493 setCondCodeAction(ISD::SETO, MVT::f32, Expand); 494 setCondCodeAction(ISD::SETO, MVT::f64, Expand); 495 setCondCodeAction(ISD::SETUO, MVT::f32, Expand); 496 setCondCodeAction(ISD::SETUO, MVT::f64, Expand); 497 } 498 setCondCodeAction(ISD::SETULT, MVT::f32, Expand); 499 setCondCodeAction(ISD::SETULT, MVT::f64, Expand); 500 setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); 501 setCondCodeAction(ISD::SETUGT, MVT::f64, Expand); 502 setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); 503 setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand); 504 setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); 505 setCondCodeAction(ISD::SETOGE, MVT::f64, Expand); 506 setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); 507 setCondCodeAction(ISD::SETOLE, MVT::f64, Expand); 508 setCondCodeAction(ISD::SETONE, MVT::f32, Expand); 509 setCondCodeAction(ISD::SETONE, MVT::f64, Expand); 510 511 if (Subtarget.has64BitSupport()) { 512 // They also have instructions for converting between i64 and fp. 513 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 514 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); 515 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 516 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 517 // This is just the low 32 bits of a (signed) fp->i64 conversion. 518 // We cannot do this with Promote because i64 is not a legal type. 519 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 520 521 if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) 522 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 523 } else { 524 // PowerPC does not have FP_TO_UINT on 32-bit implementations. 525 if (Subtarget.hasSPE()) 526 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); 527 else 528 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); 529 } 530 531 // With the instructions enabled under FPCVT, we can do everything. 532 if (Subtarget.hasFPCVT()) { 533 if (Subtarget.has64BitSupport()) { 534 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); 535 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); 536 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); 537 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); 538 } 539 540 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); 541 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); 542 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); 543 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); 544 } 545 546 if (Subtarget.use64BitRegs()) { 547 // 64-bit PowerPC implementations can support i64 types directly 548 addRegisterClass(MVT::i64, &PPC::G8RCRegClass); 549 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or 550 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); 551 // 64-bit PowerPC wants to expand i128 shifts itself. 552 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); 553 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); 554 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); 555 } else { 556 // 32-bit PowerPC wants to expand i64 shifts itself. 557 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); 558 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); 559 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); 560 } 561 562 if (Subtarget.hasVSX()) { 563 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); 564 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); 565 setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); 566 setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); 567 } 568 569 if (Subtarget.hasAltivec()) { 570 // First set operation action for all vector types to expand. Then we 571 // will selectively turn on ones that can be effectively codegen'd. 572 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { 573 // add/sub are legal for all supported vector VT's. 574 setOperationAction(ISD::ADD, VT, Legal); 575 setOperationAction(ISD::SUB, VT, Legal); 576 577 // For v2i64, these are only valid with P8Vector. This is corrected after 578 // the loop. 579 if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) { 580 setOperationAction(ISD::SMAX, VT, Legal); 581 setOperationAction(ISD::SMIN, VT, Legal); 582 setOperationAction(ISD::UMAX, VT, Legal); 583 setOperationAction(ISD::UMIN, VT, Legal); 584 } 585 else { 586 setOperationAction(ISD::SMAX, VT, Expand); 587 setOperationAction(ISD::SMIN, VT, Expand); 588 setOperationAction(ISD::UMAX, VT, Expand); 589 setOperationAction(ISD::UMIN, VT, Expand); 590 } 591 592 if (Subtarget.hasVSX()) { 593 setOperationAction(ISD::FMAXNUM, VT, Legal); 594 setOperationAction(ISD::FMINNUM, VT, Legal); 595 } 596 597 // Vector instructions introduced in P8 598 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) { 599 setOperationAction(ISD::CTPOP, VT, Legal); 600 setOperationAction(ISD::CTLZ, VT, Legal); 601 } 602 else { 603 setOperationAction(ISD::CTPOP, VT, Expand); 604 setOperationAction(ISD::CTLZ, VT, Expand); 605 } 606 607 // Vector instructions introduced in P9 608 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128)) 609 setOperationAction(ISD::CTTZ, VT, Legal); 610 else 611 setOperationAction(ISD::CTTZ, VT, Expand); 612 613 // We promote all shuffles to v16i8. 614 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote); 615 AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8); 616 617 // We promote all non-typed operations to v4i32. 618 setOperationAction(ISD::AND , VT, Promote); 619 AddPromotedToType (ISD::AND , VT, MVT::v4i32); 620 setOperationAction(ISD::OR , VT, Promote); 621 AddPromotedToType (ISD::OR , VT, MVT::v4i32); 622 setOperationAction(ISD::XOR , VT, Promote); 623 AddPromotedToType (ISD::XOR , VT, MVT::v4i32); 624 setOperationAction(ISD::LOAD , VT, Promote); 625 AddPromotedToType (ISD::LOAD , VT, MVT::v4i32); 626 setOperationAction(ISD::SELECT, VT, Promote); 627 AddPromotedToType (ISD::SELECT, VT, MVT::v4i32); 628 setOperationAction(ISD::VSELECT, VT, Legal); 629 setOperationAction(ISD::SELECT_CC, VT, Promote); 630 AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32); 631 setOperationAction(ISD::STORE, VT, Promote); 632 AddPromotedToType (ISD::STORE, VT, MVT::v4i32); 633 634 // No other operations are legal. 635 setOperationAction(ISD::MUL , VT, Expand); 636 setOperationAction(ISD::SDIV, VT, Expand); 637 setOperationAction(ISD::SREM, VT, Expand); 638 setOperationAction(ISD::UDIV, VT, Expand); 639 setOperationAction(ISD::UREM, VT, Expand); 640 setOperationAction(ISD::FDIV, VT, Expand); 641 setOperationAction(ISD::FREM, VT, Expand); 642 setOperationAction(ISD::FNEG, VT, Expand); 643 setOperationAction(ISD::FSQRT, VT, Expand); 644 setOperationAction(ISD::FLOG, VT, Expand); 645 setOperationAction(ISD::FLOG10, VT, Expand); 646 setOperationAction(ISD::FLOG2, VT, Expand); 647 setOperationAction(ISD::FEXP, VT, Expand); 648 setOperationAction(ISD::FEXP2, VT, Expand); 649 setOperationAction(ISD::FSIN, VT, Expand); 650 setOperationAction(ISD::FCOS, VT, Expand); 651 setOperationAction(ISD::FABS, VT, Expand); 652 setOperationAction(ISD::FFLOOR, VT, Expand); 653 setOperationAction(ISD::FCEIL, VT, Expand); 654 setOperationAction(ISD::FTRUNC, VT, Expand); 655 setOperationAction(ISD::FRINT, VT, Expand); 656 setOperationAction(ISD::FNEARBYINT, VT, Expand); 657 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand); 658 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); 659 setOperationAction(ISD::BUILD_VECTOR, VT, Expand); 660 setOperationAction(ISD::MULHU, VT, Expand); 661 setOperationAction(ISD::MULHS, VT, Expand); 662 setOperationAction(ISD::UMUL_LOHI, VT, Expand); 663 setOperationAction(ISD::SMUL_LOHI, VT, Expand); 664 setOperationAction(ISD::UDIVREM, VT, Expand); 665 setOperationAction(ISD::SDIVREM, VT, Expand); 666 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); 667 setOperationAction(ISD::FPOW, VT, Expand); 668 setOperationAction(ISD::BSWAP, VT, Expand); 669 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); 670 setOperationAction(ISD::ROTL, VT, Expand); 671 setOperationAction(ISD::ROTR, VT, Expand); 672 673 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) { 674 setTruncStoreAction(VT, InnerVT, Expand); 675 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 676 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 677 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 678 } 679 } 680 if (!Subtarget.hasP8Vector()) { 681 setOperationAction(ISD::SMAX, MVT::v2i64, Expand); 682 setOperationAction(ISD::SMIN, MVT::v2i64, Expand); 683 setOperationAction(ISD::UMAX, MVT::v2i64, Expand); 684 setOperationAction(ISD::UMIN, MVT::v2i64, Expand); 685 } 686 687 for (auto VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16, MVT::v16i8}) 688 setOperationAction(ISD::ABS, VT, Custom); 689 690 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle 691 // with merges, splats, etc. 692 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); 693 694 // Vector truncates to sub-word integer that fit in an Altivec/VSX register 695 // are cheap, so handle them before they get expanded to scalar. 696 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom); 697 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom); 698 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom); 699 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom); 700 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom); 701 702 setOperationAction(ISD::AND , MVT::v4i32, Legal); 703 setOperationAction(ISD::OR , MVT::v4i32, Legal); 704 setOperationAction(ISD::XOR , MVT::v4i32, Legal); 705 setOperationAction(ISD::LOAD , MVT::v4i32, Legal); 706 setOperationAction(ISD::SELECT, MVT::v4i32, 707 Subtarget.useCRBits() ? Legal : Expand); 708 setOperationAction(ISD::STORE , MVT::v4i32, Legal); 709 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); 710 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); 711 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); 712 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); 713 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 714 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 715 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 716 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 717 718 // Without hasP8Altivec set, v2i64 SMAX isn't available. 719 // But ABS custom lowering requires SMAX support. 720 if (!Subtarget.hasP8Altivec()) 721 setOperationAction(ISD::ABS, MVT::v2i64, Expand); 722 723 // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w). 724 if (Subtarget.hasAltivec()) 725 for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8}) 726 setOperationAction(ISD::ROTL, VT, Legal); 727 // With hasP8Altivec set, we can lower ISD::ROTL to vrld. 728 if (Subtarget.hasP8Altivec()) 729 setOperationAction(ISD::ROTL, MVT::v2i64, Legal); 730 731 addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass); 732 addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass); 733 addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass); 734 addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass); 735 736 setOperationAction(ISD::MUL, MVT::v4f32, Legal); 737 setOperationAction(ISD::FMA, MVT::v4f32, Legal); 738 739 if (TM.Options.UnsafeFPMath || Subtarget.hasVSX()) { 740 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 741 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 742 } 743 744 if (Subtarget.hasP8Altivec()) 745 setOperationAction(ISD::MUL, MVT::v4i32, Legal); 746 else 747 setOperationAction(ISD::MUL, MVT::v4i32, Custom); 748 749 setOperationAction(ISD::MUL, MVT::v8i16, Custom); 750 setOperationAction(ISD::MUL, MVT::v16i8, Custom); 751 752 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom); 753 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom); 754 755 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom); 756 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom); 757 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom); 758 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 759 760 // Altivec does not contain unordered floating-point compare instructions 761 setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand); 762 setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand); 763 setCondCodeAction(ISD::SETO, MVT::v4f32, Expand); 764 setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand); 765 766 if (Subtarget.hasVSX()) { 767 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); 768 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 769 if (Subtarget.hasP8Vector()) { 770 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 771 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal); 772 } 773 if (Subtarget.hasDirectMove() && isPPC64) { 774 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal); 775 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal); 776 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal); 777 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal); 778 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal); 779 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal); 780 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); 781 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); 782 } 783 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); 784 785 // The nearbyint variants are not allowed to raise the inexact exception 786 // so we can only code-gen them with unsafe math. 787 if (TM.Options.UnsafeFPMath) { 788 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); 789 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); 790 } 791 792 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); 793 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); 794 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); 795 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); 796 setOperationAction(ISD::FROUND, MVT::v2f64, Legal); 797 setOperationAction(ISD::FROUND, MVT::f64, Legal); 798 799 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 800 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 801 setOperationAction(ISD::FROUND, MVT::f32, Legal); 802 803 setOperationAction(ISD::MUL, MVT::v2f64, Legal); 804 setOperationAction(ISD::FMA, MVT::v2f64, Legal); 805 806 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 807 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 808 809 // Share the Altivec comparison restrictions. 810 setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand); 811 setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand); 812 setCondCodeAction(ISD::SETO, MVT::v2f64, Expand); 813 setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand); 814 815 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); 816 setOperationAction(ISD::STORE, MVT::v2f64, Legal); 817 818 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal); 819 820 if (Subtarget.hasP8Vector()) 821 addRegisterClass(MVT::f32, &PPC::VSSRCRegClass); 822 823 addRegisterClass(MVT::f64, &PPC::VSFRCRegClass); 824 825 addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass); 826 addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass); 827 addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass); 828 829 if (Subtarget.hasP8Altivec()) { 830 setOperationAction(ISD::SHL, MVT::v2i64, Legal); 831 setOperationAction(ISD::SRA, MVT::v2i64, Legal); 832 setOperationAction(ISD::SRL, MVT::v2i64, Legal); 833 834 // 128 bit shifts can be accomplished via 3 instructions for SHL and 835 // SRL, but not for SRA because of the instructions available: 836 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth 837 // doing 838 setOperationAction(ISD::SHL, MVT::v1i128, Expand); 839 setOperationAction(ISD::SRL, MVT::v1i128, Expand); 840 setOperationAction(ISD::SRA, MVT::v1i128, Expand); 841 842 setOperationAction(ISD::SETCC, MVT::v2i64, Legal); 843 } 844 else { 845 setOperationAction(ISD::SHL, MVT::v2i64, Expand); 846 setOperationAction(ISD::SRA, MVT::v2i64, Expand); 847 setOperationAction(ISD::SRL, MVT::v2i64, Expand); 848 849 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); 850 851 // VSX v2i64 only supports non-arithmetic operations. 852 setOperationAction(ISD::ADD, MVT::v2i64, Expand); 853 setOperationAction(ISD::SUB, MVT::v2i64, Expand); 854 } 855 856 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); 857 AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); 858 setOperationAction(ISD::STORE, MVT::v2i64, Promote); 859 AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64); 860 861 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); 862 863 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 864 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 865 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 866 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 867 868 // Custom handling for partial vectors of integers converted to 869 // floating point. We already have optimal handling for v2i32 through 870 // the DAG combine, so those aren't necessary. 871 setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom); 872 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); 873 setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom); 874 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); 875 setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom); 876 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom); 877 setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom); 878 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); 879 880 setOperationAction(ISD::FNEG, MVT::v4f32, Legal); 881 setOperationAction(ISD::FNEG, MVT::v2f64, Legal); 882 setOperationAction(ISD::FABS, MVT::v4f32, Legal); 883 setOperationAction(ISD::FABS, MVT::v2f64, Legal); 884 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); 885 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal); 886 887 if (Subtarget.hasDirectMove()) 888 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); 889 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); 890 891 addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); 892 } 893 894 if (Subtarget.hasP8Altivec()) { 895 addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass); 896 addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass); 897 } 898 899 if (Subtarget.hasP9Vector()) { 900 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); 901 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 902 903 // 128 bit shifts can be accomplished via 3 instructions for SHL and 904 // SRL, but not for SRA because of the instructions available: 905 // VS{RL} and VS{RL}O. 906 setOperationAction(ISD::SHL, MVT::v1i128, Legal); 907 setOperationAction(ISD::SRL, MVT::v1i128, Legal); 908 setOperationAction(ISD::SRA, MVT::v1i128, Expand); 909 910 if (EnableQuadPrecision) { 911 addRegisterClass(MVT::f128, &PPC::VRRCRegClass); 912 setOperationAction(ISD::FADD, MVT::f128, Legal); 913 setOperationAction(ISD::FSUB, MVT::f128, Legal); 914 setOperationAction(ISD::FDIV, MVT::f128, Legal); 915 setOperationAction(ISD::FMUL, MVT::f128, Legal); 916 setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal); 917 // No extending loads to f128 on PPC. 918 for (MVT FPT : MVT::fp_valuetypes()) 919 setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); 920 setOperationAction(ISD::FMA, MVT::f128, Legal); 921 setCondCodeAction(ISD::SETULT, MVT::f128, Expand); 922 setCondCodeAction(ISD::SETUGT, MVT::f128, Expand); 923 setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand); 924 setCondCodeAction(ISD::SETOGE, MVT::f128, Expand); 925 setCondCodeAction(ISD::SETOLE, MVT::f128, Expand); 926 setCondCodeAction(ISD::SETONE, MVT::f128, Expand); 927 928 setOperationAction(ISD::FTRUNC, MVT::f128, Legal); 929 setOperationAction(ISD::FRINT, MVT::f128, Legal); 930 setOperationAction(ISD::FFLOOR, MVT::f128, Legal); 931 setOperationAction(ISD::FCEIL, MVT::f128, Legal); 932 setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal); 933 setOperationAction(ISD::FROUND, MVT::f128, Legal); 934 935 setOperationAction(ISD::SELECT, MVT::f128, Expand); 936 setOperationAction(ISD::FP_ROUND, MVT::f64, Legal); 937 setOperationAction(ISD::FP_ROUND, MVT::f32, Legal); 938 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 939 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 940 setOperationAction(ISD::BITCAST, MVT::i128, Custom); 941 // No implementation for these ops for PowerPC. 942 setOperationAction(ISD::FSIN , MVT::f128, Expand); 943 setOperationAction(ISD::FCOS , MVT::f128, Expand); 944 setOperationAction(ISD::FPOW, MVT::f128, Expand); 945 setOperationAction(ISD::FPOWI, MVT::f128, Expand); 946 setOperationAction(ISD::FREM, MVT::f128, Expand); 947 } 948 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); 949 setOperationAction(ISD::BSWAP, MVT::v8i16, Legal); 950 setOperationAction(ISD::BSWAP, MVT::v4i32, Legal); 951 setOperationAction(ISD::BSWAP, MVT::v2i64, Legal); 952 setOperationAction(ISD::BSWAP, MVT::v1i128, Legal); 953 } 954 955 if (Subtarget.hasP9Altivec()) { 956 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); 957 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); 958 959 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Legal); 960 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal); 961 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal); 962 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal); 963 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal); 964 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); 965 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal); 966 } 967 } 968 969 if (Subtarget.hasQPX()) { 970 setOperationAction(ISD::FADD, MVT::v4f64, Legal); 971 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); 972 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); 973 setOperationAction(ISD::FREM, MVT::v4f64, Expand); 974 975 setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal); 976 setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand); 977 978 setOperationAction(ISD::LOAD , MVT::v4f64, Custom); 979 setOperationAction(ISD::STORE , MVT::v4f64, Custom); 980 981 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom); 982 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom); 983 984 if (!Subtarget.useCRBits()) 985 setOperationAction(ISD::SELECT, MVT::v4f64, Expand); 986 setOperationAction(ISD::VSELECT, MVT::v4f64, Legal); 987 988 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal); 989 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand); 990 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand); 991 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand); 992 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom); 993 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal); 994 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom); 995 996 setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal); 997 setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand); 998 999 setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal); 1000 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal); 1001 1002 setOperationAction(ISD::FNEG , MVT::v4f64, Legal); 1003 setOperationAction(ISD::FABS , MVT::v4f64, Legal); 1004 setOperationAction(ISD::FSIN , MVT::v4f64, Expand); 1005 setOperationAction(ISD::FCOS , MVT::v4f64, Expand); 1006 setOperationAction(ISD::FPOW , MVT::v4f64, Expand); 1007 setOperationAction(ISD::FLOG , MVT::v4f64, Expand); 1008 setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand); 1009 setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand); 1010 setOperationAction(ISD::FEXP , MVT::v4f64, Expand); 1011 setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand); 1012 1013 setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal); 1014 setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal); 1015 1016 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal); 1017 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal); 1018 1019 addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass); 1020 1021 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 1022 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 1023 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 1024 setOperationAction(ISD::FREM, MVT::v4f32, Expand); 1025 1026 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); 1027 setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand); 1028 1029 setOperationAction(ISD::LOAD , MVT::v4f32, Custom); 1030 setOperationAction(ISD::STORE , MVT::v4f32, Custom); 1031 1032 if (!Subtarget.useCRBits()) 1033 setOperationAction(ISD::SELECT, MVT::v4f32, Expand); 1034 setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); 1035 1036 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal); 1037 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand); 1038 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand); 1039 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand); 1040 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom); 1041 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 1042 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); 1043 1044 setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal); 1045 setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand); 1046 1047 setOperationAction(ISD::FNEG , MVT::v4f32, Legal); 1048 setOperationAction(ISD::FABS , MVT::v4f32, Legal); 1049 setOperationAction(ISD::FSIN , MVT::v4f32, Expand); 1050 setOperationAction(ISD::FCOS , MVT::v4f32, Expand); 1051 setOperationAction(ISD::FPOW , MVT::v4f32, Expand); 1052 setOperationAction(ISD::FLOG , MVT::v4f32, Expand); 1053 setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand); 1054 setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand); 1055 setOperationAction(ISD::FEXP , MVT::v4f32, Expand); 1056 setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand); 1057 1058 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); 1059 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); 1060 1061 setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal); 1062 setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal); 1063 1064 addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass); 1065 1066 setOperationAction(ISD::AND , MVT::v4i1, Legal); 1067 setOperationAction(ISD::OR , MVT::v4i1, Legal); 1068 setOperationAction(ISD::XOR , MVT::v4i1, Legal); 1069 1070 if (!Subtarget.useCRBits()) 1071 setOperationAction(ISD::SELECT, MVT::v4i1, Expand); 1072 setOperationAction(ISD::VSELECT, MVT::v4i1, Legal); 1073 1074 setOperationAction(ISD::LOAD , MVT::v4i1, Custom); 1075 setOperationAction(ISD::STORE , MVT::v4i1, Custom); 1076 1077 setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom); 1078 setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand); 1079 setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand); 1080 setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand); 1081 setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom); 1082 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand); 1083 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); 1084 1085 setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom); 1086 setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom); 1087 1088 addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass); 1089 1090 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); 1091 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); 1092 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); 1093 setOperationAction(ISD::FROUND, MVT::v4f64, Legal); 1094 1095 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 1096 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 1097 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 1098 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 1099 1100 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand); 1101 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); 1102 1103 // These need to set FE_INEXACT, and so cannot be vectorized here. 1104 setOperationAction(ISD::FRINT, MVT::v4f64, Expand); 1105 setOperationAction(ISD::FRINT, MVT::v4f32, Expand); 1106 1107 if (TM.Options.UnsafeFPMath) { 1108 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); 1109 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); 1110 1111 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 1112 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 1113 } else { 1114 setOperationAction(ISD::FDIV, MVT::v4f64, Expand); 1115 setOperationAction(ISD::FSQRT, MVT::v4f64, Expand); 1116 1117 setOperationAction(ISD::FDIV, MVT::v4f32, Expand); 1118 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); 1119 } 1120 } 1121 1122 if (Subtarget.has64BitSupport()) 1123 setOperationAction(ISD::PREFETCH, MVT::Other, Legal); 1124 1125 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom); 1126 1127 if (!isPPC64) { 1128 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand); 1129 setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); 1130 } 1131 1132 setBooleanContents(ZeroOrOneBooleanContent); 1133 1134 if (Subtarget.hasAltivec()) { 1135 // Altivec instructions set fields to all zeros or all ones. 1136 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 1137 } 1138 1139 if (!isPPC64) { 1140 // These libcalls are not available in 32-bit. 1141 setLibcallName(RTLIB::SHL_I128, nullptr); 1142 setLibcallName(RTLIB::SRL_I128, nullptr); 1143 setLibcallName(RTLIB::SRA_I128, nullptr); 1144 } 1145 1146 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1); 1147 1148 // We have target-specific dag combine patterns for the following nodes: 1149 setTargetDAGCombine(ISD::ADD); 1150 setTargetDAGCombine(ISD::SHL); 1151 setTargetDAGCombine(ISD::SRA); 1152 setTargetDAGCombine(ISD::SRL); 1153 setTargetDAGCombine(ISD::MUL); 1154 setTargetDAGCombine(ISD::SINT_TO_FP); 1155 setTargetDAGCombine(ISD::BUILD_VECTOR); 1156 if (Subtarget.hasFPCVT()) 1157 setTargetDAGCombine(ISD::UINT_TO_FP); 1158 setTargetDAGCombine(ISD::LOAD); 1159 setTargetDAGCombine(ISD::STORE); 1160 setTargetDAGCombine(ISD::BR_CC); 1161 if (Subtarget.useCRBits()) 1162 setTargetDAGCombine(ISD::BRCOND); 1163 setTargetDAGCombine(ISD::BSWAP); 1164 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); 1165 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); 1166 setTargetDAGCombine(ISD::INTRINSIC_VOID); 1167 1168 setTargetDAGCombine(ISD::SIGN_EXTEND); 1169 setTargetDAGCombine(ISD::ZERO_EXTEND); 1170 setTargetDAGCombine(ISD::ANY_EXTEND); 1171 1172 setTargetDAGCombine(ISD::TRUNCATE); 1173 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); 1174 1175 1176 if (Subtarget.useCRBits()) { 1177 setTargetDAGCombine(ISD::TRUNCATE); 1178 setTargetDAGCombine(ISD::SETCC); 1179 setTargetDAGCombine(ISD::SELECT_CC); 1180 } 1181 1182 // Use reciprocal estimates. 1183 if (TM.Options.UnsafeFPMath) { 1184 setTargetDAGCombine(ISD::FDIV); 1185 setTargetDAGCombine(ISD::FSQRT); 1186 } 1187 1188 if (Subtarget.hasP9Altivec()) { 1189 setTargetDAGCombine(ISD::ABS); 1190 setTargetDAGCombine(ISD::VSELECT); 1191 } 1192 1193 // Darwin long double math library functions have $LDBL128 appended. 1194 if (Subtarget.isDarwin()) { 1195 setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128"); 1196 setLibcallName(RTLIB::POW_PPCF128, "powl$LDBL128"); 1197 setLibcallName(RTLIB::REM_PPCF128, "fmodl$LDBL128"); 1198 setLibcallName(RTLIB::SIN_PPCF128, "sinl$LDBL128"); 1199 setLibcallName(RTLIB::SQRT_PPCF128, "sqrtl$LDBL128"); 1200 setLibcallName(RTLIB::LOG_PPCF128, "logl$LDBL128"); 1201 setLibcallName(RTLIB::LOG2_PPCF128, "log2l$LDBL128"); 1202 setLibcallName(RTLIB::LOG10_PPCF128, "log10l$LDBL128"); 1203 setLibcallName(RTLIB::EXP_PPCF128, "expl$LDBL128"); 1204 setLibcallName(RTLIB::EXP2_PPCF128, "exp2l$LDBL128"); 1205 } 1206 1207 if (EnableQuadPrecision) { 1208 setLibcallName(RTLIB::LOG_F128, "logf128"); 1209 setLibcallName(RTLIB::LOG2_F128, "log2f128"); 1210 setLibcallName(RTLIB::LOG10_F128, "log10f128"); 1211 setLibcallName(RTLIB::EXP_F128, "expf128"); 1212 setLibcallName(RTLIB::EXP2_F128, "exp2f128"); 1213 setLibcallName(RTLIB::SIN_F128, "sinf128"); 1214 setLibcallName(RTLIB::COS_F128, "cosf128"); 1215 setLibcallName(RTLIB::POW_F128, "powf128"); 1216 setLibcallName(RTLIB::FMIN_F128, "fminf128"); 1217 setLibcallName(RTLIB::FMAX_F128, "fmaxf128"); 1218 setLibcallName(RTLIB::POWI_F128, "__powikf2"); 1219 setLibcallName(RTLIB::REM_F128, "fmodf128"); 1220 } 1221 1222 // With 32 condition bits, we don't need to sink (and duplicate) compares 1223 // aggressively in CodeGenPrep. 1224 if (Subtarget.useCRBits()) { 1225 setHasMultipleConditionRegisters(); 1226 setJumpIsExpensive(); 1227 } 1228 1229 setMinFunctionAlignment(Align(4)); 1230 if (Subtarget.isDarwin()) 1231 setPrefFunctionAlignment(Align(16)); 1232 1233 switch (Subtarget.getCPUDirective()) { 1234 default: break; 1235 case PPC::DIR_970: 1236 case PPC::DIR_A2: 1237 case PPC::DIR_E500: 1238 case PPC::DIR_E500mc: 1239 case PPC::DIR_E5500: 1240 case PPC::DIR_PWR4: 1241 case PPC::DIR_PWR5: 1242 case PPC::DIR_PWR5X: 1243 case PPC::DIR_PWR6: 1244 case PPC::DIR_PWR6X: 1245 case PPC::DIR_PWR7: 1246 case PPC::DIR_PWR8: 1247 case PPC::DIR_PWR9: 1248 case PPC::DIR_PWR_FUTURE: 1249 setPrefLoopAlignment(Align(16)); 1250 setPrefFunctionAlignment(Align(16)); 1251 break; 1252 } 1253 1254 if (Subtarget.enableMachineScheduler()) 1255 setSchedulingPreference(Sched::Source); 1256 else 1257 setSchedulingPreference(Sched::Hybrid); 1258 1259 computeRegisterProperties(STI.getRegisterInfo()); 1260 1261 // The Freescale cores do better with aggressive inlining of memcpy and 1262 // friends. GCC uses same threshold of 128 bytes (= 32 word stores). 1263 if (Subtarget.getCPUDirective() == PPC::DIR_E500mc || 1264 Subtarget.getCPUDirective() == PPC::DIR_E5500) { 1265 MaxStoresPerMemset = 32; 1266 MaxStoresPerMemsetOptSize = 16; 1267 MaxStoresPerMemcpy = 32; 1268 MaxStoresPerMemcpyOptSize = 8; 1269 MaxStoresPerMemmove = 32; 1270 MaxStoresPerMemmoveOptSize = 8; 1271 } else if (Subtarget.getCPUDirective() == PPC::DIR_A2) { 1272 // The A2 also benefits from (very) aggressive inlining of memcpy and 1273 // friends. The overhead of a the function call, even when warm, can be 1274 // over one hundred cycles. 1275 MaxStoresPerMemset = 128; 1276 MaxStoresPerMemcpy = 128; 1277 MaxStoresPerMemmove = 128; 1278 MaxLoadsPerMemcmp = 128; 1279 } else { 1280 MaxLoadsPerMemcmp = 8; 1281 MaxLoadsPerMemcmpOptSize = 4; 1282 } 1283} 1284 1285/// getMaxByValAlign - Helper for getByValTypeAlignment to determine 1286/// the desired ByVal argument alignment. 1287static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign, 1288 unsigned MaxMaxAlign) { 1289 if (MaxAlign == MaxMaxAlign) 1290 return; 1291 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) { 1292 if (MaxMaxAlign >= 32 && VTy->getBitWidth() >= 256) 1293 MaxAlign = 32; 1294 else if (VTy->getBitWidth() >= 128 && MaxAlign < 16) 1295 MaxAlign = 16; 1296 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { 1297 unsigned EltAlign = 0; 1298 getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign); 1299 if (EltAlign > MaxAlign) 1300 MaxAlign = EltAlign; 1301 } else if (StructType *STy = dyn_cast<StructType>(Ty)) { 1302 for (auto *EltTy : STy->elements()) { 1303 unsigned EltAlign = 0; 1304 getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign); 1305 if (EltAlign > MaxAlign) 1306 MaxAlign = EltAlign; 1307 if (MaxAlign == MaxMaxAlign) 1308 break; 1309 } 1310 } 1311} 1312 1313/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate 1314/// function arguments in the caller parameter area. 1315unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty, 1316 const DataLayout &DL) const { 1317 // Darwin passes everything on 4 byte boundary. 1318 if (Subtarget.isDarwin()) 1319 return 4; 1320 1321 // 16byte and wider vectors are passed on 16byte boundary. 1322 // The rest is 8 on PPC64 and 4 on PPC32 boundary. 1323 unsigned Align = Subtarget.isPPC64() ? 8 : 4; 1324 if (Subtarget.hasAltivec() || Subtarget.hasQPX()) 1325 getMaxByValAlign(Ty, Align, Subtarget.hasQPX() ? 32 : 16); 1326 return Align; 1327} 1328 1329bool PPCTargetLowering::useSoftFloat() const { 1330 return Subtarget.useSoftFloat(); 1331} 1332 1333bool PPCTargetLowering::hasSPE() const { 1334 return Subtarget.hasSPE(); 1335} 1336 1337bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { 1338 return VT.isScalarInteger(); 1339} 1340 1341const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { 1342 switch ((PPCISD::NodeType)Opcode) { 1343 case PPCISD::FIRST_NUMBER: break; 1344 case PPCISD::FSEL: return "PPCISD::FSEL"; 1345 case PPCISD::XSMAXCDP: return "PPCISD::XSMAXCDP"; 1346 case PPCISD::XSMINCDP: return "PPCISD::XSMINCDP"; 1347 case PPCISD::FCFID: return "PPCISD::FCFID"; 1348 case PPCISD::FCFIDU: return "PPCISD::FCFIDU"; 1349 case PPCISD::FCFIDS: return "PPCISD::FCFIDS"; 1350 case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS"; 1351 case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ"; 1352 case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ"; 1353 case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ"; 1354 case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ"; 1355 case PPCISD::FP_TO_UINT_IN_VSR: 1356 return "PPCISD::FP_TO_UINT_IN_VSR,"; 1357 case PPCISD::FP_TO_SINT_IN_VSR: 1358 return "PPCISD::FP_TO_SINT_IN_VSR"; 1359 case PPCISD::FRE: return "PPCISD::FRE"; 1360 case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; 1361 case PPCISD::STFIWX: return "PPCISD::STFIWX"; 1362 case PPCISD::VMADDFP: return "PPCISD::VMADDFP"; 1363 case PPCISD::VNMSUBFP: return "PPCISD::VNMSUBFP"; 1364 case PPCISD::VPERM: return "PPCISD::VPERM"; 1365 case PPCISD::XXSPLT: return "PPCISD::XXSPLT"; 1366 case PPCISD::VECINSERT: return "PPCISD::VECINSERT"; 1367 case PPCISD::XXPERMDI: return "PPCISD::XXPERMDI"; 1368 case PPCISD::VECSHL: return "PPCISD::VECSHL"; 1369 case PPCISD::CMPB: return "PPCISD::CMPB"; 1370 case PPCISD::Hi: return "PPCISD::Hi"; 1371 case PPCISD::Lo: return "PPCISD::Lo"; 1372 case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY"; 1373 case PPCISD::ATOMIC_CMP_SWAP_8: return "PPCISD::ATOMIC_CMP_SWAP_8"; 1374 case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16"; 1375 case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC"; 1376 case PPCISD::DYNAREAOFFSET: return "PPCISD::DYNAREAOFFSET"; 1377 case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg"; 1378 case PPCISD::SRL: return "PPCISD::SRL"; 1379 case PPCISD::SRA: return "PPCISD::SRA"; 1380 case PPCISD::SHL: return "PPCISD::SHL"; 1381 case PPCISD::SRA_ADDZE: return "PPCISD::SRA_ADDZE"; 1382 case PPCISD::CALL: return "PPCISD::CALL"; 1383 case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP"; 1384 case PPCISD::MTCTR: return "PPCISD::MTCTR"; 1385 case PPCISD::BCTRL: return "PPCISD::BCTRL"; 1386 case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC"; 1387 case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG"; 1388 case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE"; 1389 case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP"; 1390 case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP"; 1391 case PPCISD::MFOCRF: return "PPCISD::MFOCRF"; 1392 case PPCISD::MFVSR: return "PPCISD::MFVSR"; 1393 case PPCISD::MTVSRA: return "PPCISD::MTVSRA"; 1394 case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ"; 1395 case PPCISD::SINT_VEC_TO_FP: return "PPCISD::SINT_VEC_TO_FP"; 1396 case PPCISD::UINT_VEC_TO_FP: return "PPCISD::UINT_VEC_TO_FP"; 1397 case PPCISD::ANDI_rec_1_EQ_BIT: 1398 return "PPCISD::ANDI_rec_1_EQ_BIT"; 1399 case PPCISD::ANDI_rec_1_GT_BIT: 1400 return "PPCISD::ANDI_rec_1_GT_BIT"; 1401 case PPCISD::VCMP: return "PPCISD::VCMP"; 1402 case PPCISD::VCMPo: return "PPCISD::VCMPo"; 1403 case PPCISD::LBRX: return "PPCISD::LBRX"; 1404 case PPCISD::STBRX: return "PPCISD::STBRX"; 1405 case PPCISD::LFIWAX: return "PPCISD::LFIWAX"; 1406 case PPCISD::LFIWZX: return "PPCISD::LFIWZX"; 1407 case PPCISD::LXSIZX: return "PPCISD::LXSIZX"; 1408 case PPCISD::STXSIX: return "PPCISD::STXSIX"; 1409 case PPCISD::VEXTS: return "PPCISD::VEXTS"; 1410 case PPCISD::SExtVElems: return "PPCISD::SExtVElems"; 1411 case PPCISD::LXVD2X: return "PPCISD::LXVD2X"; 1412 case PPCISD::STXVD2X: return "PPCISD::STXVD2X"; 1413 case PPCISD::LOAD_VEC_BE: return "PPCISD::LOAD_VEC_BE"; 1414 case PPCISD::STORE_VEC_BE: return "PPCISD::STORE_VEC_BE"; 1415 case PPCISD::ST_VSR_SCAL_INT: 1416 return "PPCISD::ST_VSR_SCAL_INT"; 1417 case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; 1418 case PPCISD::BDNZ: return "PPCISD::BDNZ"; 1419 case PPCISD::BDZ: return "PPCISD::BDZ"; 1420 case PPCISD::MFFS: return "PPCISD::MFFS"; 1421 case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ"; 1422 case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN"; 1423 case PPCISD::CR6SET: return "PPCISD::CR6SET"; 1424 case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET"; 1425 case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT"; 1426 case PPCISD::PPC32_PICGOT: return "PPCISD::PPC32_PICGOT"; 1427 case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA"; 1428 case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L"; 1429 case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS"; 1430 case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA"; 1431 case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; 1432 case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; 1433 case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR"; 1434 case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; 1435 case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; 1436 case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; 1437 case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR"; 1438 case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; 1439 case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; 1440 case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; 1441 case PPCISD::SC: return "PPCISD::SC"; 1442 case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB"; 1443 case PPCISD::MFBHRBE: return "PPCISD::MFBHRBE"; 1444 case PPCISD::RFEBB: return "PPCISD::RFEBB"; 1445 case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD"; 1446 case PPCISD::SWAP_NO_CHAIN: return "PPCISD::SWAP_NO_CHAIN"; 1447 case PPCISD::VABSD: return "PPCISD::VABSD"; 1448 case PPCISD::QVFPERM: return "PPCISD::QVFPERM"; 1449 case PPCISD::QVGPCI: return "PPCISD::QVGPCI"; 1450 case PPCISD::QVALIGNI: return "PPCISD::QVALIGNI"; 1451 case PPCISD::QVESPLATI: return "PPCISD::QVESPLATI"; 1452 case PPCISD::QBFLT: return "PPCISD::QBFLT"; 1453 case PPCISD::QVLFSb: return "PPCISD::QVLFSb"; 1454 case PPCISD::BUILD_FP128: return "PPCISD::BUILD_FP128"; 1455 case PPCISD::BUILD_SPE64: return "PPCISD::BUILD_SPE64"; 1456 case PPCISD::EXTRACT_SPE: return "PPCISD::EXTRACT_SPE"; 1457 case PPCISD::EXTSWSLI: return "PPCISD::EXTSWSLI"; 1458 case PPCISD::LD_VSX_LH: return "PPCISD::LD_VSX_LH"; 1459 case PPCISD::FP_EXTEND_HALF: return "PPCISD::FP_EXTEND_HALF"; 1460 case PPCISD::LD_SPLAT: return "PPCISD::LD_SPLAT"; 1461 } 1462 return nullptr; 1463} 1464 1465EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C, 1466 EVT VT) const { 1467 if (!VT.isVector()) 1468 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32; 1469 1470 if (Subtarget.hasQPX()) 1471 return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); 1472 1473 return VT.changeVectorElementTypeToInteger(); 1474} 1475 1476bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const { 1477 assert(VT.isFloatingPoint() && "Non-floating-point FMA?"); 1478 return true; 1479} 1480 1481//===----------------------------------------------------------------------===// 1482// Node matching predicates, for use by the tblgen matching code. 1483//===----------------------------------------------------------------------===// 1484 1485/// isFloatingPointZero - Return true if this is 0.0 or -0.0. 1486static bool isFloatingPointZero(SDValue Op) { 1487 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op)) 1488 return CFP->getValueAPF().isZero(); 1489 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { 1490 // Maybe this has already been legalized into the constant pool? 1491 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1))) 1492 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) 1493 return CFP->getValueAPF().isZero(); 1494 } 1495 return false; 1496} 1497 1498/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return 1499/// true if Op is undef or if it matches the specified value. 1500static bool isConstantOrUndef(int Op, int Val) { 1501 return Op < 0 || Op == Val; 1502} 1503 1504/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a 1505/// VPKUHUM instruction. 1506/// The ShuffleKind distinguishes between big-endian operations with 1507/// two different inputs (0), either-endian operations with two identical 1508/// inputs (1), and little-endian operations with two different inputs (2). 1509/// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1510bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1511 SelectionDAG &DAG) { 1512 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1513 if (ShuffleKind == 0) { 1514 if (IsLE) 1515 return false; 1516 for (unsigned i = 0; i != 16; ++i) 1517 if (!isConstantOrUndef(N->getMaskElt(i), i*2+1)) 1518 return false; 1519 } else if (ShuffleKind == 2) { 1520 if (!IsLE) 1521 return false; 1522 for (unsigned i = 0; i != 16; ++i) 1523 if (!isConstantOrUndef(N->getMaskElt(i), i*2)) 1524 return false; 1525 } else if (ShuffleKind == 1) { 1526 unsigned j = IsLE ? 0 : 1; 1527 for (unsigned i = 0; i != 8; ++i) 1528 if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) || 1529 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j)) 1530 return false; 1531 } 1532 return true; 1533} 1534 1535/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a 1536/// VPKUWUM instruction. 1537/// The ShuffleKind distinguishes between big-endian operations with 1538/// two different inputs (0), either-endian operations with two identical 1539/// inputs (1), and little-endian operations with two different inputs (2). 1540/// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1541bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1542 SelectionDAG &DAG) { 1543 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1544 if (ShuffleKind == 0) { 1545 if (IsLE) 1546 return false; 1547 for (unsigned i = 0; i != 16; i += 2) 1548 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || 1549 !isConstantOrUndef(N->getMaskElt(i+1), i*2+3)) 1550 return false; 1551 } else if (ShuffleKind == 2) { 1552 if (!IsLE) 1553 return false; 1554 for (unsigned i = 0; i != 16; i += 2) 1555 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1556 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1)) 1557 return false; 1558 } else if (ShuffleKind == 1) { 1559 unsigned j = IsLE ? 0 : 2; 1560 for (unsigned i = 0; i != 8; i += 2) 1561 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1562 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1563 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1564 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1)) 1565 return false; 1566 } 1567 return true; 1568} 1569 1570/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a 1571/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the 1572/// current subtarget. 1573/// 1574/// The ShuffleKind distinguishes between big-endian operations with 1575/// two different inputs (0), either-endian operations with two identical 1576/// inputs (1), and little-endian operations with two different inputs (2). 1577/// For the latter, the input operands are swapped (see PPCInstrAltivec.td). 1578bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, 1579 SelectionDAG &DAG) { 1580 const PPCSubtarget& Subtarget = 1581 static_cast<const PPCSubtarget&>(DAG.getSubtarget()); 1582 if (!Subtarget.hasP8Vector()) 1583 return false; 1584 1585 bool IsLE = DAG.getDataLayout().isLittleEndian(); 1586 if (ShuffleKind == 0) { 1587 if (IsLE) 1588 return false; 1589 for (unsigned i = 0; i != 16; i += 4) 1590 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+4) || 1591 !isConstantOrUndef(N->getMaskElt(i+1), i*2+5) || 1592 !isConstantOrUndef(N->getMaskElt(i+2), i*2+6) || 1593 !isConstantOrUndef(N->getMaskElt(i+3), i*2+7)) 1594 return false; 1595 } else if (ShuffleKind == 2) { 1596 if (!IsLE) 1597 return false; 1598 for (unsigned i = 0; i != 16; i += 4) 1599 if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || 1600 !isConstantOrUndef(N->getMaskElt(i+1), i*2+1) || 1601 !isConstantOrUndef(N->getMaskElt(i+2), i*2+2) || 1602 !isConstantOrUndef(N->getMaskElt(i+3), i*2+3)) 1603 return false; 1604 } else if (ShuffleKind == 1) { 1605 unsigned j = IsLE ? 0 : 4; 1606 for (unsigned i = 0; i != 8; i += 4) 1607 if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || 1608 !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || 1609 !isConstantOrUndef(N->getMaskElt(i+2), i*2+j+2) || 1610 !isConstantOrUndef(N->getMaskElt(i+3), i*2+j+3) || 1611 !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || 1612 !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1) || 1613 !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) || 1614 !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3)) 1615 return false; 1616 } 1617 return true; 1618} 1619 1620/// isVMerge - Common function, used to match vmrg* shuffles. 1621/// 1622static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize, 1623 unsigned LHSStart, unsigned RHSStart) { 1624 if (N->getValueType(0) != MVT::v16i8) 1625 return false; 1626 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && 1627 "Unsupported merge size!"); 1628 1629 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units 1630 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit 1631 if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j), 1632 LHSStart+j+i*UnitSize) || 1633 !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j), 1634 RHSStart+j+i*UnitSize)) 1635 return false; 1636 } 1637 return true; 1638} 1639 1640/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for 1641/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes). 1642/// The ShuffleKind distinguishes between big-endian merges with two 1643/// different inputs (0), either-endian merges with two identical inputs (1), 1644/// and little-endian merges with two different inputs (2). For the latter, 1645/// the input operands are swapped (see PPCInstrAltivec.td). 1646bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1647 unsigned ShuffleKind, SelectionDAG &DAG) { 1648 if (DAG.getDataLayout().isLittleEndian()) { 1649 if (ShuffleKind == 1) // unary 1650 return isVMerge(N, UnitSize, 0, 0); 1651 else if (ShuffleKind == 2) // swapped 1652 return isVMerge(N, UnitSize, 0, 16); 1653 else 1654 return false; 1655 } else { 1656 if (ShuffleKind == 1) // unary 1657 return isVMerge(N, UnitSize, 8, 8); 1658 else if (ShuffleKind == 0) // normal 1659 return isVMerge(N, UnitSize, 8, 24); 1660 else 1661 return false; 1662 } 1663} 1664 1665/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for 1666/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes). 1667/// The ShuffleKind distinguishes between big-endian merges with two 1668/// different inputs (0), either-endian merges with two identical inputs (1), 1669/// and little-endian merges with two different inputs (2). For the latter, 1670/// the input operands are swapped (see PPCInstrAltivec.td). 1671bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, 1672 unsigned ShuffleKind, SelectionDAG &DAG) { 1673 if (DAG.getDataLayout().isLittleEndian()) { 1674 if (ShuffleKind == 1) // unary 1675 return isVMerge(N, UnitSize, 8, 8); 1676 else if (ShuffleKind == 2) // swapped 1677 return isVMerge(N, UnitSize, 8, 24); 1678 else 1679 return false; 1680 } else { 1681 if (ShuffleKind == 1) // unary 1682 return isVMerge(N, UnitSize, 0, 0); 1683 else if (ShuffleKind == 0) // normal 1684 return isVMerge(N, UnitSize, 0, 16); 1685 else 1686 return false; 1687 } 1688} 1689 1690/** 1691 * Common function used to match vmrgew and vmrgow shuffles 1692 * 1693 * The indexOffset determines whether to look for even or odd words in 1694 * the shuffle mask. This is based on the of the endianness of the target 1695 * machine. 1696 * - Little Endian: 1697 * - Use offset of 0 to check for odd elements 1698 * - Use offset of 4 to check for even elements 1699 * - Big Endian: 1700 * - Use offset of 0 to check for even elements 1701 * - Use offset of 4 to check for odd elements 1702 * A detailed description of the vector element ordering for little endian and 1703 * big endian can be found at 1704 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html 1705 * Targeting your applications - what little endian and big endian IBM XL C/C++ 1706 * compiler differences mean to you 1707 * 1708 * The mask to the shuffle vector instruction specifies the indices of the 1709 * elements from the two input vectors to place in the result. The elements are 1710 * numbered in array-access order, starting with the first vector. These vectors 1711 * are always of type v16i8, thus each vector will contain 16 elements of size 1712 * 8. More info on the shuffle vector can be found in the 1713 * http://llvm.org/docs/LangRef.html#shufflevector-instruction 1714 * Language Reference. 1715 * 1716 * The RHSStartValue indicates whether the same input vectors are used (unary) 1717 * or two different input vectors are used, based on the following: 1718 * - If the instruction uses the same vector for both inputs, the range of the 1719 * indices will be 0 to 15. In this case, the RHSStart value passed should 1720 * be 0. 1721 * - If the instruction has two different vectors then the range of the 1722 * indices will be 0 to 31. In this case, the RHSStart value passed should 1723 * be 16 (indices 0-15 specify elements in the first vector while indices 16 1724 * to 31 specify elements in the second vector). 1725 * 1726 * \param[in] N The shuffle vector SD Node to analyze 1727 * \param[in] IndexOffset Specifies whether to look for even or odd elements 1728 * \param[in] RHSStartValue Specifies the starting index for the righthand input 1729 * vector to the shuffle_vector instruction 1730 * \return true iff this shuffle vector represents an even or odd word merge 1731 */ 1732static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset, 1733 unsigned RHSStartValue) { 1734 if (N->getValueType(0) != MVT::v16i8) 1735 return false; 1736 1737 for (unsigned i = 0; i < 2; ++i) 1738 for (unsigned j = 0; j < 4; ++j) 1739 if (!isConstantOrUndef(N->getMaskElt(i*4+j), 1740 i*RHSStartValue+j+IndexOffset) || 1741 !isConstantOrUndef(N->getMaskElt(i*4+j+8), 1742 i*RHSStartValue+j+IndexOffset+8)) 1743 return false; 1744 return true; 1745} 1746 1747/** 1748 * Determine if the specified shuffle mask is suitable for the vmrgew or 1749 * vmrgow instructions. 1750 * 1751 * \param[in] N The shuffle vector SD Node to analyze 1752 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false) 1753 * \param[in] ShuffleKind Identify the type of merge: 1754 * - 0 = big-endian merge with two different inputs; 1755 * - 1 = either-endian merge with two identical inputs; 1756 * - 2 = little-endian merge with two different inputs (inputs are swapped for 1757 * little-endian merges). 1758 * \param[in] DAG The current SelectionDAG 1759 * \return true iff this shuffle mask 1760 */ 1761bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven, 1762 unsigned ShuffleKind, SelectionDAG &DAG) { 1763 if (DAG.getDataLayout().isLittleEndian()) { 1764 unsigned indexOffset = CheckEven ? 4 : 0; 1765 if (ShuffleKind == 1) // Unary 1766 return isVMerge(N, indexOffset, 0); 1767 else if (ShuffleKind == 2) // swapped 1768 return isVMerge(N, indexOffset, 16); 1769 else 1770 return false; 1771 } 1772 else { 1773 unsigned indexOffset = CheckEven ? 0 : 4; 1774 if (ShuffleKind == 1) // Unary 1775 return isVMerge(N, indexOffset, 0); 1776 else if (ShuffleKind == 0) // Normal 1777 return isVMerge(N, indexOffset, 16); 1778 else 1779 return false; 1780 } 1781 return false; 1782} 1783 1784/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift 1785/// amount, otherwise return -1. 1786/// The ShuffleKind distinguishes between big-endian operations with two 1787/// different inputs (0), either-endian operations with two identical inputs 1788/// (1), and little-endian operations with two different inputs (2). For the 1789/// latter, the input operands are swapped (see PPCInstrAltivec.td). 1790int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind, 1791 SelectionDAG &DAG) { 1792 if (N->getValueType(0) != MVT::v16i8) 1793 return -1; 1794 1795 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 1796 1797 // Find the first non-undef value in the shuffle mask. 1798 unsigned i; 1799 for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i) 1800 /*search*/; 1801 1802 if (i == 16) return -1; // all undef. 1803 1804 // Otherwise, check to see if the rest of the elements are consecutively 1805 // numbered from this value. 1806 unsigned ShiftAmt = SVOp->getMaskElt(i); 1807 if (ShiftAmt < i) return -1; 1808 1809 ShiftAmt -= i; 1810 bool isLE = DAG.getDataLayout().isLittleEndian(); 1811 1812 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) { 1813 // Check the rest of the elements to see if they are consecutive. 1814 for (++i; i != 16; ++i) 1815 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 1816 return -1; 1817 } else if (ShuffleKind == 1) { 1818 // Check the rest of the elements to see if they are consecutive. 1819 for (++i; i != 16; ++i) 1820 if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15)) 1821 return -1; 1822 } else 1823 return -1; 1824 1825 if (isLE) 1826 ShiftAmt = 16 - ShiftAmt; 1827 1828 return ShiftAmt; 1829} 1830 1831/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand 1832/// specifies a splat of a single element that is suitable for input to 1833/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.). 1834bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) { 1835 assert(N->getValueType(0) == MVT::v16i8 && isPowerOf2_32(EltSize) && 1836 EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes"); 1837 1838 // The consecutive indices need to specify an element, not part of two 1839 // different elements. So abandon ship early if this isn't the case. 1840 if (N->getMaskElt(0) % EltSize != 0) 1841 return false; 1842 1843 // This is a splat operation if each element of the permute is the same, and 1844 // if the value doesn't reference the second vector. 1845 unsigned ElementBase = N->getMaskElt(0); 1846 1847 // FIXME: Handle UNDEF elements too! 1848 if (ElementBase >= 16) 1849 return false; 1850 1851 // Check that the indices are consecutive, in the case of a multi-byte element 1852 // splatted with a v16i8 mask. 1853 for (unsigned i = 1; i != EltSize; ++i) 1854 if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase)) 1855 return false; 1856 1857 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) { 1858 if (N->getMaskElt(i) < 0) continue; 1859 for (unsigned j = 0; j != EltSize; ++j) 1860 if (N->getMaskElt(i+j) != N->getMaskElt(j)) 1861 return false; 1862 } 1863 return true; 1864} 1865 1866/// Check that the mask is shuffling N byte elements. Within each N byte 1867/// element of the mask, the indices could be either in increasing or 1868/// decreasing order as long as they are consecutive. 1869/// \param[in] N the shuffle vector SD Node to analyze 1870/// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/ 1871/// Word/DoubleWord/QuadWord). 1872/// \param[in] StepLen the delta indices number among the N byte element, if 1873/// the mask is in increasing/decreasing order then it is 1/-1. 1874/// \return true iff the mask is shuffling N byte elements. 1875static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width, 1876 int StepLen) { 1877 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) && 1878 "Unexpected element width."); 1879 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width."); 1880 1881 unsigned NumOfElem = 16 / Width; 1882 unsigned MaskVal[16]; // Width is never greater than 16 1883 for (unsigned i = 0; i < NumOfElem; ++i) { 1884 MaskVal[0] = N->getMaskElt(i * Width); 1885 if ((StepLen == 1) && (MaskVal[0] % Width)) { 1886 return false; 1887 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) { 1888 return false; 1889 } 1890 1891 for (unsigned int j = 1; j < Width; ++j) { 1892 MaskVal[j] = N->getMaskElt(i * Width + j); 1893 if (MaskVal[j] != MaskVal[j-1] + StepLen) { 1894 return false; 1895 } 1896 } 1897 } 1898 1899 return true; 1900} 1901 1902bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, 1903 unsigned &InsertAtByte, bool &Swap, bool IsLE) { 1904 if (!isNByteElemShuffleMask(N, 4, 1)) 1905 return false; 1906 1907 // Now we look at mask elements 0,4,8,12 1908 unsigned M0 = N->getMaskElt(0) / 4; 1909 unsigned M1 = N->getMaskElt(4) / 4; 1910 unsigned M2 = N->getMaskElt(8) / 4; 1911 unsigned M3 = N->getMaskElt(12) / 4; 1912 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 }; 1913 unsigned BigEndianShifts[] = { 3, 0, 1, 2 }; 1914 1915 // Below, let H and L be arbitrary elements of the shuffle mask 1916 // where H is in the range [4,7] and L is in the range [0,3]. 1917 // H, 1, 2, 3 or L, 5, 6, 7 1918 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) || 1919 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) { 1920 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3]; 1921 InsertAtByte = IsLE ? 12 : 0; 1922 Swap = M0 < 4; 1923 return true; 1924 } 1925 // 0, H, 2, 3 or 4, L, 6, 7 1926 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) || 1927 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) { 1928 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3]; 1929 InsertAtByte = IsLE ? 8 : 4; 1930 Swap = M1 < 4; 1931 return true; 1932 } 1933 // 0, 1, H, 3 or 4, 5, L, 7 1934 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) || 1935 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) { 1936 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3]; 1937 InsertAtByte = IsLE ? 4 : 8; 1938 Swap = M2 < 4; 1939 return true; 1940 } 1941 // 0, 1, 2, H or 4, 5, 6, L 1942 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) || 1943 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) { 1944 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3]; 1945 InsertAtByte = IsLE ? 0 : 12; 1946 Swap = M3 < 4; 1947 return true; 1948 } 1949 1950 // If both vector operands for the shuffle are the same vector, the mask will 1951 // contain only elements from the first one and the second one will be undef. 1952 if (N->getOperand(1).isUndef()) { 1953 ShiftElts = 0; 1954 Swap = true; 1955 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1; 1956 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) { 1957 InsertAtByte = IsLE ? 12 : 0; 1958 return true; 1959 } 1960 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) { 1961 InsertAtByte = IsLE ? 8 : 4; 1962 return true; 1963 } 1964 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) { 1965 InsertAtByte = IsLE ? 4 : 8; 1966 return true; 1967 } 1968 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) { 1969 InsertAtByte = IsLE ? 0 : 12; 1970 return true; 1971 } 1972 } 1973 1974 return false; 1975} 1976 1977bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, 1978 bool &Swap, bool IsLE) { 1979 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 1980 // Ensure each byte index of the word is consecutive. 1981 if (!isNByteElemShuffleMask(N, 4, 1)) 1982 return false; 1983 1984 // Now we look at mask elements 0,4,8,12, which are the beginning of words. 1985 unsigned M0 = N->getMaskElt(0) / 4; 1986 unsigned M1 = N->getMaskElt(4) / 4; 1987 unsigned M2 = N->getMaskElt(8) / 4; 1988 unsigned M3 = N->getMaskElt(12) / 4; 1989 1990 // If both vector operands for the shuffle are the same vector, the mask will 1991 // contain only elements from the first one and the second one will be undef. 1992 if (N->getOperand(1).isUndef()) { 1993 assert(M0 < 4 && "Indexing into an undef vector?"); 1994 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4) 1995 return false; 1996 1997 ShiftElts = IsLE ? (4 - M0) % 4 : M0; 1998 Swap = false; 1999 return true; 2000 } 2001 2002 // Ensure each word index of the ShuffleVector Mask is consecutive. 2003 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8) 2004 return false; 2005 2006 if (IsLE) { 2007 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) { 2008 // Input vectors don't need to be swapped if the leading element 2009 // of the result is one of the 3 left elements of the second vector 2010 // (or if there is no shift to be done at all). 2011 Swap = false; 2012 ShiftElts = (8 - M0) % 8; 2013 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) { 2014 // Input vectors need to be swapped if the leading element 2015 // of the result is one of the 3 left elements of the first vector 2016 // (or if we're shifting by 4 - thereby simply swapping the vectors). 2017 Swap = true; 2018 ShiftElts = (4 - M0) % 4; 2019 } 2020 2021 return true; 2022 } else { // BE 2023 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) { 2024 // Input vectors don't need to be swapped if the leading element 2025 // of the result is one of the 4 elements of the first vector. 2026 Swap = false; 2027 ShiftElts = M0; 2028 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) { 2029 // Input vectors need to be swapped if the leading element 2030 // of the result is one of the 4 elements of the right vector. 2031 Swap = true; 2032 ShiftElts = M0 - 4; 2033 } 2034 2035 return true; 2036 } 2037} 2038 2039bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) { 2040 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 2041 2042 if (!isNByteElemShuffleMask(N, Width, -1)) 2043 return false; 2044 2045 for (int i = 0; i < 16; i += Width) 2046 if (N->getMaskElt(i) != i + Width - 1) 2047 return false; 2048 2049 return true; 2050} 2051 2052bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) { 2053 return isXXBRShuffleMaskHelper(N, 2); 2054} 2055 2056bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) { 2057 return isXXBRShuffleMaskHelper(N, 4); 2058} 2059 2060bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) { 2061 return isXXBRShuffleMaskHelper(N, 8); 2062} 2063 2064bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) { 2065 return isXXBRShuffleMaskHelper(N, 16); 2066} 2067 2068/// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap 2069/// if the inputs to the instruction should be swapped and set \p DM to the 2070/// value for the immediate. 2071/// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI 2072/// AND element 0 of the result comes from the first input (LE) or second input 2073/// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered. 2074/// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle 2075/// mask. 2076bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM, 2077 bool &Swap, bool IsLE) { 2078 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); 2079 2080 // Ensure each byte index of the double word is consecutive. 2081 if (!isNByteElemShuffleMask(N, 8, 1)) 2082 return false; 2083 2084 unsigned M0 = N->getMaskElt(0) / 8; 2085 unsigned M1 = N->getMaskElt(8) / 8; 2086 assert(((M0 | M1) < 4) && "A mask element out of bounds?"); 2087 2088 // If both vector operands for the shuffle are the same vector, the mask will 2089 // contain only elements from the first one and the second one will be undef. 2090 if (N->getOperand(1).isUndef()) { 2091 if ((M0 | M1) < 2) { 2092 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1); 2093 Swap = false; 2094 return true; 2095 } else 2096 return false; 2097 } 2098 2099 if (IsLE) { 2100 if (M0 > 1 && M1 < 2) { 2101 Swap = false; 2102 } else if (M0 < 2 && M1 > 1) { 2103 M0 = (M0 + 2) % 4; 2104 M1 = (M1 + 2) % 4; 2105 Swap = true; 2106 } else 2107 return false; 2108 2109 // Note: if control flow comes here that means Swap is already set above 2110 DM = (((~M1) & 1) << 1) + ((~M0) & 1); 2111 return true; 2112 } else { // BE 2113 if (M0 < 2 && M1 > 1) { 2114 Swap = false; 2115 } else if (M0 > 1 && M1 < 2) { 2116 M0 = (M0 + 2) % 4; 2117 M1 = (M1 + 2) % 4; 2118 Swap = true; 2119 } else 2120 return false; 2121 2122 // Note: if control flow comes here that means Swap is already set above 2123 DM = (M0 << 1) + (M1 & 1); 2124 return true; 2125 } 2126} 2127 2128 2129/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is 2130/// appropriate for PPC mnemonics (which have a big endian bias - namely 2131/// elements are counted from the left of the vector register). 2132unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize, 2133 SelectionDAG &DAG) { 2134 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 2135 assert(isSplatShuffleMask(SVOp, EltSize)); 2136 if (DAG.getDataLayout().isLittleEndian()) 2137 return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize); 2138 else 2139 return SVOp->getMaskElt(0) / EltSize; 2140} 2141 2142/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed 2143/// by using a vspltis[bhw] instruction of the specified element size, return 2144/// the constant being splatted. The ByteSize field indicates the number of 2145/// bytes of each element [124] -> [bhw]. 2146SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { 2147 SDValue OpVal(nullptr, 0); 2148 2149 // If ByteSize of the splat is bigger than the element size of the 2150 // build_vector, then we have a case where we are checking for a splat where 2151 // multiple elements of the buildvector are folded together into a single 2152 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8). 2153 unsigned EltSize = 16/N->getNumOperands(); 2154 if (EltSize < ByteSize) { 2155 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval. 2156 SDValue UniquedVals[4]; 2157 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?"); 2158 2159 // See if all of the elements in the buildvector agree across. 2160 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2161 if (N->getOperand(i).isUndef()) continue; 2162 // If the element isn't a constant, bail fully out. 2163 if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue(); 2164 2165 if (!UniquedVals[i&(Multiple-1)].getNode()) 2166 UniquedVals[i&(Multiple-1)] = N->getOperand(i); 2167 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i)) 2168 return SDValue(); // no match. 2169 } 2170 2171 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains 2172 // either constant or undef values that are identical for each chunk. See 2173 // if these chunks can form into a larger vspltis*. 2174 2175 // Check to see if all of the leading entries are either 0 or -1. If 2176 // neither, then this won't fit into the immediate field. 2177 bool LeadingZero = true; 2178 bool LeadingOnes = true; 2179 for (unsigned i = 0; i != Multiple-1; ++i) { 2180 if (!UniquedVals[i].getNode()) continue; // Must have been undefs. 2181 2182 LeadingZero &= isNullConstant(UniquedVals[i]); 2183 LeadingOnes &= isAllOnesConstant(UniquedVals[i]); 2184 } 2185 // Finally, check the least significant entry. 2186 if (LeadingZero) { 2187 if (!UniquedVals[Multiple-1].getNode()) 2188 return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef 2189 int Val = cast<ConstantSDNode>(UniquedVals[Multiple-1])->getZExtValue(); 2190 if (Val < 16) // 0,0,0,4 -> vspltisw(4) 2191 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 2192 } 2193 if (LeadingOnes) { 2194 if (!UniquedVals[Multiple-1].getNode()) 2195 return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef 2196 int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue(); 2197 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2) 2198 return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); 2199 } 2200 2201 return SDValue(); 2202 } 2203 2204 // Check to see if this buildvec has a single non-undef value in its elements. 2205 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 2206 if (N->getOperand(i).isUndef()) continue; 2207 if (!OpVal.getNode()) 2208 OpVal = N->getOperand(i); 2209 else if (OpVal != N->getOperand(i)) 2210 return SDValue(); 2211 } 2212 2213 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def. 2214 2215 unsigned ValSizeInBytes = EltSize; 2216 uint64_t Value = 0; 2217 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) { 2218 Value = CN->getZExtValue(); 2219 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) { 2220 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!"); 2221 Value = FloatToBits(CN->getValueAPF().convertToFloat()); 2222 } 2223 2224 // If the splat value is larger than the element value, then we can never do 2225 // this splat. The only case that we could fit the replicated bits into our 2226 // immediate field for would be zero, and we prefer to use vxor for it. 2227 if (ValSizeInBytes < ByteSize) return SDValue(); 2228 2229 // If the element value is larger than the splat value, check if it consists 2230 // of a repeated bit pattern of size ByteSize. 2231 if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8)) 2232 return SDValue(); 2233 2234 // Properly sign extend the value. 2235 int MaskVal = SignExtend32(Value, ByteSize * 8); 2236 2237 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros. 2238 if (MaskVal == 0) return SDValue(); 2239 2240 // Finally, if this value fits in a 5 bit sext field, return it 2241 if (SignExtend32<5>(MaskVal) == MaskVal) 2242 return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32); 2243 return SDValue(); 2244} 2245 2246/// isQVALIGNIShuffleMask - If this is a qvaligni shuffle mask, return the shift 2247/// amount, otherwise return -1. 2248int PPC::isQVALIGNIShuffleMask(SDNode *N) { 2249 EVT VT = N->getValueType(0); 2250 if (VT != MVT::v4f64 && VT != MVT::v4f32 && VT != MVT::v4i1) 2251 return -1; 2252 2253 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N); 2254 2255 // Find the first non-undef value in the shuffle mask. 2256 unsigned i; 2257 for (i = 0; i != 4 && SVOp->getMaskElt(i) < 0; ++i) 2258 /*search*/; 2259 2260 if (i == 4) return -1; // all undef. 2261 2262 // Otherwise, check to see if the rest of the elements are consecutively 2263 // numbered from this value. 2264 unsigned ShiftAmt = SVOp->getMaskElt(i); 2265 if (ShiftAmt < i) return -1; 2266 ShiftAmt -= i; 2267 2268 // Check the rest of the elements to see if they are consecutive. 2269 for (++i; i != 4; ++i) 2270 if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) 2271 return -1; 2272 2273 return ShiftAmt; 2274} 2275 2276//===----------------------------------------------------------------------===// 2277// Addressing Mode Selection 2278//===----------------------------------------------------------------------===// 2279 2280/// isIntS16Immediate - This method tests to see if the node is either a 32-bit 2281/// or 64-bit immediate, and if the value can be accurately represented as a 2282/// sign extension from a 16-bit value. If so, this returns true and the 2283/// immediate. 2284bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) { 2285 if (!isa<ConstantSDNode>(N)) 2286 return false; 2287 2288 Imm = (int16_t)cast<ConstantSDNode>(N)->getZExtValue(); 2289 if (N->getValueType(0) == MVT::i32) 2290 return Imm == (int32_t)cast<ConstantSDNode>(N)->getZExtValue(); 2291 else 2292 return Imm == (int64_t)cast<ConstantSDNode>(N)->getZExtValue(); 2293} 2294bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) { 2295 return isIntS16Immediate(Op.getNode(), Imm); 2296} 2297 2298 2299/// SelectAddressEVXRegReg - Given the specified address, check to see if it can 2300/// be represented as an indexed [r+r] operation. 2301bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base, 2302 SDValue &Index, 2303 SelectionDAG &DAG) const { 2304 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); 2305 UI != E; ++UI) { 2306 if (MemSDNode *Memop = dyn_cast<MemSDNode>(*UI)) { 2307 if (Memop->getMemoryVT() == MVT::f64) { 2308 Base = N.getOperand(0); 2309 Index = N.getOperand(1); 2310 return true; 2311 } 2312 } 2313 } 2314 return false; 2315} 2316 2317/// SelectAddressRegReg - Given the specified addressed, check to see if it 2318/// can be represented as an indexed [r+r] operation. Returns false if it 2319/// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is 2320/// non-zero and N can be represented by a base register plus a signed 16-bit 2321/// displacement, make a more precise judgement by checking (displacement % \p 2322/// EncodingAlignment). 2323bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base, 2324 SDValue &Index, SelectionDAG &DAG, 2325 unsigned EncodingAlignment) const { 2326 int16_t imm = 0; 2327 if (N.getOpcode() == ISD::ADD) { 2328 // Is there any SPE load/store (f64), which can't handle 16bit offset? 2329 // SPE load/store can only handle 8-bit offsets. 2330 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG)) 2331 return true; 2332 if (isIntS16Immediate(N.getOperand(1), imm) && 2333 (!EncodingAlignment || !(imm % EncodingAlignment))) 2334 return false; // r+i 2335 if (N.getOperand(1).getOpcode() == PPCISD::Lo) 2336 return false; // r+i 2337 2338 Base = N.getOperand(0); 2339 Index = N.getOperand(1); 2340 return true; 2341 } else if (N.getOpcode() == ISD::OR) { 2342 if (isIntS16Immediate(N.getOperand(1), imm) && 2343 (!EncodingAlignment || !(imm % EncodingAlignment))) 2344 return false; // r+i can fold it if we can. 2345 2346 // If this is an or of disjoint bitfields, we can codegen this as an add 2347 // (for better address arithmetic) if the LHS and RHS of the OR are provably 2348 // disjoint. 2349 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2350 2351 if (LHSKnown.Zero.getBoolValue()) { 2352 KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1)); 2353 // If all of the bits are known zero on the LHS or RHS, the add won't 2354 // carry. 2355 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) { 2356 Base = N.getOperand(0); 2357 Index = N.getOperand(1); 2358 return true; 2359 } 2360 } 2361 } 2362 2363 return false; 2364} 2365 2366// If we happen to be doing an i64 load or store into a stack slot that has 2367// less than a 4-byte alignment, then the frame-index elimination may need to 2368// use an indexed load or store instruction (because the offset may not be a 2369// multiple of 4). The extra register needed to hold the offset comes from the 2370// register scavenger, and it is possible that the scavenger will need to use 2371// an emergency spill slot. As a result, we need to make sure that a spill slot 2372// is allocated when doing an i64 load/store into a less-than-4-byte-aligned 2373// stack slot. 2374static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) { 2375 // FIXME: This does not handle the LWA case. 2376 if (VT != MVT::i64) 2377 return; 2378 2379 // NOTE: We'll exclude negative FIs here, which come from argument 2380 // lowering, because there are no known test cases triggering this problem 2381 // using packed structures (or similar). We can remove this exclusion if 2382 // we find such a test case. The reason why this is so test-case driven is 2383 // because this entire 'fixup' is only to prevent crashes (from the 2384 // register scavenger) on not-really-valid inputs. For example, if we have: 2385 // %a = alloca i1 2386 // %b = bitcast i1* %a to i64* 2387 // store i64* a, i64 b 2388 // then the store should really be marked as 'align 1', but is not. If it 2389 // were marked as 'align 1' then the indexed form would have been 2390 // instruction-selected initially, and the problem this 'fixup' is preventing 2391 // won't happen regardless. 2392 if (FrameIdx < 0) 2393 return; 2394 2395 MachineFunction &MF = DAG.getMachineFunction(); 2396 MachineFrameInfo &MFI = MF.getFrameInfo(); 2397 2398 unsigned Align = MFI.getObjectAlignment(FrameIdx); 2399 if (Align >= 4) 2400 return; 2401 2402 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2403 FuncInfo->setHasNonRISpills(); 2404} 2405 2406/// Returns true if the address N can be represented by a base register plus 2407/// a signed 16-bit displacement [r+imm], and if it is not better 2408/// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept 2409/// displacements that are multiples of that value. 2410bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, 2411 SDValue &Base, 2412 SelectionDAG &DAG, 2413 unsigned EncodingAlignment) const { 2414 // FIXME dl should come from parent load or store, not from address 2415 SDLoc dl(N); 2416 // If this can be more profitably realized as r+r, fail. 2417 if (SelectAddressRegReg(N, Disp, Base, DAG, EncodingAlignment)) 2418 return false; 2419 2420 if (N.getOpcode() == ISD::ADD) { 2421 int16_t imm = 0; 2422 if (isIntS16Immediate(N.getOperand(1), imm) && 2423 (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { 2424 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 2425 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 2426 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2427 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2428 } else { 2429 Base = N.getOperand(0); 2430 } 2431 return true; // [r+i] 2432 } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) { 2433 // Match LOAD (ADD (X, Lo(G))). 2434 assert(!cast<ConstantSDNode>(N.getOperand(1).getOperand(1))->getZExtValue() 2435 && "Cannot handle constant offsets yet!"); 2436 Disp = N.getOperand(1).getOperand(0); // The global address. 2437 assert(Disp.getOpcode() == ISD::TargetGlobalAddress || 2438 Disp.getOpcode() == ISD::TargetGlobalTLSAddress || 2439 Disp.getOpcode() == ISD::TargetConstantPool || 2440 Disp.getOpcode() == ISD::TargetJumpTable); 2441 Base = N.getOperand(0); 2442 return true; // [&g+r] 2443 } 2444 } else if (N.getOpcode() == ISD::OR) { 2445 int16_t imm = 0; 2446 if (isIntS16Immediate(N.getOperand(1), imm) && 2447 (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { 2448 // If this is an or of disjoint bitfields, we can codegen this as an add 2449 // (for better address arithmetic) if the LHS and RHS of the OR are 2450 // provably disjoint. 2451 KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); 2452 2453 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) { 2454 // If all of the bits are known zero on the LHS or RHS, the add won't 2455 // carry. 2456 if (FrameIndexSDNode *FI = 2457 dyn_cast<FrameIndexSDNode>(N.getOperand(0))) { 2458 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2459 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2460 } else { 2461 Base = N.getOperand(0); 2462 } 2463 Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); 2464 return true; 2465 } 2466 } 2467 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) { 2468 // Loading from a constant address. 2469 2470 // If this address fits entirely in a 16-bit sext immediate field, codegen 2471 // this as "d, 0" 2472 int16_t Imm; 2473 if (isIntS16Immediate(CN, Imm) && 2474 (!EncodingAlignment || (Imm % EncodingAlignment) == 0)) { 2475 Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0)); 2476 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 2477 CN->getValueType(0)); 2478 return true; 2479 } 2480 2481 // Handle 32-bit sext immediates with LIS + addr mode. 2482 if ((CN->getValueType(0) == MVT::i32 || 2483 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) && 2484 (!EncodingAlignment || (CN->getZExtValue() % EncodingAlignment) == 0)) { 2485 int Addr = (int)CN->getZExtValue(); 2486 2487 // Otherwise, break this down into an LIS + disp. 2488 Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32); 2489 2490 Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl, 2491 MVT::i32); 2492 unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8; 2493 Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0); 2494 return true; 2495 } 2496 } 2497 2498 Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout())); 2499 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) { 2500 Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); 2501 fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); 2502 } else 2503 Base = N; 2504 return true; // [r+0] 2505} 2506 2507/// SelectAddressRegRegOnly - Given the specified addressed, force it to be 2508/// represented as an indexed [r+r] operation. 2509bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, 2510 SDValue &Index, 2511 SelectionDAG &DAG) const { 2512 // Check to see if we can easily represent this as an [r+r] address. This 2513 // will fail if it thinks that the address is more profitably represented as 2514 // reg+imm, e.g. where imm = 0. 2515 if (SelectAddressRegReg(N, Base, Index, DAG)) 2516 return true; 2517 2518 // If the address is the result of an add, we will utilize the fact that the 2519 // address calculation includes an implicit add. However, we can reduce 2520 // register pressure if we do not materialize a constant just for use as the 2521 // index register. We only get rid of the add if it is not an add of a 2522 // value and a 16-bit signed constant and both have a single use. 2523 int16_t imm = 0; 2524 if (N.getOpcode() == ISD::ADD && 2525 (!isIntS16Immediate(N.getOperand(1), imm) || 2526 !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { 2527 Base = N.getOperand(0); 2528 Index = N.getOperand(1); 2529 return true; 2530 } 2531 2532 // Otherwise, do it the hard way, using R0 as the base register. 2533 Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, 2534 N.getValueType()); 2535 Index = N; 2536 return true; 2537} 2538 2539/// Returns true if we should use a direct load into vector instruction 2540/// (such as lxsd or lfd), instead of a load into gpr + direct move sequence. 2541static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) { 2542 2543 // If there are any other uses other than scalar to vector, then we should 2544 // keep it as a scalar load -> direct move pattern to prevent multiple 2545 // loads. 2546 LoadSDNode *LD = dyn_cast<LoadSDNode>(N); 2547 if (!LD) 2548 return false; 2549 2550 EVT MemVT = LD->getMemoryVT(); 2551 if (!MemVT.isSimple()) 2552 return false; 2553 switch(MemVT.getSimpleVT().SimpleTy) { 2554 case MVT::i64: 2555 break; 2556 case MVT::i32: 2557 if (!ST.hasP8Vector()) 2558 return false; 2559 break; 2560 case MVT::i16: 2561 case MVT::i8: 2562 if (!ST.hasP9Vector()) 2563 return false; 2564 break; 2565 default: 2566 return false; 2567 } 2568 2569 SDValue LoadedVal(N, 0); 2570 if (!LoadedVal.hasOneUse()) 2571 return false; 2572 2573 for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); 2574 UI != UE; ++UI) 2575 if (UI.getUse().get().getResNo() == 0 && 2576 UI->getOpcode() != ISD::SCALAR_TO_VECTOR) 2577 return false; 2578 2579 return true; 2580} 2581 2582/// getPreIndexedAddressParts - returns true by value, base pointer and 2583/// offset pointer and addressing mode by reference if the node's address 2584/// can be legally represented as pre-indexed load / store address. 2585bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, 2586 SDValue &Offset, 2587 ISD::MemIndexedMode &AM, 2588 SelectionDAG &DAG) const { 2589 if (DisablePPCPreinc) return false; 2590 2591 bool isLoad = true; 2592 SDValue Ptr; 2593 EVT VT; 2594 unsigned Alignment; 2595 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 2596 Ptr = LD->getBasePtr(); 2597 VT = LD->getMemoryVT(); 2598 Alignment = LD->getAlignment(); 2599 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 2600 Ptr = ST->getBasePtr(); 2601 VT = ST->getMemoryVT(); 2602 Alignment = ST->getAlignment(); 2603 isLoad = false; 2604 } else 2605 return false; 2606 2607 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector 2608 // instructions because we can fold these into a more efficient instruction 2609 // instead, (such as LXSD). 2610 if (isLoad && usePartialVectorLoads(N, Subtarget)) { 2611 return false; 2612 } 2613 2614 // PowerPC doesn't have preinc load/store instructions for vectors (except 2615 // for QPX, which does have preinc r+r forms). 2616 if (VT.isVector()) { 2617 if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) { 2618 return false; 2619 } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) { 2620 AM = ISD::PRE_INC; 2621 return true; 2622 } 2623 } 2624 2625 if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { 2626 // Common code will reject creating a pre-inc form if the base pointer 2627 // is a frame index, or if N is a store and the base pointer is either 2628 // the same as or a predecessor of the value being stored. Check for 2629 // those situations here, and try with swapped Base/Offset instead. 2630 bool Swap = false; 2631 2632 if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base)) 2633 Swap = true; 2634 else if (!isLoad) { 2635 SDValue Val = cast<StoreSDNode>(N)->getValue(); 2636 if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode())) 2637 Swap = true; 2638 } 2639 2640 if (Swap) 2641 std::swap(Base, Offset); 2642 2643 AM = ISD::PRE_INC; 2644 return true; 2645 } 2646 2647 // LDU/STU can only handle immediates that are a multiple of 4. 2648 if (VT != MVT::i64) { 2649 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 0)) 2650 return false; 2651 } else { 2652 // LDU/STU need an address with at least 4-byte alignment. 2653 if (Alignment < 4) 2654 return false; 2655 2656 if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 4)) 2657 return false; 2658 } 2659 2660 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 2661 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of 2662 // sext i32 to i64 when addr mode is r+i. 2663 if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 && 2664 LD->getExtensionType() == ISD::SEXTLOAD && 2665 isa<ConstantSDNode>(Offset)) 2666 return false; 2667 } 2668 2669 AM = ISD::PRE_INC; 2670 return true; 2671} 2672 2673//===----------------------------------------------------------------------===// 2674// LowerOperation implementation 2675//===----------------------------------------------------------------------===// 2676 2677/// Return true if we should reference labels using a PICBase, set the HiOpFlags 2678/// and LoOpFlags to the target MO flags. 2679static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget, 2680 unsigned &HiOpFlags, unsigned &LoOpFlags, 2681 const GlobalValue *GV = nullptr) { 2682 HiOpFlags = PPCII::MO_HA; 2683 LoOpFlags = PPCII::MO_LO; 2684 2685 // Don't use the pic base if not in PIC relocation model. 2686 if (IsPIC) { 2687 HiOpFlags |= PPCII::MO_PIC_FLAG; 2688 LoOpFlags |= PPCII::MO_PIC_FLAG; 2689 } 2690 2691 // If this is a reference to a global value that requires a non-lazy-ptr, make 2692 // sure that instruction lowering adds it. 2693 if (GV && Subtarget.hasLazyResolverStub(GV)) { 2694 HiOpFlags |= PPCII::MO_NLP_FLAG; 2695 LoOpFlags |= PPCII::MO_NLP_FLAG; 2696 2697 if (GV->hasHiddenVisibility()) { 2698 HiOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 2699 LoOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; 2700 } 2701 } 2702} 2703 2704static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC, 2705 SelectionDAG &DAG) { 2706 SDLoc DL(HiPart); 2707 EVT PtrVT = HiPart.getValueType(); 2708 SDValue Zero = DAG.getConstant(0, DL, PtrVT); 2709 2710 SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero); 2711 SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero); 2712 2713 // With PIC, the first instruction is actually "GR+hi(&G)". 2714 if (isPIC) 2715 Hi = DAG.getNode(ISD::ADD, DL, PtrVT, 2716 DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi); 2717 2718 // Generate non-pic code that has direct accesses to the constant pool. 2719 // The address of the global is just (hi(&g)+lo(&g)). 2720 return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo); 2721} 2722 2723static void setUsesTOCBasePtr(MachineFunction &MF) { 2724 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2725 FuncInfo->setUsesTOCBasePtr(); 2726} 2727 2728static void setUsesTOCBasePtr(SelectionDAG &DAG) { 2729 setUsesTOCBasePtr(DAG.getMachineFunction()); 2730} 2731 2732SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, 2733 SDValue GA) const { 2734 const bool Is64Bit = Subtarget.isPPC64(); 2735 EVT VT = Is64Bit ? MVT::i64 : MVT::i32; 2736 SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT) 2737 : Subtarget.isAIXABI() 2738 ? DAG.getRegister(PPC::R2, VT) 2739 : DAG.getNode(PPCISD::GlobalBaseReg, dl, VT); 2740 SDValue Ops[] = { GA, Reg }; 2741 return DAG.getMemIntrinsicNode( 2742 PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT, 2743 MachinePointerInfo::getGOT(DAG.getMachineFunction()), 0, 2744 MachineMemOperand::MOLoad); 2745} 2746 2747SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, 2748 SelectionDAG &DAG) const { 2749 EVT PtrVT = Op.getValueType(); 2750 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op); 2751 const Constant *C = CP->getConstVal(); 2752 2753 // 64-bit SVR4 ABI and AIX ABI code are always position-independent. 2754 // The actual address of the GlobalValue is stored in the TOC. 2755 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 2756 setUsesTOCBasePtr(DAG); 2757 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0); 2758 return getTOCEntry(DAG, SDLoc(CP), GA); 2759 } 2760 2761 unsigned MOHiFlag, MOLoFlag; 2762 bool IsPIC = isPositionIndependent(); 2763 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 2764 2765 if (IsPIC && Subtarget.isSVR4ABI()) { 2766 SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 2767 PPCII::MO_PIC_FLAG); 2768 return getTOCEntry(DAG, SDLoc(CP), GA); 2769 } 2770 2771 SDValue CPIHi = 2772 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOHiFlag); 2773 SDValue CPILo = 2774 DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOLoFlag); 2775 return LowerLabelRef(CPIHi, CPILo, IsPIC, DAG); 2776} 2777 2778// For 64-bit PowerPC, prefer the more compact relative encodings. 2779// This trades 32 bits per jump table entry for one or two instructions 2780// on the jump site. 2781unsigned PPCTargetLowering::getJumpTableEncoding() const { 2782 if (isJumpTableRelative()) 2783 return MachineJumpTableInfo::EK_LabelDifference32; 2784 2785 return TargetLowering::getJumpTableEncoding(); 2786} 2787 2788bool PPCTargetLowering::isJumpTableRelative() const { 2789 if (UseAbsoluteJumpTables) 2790 return false; 2791 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) 2792 return true; 2793 return TargetLowering::isJumpTableRelative(); 2794} 2795 2796SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table, 2797 SelectionDAG &DAG) const { 2798 if (!Subtarget.isPPC64() || Subtarget.isAIXABI()) 2799 return TargetLowering::getPICJumpTableRelocBase(Table, DAG); 2800 2801 switch (getTargetMachine().getCodeModel()) { 2802 case CodeModel::Small: 2803 case CodeModel::Medium: 2804 return TargetLowering::getPICJumpTableRelocBase(Table, DAG); 2805 default: 2806 return DAG.getNode(PPCISD::GlobalBaseReg, SDLoc(), 2807 getPointerTy(DAG.getDataLayout())); 2808 } 2809} 2810 2811const MCExpr * 2812PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF, 2813 unsigned JTI, 2814 MCContext &Ctx) const { 2815 if (!Subtarget.isPPC64() || Subtarget.isAIXABI()) 2816 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 2817 2818 switch (getTargetMachine().getCodeModel()) { 2819 case CodeModel::Small: 2820 case CodeModel::Medium: 2821 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); 2822 default: 2823 return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); 2824 } 2825} 2826 2827SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { 2828 EVT PtrVT = Op.getValueType(); 2829 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 2830 2831 // 64-bit SVR4 ABI and AIX ABI code are always position-independent. 2832 // The actual address of the GlobalValue is stored in the TOC. 2833 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 2834 setUsesTOCBasePtr(DAG); 2835 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 2836 return getTOCEntry(DAG, SDLoc(JT), GA); 2837 } 2838 2839 unsigned MOHiFlag, MOLoFlag; 2840 bool IsPIC = isPositionIndependent(); 2841 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 2842 2843 if (IsPIC && Subtarget.isSVR4ABI()) { 2844 SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, 2845 PPCII::MO_PIC_FLAG); 2846 return getTOCEntry(DAG, SDLoc(GA), GA); 2847 } 2848 2849 SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag); 2850 SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag); 2851 return LowerLabelRef(JTIHi, JTILo, IsPIC, DAG); 2852} 2853 2854SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, 2855 SelectionDAG &DAG) const { 2856 EVT PtrVT = Op.getValueType(); 2857 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Op); 2858 const BlockAddress *BA = BASDN->getBlockAddress(); 2859 2860 // 64-bit SVR4 ABI and AIX ABI code are always position-independent. 2861 // The actual BlockAddress is stored in the TOC. 2862 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 2863 setUsesTOCBasePtr(DAG); 2864 SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()); 2865 return getTOCEntry(DAG, SDLoc(BASDN), GA); 2866 } 2867 2868 // 32-bit position-independent ELF stores the BlockAddress in the .got. 2869 if (Subtarget.is32BitELFABI() && isPositionIndependent()) 2870 return getTOCEntry( 2871 DAG, SDLoc(BASDN), 2872 DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset())); 2873 2874 unsigned MOHiFlag, MOLoFlag; 2875 bool IsPIC = isPositionIndependent(); 2876 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); 2877 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag); 2878 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag); 2879 return LowerLabelRef(TgtBAHi, TgtBALo, IsPIC, DAG); 2880} 2881 2882SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, 2883 SelectionDAG &DAG) const { 2884 // FIXME: TLS addresses currently use medium model code sequences, 2885 // which is the most useful form. Eventually support for small and 2886 // large models could be added if users need it, at the cost of 2887 // additional complexity. 2888 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op); 2889 if (DAG.getTarget().useEmulatedTLS()) 2890 return LowerToTLSEmulatedModel(GA, DAG); 2891 2892 SDLoc dl(GA); 2893 const GlobalValue *GV = GA->getGlobal(); 2894 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2895 bool is64bit = Subtarget.isPPC64(); 2896 const Module *M = DAG.getMachineFunction().getFunction().getParent(); 2897 PICLevel::Level picLevel = M->getPICLevel(); 2898 2899 const TargetMachine &TM = getTargetMachine(); 2900 TLSModel::Model Model = TM.getTLSModel(GV); 2901 2902 if (Model == TLSModel::LocalExec) { 2903 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2904 PPCII::MO_TPREL_HA); 2905 SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2906 PPCII::MO_TPREL_LO); 2907 SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64) 2908 : DAG.getRegister(PPC::R2, MVT::i32); 2909 2910 SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg); 2911 return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi); 2912 } 2913 2914 if (Model == TLSModel::InitialExec) { 2915 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2916 SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 2917 PPCII::MO_TLS); 2918 SDValue GOTPtr; 2919 if (is64bit) { 2920 setUsesTOCBasePtr(DAG); 2921 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2922 GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, 2923 PtrVT, GOTReg, TGA); 2924 } else { 2925 if (!TM.isPositionIndependent()) 2926 GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); 2927 else if (picLevel == PICLevel::SmallPIC) 2928 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2929 else 2930 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2931 } 2932 SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, 2933 PtrVT, TGA, GOTPtr); 2934 return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); 2935 } 2936 2937 if (Model == TLSModel::GeneralDynamic) { 2938 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2939 SDValue GOTPtr; 2940 if (is64bit) { 2941 setUsesTOCBasePtr(DAG); 2942 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2943 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT, 2944 GOTReg, TGA); 2945 } else { 2946 if (picLevel == PICLevel::SmallPIC) 2947 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2948 else 2949 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2950 } 2951 return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT, 2952 GOTPtr, TGA, TGA); 2953 } 2954 2955 if (Model == TLSModel::LocalDynamic) { 2956 SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); 2957 SDValue GOTPtr; 2958 if (is64bit) { 2959 setUsesTOCBasePtr(DAG); 2960 SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); 2961 GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT, 2962 GOTReg, TGA); 2963 } else { 2964 if (picLevel == PICLevel::SmallPIC) 2965 GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); 2966 else 2967 GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); 2968 } 2969 SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl, 2970 PtrVT, GOTPtr, TGA, TGA); 2971 SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl, 2972 PtrVT, TLSAddr, TGA); 2973 return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA); 2974 } 2975 2976 llvm_unreachable("Unknown TLS model!"); 2977} 2978 2979SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, 2980 SelectionDAG &DAG) const { 2981 EVT PtrVT = Op.getValueType(); 2982 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op); 2983 SDLoc DL(GSDN); 2984 const GlobalValue *GV = GSDN->getGlobal(); 2985 2986 // 64-bit SVR4 ABI & AIX ABI code is always position-independent. 2987 // The actual address of the GlobalValue is stored in the TOC. 2988 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 2989 setUsesTOCBasePtr(DAG); 2990 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset()); 2991 return getTOCEntry(DAG, DL, GA); 2992 } 2993 2994 unsigned MOHiFlag, MOLoFlag; 2995 bool IsPIC = isPositionIndependent(); 2996 getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag, GV); 2997 2998 if (IsPIC && Subtarget.isSVR4ABI()) { 2999 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 3000 GSDN->getOffset(), 3001 PPCII::MO_PIC_FLAG); 3002 return getTOCEntry(DAG, DL, GA); 3003 } 3004 3005 SDValue GAHi = 3006 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag); 3007 SDValue GALo = 3008 DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag); 3009 3010 SDValue Ptr = LowerLabelRef(GAHi, GALo, IsPIC, DAG); 3011 3012 // If the global reference is actually to a non-lazy-pointer, we have to do an 3013 // extra load to get the address of the global. 3014 if (MOHiFlag & PPCII::MO_NLP_FLAG) 3015 Ptr = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo()); 3016 return Ptr; 3017} 3018 3019SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { 3020 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 3021 SDLoc dl(Op); 3022 3023 if (Op.getValueType() == MVT::v2i64) { 3024 // When the operands themselves are v2i64 values, we need to do something 3025 // special because VSX has no underlying comparison operations for these. 3026 if (Op.getOperand(0).getValueType() == MVT::v2i64) { 3027 // Equality can be handled by casting to the legal type for Altivec 3028 // comparisons, everything else needs to be expanded. 3029 if (CC == ISD::SETEQ || CC == ISD::SETNE) { 3030 return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, 3031 DAG.getSetCC(dl, MVT::v4i32, 3032 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)), 3033 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)), 3034 CC)); 3035 } 3036 3037 return SDValue(); 3038 } 3039 3040 // We handle most of these in the usual way. 3041 return Op; 3042 } 3043 3044 // If we're comparing for equality to zero, expose the fact that this is 3045 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can 3046 // fold the new nodes. 3047 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG)) 3048 return V; 3049 3050 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 3051 // Leave comparisons against 0 and -1 alone for now, since they're usually 3052 // optimized. FIXME: revisit this when we can custom lower all setcc 3053 // optimizations. 3054 if (C->isAllOnesValue() || C->isNullValue()) 3055 return SDValue(); 3056 } 3057 3058 // If we have an integer seteq/setne, turn it into a compare against zero 3059 // by xor'ing the rhs with the lhs, which is faster than setting a 3060 // condition register, reading it back out, and masking the correct bit. The 3061 // normal approach here uses sub to do this instead of xor. Using xor exposes 3062 // the result to other bit-twiddling opportunities. 3063 EVT LHSVT = Op.getOperand(0).getValueType(); 3064 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { 3065 EVT VT = Op.getValueType(); 3066 SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0), 3067 Op.getOperand(1)); 3068 return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC); 3069 } 3070 return SDValue(); 3071} 3072 3073SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { 3074 SDNode *Node = Op.getNode(); 3075 EVT VT = Node->getValueType(0); 3076 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3077 SDValue InChain = Node->getOperand(0); 3078 SDValue VAListPtr = Node->getOperand(1); 3079 const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); 3080 SDLoc dl(Node); 3081 3082 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"); 3083 3084 // gpr_index 3085 SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 3086 VAListPtr, MachinePointerInfo(SV), MVT::i8); 3087 InChain = GprIndex.getValue(1); 3088 3089 if (VT == MVT::i64) { 3090 // Check if GprIndex is even 3091 SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex, 3092 DAG.getConstant(1, dl, MVT::i32)); 3093 SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd, 3094 DAG.getConstant(0, dl, MVT::i32), ISD::SETNE); 3095 SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex, 3096 DAG.getConstant(1, dl, MVT::i32)); 3097 // Align GprIndex to be even if it isn't 3098 GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne, 3099 GprIndex); 3100 } 3101 3102 // fpr index is 1 byte after gpr 3103 SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3104 DAG.getConstant(1, dl, MVT::i32)); 3105 3106 // fpr 3107 SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, 3108 FprPtr, MachinePointerInfo(SV), MVT::i8); 3109 InChain = FprIndex.getValue(1); 3110 3111 SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3112 DAG.getConstant(8, dl, MVT::i32)); 3113 3114 SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, 3115 DAG.getConstant(4, dl, MVT::i32)); 3116 3117 // areas 3118 SDValue OverflowArea = 3119 DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, MachinePointerInfo()); 3120 InChain = OverflowArea.getValue(1); 3121 3122 SDValue RegSaveArea = 3123 DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, MachinePointerInfo()); 3124 InChain = RegSaveArea.getValue(1); 3125 3126 // select overflow_area if index > 8 3127 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, 3128 DAG.getConstant(8, dl, MVT::i32), ISD::SETLT); 3129 3130 // adjustment constant gpr_index * 4/8 3131 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32, 3132 VT.isInteger() ? GprIndex : FprIndex, 3133 DAG.getConstant(VT.isInteger() ? 4 : 8, dl, 3134 MVT::i32)); 3135 3136 // OurReg = RegSaveArea + RegConstant 3137 SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea, 3138 RegConstant); 3139 3140 // Floating types are 32 bytes into RegSaveArea 3141 if (VT.isFloatingPoint()) 3142 OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg, 3143 DAG.getConstant(32, dl, MVT::i32)); 3144 3145 // increase {f,g}pr_index by 1 (or 2 if VT is i64) 3146 SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32, 3147 VT.isInteger() ? GprIndex : FprIndex, 3148 DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl, 3149 MVT::i32)); 3150 3151 InChain = DAG.getTruncStore(InChain, dl, IndexPlus1, 3152 VT.isInteger() ? VAListPtr : FprPtr, 3153 MachinePointerInfo(SV), MVT::i8); 3154 3155 // determine if we should load from reg_save_area or overflow_area 3156 SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea); 3157 3158 // increase overflow_area by 4/8 if gpr/fpr > 8 3159 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea, 3160 DAG.getConstant(VT.isInteger() ? 4 : 8, 3161 dl, MVT::i32)); 3162 3163 OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea, 3164 OverflowAreaPlusN); 3165 3166 InChain = DAG.getTruncStore(InChain, dl, OverflowArea, OverflowAreaPtr, 3167 MachinePointerInfo(), MVT::i32); 3168 3169 return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo()); 3170} 3171 3172SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { 3173 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"); 3174 3175 // We have to copy the entire va_list struct: 3176 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte 3177 return DAG.getMemcpy(Op.getOperand(0), Op, 3178 Op.getOperand(1), Op.getOperand(2), 3179 DAG.getConstant(12, SDLoc(Op), MVT::i32), 8, false, true, 3180 false, MachinePointerInfo(), MachinePointerInfo()); 3181} 3182 3183SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, 3184 SelectionDAG &DAG) const { 3185 if (Subtarget.isAIXABI()) 3186 report_fatal_error("ADJUST_TRAMPOLINE operation is not supported on AIX."); 3187 3188 return Op.getOperand(0); 3189} 3190 3191SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, 3192 SelectionDAG &DAG) const { 3193 if (Subtarget.isAIXABI()) 3194 report_fatal_error("INIT_TRAMPOLINE operation is not supported on AIX."); 3195 3196 SDValue Chain = Op.getOperand(0); 3197 SDValue Trmp = Op.getOperand(1); // trampoline 3198 SDValue FPtr = Op.getOperand(2); // nested function 3199 SDValue Nest = Op.getOperand(3); // 'nest' parameter value 3200 SDLoc dl(Op); 3201 3202 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 3203 bool isPPC64 = (PtrVT == MVT::i64); 3204 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); 3205 3206 TargetLowering::ArgListTy Args; 3207 TargetLowering::ArgListEntry Entry; 3208 3209 Entry.Ty = IntPtrTy; 3210 Entry.Node = Trmp; Args.push_back(Entry); 3211 3212 // TrampSize == (isPPC64 ? 48 : 40); 3213 Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl, 3214 isPPC64 ? MVT::i64 : MVT::i32); 3215 Args.push_back(Entry); 3216 3217 Entry.Node = FPtr; Args.push_back(Entry); 3218 Entry.Node = Nest; Args.push_back(Entry); 3219 3220 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg) 3221 TargetLowering::CallLoweringInfo CLI(DAG); 3222 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee( 3223 CallingConv::C, Type::getVoidTy(*DAG.getContext()), 3224 DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args)); 3225 3226 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI); 3227 return CallResult.second; 3228} 3229 3230SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { 3231 MachineFunction &MF = DAG.getMachineFunction(); 3232 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3233 EVT PtrVT = getPointerTy(MF.getDataLayout()); 3234 3235 SDLoc dl(Op); 3236 3237 if (Subtarget.isDarwinABI() || Subtarget.isPPC64()) { 3238 // vastart just stores the address of the VarArgsFrameIndex slot into the 3239 // memory location argument. 3240 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3241 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3242 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), 3243 MachinePointerInfo(SV)); 3244 } 3245 3246 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct. 3247 // We suppose the given va_list is already allocated. 3248 // 3249 // typedef struct { 3250 // char gpr; /* index into the array of 8 GPRs 3251 // * stored in the register save area 3252 // * gpr=0 corresponds to r3, 3253 // * gpr=1 to r4, etc. 3254 // */ 3255 // char fpr; /* index into the array of 8 FPRs 3256 // * stored in the register save area 3257 // * fpr=0 corresponds to f1, 3258 // * fpr=1 to f2, etc. 3259 // */ 3260 // char *overflow_arg_area; 3261 // /* location on stack that holds 3262 // * the next overflow argument 3263 // */ 3264 // char *reg_save_area; 3265 // /* where r3:r10 and f1:f8 (if saved) 3266 // * are stored 3267 // */ 3268 // } va_list[1]; 3269 3270 SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32); 3271 SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32); 3272 SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(), 3273 PtrVT); 3274 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), 3275 PtrVT); 3276 3277 uint64_t FrameOffset = PtrVT.getSizeInBits()/8; 3278 SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT); 3279 3280 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1; 3281 SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT); 3282 3283 uint64_t FPROffset = 1; 3284 SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT); 3285 3286 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 3287 3288 // Store first byte : number of int regs 3289 SDValue firstStore = 3290 DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, Op.getOperand(1), 3291 MachinePointerInfo(SV), MVT::i8); 3292 uint64_t nextOffset = FPROffset; 3293 SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1), 3294 ConstFPROffset); 3295 3296 // Store second byte : number of float regs 3297 SDValue secondStore = 3298 DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr, 3299 MachinePointerInfo(SV, nextOffset), MVT::i8); 3300 nextOffset += StackOffset; 3301 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset); 3302 3303 // Store second word : arguments given on stack 3304 SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr, 3305 MachinePointerInfo(SV, nextOffset)); 3306 nextOffset += FrameOffset; 3307 nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset); 3308 3309 // Store third word : arguments given in registers 3310 return DAG.getStore(thirdStore, dl, FR, nextPtr, 3311 MachinePointerInfo(SV, nextOffset)); 3312} 3313 3314/// FPR - The set of FP registers that should be allocated for arguments 3315/// on Darwin and AIX. 3316static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, 3317 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10, 3318 PPC::F11, PPC::F12, PPC::F13}; 3319 3320/// QFPR - The set of QPX registers that should be allocated for arguments. 3321static const MCPhysReg QFPR[] = { 3322 PPC::QF1, PPC::QF2, PPC::QF3, PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7, 3323 PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13}; 3324 3325/// CalculateStackSlotSize - Calculates the size reserved for this argument on 3326/// the stack. 3327static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, 3328 unsigned PtrByteSize) { 3329 unsigned ArgSize = ArgVT.getStoreSize(); 3330 if (Flags.isByVal()) 3331 ArgSize = Flags.getByValSize(); 3332 3333 // Round up to multiples of the pointer size, except for array members, 3334 // which are always packed. 3335 if (!Flags.isInConsecutiveRegs()) 3336 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3337 3338 return ArgSize; 3339} 3340 3341/// CalculateStackSlotAlignment - Calculates the alignment of this argument 3342/// on the stack. 3343static unsigned CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, 3344 ISD::ArgFlagsTy Flags, 3345 unsigned PtrByteSize) { 3346 unsigned Align = PtrByteSize; 3347 3348 // Altivec parameters are padded to a 16 byte boundary. 3349 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 3350 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 3351 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 3352 ArgVT == MVT::v1i128 || ArgVT == MVT::f128) 3353 Align = 16; 3354 // QPX vector types stored in double-precision are padded to a 32 byte 3355 // boundary. 3356 else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1) 3357 Align = 32; 3358 3359 // ByVal parameters are aligned as requested. 3360 if (Flags.isByVal()) { 3361 unsigned BVAlign = Flags.getByValAlign(); 3362 if (BVAlign > PtrByteSize) { 3363 if (BVAlign % PtrByteSize != 0) 3364 llvm_unreachable( 3365 "ByVal alignment is not a multiple of the pointer size"); 3366 3367 Align = BVAlign; 3368 } 3369 } 3370 3371 // Array members are always packed to their original alignment. 3372 if (Flags.isInConsecutiveRegs()) { 3373 // If the array member was split into multiple registers, the first 3374 // needs to be aligned to the size of the full type. (Except for 3375 // ppcf128, which is only aligned as its f64 components.) 3376 if (Flags.isSplit() && OrigVT != MVT::ppcf128) 3377 Align = OrigVT.getStoreSize(); 3378 else 3379 Align = ArgVT.getStoreSize(); 3380 } 3381 3382 return Align; 3383} 3384 3385/// CalculateStackSlotUsed - Return whether this argument will use its 3386/// stack slot (instead of being passed in registers). ArgOffset, 3387/// AvailableFPRs, and AvailableVRs must hold the current argument 3388/// position, and will be updated to account for this argument. 3389static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, 3390 ISD::ArgFlagsTy Flags, 3391 unsigned PtrByteSize, 3392 unsigned LinkageSize, 3393 unsigned ParamAreaSize, 3394 unsigned &ArgOffset, 3395 unsigned &AvailableFPRs, 3396 unsigned &AvailableVRs, bool HasQPX) { 3397 bool UseMemory = false; 3398 3399 // Respect alignment of argument on the stack. 3400 unsigned Align = 3401 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 3402 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 3403 // If there's no space left in the argument save area, we must 3404 // use memory (this check also catches zero-sized arguments). 3405 if (ArgOffset >= LinkageSize + ParamAreaSize) 3406 UseMemory = true; 3407 3408 // Allocate argument on the stack. 3409 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 3410 if (Flags.isInConsecutiveRegsLast()) 3411 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3412 // If we overran the argument save area, we must use memory 3413 // (this check catches arguments passed partially in memory) 3414 if (ArgOffset > LinkageSize + ParamAreaSize) 3415 UseMemory = true; 3416 3417 // However, if the argument is actually passed in an FPR or a VR, 3418 // we don't use memory after all. 3419 if (!Flags.isByVal()) { 3420 if (ArgVT == MVT::f32 || ArgVT == MVT::f64 || 3421 // QPX registers overlap with the scalar FP registers. 3422 (HasQPX && (ArgVT == MVT::v4f32 || 3423 ArgVT == MVT::v4f64 || 3424 ArgVT == MVT::v4i1))) 3425 if (AvailableFPRs > 0) { 3426 --AvailableFPRs; 3427 return false; 3428 } 3429 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 3430 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 3431 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || 3432 ArgVT == MVT::v1i128 || ArgVT == MVT::f128) 3433 if (AvailableVRs > 0) { 3434 --AvailableVRs; 3435 return false; 3436 } 3437 } 3438 3439 return UseMemory; 3440} 3441 3442/// EnsureStackAlignment - Round stack frame size up from NumBytes to 3443/// ensure minimum alignment required for target. 3444static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering, 3445 unsigned NumBytes) { 3446 unsigned TargetAlign = Lowering->getStackAlignment(); 3447 unsigned AlignMask = TargetAlign - 1; 3448 NumBytes = (NumBytes + AlignMask) & ~AlignMask; 3449 return NumBytes; 3450} 3451 3452SDValue PPCTargetLowering::LowerFormalArguments( 3453 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3454 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3455 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3456 if (Subtarget.isAIXABI()) 3457 return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG, 3458 InVals); 3459 if (Subtarget.is64BitELFABI()) 3460 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, 3461 InVals); 3462 if (Subtarget.is32BitELFABI()) 3463 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, 3464 InVals); 3465 3466 return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins, dl, DAG, 3467 InVals); 3468} 3469 3470SDValue PPCTargetLowering::LowerFormalArguments_32SVR4( 3471 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3472 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3473 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3474 3475 // 32-bit SVR4 ABI Stack Frame Layout: 3476 // +-----------------------------------+ 3477 // +--> | Back chain | 3478 // | +-----------------------------------+ 3479 // | | Floating-point register save area | 3480 // | +-----------------------------------+ 3481 // | | General register save area | 3482 // | +-----------------------------------+ 3483 // | | CR save word | 3484 // | +-----------------------------------+ 3485 // | | VRSAVE save word | 3486 // | +-----------------------------------+ 3487 // | | Alignment padding | 3488 // | +-----------------------------------+ 3489 // | | Vector register save area | 3490 // | +-----------------------------------+ 3491 // | | Local variable space | 3492 // | +-----------------------------------+ 3493 // | | Parameter list area | 3494 // | +-----------------------------------+ 3495 // | | LR save word | 3496 // | +-----------------------------------+ 3497 // SP--> +--- | Back chain | 3498 // +-----------------------------------+ 3499 // 3500 // Specifications: 3501 // System V Application Binary Interface PowerPC Processor Supplement 3502 // AltiVec Technology Programming Interface Manual 3503 3504 MachineFunction &MF = DAG.getMachineFunction(); 3505 MachineFrameInfo &MFI = MF.getFrameInfo(); 3506 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3507 3508 EVT PtrVT = getPointerTy(MF.getDataLayout()); 3509 // Potential tail calls could cause overwriting of argument stack slots. 3510 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 3511 (CallConv == CallingConv::Fast)); 3512 unsigned PtrByteSize = 4; 3513 3514 // Assign locations to all of the incoming arguments. 3515 SmallVector<CCValAssign, 16> ArgLocs; 3516 PPCCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, 3517 *DAG.getContext()); 3518 3519 // Reserve space for the linkage area on the stack. 3520 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 3521 CCInfo.AllocateStack(LinkageSize, PtrByteSize); 3522 if (useSoftFloat()) 3523 CCInfo.PreAnalyzeFormalArguments(Ins); 3524 3525 CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4); 3526 CCInfo.clearWasPPCF128(); 3527 3528 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 3529 CCValAssign &VA = ArgLocs[i]; 3530 3531 // Arguments stored in registers. 3532 if (VA.isRegLoc()) { 3533 const TargetRegisterClass *RC; 3534 EVT ValVT = VA.getValVT(); 3535 3536 switch (ValVT.getSimpleVT().SimpleTy) { 3537 default: 3538 llvm_unreachable("ValVT not supported by formal arguments Lowering"); 3539 case MVT::i1: 3540 case MVT::i32: 3541 RC = &PPC::GPRCRegClass; 3542 break; 3543 case MVT::f32: 3544 if (Subtarget.hasP8Vector()) 3545 RC = &PPC::VSSRCRegClass; 3546 else if (Subtarget.hasSPE()) 3547 RC = &PPC::GPRCRegClass; 3548 else 3549 RC = &PPC::F4RCRegClass; 3550 break; 3551 case MVT::f64: 3552 if (Subtarget.hasVSX()) 3553 RC = &PPC::VSFRCRegClass; 3554 else if (Subtarget.hasSPE()) 3555 // SPE passes doubles in GPR pairs. 3556 RC = &PPC::GPRCRegClass; 3557 else 3558 RC = &PPC::F8RCRegClass; 3559 break; 3560 case MVT::v16i8: 3561 case MVT::v8i16: 3562 case MVT::v4i32: 3563 RC = &PPC::VRRCRegClass; 3564 break; 3565 case MVT::v4f32: 3566 RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass; 3567 break; 3568 case MVT::v2f64: 3569 case MVT::v2i64: 3570 RC = &PPC::VRRCRegClass; 3571 break; 3572 case MVT::v4f64: 3573 RC = &PPC::QFRCRegClass; 3574 break; 3575 case MVT::v4i1: 3576 RC = &PPC::QBRCRegClass; 3577 break; 3578 } 3579 3580 SDValue ArgValue; 3581 // Transform the arguments stored in physical registers into 3582 // virtual ones. 3583 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) { 3584 assert(i + 1 < e && "No second half of double precision argument"); 3585 unsigned RegLo = MF.addLiveIn(VA.getLocReg(), RC); 3586 unsigned RegHi = MF.addLiveIn(ArgLocs[++i].getLocReg(), RC); 3587 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32); 3588 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32); 3589 if (!Subtarget.isLittleEndian()) 3590 std::swap (ArgValueLo, ArgValueHi); 3591 ArgValue = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, ArgValueLo, 3592 ArgValueHi); 3593 } else { 3594 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); 3595 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, 3596 ValVT == MVT::i1 ? MVT::i32 : ValVT); 3597 if (ValVT == MVT::i1) 3598 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue); 3599 } 3600 3601 InVals.push_back(ArgValue); 3602 } else { 3603 // Argument stored in memory. 3604 assert(VA.isMemLoc()); 3605 3606 // Get the extended size of the argument type in stack 3607 unsigned ArgSize = VA.getLocVT().getStoreSize(); 3608 // Get the actual size of the argument type 3609 unsigned ObjSize = VA.getValVT().getStoreSize(); 3610 unsigned ArgOffset = VA.getLocMemOffset(); 3611 // Stack objects in PPC32 are right justified. 3612 ArgOffset += ArgSize - ObjSize; 3613 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, isImmutable); 3614 3615 // Create load nodes to retrieve arguments from the stack. 3616 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3617 InVals.push_back( 3618 DAG.getLoad(VA.getValVT(), dl, Chain, FIN, MachinePointerInfo())); 3619 } 3620 } 3621 3622 // Assign locations to all of the incoming aggregate by value arguments. 3623 // Aggregates passed by value are stored in the local variable space of the 3624 // caller's stack frame, right above the parameter list area. 3625 SmallVector<CCValAssign, 16> ByValArgLocs; 3626 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), 3627 ByValArgLocs, *DAG.getContext()); 3628 3629 // Reserve stack space for the allocations in CCInfo. 3630 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 3631 3632 CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal); 3633 3634 // Area that is at least reserved in the caller of this function. 3635 unsigned MinReservedArea = CCByValInfo.getNextStackOffset(); 3636 MinReservedArea = std::max(MinReservedArea, LinkageSize); 3637 3638 // Set the size that is at least reserved in caller of this function. Tail 3639 // call optimized function's reserved stack space needs to be aligned so that 3640 // taking the difference between two stack areas will result in an aligned 3641 // stack. 3642 MinReservedArea = 3643 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 3644 FuncInfo->setMinReservedArea(MinReservedArea); 3645 3646 SmallVector<SDValue, 8> MemOps; 3647 3648 // If the function takes variable number of arguments, make a frame index for 3649 // the start of the first vararg value... for expansion of llvm.va_start. 3650 if (isVarArg) { 3651 static const MCPhysReg GPArgRegs[] = { 3652 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 3653 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 3654 }; 3655 const unsigned NumGPArgRegs = array_lengthof(GPArgRegs); 3656 3657 static const MCPhysReg FPArgRegs[] = { 3658 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, 3659 PPC::F8 3660 }; 3661 unsigned NumFPArgRegs = array_lengthof(FPArgRegs); 3662 3663 if (useSoftFloat() || hasSPE()) 3664 NumFPArgRegs = 0; 3665 3666 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs)); 3667 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs)); 3668 3669 // Make room for NumGPArgRegs and NumFPArgRegs. 3670 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 + 3671 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8; 3672 3673 FuncInfo->setVarArgsStackOffset( 3674 MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, 3675 CCInfo.getNextStackOffset(), true)); 3676 3677 FuncInfo->setVarArgsFrameIndex(MFI.CreateStackObject(Depth, 8, false)); 3678 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 3679 3680 // The fixed integer arguments of a variadic function are stored to the 3681 // VarArgsFrameIndex on the stack so that they may be loaded by 3682 // dereferencing the result of va_next. 3683 for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) { 3684 // Get an existing live-in vreg, or add a new one. 3685 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]); 3686 if (!VReg) 3687 VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass); 3688 3689 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3690 SDValue Store = 3691 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 3692 MemOps.push_back(Store); 3693 // Increment the address by four for the next argument to store 3694 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); 3695 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3696 } 3697 3698 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6 3699 // is set. 3700 // The double arguments are stored to the VarArgsFrameIndex 3701 // on the stack. 3702 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) { 3703 // Get an existing live-in vreg, or add a new one. 3704 unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]); 3705 if (!VReg) 3706 VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass); 3707 3708 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64); 3709 SDValue Store = 3710 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 3711 MemOps.push_back(Store); 3712 // Increment the address by eight for the next argument to store 3713 SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl, 3714 PtrVT); 3715 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 3716 } 3717 } 3718 3719 if (!MemOps.empty()) 3720 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 3721 3722 return Chain; 3723} 3724 3725// PPC64 passes i8, i16, and i32 values in i64 registers. Promote 3726// value to MVT::i64 and then truncate to the correct register size. 3727SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, 3728 EVT ObjectVT, SelectionDAG &DAG, 3729 SDValue ArgVal, 3730 const SDLoc &dl) const { 3731 if (Flags.isSExt()) 3732 ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal, 3733 DAG.getValueType(ObjectVT)); 3734 else if (Flags.isZExt()) 3735 ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal, 3736 DAG.getValueType(ObjectVT)); 3737 3738 return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal); 3739} 3740 3741SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( 3742 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 3743 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 3744 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 3745 // TODO: add description of PPC stack frame format, or at least some docs. 3746 // 3747 bool isELFv2ABI = Subtarget.isELFv2ABI(); 3748 bool isLittleEndian = Subtarget.isLittleEndian(); 3749 MachineFunction &MF = DAG.getMachineFunction(); 3750 MachineFrameInfo &MFI = MF.getFrameInfo(); 3751 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 3752 3753 assert(!(CallConv == CallingConv::Fast && isVarArg) && 3754 "fastcc not supported on varargs functions"); 3755 3756 EVT PtrVT = getPointerTy(MF.getDataLayout()); 3757 // Potential tail calls could cause overwriting of argument stack slots. 3758 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 3759 (CallConv == CallingConv::Fast)); 3760 unsigned PtrByteSize = 8; 3761 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 3762 3763 static const MCPhysReg GPR[] = { 3764 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 3765 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 3766 }; 3767 static const MCPhysReg VR[] = { 3768 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 3769 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 3770 }; 3771 3772 const unsigned Num_GPR_Regs = array_lengthof(GPR); 3773 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; 3774 const unsigned Num_VR_Regs = array_lengthof(VR); 3775 const unsigned Num_QFPR_Regs = Num_FPR_Regs; 3776 3777 // Do a first pass over the arguments to determine whether the ABI 3778 // guarantees that our caller has allocated the parameter save area 3779 // on its stack frame. In the ELFv1 ABI, this is always the case; 3780 // in the ELFv2 ABI, it is true if this is a vararg function or if 3781 // any parameter is located in a stack slot. 3782 3783 bool HasParameterArea = !isELFv2ABI || isVarArg; 3784 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize; 3785 unsigned NumBytes = LinkageSize; 3786 unsigned AvailableFPRs = Num_FPR_Regs; 3787 unsigned AvailableVRs = Num_VR_Regs; 3788 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 3789 if (Ins[i].Flags.isNest()) 3790 continue; 3791 3792 if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags, 3793 PtrByteSize, LinkageSize, ParamAreaSize, 3794 NumBytes, AvailableFPRs, AvailableVRs, 3795 Subtarget.hasQPX())) 3796 HasParameterArea = true; 3797 } 3798 3799 // Add DAG nodes to load the arguments or copy them out of registers. On 3800 // entry to a function on PPC, the arguments start after the linkage area, 3801 // although the first ones are often in registers. 3802 3803 unsigned ArgOffset = LinkageSize; 3804 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 3805 unsigned &QFPR_idx = FPR_idx; 3806 SmallVector<SDValue, 8> MemOps; 3807 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); 3808 unsigned CurArgIdx = 0; 3809 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 3810 SDValue ArgVal; 3811 bool needsLoad = false; 3812 EVT ObjectVT = Ins[ArgNo].VT; 3813 EVT OrigVT = Ins[ArgNo].ArgVT; 3814 unsigned ObjSize = ObjectVT.getStoreSize(); 3815 unsigned ArgSize = ObjSize; 3816 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 3817 if (Ins[ArgNo].isOrigArg()) { 3818 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); 3819 CurArgIdx = Ins[ArgNo].getOrigArgIndex(); 3820 } 3821 // We re-align the argument offset for each argument, except when using the 3822 // fast calling convention, when we need to make sure we do that only when 3823 // we'll actually use a stack slot. 3824 unsigned CurArgOffset, Align; 3825 auto ComputeArgOffset = [&]() { 3826 /* Respect alignment of argument on the stack. */ 3827 Align = CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize); 3828 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 3829 CurArgOffset = ArgOffset; 3830 }; 3831 3832 if (CallConv != CallingConv::Fast) { 3833 ComputeArgOffset(); 3834 3835 /* Compute GPR index associated with argument offset. */ 3836 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 3837 GPR_idx = std::min(GPR_idx, Num_GPR_Regs); 3838 } 3839 3840 // FIXME the codegen can be much improved in some cases. 3841 // We do not have to keep everything in memory. 3842 if (Flags.isByVal()) { 3843 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); 3844 3845 if (CallConv == CallingConv::Fast) 3846 ComputeArgOffset(); 3847 3848 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 3849 ObjSize = Flags.getByValSize(); 3850 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 3851 // Empty aggregate parameters do not take up registers. Examples: 3852 // struct { } a; 3853 // union { } b; 3854 // int c[0]; 3855 // etc. However, we have to provide a place-holder in InVals, so 3856 // pretend we have an 8-byte item at the current address for that 3857 // purpose. 3858 if (!ObjSize) { 3859 int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); 3860 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3861 InVals.push_back(FIN); 3862 continue; 3863 } 3864 3865 // Create a stack object covering all stack doublewords occupied 3866 // by the argument. If the argument is (fully or partially) on 3867 // the stack, or if the argument is fully in registers but the 3868 // caller has allocated the parameter save anyway, we can refer 3869 // directly to the caller's stack frame. Otherwise, create a 3870 // local copy in our own frame. 3871 int FI; 3872 if (HasParameterArea || 3873 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize) 3874 FI = MFI.CreateFixedObject(ArgSize, ArgOffset, false, true); 3875 else 3876 FI = MFI.CreateStackObject(ArgSize, Align, false); 3877 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 3878 3879 // Handle aggregates smaller than 8 bytes. 3880 if (ObjSize < PtrByteSize) { 3881 // The value of the object is its address, which differs from the 3882 // address of the enclosing doubleword on big-endian systems. 3883 SDValue Arg = FIN; 3884 if (!isLittleEndian) { 3885 SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT); 3886 Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff); 3887 } 3888 InVals.push_back(Arg); 3889 3890 if (GPR_idx != Num_GPR_Regs) { 3891 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3892 FuncInfo->addLiveInAttr(VReg, Flags); 3893 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3894 SDValue Store; 3895 3896 if (ObjSize==1 || ObjSize==2 || ObjSize==4) { 3897 EVT ObjType = (ObjSize == 1 ? MVT::i8 : 3898 (ObjSize == 2 ? MVT::i16 : MVT::i32)); 3899 Store = DAG.getTruncStore(Val.getValue(1), dl, Val, Arg, 3900 MachinePointerInfo(&*FuncArg), ObjType); 3901 } else { 3902 // For sizes that don't fit a truncating store (3, 5, 6, 7), 3903 // store the whole register as-is to the parameter save area 3904 // slot. 3905 Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 3906 MachinePointerInfo(&*FuncArg)); 3907 } 3908 3909 MemOps.push_back(Store); 3910 } 3911 // Whether we copied from a register or not, advance the offset 3912 // into the parameter save area by a full doubleword. 3913 ArgOffset += PtrByteSize; 3914 continue; 3915 } 3916 3917 // The value of the object is its address, which is the address of 3918 // its first stack doubleword. 3919 InVals.push_back(FIN); 3920 3921 // Store whatever pieces of the object are in registers to memory. 3922 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 3923 if (GPR_idx == Num_GPR_Regs) 3924 break; 3925 3926 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 3927 FuncInfo->addLiveInAttr(VReg, Flags); 3928 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 3929 SDValue Addr = FIN; 3930 if (j) { 3931 SDValue Off = DAG.getConstant(j, dl, PtrVT); 3932 Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off); 3933 } 3934 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr, 3935 MachinePointerInfo(&*FuncArg, j)); 3936 MemOps.push_back(Store); 3937 ++GPR_idx; 3938 } 3939 ArgOffset += ArgSize; 3940 continue; 3941 } 3942 3943 switch (ObjectVT.getSimpleVT().SimpleTy) { 3944 default: llvm_unreachable("Unhandled argument type!"); 3945 case MVT::i1: 3946 case MVT::i32: 3947 case MVT::i64: 3948 if (Flags.isNest()) { 3949 // The 'nest' parameter, if any, is passed in R11. 3950 unsigned VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass); 3951 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3952 3953 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 3954 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 3955 3956 break; 3957 } 3958 3959 // These can be scalar arguments or elements of an integer array type 3960 // passed directly. Clang may use those instead of "byval" aggregate 3961 // types to avoid forcing arguments to memory unnecessarily. 3962 if (GPR_idx != Num_GPR_Regs) { 3963 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 3964 FuncInfo->addLiveInAttr(VReg, Flags); 3965 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 3966 3967 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 3968 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 3969 // value to MVT::i64 and then truncate to the correct register size. 3970 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 3971 } else { 3972 if (CallConv == CallingConv::Fast) 3973 ComputeArgOffset(); 3974 3975 needsLoad = true; 3976 ArgSize = PtrByteSize; 3977 } 3978 if (CallConv != CallingConv::Fast || needsLoad) 3979 ArgOffset += 8; 3980 break; 3981 3982 case MVT::f32: 3983 case MVT::f64: 3984 // These can be scalar arguments or elements of a float array type 3985 // passed directly. The latter are used to implement ELFv2 homogenous 3986 // float aggregates. 3987 if (FPR_idx != Num_FPR_Regs) { 3988 unsigned VReg; 3989 3990 if (ObjectVT == MVT::f32) 3991 VReg = MF.addLiveIn(FPR[FPR_idx], 3992 Subtarget.hasP8Vector() 3993 ? &PPC::VSSRCRegClass 3994 : &PPC::F4RCRegClass); 3995 else 3996 VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX() 3997 ? &PPC::VSFRCRegClass 3998 : &PPC::F8RCRegClass); 3999 4000 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4001 ++FPR_idx; 4002 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) { 4003 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 4004 // once we support fp <-> gpr moves. 4005 4006 // This can only ever happen in the presence of f32 array types, 4007 // since otherwise we never run out of FPRs before running out 4008 // of GPRs. 4009 unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); 4010 FuncInfo->addLiveInAttr(VReg, Flags); 4011 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 4012 4013 if (ObjectVT == MVT::f32) { 4014 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0)) 4015 ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal, 4016 DAG.getConstant(32, dl, MVT::i32)); 4017 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal); 4018 } 4019 4020 ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal); 4021 } else { 4022 if (CallConv == CallingConv::Fast) 4023 ComputeArgOffset(); 4024 4025 needsLoad = true; 4026 } 4027 4028 // When passing an array of floats, the array occupies consecutive 4029 // space in the argument area; only round up to the next doubleword 4030 // at the end of the array. Otherwise, each float takes 8 bytes. 4031 if (CallConv != CallingConv::Fast || needsLoad) { 4032 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize; 4033 ArgOffset += ArgSize; 4034 if (Flags.isInConsecutiveRegsLast()) 4035 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4036 } 4037 break; 4038 case MVT::v4f32: 4039 case MVT::v4i32: 4040 case MVT::v8i16: 4041 case MVT::v16i8: 4042 case MVT::v2f64: 4043 case MVT::v2i64: 4044 case MVT::v1i128: 4045 case MVT::f128: 4046 if (!Subtarget.hasQPX()) { 4047 // These can be scalar arguments or elements of a vector array type 4048 // passed directly. The latter are used to implement ELFv2 homogenous 4049 // vector aggregates. 4050 if (VR_idx != Num_VR_Regs) { 4051 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 4052 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4053 ++VR_idx; 4054 } else { 4055 if (CallConv == CallingConv::Fast) 4056 ComputeArgOffset(); 4057 needsLoad = true; 4058 } 4059 if (CallConv != CallingConv::Fast || needsLoad) 4060 ArgOffset += 16; 4061 break; 4062 } // not QPX 4063 4064 assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 && 4065 "Invalid QPX parameter type"); 4066 LLVM_FALLTHROUGH; 4067 4068 case MVT::v4f64: 4069 case MVT::v4i1: 4070 // QPX vectors are treated like their scalar floating-point subregisters 4071 // (except that they're larger). 4072 unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32; 4073 if (QFPR_idx != Num_QFPR_Regs) { 4074 const TargetRegisterClass *RC; 4075 switch (ObjectVT.getSimpleVT().SimpleTy) { 4076 case MVT::v4f64: RC = &PPC::QFRCRegClass; break; 4077 case MVT::v4f32: RC = &PPC::QSRCRegClass; break; 4078 default: RC = &PPC::QBRCRegClass; break; 4079 } 4080 4081 unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC); 4082 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4083 ++QFPR_idx; 4084 } else { 4085 if (CallConv == CallingConv::Fast) 4086 ComputeArgOffset(); 4087 needsLoad = true; 4088 } 4089 if (CallConv != CallingConv::Fast || needsLoad) 4090 ArgOffset += Sz; 4091 break; 4092 } 4093 4094 // We need to load the argument to a virtual register if we determined 4095 // above that we ran out of physical registers of the appropriate type. 4096 if (needsLoad) { 4097 if (ObjSize < ArgSize && !isLittleEndian) 4098 CurArgOffset += ArgSize - ObjSize; 4099 int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, isImmutable); 4100 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4101 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); 4102 } 4103 4104 InVals.push_back(ArgVal); 4105 } 4106 4107 // Area that is at least reserved in the caller of this function. 4108 unsigned MinReservedArea; 4109 if (HasParameterArea) 4110 MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize); 4111 else 4112 MinReservedArea = LinkageSize; 4113 4114 // Set the size that is at least reserved in caller of this function. Tail 4115 // call optimized functions' reserved stack space needs to be aligned so that 4116 // taking the difference between two stack areas will result in an aligned 4117 // stack. 4118 MinReservedArea = 4119 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 4120 FuncInfo->setMinReservedArea(MinReservedArea); 4121 4122 // If the function takes variable number of arguments, make a frame index for 4123 // the start of the first vararg value... for expansion of llvm.va_start. 4124 if (isVarArg) { 4125 int Depth = ArgOffset; 4126 4127 FuncInfo->setVarArgsFrameIndex( 4128 MFI.CreateFixedObject(PtrByteSize, Depth, true)); 4129 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 4130 4131 // If this function is vararg, store any remaining integer argument regs 4132 // to their spots on the stack so that they may be loaded by dereferencing 4133 // the result of va_next. 4134 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 4135 GPR_idx < Num_GPR_Regs; ++GPR_idx) { 4136 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4137 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4138 SDValue Store = 4139 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 4140 MemOps.push_back(Store); 4141 // Increment the address by four for the next argument to store 4142 SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT); 4143 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 4144 } 4145 } 4146 4147 if (!MemOps.empty()) 4148 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 4149 4150 return Chain; 4151} 4152 4153SDValue PPCTargetLowering::LowerFormalArguments_Darwin( 4154 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 4155 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 4156 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 4157 // TODO: add description of PPC stack frame format, or at least some docs. 4158 // 4159 MachineFunction &MF = DAG.getMachineFunction(); 4160 MachineFrameInfo &MFI = MF.getFrameInfo(); 4161 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 4162 4163 EVT PtrVT = getPointerTy(MF.getDataLayout()); 4164 bool isPPC64 = PtrVT == MVT::i64; 4165 // Potential tail calls could cause overwriting of argument stack slots. 4166 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && 4167 (CallConv == CallingConv::Fast)); 4168 unsigned PtrByteSize = isPPC64 ? 8 : 4; 4169 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4170 unsigned ArgOffset = LinkageSize; 4171 // Area that is at least reserved in caller of this function. 4172 unsigned MinReservedArea = ArgOffset; 4173 4174 static const MCPhysReg GPR_32[] = { // 32-bit registers. 4175 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 4176 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 4177 }; 4178 static const MCPhysReg GPR_64[] = { // 64-bit registers. 4179 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4180 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4181 }; 4182 static const MCPhysReg VR[] = { 4183 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4184 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4185 }; 4186 4187 const unsigned Num_GPR_Regs = array_lengthof(GPR_32); 4188 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; 4189 const unsigned Num_VR_Regs = array_lengthof( VR); 4190 4191 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 4192 4193 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 4194 4195 // In 32-bit non-varargs functions, the stack space for vectors is after the 4196 // stack space for non-vectors. We do not use this space unless we have 4197 // too many vectors to fit in registers, something that only occurs in 4198 // constructed examples:), but we have to walk the arglist to figure 4199 // that out...for the pathological case, compute VecArgOffset as the 4200 // start of the vector parameter area. Computing VecArgOffset is the 4201 // entire point of the following loop. 4202 unsigned VecArgOffset = ArgOffset; 4203 if (!isVarArg && !isPPC64) { 4204 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; 4205 ++ArgNo) { 4206 EVT ObjectVT = Ins[ArgNo].VT; 4207 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 4208 4209 if (Flags.isByVal()) { 4210 // ObjSize is the true size, ArgSize rounded up to multiple of regs. 4211 unsigned ObjSize = Flags.getByValSize(); 4212 unsigned ArgSize = 4213 ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4214 VecArgOffset += ArgSize; 4215 continue; 4216 } 4217 4218 switch(ObjectVT.getSimpleVT().SimpleTy) { 4219 default: llvm_unreachable("Unhandled argument type!"); 4220 case MVT::i1: 4221 case MVT::i32: 4222 case MVT::f32: 4223 VecArgOffset += 4; 4224 break; 4225 case MVT::i64: // PPC64 4226 case MVT::f64: 4227 // FIXME: We are guaranteed to be !isPPC64 at this point. 4228 // Does MVT::i64 apply? 4229 VecArgOffset += 8; 4230 break; 4231 case MVT::v4f32: 4232 case MVT::v4i32: 4233 case MVT::v8i16: 4234 case MVT::v16i8: 4235 // Nothing to do, we're only looking at Nonvector args here. 4236 break; 4237 } 4238 } 4239 } 4240 // We've found where the vector parameter area in memory is. Skip the 4241 // first 12 parameters; these don't use that memory. 4242 VecArgOffset = ((VecArgOffset+15)/16)*16; 4243 VecArgOffset += 12*16; 4244 4245 // Add DAG nodes to load the arguments or copy them out of registers. On 4246 // entry to a function on PPC, the arguments start after the linkage area, 4247 // although the first ones are often in registers. 4248 4249 SmallVector<SDValue, 8> MemOps; 4250 unsigned nAltivecParamsAtEnd = 0; 4251 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); 4252 unsigned CurArgIdx = 0; 4253 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { 4254 SDValue ArgVal; 4255 bool needsLoad = false; 4256 EVT ObjectVT = Ins[ArgNo].VT; 4257 unsigned ObjSize = ObjectVT.getSizeInBits()/8; 4258 unsigned ArgSize = ObjSize; 4259 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; 4260 if (Ins[ArgNo].isOrigArg()) { 4261 std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); 4262 CurArgIdx = Ins[ArgNo].getOrigArgIndex(); 4263 } 4264 unsigned CurArgOffset = ArgOffset; 4265 4266 // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. 4267 if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || 4268 ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) { 4269 if (isVarArg || isPPC64) { 4270 MinReservedArea = ((MinReservedArea+15)/16)*16; 4271 MinReservedArea += CalculateStackSlotSize(ObjectVT, 4272 Flags, 4273 PtrByteSize); 4274 } else nAltivecParamsAtEnd++; 4275 } else 4276 // Calculate min reserved area. 4277 MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, 4278 Flags, 4279 PtrByteSize); 4280 4281 // FIXME the codegen can be much improved in some cases. 4282 // We do not have to keep everything in memory. 4283 if (Flags.isByVal()) { 4284 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); 4285 4286 // ObjSize is the true size, ArgSize rounded up to multiple of registers. 4287 ObjSize = Flags.getByValSize(); 4288 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 4289 // Objects of size 1 and 2 are right justified, everything else is 4290 // left justified. This means the memory address is adjusted forwards. 4291 if (ObjSize==1 || ObjSize==2) { 4292 CurArgOffset = CurArgOffset + (4 - ObjSize); 4293 } 4294 // The value of the object is its address. 4295 int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, false, true); 4296 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4297 InVals.push_back(FIN); 4298 if (ObjSize==1 || ObjSize==2) { 4299 if (GPR_idx != Num_GPR_Regs) { 4300 unsigned VReg; 4301 if (isPPC64) 4302 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4303 else 4304 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4305 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4306 EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16; 4307 SDValue Store = 4308 DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, 4309 MachinePointerInfo(&*FuncArg), ObjType); 4310 MemOps.push_back(Store); 4311 ++GPR_idx; 4312 } 4313 4314 ArgOffset += PtrByteSize; 4315 4316 continue; 4317 } 4318 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { 4319 // Store whatever pieces of the object are in registers 4320 // to memory. ArgOffset will be the address of the beginning 4321 // of the object. 4322 if (GPR_idx != Num_GPR_Regs) { 4323 unsigned VReg; 4324 if (isPPC64) 4325 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4326 else 4327 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4328 int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); 4329 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4330 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4331 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, 4332 MachinePointerInfo(&*FuncArg, j)); 4333 MemOps.push_back(Store); 4334 ++GPR_idx; 4335 ArgOffset += PtrByteSize; 4336 } else { 4337 ArgOffset += ArgSize - (ArgOffset-CurArgOffset); 4338 break; 4339 } 4340 } 4341 continue; 4342 } 4343 4344 switch (ObjectVT.getSimpleVT().SimpleTy) { 4345 default: llvm_unreachable("Unhandled argument type!"); 4346 case MVT::i1: 4347 case MVT::i32: 4348 if (!isPPC64) { 4349 if (GPR_idx != Num_GPR_Regs) { 4350 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4351 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); 4352 4353 if (ObjectVT == MVT::i1) 4354 ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal); 4355 4356 ++GPR_idx; 4357 } else { 4358 needsLoad = true; 4359 ArgSize = PtrByteSize; 4360 } 4361 // All int arguments reserve stack space in the Darwin ABI. 4362 ArgOffset += PtrByteSize; 4363 break; 4364 } 4365 LLVM_FALLTHROUGH; 4366 case MVT::i64: // PPC64 4367 if (GPR_idx != Num_GPR_Regs) { 4368 unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4369 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); 4370 4371 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) 4372 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote 4373 // value to MVT::i64 and then truncate to the correct register size. 4374 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); 4375 4376 ++GPR_idx; 4377 } else { 4378 needsLoad = true; 4379 ArgSize = PtrByteSize; 4380 } 4381 // All int arguments reserve stack space in the Darwin ABI. 4382 ArgOffset += 8; 4383 break; 4384 4385 case MVT::f32: 4386 case MVT::f64: 4387 // Every 4 bytes of argument space consumes one of the GPRs available for 4388 // argument passing. 4389 if (GPR_idx != Num_GPR_Regs) { 4390 ++GPR_idx; 4391 if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64) 4392 ++GPR_idx; 4393 } 4394 if (FPR_idx != Num_FPR_Regs) { 4395 unsigned VReg; 4396 4397 if (ObjectVT == MVT::f32) 4398 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); 4399 else 4400 VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass); 4401 4402 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4403 ++FPR_idx; 4404 } else { 4405 needsLoad = true; 4406 } 4407 4408 // All FP arguments reserve stack space in the Darwin ABI. 4409 ArgOffset += isPPC64 ? 8 : ObjSize; 4410 break; 4411 case MVT::v4f32: 4412 case MVT::v4i32: 4413 case MVT::v8i16: 4414 case MVT::v16i8: 4415 // Note that vector arguments in registers don't reserve stack space, 4416 // except in varargs functions. 4417 if (VR_idx != Num_VR_Regs) { 4418 unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); 4419 ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); 4420 if (isVarArg) { 4421 while ((ArgOffset % 16) != 0) { 4422 ArgOffset += PtrByteSize; 4423 if (GPR_idx != Num_GPR_Regs) 4424 GPR_idx++; 4425 } 4426 ArgOffset += 16; 4427 GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? 4428 } 4429 ++VR_idx; 4430 } else { 4431 if (!isVarArg && !isPPC64) { 4432 // Vectors go after all the nonvectors. 4433 CurArgOffset = VecArgOffset; 4434 VecArgOffset += 16; 4435 } else { 4436 // Vectors are aligned. 4437 ArgOffset = ((ArgOffset+15)/16)*16; 4438 CurArgOffset = ArgOffset; 4439 ArgOffset += 16; 4440 } 4441 needsLoad = true; 4442 } 4443 break; 4444 } 4445 4446 // We need to load the argument to a virtual register if we determined above 4447 // that we ran out of physical registers of the appropriate type. 4448 if (needsLoad) { 4449 int FI = MFI.CreateFixedObject(ObjSize, 4450 CurArgOffset + (ArgSize - ObjSize), 4451 isImmutable); 4452 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 4453 ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); 4454 } 4455 4456 InVals.push_back(ArgVal); 4457 } 4458 4459 // Allow for Altivec parameters at the end, if needed. 4460 if (nAltivecParamsAtEnd) { 4461 MinReservedArea = ((MinReservedArea+15)/16)*16; 4462 MinReservedArea += 16*nAltivecParamsAtEnd; 4463 } 4464 4465 // Area that is at least reserved in the caller of this function. 4466 MinReservedArea = std::max(MinReservedArea, LinkageSize + 8 * PtrByteSize); 4467 4468 // Set the size that is at least reserved in caller of this function. Tail 4469 // call optimized functions' reserved stack space needs to be aligned so that 4470 // taking the difference between two stack areas will result in an aligned 4471 // stack. 4472 MinReservedArea = 4473 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 4474 FuncInfo->setMinReservedArea(MinReservedArea); 4475 4476 // If the function takes variable number of arguments, make a frame index for 4477 // the start of the first vararg value... for expansion of llvm.va_start. 4478 if (isVarArg) { 4479 int Depth = ArgOffset; 4480 4481 FuncInfo->setVarArgsFrameIndex( 4482 MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, 4483 Depth, true)); 4484 SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); 4485 4486 // If this function is vararg, store any remaining integer argument regs 4487 // to their spots on the stack so that they may be loaded by dereferencing 4488 // the result of va_next. 4489 for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { 4490 unsigned VReg; 4491 4492 if (isPPC64) 4493 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); 4494 else 4495 VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); 4496 4497 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); 4498 SDValue Store = 4499 DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); 4500 MemOps.push_back(Store); 4501 // Increment the address by four for the next argument to store 4502 SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); 4503 FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); 4504 } 4505 } 4506 4507 if (!MemOps.empty()) 4508 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); 4509 4510 return Chain; 4511} 4512 4513/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be 4514/// adjusted to accommodate the arguments for the tailcall. 4515static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, 4516 unsigned ParamSize) { 4517 4518 if (!isTailCall) return 0; 4519 4520 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>(); 4521 unsigned CallerMinReservedArea = FI->getMinReservedArea(); 4522 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize; 4523 // Remember only if the new adjustment is bigger. 4524 if (SPDiff < FI->getTailCallSPDelta()) 4525 FI->setTailCallSPDelta(SPDiff); 4526 4527 return SPDiff; 4528} 4529 4530static bool isFunctionGlobalAddress(SDValue Callee); 4531 4532static bool 4533callsShareTOCBase(const Function *Caller, SDValue Callee, 4534 const TargetMachine &TM) { 4535 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols 4536 // don't have enough information to determine if the caller and calle share 4537 // the same TOC base, so we have to pessimistically assume they don't for 4538 // correctness. 4539 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 4540 if (!G) 4541 return false; 4542 4543 const GlobalValue *GV = G->getGlobal(); 4544 // The medium and large code models are expected to provide a sufficiently 4545 // large TOC to provide all data addressing needs of a module with a 4546 // single TOC. Since each module will be addressed with a single TOC then we 4547 // only need to check that caller and callee don't cross dso boundaries. 4548 if (CodeModel::Medium == TM.getCodeModel() || 4549 CodeModel::Large == TM.getCodeModel()) 4550 return TM.shouldAssumeDSOLocal(*Caller->getParent(), GV); 4551 4552 // Otherwise we need to ensure callee and caller are in the same section, 4553 // since the linker may allocate multiple TOCs, and we don't know which 4554 // sections will belong to the same TOC base. 4555 4556 if (!GV->isStrongDefinitionForLinker()) 4557 return false; 4558 4559 // Any explicitly-specified sections and section prefixes must also match. 4560 // Also, if we're using -ffunction-sections, then each function is always in 4561 // a different section (the same is true for COMDAT functions). 4562 if (TM.getFunctionSections() || GV->hasComdat() || Caller->hasComdat() || 4563 GV->getSection() != Caller->getSection()) 4564 return false; 4565 if (const auto *F = dyn_cast<Function>(GV)) { 4566 if (F->getSectionPrefix() != Caller->getSectionPrefix()) 4567 return false; 4568 } 4569 4570 // If the callee might be interposed, then we can't assume the ultimate call 4571 // target will be in the same section. Even in cases where we can assume that 4572 // interposition won't happen, in any case where the linker might insert a 4573 // stub to allow for interposition, we must generate code as though 4574 // interposition might occur. To understand why this matters, consider a 4575 // situation where: a -> b -> c where the arrows indicate calls. b and c are 4576 // in the same section, but a is in a different module (i.e. has a different 4577 // TOC base pointer). If the linker allows for interposition between b and c, 4578 // then it will generate a stub for the call edge between b and c which will 4579 // save the TOC pointer into the designated stack slot allocated by b. If we 4580 // return true here, and therefore allow a tail call between b and c, that 4581 // stack slot won't exist and the b -> c stub will end up saving b'c TOC base 4582 // pointer into the stack slot allocated by a (where the a -> b stub saved 4583 // a's TOC base pointer). If we're not considering a tail call, but rather, 4584 // whether a nop is needed after the call instruction in b, because the linker 4585 // will insert a stub, it might complain about a missing nop if we omit it 4586 // (although many don't complain in this case). 4587 if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV)) 4588 return false; 4589 4590 return true; 4591} 4592 4593static bool 4594needStackSlotPassParameters(const PPCSubtarget &Subtarget, 4595 const SmallVectorImpl<ISD::OutputArg> &Outs) { 4596 assert(Subtarget.is64BitELFABI()); 4597 4598 const unsigned PtrByteSize = 8; 4599 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 4600 4601 static const MCPhysReg GPR[] = { 4602 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 4603 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 4604 }; 4605 static const MCPhysReg VR[] = { 4606 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 4607 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 4608 }; 4609 4610 const unsigned NumGPRs = array_lengthof(GPR); 4611 const unsigned NumFPRs = 13; 4612 const unsigned NumVRs = array_lengthof(VR); 4613 const unsigned ParamAreaSize = NumGPRs * PtrByteSize; 4614 4615 unsigned NumBytes = LinkageSize; 4616 unsigned AvailableFPRs = NumFPRs; 4617 unsigned AvailableVRs = NumVRs; 4618 4619 for (const ISD::OutputArg& Param : Outs) { 4620 if (Param.Flags.isNest()) continue; 4621 4622 if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, 4623 PtrByteSize, LinkageSize, ParamAreaSize, 4624 NumBytes, AvailableFPRs, AvailableVRs, 4625 Subtarget.hasQPX())) 4626 return true; 4627 } 4628 return false; 4629} 4630 4631static bool 4632hasSameArgumentList(const Function *CallerFn, ImmutableCallSite CS) { 4633 if (CS.arg_size() != CallerFn->arg_size()) 4634 return false; 4635 4636 ImmutableCallSite::arg_iterator CalleeArgIter = CS.arg_begin(); 4637 ImmutableCallSite::arg_iterator CalleeArgEnd = CS.arg_end(); 4638 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin(); 4639 4640 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) { 4641 const Value* CalleeArg = *CalleeArgIter; 4642 const Value* CallerArg = &(*CallerArgIter); 4643 if (CalleeArg == CallerArg) 4644 continue; 4645 4646 // e.g. @caller([4 x i64] %a, [4 x i64] %b) { 4647 // tail call @callee([4 x i64] undef, [4 x i64] %b) 4648 // } 4649 // 1st argument of callee is undef and has the same type as caller. 4650 if (CalleeArg->getType() == CallerArg->getType() && 4651 isa<UndefValue>(CalleeArg)) 4652 continue; 4653 4654 return false; 4655 } 4656 4657 return true; 4658} 4659 4660// Returns true if TCO is possible between the callers and callees 4661// calling conventions. 4662static bool 4663areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC, 4664 CallingConv::ID CalleeCC) { 4665 // Tail calls are possible with fastcc and ccc. 4666 auto isTailCallableCC = [] (CallingConv::ID CC){ 4667 return CC == CallingConv::C || CC == CallingConv::Fast; 4668 }; 4669 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC)) 4670 return false; 4671 4672 // We can safely tail call both fastcc and ccc callees from a c calling 4673 // convention caller. If the caller is fastcc, we may have less stack space 4674 // than a non-fastcc caller with the same signature so disable tail-calls in 4675 // that case. 4676 return CallerCC == CallingConv::C || CallerCC == CalleeCC; 4677} 4678 4679bool 4680PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4( 4681 SDValue Callee, 4682 CallingConv::ID CalleeCC, 4683 ImmutableCallSite CS, 4684 bool isVarArg, 4685 const SmallVectorImpl<ISD::OutputArg> &Outs, 4686 const SmallVectorImpl<ISD::InputArg> &Ins, 4687 SelectionDAG& DAG) const { 4688 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt; 4689 4690 if (DisableSCO && !TailCallOpt) return false; 4691 4692 // Variadic argument functions are not supported. 4693 if (isVarArg) return false; 4694 4695 auto &Caller = DAG.getMachineFunction().getFunction(); 4696 // Check that the calling conventions are compatible for tco. 4697 if (!areCallingConvEligibleForTCO_64SVR4(Caller.getCallingConv(), CalleeCC)) 4698 return false; 4699 4700 // Caller contains any byval parameter is not supported. 4701 if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); })) 4702 return false; 4703 4704 // Callee contains any byval parameter is not supported, too. 4705 // Note: This is a quick work around, because in some cases, e.g. 4706 // caller's stack size > callee's stack size, we are still able to apply 4707 // sibling call optimization. For example, gcc is able to do SCO for caller1 4708 // in the following example, but not for caller2. 4709 // struct test { 4710 // long int a; 4711 // char ary[56]; 4712 // } gTest; 4713 // __attribute__((noinline)) int callee(struct test v, struct test *b) { 4714 // b->a = v.a; 4715 // return 0; 4716 // } 4717 // void caller1(struct test a, struct test c, struct test *b) { 4718 // callee(gTest, b); } 4719 // void caller2(struct test *b) { callee(gTest, b); } 4720 if (any_of(Outs, [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); })) 4721 return false; 4722 4723 // If callee and caller use different calling conventions, we cannot pass 4724 // parameters on stack since offsets for the parameter area may be different. 4725 if (Caller.getCallingConv() != CalleeCC && 4726 needStackSlotPassParameters(Subtarget, Outs)) 4727 return false; 4728 4729 // No TCO/SCO on indirect call because Caller have to restore its TOC 4730 if (!isFunctionGlobalAddress(Callee) && 4731 !isa<ExternalSymbolSDNode>(Callee)) 4732 return false; 4733 4734 // If the caller and callee potentially have different TOC bases then we 4735 // cannot tail call since we need to restore the TOC pointer after the call. 4736 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977 4737 if (!callsShareTOCBase(&Caller, Callee, getTargetMachine())) 4738 return false; 4739 4740 // TCO allows altering callee ABI, so we don't have to check further. 4741 if (CalleeCC == CallingConv::Fast && TailCallOpt) 4742 return true; 4743 4744 if (DisableSCO) return false; 4745 4746 // If callee use the same argument list that caller is using, then we can 4747 // apply SCO on this case. If it is not, then we need to check if callee needs 4748 // stack for passing arguments. 4749 if (!hasSameArgumentList(&Caller, CS) && 4750 needStackSlotPassParameters(Subtarget, Outs)) { 4751 return false; 4752 } 4753 4754 return true; 4755} 4756 4757/// IsEligibleForTailCallOptimization - Check whether the call is eligible 4758/// for tail call optimization. Targets which want to do tail call 4759/// optimization should implement this function. 4760bool 4761PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, 4762 CallingConv::ID CalleeCC, 4763 bool isVarArg, 4764 const SmallVectorImpl<ISD::InputArg> &Ins, 4765 SelectionDAG& DAG) const { 4766 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 4767 return false; 4768 4769 // Variable argument functions are not supported. 4770 if (isVarArg) 4771 return false; 4772 4773 MachineFunction &MF = DAG.getMachineFunction(); 4774 CallingConv::ID CallerCC = MF.getFunction().getCallingConv(); 4775 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { 4776 // Functions containing by val parameters are not supported. 4777 for (unsigned i = 0; i != Ins.size(); i++) { 4778 ISD::ArgFlagsTy Flags = Ins[i].Flags; 4779 if (Flags.isByVal()) return false; 4780 } 4781 4782 // Non-PIC/GOT tail calls are supported. 4783 if (getTargetMachine().getRelocationModel() != Reloc::PIC_) 4784 return true; 4785 4786 // At the moment we can only do local tail calls (in same module, hidden 4787 // or protected) if we are generating PIC. 4788 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) 4789 return G->getGlobal()->hasHiddenVisibility() 4790 || G->getGlobal()->hasProtectedVisibility(); 4791 } 4792 4793 return false; 4794} 4795 4796/// isCallCompatibleAddress - Return the immediate to use if the specified 4797/// 32-bit value is representable in the immediate field of a BxA instruction. 4798static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) { 4799 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op); 4800 if (!C) return nullptr; 4801 4802 int Addr = C->getZExtValue(); 4803 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero. 4804 SignExtend32<26>(Addr) != Addr) 4805 return nullptr; // Top 6 bits have to be sext of immediate. 4806 4807 return DAG 4808 .getConstant( 4809 (int)C->getZExtValue() >> 2, SDLoc(Op), 4810 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())) 4811 .getNode(); 4812} 4813 4814namespace { 4815 4816struct TailCallArgumentInfo { 4817 SDValue Arg; 4818 SDValue FrameIdxOp; 4819 int FrameIdx = 0; 4820 4821 TailCallArgumentInfo() = default; 4822}; 4823 4824} // end anonymous namespace 4825 4826/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot. 4827static void StoreTailCallArgumentsToStackSlot( 4828 SelectionDAG &DAG, SDValue Chain, 4829 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs, 4830 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) { 4831 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) { 4832 SDValue Arg = TailCallArgs[i].Arg; 4833 SDValue FIN = TailCallArgs[i].FrameIdxOp; 4834 int FI = TailCallArgs[i].FrameIdx; 4835 // Store relative to framepointer. 4836 MemOpChains.push_back(DAG.getStore( 4837 Chain, dl, Arg, FIN, 4838 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI))); 4839 } 4840} 4841 4842/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to 4843/// the appropriate stack slot for the tail call optimized function call. 4844static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain, 4845 SDValue OldRetAddr, SDValue OldFP, 4846 int SPDiff, const SDLoc &dl) { 4847 if (SPDiff) { 4848 // Calculate the new stack slot for the return address. 4849 MachineFunction &MF = DAG.getMachineFunction(); 4850 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>(); 4851 const PPCFrameLowering *FL = Subtarget.getFrameLowering(); 4852 bool isPPC64 = Subtarget.isPPC64(); 4853 int SlotSize = isPPC64 ? 8 : 4; 4854 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset(); 4855 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(SlotSize, 4856 NewRetAddrLoc, true); 4857 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 4858 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT); 4859 Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx, 4860 MachinePointerInfo::getFixedStack(MF, NewRetAddr)); 4861 4862 // When using the 32/64-bit SVR4 ABI there is no need to move the FP stack 4863 // slot as the FP is never overwritten. 4864 if (Subtarget.isDarwinABI()) { 4865 int NewFPLoc = SPDiff + FL->getFramePointerSaveOffset(); 4866 int NewFPIdx = MF.getFrameInfo().CreateFixedObject(SlotSize, NewFPLoc, 4867 true); 4868 SDValue NewFramePtrIdx = DAG.getFrameIndex(NewFPIdx, VT); 4869 Chain = DAG.getStore(Chain, dl, OldFP, NewFramePtrIdx, 4870 MachinePointerInfo::getFixedStack( 4871 DAG.getMachineFunction(), NewFPIdx)); 4872 } 4873 } 4874 return Chain; 4875} 4876 4877/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate 4878/// the position of the argument. 4879static void 4880CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64, 4881 SDValue Arg, int SPDiff, unsigned ArgOffset, 4882 SmallVectorImpl<TailCallArgumentInfo>& TailCallArguments) { 4883 int Offset = ArgOffset + SPDiff; 4884 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8; 4885 int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true); 4886 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 4887 SDValue FIN = DAG.getFrameIndex(FI, VT); 4888 TailCallArgumentInfo Info; 4889 Info.Arg = Arg; 4890 Info.FrameIdxOp = FIN; 4891 Info.FrameIdx = FI; 4892 TailCallArguments.push_back(Info); 4893} 4894 4895/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address 4896/// stack slot. Returns the chain as result and the loaded frame pointers in 4897/// LROpOut/FPOpout. Used when tail calling. 4898SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr( 4899 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut, 4900 SDValue &FPOpOut, const SDLoc &dl) const { 4901 if (SPDiff) { 4902 // Load the LR and FP stack slot for later adjusting. 4903 EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 4904 LROpOut = getReturnAddrFrameIndex(DAG); 4905 LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo()); 4906 Chain = SDValue(LROpOut.getNode(), 1); 4907 4908 // When using the 32/64-bit SVR4 ABI there is no need to load the FP stack 4909 // slot as the FP is never overwritten. 4910 if (Subtarget.isDarwinABI()) { 4911 FPOpOut = getFramePointerFrameIndex(DAG); 4912 FPOpOut = DAG.getLoad(VT, dl, Chain, FPOpOut, MachinePointerInfo()); 4913 Chain = SDValue(FPOpOut.getNode(), 1); 4914 } 4915 } 4916 return Chain; 4917} 4918 4919/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified 4920/// by "Src" to address "Dst" of size "Size". Alignment information is 4921/// specified by the specific parameter attribute. The copy will be passed as 4922/// a byval function parameter. 4923/// Sometimes what we are copying is the end of a larger object, the part that 4924/// does not fit in registers. 4925static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, 4926 SDValue Chain, ISD::ArgFlagsTy Flags, 4927 SelectionDAG &DAG, const SDLoc &dl) { 4928 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); 4929 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), 4930 false, false, false, MachinePointerInfo(), 4931 MachinePointerInfo()); 4932} 4933 4934/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of 4935/// tail calls. 4936static void LowerMemOpCallTo( 4937 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg, 4938 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64, 4939 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains, 4940 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) { 4941 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 4942 if (!isTailCall) { 4943 if (isVector) { 4944 SDValue StackPtr; 4945 if (isPPC64) 4946 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 4947 else 4948 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 4949 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 4950 DAG.getConstant(ArgOffset, dl, PtrVT)); 4951 } 4952 MemOpChains.push_back( 4953 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); 4954 // Calculate and remember argument location. 4955 } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset, 4956 TailCallArguments); 4957} 4958 4959static void 4960PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain, 4961 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp, 4962 SDValue FPOp, 4963 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) { 4964 // Emit a sequence of copyto/copyfrom virtual registers for arguments that 4965 // might overwrite each other in case of tail call optimization. 4966 SmallVector<SDValue, 8> MemOpChains2; 4967 // Do not flag preceding copytoreg stuff together with the following stuff. 4968 InFlag = SDValue(); 4969 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments, 4970 MemOpChains2, dl); 4971 if (!MemOpChains2.empty()) 4972 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); 4973 4974 // Store the return address to the appropriate stack slot. 4975 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, LROp, FPOp, SPDiff, dl); 4976 4977 // Emit callseq_end just before tailcall node. 4978 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 4979 DAG.getIntPtrConstant(0, dl, true), InFlag, dl); 4980 InFlag = Chain.getValue(1); 4981} 4982 4983// Is this global address that of a function that can be called by name? (as 4984// opposed to something that must hold a descriptor for an indirect call). 4985static bool isFunctionGlobalAddress(SDValue Callee) { 4986 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 4987 if (Callee.getOpcode() == ISD::GlobalTLSAddress || 4988 Callee.getOpcode() == ISD::TargetGlobalTLSAddress) 4989 return false; 4990 4991 return G->getGlobal()->getValueType()->isFunctionTy(); 4992 } 4993 4994 return false; 4995} 4996 4997SDValue PPCTargetLowering::LowerCallResult( 4998 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, 4999 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5000 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 5001 SmallVector<CCValAssign, 16> RVLocs; 5002 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 5003 *DAG.getContext()); 5004 5005 CCRetInfo.AnalyzeCallResult( 5006 Ins, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 5007 ? RetCC_PPC_Cold 5008 : RetCC_PPC); 5009 5010 // Copy all of the result registers out of their specified physreg. 5011 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { 5012 CCValAssign &VA = RVLocs[i]; 5013 assert(VA.isRegLoc() && "Can only return in registers!"); 5014 5015 SDValue Val; 5016 5017 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { 5018 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 5019 InFlag); 5020 Chain = Lo.getValue(1); 5021 InFlag = Lo.getValue(2); 5022 VA = RVLocs[++i]; // skip ahead to next loc 5023 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, 5024 InFlag); 5025 Chain = Hi.getValue(1); 5026 InFlag = Hi.getValue(2); 5027 if (!Subtarget.isLittleEndian()) 5028 std::swap (Lo, Hi); 5029 Val = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, Lo, Hi); 5030 } else { 5031 Val = DAG.getCopyFromReg(Chain, dl, 5032 VA.getLocReg(), VA.getLocVT(), InFlag); 5033 Chain = Val.getValue(1); 5034 InFlag = Val.getValue(2); 5035 } 5036 5037 switch (VA.getLocInfo()) { 5038 default: llvm_unreachable("Unknown loc info!"); 5039 case CCValAssign::Full: break; 5040 case CCValAssign::AExt: 5041 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5042 break; 5043 case CCValAssign::ZExt: 5044 Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val, 5045 DAG.getValueType(VA.getValVT())); 5046 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5047 break; 5048 case CCValAssign::SExt: 5049 Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val, 5050 DAG.getValueType(VA.getValVT())); 5051 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); 5052 break; 5053 } 5054 5055 InVals.push_back(Val); 5056 } 5057 5058 return Chain; 5059} 5060 5061static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG, 5062 const PPCSubtarget &Subtarget, bool isPatchPoint) { 5063 // PatchPoint calls are not indirect. 5064 if (isPatchPoint) 5065 return false; 5066 5067 if (isFunctionGlobalAddress(Callee) || dyn_cast<ExternalSymbolSDNode>(Callee)) 5068 return false; 5069 5070 // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not 5071 // becuase the immediate function pointer points to a descriptor instead of 5072 // a function entry point. The ELFv2 ABI cannot use a BLA because the function 5073 // pointer immediate points to the global entry point, while the BLA would 5074 // need to jump to the local entry point (see rL211174). 5075 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() && 5076 isBLACompatibleAddress(Callee, DAG)) 5077 return false; 5078 5079 return true; 5080} 5081 5082static unsigned getCallOpcode(bool isIndirectCall, bool isPatchPoint, 5083 bool isTailCall, const Function &Caller, 5084 const SDValue &Callee, 5085 const PPCSubtarget &Subtarget, 5086 const TargetMachine &TM) { 5087 if (isTailCall) 5088 return PPCISD::TC_RETURN; 5089 5090 // This is a call through a function pointer. 5091 if (isIndirectCall) { 5092 // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross 5093 // indirect calls. The save of the caller's TOC pointer to the stack will be 5094 // inserted into the DAG as part of call lowering. The restore of the TOC 5095 // pointer is modeled by using a pseudo instruction for the call opcode that 5096 // represents the 2 instruction sequence of an indirect branch and link, 5097 // immediately followed by a load of the TOC pointer from the the stack save 5098 // slot into gpr2. 5099 if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) 5100 return PPCISD::BCTRL_LOAD_TOC; 5101 5102 // An indirect call that does not need a TOC restore. 5103 return PPCISD::BCTRL; 5104 } 5105 5106 // The ABIs that maintain a TOC pointer accross calls need to have a nop 5107 // immediately following the call instruction if the caller and callee may 5108 // have different TOC bases. At link time if the linker determines the calls 5109 // may not share a TOC base, the call is redirected to a trampoline inserted 5110 // by the linker. The trampoline will (among other things) save the callers 5111 // TOC pointer at an ABI designated offset in the linkage area and the linker 5112 // will rewrite the nop to be a load of the TOC pointer from the linkage area 5113 // into gpr2. 5114 if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) 5115 return callsShareTOCBase(&Caller, Callee, TM) ? PPCISD::CALL 5116 : PPCISD::CALL_NOP; 5117 5118 return PPCISD::CALL; 5119} 5120 5121static bool isValidAIXExternalSymSDNode(StringRef SymName) { 5122 return StringSwitch<bool>(SymName) 5123 .Cases("__divdi3", "__fixunsdfdi", "__floatundidf", "__floatundisf", 5124 "__moddi3", "__udivdi3", "__umoddi3", true) 5125 .Cases("ceil", "floor", "memcpy", "memmove", "memset", "round", true) 5126 .Default(false); 5127} 5128 5129static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG, 5130 const SDLoc &dl, const PPCSubtarget &Subtarget) { 5131 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI()) 5132 if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) 5133 return SDValue(Dest, 0); 5134 5135 // Returns true if the callee is local, and false otherwise. 5136 auto isLocalCallee = [&]() { 5137 const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee); 5138 const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); 5139 const GlobalValue *GV = G ? G->getGlobal() : nullptr; 5140 5141 return DAG.getTarget().shouldAssumeDSOLocal(*Mod, GV) && 5142 !dyn_cast_or_null<GlobalIFunc>(GV); 5143 }; 5144 5145 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in 5146 // a static relocation model causes some versions of GNU LD (2.17.50, at 5147 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are 5148 // built with secure-PLT. 5149 bool UsePlt = 5150 Subtarget.is32BitELFABI() && !isLocalCallee() && 5151 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_; 5152 5153 // On AIX, direct function calls reference the symbol for the function's 5154 // entry point, which is named by prepending a "." before the function's 5155 // C-linkage name. 5156 const auto getAIXFuncEntryPointSymbolSDNode = 5157 [&](StringRef FuncName, bool IsDeclaration, 5158 const XCOFF::StorageClass &SC) { 5159 auto &Context = DAG.getMachineFunction().getMMI().getContext(); 5160 5161 MCSymbolXCOFF *S = cast<MCSymbolXCOFF>( 5162 Context.getOrCreateSymbol(Twine(".") + Twine(FuncName))); 5163 5164 if (IsDeclaration && !S->hasContainingCsect()) { 5165 // On AIX, an undefined symbol needs to be associated with a 5166 // MCSectionXCOFF to get the correct storage mapping class. 5167 // In this case, XCOFF::XMC_PR. 5168 MCSectionXCOFF *Sec = Context.getXCOFFSection( 5169 S->getName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC, 5170 SectionKind::getMetadata()); 5171 S->setContainingCsect(Sec); 5172 } 5173 5174 MVT PtrVT = 5175 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 5176 return DAG.getMCSymbol(S, PtrVT); 5177 }; 5178 5179 if (isFunctionGlobalAddress(Callee)) { 5180 const GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Callee); 5181 const GlobalValue *GV = G->getGlobal(); 5182 5183 if (!Subtarget.isAIXABI()) 5184 return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0, 5185 UsePlt ? PPCII::MO_PLT : 0); 5186 5187 assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX."); 5188 const GlobalObject *GO = cast<GlobalObject>(GV); 5189 const XCOFF::StorageClass SC = 5190 TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GO); 5191 return getAIXFuncEntryPointSymbolSDNode(GO->getName(), GO->isDeclaration(), 5192 SC); 5193 } 5194 5195 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { 5196 const char *SymName = S->getSymbol(); 5197 if (!Subtarget.isAIXABI()) 5198 return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(), 5199 UsePlt ? PPCII::MO_PLT : 0); 5200 5201 // If there exists a user-declared function whose name is the same as the 5202 // ExternalSymbol's, then we pick up the user-declared version. 5203 const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); 5204 if (const Function *F = 5205 dyn_cast_or_null<Function>(Mod->getNamedValue(SymName))) { 5206 const XCOFF::StorageClass SC = 5207 TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(F); 5208 return getAIXFuncEntryPointSymbolSDNode(F->getName(), F->isDeclaration(), 5209 SC); 5210 } 5211 5212 // TODO: Remove this when the support for ExternalSymbolSDNode is complete. 5213 if (isValidAIXExternalSymSDNode(SymName)) { 5214 return getAIXFuncEntryPointSymbolSDNode(SymName, true, XCOFF::C_EXT); 5215 } 5216 5217 report_fatal_error("Unexpected ExternalSymbolSDNode: " + Twine(SymName)); 5218 } 5219 5220 // No transformation needed. 5221 assert(Callee.getNode() && "What no callee?"); 5222 return Callee; 5223} 5224 5225static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) { 5226 assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START && 5227 "Expected a CALLSEQ_STARTSDNode."); 5228 5229 // The last operand is the chain, except when the node has glue. If the node 5230 // has glue, then the last operand is the glue, and the chain is the second 5231 // last operand. 5232 SDValue LastValue = CallSeqStart.getValue(CallSeqStart->getNumValues() - 1); 5233 if (LastValue.getValueType() != MVT::Glue) 5234 return LastValue; 5235 5236 return CallSeqStart.getValue(CallSeqStart->getNumValues() - 2); 5237} 5238 5239// Creates the node that moves a functions address into the count register 5240// to prepare for an indirect call instruction. 5241static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee, 5242 SDValue &Glue, SDValue &Chain, 5243 const SDLoc &dl) { 5244 SDValue MTCTROps[] = {Chain, Callee, Glue}; 5245 EVT ReturnTypes[] = {MVT::Other, MVT::Glue}; 5246 Chain = DAG.getNode(PPCISD::MTCTR, dl, makeArrayRef(ReturnTypes, 2), 5247 makeArrayRef(MTCTROps, Glue.getNode() ? 3 : 2)); 5248 // The glue is the second value produced. 5249 Glue = Chain.getValue(1); 5250} 5251 5252static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee, 5253 SDValue &Glue, SDValue &Chain, 5254 SDValue CallSeqStart, 5255 ImmutableCallSite CS, const SDLoc &dl, 5256 bool hasNest, 5257 const PPCSubtarget &Subtarget) { 5258 // Function pointers in the 64-bit SVR4 ABI do not point to the function 5259 // entry point, but to the function descriptor (the function entry point 5260 // address is part of the function descriptor though). 5261 // The function descriptor is a three doubleword structure with the 5262 // following fields: function entry point, TOC base address and 5263 // environment pointer. 5264 // Thus for a call through a function pointer, the following actions need 5265 // to be performed: 5266 // 1. Save the TOC of the caller in the TOC save area of its stack 5267 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()). 5268 // 2. Load the address of the function entry point from the function 5269 // descriptor. 5270 // 3. Load the TOC of the callee from the function descriptor into r2. 5271 // 4. Load the environment pointer from the function descriptor into 5272 // r11. 5273 // 5. Branch to the function entry point address. 5274 // 6. On return of the callee, the TOC of the caller needs to be 5275 // restored (this is done in FinishCall()). 5276 // 5277 // The loads are scheduled at the beginning of the call sequence, and the 5278 // register copies are flagged together to ensure that no other 5279 // operations can be scheduled in between. E.g. without flagging the 5280 // copies together, a TOC access in the caller could be scheduled between 5281 // the assignment of the callee TOC and the branch to the callee, which leads 5282 // to incorrect code. 5283 5284 // Start by loading the function address from the descriptor. 5285 SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart); 5286 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors() 5287 ? (MachineMemOperand::MODereferenceable | 5288 MachineMemOperand::MOInvariant) 5289 : MachineMemOperand::MONone; 5290 5291 MachinePointerInfo MPI(CS ? CS.getCalledValue() : nullptr); 5292 5293 // Registers used in building the DAG. 5294 const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister(); 5295 const MCRegister TOCReg = Subtarget.getTOCPointerRegister(); 5296 5297 // Offsets of descriptor members. 5298 const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset(); 5299 const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset(); 5300 5301 const MVT RegVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 5302 const unsigned Alignment = Subtarget.isPPC64() ? 8 : 4; 5303 5304 // One load for the functions entry point address. 5305 SDValue LoadFuncPtr = DAG.getLoad(RegVT, dl, LDChain, Callee, MPI, 5306 Alignment, MMOFlags); 5307 5308 // One for loading the TOC anchor for the module that contains the called 5309 // function. 5310 SDValue TOCOff = DAG.getIntPtrConstant(TOCAnchorOffset, dl); 5311 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, Callee, TOCOff); 5312 SDValue TOCPtr = 5313 DAG.getLoad(RegVT, dl, LDChain, AddTOC, 5314 MPI.getWithOffset(TOCAnchorOffset), Alignment, MMOFlags); 5315 5316 // One for loading the environment pointer. 5317 SDValue PtrOff = DAG.getIntPtrConstant(EnvPtrOffset, dl); 5318 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, RegVT, Callee, PtrOff); 5319 SDValue LoadEnvPtr = 5320 DAG.getLoad(RegVT, dl, LDChain, AddPtr, 5321 MPI.getWithOffset(EnvPtrOffset), Alignment, MMOFlags); 5322 5323 5324 // Then copy the newly loaded TOC anchor to the TOC pointer. 5325 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, TOCReg, TOCPtr, Glue); 5326 Chain = TOCVal.getValue(0); 5327 Glue = TOCVal.getValue(1); 5328 5329 // If the function call has an explicit 'nest' parameter, it takes the 5330 // place of the environment pointer. 5331 assert((!hasNest || !Subtarget.isAIXABI()) && 5332 "Nest parameter is not supported on AIX."); 5333 if (!hasNest) { 5334 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, EnvPtrReg, LoadEnvPtr, Glue); 5335 Chain = EnvVal.getValue(0); 5336 Glue = EnvVal.getValue(1); 5337 } 5338 5339 // The rest of the indirect call sequence is the same as the non-descriptor 5340 // DAG. 5341 prepareIndirectCall(DAG, LoadFuncPtr, Glue, Chain, dl); 5342} 5343 5344static void 5345buildCallOperands(SmallVectorImpl<SDValue> &Ops, CallingConv::ID CallConv, 5346 const SDLoc &dl, bool isTailCall, bool isVarArg, 5347 bool isPatchPoint, bool hasNest, SelectionDAG &DAG, 5348 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, 5349 SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff, 5350 const PPCSubtarget &Subtarget, bool isIndirect) { 5351 const bool IsPPC64 = Subtarget.isPPC64(); 5352 // MVT for a general purpose register. 5353 const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; 5354 5355 // First operand is always the chain. 5356 Ops.push_back(Chain); 5357 5358 // If it's a direct call pass the callee as the second operand. 5359 if (!isIndirect) 5360 Ops.push_back(Callee); 5361 else { 5362 assert(!isPatchPoint && "Patch point call are not indirect."); 5363 5364 // For the TOC based ABIs, we have saved the TOC pointer to the linkage area 5365 // on the stack (this would have been done in `LowerCall_64SVR4` or 5366 // `LowerCall_AIX`). The call instruction is a pseudo instruction that 5367 // represents both the indirect branch and a load that restores the TOC 5368 // pointer from the linkage area. The operand for the TOC restore is an add 5369 // of the TOC save offset to the stack pointer. This must be the second 5370 // operand: after the chain input but before any other variadic arguments. 5371 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { 5372 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister(); 5373 5374 SDValue StackPtr = DAG.getRegister(StackPtrReg, RegVT); 5375 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 5376 SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 5377 SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, StackPtr, TOCOff); 5378 Ops.push_back(AddTOC); 5379 } 5380 5381 // Add the register used for the environment pointer. 5382 if (Subtarget.usesFunctionDescriptors() && !hasNest) 5383 Ops.push_back(DAG.getRegister(Subtarget.getEnvironmentPointerRegister(), 5384 RegVT)); 5385 5386 5387 // Add CTR register as callee so a bctr can be emitted later. 5388 if (isTailCall) 5389 Ops.push_back(DAG.getRegister(IsPPC64 ? PPC::CTR8 : PPC::CTR, RegVT)); 5390 } 5391 5392 // If this is a tail call add stack pointer delta. 5393 if (isTailCall) 5394 Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32)); 5395 5396 // Add argument registers to the end of the list so that they are known live 5397 // into the call. 5398 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) 5399 Ops.push_back(DAG.getRegister(RegsToPass[i].first, 5400 RegsToPass[i].second.getValueType())); 5401 5402 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is 5403 // no way to mark dependencies as implicit here. 5404 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter. 5405 if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) && !isPatchPoint) 5406 Ops.push_back(DAG.getRegister(Subtarget.getTOCPointerRegister(), RegVT)); 5407 5408 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls 5409 if (isVarArg && Subtarget.is32BitELFABI()) 5410 Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32)); 5411 5412 // Add a register mask operand representing the call-preserved registers. 5413 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 5414 const uint32_t *Mask = 5415 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallConv); 5416 assert(Mask && "Missing call preserved mask for calling convention"); 5417 Ops.push_back(DAG.getRegisterMask(Mask)); 5418 5419 // If the glue is valid, it is the last operand. 5420 if (Glue.getNode()) 5421 Ops.push_back(Glue); 5422} 5423 5424SDValue PPCTargetLowering::FinishCall( 5425 CallingConv::ID CallConv, const SDLoc &dl, bool isTailCall, bool isVarArg, 5426 bool isPatchPoint, bool hasNest, SelectionDAG &DAG, 5427 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue, 5428 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff, 5429 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins, 5430 SmallVectorImpl<SDValue> &InVals, ImmutableCallSite CS) const { 5431 5432 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) 5433 setUsesTOCBasePtr(DAG); 5434 5435 const bool isIndirect = isIndirectCall(Callee, DAG, Subtarget, isPatchPoint); 5436 unsigned CallOpc = getCallOpcode(isIndirect, isPatchPoint, isTailCall, 5437 DAG.getMachineFunction().getFunction(), 5438 Callee, Subtarget, DAG.getTarget()); 5439 5440 if (!isIndirect) 5441 Callee = transformCallee(Callee, DAG, dl, Subtarget); 5442 else if (Subtarget.usesFunctionDescriptors()) 5443 prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CS, 5444 dl, hasNest, Subtarget); 5445 else 5446 prepareIndirectCall(DAG, Callee, Glue, Chain, dl); 5447 5448 // Build the operand list for the call instruction. 5449 SmallVector<SDValue, 8> Ops; 5450 buildCallOperands(Ops, CallConv, dl, isTailCall, isVarArg, isPatchPoint, 5451 hasNest, DAG, RegsToPass, Glue, Chain, Callee, SPDiff, 5452 Subtarget, isIndirect); 5453 5454 // Emit tail call. 5455 if (isTailCall) { 5456 assert(((Callee.getOpcode() == ISD::Register && 5457 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) || 5458 Callee.getOpcode() == ISD::TargetExternalSymbol || 5459 Callee.getOpcode() == ISD::TargetGlobalAddress || 5460 isa<ConstantSDNode>(Callee)) && 5461 "Expecting a global address, external symbol, absolute value or " 5462 "register"); 5463 assert(CallOpc == PPCISD::TC_RETURN && 5464 "Unexpected call opcode for a tail call."); 5465 DAG.getMachineFunction().getFrameInfo().setHasTailCall(); 5466 return DAG.getNode(CallOpc, dl, MVT::Other, Ops); 5467 } 5468 5469 std::array<EVT, 2> ReturnTypes = {{MVT::Other, MVT::Glue}}; 5470 Chain = DAG.getNode(CallOpc, dl, ReturnTypes, Ops); 5471 Glue = Chain.getValue(1); 5472 5473 // When performing tail call optimization the callee pops its arguments off 5474 // the stack. Account for this here so these bytes can be pushed back on in 5475 // PPCFrameLowering::eliminateCallFramePseudoInstr. 5476 int BytesCalleePops = (CallConv == CallingConv::Fast && 5477 getTargetMachine().Options.GuaranteedTailCallOpt) 5478 ? NumBytes 5479 : 0; 5480 5481 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), 5482 DAG.getIntPtrConstant(BytesCalleePops, dl, true), 5483 Glue, dl); 5484 Glue = Chain.getValue(1); 5485 5486 return LowerCallResult(Chain, Glue, CallConv, isVarArg, Ins, dl, DAG, InVals); 5487} 5488 5489SDValue 5490PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 5491 SmallVectorImpl<SDValue> &InVals) const { 5492 SelectionDAG &DAG = CLI.DAG; 5493 SDLoc &dl = CLI.DL; 5494 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 5495 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 5496 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 5497 SDValue Chain = CLI.Chain; 5498 SDValue Callee = CLI.Callee; 5499 bool &isTailCall = CLI.IsTailCall; 5500 CallingConv::ID CallConv = CLI.CallConv; 5501 bool isVarArg = CLI.IsVarArg; 5502 bool isPatchPoint = CLI.IsPatchPoint; 5503 ImmutableCallSite CS = CLI.CS; 5504 5505 if (isTailCall) { 5506 if (Subtarget.useLongCalls() && !(CS && CS.isMustTailCall())) 5507 isTailCall = false; 5508 else if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) 5509 isTailCall = 5510 IsEligibleForTailCallOptimization_64SVR4(Callee, CallConv, CS, 5511 isVarArg, Outs, Ins, DAG); 5512 else 5513 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg, 5514 Ins, DAG); 5515 if (isTailCall) { 5516 ++NumTailCalls; 5517 if (!getTargetMachine().Options.GuaranteedTailCallOpt) 5518 ++NumSiblingCalls; 5519 5520 assert(isa<GlobalAddressSDNode>(Callee) && 5521 "Callee should be an llvm::Function object."); 5522 LLVM_DEBUG( 5523 const GlobalValue *GV = 5524 cast<GlobalAddressSDNode>(Callee)->getGlobal(); 5525 const unsigned Width = 5526 80 - strlen("TCO caller: ") - strlen(", callee linkage: 0, 0"); 5527 dbgs() << "TCO caller: " 5528 << left_justify(DAG.getMachineFunction().getName(), Width) 5529 << ", callee linkage: " << GV->getVisibility() << ", " 5530 << GV->getLinkage() << "\n"); 5531 } 5532 } 5533 5534 if (!isTailCall && CS && CS.isMustTailCall()) 5535 report_fatal_error("failed to perform tail call elimination on a call " 5536 "site marked musttail"); 5537 5538 // When long calls (i.e. indirect calls) are always used, calls are always 5539 // made via function pointer. If we have a function name, first translate it 5540 // into a pointer. 5541 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Callee) && 5542 !isTailCall) 5543 Callee = LowerGlobalAddress(Callee, DAG); 5544 5545 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) 5546 return LowerCall_64SVR4(Chain, Callee, CallConv, isVarArg, 5547 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5548 dl, DAG, InVals, CS); 5549 5550 if (Subtarget.isSVR4ABI()) 5551 return LowerCall_32SVR4(Chain, Callee, CallConv, isVarArg, 5552 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5553 dl, DAG, InVals, CS); 5554 5555 if (Subtarget.isAIXABI()) 5556 return LowerCall_AIX(Chain, Callee, CallConv, isVarArg, 5557 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5558 dl, DAG, InVals, CS); 5559 5560 return LowerCall_Darwin(Chain, Callee, CallConv, isVarArg, 5561 isTailCall, isPatchPoint, Outs, OutVals, Ins, 5562 dl, DAG, InVals, CS); 5563} 5564 5565SDValue PPCTargetLowering::LowerCall_32SVR4( 5566 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 5567 bool isTailCall, bool isPatchPoint, 5568 const SmallVectorImpl<ISD::OutputArg> &Outs, 5569 const SmallVectorImpl<SDValue> &OutVals, 5570 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5571 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 5572 ImmutableCallSite CS) const { 5573 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description 5574 // of the 32-bit SVR4 ABI stack frame layout. 5575 5576 assert((CallConv == CallingConv::C || 5577 CallConv == CallingConv::Cold || 5578 CallConv == CallingConv::Fast) && "Unknown calling convention!"); 5579 5580 unsigned PtrByteSize = 4; 5581 5582 MachineFunction &MF = DAG.getMachineFunction(); 5583 5584 // Mark this function as potentially containing a function that contains a 5585 // tail call. As a consequence the frame pointer will be used for dynamicalloc 5586 // and restoring the callers stack pointer in this functions epilog. This is 5587 // done because by tail calling the called function might overwrite the value 5588 // in this function's (MF) stack pointer stack slot 0(SP). 5589 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5590 CallConv == CallingConv::Fast) 5591 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 5592 5593 // Count how many bytes are to be pushed on the stack, including the linkage 5594 // area, parameter list area and the part of the local variable space which 5595 // contains copies of aggregates which are passed by value. 5596 5597 // Assign locations to all of the outgoing arguments. 5598 SmallVector<CCValAssign, 16> ArgLocs; 5599 PPCCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); 5600 5601 // Reserve space for the linkage area on the stack. 5602 CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(), 5603 PtrByteSize); 5604 if (useSoftFloat()) 5605 CCInfo.PreAnalyzeCallOperands(Outs); 5606 5607 if (isVarArg) { 5608 // Handle fixed and variable vector arguments differently. 5609 // Fixed vector arguments go into registers as long as registers are 5610 // available. Variable vector arguments always go into memory. 5611 unsigned NumArgs = Outs.size(); 5612 5613 for (unsigned i = 0; i != NumArgs; ++i) { 5614 MVT ArgVT = Outs[i].VT; 5615 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; 5616 bool Result; 5617 5618 if (Outs[i].IsFixed) { 5619 Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, 5620 CCInfo); 5621 } else { 5622 Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, 5623 ArgFlags, CCInfo); 5624 } 5625 5626 if (Result) { 5627#ifndef NDEBUG 5628 errs() << "Call operand #" << i << " has unhandled type " 5629 << EVT(ArgVT).getEVTString() << "\n"; 5630#endif 5631 llvm_unreachable(nullptr); 5632 } 5633 } 5634 } else { 5635 // All arguments are treated the same. 5636 CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4); 5637 } 5638 CCInfo.clearWasPPCF128(); 5639 5640 // Assign locations to all of the outgoing aggregate by value arguments. 5641 SmallVector<CCValAssign, 16> ByValArgLocs; 5642 CCState CCByValInfo(CallConv, isVarArg, MF, ByValArgLocs, *DAG.getContext()); 5643 5644 // Reserve stack space for the allocations in CCInfo. 5645 CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); 5646 5647 CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal); 5648 5649 // Size of the linkage area, parameter list area and the part of the local 5650 // space variable where copies of aggregates which are passed by value are 5651 // stored. 5652 unsigned NumBytes = CCByValInfo.getNextStackOffset(); 5653 5654 // Calculate by how many bytes the stack has to be adjusted in case of tail 5655 // call optimization. 5656 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 5657 5658 // Adjust the stack pointer for the new arguments... 5659 // These operations are automatically eliminated by the prolog/epilog pass 5660 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 5661 SDValue CallSeqStart = Chain; 5662 5663 // Load the return address and frame pointer so it can be moved somewhere else 5664 // later. 5665 SDValue LROp, FPOp; 5666 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 5667 5668 // Set up a copy of the stack pointer for use loading and storing any 5669 // arguments that may not fit in the registers available for argument 5670 // passing. 5671 SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 5672 5673 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 5674 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 5675 SmallVector<SDValue, 8> MemOpChains; 5676 5677 bool seenFloatArg = false; 5678 // Walk the register/memloc assignments, inserting copies/loads. 5679 // i - Tracks the index into the list of registers allocated for the call 5680 // RealArgIdx - Tracks the index into the list of actual function arguments 5681 // j - Tracks the index into the list of byval arguments 5682 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size(); 5683 i != e; 5684 ++i, ++RealArgIdx) { 5685 CCValAssign &VA = ArgLocs[i]; 5686 SDValue Arg = OutVals[RealArgIdx]; 5687 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags; 5688 5689 if (Flags.isByVal()) { 5690 // Argument is an aggregate which is passed by value, thus we need to 5691 // create a copy of it in the local variable space of the current stack 5692 // frame (which is the stack frame of the caller) and pass the address of 5693 // this copy to the callee. 5694 assert((j < ByValArgLocs.size()) && "Index out of bounds!"); 5695 CCValAssign &ByValVA = ByValArgLocs[j++]; 5696 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!"); 5697 5698 // Memory reserved in the local variable space of the callers stack frame. 5699 unsigned LocMemOffset = ByValVA.getLocMemOffset(); 5700 5701 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 5702 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 5703 StackPtr, PtrOff); 5704 5705 // Create a copy of the argument in the local area of the current 5706 // stack frame. 5707 SDValue MemcpyCall = 5708 CreateCopyOfByValArgument(Arg, PtrOff, 5709 CallSeqStart.getNode()->getOperand(0), 5710 Flags, DAG, dl); 5711 5712 // This must go outside the CALLSEQ_START..END. 5713 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0, 5714 SDLoc(MemcpyCall)); 5715 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 5716 NewCallSeqStart.getNode()); 5717 Chain = CallSeqStart = NewCallSeqStart; 5718 5719 // Pass the address of the aggregate copy on the stack either in a 5720 // physical register or in the parameter list area of the current stack 5721 // frame to the callee. 5722 Arg = PtrOff; 5723 } 5724 5725 // When useCRBits() is true, there can be i1 arguments. 5726 // It is because getRegisterType(MVT::i1) => MVT::i1, 5727 // and for other integer types getRegisterType() => MVT::i32. 5728 // Extend i1 and ensure callee will get i32. 5729 if (Arg.getValueType() == MVT::i1) 5730 Arg = DAG.getNode(Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, 5731 dl, MVT::i32, Arg); 5732 5733 if (VA.isRegLoc()) { 5734 seenFloatArg |= VA.getLocVT().isFloatingPoint(); 5735 // Put argument in a physical register. 5736 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) { 5737 bool IsLE = Subtarget.isLittleEndian(); 5738 SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 5739 DAG.getIntPtrConstant(IsLE ? 0 : 1, dl)); 5740 RegsToPass.push_back(std::make_pair(VA.getLocReg(), SVal.getValue(0))); 5741 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 5742 DAG.getIntPtrConstant(IsLE ? 1 : 0, dl)); 5743 RegsToPass.push_back(std::make_pair(ArgLocs[++i].getLocReg(), 5744 SVal.getValue(0))); 5745 } else 5746 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 5747 } else { 5748 // Put argument in the parameter list area of the current stack frame. 5749 assert(VA.isMemLoc()); 5750 unsigned LocMemOffset = VA.getLocMemOffset(); 5751 5752 if (!isTailCall) { 5753 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); 5754 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), 5755 StackPtr, PtrOff); 5756 5757 MemOpChains.push_back( 5758 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); 5759 } else { 5760 // Calculate and remember argument location. 5761 CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset, 5762 TailCallArguments); 5763 } 5764 } 5765 } 5766 5767 if (!MemOpChains.empty()) 5768 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 5769 5770 // Build a sequence of copy-to-reg nodes chained together with token chain 5771 // and flag operands which copy the outgoing args into the appropriate regs. 5772 SDValue InFlag; 5773 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 5774 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 5775 RegsToPass[i].second, InFlag); 5776 InFlag = Chain.getValue(1); 5777 } 5778 5779 // Set CR bit 6 to true if this is a vararg call with floating args passed in 5780 // registers. 5781 if (isVarArg) { 5782 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); 5783 SDValue Ops[] = { Chain, InFlag }; 5784 5785 Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, 5786 dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1)); 5787 5788 InFlag = Chain.getValue(1); 5789 } 5790 5791 if (isTailCall) 5792 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 5793 TailCallArguments); 5794 5795 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, 5796 /* unused except on PPC64 ELFv1 */ false, DAG, 5797 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, 5798 NumBytes, Ins, InVals, CS); 5799} 5800 5801// Copy an argument into memory, being careful to do this outside the 5802// call sequence for the call to which the argument belongs. 5803SDValue PPCTargetLowering::createMemcpyOutsideCallSeq( 5804 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags, 5805 SelectionDAG &DAG, const SDLoc &dl) const { 5806 SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, 5807 CallSeqStart.getNode()->getOperand(0), 5808 Flags, DAG, dl); 5809 // The MEMCPY must go outside the CALLSEQ_START..END. 5810 int64_t FrameSize = CallSeqStart.getConstantOperandVal(1); 5811 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0, 5812 SDLoc(MemcpyCall)); 5813 DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), 5814 NewCallSeqStart.getNode()); 5815 return NewCallSeqStart; 5816} 5817 5818SDValue PPCTargetLowering::LowerCall_64SVR4( 5819 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 5820 bool isTailCall, bool isPatchPoint, 5821 const SmallVectorImpl<ISD::OutputArg> &Outs, 5822 const SmallVectorImpl<SDValue> &OutVals, 5823 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 5824 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 5825 ImmutableCallSite CS) const { 5826 bool isELFv2ABI = Subtarget.isELFv2ABI(); 5827 bool isLittleEndian = Subtarget.isLittleEndian(); 5828 unsigned NumOps = Outs.size(); 5829 bool hasNest = false; 5830 bool IsSibCall = false; 5831 5832 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 5833 unsigned PtrByteSize = 8; 5834 5835 MachineFunction &MF = DAG.getMachineFunction(); 5836 5837 if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt) 5838 IsSibCall = true; 5839 5840 // Mark this function as potentially containing a function that contains a 5841 // tail call. As a consequence the frame pointer will be used for dynamicalloc 5842 // and restoring the callers stack pointer in this functions epilog. This is 5843 // done because by tail calling the called function might overwrite the value 5844 // in this function's (MF) stack pointer stack slot 0(SP). 5845 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5846 CallConv == CallingConv::Fast) 5847 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 5848 5849 assert(!(CallConv == CallingConv::Fast && isVarArg) && 5850 "fastcc not supported on varargs functions"); 5851 5852 // Count how many bytes are to be pushed on the stack, including the linkage 5853 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes 5854 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage 5855 // area is 32 bytes reserved space for [SP][CR][LR][TOC]. 5856 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 5857 unsigned NumBytes = LinkageSize; 5858 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 5859 unsigned &QFPR_idx = FPR_idx; 5860 5861 static const MCPhysReg GPR[] = { 5862 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 5863 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 5864 }; 5865 static const MCPhysReg VR[] = { 5866 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 5867 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 5868 }; 5869 5870 const unsigned NumGPRs = array_lengthof(GPR); 5871 const unsigned NumFPRs = useSoftFloat() ? 0 : 13; 5872 const unsigned NumVRs = array_lengthof(VR); 5873 const unsigned NumQFPRs = NumFPRs; 5874 5875 // On ELFv2, we can avoid allocating the parameter area if all the arguments 5876 // can be passed to the callee in registers. 5877 // For the fast calling convention, there is another check below. 5878 // Note: We should keep consistent with LowerFormalArguments_64SVR4() 5879 bool HasParameterArea = !isELFv2ABI || isVarArg || CallConv == CallingConv::Fast; 5880 if (!HasParameterArea) { 5881 unsigned ParamAreaSize = NumGPRs * PtrByteSize; 5882 unsigned AvailableFPRs = NumFPRs; 5883 unsigned AvailableVRs = NumVRs; 5884 unsigned NumBytesTmp = NumBytes; 5885 for (unsigned i = 0; i != NumOps; ++i) { 5886 if (Outs[i].Flags.isNest()) continue; 5887 if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags, 5888 PtrByteSize, LinkageSize, ParamAreaSize, 5889 NumBytesTmp, AvailableFPRs, AvailableVRs, 5890 Subtarget.hasQPX())) 5891 HasParameterArea = true; 5892 } 5893 } 5894 5895 // When using the fast calling convention, we don't provide backing for 5896 // arguments that will be in registers. 5897 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0; 5898 5899 // Avoid allocating parameter area for fastcc functions if all the arguments 5900 // can be passed in the registers. 5901 if (CallConv == CallingConv::Fast) 5902 HasParameterArea = false; 5903 5904 // Add up all the space actually used. 5905 for (unsigned i = 0; i != NumOps; ++i) { 5906 ISD::ArgFlagsTy Flags = Outs[i].Flags; 5907 EVT ArgVT = Outs[i].VT; 5908 EVT OrigVT = Outs[i].ArgVT; 5909 5910 if (Flags.isNest()) 5911 continue; 5912 5913 if (CallConv == CallingConv::Fast) { 5914 if (Flags.isByVal()) { 5915 NumGPRsUsed += (Flags.getByValSize()+7)/8; 5916 if (NumGPRsUsed > NumGPRs) 5917 HasParameterArea = true; 5918 } else { 5919 switch (ArgVT.getSimpleVT().SimpleTy) { 5920 default: llvm_unreachable("Unexpected ValueType for argument!"); 5921 case MVT::i1: 5922 case MVT::i32: 5923 case MVT::i64: 5924 if (++NumGPRsUsed <= NumGPRs) 5925 continue; 5926 break; 5927 case MVT::v4i32: 5928 case MVT::v8i16: 5929 case MVT::v16i8: 5930 case MVT::v2f64: 5931 case MVT::v2i64: 5932 case MVT::v1i128: 5933 case MVT::f128: 5934 if (++NumVRsUsed <= NumVRs) 5935 continue; 5936 break; 5937 case MVT::v4f32: 5938 // When using QPX, this is handled like a FP register, otherwise, it 5939 // is an Altivec register. 5940 if (Subtarget.hasQPX()) { 5941 if (++NumFPRsUsed <= NumFPRs) 5942 continue; 5943 } else { 5944 if (++NumVRsUsed <= NumVRs) 5945 continue; 5946 } 5947 break; 5948 case MVT::f32: 5949 case MVT::f64: 5950 case MVT::v4f64: // QPX 5951 case MVT::v4i1: // QPX 5952 if (++NumFPRsUsed <= NumFPRs) 5953 continue; 5954 break; 5955 } 5956 HasParameterArea = true; 5957 } 5958 } 5959 5960 /* Respect alignment of argument on the stack. */ 5961 unsigned Align = 5962 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 5963 NumBytes = ((NumBytes + Align - 1) / Align) * Align; 5964 5965 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 5966 if (Flags.isInConsecutiveRegsLast()) 5967 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 5968 } 5969 5970 unsigned NumBytesActuallyUsed = NumBytes; 5971 5972 // In the old ELFv1 ABI, 5973 // the prolog code of the callee may store up to 8 GPR argument registers to 5974 // the stack, allowing va_start to index over them in memory if its varargs. 5975 // Because we cannot tell if this is needed on the caller side, we have to 5976 // conservatively assume that it is needed. As such, make sure we have at 5977 // least enough stack space for the caller to store the 8 GPRs. 5978 // In the ELFv2 ABI, we allocate the parameter area iff a callee 5979 // really requires memory operands, e.g. a vararg function. 5980 if (HasParameterArea) 5981 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); 5982 else 5983 NumBytes = LinkageSize; 5984 5985 // Tail call needs the stack to be aligned. 5986 if (getTargetMachine().Options.GuaranteedTailCallOpt && 5987 CallConv == CallingConv::Fast) 5988 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); 5989 5990 int SPDiff = 0; 5991 5992 // Calculate by how many bytes the stack has to be adjusted in case of tail 5993 // call optimization. 5994 if (!IsSibCall) 5995 SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 5996 5997 // To protect arguments on the stack from being clobbered in a tail call, 5998 // force all the loads to happen before doing any other lowering. 5999 if (isTailCall) 6000 Chain = DAG.getStackArgumentTokenFactor(Chain); 6001 6002 // Adjust the stack pointer for the new arguments... 6003 // These operations are automatically eliminated by the prolog/epilog pass 6004 if (!IsSibCall) 6005 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 6006 SDValue CallSeqStart = Chain; 6007 6008 // Load the return address and frame pointer so it can be move somewhere else 6009 // later. 6010 SDValue LROp, FPOp; 6011 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 6012 6013 // Set up a copy of the stack pointer for use loading and storing any 6014 // arguments that may not fit in the registers available for argument 6015 // passing. 6016 SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 6017 6018 // Figure out which arguments are going to go in registers, and which in 6019 // memory. Also, if this is a vararg function, floating point operations 6020 // must be stored to our stack, and loaded into integer regs as well, if 6021 // any integer regs are available for argument passing. 6022 unsigned ArgOffset = LinkageSize; 6023 6024 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 6025 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 6026 6027 SmallVector<SDValue, 8> MemOpChains; 6028 for (unsigned i = 0; i != NumOps; ++i) { 6029 SDValue Arg = OutVals[i]; 6030 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6031 EVT ArgVT = Outs[i].VT; 6032 EVT OrigVT = Outs[i].ArgVT; 6033 6034 // PtrOff will be used to store the current argument to the stack if a 6035 // register cannot be found for it. 6036 SDValue PtrOff; 6037 6038 // We re-align the argument offset for each argument, except when using the 6039 // fast calling convention, when we need to make sure we do that only when 6040 // we'll actually use a stack slot. 6041 auto ComputePtrOff = [&]() { 6042 /* Respect alignment of argument on the stack. */ 6043 unsigned Align = 6044 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); 6045 ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; 6046 6047 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); 6048 6049 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 6050 }; 6051 6052 if (CallConv != CallingConv::Fast) { 6053 ComputePtrOff(); 6054 6055 /* Compute GPR index associated with argument offset. */ 6056 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; 6057 GPR_idx = std::min(GPR_idx, NumGPRs); 6058 } 6059 6060 // Promote integers to 64-bit values. 6061 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) { 6062 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 6063 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 6064 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 6065 } 6066 6067 // FIXME memcpy is used way more than necessary. Correctness first. 6068 // Note: "by value" is code for passing a structure by value, not 6069 // basic types. 6070 if (Flags.isByVal()) { 6071 // Note: Size includes alignment padding, so 6072 // struct x { short a; char b; } 6073 // will have Size = 4. With #pragma pack(1), it will have Size = 3. 6074 // These are the proper values we need for right-justifying the 6075 // aggregate in a parameter register. 6076 unsigned Size = Flags.getByValSize(); 6077 6078 // An empty aggregate parameter takes up no storage and no 6079 // registers. 6080 if (Size == 0) 6081 continue; 6082 6083 if (CallConv == CallingConv::Fast) 6084 ComputePtrOff(); 6085 6086 // All aggregates smaller than 8 bytes must be passed right-justified. 6087 if (Size==1 || Size==2 || Size==4) { 6088 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32); 6089 if (GPR_idx != NumGPRs) { 6090 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 6091 MachinePointerInfo(), VT); 6092 MemOpChains.push_back(Load.getValue(1)); 6093 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6094 6095 ArgOffset += PtrByteSize; 6096 continue; 6097 } 6098 } 6099 6100 if (GPR_idx == NumGPRs && Size < 8) { 6101 SDValue AddPtr = PtrOff; 6102 if (!isLittleEndian) { 6103 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, 6104 PtrOff.getValueType()); 6105 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 6106 } 6107 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 6108 CallSeqStart, 6109 Flags, DAG, dl); 6110 ArgOffset += PtrByteSize; 6111 continue; 6112 } 6113 // Copy entire object into memory. There are cases where gcc-generated 6114 // code assumes it is there, even if it could be put entirely into 6115 // registers. (This is not what the doc says.) 6116 6117 // FIXME: The above statement is likely due to a misunderstanding of the 6118 // documents. All arguments must be copied into the parameter area BY 6119 // THE CALLEE in the event that the callee takes the address of any 6120 // formal argument. That has not yet been implemented. However, it is 6121 // reasonable to use the stack area as a staging area for the register 6122 // load. 6123 6124 // Skip this for small aggregates, as we will use the same slot for a 6125 // right-justified copy, below. 6126 if (Size >= 8) 6127 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 6128 CallSeqStart, 6129 Flags, DAG, dl); 6130 6131 // When a register is available, pass a small aggregate right-justified. 6132 if (Size < 8 && GPR_idx != NumGPRs) { 6133 // The easiest way to get this right-justified in a register 6134 // is to copy the structure into the rightmost portion of a 6135 // local variable slot, then load the whole slot into the 6136 // register. 6137 // FIXME: The memcpy seems to produce pretty awful code for 6138 // small aggregates, particularly for packed ones. 6139 // FIXME: It would be preferable to use the slot in the 6140 // parameter save area instead of a new local variable. 6141 SDValue AddPtr = PtrOff; 6142 if (!isLittleEndian) { 6143 SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType()); 6144 AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 6145 } 6146 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 6147 CallSeqStart, 6148 Flags, DAG, dl); 6149 6150 // Load the slot into the register. 6151 SDValue Load = 6152 DAG.getLoad(PtrVT, dl, Chain, PtrOff, MachinePointerInfo()); 6153 MemOpChains.push_back(Load.getValue(1)); 6154 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6155 6156 // Done with this argument. 6157 ArgOffset += PtrByteSize; 6158 continue; 6159 } 6160 6161 // For aggregates larger than PtrByteSize, copy the pieces of the 6162 // object that fit into registers from the parameter save area. 6163 for (unsigned j=0; j<Size; j+=PtrByteSize) { 6164 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); 6165 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 6166 if (GPR_idx != NumGPRs) { 6167 SDValue Load = 6168 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo()); 6169 MemOpChains.push_back(Load.getValue(1)); 6170 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6171 ArgOffset += PtrByteSize; 6172 } else { 6173 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 6174 break; 6175 } 6176 } 6177 continue; 6178 } 6179 6180 switch (Arg.getSimpleValueType().SimpleTy) { 6181 default: llvm_unreachable("Unexpected ValueType for argument!"); 6182 case MVT::i1: 6183 case MVT::i32: 6184 case MVT::i64: 6185 if (Flags.isNest()) { 6186 // The 'nest' parameter, if any, is passed in R11. 6187 RegsToPass.push_back(std::make_pair(PPC::X11, Arg)); 6188 hasNest = true; 6189 break; 6190 } 6191 6192 // These can be scalar arguments or elements of an integer array type 6193 // passed directly. Clang may use those instead of "byval" aggregate 6194 // types to avoid forcing arguments to memory unnecessarily. 6195 if (GPR_idx != NumGPRs) { 6196 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 6197 } else { 6198 if (CallConv == CallingConv::Fast) 6199 ComputePtrOff(); 6200 6201 assert(HasParameterArea && 6202 "Parameter area must exist to pass an argument in memory."); 6203 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6204 true, isTailCall, false, MemOpChains, 6205 TailCallArguments, dl); 6206 if (CallConv == CallingConv::Fast) 6207 ArgOffset += PtrByteSize; 6208 } 6209 if (CallConv != CallingConv::Fast) 6210 ArgOffset += PtrByteSize; 6211 break; 6212 case MVT::f32: 6213 case MVT::f64: { 6214 // These can be scalar arguments or elements of a float array type 6215 // passed directly. The latter are used to implement ELFv2 homogenous 6216 // float aggregates. 6217 6218 // Named arguments go into FPRs first, and once they overflow, the 6219 // remaining arguments go into GPRs and then the parameter save area. 6220 // Unnamed arguments for vararg functions always go to GPRs and 6221 // then the parameter save area. For now, put all arguments to vararg 6222 // routines always in both locations (FPR *and* GPR or stack slot). 6223 bool NeedGPROrStack = isVarArg || FPR_idx == NumFPRs; 6224 bool NeededLoad = false; 6225 6226 // First load the argument into the next available FPR. 6227 if (FPR_idx != NumFPRs) 6228 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 6229 6230 // Next, load the argument into GPR or stack slot if needed. 6231 if (!NeedGPROrStack) 6232 ; 6233 else if (GPR_idx != NumGPRs && CallConv != CallingConv::Fast) { 6234 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 6235 // once we support fp <-> gpr moves. 6236 6237 // In the non-vararg case, this can only ever happen in the 6238 // presence of f32 array types, since otherwise we never run 6239 // out of FPRs before running out of GPRs. 6240 SDValue ArgVal; 6241 6242 // Double values are always passed in a single GPR. 6243 if (Arg.getValueType() != MVT::f32) { 6244 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); 6245 6246 // Non-array float values are extended and passed in a GPR. 6247 } else if (!Flags.isInConsecutiveRegs()) { 6248 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6249 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 6250 6251 // If we have an array of floats, we collect every odd element 6252 // together with its predecessor into one GPR. 6253 } else if (ArgOffset % PtrByteSize != 0) { 6254 SDValue Lo, Hi; 6255 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]); 6256 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6257 if (!isLittleEndian) 6258 std::swap(Lo, Hi); 6259 ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); 6260 6261 // The final element, if even, goes into the first half of a GPR. 6262 } else if (Flags.isInConsecutiveRegsLast()) { 6263 ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); 6264 ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); 6265 if (!isLittleEndian) 6266 ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal, 6267 DAG.getConstant(32, dl, MVT::i32)); 6268 6269 // Non-final even elements are skipped; they will be handled 6270 // together the with subsequent argument on the next go-around. 6271 } else 6272 ArgVal = SDValue(); 6273 6274 if (ArgVal.getNode()) 6275 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal)); 6276 } else { 6277 if (CallConv == CallingConv::Fast) 6278 ComputePtrOff(); 6279 6280 // Single-precision floating-point values are mapped to the 6281 // second (rightmost) word of the stack doubleword. 6282 if (Arg.getValueType() == MVT::f32 && 6283 !isLittleEndian && !Flags.isInConsecutiveRegs()) { 6284 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); 6285 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 6286 } 6287 6288 assert(HasParameterArea && 6289 "Parameter area must exist to pass an argument in memory."); 6290 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6291 true, isTailCall, false, MemOpChains, 6292 TailCallArguments, dl); 6293 6294 NeededLoad = true; 6295 } 6296 // When passing an array of floats, the array occupies consecutive 6297 // space in the argument area; only round up to the next doubleword 6298 // at the end of the array. Otherwise, each float takes 8 bytes. 6299 if (CallConv != CallingConv::Fast || NeededLoad) { 6300 ArgOffset += (Arg.getValueType() == MVT::f32 && 6301 Flags.isInConsecutiveRegs()) ? 4 : 8; 6302 if (Flags.isInConsecutiveRegsLast()) 6303 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; 6304 } 6305 break; 6306 } 6307 case MVT::v4f32: 6308 case MVT::v4i32: 6309 case MVT::v8i16: 6310 case MVT::v16i8: 6311 case MVT::v2f64: 6312 case MVT::v2i64: 6313 case MVT::v1i128: 6314 case MVT::f128: 6315 if (!Subtarget.hasQPX()) { 6316 // These can be scalar arguments or elements of a vector array type 6317 // passed directly. The latter are used to implement ELFv2 homogenous 6318 // vector aggregates. 6319 6320 // For a varargs call, named arguments go into VRs or on the stack as 6321 // usual; unnamed arguments always go to the stack or the corresponding 6322 // GPRs when within range. For now, we always put the value in both 6323 // locations (or even all three). 6324 if (isVarArg) { 6325 assert(HasParameterArea && 6326 "Parameter area must exist if we have a varargs call."); 6327 // We could elide this store in the case where the object fits 6328 // entirely in R registers. Maybe later. 6329 SDValue Store = 6330 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6331 MemOpChains.push_back(Store); 6332 if (VR_idx != NumVRs) { 6333 SDValue Load = 6334 DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); 6335 MemOpChains.push_back(Load.getValue(1)); 6336 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); 6337 } 6338 ArgOffset += 16; 6339 for (unsigned i=0; i<16; i+=PtrByteSize) { 6340 if (GPR_idx == NumGPRs) 6341 break; 6342 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 6343 DAG.getConstant(i, dl, PtrVT)); 6344 SDValue Load = 6345 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); 6346 MemOpChains.push_back(Load.getValue(1)); 6347 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6348 } 6349 break; 6350 } 6351 6352 // Non-varargs Altivec params go into VRs or on the stack. 6353 if (VR_idx != NumVRs) { 6354 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); 6355 } else { 6356 if (CallConv == CallingConv::Fast) 6357 ComputePtrOff(); 6358 6359 assert(HasParameterArea && 6360 "Parameter area must exist to pass an argument in memory."); 6361 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6362 true, isTailCall, true, MemOpChains, 6363 TailCallArguments, dl); 6364 if (CallConv == CallingConv::Fast) 6365 ArgOffset += 16; 6366 } 6367 6368 if (CallConv != CallingConv::Fast) 6369 ArgOffset += 16; 6370 break; 6371 } // not QPX 6372 6373 assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 && 6374 "Invalid QPX parameter type"); 6375 6376 LLVM_FALLTHROUGH; 6377 case MVT::v4f64: 6378 case MVT::v4i1: { 6379 bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32; 6380 if (isVarArg) { 6381 assert(HasParameterArea && 6382 "Parameter area must exist if we have a varargs call."); 6383 // We could elide this store in the case where the object fits 6384 // entirely in R registers. Maybe later. 6385 SDValue Store = 6386 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6387 MemOpChains.push_back(Store); 6388 if (QFPR_idx != NumQFPRs) { 6389 SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store, 6390 PtrOff, MachinePointerInfo()); 6391 MemOpChains.push_back(Load.getValue(1)); 6392 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load)); 6393 } 6394 ArgOffset += (IsF32 ? 16 : 32); 6395 for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) { 6396 if (GPR_idx == NumGPRs) 6397 break; 6398 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 6399 DAG.getConstant(i, dl, PtrVT)); 6400 SDValue Load = 6401 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); 6402 MemOpChains.push_back(Load.getValue(1)); 6403 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6404 } 6405 break; 6406 } 6407 6408 // Non-varargs QPX params go into registers or on the stack. 6409 if (QFPR_idx != NumQFPRs) { 6410 RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg)); 6411 } else { 6412 if (CallConv == CallingConv::Fast) 6413 ComputePtrOff(); 6414 6415 assert(HasParameterArea && 6416 "Parameter area must exist to pass an argument in memory."); 6417 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6418 true, isTailCall, true, MemOpChains, 6419 TailCallArguments, dl); 6420 if (CallConv == CallingConv::Fast) 6421 ArgOffset += (IsF32 ? 16 : 32); 6422 } 6423 6424 if (CallConv != CallingConv::Fast) 6425 ArgOffset += (IsF32 ? 16 : 32); 6426 break; 6427 } 6428 } 6429 } 6430 6431 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) && 6432 "mismatch in size of parameter area"); 6433 (void)NumBytesActuallyUsed; 6434 6435 if (!MemOpChains.empty()) 6436 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 6437 6438 // Check if this is an indirect call (MTCTR/BCTRL). 6439 // See prepareDescriptorIndirectCall and buildCallOperands for more 6440 // information about calls through function pointers in the 64-bit SVR4 ABI. 6441 if (!isTailCall && !isPatchPoint && 6442 !isFunctionGlobalAddress(Callee) && 6443 !isa<ExternalSymbolSDNode>(Callee)) { 6444 // Load r2 into a virtual register and store it to the TOC save area. 6445 setUsesTOCBasePtr(DAG); 6446 SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64); 6447 // TOC save area offset. 6448 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 6449 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 6450 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 6451 Chain = DAG.getStore( 6452 Val.getValue(1), dl, Val, AddPtr, 6453 MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset)); 6454 // In the ELFv2 ABI, R12 must contain the address of an indirect callee. 6455 // This does not mean the MTCTR instruction must use R12; it's easier 6456 // to model this as an extra parameter, so do that. 6457 if (isELFv2ABI && !isPatchPoint) 6458 RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee)); 6459 } 6460 6461 // Build a sequence of copy-to-reg nodes chained together with token chain 6462 // and flag operands which copy the outgoing args into the appropriate regs. 6463 SDValue InFlag; 6464 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 6465 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 6466 RegsToPass[i].second, InFlag); 6467 InFlag = Chain.getValue(1); 6468 } 6469 6470 if (isTailCall && !IsSibCall) 6471 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 6472 TailCallArguments); 6473 6474 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, hasNest, 6475 DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, 6476 SPDiff, NumBytes, Ins, InVals, CS); 6477} 6478 6479SDValue PPCTargetLowering::LowerCall_Darwin( 6480 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 6481 bool isTailCall, bool isPatchPoint, 6482 const SmallVectorImpl<ISD::OutputArg> &Outs, 6483 const SmallVectorImpl<SDValue> &OutVals, 6484 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 6485 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 6486 ImmutableCallSite CS) const { 6487 unsigned NumOps = Outs.size(); 6488 6489 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 6490 bool isPPC64 = PtrVT == MVT::i64; 6491 unsigned PtrByteSize = isPPC64 ? 8 : 4; 6492 6493 MachineFunction &MF = DAG.getMachineFunction(); 6494 6495 // Mark this function as potentially containing a function that contains a 6496 // tail call. As a consequence the frame pointer will be used for dynamicalloc 6497 // and restoring the callers stack pointer in this functions epilog. This is 6498 // done because by tail calling the called function might overwrite the value 6499 // in this function's (MF) stack pointer stack slot 0(SP). 6500 if (getTargetMachine().Options.GuaranteedTailCallOpt && 6501 CallConv == CallingConv::Fast) 6502 MF.getInfo<PPCFunctionInfo>()->setHasFastCall(); 6503 6504 // Count how many bytes are to be pushed on the stack, including the linkage 6505 // area, and parameter passing area. We start with 24/48 bytes, which is 6506 // prereserved space for [SP][CR][LR][3 x unused]. 6507 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 6508 unsigned NumBytes = LinkageSize; 6509 6510 // Add up all the space actually used. 6511 // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually 6512 // they all go in registers, but we must reserve stack space for them for 6513 // possible use by the caller. In varargs or 64-bit calls, parameters are 6514 // assigned stack space in order, with padding so Altivec parameters are 6515 // 16-byte aligned. 6516 unsigned nAltivecParamsAtEnd = 0; 6517 for (unsigned i = 0; i != NumOps; ++i) { 6518 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6519 EVT ArgVT = Outs[i].VT; 6520 // Varargs Altivec parameters are padded to a 16 byte boundary. 6521 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || 6522 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || 6523 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64) { 6524 if (!isVarArg && !isPPC64) { 6525 // Non-varargs Altivec parameters go after all the non-Altivec 6526 // parameters; handle those later so we know how much padding we need. 6527 nAltivecParamsAtEnd++; 6528 continue; 6529 } 6530 // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary. 6531 NumBytes = ((NumBytes+15)/16)*16; 6532 } 6533 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); 6534 } 6535 6536 // Allow for Altivec parameters at the end, if needed. 6537 if (nAltivecParamsAtEnd) { 6538 NumBytes = ((NumBytes+15)/16)*16; 6539 NumBytes += 16*nAltivecParamsAtEnd; 6540 } 6541 6542 // The prolog code of the callee may store up to 8 GPR argument registers to 6543 // the stack, allowing va_start to index over them in memory if its varargs. 6544 // Because we cannot tell if this is needed on the caller side, we have to 6545 // conservatively assume that it is needed. As such, make sure we have at 6546 // least enough stack space for the caller to store the 8 GPRs. 6547 NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); 6548 6549 // Tail call needs the stack to be aligned. 6550 if (getTargetMachine().Options.GuaranteedTailCallOpt && 6551 CallConv == CallingConv::Fast) 6552 NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); 6553 6554 // Calculate by how many bytes the stack has to be adjusted in case of tail 6555 // call optimization. 6556 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); 6557 6558 // To protect arguments on the stack from being clobbered in a tail call, 6559 // force all the loads to happen before doing any other lowering. 6560 if (isTailCall) 6561 Chain = DAG.getStackArgumentTokenFactor(Chain); 6562 6563 // Adjust the stack pointer for the new arguments... 6564 // These operations are automatically eliminated by the prolog/epilog pass 6565 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 6566 SDValue CallSeqStart = Chain; 6567 6568 // Load the return address and frame pointer so it can be move somewhere else 6569 // later. 6570 SDValue LROp, FPOp; 6571 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); 6572 6573 // Set up a copy of the stack pointer for use loading and storing any 6574 // arguments that may not fit in the registers available for argument 6575 // passing. 6576 SDValue StackPtr; 6577 if (isPPC64) 6578 StackPtr = DAG.getRegister(PPC::X1, MVT::i64); 6579 else 6580 StackPtr = DAG.getRegister(PPC::R1, MVT::i32); 6581 6582 // Figure out which arguments are going to go in registers, and which in 6583 // memory. Also, if this is a vararg function, floating point operations 6584 // must be stored to our stack, and loaded into integer regs as well, if 6585 // any integer regs are available for argument passing. 6586 unsigned ArgOffset = LinkageSize; 6587 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; 6588 6589 static const MCPhysReg GPR_32[] = { // 32-bit registers. 6590 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 6591 PPC::R7, PPC::R8, PPC::R9, PPC::R10, 6592 }; 6593 static const MCPhysReg GPR_64[] = { // 64-bit registers. 6594 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 6595 PPC::X7, PPC::X8, PPC::X9, PPC::X10, 6596 }; 6597 static const MCPhysReg VR[] = { 6598 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, 6599 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 6600 }; 6601 const unsigned NumGPRs = array_lengthof(GPR_32); 6602 const unsigned NumFPRs = 13; 6603 const unsigned NumVRs = array_lengthof(VR); 6604 6605 const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; 6606 6607 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 6608 SmallVector<TailCallArgumentInfo, 8> TailCallArguments; 6609 6610 SmallVector<SDValue, 8> MemOpChains; 6611 for (unsigned i = 0; i != NumOps; ++i) { 6612 SDValue Arg = OutVals[i]; 6613 ISD::ArgFlagsTy Flags = Outs[i].Flags; 6614 6615 // PtrOff will be used to store the current argument to the stack if a 6616 // register cannot be found for it. 6617 SDValue PtrOff; 6618 6619 PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); 6620 6621 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 6622 6623 // On PPC64, promote integers to 64-bit values. 6624 if (isPPC64 && Arg.getValueType() == MVT::i32) { 6625 // FIXME: Should this use ANY_EXTEND if neither sext nor zext? 6626 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 6627 Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); 6628 } 6629 6630 // FIXME memcpy is used way more than necessary. Correctness first. 6631 // Note: "by value" is code for passing a structure by value, not 6632 // basic types. 6633 if (Flags.isByVal()) { 6634 unsigned Size = Flags.getByValSize(); 6635 // Very small objects are passed right-justified. Everything else is 6636 // passed left-justified. 6637 if (Size==1 || Size==2) { 6638 EVT VT = (Size==1) ? MVT::i8 : MVT::i16; 6639 if (GPR_idx != NumGPRs) { 6640 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, 6641 MachinePointerInfo(), VT); 6642 MemOpChains.push_back(Load.getValue(1)); 6643 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6644 6645 ArgOffset += PtrByteSize; 6646 } else { 6647 SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, 6648 PtrOff.getValueType()); 6649 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); 6650 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, 6651 CallSeqStart, 6652 Flags, DAG, dl); 6653 ArgOffset += PtrByteSize; 6654 } 6655 continue; 6656 } 6657 // Copy entire object into memory. There are cases where gcc-generated 6658 // code assumes it is there, even if it could be put entirely into 6659 // registers. (This is not what the doc says.) 6660 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, 6661 CallSeqStart, 6662 Flags, DAG, dl); 6663 6664 // For small aggregates (Darwin only) and aggregates >= PtrByteSize, 6665 // copy the pieces of the object that fit into registers from the 6666 // parameter save area. 6667 for (unsigned j=0; j<Size; j+=PtrByteSize) { 6668 SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType()); 6669 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const); 6670 if (GPR_idx != NumGPRs) { 6671 SDValue Load = 6672 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo()); 6673 MemOpChains.push_back(Load.getValue(1)); 6674 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6675 ArgOffset += PtrByteSize; 6676 } else { 6677 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize; 6678 break; 6679 } 6680 } 6681 continue; 6682 } 6683 6684 switch (Arg.getSimpleValueType().SimpleTy) { 6685 default: llvm_unreachable("Unexpected ValueType for argument!"); 6686 case MVT::i1: 6687 case MVT::i32: 6688 case MVT::i64: 6689 if (GPR_idx != NumGPRs) { 6690 if (Arg.getValueType() == MVT::i1) 6691 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, PtrVT, Arg); 6692 6693 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); 6694 } else { 6695 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6696 isPPC64, isTailCall, false, MemOpChains, 6697 TailCallArguments, dl); 6698 } 6699 ArgOffset += PtrByteSize; 6700 break; 6701 case MVT::f32: 6702 case MVT::f64: 6703 if (FPR_idx != NumFPRs) { 6704 RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg)); 6705 6706 if (isVarArg) { 6707 SDValue Store = 6708 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6709 MemOpChains.push_back(Store); 6710 6711 // Float varargs are always shadowed in available integer registers 6712 if (GPR_idx != NumGPRs) { 6713 SDValue Load = 6714 DAG.getLoad(PtrVT, dl, Store, PtrOff, MachinePointerInfo()); 6715 MemOpChains.push_back(Load.getValue(1)); 6716 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6717 } 6718 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && !isPPC64){ 6719 SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); 6720 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); 6721 SDValue Load = 6722 DAG.getLoad(PtrVT, dl, Store, PtrOff, MachinePointerInfo()); 6723 MemOpChains.push_back(Load.getValue(1)); 6724 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6725 } 6726 } else { 6727 // If we have any FPRs remaining, we may also have GPRs remaining. 6728 // Args passed in FPRs consume either 1 (f32) or 2 (f64) available 6729 // GPRs. 6730 if (GPR_idx != NumGPRs) 6731 ++GPR_idx; 6732 if (GPR_idx != NumGPRs && Arg.getValueType() == MVT::f64 && 6733 !isPPC64) // PPC64 has 64-bit GPR's obviously :) 6734 ++GPR_idx; 6735 } 6736 } else 6737 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6738 isPPC64, isTailCall, false, MemOpChains, 6739 TailCallArguments, dl); 6740 if (isPPC64) 6741 ArgOffset += 8; 6742 else 6743 ArgOffset += Arg.getValueType() == MVT::f32 ? 4 : 8; 6744 break; 6745 case MVT::v4f32: 6746 case MVT::v4i32: 6747 case MVT::v8i16: 6748 case MVT::v16i8: 6749 if (isVarArg) { 6750 // These go aligned on the stack, or in the corresponding R registers 6751 // when within range. The Darwin PPC ABI doc claims they also go in 6752 // V registers; in fact gcc does this only for arguments that are 6753 // prototyped, not for those that match the ... We do it for all 6754 // arguments, seems to work. 6755 while (ArgOffset % 16 !=0) { 6756 ArgOffset += PtrByteSize; 6757 if (GPR_idx != NumGPRs) 6758 GPR_idx++; 6759 } 6760 // We could elide this store in the case where the object fits 6761 // entirely in R registers. Maybe later. 6762 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, 6763 DAG.getConstant(ArgOffset, dl, PtrVT)); 6764 SDValue Store = 6765 DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); 6766 MemOpChains.push_back(Store); 6767 if (VR_idx != NumVRs) { 6768 SDValue Load = 6769 DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); 6770 MemOpChains.push_back(Load.getValue(1)); 6771 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); 6772 } 6773 ArgOffset += 16; 6774 for (unsigned i=0; i<16; i+=PtrByteSize) { 6775 if (GPR_idx == NumGPRs) 6776 break; 6777 SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, 6778 DAG.getConstant(i, dl, PtrVT)); 6779 SDValue Load = 6780 DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); 6781 MemOpChains.push_back(Load.getValue(1)); 6782 RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); 6783 } 6784 break; 6785 } 6786 6787 // Non-varargs Altivec params generally go in registers, but have 6788 // stack space allocated at the end. 6789 if (VR_idx != NumVRs) { 6790 // Doesn't have GPR space allocated. 6791 RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); 6792 } else if (nAltivecParamsAtEnd==0) { 6793 // We are emitting Altivec params in order. 6794 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6795 isPPC64, isTailCall, true, MemOpChains, 6796 TailCallArguments, dl); 6797 ArgOffset += 16; 6798 } 6799 break; 6800 } 6801 } 6802 // If all Altivec parameters fit in registers, as they usually do, 6803 // they get stack space following the non-Altivec parameters. We 6804 // don't track this here because nobody below needs it. 6805 // If there are more Altivec parameters than fit in registers emit 6806 // the stores here. 6807 if (!isVarArg && nAltivecParamsAtEnd > NumVRs) { 6808 unsigned j = 0; 6809 // Offset is aligned; skip 1st 12 params which go in V registers. 6810 ArgOffset = ((ArgOffset+15)/16)*16; 6811 ArgOffset += 12*16; 6812 for (unsigned i = 0; i != NumOps; ++i) { 6813 SDValue Arg = OutVals[i]; 6814 EVT ArgType = Outs[i].VT; 6815 if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 || 6816 ArgType==MVT::v8i16 || ArgType==MVT::v16i8) { 6817 if (++j > NumVRs) { 6818 SDValue PtrOff; 6819 // We are emitting Altivec params in order. 6820 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, 6821 isPPC64, isTailCall, true, MemOpChains, 6822 TailCallArguments, dl); 6823 ArgOffset += 16; 6824 } 6825 } 6826 } 6827 } 6828 6829 if (!MemOpChains.empty()) 6830 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); 6831 6832 // On Darwin, R12 must contain the address of an indirect callee. This does 6833 // not mean the MTCTR instruction must use R12; it's easier to model this as 6834 // an extra parameter, so do that. 6835 if (!isTailCall && 6836 !isFunctionGlobalAddress(Callee) && 6837 !isa<ExternalSymbolSDNode>(Callee) && 6838 !isBLACompatibleAddress(Callee, DAG)) 6839 RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 : 6840 PPC::R12), Callee)); 6841 6842 // Build a sequence of copy-to-reg nodes chained together with token chain 6843 // and flag operands which copy the outgoing args into the appropriate regs. 6844 SDValue InFlag; 6845 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { 6846 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, 6847 RegsToPass[i].second, InFlag); 6848 InFlag = Chain.getValue(1); 6849 } 6850 6851 if (isTailCall) 6852 PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, 6853 TailCallArguments); 6854 6855 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, 6856 /* unused except on PPC64 ELFv1 */ false, DAG, 6857 RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, 6858 NumBytes, Ins, InVals, CS); 6859} 6860 6861static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, 6862 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, 6863 CCState &State) { 6864 6865 if (ValVT == MVT::f128) 6866 report_fatal_error("f128 is unimplemented on AIX."); 6867 6868 if (ArgFlags.isByVal()) 6869 report_fatal_error("Passing structure by value is unimplemented."); 6870 6871 if (ArgFlags.isNest()) 6872 report_fatal_error("Nest arguments are unimplemented."); 6873 6874 if (ValVT.isVector() || LocVT.isVector()) 6875 report_fatal_error("Vector arguments are unimplemented on AIX."); 6876 6877 const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>( 6878 State.getMachineFunction().getSubtarget()); 6879 const bool IsPPC64 = Subtarget.isPPC64(); 6880 const unsigned PtrByteSize = IsPPC64 ? 8 : 4; 6881 6882 static const MCPhysReg GPR_32[] = {// 32-bit registers. 6883 PPC::R3, PPC::R4, PPC::R5, PPC::R6, 6884 PPC::R7, PPC::R8, PPC::R9, PPC::R10}; 6885 static const MCPhysReg GPR_64[] = {// 64-bit registers. 6886 PPC::X3, PPC::X4, PPC::X5, PPC::X6, 6887 PPC::X7, PPC::X8, PPC::X9, PPC::X10}; 6888 6889 // Arguments always reserve parameter save area. 6890 switch (ValVT.SimpleTy) { 6891 default: 6892 report_fatal_error("Unhandled value type for argument."); 6893 case MVT::i64: 6894 // i64 arguments should have been split to i32 for PPC32. 6895 assert(IsPPC64 && "PPC32 should have split i64 values."); 6896 LLVM_FALLTHROUGH; 6897 case MVT::i1: 6898 case MVT::i32: 6899 State.AllocateStack(PtrByteSize, PtrByteSize); 6900 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) { 6901 MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; 6902 // Promote integers if needed. 6903 if (ValVT.getSizeInBits() < RegVT.getSizeInBits()) 6904 LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt 6905 : CCValAssign::LocInfo::ZExt; 6906 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo)); 6907 } 6908 else 6909 report_fatal_error("Handling of placing parameters on the stack is " 6910 "unimplemented!"); 6911 return false; 6912 6913 case MVT::f32: 6914 case MVT::f64: { 6915 // Parameter save area (PSA) is reserved even if the float passes in fpr. 6916 const unsigned StoreSize = LocVT.getStoreSize(); 6917 // Floats are always 4-byte aligned in the PSA on AIX. 6918 // This includes f64 in 64-bit mode for ABI compatibility. 6919 State.AllocateStack(IsPPC64 ? 8 : StoreSize, 4); 6920 if (unsigned Reg = State.AllocateReg(FPR)) 6921 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); 6922 else 6923 report_fatal_error("Handling of placing parameters on the stack is " 6924 "unimplemented!"); 6925 6926 // AIX requires that GPRs are reserved for float arguments. 6927 // Successfully reserved GPRs are only initialized for vararg calls. 6928 MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; 6929 for (unsigned I = 0; I < StoreSize; I += PtrByteSize) { 6930 if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) { 6931 if (State.isVarArg()) { 6932 // Custom handling is required for: 6933 // f64 in PPC32 needs to be split into 2 GPRs. 6934 // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR. 6935 State.addLoc( 6936 CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo)); 6937 } 6938 } else if (State.isVarArg()) { 6939 report_fatal_error("Handling of placing parameters on the stack is " 6940 "unimplemented!"); 6941 } 6942 } 6943 6944 return false; 6945 } 6946 } 6947 return true; 6948} 6949 6950static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT, 6951 bool IsPPC64) { 6952 assert((IsPPC64 || SVT != MVT::i64) && 6953 "i64 should have been split for 32-bit codegen."); 6954 6955 switch (SVT) { 6956 default: 6957 report_fatal_error("Unexpected value type for formal argument"); 6958 case MVT::i1: 6959 case MVT::i32: 6960 case MVT::i64: 6961 return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 6962 case MVT::f32: 6963 return &PPC::F4RCRegClass; 6964 case MVT::f64: 6965 return &PPC::F8RCRegClass; 6966 } 6967} 6968 6969static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT, 6970 SelectionDAG &DAG, SDValue ArgValue, 6971 MVT LocVT, const SDLoc &dl) { 6972 assert(ValVT.isScalarInteger() && LocVT.isScalarInteger()); 6973 assert(ValVT.getSizeInBits() < LocVT.getSizeInBits()); 6974 6975 if (Flags.isSExt()) 6976 ArgValue = DAG.getNode(ISD::AssertSext, dl, LocVT, ArgValue, 6977 DAG.getValueType(ValVT)); 6978 else if (Flags.isZExt()) 6979 ArgValue = DAG.getNode(ISD::AssertZext, dl, LocVT, ArgValue, 6980 DAG.getValueType(ValVT)); 6981 6982 return DAG.getNode(ISD::TRUNCATE, dl, ValVT, ArgValue); 6983} 6984 6985SDValue PPCTargetLowering::LowerFormalArguments_AIX( 6986 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 6987 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 6988 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 6989 6990 assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold || 6991 CallConv == CallingConv::Fast) && 6992 "Unexpected calling convention!"); 6993 6994 if (isVarArg) 6995 report_fatal_error("This call type is unimplemented on AIX."); 6996 6997 if (getTargetMachine().Options.GuaranteedTailCallOpt) 6998 report_fatal_error("Tail call support is unimplemented on AIX."); 6999 7000 if (useSoftFloat()) 7001 report_fatal_error("Soft float support is unimplemented on AIX."); 7002 7003 const PPCSubtarget &Subtarget = 7004 static_cast<const PPCSubtarget &>(DAG.getSubtarget()); 7005 if (Subtarget.hasQPX()) 7006 report_fatal_error("QPX support is not supported on AIX."); 7007 7008 const bool IsPPC64 = Subtarget.isPPC64(); 7009 const unsigned PtrByteSize = IsPPC64 ? 8 : 4; 7010 7011 // Assign locations to all of the incoming arguments. 7012 SmallVector<CCValAssign, 16> ArgLocs; 7013 MachineFunction &MF = DAG.getMachineFunction(); 7014 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); 7015 7016 // Reserve space for the linkage area on the stack. 7017 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 7018 // On AIX a minimum of 8 words is saved to the parameter save area. 7019 const unsigned MinParameterSaveArea = 8 * PtrByteSize; 7020 CCInfo.AllocateStack(LinkageSize + MinParameterSaveArea, PtrByteSize); 7021 CCInfo.AnalyzeFormalArguments(Ins, CC_AIX); 7022 7023 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { 7024 CCValAssign &VA = ArgLocs[i]; 7025 SDValue ArgValue; 7026 ISD::ArgFlagsTy Flags = Ins[i].Flags; 7027 if (VA.isRegLoc()) { 7028 EVT ValVT = VA.getValVT(); 7029 MVT LocVT = VA.getLocVT(); 7030 MVT::SimpleValueType SVT = ValVT.getSimpleVT().SimpleTy; 7031 unsigned VReg = 7032 MF.addLiveIn(VA.getLocReg(), getRegClassForSVT(SVT, IsPPC64)); 7033 ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, LocVT); 7034 if (ValVT.isScalarInteger() && 7035 (ValVT.getSizeInBits() < LocVT.getSizeInBits())) { 7036 ArgValue = 7037 truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl); 7038 } 7039 InVals.push_back(ArgValue); 7040 } else { 7041 report_fatal_error("Handling of formal arguments on the stack is " 7042 "unimplemented!"); 7043 } 7044 } 7045 7046 // Area that is at least reserved in the caller of this function. 7047 unsigned MinReservedArea = CCInfo.getNextStackOffset(); 7048 7049 // Set the size that is at least reserved in caller of this function. Tail 7050 // call optimized function's reserved stack space needs to be aligned so 7051 // that taking the difference between two stack areas will result in an 7052 // aligned stack. 7053 MinReservedArea = 7054 EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); 7055 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 7056 FuncInfo->setMinReservedArea(MinReservedArea); 7057 7058 return Chain; 7059} 7060 7061SDValue PPCTargetLowering::LowerCall_AIX( 7062 SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, 7063 bool isTailCall, bool isPatchPoint, 7064 const SmallVectorImpl<ISD::OutputArg> &Outs, 7065 const SmallVectorImpl<SDValue> &OutVals, 7066 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 7067 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, 7068 ImmutableCallSite CS) const { 7069 7070 assert((CallConv == CallingConv::C || 7071 CallConv == CallingConv::Cold || 7072 CallConv == CallingConv::Fast) && "Unexpected calling convention!"); 7073 7074 if (isPatchPoint) 7075 report_fatal_error("This call type is unimplemented on AIX."); 7076 7077 const PPCSubtarget& Subtarget = 7078 static_cast<const PPCSubtarget&>(DAG.getSubtarget()); 7079 if (Subtarget.hasQPX()) 7080 report_fatal_error("QPX is not supported on AIX."); 7081 if (Subtarget.hasAltivec()) 7082 report_fatal_error("Altivec support is unimplemented on AIX."); 7083 7084 MachineFunction &MF = DAG.getMachineFunction(); 7085 SmallVector<CCValAssign, 16> ArgLocs; 7086 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); 7087 7088 // Reserve space for the linkage save area (LSA) on the stack. 7089 // In both PPC32 and PPC64 there are 6 reserved slots in the LSA: 7090 // [SP][CR][LR][2 x reserved][TOC]. 7091 // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64. 7092 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); 7093 const bool IsPPC64 = Subtarget.isPPC64(); 7094 const unsigned PtrByteSize = IsPPC64 ? 8 : 4; 7095 CCInfo.AllocateStack(LinkageSize, PtrByteSize); 7096 CCInfo.AnalyzeCallOperands(Outs, CC_AIX); 7097 7098 // The prolog code of the callee may store up to 8 GPR argument registers to 7099 // the stack, allowing va_start to index over them in memory if the callee 7100 // is variadic. 7101 // Because we cannot tell if this is needed on the caller side, we have to 7102 // conservatively assume that it is needed. As such, make sure we have at 7103 // least enough stack space for the caller to store the 8 GPRs. 7104 const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize; 7105 const unsigned NumBytes = LinkageSize + MinParameterSaveAreaSize; 7106 7107 // Adjust the stack pointer for the new arguments... 7108 // These operations are automatically eliminated by the prolog/epilog pass. 7109 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); 7110 SDValue CallSeqStart = Chain; 7111 7112 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass; 7113 7114 for (unsigned I = 0, E = ArgLocs.size(); I != E;) { 7115 CCValAssign &VA = ArgLocs[I++]; 7116 7117 if (VA.isMemLoc()) 7118 report_fatal_error("Handling of placing parameters on the stack is " 7119 "unimplemented!"); 7120 if (!VA.isRegLoc()) 7121 report_fatal_error( 7122 "Unexpected non-register location for function call argument."); 7123 7124 SDValue Arg = OutVals[VA.getValNo()]; 7125 7126 if (!VA.needsCustom()) { 7127 switch (VA.getLocInfo()) { 7128 default: 7129 report_fatal_error("Unexpected argument extension type."); 7130 case CCValAssign::Full: 7131 break; 7132 case CCValAssign::ZExt: 7133 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 7134 break; 7135 case CCValAssign::SExt: 7136 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 7137 break; 7138 } 7139 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); 7140 7141 continue; 7142 } 7143 7144 // Custom handling is used for GPR initializations for vararg float 7145 // arguments. 7146 assert(isVarArg && VA.getValVT().isFloatingPoint() && 7147 VA.getLocVT().isInteger() && 7148 "Unexpected custom register handling for calling convention."); 7149 7150 SDValue ArgAsInt = 7151 DAG.getBitcast(MVT::getIntegerVT(VA.getValVT().getSizeInBits()), Arg); 7152 7153 if (Arg.getValueType().getStoreSize() == VA.getLocVT().getStoreSize()) 7154 // f32 in 32-bit GPR 7155 // f64 in 64-bit GPR 7156 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgAsInt)); 7157 else if (Arg.getValueType().getSizeInBits() < VA.getLocVT().getSizeInBits()) 7158 // f32 in 64-bit GPR. 7159 RegsToPass.push_back(std::make_pair( 7160 VA.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, VA.getLocVT()))); 7161 else { 7162 // f64 in two 32-bit GPRs 7163 // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs. 7164 assert(Arg.getValueType() == MVT::f64 && isVarArg && !IsPPC64 && 7165 "Unexpected custom register for argument!"); 7166 CCValAssign &GPR1 = VA; 7167 SDValue MSWAsI64 = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgAsInt, 7168 DAG.getConstant(32, dl, MVT::i8)); 7169 RegsToPass.push_back(std::make_pair( 7170 GPR1.getLocReg(), DAG.getZExtOrTrunc(MSWAsI64, dl, MVT::i32))); 7171 assert(I != E && "A second custom GPR is expected!"); 7172 CCValAssign &GPR2 = ArgLocs[I++]; 7173 assert(GPR2.isRegLoc() && GPR2.getValNo() == GPR1.getValNo() && 7174 GPR2.needsCustom() && "A second custom GPR is expected!"); 7175 RegsToPass.push_back(std::make_pair( 7176 GPR2.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, MVT::i32))); 7177 } 7178 } 7179 7180 // For indirect calls, we need to save the TOC base to the stack for 7181 // restoration after the call. 7182 if (!isTailCall && !isPatchPoint && 7183 !isFunctionGlobalAddress(Callee) && !isa<ExternalSymbolSDNode>(Callee)) { 7184 const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister(); 7185 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister(); 7186 const MVT PtrVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; 7187 const unsigned TOCSaveOffset = 7188 Subtarget.getFrameLowering()->getTOCSaveOffset(); 7189 7190 setUsesTOCBasePtr(DAG); 7191 SDValue Val = DAG.getCopyFromReg(Chain, dl, TOCBaseReg, PtrVT); 7192 SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); 7193 SDValue StackPtr = DAG.getRegister(StackPtrReg, PtrVT); 7194 SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); 7195 Chain = DAG.getStore( 7196 Val.getValue(1), dl, Val, AddPtr, 7197 MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset)); 7198 } 7199 7200 // Build a sequence of copy-to-reg nodes chained together with token chain 7201 // and flag operands which copy the outgoing args into the appropriate regs. 7202 SDValue InFlag; 7203 for (auto Reg : RegsToPass) { 7204 Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, InFlag); 7205 InFlag = Chain.getValue(1); 7206 } 7207 7208 const int SPDiff = 0; 7209 return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, 7210 /* unused except on PPC64 ELFv1 */ false, DAG, RegsToPass, 7211 InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, 7212 InVals, CS); 7213} 7214 7215bool 7216PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv, 7217 MachineFunction &MF, bool isVarArg, 7218 const SmallVectorImpl<ISD::OutputArg> &Outs, 7219 LLVMContext &Context) const { 7220 SmallVector<CCValAssign, 16> RVLocs; 7221 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); 7222 return CCInfo.CheckReturn( 7223 Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 7224 ? RetCC_PPC_Cold 7225 : RetCC_PPC); 7226} 7227 7228SDValue 7229PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 7230 bool isVarArg, 7231 const SmallVectorImpl<ISD::OutputArg> &Outs, 7232 const SmallVectorImpl<SDValue> &OutVals, 7233 const SDLoc &dl, SelectionDAG &DAG) const { 7234 SmallVector<CCValAssign, 16> RVLocs; 7235 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, 7236 *DAG.getContext()); 7237 CCInfo.AnalyzeReturn(Outs, 7238 (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) 7239 ? RetCC_PPC_Cold 7240 : RetCC_PPC); 7241 7242 SDValue Flag; 7243 SmallVector<SDValue, 4> RetOps(1, Chain); 7244 7245 // Copy the result values into the output registers. 7246 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) { 7247 CCValAssign &VA = RVLocs[i]; 7248 assert(VA.isRegLoc() && "Can only return in registers!"); 7249 7250 SDValue Arg = OutVals[RealResIdx]; 7251 7252 switch (VA.getLocInfo()) { 7253 default: llvm_unreachable("Unknown loc info!"); 7254 case CCValAssign::Full: break; 7255 case CCValAssign::AExt: 7256 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); 7257 break; 7258 case CCValAssign::ZExt: 7259 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); 7260 break; 7261 case CCValAssign::SExt: 7262 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); 7263 break; 7264 } 7265 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { 7266 bool isLittleEndian = Subtarget.isLittleEndian(); 7267 // Legalize ret f64 -> ret 2 x i32. 7268 SDValue SVal = 7269 DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 7270 DAG.getIntPtrConstant(isLittleEndian ? 0 : 1, dl)); 7271 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); 7272 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 7273 SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, 7274 DAG.getIntPtrConstant(isLittleEndian ? 1 : 0, dl)); 7275 Flag = Chain.getValue(1); 7276 VA = RVLocs[++i]; // skip ahead to next loc 7277 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); 7278 } else 7279 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); 7280 Flag = Chain.getValue(1); 7281 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); 7282 } 7283 7284 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 7285 const MCPhysReg *I = 7286 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); 7287 if (I) { 7288 for (; *I; ++I) { 7289 7290 if (PPC::G8RCRegClass.contains(*I)) 7291 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); 7292 else if (PPC::F8RCRegClass.contains(*I)) 7293 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); 7294 else if (PPC::CRRCRegClass.contains(*I)) 7295 RetOps.push_back(DAG.getRegister(*I, MVT::i1)); 7296 else if (PPC::VRRCRegClass.contains(*I)) 7297 RetOps.push_back(DAG.getRegister(*I, MVT::Other)); 7298 else 7299 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 7300 } 7301 } 7302 7303 RetOps[0] = Chain; // Update chain. 7304 7305 // Add the flag if we have it. 7306 if (Flag.getNode()) 7307 RetOps.push_back(Flag); 7308 7309 return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps); 7310} 7311 7312SDValue 7313PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, 7314 SelectionDAG &DAG) const { 7315 SDLoc dl(Op); 7316 7317 // Get the correct type for integers. 7318 EVT IntVT = Op.getValueType(); 7319 7320 // Get the inputs. 7321 SDValue Chain = Op.getOperand(0); 7322 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 7323 // Build a DYNAREAOFFSET node. 7324 SDValue Ops[2] = {Chain, FPSIdx}; 7325 SDVTList VTs = DAG.getVTList(IntVT); 7326 return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops); 7327} 7328 7329SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, 7330 SelectionDAG &DAG) const { 7331 // When we pop the dynamic allocation we need to restore the SP link. 7332 SDLoc dl(Op); 7333 7334 // Get the correct type for pointers. 7335 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7336 7337 // Construct the stack pointer operand. 7338 bool isPPC64 = Subtarget.isPPC64(); 7339 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1; 7340 SDValue StackPtr = DAG.getRegister(SP, PtrVT); 7341 7342 // Get the operands for the STACKRESTORE. 7343 SDValue Chain = Op.getOperand(0); 7344 SDValue SaveSP = Op.getOperand(1); 7345 7346 // Load the old link SP. 7347 SDValue LoadLinkSP = 7348 DAG.getLoad(PtrVT, dl, Chain, StackPtr, MachinePointerInfo()); 7349 7350 // Restore the stack pointer. 7351 Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP); 7352 7353 // Store the old link SP. 7354 return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo()); 7355} 7356 7357SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const { 7358 MachineFunction &MF = DAG.getMachineFunction(); 7359 bool isPPC64 = Subtarget.isPPC64(); 7360 EVT PtrVT = getPointerTy(MF.getDataLayout()); 7361 7362 // Get current frame pointer save index. The users of this index will be 7363 // primarily DYNALLOC instructions. 7364 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 7365 int RASI = FI->getReturnAddrSaveIndex(); 7366 7367 // If the frame pointer save index hasn't been defined yet. 7368 if (!RASI) { 7369 // Find out what the fix offset of the frame pointer save area. 7370 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset(); 7371 // Allocate the frame index for frame pointer save area. 7372 RASI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, LROffset, false); 7373 // Save the result. 7374 FI->setReturnAddrSaveIndex(RASI); 7375 } 7376 return DAG.getFrameIndex(RASI, PtrVT); 7377} 7378 7379SDValue 7380PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const { 7381 MachineFunction &MF = DAG.getMachineFunction(); 7382 bool isPPC64 = Subtarget.isPPC64(); 7383 EVT PtrVT = getPointerTy(MF.getDataLayout()); 7384 7385 // Get current frame pointer save index. The users of this index will be 7386 // primarily DYNALLOC instructions. 7387 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 7388 int FPSI = FI->getFramePointerSaveIndex(); 7389 7390 // If the frame pointer save index hasn't been defined yet. 7391 if (!FPSI) { 7392 // Find out what the fix offset of the frame pointer save area. 7393 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset(); 7394 // Allocate the frame index for frame pointer save area. 7395 FPSI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, FPOffset, true); 7396 // Save the result. 7397 FI->setFramePointerSaveIndex(FPSI); 7398 } 7399 return DAG.getFrameIndex(FPSI, PtrVT); 7400} 7401 7402SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, 7403 SelectionDAG &DAG) const { 7404 // Get the inputs. 7405 SDValue Chain = Op.getOperand(0); 7406 SDValue Size = Op.getOperand(1); 7407 SDLoc dl(Op); 7408 7409 // Get the correct type for pointers. 7410 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7411 // Negate the size. 7412 SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, 7413 DAG.getConstant(0, dl, PtrVT), Size); 7414 // Construct a node for the frame pointer save index. 7415 SDValue FPSIdx = getFramePointerFrameIndex(DAG); 7416 // Build a DYNALLOC node. 7417 SDValue Ops[3] = { Chain, NegSize, FPSIdx }; 7418 SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other); 7419 return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops); 7420} 7421 7422SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op, 7423 SelectionDAG &DAG) const { 7424 MachineFunction &MF = DAG.getMachineFunction(); 7425 7426 bool isPPC64 = Subtarget.isPPC64(); 7427 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 7428 7429 int FI = MF.getFrameInfo().CreateFixedObject(isPPC64 ? 8 : 4, 0, false); 7430 return DAG.getFrameIndex(FI, PtrVT); 7431} 7432 7433SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op, 7434 SelectionDAG &DAG) const { 7435 SDLoc DL(Op); 7436 return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL, 7437 DAG.getVTList(MVT::i32, MVT::Other), 7438 Op.getOperand(0), Op.getOperand(1)); 7439} 7440 7441SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op, 7442 SelectionDAG &DAG) const { 7443 SDLoc DL(Op); 7444 return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other, 7445 Op.getOperand(0), Op.getOperand(1)); 7446} 7447 7448SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 7449 if (Op.getValueType().isVector()) 7450 return LowerVectorLoad(Op, DAG); 7451 7452 assert(Op.getValueType() == MVT::i1 && 7453 "Custom lowering only for i1 loads"); 7454 7455 // First, load 8 bits into 32 bits, then truncate to 1 bit. 7456 7457 SDLoc dl(Op); 7458 LoadSDNode *LD = cast<LoadSDNode>(Op); 7459 7460 SDValue Chain = LD->getChain(); 7461 SDValue BasePtr = LD->getBasePtr(); 7462 MachineMemOperand *MMO = LD->getMemOperand(); 7463 7464 SDValue NewLD = 7465 DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain, 7466 BasePtr, MVT::i8, MMO); 7467 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD); 7468 7469 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) }; 7470 return DAG.getMergeValues(Ops, dl); 7471} 7472 7473SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 7474 if (Op.getOperand(1).getValueType().isVector()) 7475 return LowerVectorStore(Op, DAG); 7476 7477 assert(Op.getOperand(1).getValueType() == MVT::i1 && 7478 "Custom lowering only for i1 stores"); 7479 7480 // First, zero extend to 32 bits, then use a truncating store to 8 bits. 7481 7482 SDLoc dl(Op); 7483 StoreSDNode *ST = cast<StoreSDNode>(Op); 7484 7485 SDValue Chain = ST->getChain(); 7486 SDValue BasePtr = ST->getBasePtr(); 7487 SDValue Value = ST->getValue(); 7488 MachineMemOperand *MMO = ST->getMemOperand(); 7489 7490 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()), 7491 Value); 7492 return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO); 7493} 7494 7495// FIXME: Remove this once the ANDI glue bug is fixed: 7496SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { 7497 assert(Op.getValueType() == MVT::i1 && 7498 "Custom lowering only for i1 results"); 7499 7500 SDLoc DL(Op); 7501 return DAG.getNode(PPCISD::ANDI_rec_1_GT_BIT, DL, MVT::i1, Op.getOperand(0)); 7502} 7503 7504SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, 7505 SelectionDAG &DAG) const { 7506 7507 // Implements a vector truncate that fits in a vector register as a shuffle. 7508 // We want to legalize vector truncates down to where the source fits in 7509 // a vector register (and target is therefore smaller than vector register 7510 // size). At that point legalization will try to custom lower the sub-legal 7511 // result and get here - where we can contain the truncate as a single target 7512 // operation. 7513 7514 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows: 7515 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2> 7516 // 7517 // We will implement it for big-endian ordering as this (where x denotes 7518 // undefined): 7519 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to 7520 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u> 7521 // 7522 // The same operation in little-endian ordering will be: 7523 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to 7524 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1> 7525 7526 assert(Op.getValueType().isVector() && "Vector type expected."); 7527 7528 SDLoc DL(Op); 7529 SDValue N1 = Op.getOperand(0); 7530 unsigned SrcSize = N1.getValueType().getSizeInBits(); 7531 assert(SrcSize <= 128 && "Source must fit in an Altivec/VSX vector"); 7532 SDValue WideSrc = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL); 7533 7534 EVT TrgVT = Op.getValueType(); 7535 unsigned TrgNumElts = TrgVT.getVectorNumElements(); 7536 EVT EltVT = TrgVT.getVectorElementType(); 7537 unsigned WideNumElts = 128 / EltVT.getSizeInBits(); 7538 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); 7539 7540 // First list the elements we want to keep. 7541 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits(); 7542 SmallVector<int, 16> ShuffV; 7543 if (Subtarget.isLittleEndian()) 7544 for (unsigned i = 0; i < TrgNumElts; ++i) 7545 ShuffV.push_back(i * SizeMult); 7546 else 7547 for (unsigned i = 1; i <= TrgNumElts; ++i) 7548 ShuffV.push_back(i * SizeMult - 1); 7549 7550 // Populate the remaining elements with undefs. 7551 for (unsigned i = TrgNumElts; i < WideNumElts; ++i) 7552 // ShuffV.push_back(i + WideNumElts); 7553 ShuffV.push_back(WideNumElts + 1); 7554 7555 SDValue Conv = DAG.getNode(ISD::BITCAST, DL, WideVT, WideSrc); 7556 return DAG.getVectorShuffle(WideVT, DL, Conv, DAG.getUNDEF(WideVT), ShuffV); 7557} 7558 7559/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when 7560/// possible. 7561SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 7562 // Not FP? Not a fsel. 7563 if (!Op.getOperand(0).getValueType().isFloatingPoint() || 7564 !Op.getOperand(2).getValueType().isFloatingPoint()) 7565 return Op; 7566 7567 bool HasNoInfs = DAG.getTarget().Options.NoInfsFPMath; 7568 bool HasNoNaNs = DAG.getTarget().Options.NoNaNsFPMath; 7569 // We might be able to do better than this under some circumstances, but in 7570 // general, fsel-based lowering of select is a finite-math-only optimization. 7571 // For more information, see section F.3 of the 2.06 ISA specification. 7572 // With ISA 3.0, we have xsmaxcdp/xsmincdp which are OK to emit even in the 7573 // presence of infinities. 7574 if (!Subtarget.hasP9Vector() && (!HasNoInfs || !HasNoNaNs)) 7575 return Op; 7576 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 7577 7578 EVT ResVT = Op.getValueType(); 7579 EVT CmpVT = Op.getOperand(0).getValueType(); 7580 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 7581 SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); 7582 SDLoc dl(Op); 7583 7584 if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) { 7585 switch (CC) { 7586 default: 7587 // Not a min/max but with finite math, we may still be able to use fsel. 7588 if (HasNoInfs && HasNoNaNs) 7589 break; 7590 return Op; 7591 case ISD::SETOGT: 7592 case ISD::SETGT: 7593 return DAG.getNode(PPCISD::XSMAXCDP, dl, Op.getValueType(), LHS, RHS); 7594 case ISD::SETOLT: 7595 case ISD::SETLT: 7596 return DAG.getNode(PPCISD::XSMINCDP, dl, Op.getValueType(), LHS, RHS); 7597 } 7598 } 7599 7600 // TODO: Propagate flags from the select rather than global settings. 7601 SDNodeFlags Flags; 7602 Flags.setNoInfs(true); 7603 Flags.setNoNaNs(true); 7604 7605 // If the RHS of the comparison is a 0.0, we don't need to do the 7606 // subtraction at all. 7607 SDValue Sel1; 7608 if (isFloatingPointZero(RHS)) 7609 switch (CC) { 7610 default: break; // SETUO etc aren't handled by fsel. 7611 case ISD::SETNE: 7612 std::swap(TV, FV); 7613 LLVM_FALLTHROUGH; 7614 case ISD::SETEQ: 7615 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7616 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7617 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 7618 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 7619 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 7620 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7621 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV); 7622 case ISD::SETULT: 7623 case ISD::SETLT: 7624 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 7625 LLVM_FALLTHROUGH; 7626 case ISD::SETOGE: 7627 case ISD::SETGE: 7628 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7629 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7630 return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); 7631 case ISD::SETUGT: 7632 case ISD::SETGT: 7633 std::swap(TV, FV); // fsel is natively setge, swap operands for setlt 7634 LLVM_FALLTHROUGH; 7635 case ISD::SETOLE: 7636 case ISD::SETLE: 7637 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits 7638 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); 7639 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7640 DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV); 7641 } 7642 7643 SDValue Cmp; 7644 switch (CC) { 7645 default: break; // SETUO etc aren't handled by fsel. 7646 case ISD::SETNE: 7647 std::swap(TV, FV); 7648 LLVM_FALLTHROUGH; 7649 case ISD::SETEQ: 7650 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7651 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7652 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7653 Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7654 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits 7655 Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); 7656 return DAG.getNode(PPCISD::FSEL, dl, ResVT, 7657 DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV); 7658 case ISD::SETULT: 7659 case ISD::SETLT: 7660 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7661 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7662 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7663 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 7664 case ISD::SETOGE: 7665 case ISD::SETGE: 7666 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); 7667 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7668 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7669 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7670 case ISD::SETUGT: 7671 case ISD::SETGT: 7672 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); 7673 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7674 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7675 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); 7676 case ISD::SETOLE: 7677 case ISD::SETLE: 7678 Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); 7679 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits 7680 Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); 7681 return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); 7682 } 7683 return Op; 7684} 7685 7686void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, 7687 SelectionDAG &DAG, 7688 const SDLoc &dl) const { 7689 assert(Op.getOperand(0).getValueType().isFloatingPoint()); 7690 SDValue Src = Op.getOperand(0); 7691 if (Src.getValueType() == MVT::f32) 7692 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 7693 7694 SDValue Tmp; 7695 switch (Op.getSimpleValueType().SimpleTy) { 7696 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 7697 case MVT::i32: 7698 Tmp = DAG.getNode( 7699 Op.getOpcode() == ISD::FP_TO_SINT 7700 ? PPCISD::FCTIWZ 7701 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), 7702 dl, MVT::f64, Src); 7703 break; 7704 case MVT::i64: 7705 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && 7706 "i64 FP_TO_UINT is supported only with FPCVT"); 7707 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 7708 PPCISD::FCTIDUZ, 7709 dl, MVT::f64, Src); 7710 break; 7711 } 7712 7713 // Convert the FP value to an int value through memory. 7714 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() && 7715 (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()); 7716 SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); 7717 int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex(); 7718 MachinePointerInfo MPI = 7719 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); 7720 7721 // Emit a store to the stack slot. 7722 SDValue Chain; 7723 if (i32Stack) { 7724 MachineFunction &MF = DAG.getMachineFunction(); 7725 MachineMemOperand *MMO = 7726 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4); 7727 SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; 7728 Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, 7729 DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); 7730 } else 7731 Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, MPI); 7732 7733 // Result is a load from the stack slot. If loading 4 bytes, make sure to 7734 // add in a bias on big endian. 7735 if (Op.getValueType() == MVT::i32 && !i32Stack) { 7736 FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, 7737 DAG.getConstant(4, dl, FIPtr.getValueType())); 7738 MPI = MPI.getWithOffset(Subtarget.isLittleEndian() ? 0 : 4); 7739 } 7740 7741 RLI.Chain = Chain; 7742 RLI.Ptr = FIPtr; 7743 RLI.MPI = MPI; 7744} 7745 7746/// Custom lowers floating point to integer conversions to use 7747/// the direct move instructions available in ISA 2.07 to avoid the 7748/// need for load/store combinations. 7749SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op, 7750 SelectionDAG &DAG, 7751 const SDLoc &dl) const { 7752 assert(Op.getOperand(0).getValueType().isFloatingPoint()); 7753 SDValue Src = Op.getOperand(0); 7754 7755 if (Src.getValueType() == MVT::f32) 7756 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 7757 7758 SDValue Tmp; 7759 switch (Op.getSimpleValueType().SimpleTy) { 7760 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); 7761 case MVT::i32: 7762 Tmp = DAG.getNode( 7763 Op.getOpcode() == ISD::FP_TO_SINT 7764 ? PPCISD::FCTIWZ 7765 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), 7766 dl, MVT::f64, Src); 7767 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp); 7768 break; 7769 case MVT::i64: 7770 assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && 7771 "i64 FP_TO_UINT is supported only with FPCVT"); 7772 Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 7773 PPCISD::FCTIDUZ, 7774 dl, MVT::f64, Src); 7775 Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp); 7776 break; 7777 } 7778 return Tmp; 7779} 7780 7781SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, 7782 const SDLoc &dl) const { 7783 7784 // FP to INT conversions are legal for f128. 7785 if (EnableQuadPrecision && (Op->getOperand(0).getValueType() == MVT::f128)) 7786 return Op; 7787 7788 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on 7789 // PPC (the libcall is not available). 7790 if (Op.getOperand(0).getValueType() == MVT::ppcf128) { 7791 if (Op.getValueType() == MVT::i32) { 7792 if (Op.getOpcode() == ISD::FP_TO_SINT) { 7793 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 7794 MVT::f64, Op.getOperand(0), 7795 DAG.getIntPtrConstant(0, dl)); 7796 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, 7797 MVT::f64, Op.getOperand(0), 7798 DAG.getIntPtrConstant(1, dl)); 7799 7800 // Add the two halves of the long double in round-to-zero mode. 7801 SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); 7802 7803 // Now use a smaller FP_TO_SINT. 7804 return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res); 7805 } 7806 if (Op.getOpcode() == ISD::FP_TO_UINT) { 7807 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; 7808 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31)); 7809 SDValue Tmp = DAG.getConstantFP(APF, dl, MVT::ppcf128); 7810 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X 7811 // FIXME: generated code sucks. 7812 // TODO: Are there fast-math-flags to propagate to this FSUB? 7813 SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, 7814 Op.getOperand(0), Tmp); 7815 True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True); 7816 True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, 7817 DAG.getConstant(0x80000000, dl, MVT::i32)); 7818 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, 7819 Op.getOperand(0)); 7820 return DAG.getSelectCC(dl, Op.getOperand(0), Tmp, True, False, 7821 ISD::SETGE); 7822 } 7823 } 7824 7825 return SDValue(); 7826 } 7827 7828 if (Subtarget.hasDirectMove() && Subtarget.isPPC64()) 7829 return LowerFP_TO_INTDirectMove(Op, DAG, dl); 7830 7831 ReuseLoadInfo RLI; 7832 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 7833 7834 return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, 7835 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); 7836} 7837 7838// We're trying to insert a regular store, S, and then a load, L. If the 7839// incoming value, O, is a load, we might just be able to have our load use the 7840// address used by O. However, we don't know if anything else will store to 7841// that address before we can load from it. To prevent this situation, we need 7842// to insert our load, L, into the chain as a peer of O. To do this, we give L 7843// the same chain operand as O, we create a token factor from the chain results 7844// of O and L, and we replace all uses of O's chain result with that token 7845// factor (see spliceIntoChain below for this last part). 7846bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, 7847 ReuseLoadInfo &RLI, 7848 SelectionDAG &DAG, 7849 ISD::LoadExtType ET) const { 7850 SDLoc dl(Op); 7851 bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT && 7852 (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32); 7853 if (ET == ISD::NON_EXTLOAD && 7854 (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) && 7855 isOperationLegalOrCustom(Op.getOpcode(), 7856 Op.getOperand(0).getValueType())) { 7857 7858 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); 7859 return true; 7860 } 7861 7862 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op); 7863 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() || 7864 LD->isNonTemporal()) 7865 return false; 7866 if (LD->getMemoryVT() != MemVT) 7867 return false; 7868 7869 RLI.Ptr = LD->getBasePtr(); 7870 if (LD->isIndexed() && !LD->getOffset().isUndef()) { 7871 assert(LD->getAddressingMode() == ISD::PRE_INC && 7872 "Non-pre-inc AM on PPC?"); 7873 RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr, 7874 LD->getOffset()); 7875 } 7876 7877 RLI.Chain = LD->getChain(); 7878 RLI.MPI = LD->getPointerInfo(); 7879 RLI.IsDereferenceable = LD->isDereferenceable(); 7880 RLI.IsInvariant = LD->isInvariant(); 7881 RLI.Alignment = LD->getAlignment(); 7882 RLI.AAInfo = LD->getAAInfo(); 7883 RLI.Ranges = LD->getRanges(); 7884 7885 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1); 7886 return true; 7887} 7888 7889// Given the head of the old chain, ResChain, insert a token factor containing 7890// it and NewResChain, and make users of ResChain now be users of that token 7891// factor. 7892// TODO: Remove and use DAG::makeEquivalentMemoryOrdering() instead. 7893void PPCTargetLowering::spliceIntoChain(SDValue ResChain, 7894 SDValue NewResChain, 7895 SelectionDAG &DAG) const { 7896 if (!ResChain) 7897 return; 7898 7899 SDLoc dl(NewResChain); 7900 7901 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 7902 NewResChain, DAG.getUNDEF(MVT::Other)); 7903 assert(TF.getNode() != NewResChain.getNode() && 7904 "A new TF really is required here"); 7905 7906 DAG.ReplaceAllUsesOfValueWith(ResChain, TF); 7907 DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain); 7908} 7909 7910/// Analyze profitability of direct move 7911/// prefer float load to int load plus direct move 7912/// when there is no integer use of int load 7913bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const { 7914 SDNode *Origin = Op.getOperand(0).getNode(); 7915 if (Origin->getOpcode() != ISD::LOAD) 7916 return true; 7917 7918 // If there is no LXSIBZX/LXSIHZX, like Power8, 7919 // prefer direct move if the memory size is 1 or 2 bytes. 7920 MachineMemOperand *MMO = cast<LoadSDNode>(Origin)->getMemOperand(); 7921 if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2) 7922 return true; 7923 7924 for (SDNode::use_iterator UI = Origin->use_begin(), 7925 UE = Origin->use_end(); 7926 UI != UE; ++UI) { 7927 7928 // Only look at the users of the loaded value. 7929 if (UI.getUse().get().getResNo() != 0) 7930 continue; 7931 7932 if (UI->getOpcode() != ISD::SINT_TO_FP && 7933 UI->getOpcode() != ISD::UINT_TO_FP) 7934 return true; 7935 } 7936 7937 return false; 7938} 7939 7940/// Custom lowers integer to floating point conversions to use 7941/// the direct move instructions available in ISA 2.07 to avoid the 7942/// need for load/store combinations. 7943SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op, 7944 SelectionDAG &DAG, 7945 const SDLoc &dl) const { 7946 assert((Op.getValueType() == MVT::f32 || 7947 Op.getValueType() == MVT::f64) && 7948 "Invalid floating point type as target of conversion"); 7949 assert(Subtarget.hasFPCVT() && 7950 "Int to FP conversions with direct moves require FPCVT"); 7951 SDValue FP; 7952 SDValue Src = Op.getOperand(0); 7953 bool SinglePrec = Op.getValueType() == MVT::f32; 7954 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32; 7955 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP; 7956 unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) : 7957 (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU); 7958 7959 if (WordInt) { 7960 FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ, 7961 dl, MVT::f64, Src); 7962 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); 7963 } 7964 else { 7965 FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src); 7966 FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); 7967 } 7968 7969 return FP; 7970} 7971 7972static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) { 7973 7974 EVT VecVT = Vec.getValueType(); 7975 assert(VecVT.isVector() && "Expected a vector type."); 7976 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width."); 7977 7978 EVT EltVT = VecVT.getVectorElementType(); 7979 unsigned WideNumElts = 128 / EltVT.getSizeInBits(); 7980 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); 7981 7982 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements(); 7983 SmallVector<SDValue, 16> Ops(NumConcat); 7984 Ops[0] = Vec; 7985 SDValue UndefVec = DAG.getUNDEF(VecVT); 7986 for (unsigned i = 1; i < NumConcat; ++i) 7987 Ops[i] = UndefVec; 7988 7989 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Ops); 7990} 7991 7992SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, 7993 const SDLoc &dl) const { 7994 7995 unsigned Opc = Op.getOpcode(); 7996 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP) && 7997 "Unexpected conversion type"); 7998 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && 7999 "Supports conversions to v2f64/v4f32 only."); 8000 8001 bool SignedConv = Opc == ISD::SINT_TO_FP; 8002 bool FourEltRes = Op.getValueType() == MVT::v4f32; 8003 8004 SDValue Wide = widenVec(DAG, Op.getOperand(0), dl); 8005 EVT WideVT = Wide.getValueType(); 8006 unsigned WideNumElts = WideVT.getVectorNumElements(); 8007 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64; 8008 8009 SmallVector<int, 16> ShuffV; 8010 for (unsigned i = 0; i < WideNumElts; ++i) 8011 ShuffV.push_back(i + WideNumElts); 8012 8013 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2; 8014 int SaveElts = FourEltRes ? 4 : 2; 8015 if (Subtarget.isLittleEndian()) 8016 for (int i = 0; i < SaveElts; i++) 8017 ShuffV[i * Stride] = i; 8018 else 8019 for (int i = 1; i <= SaveElts; i++) 8020 ShuffV[i * Stride - 1] = i - 1; 8021 8022 SDValue ShuffleSrc2 = 8023 SignedConv ? DAG.getUNDEF(WideVT) : DAG.getConstant(0, dl, WideVT); 8024 SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV); 8025 unsigned ExtendOp = 8026 SignedConv ? (unsigned)PPCISD::SExtVElems : (unsigned)ISD::BITCAST; 8027 8028 SDValue Extend; 8029 if (!Subtarget.hasP9Altivec() && SignedConv) { 8030 Arrange = DAG.getBitcast(IntermediateVT, Arrange); 8031 Extend = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, IntermediateVT, Arrange, 8032 DAG.getValueType(Op.getOperand(0).getValueType())); 8033 } else 8034 Extend = DAG.getNode(ExtendOp, dl, IntermediateVT, Arrange); 8035 8036 return DAG.getNode(Opc, dl, Op.getValueType(), Extend); 8037} 8038 8039SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, 8040 SelectionDAG &DAG) const { 8041 SDLoc dl(Op); 8042 8043 EVT InVT = Op.getOperand(0).getValueType(); 8044 EVT OutVT = Op.getValueType(); 8045 if (OutVT.isVector() && OutVT.isFloatingPoint() && 8046 isOperationCustom(Op.getOpcode(), InVT)) 8047 return LowerINT_TO_FPVector(Op, DAG, dl); 8048 8049 // Conversions to f128 are legal. 8050 if (EnableQuadPrecision && (Op.getValueType() == MVT::f128)) 8051 return Op; 8052 8053 if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) { 8054 if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64) 8055 return SDValue(); 8056 8057 SDValue Value = Op.getOperand(0); 8058 // The values are now known to be -1 (false) or 1 (true). To convert this 8059 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 8060 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 8061 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 8062 8063 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); 8064 8065 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 8066 8067 if (Op.getValueType() != MVT::v4f64) 8068 Value = DAG.getNode(ISD::FP_ROUND, dl, 8069 Op.getValueType(), Value, 8070 DAG.getIntPtrConstant(1, dl)); 8071 return Value; 8072 } 8073 8074 // Don't handle ppc_fp128 here; let it be lowered to a libcall. 8075 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 8076 return SDValue(); 8077 8078 if (Op.getOperand(0).getValueType() == MVT::i1) 8079 return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0), 8080 DAG.getConstantFP(1.0, dl, Op.getValueType()), 8081 DAG.getConstantFP(0.0, dl, Op.getValueType())); 8082 8083 // If we have direct moves, we can do all the conversion, skip the store/load 8084 // however, without FPCVT we can't do most conversions. 8085 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) && 8086 Subtarget.isPPC64() && Subtarget.hasFPCVT()) 8087 return LowerINT_TO_FPDirectMove(Op, DAG, dl); 8088 8089 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && 8090 "UINT_TO_FP is supported only with FPCVT"); 8091 8092 // If we have FCFIDS, then use it when converting to single-precision. 8093 // Otherwise, convert to double-precision and then round. 8094 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 8095 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS 8096 : PPCISD::FCFIDS) 8097 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU 8098 : PPCISD::FCFID); 8099 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 8100 ? MVT::f32 8101 : MVT::f64; 8102 8103 if (Op.getOperand(0).getValueType() == MVT::i64) { 8104 SDValue SINT = Op.getOperand(0); 8105 // When converting to single-precision, we actually need to convert 8106 // to double-precision first and then round to single-precision. 8107 // To avoid double-rounding effects during that operation, we have 8108 // to prepare the input operand. Bits that might be truncated when 8109 // converting to double-precision are replaced by a bit that won't 8110 // be lost at this stage, but is below the single-precision rounding 8111 // position. 8112 // 8113 // However, if -enable-unsafe-fp-math is in effect, accept double 8114 // rounding to avoid the extra overhead. 8115 if (Op.getValueType() == MVT::f32 && 8116 !Subtarget.hasFPCVT() && 8117 !DAG.getTarget().Options.UnsafeFPMath) { 8118 8119 // Twiddle input to make sure the low 11 bits are zero. (If this 8120 // is the case, we are guaranteed the value will fit into the 53 bit 8121 // mantissa of an IEEE double-precision value without rounding.) 8122 // If any of those low 11 bits were not zero originally, make sure 8123 // bit 12 (value 2048) is set instead, so that the final rounding 8124 // to single-precision gets the correct result. 8125 SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64, 8126 SINT, DAG.getConstant(2047, dl, MVT::i64)); 8127 Round = DAG.getNode(ISD::ADD, dl, MVT::i64, 8128 Round, DAG.getConstant(2047, dl, MVT::i64)); 8129 Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT); 8130 Round = DAG.getNode(ISD::AND, dl, MVT::i64, 8131 Round, DAG.getConstant(-2048, dl, MVT::i64)); 8132 8133 // However, we cannot use that value unconditionally: if the magnitude 8134 // of the input value is small, the bit-twiddling we did above might 8135 // end up visibly changing the output. Fortunately, in that case, we 8136 // don't need to twiddle bits since the original input will convert 8137 // exactly to double-precision floating-point already. Therefore, 8138 // construct a conditional to use the original value if the top 11 8139 // bits are all sign-bit copies, and use the rounded value computed 8140 // above otherwise. 8141 SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64, 8142 SINT, DAG.getConstant(53, dl, MVT::i32)); 8143 Cond = DAG.getNode(ISD::ADD, dl, MVT::i64, 8144 Cond, DAG.getConstant(1, dl, MVT::i64)); 8145 Cond = DAG.getSetCC(dl, MVT::i32, 8146 Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT); 8147 8148 SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT); 8149 } 8150 8151 ReuseLoadInfo RLI; 8152 SDValue Bits; 8153 8154 MachineFunction &MF = DAG.getMachineFunction(); 8155 if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { 8156 Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, 8157 RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); 8158 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 8159 } else if (Subtarget.hasLFIWAX() && 8160 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { 8161 MachineMemOperand *MMO = 8162 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8163 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8164 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8165 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl, 8166 DAG.getVTList(MVT::f64, MVT::Other), 8167 Ops, MVT::i32, MMO); 8168 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 8169 } else if (Subtarget.hasFPCVT() && 8170 canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) { 8171 MachineMemOperand *MMO = 8172 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8173 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8174 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8175 Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl, 8176 DAG.getVTList(MVT::f64, MVT::Other), 8177 Ops, MVT::i32, MMO); 8178 spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); 8179 } else if (((Subtarget.hasLFIWAX() && 8180 SINT.getOpcode() == ISD::SIGN_EXTEND) || 8181 (Subtarget.hasFPCVT() && 8182 SINT.getOpcode() == ISD::ZERO_EXTEND)) && 8183 SINT.getOperand(0).getValueType() == MVT::i32) { 8184 MachineFrameInfo &MFI = MF.getFrameInfo(); 8185 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 8186 8187 int FrameIdx = MFI.CreateStackObject(4, 4, false); 8188 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8189 8190 SDValue Store = 8191 DAG.getStore(DAG.getEntryNode(), dl, SINT.getOperand(0), FIdx, 8192 MachinePointerInfo::getFixedStack( 8193 DAG.getMachineFunction(), FrameIdx)); 8194 8195 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 8196 "Expected an i32 store"); 8197 8198 RLI.Ptr = FIdx; 8199 RLI.Chain = Store; 8200 RLI.MPI = 8201 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 8202 RLI.Alignment = 4; 8203 8204 MachineMemOperand *MMO = 8205 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8206 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8207 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8208 Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ? 8209 PPCISD::LFIWZX : PPCISD::LFIWAX, 8210 dl, DAG.getVTList(MVT::f64, MVT::Other), 8211 Ops, MVT::i32, MMO); 8212 } else 8213 Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); 8214 8215 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits); 8216 8217 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) 8218 FP = DAG.getNode(ISD::FP_ROUND, dl, 8219 MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); 8220 return FP; 8221 } 8222 8223 assert(Op.getOperand(0).getValueType() == MVT::i32 && 8224 "Unhandled INT_TO_FP type in custom expander!"); 8225 // Since we only generate this in 64-bit mode, we can take advantage of 8226 // 64-bit registers. In particular, sign extend the input value into the 8227 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack 8228 // then lfd it and fcfid it. 8229 MachineFunction &MF = DAG.getMachineFunction(); 8230 MachineFrameInfo &MFI = MF.getFrameInfo(); 8231 EVT PtrVT = getPointerTy(MF.getDataLayout()); 8232 8233 SDValue Ld; 8234 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) { 8235 ReuseLoadInfo RLI; 8236 bool ReusingLoad; 8237 if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI, 8238 DAG))) { 8239 int FrameIdx = MFI.CreateStackObject(4, 4, false); 8240 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8241 8242 SDValue Store = 8243 DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, 8244 MachinePointerInfo::getFixedStack( 8245 DAG.getMachineFunction(), FrameIdx)); 8246 8247 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 && 8248 "Expected an i32 store"); 8249 8250 RLI.Ptr = FIdx; 8251 RLI.Chain = Store; 8252 RLI.MPI = 8253 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 8254 RLI.Alignment = 4; 8255 } 8256 8257 MachineMemOperand *MMO = 8258 MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, 8259 RLI.Alignment, RLI.AAInfo, RLI.Ranges); 8260 SDValue Ops[] = { RLI.Chain, RLI.Ptr }; 8261 Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ? 8262 PPCISD::LFIWZX : PPCISD::LFIWAX, 8263 dl, DAG.getVTList(MVT::f64, MVT::Other), 8264 Ops, MVT::i32, MMO); 8265 if (ReusingLoad) 8266 spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG); 8267 } else { 8268 assert(Subtarget.isPPC64() && 8269 "i32->FP without LFIWAX supported only on PPC64"); 8270 8271 int FrameIdx = MFI.CreateStackObject(8, 8, false); 8272 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8273 8274 SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, 8275 Op.getOperand(0)); 8276 8277 // STD the extended value into the stack slot. 8278 SDValue Store = DAG.getStore( 8279 DAG.getEntryNode(), dl, Ext64, FIdx, 8280 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); 8281 8282 // Load the value as a double. 8283 Ld = DAG.getLoad( 8284 MVT::f64, dl, Store, FIdx, 8285 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); 8286 } 8287 8288 // FCFID it and return it. 8289 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld); 8290 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) 8291 FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, 8292 DAG.getIntPtrConstant(0, dl)); 8293 return FP; 8294} 8295 8296SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, 8297 SelectionDAG &DAG) const { 8298 SDLoc dl(Op); 8299 /* 8300 The rounding mode is in bits 30:31 of FPSR, and has the following 8301 settings: 8302 00 Round to nearest 8303 01 Round to 0 8304 10 Round to +inf 8305 11 Round to -inf 8306 8307 FLT_ROUNDS, on the other hand, expects the following: 8308 -1 Undefined 8309 0 Round to 0 8310 1 Round to nearest 8311 2 Round to +inf 8312 3 Round to -inf 8313 8314 To perform the conversion, we do: 8315 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1)) 8316 */ 8317 8318 MachineFunction &MF = DAG.getMachineFunction(); 8319 EVT VT = Op.getValueType(); 8320 EVT PtrVT = getPointerTy(MF.getDataLayout()); 8321 8322 // Save FP Control Word to register 8323 EVT NodeTys[] = { 8324 MVT::f64, // return register 8325 MVT::Glue // unused in this context 8326 }; 8327 SDValue Chain = DAG.getNode(PPCISD::MFFS, dl, NodeTys, None); 8328 8329 // Save FP register to stack slot 8330 int SSFI = MF.getFrameInfo().CreateStackObject(8, 8, false); 8331 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); 8332 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Chain, StackSlot, 8333 MachinePointerInfo()); 8334 8335 // Load FP Control Word from low 32 bits of stack slot. 8336 SDValue Four = DAG.getConstant(4, dl, PtrVT); 8337 SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); 8338 SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo()); 8339 8340 // Transform as necessary 8341 SDValue CWD1 = 8342 DAG.getNode(ISD::AND, dl, MVT::i32, 8343 CWD, DAG.getConstant(3, dl, MVT::i32)); 8344 SDValue CWD2 = 8345 DAG.getNode(ISD::SRL, dl, MVT::i32, 8346 DAG.getNode(ISD::AND, dl, MVT::i32, 8347 DAG.getNode(ISD::XOR, dl, MVT::i32, 8348 CWD, DAG.getConstant(3, dl, MVT::i32)), 8349 DAG.getConstant(3, dl, MVT::i32)), 8350 DAG.getConstant(1, dl, MVT::i32)); 8351 8352 SDValue RetVal = 8353 DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2); 8354 8355 return DAG.getNode((VT.getSizeInBits() < 16 ? 8356 ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal); 8357} 8358 8359SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const { 8360 EVT VT = Op.getValueType(); 8361 unsigned BitWidth = VT.getSizeInBits(); 8362 SDLoc dl(Op); 8363 assert(Op.getNumOperands() == 3 && 8364 VT == Op.getOperand(1).getValueType() && 8365 "Unexpected SHL!"); 8366 8367 // Expand into a bunch of logical ops. Note that these ops 8368 // depend on the PPC behavior for oversized shift amounts. 8369 SDValue Lo = Op.getOperand(0); 8370 SDValue Hi = Op.getOperand(1); 8371 SDValue Amt = Op.getOperand(2); 8372 EVT AmtVT = Amt.getValueType(); 8373 8374 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 8375 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 8376 SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt); 8377 SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1); 8378 SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3); 8379 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 8380 DAG.getConstant(-BitWidth, dl, AmtVT)); 8381 SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5); 8382 SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 8383 SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt); 8384 SDValue OutOps[] = { OutLo, OutHi }; 8385 return DAG.getMergeValues(OutOps, dl); 8386} 8387 8388SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const { 8389 EVT VT = Op.getValueType(); 8390 SDLoc dl(Op); 8391 unsigned BitWidth = VT.getSizeInBits(); 8392 assert(Op.getNumOperands() == 3 && 8393 VT == Op.getOperand(1).getValueType() && 8394 "Unexpected SRL!"); 8395 8396 // Expand into a bunch of logical ops. Note that these ops 8397 // depend on the PPC behavior for oversized shift amounts. 8398 SDValue Lo = Op.getOperand(0); 8399 SDValue Hi = Op.getOperand(1); 8400 SDValue Amt = Op.getOperand(2); 8401 EVT AmtVT = Amt.getValueType(); 8402 8403 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 8404 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 8405 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 8406 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 8407 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 8408 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 8409 DAG.getConstant(-BitWidth, dl, AmtVT)); 8410 SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5); 8411 SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); 8412 SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt); 8413 SDValue OutOps[] = { OutLo, OutHi }; 8414 return DAG.getMergeValues(OutOps, dl); 8415} 8416 8417SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { 8418 SDLoc dl(Op); 8419 EVT VT = Op.getValueType(); 8420 unsigned BitWidth = VT.getSizeInBits(); 8421 assert(Op.getNumOperands() == 3 && 8422 VT == Op.getOperand(1).getValueType() && 8423 "Unexpected SRA!"); 8424 8425 // Expand into a bunch of logical ops, followed by a select_cc. 8426 SDValue Lo = Op.getOperand(0); 8427 SDValue Hi = Op.getOperand(1); 8428 SDValue Amt = Op.getOperand(2); 8429 EVT AmtVT = Amt.getValueType(); 8430 8431 SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, 8432 DAG.getConstant(BitWidth, dl, AmtVT), Amt); 8433 SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); 8434 SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); 8435 SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); 8436 SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, 8437 DAG.getConstant(-BitWidth, dl, AmtVT)); 8438 SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5); 8439 SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt); 8440 SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT), 8441 Tmp4, Tmp6, ISD::SETLE); 8442 SDValue OutOps[] = { OutLo, OutHi }; 8443 return DAG.getMergeValues(OutOps, dl); 8444} 8445 8446//===----------------------------------------------------------------------===// 8447// Vector related lowering. 8448// 8449 8450/// BuildSplatI - Build a canonical splati of Val with an element size of 8451/// SplatSize. Cast the result to VT. 8452static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT, 8453 SelectionDAG &DAG, const SDLoc &dl) { 8454 static const MVT VTys[] = { // canonical VT to use for each size. 8455 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32 8456 }; 8457 8458 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; 8459 8460 // Force vspltis[hw] -1 to vspltisb -1 to canonicalize. 8461 if (Val == -1) 8462 SplatSize = 1; 8463 8464 EVT CanonicalVT = VTys[SplatSize-1]; 8465 8466 // Build a canonical splat for this value. 8467 return DAG.getBitcast(ReqVT, DAG.getConstant(Val, dl, CanonicalVT)); 8468} 8469 8470/// BuildIntrinsicOp - Return a unary operator intrinsic node with the 8471/// specified intrinsic ID. 8472static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG, 8473 const SDLoc &dl, EVT DestVT = MVT::Other) { 8474 if (DestVT == MVT::Other) DestVT = Op.getValueType(); 8475 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8476 DAG.getConstant(IID, dl, MVT::i32), Op); 8477} 8478 8479/// BuildIntrinsicOp - Return a binary operator intrinsic node with the 8480/// specified intrinsic ID. 8481static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS, 8482 SelectionDAG &DAG, const SDLoc &dl, 8483 EVT DestVT = MVT::Other) { 8484 if (DestVT == MVT::Other) DestVT = LHS.getValueType(); 8485 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8486 DAG.getConstant(IID, dl, MVT::i32), LHS, RHS); 8487} 8488 8489/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the 8490/// specified intrinsic ID. 8491static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1, 8492 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl, 8493 EVT DestVT = MVT::Other) { 8494 if (DestVT == MVT::Other) DestVT = Op0.getValueType(); 8495 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, 8496 DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2); 8497} 8498 8499/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified 8500/// amount. The result has the specified value type. 8501static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT, 8502 SelectionDAG &DAG, const SDLoc &dl) { 8503 // Force LHS/RHS to be the right type. 8504 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS); 8505 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS); 8506 8507 int Ops[16]; 8508 for (unsigned i = 0; i != 16; ++i) 8509 Ops[i] = i + Amt; 8510 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops); 8511 return DAG.getNode(ISD::BITCAST, dl, VT, T); 8512} 8513 8514/// Do we have an efficient pattern in a .td file for this node? 8515/// 8516/// \param V - pointer to the BuildVectorSDNode being matched 8517/// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves? 8518/// 8519/// There are some patterns where it is beneficial to keep a BUILD_VECTOR 8520/// node as a BUILD_VECTOR node rather than expanding it. The patterns where 8521/// the opposite is true (expansion is beneficial) are: 8522/// - The node builds a vector out of integers that are not 32 or 64-bits 8523/// - The node builds a vector out of constants 8524/// - The node is a "load-and-splat" 8525/// In all other cases, we will choose to keep the BUILD_VECTOR. 8526static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V, 8527 bool HasDirectMove, 8528 bool HasP8Vector) { 8529 EVT VecVT = V->getValueType(0); 8530 bool RightType = VecVT == MVT::v2f64 || 8531 (HasP8Vector && VecVT == MVT::v4f32) || 8532 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32)); 8533 if (!RightType) 8534 return false; 8535 8536 bool IsSplat = true; 8537 bool IsLoad = false; 8538 SDValue Op0 = V->getOperand(0); 8539 8540 // This function is called in a block that confirms the node is not a constant 8541 // splat. So a constant BUILD_VECTOR here means the vector is built out of 8542 // different constants. 8543 if (V->isConstant()) 8544 return false; 8545 for (int i = 0, e = V->getNumOperands(); i < e; ++i) { 8546 if (V->getOperand(i).isUndef()) 8547 return false; 8548 // We want to expand nodes that represent load-and-splat even if the 8549 // loaded value is a floating point truncation or conversion to int. 8550 if (V->getOperand(i).getOpcode() == ISD::LOAD || 8551 (V->getOperand(i).getOpcode() == ISD::FP_ROUND && 8552 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || 8553 (V->getOperand(i).getOpcode() == ISD::FP_TO_SINT && 8554 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || 8555 (V->getOperand(i).getOpcode() == ISD::FP_TO_UINT && 8556 V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD)) 8557 IsLoad = true; 8558 // If the operands are different or the input is not a load and has more 8559 // uses than just this BV node, then it isn't a splat. 8560 if (V->getOperand(i) != Op0 || 8561 (!IsLoad && !V->isOnlyUserOf(V->getOperand(i).getNode()))) 8562 IsSplat = false; 8563 } 8564 return !(IsSplat && IsLoad); 8565} 8566 8567// Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128. 8568SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { 8569 8570 SDLoc dl(Op); 8571 SDValue Op0 = Op->getOperand(0); 8572 8573 if (!EnableQuadPrecision || 8574 (Op.getValueType() != MVT::f128 ) || 8575 (Op0.getOpcode() != ISD::BUILD_PAIR) || 8576 (Op0.getOperand(0).getValueType() != MVT::i64) || 8577 (Op0.getOperand(1).getValueType() != MVT::i64)) 8578 return SDValue(); 8579 8580 return DAG.getNode(PPCISD::BUILD_FP128, dl, MVT::f128, Op0.getOperand(0), 8581 Op0.getOperand(1)); 8582} 8583 8584static const SDValue *getNormalLoadInput(const SDValue &Op) { 8585 const SDValue *InputLoad = &Op; 8586 if (InputLoad->getOpcode() == ISD::BITCAST) 8587 InputLoad = &InputLoad->getOperand(0); 8588 if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR) 8589 InputLoad = &InputLoad->getOperand(0); 8590 if (InputLoad->getOpcode() != ISD::LOAD) 8591 return nullptr; 8592 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); 8593 return ISD::isNormalLoad(LD) ? InputLoad : nullptr; 8594} 8595 8596// If this is a case we can't handle, return null and let the default 8597// expansion code take care of it. If we CAN select this case, and if it 8598// selects to a single instruction, return Op. Otherwise, if we can codegen 8599// this case more efficiently than a constant pool load, lower it to the 8600// sequence of ops that should be used. 8601SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, 8602 SelectionDAG &DAG) const { 8603 SDLoc dl(Op); 8604 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode()); 8605 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); 8606 8607 if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) { 8608 // We first build an i32 vector, load it into a QPX register, 8609 // then convert it to a floating-point vector and compare it 8610 // to a zero vector to get the boolean result. 8611 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 8612 int FrameIdx = MFI.CreateStackObject(16, 16, false); 8613 MachinePointerInfo PtrInfo = 8614 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 8615 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 8616 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 8617 8618 assert(BVN->getNumOperands() == 4 && 8619 "BUILD_VECTOR for v4i1 does not have 4 operands"); 8620 8621 bool IsConst = true; 8622 for (unsigned i = 0; i < 4; ++i) { 8623 if (BVN->getOperand(i).isUndef()) continue; 8624 if (!isa<ConstantSDNode>(BVN->getOperand(i))) { 8625 IsConst = false; 8626 break; 8627 } 8628 } 8629 8630 if (IsConst) { 8631 Constant *One = 8632 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0); 8633 Constant *NegOne = 8634 ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0); 8635 8636 Constant *CV[4]; 8637 for (unsigned i = 0; i < 4; ++i) { 8638 if (BVN->getOperand(i).isUndef()) 8639 CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext())); 8640 else if (isNullConstant(BVN->getOperand(i))) 8641 CV[i] = NegOne; 8642 else 8643 CV[i] = One; 8644 } 8645 8646 Constant *CP = ConstantVector::get(CV); 8647 SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), 8648 16 /* alignment */); 8649 8650 SDValue Ops[] = {DAG.getEntryNode(), CPIdx}; 8651 SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other}); 8652 return DAG.getMemIntrinsicNode( 8653 PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32, 8654 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 8655 } 8656 8657 SmallVector<SDValue, 4> Stores; 8658 for (unsigned i = 0; i < 4; ++i) { 8659 if (BVN->getOperand(i).isUndef()) continue; 8660 8661 unsigned Offset = 4*i; 8662 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 8663 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 8664 8665 unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize(); 8666 if (StoreSize > 4) { 8667 Stores.push_back( 8668 DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx, 8669 PtrInfo.getWithOffset(Offset), MVT::i32)); 8670 } else { 8671 SDValue StoreValue = BVN->getOperand(i); 8672 if (StoreSize < 4) 8673 StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue); 8674 8675 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx, 8676 PtrInfo.getWithOffset(Offset))); 8677 } 8678 } 8679 8680 SDValue StoreChain; 8681 if (!Stores.empty()) 8682 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 8683 else 8684 StoreChain = DAG.getEntryNode(); 8685 8686 // Now load from v4i32 into the QPX register; this will extend it to 8687 // v4i64 but not yet convert it to a floating point. Nevertheless, this 8688 // is typed as v4f64 because the QPX register integer states are not 8689 // explicitly represented. 8690 8691 SDValue Ops[] = {StoreChain, 8692 DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32), 8693 FIdx}; 8694 SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other}); 8695 8696 SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, 8697 dl, VTs, Ops, MVT::v4i32, PtrInfo); 8698 LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 8699 DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32), 8700 LoadedVect); 8701 8702 SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64); 8703 8704 return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ); 8705 } 8706 8707 // All other QPX vectors are handled by generic code. 8708 if (Subtarget.hasQPX()) 8709 return SDValue(); 8710 8711 // Check if this is a splat of a constant value. 8712 APInt APSplatBits, APSplatUndef; 8713 unsigned SplatBitSize; 8714 bool HasAnyUndefs; 8715 if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize, 8716 HasAnyUndefs, 0, !Subtarget.isLittleEndian()) || 8717 SplatBitSize > 32) { 8718 8719 const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0)); 8720 // Handle load-and-splat patterns as we have instructions that will do this 8721 // in one go. 8722 if (InputLoad && DAG.isSplatValue(Op, true)) { 8723 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); 8724 8725 // We have handling for 4 and 8 byte elements. 8726 unsigned ElementSize = LD->getMemoryVT().getScalarSizeInBits(); 8727 8728 // Checking for a single use of this load, we have to check for vector 8729 // width (128 bits) / ElementSize uses (since each operand of the 8730 // BUILD_VECTOR is a separate use of the value. 8731 if (InputLoad->getNode()->hasNUsesOfValue(128 / ElementSize, 0) && 8732 ((Subtarget.hasVSX() && ElementSize == 64) || 8733 (Subtarget.hasP9Vector() && ElementSize == 32))) { 8734 SDValue Ops[] = { 8735 LD->getChain(), // Chain 8736 LD->getBasePtr(), // Ptr 8737 DAG.getValueType(Op.getValueType()) // VT 8738 }; 8739 return 8740 DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, 8741 DAG.getVTList(Op.getValueType(), MVT::Other), 8742 Ops, LD->getMemoryVT(), LD->getMemOperand()); 8743 } 8744 } 8745 8746 // BUILD_VECTOR nodes that are not constant splats of up to 32-bits can be 8747 // lowered to VSX instructions under certain conditions. 8748 // Without VSX, there is no pattern more efficient than expanding the node. 8749 if (Subtarget.hasVSX() && 8750 haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(), 8751 Subtarget.hasP8Vector())) 8752 return Op; 8753 return SDValue(); 8754 } 8755 8756 unsigned SplatBits = APSplatBits.getZExtValue(); 8757 unsigned SplatUndef = APSplatUndef.getZExtValue(); 8758 unsigned SplatSize = SplatBitSize / 8; 8759 8760 // First, handle single instruction cases. 8761 8762 // All zeros? 8763 if (SplatBits == 0) { 8764 // Canonicalize all zero vectors to be v4i32. 8765 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) { 8766 SDValue Z = DAG.getConstant(0, dl, MVT::v4i32); 8767 Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z); 8768 } 8769 return Op; 8770 } 8771 8772 // We have XXSPLTIB for constant splats one byte wide 8773 // FIXME: SplatBits is an unsigned int being cast to an int while passing it 8774 // as an argument to BuildSplatiI. Given SplatSize == 1 it is okay here. 8775 if (Subtarget.hasP9Vector() && SplatSize == 1) 8776 return BuildSplatI(SplatBits, SplatSize, Op.getValueType(), DAG, dl); 8777 8778 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw]. 8779 int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >> 8780 (32-SplatBitSize)); 8781 if (SextVal >= -16 && SextVal <= 15) 8782 return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl); 8783 8784 // Two instruction sequences. 8785 8786 // If this value is in the range [-32,30] and is even, use: 8787 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2) 8788 // If this value is in the range [17,31] and is odd, use: 8789 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16) 8790 // If this value is in the range [-31,-17] and is odd, use: 8791 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16) 8792 // Note the last two are three-instruction sequences. 8793 if (SextVal >= -32 && SextVal <= 31) { 8794 // To avoid having these optimizations undone by constant folding, 8795 // we convert to a pseudo that will be expanded later into one of 8796 // the above forms. 8797 SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32); 8798 EVT VT = (SplatSize == 1 ? MVT::v16i8 : 8799 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32)); 8800 SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32); 8801 SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize); 8802 if (VT == Op.getValueType()) 8803 return RetVal; 8804 else 8805 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal); 8806 } 8807 8808 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is 8809 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important 8810 // for fneg/fabs. 8811 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) { 8812 // Make -1 and vspltisw -1: 8813 SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl); 8814 8815 // Make the VSLW intrinsic, computing 0x8000_0000. 8816 SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV, 8817 OnesV, DAG, dl); 8818 8819 // xor by OnesV to invert it. 8820 Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV); 8821 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8822 } 8823 8824 // Check to see if this is a wide variety of vsplti*, binop self cases. 8825 static const signed char SplatCsts[] = { 8826 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, 8827 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16 8828 }; 8829 8830 for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) { 8831 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for 8832 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1' 8833 int i = SplatCsts[idx]; 8834 8835 // Figure out what shift amount will be used by altivec if shifted by i in 8836 // this splat size. 8837 unsigned TypeShiftAmt = i & (SplatBitSize-1); 8838 8839 // vsplti + shl self. 8840 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) { 8841 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8842 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8843 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0, 8844 Intrinsic::ppc_altivec_vslw 8845 }; 8846 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8847 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8848 } 8849 8850 // vsplti + srl self. 8851 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 8852 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8853 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8854 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0, 8855 Intrinsic::ppc_altivec_vsrw 8856 }; 8857 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8858 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8859 } 8860 8861 // vsplti + sra self. 8862 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { 8863 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8864 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8865 Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0, 8866 Intrinsic::ppc_altivec_vsraw 8867 }; 8868 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8869 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8870 } 8871 8872 // vsplti + rol self. 8873 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | 8874 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { 8875 SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); 8876 static const unsigned IIDs[] = { // Intrinsic to use for each size. 8877 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0, 8878 Intrinsic::ppc_altivec_vrlw 8879 }; 8880 Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); 8881 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); 8882 } 8883 8884 // t = vsplti c, result = vsldoi t, t, 1 8885 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) { 8886 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 8887 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1; 8888 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 8889 } 8890 // t = vsplti c, result = vsldoi t, t, 2 8891 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) { 8892 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 8893 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2; 8894 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 8895 } 8896 // t = vsplti c, result = vsldoi t, t, 3 8897 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) { 8898 SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); 8899 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3; 8900 return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); 8901 } 8902 } 8903 8904 return SDValue(); 8905} 8906 8907/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit 8908/// the specified operations to build the shuffle. 8909static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, 8910 SDValue RHS, SelectionDAG &DAG, 8911 const SDLoc &dl) { 8912 unsigned OpNum = (PFEntry >> 26) & 0x0F; 8913 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); 8914 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); 8915 8916 enum { 8917 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3> 8918 OP_VMRGHW, 8919 OP_VMRGLW, 8920 OP_VSPLTISW0, 8921 OP_VSPLTISW1, 8922 OP_VSPLTISW2, 8923 OP_VSPLTISW3, 8924 OP_VSLDOI4, 8925 OP_VSLDOI8, 8926 OP_VSLDOI12 8927 }; 8928 8929 if (OpNum == OP_COPY) { 8930 if (LHSID == (1*9+2)*9+3) return LHS; 8931 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); 8932 return RHS; 8933 } 8934 8935 SDValue OpLHS, OpRHS; 8936 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); 8937 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); 8938 8939 int ShufIdxs[16]; 8940 switch (OpNum) { 8941 default: llvm_unreachable("Unknown i32 permute!"); 8942 case OP_VMRGHW: 8943 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3; 8944 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19; 8945 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7; 8946 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23; 8947 break; 8948 case OP_VMRGLW: 8949 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11; 8950 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27; 8951 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15; 8952 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31; 8953 break; 8954 case OP_VSPLTISW0: 8955 for (unsigned i = 0; i != 16; ++i) 8956 ShufIdxs[i] = (i&3)+0; 8957 break; 8958 case OP_VSPLTISW1: 8959 for (unsigned i = 0; i != 16; ++i) 8960 ShufIdxs[i] = (i&3)+4; 8961 break; 8962 case OP_VSPLTISW2: 8963 for (unsigned i = 0; i != 16; ++i) 8964 ShufIdxs[i] = (i&3)+8; 8965 break; 8966 case OP_VSPLTISW3: 8967 for (unsigned i = 0; i != 16; ++i) 8968 ShufIdxs[i] = (i&3)+12; 8969 break; 8970 case OP_VSLDOI4: 8971 return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl); 8972 case OP_VSLDOI8: 8973 return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl); 8974 case OP_VSLDOI12: 8975 return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl); 8976 } 8977 EVT VT = OpLHS.getValueType(); 8978 OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS); 8979 OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS); 8980 SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs); 8981 return DAG.getNode(ISD::BITCAST, dl, VT, T); 8982} 8983 8984/// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled 8985/// by the VINSERTB instruction introduced in ISA 3.0, else just return default 8986/// SDValue. 8987SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N, 8988 SelectionDAG &DAG) const { 8989 const unsigned BytesInVector = 16; 8990 bool IsLE = Subtarget.isLittleEndian(); 8991 SDLoc dl(N); 8992 SDValue V1 = N->getOperand(0); 8993 SDValue V2 = N->getOperand(1); 8994 unsigned ShiftElts = 0, InsertAtByte = 0; 8995 bool Swap = false; 8996 8997 // Shifts required to get the byte we want at element 7. 8998 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1, 8999 0, 15, 14, 13, 12, 11, 10, 9}; 9000 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0, 9001 1, 2, 3, 4, 5, 6, 7, 8}; 9002 9003 ArrayRef<int> Mask = N->getMask(); 9004 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; 9005 9006 // For each mask element, find out if we're just inserting something 9007 // from V2 into V1 or vice versa. 9008 // Possible permutations inserting an element from V2 into V1: 9009 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 9010 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 9011 // ... 9012 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X 9013 // Inserting from V1 into V2 will be similar, except mask range will be 9014 // [16,31]. 9015 9016 bool FoundCandidate = false; 9017 // If both vector operands for the shuffle are the same vector, the mask 9018 // will contain only elements from the first one and the second one will be 9019 // undef. 9020 unsigned VINSERTBSrcElem = IsLE ? 8 : 7; 9021 // Go through the mask of half-words to find an element that's being moved 9022 // from one vector to the other. 9023 for (unsigned i = 0; i < BytesInVector; ++i) { 9024 unsigned CurrentElement = Mask[i]; 9025 // If 2nd operand is undefined, we should only look for element 7 in the 9026 // Mask. 9027 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem) 9028 continue; 9029 9030 bool OtherElementsInOrder = true; 9031 // Examine the other elements in the Mask to see if they're in original 9032 // order. 9033 for (unsigned j = 0; j < BytesInVector; ++j) { 9034 if (j == i) 9035 continue; 9036 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be 9037 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined, 9038 // in which we always assume we're always picking from the 1st operand. 9039 int MaskOffset = 9040 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0; 9041 if (Mask[j] != OriginalOrder[j] + MaskOffset) { 9042 OtherElementsInOrder = false; 9043 break; 9044 } 9045 } 9046 // If other elements are in original order, we record the number of shifts 9047 // we need to get the element we want into element 7. Also record which byte 9048 // in the vector we should insert into. 9049 if (OtherElementsInOrder) { 9050 // If 2nd operand is undefined, we assume no shifts and no swapping. 9051 if (V2.isUndef()) { 9052 ShiftElts = 0; 9053 Swap = false; 9054 } else { 9055 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4. 9056 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF] 9057 : BigEndianShifts[CurrentElement & 0xF]; 9058 Swap = CurrentElement < BytesInVector; 9059 } 9060 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i; 9061 FoundCandidate = true; 9062 break; 9063 } 9064 } 9065 9066 if (!FoundCandidate) 9067 return SDValue(); 9068 9069 // Candidate found, construct the proper SDAG sequence with VINSERTB, 9070 // optionally with VECSHL if shift is required. 9071 if (Swap) 9072 std::swap(V1, V2); 9073 if (V2.isUndef()) 9074 V2 = V1; 9075 if (ShiftElts) { 9076 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, 9077 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9078 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, Shl, 9079 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9080 } 9081 return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, V2, 9082 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9083} 9084 9085/// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled 9086/// by the VINSERTH instruction introduced in ISA 3.0, else just return default 9087/// SDValue. 9088SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N, 9089 SelectionDAG &DAG) const { 9090 const unsigned NumHalfWords = 8; 9091 const unsigned BytesInVector = NumHalfWords * 2; 9092 // Check that the shuffle is on half-words. 9093 if (!isNByteElemShuffleMask(N, 2, 1)) 9094 return SDValue(); 9095 9096 bool IsLE = Subtarget.isLittleEndian(); 9097 SDLoc dl(N); 9098 SDValue V1 = N->getOperand(0); 9099 SDValue V2 = N->getOperand(1); 9100 unsigned ShiftElts = 0, InsertAtByte = 0; 9101 bool Swap = false; 9102 9103 // Shifts required to get the half-word we want at element 3. 9104 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5}; 9105 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4}; 9106 9107 uint32_t Mask = 0; 9108 uint32_t OriginalOrderLow = 0x1234567; 9109 uint32_t OriginalOrderHigh = 0x89ABCDEF; 9110 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a 9111 // 32-bit space, only need 4-bit nibbles per element. 9112 for (unsigned i = 0; i < NumHalfWords; ++i) { 9113 unsigned MaskShift = (NumHalfWords - 1 - i) * 4; 9114 Mask |= ((uint32_t)(N->getMaskElt(i * 2) / 2) << MaskShift); 9115 } 9116 9117 // For each mask element, find out if we're just inserting something 9118 // from V2 into V1 or vice versa. Possible permutations inserting an element 9119 // from V2 into V1: 9120 // X, 1, 2, 3, 4, 5, 6, 7 9121 // 0, X, 2, 3, 4, 5, 6, 7 9122 // 0, 1, X, 3, 4, 5, 6, 7 9123 // 0, 1, 2, X, 4, 5, 6, 7 9124 // 0, 1, 2, 3, X, 5, 6, 7 9125 // 0, 1, 2, 3, 4, X, 6, 7 9126 // 0, 1, 2, 3, 4, 5, X, 7 9127 // 0, 1, 2, 3, 4, 5, 6, X 9128 // Inserting from V1 into V2 will be similar, except mask range will be [8,15]. 9129 9130 bool FoundCandidate = false; 9131 // Go through the mask of half-words to find an element that's being moved 9132 // from one vector to the other. 9133 for (unsigned i = 0; i < NumHalfWords; ++i) { 9134 unsigned MaskShift = (NumHalfWords - 1 - i) * 4; 9135 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF; 9136 uint32_t MaskOtherElts = ~(0xF << MaskShift); 9137 uint32_t TargetOrder = 0x0; 9138 9139 // If both vector operands for the shuffle are the same vector, the mask 9140 // will contain only elements from the first one and the second one will be 9141 // undef. 9142 if (V2.isUndef()) { 9143 ShiftElts = 0; 9144 unsigned VINSERTHSrcElem = IsLE ? 4 : 3; 9145 TargetOrder = OriginalOrderLow; 9146 Swap = false; 9147 // Skip if not the correct element or mask of other elements don't equal 9148 // to our expected order. 9149 if (MaskOneElt == VINSERTHSrcElem && 9150 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { 9151 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; 9152 FoundCandidate = true; 9153 break; 9154 } 9155 } else { // If both operands are defined. 9156 // Target order is [8,15] if the current mask is between [0,7]. 9157 TargetOrder = 9158 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow; 9159 // Skip if mask of other elements don't equal our expected order. 9160 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { 9161 // We only need the last 3 bits for the number of shifts. 9162 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7] 9163 : BigEndianShifts[MaskOneElt & 0x7]; 9164 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; 9165 Swap = MaskOneElt < NumHalfWords; 9166 FoundCandidate = true; 9167 break; 9168 } 9169 } 9170 } 9171 9172 if (!FoundCandidate) 9173 return SDValue(); 9174 9175 // Candidate found, construct the proper SDAG sequence with VINSERTH, 9176 // optionally with VECSHL if shift is required. 9177 if (Swap) 9178 std::swap(V1, V2); 9179 if (V2.isUndef()) 9180 V2 = V1; 9181 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 9182 if (ShiftElts) { 9183 // Double ShiftElts because we're left shifting on v16i8 type. 9184 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, 9185 DAG.getConstant(2 * ShiftElts, dl, MVT::i32)); 9186 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, Shl); 9187 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, 9188 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9189 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9190 } 9191 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); 9192 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, 9193 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9194 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9195} 9196 9197/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this 9198/// is a shuffle we can handle in a single instruction, return it. Otherwise, 9199/// return the code it can be lowered into. Worst case, it can always be 9200/// lowered into a vperm. 9201SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 9202 SelectionDAG &DAG) const { 9203 SDLoc dl(Op); 9204 SDValue V1 = Op.getOperand(0); 9205 SDValue V2 = Op.getOperand(1); 9206 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); 9207 EVT VT = Op.getValueType(); 9208 bool isLittleEndian = Subtarget.isLittleEndian(); 9209 9210 unsigned ShiftElts, InsertAtByte; 9211 bool Swap = false; 9212 9213 // If this is a load-and-splat, we can do that with a single instruction 9214 // in some cases. However if the load has multiple uses, we don't want to 9215 // combine it because that will just produce multiple loads. 9216 const SDValue *InputLoad = getNormalLoadInput(V1); 9217 if (InputLoad && Subtarget.hasVSX() && V2.isUndef() && 9218 (PPC::isSplatShuffleMask(SVOp, 4) || PPC::isSplatShuffleMask(SVOp, 8)) && 9219 InputLoad->hasOneUse()) { 9220 bool IsFourByte = PPC::isSplatShuffleMask(SVOp, 4); 9221 int SplatIdx = 9222 PPC::getSplatIdxForPPCMnemonics(SVOp, IsFourByte ? 4 : 8, DAG); 9223 9224 LoadSDNode *LD = cast<LoadSDNode>(*InputLoad); 9225 // For 4-byte load-and-splat, we need Power9. 9226 if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) { 9227 uint64_t Offset = 0; 9228 if (IsFourByte) 9229 Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4; 9230 else 9231 Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8; 9232 SDValue BasePtr = LD->getBasePtr(); 9233 if (Offset != 0) 9234 BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), 9235 BasePtr, DAG.getIntPtrConstant(Offset, dl)); 9236 SDValue Ops[] = { 9237 LD->getChain(), // Chain 9238 BasePtr, // BasePtr 9239 DAG.getValueType(Op.getValueType()) // VT 9240 }; 9241 SDVTList VTL = 9242 DAG.getVTList(IsFourByte ? MVT::v4i32 : MVT::v2i64, MVT::Other); 9243 SDValue LdSplt = 9244 DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, VTL, 9245 Ops, LD->getMemoryVT(), LD->getMemOperand()); 9246 if (LdSplt.getValueType() != SVOp->getValueType(0)) 9247 LdSplt = DAG.getBitcast(SVOp->getValueType(0), LdSplt); 9248 return LdSplt; 9249 } 9250 } 9251 if (Subtarget.hasP9Vector() && 9252 PPC::isXXINSERTWMask(SVOp, ShiftElts, InsertAtByte, Swap, 9253 isLittleEndian)) { 9254 if (Swap) 9255 std::swap(V1, V2); 9256 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9257 SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2); 9258 if (ShiftElts) { 9259 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv2, Conv2, 9260 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9261 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Shl, 9262 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9263 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9264 } 9265 SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Conv2, 9266 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9267 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); 9268 } 9269 9270 if (Subtarget.hasP9Altivec()) { 9271 SDValue NewISDNode; 9272 if ((NewISDNode = lowerToVINSERTH(SVOp, DAG))) 9273 return NewISDNode; 9274 9275 if ((NewISDNode = lowerToVINSERTB(SVOp, DAG))) 9276 return NewISDNode; 9277 } 9278 9279 if (Subtarget.hasVSX() && 9280 PPC::isXXSLDWIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { 9281 if (Swap) 9282 std::swap(V1, V2); 9283 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9284 SDValue Conv2 = 9285 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2.isUndef() ? V1 : V2); 9286 9287 SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv1, Conv2, 9288 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9289 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Shl); 9290 } 9291 9292 if (Subtarget.hasVSX() && 9293 PPC::isXXPERMDIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { 9294 if (Swap) 9295 std::swap(V1, V2); 9296 SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); 9297 SDValue Conv2 = 9298 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2.isUndef() ? V1 : V2); 9299 9300 SDValue PermDI = DAG.getNode(PPCISD::XXPERMDI, dl, MVT::v2i64, Conv1, Conv2, 9301 DAG.getConstant(ShiftElts, dl, MVT::i32)); 9302 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, PermDI); 9303 } 9304 9305 if (Subtarget.hasP9Vector()) { 9306 if (PPC::isXXBRHShuffleMask(SVOp)) { 9307 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); 9308 SDValue ReveHWord = DAG.getNode(ISD::BSWAP, dl, MVT::v8i16, Conv); 9309 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveHWord); 9310 } else if (PPC::isXXBRWShuffleMask(SVOp)) { 9311 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9312 SDValue ReveWord = DAG.getNode(ISD::BSWAP, dl, MVT::v4i32, Conv); 9313 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveWord); 9314 } else if (PPC::isXXBRDShuffleMask(SVOp)) { 9315 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); 9316 SDValue ReveDWord = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Conv); 9317 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveDWord); 9318 } else if (PPC::isXXBRQShuffleMask(SVOp)) { 9319 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, V1); 9320 SDValue ReveQWord = DAG.getNode(ISD::BSWAP, dl, MVT::v1i128, Conv); 9321 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveQWord); 9322 } 9323 } 9324 9325 if (Subtarget.hasVSX()) { 9326 if (V2.isUndef() && PPC::isSplatShuffleMask(SVOp, 4)) { 9327 int SplatIdx = PPC::getSplatIdxForPPCMnemonics(SVOp, 4, DAG); 9328 9329 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); 9330 SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv, 9331 DAG.getConstant(SplatIdx, dl, MVT::i32)); 9332 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Splat); 9333 } 9334 9335 // Left shifts of 8 bytes are actually swaps. Convert accordingly. 9336 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) == 8) { 9337 SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); 9338 SDValue Swap = DAG.getNode(PPCISD::SWAP_NO_CHAIN, dl, MVT::v2f64, Conv); 9339 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Swap); 9340 } 9341 } 9342 9343 if (Subtarget.hasQPX()) { 9344 if (VT.getVectorNumElements() != 4) 9345 return SDValue(); 9346 9347 if (V2.isUndef()) V2 = V1; 9348 9349 int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp); 9350 if (AlignIdx != -1) { 9351 return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2, 9352 DAG.getConstant(AlignIdx, dl, MVT::i32)); 9353 } else if (SVOp->isSplat()) { 9354 int SplatIdx = SVOp->getSplatIndex(); 9355 if (SplatIdx >= 4) { 9356 std::swap(V1, V2); 9357 SplatIdx -= 4; 9358 } 9359 9360 return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1, 9361 DAG.getConstant(SplatIdx, dl, MVT::i32)); 9362 } 9363 9364 // Lower this into a qvgpci/qvfperm pair. 9365 9366 // Compute the qvgpci literal 9367 unsigned idx = 0; 9368 for (unsigned i = 0; i < 4; ++i) { 9369 int m = SVOp->getMaskElt(i); 9370 unsigned mm = m >= 0 ? (unsigned) m : i; 9371 idx |= mm << (3-i)*3; 9372 } 9373 9374 SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64, 9375 DAG.getConstant(idx, dl, MVT::i32)); 9376 return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3); 9377 } 9378 9379 // Cases that are handled by instructions that take permute immediates 9380 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be 9381 // selected by the instruction selector. 9382 if (V2.isUndef()) { 9383 if (PPC::isSplatShuffleMask(SVOp, 1) || 9384 PPC::isSplatShuffleMask(SVOp, 2) || 9385 PPC::isSplatShuffleMask(SVOp, 4) || 9386 PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) || 9387 PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) || 9388 PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 || 9389 PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) || 9390 PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) || 9391 PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) || 9392 PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) || 9393 PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) || 9394 PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) || 9395 (Subtarget.hasP8Altivec() && ( 9396 PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) || 9397 PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG) || 9398 PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)))) { 9399 return Op; 9400 } 9401 } 9402 9403 // Altivec has a variety of "shuffle immediates" that take two vector inputs 9404 // and produce a fixed permutation. If any of these match, do not lower to 9405 // VPERM. 9406 unsigned int ShuffleKind = isLittleEndian ? 2 : 0; 9407 if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) || 9408 PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) || 9409 PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 || 9410 PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) || 9411 PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) || 9412 PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) || 9413 PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) || 9414 PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) || 9415 PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) || 9416 (Subtarget.hasP8Altivec() && ( 9417 PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) || 9418 PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG) || 9419 PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG)))) 9420 return Op; 9421 9422 // Check to see if this is a shuffle of 4-byte values. If so, we can use our 9423 // perfect shuffle table to emit an optimal matching sequence. 9424 ArrayRef<int> PermMask = SVOp->getMask(); 9425 9426 unsigned PFIndexes[4]; 9427 bool isFourElementShuffle = true; 9428 for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number 9429 unsigned EltNo = 8; // Start out undef. 9430 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte. 9431 if (PermMask[i*4+j] < 0) 9432 continue; // Undef, ignore it. 9433 9434 unsigned ByteSource = PermMask[i*4+j]; 9435 if ((ByteSource & 3) != j) { 9436 isFourElementShuffle = false; 9437 break; 9438 } 9439 9440 if (EltNo == 8) { 9441 EltNo = ByteSource/4; 9442 } else if (EltNo != ByteSource/4) { 9443 isFourElementShuffle = false; 9444 break; 9445 } 9446 } 9447 PFIndexes[i] = EltNo; 9448 } 9449 9450 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the 9451 // perfect shuffle vector to determine if it is cost effective to do this as 9452 // discrete instructions, or whether we should use a vperm. 9453 // For now, we skip this for little endian until such time as we have a 9454 // little-endian perfect shuffle table. 9455 if (isFourElementShuffle && !isLittleEndian) { 9456 // Compute the index in the perfect shuffle table. 9457 unsigned PFTableIndex = 9458 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; 9459 9460 unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; 9461 unsigned Cost = (PFEntry >> 30); 9462 9463 // Determining when to avoid vperm is tricky. Many things affect the cost 9464 // of vperm, particularly how many times the perm mask needs to be computed. 9465 // For example, if the perm mask can be hoisted out of a loop or is already 9466 // used (perhaps because there are multiple permutes with the same shuffle 9467 // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of 9468 // the loop requires an extra register. 9469 // 9470 // As a compromise, we only emit discrete instructions if the shuffle can be 9471 // generated in 3 or fewer operations. When we have loop information 9472 // available, if this block is within a loop, we should avoid using vperm 9473 // for 3-operation perms and use a constant pool load instead. 9474 if (Cost < 3) 9475 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); 9476 } 9477 9478 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant 9479 // vector that will get spilled to the constant pool. 9480 if (V2.isUndef()) V2 = V1; 9481 9482 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except 9483 // that it is in input element units, not in bytes. Convert now. 9484 9485 // For little endian, the order of the input vectors is reversed, and 9486 // the permutation mask is complemented with respect to 31. This is 9487 // necessary to produce proper semantics with the big-endian-biased vperm 9488 // instruction. 9489 EVT EltVT = V1.getValueType().getVectorElementType(); 9490 unsigned BytesPerElement = EltVT.getSizeInBits()/8; 9491 9492 SmallVector<SDValue, 16> ResultMask; 9493 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { 9494 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i]; 9495 9496 for (unsigned j = 0; j != BytesPerElement; ++j) 9497 if (isLittleEndian) 9498 ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j), 9499 dl, MVT::i32)); 9500 else 9501 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl, 9502 MVT::i32)); 9503 } 9504 9505 SDValue VPermMask = DAG.getBuildVector(MVT::v16i8, dl, ResultMask); 9506 if (isLittleEndian) 9507 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 9508 V2, V1, VPermMask); 9509 else 9510 return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), 9511 V1, V2, VPermMask); 9512} 9513 9514/// getVectorCompareInfo - Given an intrinsic, return false if it is not a 9515/// vector comparison. If it is, return true and fill in Opc/isDot with 9516/// information about the intrinsic. 9517static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, 9518 bool &isDot, const PPCSubtarget &Subtarget) { 9519 unsigned IntrinsicID = 9520 cast<ConstantSDNode>(Intrin.getOperand(0))->getZExtValue(); 9521 CompareOpc = -1; 9522 isDot = false; 9523 switch (IntrinsicID) { 9524 default: 9525 return false; 9526 // Comparison predicates. 9527 case Intrinsic::ppc_altivec_vcmpbfp_p: 9528 CompareOpc = 966; 9529 isDot = true; 9530 break; 9531 case Intrinsic::ppc_altivec_vcmpeqfp_p: 9532 CompareOpc = 198; 9533 isDot = true; 9534 break; 9535 case Intrinsic::ppc_altivec_vcmpequb_p: 9536 CompareOpc = 6; 9537 isDot = true; 9538 break; 9539 case Intrinsic::ppc_altivec_vcmpequh_p: 9540 CompareOpc = 70; 9541 isDot = true; 9542 break; 9543 case Intrinsic::ppc_altivec_vcmpequw_p: 9544 CompareOpc = 134; 9545 isDot = true; 9546 break; 9547 case Intrinsic::ppc_altivec_vcmpequd_p: 9548 if (Subtarget.hasP8Altivec()) { 9549 CompareOpc = 199; 9550 isDot = true; 9551 } else 9552 return false; 9553 break; 9554 case Intrinsic::ppc_altivec_vcmpneb_p: 9555 case Intrinsic::ppc_altivec_vcmpneh_p: 9556 case Intrinsic::ppc_altivec_vcmpnew_p: 9557 case Intrinsic::ppc_altivec_vcmpnezb_p: 9558 case Intrinsic::ppc_altivec_vcmpnezh_p: 9559 case Intrinsic::ppc_altivec_vcmpnezw_p: 9560 if (Subtarget.hasP9Altivec()) { 9561 switch (IntrinsicID) { 9562 default: 9563 llvm_unreachable("Unknown comparison intrinsic."); 9564 case Intrinsic::ppc_altivec_vcmpneb_p: 9565 CompareOpc = 7; 9566 break; 9567 case Intrinsic::ppc_altivec_vcmpneh_p: 9568 CompareOpc = 71; 9569 break; 9570 case Intrinsic::ppc_altivec_vcmpnew_p: 9571 CompareOpc = 135; 9572 break; 9573 case Intrinsic::ppc_altivec_vcmpnezb_p: 9574 CompareOpc = 263; 9575 break; 9576 case Intrinsic::ppc_altivec_vcmpnezh_p: 9577 CompareOpc = 327; 9578 break; 9579 case Intrinsic::ppc_altivec_vcmpnezw_p: 9580 CompareOpc = 391; 9581 break; 9582 } 9583 isDot = true; 9584 } else 9585 return false; 9586 break; 9587 case Intrinsic::ppc_altivec_vcmpgefp_p: 9588 CompareOpc = 454; 9589 isDot = true; 9590 break; 9591 case Intrinsic::ppc_altivec_vcmpgtfp_p: 9592 CompareOpc = 710; 9593 isDot = true; 9594 break; 9595 case Intrinsic::ppc_altivec_vcmpgtsb_p: 9596 CompareOpc = 774; 9597 isDot = true; 9598 break; 9599 case Intrinsic::ppc_altivec_vcmpgtsh_p: 9600 CompareOpc = 838; 9601 isDot = true; 9602 break; 9603 case Intrinsic::ppc_altivec_vcmpgtsw_p: 9604 CompareOpc = 902; 9605 isDot = true; 9606 break; 9607 case Intrinsic::ppc_altivec_vcmpgtsd_p: 9608 if (Subtarget.hasP8Altivec()) { 9609 CompareOpc = 967; 9610 isDot = true; 9611 } else 9612 return false; 9613 break; 9614 case Intrinsic::ppc_altivec_vcmpgtub_p: 9615 CompareOpc = 518; 9616 isDot = true; 9617 break; 9618 case Intrinsic::ppc_altivec_vcmpgtuh_p: 9619 CompareOpc = 582; 9620 isDot = true; 9621 break; 9622 case Intrinsic::ppc_altivec_vcmpgtuw_p: 9623 CompareOpc = 646; 9624 isDot = true; 9625 break; 9626 case Intrinsic::ppc_altivec_vcmpgtud_p: 9627 if (Subtarget.hasP8Altivec()) { 9628 CompareOpc = 711; 9629 isDot = true; 9630 } else 9631 return false; 9632 break; 9633 9634 // VSX predicate comparisons use the same infrastructure 9635 case Intrinsic::ppc_vsx_xvcmpeqdp_p: 9636 case Intrinsic::ppc_vsx_xvcmpgedp_p: 9637 case Intrinsic::ppc_vsx_xvcmpgtdp_p: 9638 case Intrinsic::ppc_vsx_xvcmpeqsp_p: 9639 case Intrinsic::ppc_vsx_xvcmpgesp_p: 9640 case Intrinsic::ppc_vsx_xvcmpgtsp_p: 9641 if (Subtarget.hasVSX()) { 9642 switch (IntrinsicID) { 9643 case Intrinsic::ppc_vsx_xvcmpeqdp_p: 9644 CompareOpc = 99; 9645 break; 9646 case Intrinsic::ppc_vsx_xvcmpgedp_p: 9647 CompareOpc = 115; 9648 break; 9649 case Intrinsic::ppc_vsx_xvcmpgtdp_p: 9650 CompareOpc = 107; 9651 break; 9652 case Intrinsic::ppc_vsx_xvcmpeqsp_p: 9653 CompareOpc = 67; 9654 break; 9655 case Intrinsic::ppc_vsx_xvcmpgesp_p: 9656 CompareOpc = 83; 9657 break; 9658 case Intrinsic::ppc_vsx_xvcmpgtsp_p: 9659 CompareOpc = 75; 9660 break; 9661 } 9662 isDot = true; 9663 } else 9664 return false; 9665 break; 9666 9667 // Normal Comparisons. 9668 case Intrinsic::ppc_altivec_vcmpbfp: 9669 CompareOpc = 966; 9670 break; 9671 case Intrinsic::ppc_altivec_vcmpeqfp: 9672 CompareOpc = 198; 9673 break; 9674 case Intrinsic::ppc_altivec_vcmpequb: 9675 CompareOpc = 6; 9676 break; 9677 case Intrinsic::ppc_altivec_vcmpequh: 9678 CompareOpc = 70; 9679 break; 9680 case Intrinsic::ppc_altivec_vcmpequw: 9681 CompareOpc = 134; 9682 break; 9683 case Intrinsic::ppc_altivec_vcmpequd: 9684 if (Subtarget.hasP8Altivec()) 9685 CompareOpc = 199; 9686 else 9687 return false; 9688 break; 9689 case Intrinsic::ppc_altivec_vcmpneb: 9690 case Intrinsic::ppc_altivec_vcmpneh: 9691 case Intrinsic::ppc_altivec_vcmpnew: 9692 case Intrinsic::ppc_altivec_vcmpnezb: 9693 case Intrinsic::ppc_altivec_vcmpnezh: 9694 case Intrinsic::ppc_altivec_vcmpnezw: 9695 if (Subtarget.hasP9Altivec()) 9696 switch (IntrinsicID) { 9697 default: 9698 llvm_unreachable("Unknown comparison intrinsic."); 9699 case Intrinsic::ppc_altivec_vcmpneb: 9700 CompareOpc = 7; 9701 break; 9702 case Intrinsic::ppc_altivec_vcmpneh: 9703 CompareOpc = 71; 9704 break; 9705 case Intrinsic::ppc_altivec_vcmpnew: 9706 CompareOpc = 135; 9707 break; 9708 case Intrinsic::ppc_altivec_vcmpnezb: 9709 CompareOpc = 263; 9710 break; 9711 case Intrinsic::ppc_altivec_vcmpnezh: 9712 CompareOpc = 327; 9713 break; 9714 case Intrinsic::ppc_altivec_vcmpnezw: 9715 CompareOpc = 391; 9716 break; 9717 } 9718 else 9719 return false; 9720 break; 9721 case Intrinsic::ppc_altivec_vcmpgefp: 9722 CompareOpc = 454; 9723 break; 9724 case Intrinsic::ppc_altivec_vcmpgtfp: 9725 CompareOpc = 710; 9726 break; 9727 case Intrinsic::ppc_altivec_vcmpgtsb: 9728 CompareOpc = 774; 9729 break; 9730 case Intrinsic::ppc_altivec_vcmpgtsh: 9731 CompareOpc = 838; 9732 break; 9733 case Intrinsic::ppc_altivec_vcmpgtsw: 9734 CompareOpc = 902; 9735 break; 9736 case Intrinsic::ppc_altivec_vcmpgtsd: 9737 if (Subtarget.hasP8Altivec()) 9738 CompareOpc = 967; 9739 else 9740 return false; 9741 break; 9742 case Intrinsic::ppc_altivec_vcmpgtub: 9743 CompareOpc = 518; 9744 break; 9745 case Intrinsic::ppc_altivec_vcmpgtuh: 9746 CompareOpc = 582; 9747 break; 9748 case Intrinsic::ppc_altivec_vcmpgtuw: 9749 CompareOpc = 646; 9750 break; 9751 case Intrinsic::ppc_altivec_vcmpgtud: 9752 if (Subtarget.hasP8Altivec()) 9753 CompareOpc = 711; 9754 else 9755 return false; 9756 break; 9757 } 9758 return true; 9759} 9760 9761/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom 9762/// lower, do it, otherwise return null. 9763SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 9764 SelectionDAG &DAG) const { 9765 unsigned IntrinsicID = 9766 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 9767 9768 SDLoc dl(Op); 9769 9770 if (IntrinsicID == Intrinsic::thread_pointer) { 9771 // Reads the thread pointer register, used for __builtin_thread_pointer. 9772 if (Subtarget.isPPC64()) 9773 return DAG.getRegister(PPC::X13, MVT::i64); 9774 return DAG.getRegister(PPC::R2, MVT::i32); 9775 } 9776 9777 // If this is a lowered altivec predicate compare, CompareOpc is set to the 9778 // opcode number of the comparison. 9779 int CompareOpc; 9780 bool isDot; 9781 if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget)) 9782 return SDValue(); // Don't custom lower most intrinsics. 9783 9784 // If this is a non-dot comparison, make the VCMP node and we are done. 9785 if (!isDot) { 9786 SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(), 9787 Op.getOperand(1), Op.getOperand(2), 9788 DAG.getConstant(CompareOpc, dl, MVT::i32)); 9789 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp); 9790 } 9791 9792 // Create the PPCISD altivec 'dot' comparison node. 9793 SDValue Ops[] = { 9794 Op.getOperand(2), // LHS 9795 Op.getOperand(3), // RHS 9796 DAG.getConstant(CompareOpc, dl, MVT::i32) 9797 }; 9798 EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; 9799 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 9800 9801 // Now that we have the comparison, emit a copy from the CR to a GPR. 9802 // This is flagged to the above dot comparison. 9803 SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32, 9804 DAG.getRegister(PPC::CR6, MVT::i32), 9805 CompNode.getValue(1)); 9806 9807 // Unpack the result based on how the target uses it. 9808 unsigned BitNo; // Bit # of CR6. 9809 bool InvertBit; // Invert result? 9810 switch (cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue()) { 9811 default: // Can't happen, don't crash on invalid number though. 9812 case 0: // Return the value of the EQ bit of CR6. 9813 BitNo = 0; InvertBit = false; 9814 break; 9815 case 1: // Return the inverted value of the EQ bit of CR6. 9816 BitNo = 0; InvertBit = true; 9817 break; 9818 case 2: // Return the value of the LT bit of CR6. 9819 BitNo = 2; InvertBit = false; 9820 break; 9821 case 3: // Return the inverted value of the LT bit of CR6. 9822 BitNo = 2; InvertBit = true; 9823 break; 9824 } 9825 9826 // Shift the bit into the low position. 9827 Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags, 9828 DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32)); 9829 // Isolate the bit. 9830 Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags, 9831 DAG.getConstant(1, dl, MVT::i32)); 9832 9833 // If we are supposed to, toggle the bit. 9834 if (InvertBit) 9835 Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags, 9836 DAG.getConstant(1, dl, MVT::i32)); 9837 return Flags; 9838} 9839 9840SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op, 9841 SelectionDAG &DAG) const { 9842 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to 9843 // the beginning of the argument list. 9844 int ArgStart = isa<ConstantSDNode>(Op.getOperand(0)) ? 0 : 1; 9845 SDLoc DL(Op); 9846 switch (cast<ConstantSDNode>(Op.getOperand(ArgStart))->getZExtValue()) { 9847 case Intrinsic::ppc_cfence: { 9848 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument."); 9849 assert(Subtarget.isPPC64() && "Only 64-bit is supported for now."); 9850 return SDValue(DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other, 9851 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, 9852 Op.getOperand(ArgStart + 1)), 9853 Op.getOperand(0)), 9854 0); 9855 } 9856 default: 9857 break; 9858 } 9859 return SDValue(); 9860} 9861 9862SDValue PPCTargetLowering::LowerREM(SDValue Op, SelectionDAG &DAG) const { 9863 // Check for a DIV with the same operands as this REM. 9864 for (auto UI : Op.getOperand(1)->uses()) { 9865 if ((Op.getOpcode() == ISD::SREM && UI->getOpcode() == ISD::SDIV) || 9866 (Op.getOpcode() == ISD::UREM && UI->getOpcode() == ISD::UDIV)) 9867 if (UI->getOperand(0) == Op.getOperand(0) && 9868 UI->getOperand(1) == Op.getOperand(1)) 9869 return SDValue(); 9870 } 9871 return Op; 9872} 9873 9874// Lower scalar BSWAP64 to xxbrd. 9875SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const { 9876 SDLoc dl(Op); 9877 // MTVSRDD 9878 Op = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, Op.getOperand(0), 9879 Op.getOperand(0)); 9880 // XXBRD 9881 Op = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Op); 9882 // MFVSRD 9883 int VectorIndex = 0; 9884 if (Subtarget.isLittleEndian()) 9885 VectorIndex = 1; 9886 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op, 9887 DAG.getTargetConstant(VectorIndex, dl, MVT::i32)); 9888 return Op; 9889} 9890 9891// ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be 9892// compared to a value that is atomically loaded (atomic loads zero-extend). 9893SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, 9894 SelectionDAG &DAG) const { 9895 assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP && 9896 "Expecting an atomic compare-and-swap here."); 9897 SDLoc dl(Op); 9898 auto *AtomicNode = cast<AtomicSDNode>(Op.getNode()); 9899 EVT MemVT = AtomicNode->getMemoryVT(); 9900 if (MemVT.getSizeInBits() >= 32) 9901 return Op; 9902 9903 SDValue CmpOp = Op.getOperand(2); 9904 // If this is already correctly zero-extended, leave it alone. 9905 auto HighBits = APInt::getHighBitsSet(32, 32 - MemVT.getSizeInBits()); 9906 if (DAG.MaskedValueIsZero(CmpOp, HighBits)) 9907 return Op; 9908 9909 // Clear the high bits of the compare operand. 9910 unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1; 9911 SDValue NewCmpOp = 9912 DAG.getNode(ISD::AND, dl, MVT::i32, CmpOp, 9913 DAG.getConstant(MaskVal, dl, MVT::i32)); 9914 9915 // Replace the existing compare operand with the properly zero-extended one. 9916 SmallVector<SDValue, 4> Ops; 9917 for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++) 9918 Ops.push_back(AtomicNode->getOperand(i)); 9919 Ops[2] = NewCmpOp; 9920 MachineMemOperand *MMO = AtomicNode->getMemOperand(); 9921 SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other); 9922 auto NodeTy = 9923 (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16; 9924 return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO); 9925} 9926 9927SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, 9928 SelectionDAG &DAG) const { 9929 SDLoc dl(Op); 9930 // Create a stack slot that is 16-byte aligned. 9931 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 9932 int FrameIdx = MFI.CreateStackObject(16, 16, false); 9933 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 9934 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 9935 9936 // Store the input value into Value#0 of the stack slot. 9937 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, 9938 MachinePointerInfo()); 9939 // Load it out. 9940 return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo()); 9941} 9942 9943SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, 9944 SelectionDAG &DAG) const { 9945 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && 9946 "Should only be called for ISD::INSERT_VECTOR_ELT"); 9947 9948 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(2)); 9949 // We have legal lowering for constant indices but not for variable ones. 9950 if (!C) 9951 return SDValue(); 9952 9953 EVT VT = Op.getValueType(); 9954 SDLoc dl(Op); 9955 SDValue V1 = Op.getOperand(0); 9956 SDValue V2 = Op.getOperand(1); 9957 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types. 9958 if (VT == MVT::v8i16 || VT == MVT::v16i8) { 9959 SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2); 9960 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8; 9961 unsigned InsertAtElement = C->getZExtValue(); 9962 unsigned InsertAtByte = InsertAtElement * BytesInEachElement; 9963 if (Subtarget.isLittleEndian()) { 9964 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte; 9965 } 9966 return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, Mtvsrz, 9967 DAG.getConstant(InsertAtByte, dl, MVT::i32)); 9968 } 9969 return Op; 9970} 9971 9972SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 9973 SelectionDAG &DAG) const { 9974 SDLoc dl(Op); 9975 SDNode *N = Op.getNode(); 9976 9977 assert(N->getOperand(0).getValueType() == MVT::v4i1 && 9978 "Unknown extract_vector_elt type"); 9979 9980 SDValue Value = N->getOperand(0); 9981 9982 // The first part of this is like the store lowering except that we don't 9983 // need to track the chain. 9984 9985 // The values are now known to be -1 (false) or 1 (true). To convert this 9986 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 9987 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 9988 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 9989 9990 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to 9991 // understand how to form the extending load. 9992 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); 9993 9994 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 9995 9996 // Now convert to an integer and store. 9997 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 9998 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), 9999 Value); 10000 10001 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 10002 int FrameIdx = MFI.CreateStackObject(16, 16, false); 10003 MachinePointerInfo PtrInfo = 10004 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 10005 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 10006 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 10007 10008 SDValue StoreChain = DAG.getEntryNode(); 10009 SDValue Ops[] = {StoreChain, 10010 DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), 10011 Value, FIdx}; 10012 SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); 10013 10014 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, 10015 dl, VTs, Ops, MVT::v4i32, PtrInfo); 10016 10017 // Extract the value requested. 10018 unsigned Offset = 4*cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 10019 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 10020 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 10021 10022 SDValue IntVal = 10023 DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset)); 10024 10025 if (!Subtarget.useCRBits()) 10026 return IntVal; 10027 10028 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal); 10029} 10030 10031/// Lowering for QPX v4i1 loads 10032SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, 10033 SelectionDAG &DAG) const { 10034 SDLoc dl(Op); 10035 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); 10036 SDValue LoadChain = LN->getChain(); 10037 SDValue BasePtr = LN->getBasePtr(); 10038 10039 if (Op.getValueType() == MVT::v4f64 || 10040 Op.getValueType() == MVT::v4f32) { 10041 EVT MemVT = LN->getMemoryVT(); 10042 unsigned Alignment = LN->getAlignment(); 10043 10044 // If this load is properly aligned, then it is legal. 10045 if (Alignment >= MemVT.getStoreSize()) 10046 return Op; 10047 10048 EVT ScalarVT = Op.getValueType().getScalarType(), 10049 ScalarMemVT = MemVT.getScalarType(); 10050 unsigned Stride = ScalarMemVT.getStoreSize(); 10051 10052 SDValue Vals[4], LoadChains[4]; 10053 for (unsigned Idx = 0; Idx < 4; ++Idx) { 10054 SDValue Load; 10055 if (ScalarVT != ScalarMemVT) 10056 Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain, 10057 BasePtr, 10058 LN->getPointerInfo().getWithOffset(Idx * Stride), 10059 ScalarMemVT, MinAlign(Alignment, Idx * Stride), 10060 LN->getMemOperand()->getFlags(), LN->getAAInfo()); 10061 else 10062 Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr, 10063 LN->getPointerInfo().getWithOffset(Idx * Stride), 10064 MinAlign(Alignment, Idx * Stride), 10065 LN->getMemOperand()->getFlags(), LN->getAAInfo()); 10066 10067 if (Idx == 0 && LN->isIndexed()) { 10068 assert(LN->getAddressingMode() == ISD::PRE_INC && 10069 "Unknown addressing mode on vector load"); 10070 Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(), 10071 LN->getAddressingMode()); 10072 } 10073 10074 Vals[Idx] = Load; 10075 LoadChains[Idx] = Load.getValue(1); 10076 10077 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 10078 DAG.getConstant(Stride, dl, 10079 BasePtr.getValueType())); 10080 } 10081 10082 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 10083 SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals); 10084 10085 if (LN->isIndexed()) { 10086 SDValue RetOps[] = { Value, Vals[0].getValue(1), TF }; 10087 return DAG.getMergeValues(RetOps, dl); 10088 } 10089 10090 SDValue RetOps[] = { Value, TF }; 10091 return DAG.getMergeValues(RetOps, dl); 10092 } 10093 10094 assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower"); 10095 assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported"); 10096 10097 // To lower v4i1 from a byte array, we load the byte elements of the 10098 // vector and then reuse the BUILD_VECTOR logic. 10099 10100 SDValue VectElmts[4], VectElmtChains[4]; 10101 for (unsigned i = 0; i < 4; ++i) { 10102 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); 10103 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); 10104 10105 VectElmts[i] = DAG.getExtLoad( 10106 ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx, 10107 LN->getPointerInfo().getWithOffset(i), MVT::i8, 10108 /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo()); 10109 VectElmtChains[i] = VectElmts[i].getValue(1); 10110 } 10111 10112 LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains); 10113 SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts); 10114 10115 SDValue RVals[] = { Value, LoadChain }; 10116 return DAG.getMergeValues(RVals, dl); 10117} 10118 10119/// Lowering for QPX v4i1 stores 10120SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, 10121 SelectionDAG &DAG) const { 10122 SDLoc dl(Op); 10123 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); 10124 SDValue StoreChain = SN->getChain(); 10125 SDValue BasePtr = SN->getBasePtr(); 10126 SDValue Value = SN->getValue(); 10127 10128 if (Value.getValueType() == MVT::v4f64 || 10129 Value.getValueType() == MVT::v4f32) { 10130 EVT MemVT = SN->getMemoryVT(); 10131 unsigned Alignment = SN->getAlignment(); 10132 10133 // If this store is properly aligned, then it is legal. 10134 if (Alignment >= MemVT.getStoreSize()) 10135 return Op; 10136 10137 EVT ScalarVT = Value.getValueType().getScalarType(), 10138 ScalarMemVT = MemVT.getScalarType(); 10139 unsigned Stride = ScalarMemVT.getStoreSize(); 10140 10141 SDValue Stores[4]; 10142 for (unsigned Idx = 0; Idx < 4; ++Idx) { 10143 SDValue Ex = DAG.getNode( 10144 ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value, 10145 DAG.getConstant(Idx, dl, getVectorIdxTy(DAG.getDataLayout()))); 10146 SDValue Store; 10147 if (ScalarVT != ScalarMemVT) 10148 Store = 10149 DAG.getTruncStore(StoreChain, dl, Ex, BasePtr, 10150 SN->getPointerInfo().getWithOffset(Idx * Stride), 10151 ScalarMemVT, MinAlign(Alignment, Idx * Stride), 10152 SN->getMemOperand()->getFlags(), SN->getAAInfo()); 10153 else 10154 Store = DAG.getStore(StoreChain, dl, Ex, BasePtr, 10155 SN->getPointerInfo().getWithOffset(Idx * Stride), 10156 MinAlign(Alignment, Idx * Stride), 10157 SN->getMemOperand()->getFlags(), SN->getAAInfo()); 10158 10159 if (Idx == 0 && SN->isIndexed()) { 10160 assert(SN->getAddressingMode() == ISD::PRE_INC && 10161 "Unknown addressing mode on vector store"); 10162 Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(), 10163 SN->getAddressingMode()); 10164 } 10165 10166 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 10167 DAG.getConstant(Stride, dl, 10168 BasePtr.getValueType())); 10169 Stores[Idx] = Store; 10170 } 10171 10172 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 10173 10174 if (SN->isIndexed()) { 10175 SDValue RetOps[] = { TF, Stores[0].getValue(1) }; 10176 return DAG.getMergeValues(RetOps, dl); 10177 } 10178 10179 return TF; 10180 } 10181 10182 assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported"); 10183 assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower"); 10184 10185 // The values are now known to be -1 (false) or 1 (true). To convert this 10186 // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). 10187 // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 10188 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); 10189 10190 // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to 10191 // understand how to form the extending load. 10192 SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); 10193 10194 Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); 10195 10196 // Now convert to an integer and store. 10197 Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, 10198 DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), 10199 Value); 10200 10201 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 10202 int FrameIdx = MFI.CreateStackObject(16, 16, false); 10203 MachinePointerInfo PtrInfo = 10204 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 10205 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 10206 SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); 10207 10208 SDValue Ops[] = {StoreChain, 10209 DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), 10210 Value, FIdx}; 10211 SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); 10212 10213 StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, 10214 dl, VTs, Ops, MVT::v4i32, PtrInfo); 10215 10216 // Move data into the byte array. 10217 SDValue Loads[4], LoadChains[4]; 10218 for (unsigned i = 0; i < 4; ++i) { 10219 unsigned Offset = 4*i; 10220 SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); 10221 Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); 10222 10223 Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, 10224 PtrInfo.getWithOffset(Offset)); 10225 LoadChains[i] = Loads[i].getValue(1); 10226 } 10227 10228 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); 10229 10230 SDValue Stores[4]; 10231 for (unsigned i = 0; i < 4; ++i) { 10232 SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); 10233 Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); 10234 10235 Stores[i] = DAG.getTruncStore( 10236 StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i), 10237 MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(), 10238 SN->getAAInfo()); 10239 } 10240 10241 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); 10242 10243 return StoreChain; 10244} 10245 10246SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { 10247 SDLoc dl(Op); 10248 if (Op.getValueType() == MVT::v4i32) { 10249 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 10250 10251 SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl); 10252 SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt. 10253 10254 SDValue RHSSwap = // = vrlw RHS, 16 10255 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl); 10256 10257 // Shrinkify inputs to v8i16. 10258 LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS); 10259 RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS); 10260 RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap); 10261 10262 // Low parts multiplied together, generating 32-bit results (we ignore the 10263 // top parts). 10264 SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh, 10265 LHS, RHS, DAG, dl, MVT::v4i32); 10266 10267 SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm, 10268 LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32); 10269 // Shift the high parts up 16 bits. 10270 HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd, 10271 Neg16, DAG, dl); 10272 return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd); 10273 } else if (Op.getValueType() == MVT::v8i16) { 10274 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 10275 10276 SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl); 10277 10278 return BuildIntrinsicOp(Intrinsic::ppc_altivec_vmladduhm, 10279 LHS, RHS, Zero, DAG, dl); 10280 } else if (Op.getValueType() == MVT::v16i8) { 10281 SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); 10282 bool isLittleEndian = Subtarget.isLittleEndian(); 10283 10284 // Multiply the even 8-bit parts, producing 16-bit sums. 10285 SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub, 10286 LHS, RHS, DAG, dl, MVT::v8i16); 10287 EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts); 10288 10289 // Multiply the odd 8-bit parts, producing 16-bit sums. 10290 SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub, 10291 LHS, RHS, DAG, dl, MVT::v8i16); 10292 OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts); 10293 10294 // Merge the results together. Because vmuleub and vmuloub are 10295 // instructions with a big-endian bias, we must reverse the 10296 // element numbering and reverse the meaning of "odd" and "even" 10297 // when generating little endian code. 10298 int Ops[16]; 10299 for (unsigned i = 0; i != 8; ++i) { 10300 if (isLittleEndian) { 10301 Ops[i*2 ] = 2*i; 10302 Ops[i*2+1] = 2*i+16; 10303 } else { 10304 Ops[i*2 ] = 2*i+1; 10305 Ops[i*2+1] = 2*i+1+16; 10306 } 10307 } 10308 if (isLittleEndian) 10309 return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops); 10310 else 10311 return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops); 10312 } else { 10313 llvm_unreachable("Unknown mul to lower!"); 10314 } 10315} 10316 10317SDValue PPCTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const { 10318 10319 assert(Op.getOpcode() == ISD::ABS && "Should only be called for ISD::ABS"); 10320 10321 EVT VT = Op.getValueType(); 10322 assert(VT.isVector() && 10323 "Only set vector abs as custom, scalar abs shouldn't reach here!"); 10324 assert((VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || 10325 VT == MVT::v16i8) && 10326 "Unexpected vector element type!"); 10327 assert((VT != MVT::v2i64 || Subtarget.hasP8Altivec()) && 10328 "Current subtarget doesn't support smax v2i64!"); 10329 10330 // For vector abs, it can be lowered to: 10331 // abs x 10332 // ==> 10333 // y = -x 10334 // smax(x, y) 10335 10336 SDLoc dl(Op); 10337 SDValue X = Op.getOperand(0); 10338 SDValue Zero = DAG.getConstant(0, dl, VT); 10339 SDValue Y = DAG.getNode(ISD::SUB, dl, VT, Zero, X); 10340 10341 // SMAX patch https://reviews.llvm.org/D47332 10342 // hasn't landed yet, so use intrinsic first here. 10343 // TODO: Should use SMAX directly once SMAX patch landed 10344 Intrinsic::ID BifID = Intrinsic::ppc_altivec_vmaxsw; 10345 if (VT == MVT::v2i64) 10346 BifID = Intrinsic::ppc_altivec_vmaxsd; 10347 else if (VT == MVT::v8i16) 10348 BifID = Intrinsic::ppc_altivec_vmaxsh; 10349 else if (VT == MVT::v16i8) 10350 BifID = Intrinsic::ppc_altivec_vmaxsb; 10351 10352 return BuildIntrinsicOp(BifID, X, Y, DAG, dl, VT); 10353} 10354 10355// Custom lowering for fpext vf32 to v2f64 10356SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { 10357 10358 assert(Op.getOpcode() == ISD::FP_EXTEND && 10359 "Should only be called for ISD::FP_EXTEND"); 10360 10361 // We only want to custom lower an extend from v2f32 to v2f64. 10362 if (Op.getValueType() != MVT::v2f64 || 10363 Op.getOperand(0).getValueType() != MVT::v2f32) 10364 return SDValue(); 10365 10366 SDLoc dl(Op); 10367 SDValue Op0 = Op.getOperand(0); 10368 10369 switch (Op0.getOpcode()) { 10370 default: 10371 return SDValue(); 10372 case ISD::EXTRACT_SUBVECTOR: { 10373 assert(Op0.getNumOperands() == 2 && 10374 isa<ConstantSDNode>(Op0->getOperand(1)) && 10375 "Node should have 2 operands with second one being a constant!"); 10376 10377 if (Op0.getOperand(0).getValueType() != MVT::v4f32) 10378 return SDValue(); 10379 10380 // Custom lower is only done for high or low doubleword. 10381 int Idx = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue(); 10382 if (Idx % 2 != 0) 10383 return SDValue(); 10384 10385 // Since input is v4f32, at this point Idx is either 0 or 2. 10386 // Shift to get the doubleword position we want. 10387 int DWord = Idx >> 1; 10388 10389 // High and low word positions are different on little endian. 10390 if (Subtarget.isLittleEndian()) 10391 DWord ^= 0x1; 10392 10393 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, 10394 Op0.getOperand(0), DAG.getConstant(DWord, dl, MVT::i32)); 10395 } 10396 case ISD::FADD: 10397 case ISD::FMUL: 10398 case ISD::FSUB: { 10399 SDValue NewLoad[2]; 10400 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) { 10401 // Ensure both input are loads. 10402 SDValue LdOp = Op0.getOperand(i); 10403 if (LdOp.getOpcode() != ISD::LOAD) 10404 return SDValue(); 10405 // Generate new load node. 10406 LoadSDNode *LD = cast<LoadSDNode>(LdOp); 10407 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()}; 10408 NewLoad[i] = DAG.getMemIntrinsicNode( 10409 PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, 10410 LD->getMemoryVT(), LD->getMemOperand()); 10411 } 10412 SDValue NewOp = 10413 DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32, NewLoad[0], 10414 NewLoad[1], Op0.getNode()->getFlags()); 10415 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewOp, 10416 DAG.getConstant(0, dl, MVT::i32)); 10417 } 10418 case ISD::LOAD: { 10419 LoadSDNode *LD = cast<LoadSDNode>(Op0); 10420 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()}; 10421 SDValue NewLd = DAG.getMemIntrinsicNode( 10422 PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, 10423 LD->getMemoryVT(), LD->getMemOperand()); 10424 return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewLd, 10425 DAG.getConstant(0, dl, MVT::i32)); 10426 } 10427 } 10428 llvm_unreachable("ERROR:Should return for all cases within swtich."); 10429} 10430 10431/// LowerOperation - Provide custom lowering hooks for some operations. 10432/// 10433SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 10434 switch (Op.getOpcode()) { 10435 default: llvm_unreachable("Wasn't expecting to be able to lower this!"); 10436 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); 10437 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); 10438 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); 10439 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); 10440 case ISD::JumpTable: return LowerJumpTable(Op, DAG); 10441 case ISD::SETCC: return LowerSETCC(Op, DAG); 10442 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); 10443 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); 10444 10445 // Variable argument lowering. 10446 case ISD::VASTART: return LowerVASTART(Op, DAG); 10447 case ISD::VAARG: return LowerVAARG(Op, DAG); 10448 case ISD::VACOPY: return LowerVACOPY(Op, DAG); 10449 10450 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG); 10451 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); 10452 case ISD::GET_DYNAMIC_AREA_OFFSET: 10453 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG); 10454 10455 // Exception handling lowering. 10456 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG); 10457 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); 10458 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); 10459 10460 case ISD::LOAD: return LowerLOAD(Op, DAG); 10461 case ISD::STORE: return LowerSTORE(Op, DAG); 10462 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); 10463 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 10464 case ISD::FP_TO_UINT: 10465 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, SDLoc(Op)); 10466 case ISD::UINT_TO_FP: 10467 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); 10468 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); 10469 10470 // Lower 64-bit shifts. 10471 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG); 10472 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); 10473 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); 10474 10475 // Vector-related lowering. 10476 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 10477 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); 10478 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 10479 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); 10480 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); 10481 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); 10482 case ISD::MUL: return LowerMUL(Op, DAG); 10483 case ISD::ABS: return LowerABS(Op, DAG); 10484 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); 10485 10486 // For counter-based loop handling. 10487 case ISD::INTRINSIC_W_CHAIN: return SDValue(); 10488 10489 case ISD::BITCAST: return LowerBITCAST(Op, DAG); 10490 10491 // Frame & Return address. 10492 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); 10493 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); 10494 10495 case ISD::INTRINSIC_VOID: 10496 return LowerINTRINSIC_VOID(Op, DAG); 10497 case ISD::SREM: 10498 case ISD::UREM: 10499 return LowerREM(Op, DAG); 10500 case ISD::BSWAP: 10501 return LowerBSWAP(Op, DAG); 10502 case ISD::ATOMIC_CMP_SWAP: 10503 return LowerATOMIC_CMP_SWAP(Op, DAG); 10504 } 10505} 10506 10507void PPCTargetLowering::ReplaceNodeResults(SDNode *N, 10508 SmallVectorImpl<SDValue>&Results, 10509 SelectionDAG &DAG) const { 10510 SDLoc dl(N); 10511 switch (N->getOpcode()) { 10512 default: 10513 llvm_unreachable("Do not know how to custom type legalize this operation!"); 10514 case ISD::READCYCLECOUNTER: { 10515 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); 10516 SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0)); 10517 10518 Results.push_back(RTB); 10519 Results.push_back(RTB.getValue(1)); 10520 Results.push_back(RTB.getValue(2)); 10521 break; 10522 } 10523 case ISD::INTRINSIC_W_CHAIN: { 10524 if (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() != 10525 Intrinsic::loop_decrement) 10526 break; 10527 10528 assert(N->getValueType(0) == MVT::i1 && 10529 "Unexpected result type for CTR decrement intrinsic"); 10530 EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), 10531 N->getValueType(0)); 10532 SDVTList VTs = DAG.getVTList(SVT, MVT::Other); 10533 SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0), 10534 N->getOperand(1)); 10535 10536 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewInt)); 10537 Results.push_back(NewInt.getValue(1)); 10538 break; 10539 } 10540 case ISD::VAARG: { 10541 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64()) 10542 return; 10543 10544 EVT VT = N->getValueType(0); 10545 10546 if (VT == MVT::i64) { 10547 SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG); 10548 10549 Results.push_back(NewNode); 10550 Results.push_back(NewNode.getValue(1)); 10551 } 10552 return; 10553 } 10554 case ISD::FP_TO_SINT: 10555 case ISD::FP_TO_UINT: 10556 // LowerFP_TO_INT() can only handle f32 and f64. 10557 if (N->getOperand(0).getValueType() == MVT::ppcf128) 10558 return; 10559 Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl)); 10560 return; 10561 case ISD::TRUNCATE: { 10562 EVT TrgVT = N->getValueType(0); 10563 EVT OpVT = N->getOperand(0).getValueType(); 10564 if (TrgVT.isVector() && 10565 isOperationCustom(N->getOpcode(), TrgVT) && 10566 OpVT.getSizeInBits() <= 128 && 10567 isPowerOf2_32(OpVT.getVectorElementType().getSizeInBits())) 10568 Results.push_back(LowerTRUNCATEVector(SDValue(N, 0), DAG)); 10569 return; 10570 } 10571 case ISD::BITCAST: 10572 // Don't handle bitcast here. 10573 return; 10574 } 10575} 10576 10577//===----------------------------------------------------------------------===// 10578// Other Lowering Code 10579//===----------------------------------------------------------------------===// 10580 10581static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) { 10582 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 10583 Function *Func = Intrinsic::getDeclaration(M, Id); 10584 return Builder.CreateCall(Func, {}); 10585} 10586 10587// The mappings for emitLeading/TrailingFence is taken from 10588// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html 10589Instruction *PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder, 10590 Instruction *Inst, 10591 AtomicOrdering Ord) const { 10592 if (Ord == AtomicOrdering::SequentiallyConsistent) 10593 return callIntrinsic(Builder, Intrinsic::ppc_sync); 10594 if (isReleaseOrStronger(Ord)) 10595 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 10596 return nullptr; 10597} 10598 10599Instruction *PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder, 10600 Instruction *Inst, 10601 AtomicOrdering Ord) const { 10602 if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord)) { 10603 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and 10604 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html 10605 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification. 10606 if (isa<LoadInst>(Inst) && Subtarget.isPPC64()) 10607 return Builder.CreateCall( 10608 Intrinsic::getDeclaration( 10609 Builder.GetInsertBlock()->getParent()->getParent(), 10610 Intrinsic::ppc_cfence, {Inst->getType()}), 10611 {Inst}); 10612 // FIXME: Can use isync for rmw operation. 10613 return callIntrinsic(Builder, Intrinsic::ppc_lwsync); 10614 } 10615 return nullptr; 10616} 10617 10618MachineBasicBlock * 10619PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB, 10620 unsigned AtomicSize, 10621 unsigned BinOpcode, 10622 unsigned CmpOpcode, 10623 unsigned CmpPred) const { 10624 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 10625 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10626 10627 auto LoadMnemonic = PPC::LDARX; 10628 auto StoreMnemonic = PPC::STDCX; 10629 switch (AtomicSize) { 10630 default: 10631 llvm_unreachable("Unexpected size of atomic entity"); 10632 case 1: 10633 LoadMnemonic = PPC::LBARX; 10634 StoreMnemonic = PPC::STBCX; 10635 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 10636 break; 10637 case 2: 10638 LoadMnemonic = PPC::LHARX; 10639 StoreMnemonic = PPC::STHCX; 10640 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); 10641 break; 10642 case 4: 10643 LoadMnemonic = PPC::LWARX; 10644 StoreMnemonic = PPC::STWCX; 10645 break; 10646 case 8: 10647 LoadMnemonic = PPC::LDARX; 10648 StoreMnemonic = PPC::STDCX; 10649 break; 10650 } 10651 10652 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 10653 MachineFunction *F = BB->getParent(); 10654 MachineFunction::iterator It = ++BB->getIterator(); 10655 10656 Register dest = MI.getOperand(0).getReg(); 10657 Register ptrA = MI.getOperand(1).getReg(); 10658 Register ptrB = MI.getOperand(2).getReg(); 10659 Register incr = MI.getOperand(3).getReg(); 10660 DebugLoc dl = MI.getDebugLoc(); 10661 10662 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 10663 MachineBasicBlock *loop2MBB = 10664 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; 10665 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 10666 F->insert(It, loopMBB); 10667 if (CmpOpcode) 10668 F->insert(It, loop2MBB); 10669 F->insert(It, exitMBB); 10670 exitMBB->splice(exitMBB->begin(), BB, 10671 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 10672 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 10673 10674 MachineRegisterInfo &RegInfo = F->getRegInfo(); 10675 Register TmpReg = (!BinOpcode) ? incr : 10676 RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass 10677 : &PPC::GPRCRegClass); 10678 10679 // thisMBB: 10680 // ... 10681 // fallthrough --> loopMBB 10682 BB->addSuccessor(loopMBB); 10683 10684 // loopMBB: 10685 // l[wd]arx dest, ptr 10686 // add r0, dest, incr 10687 // st[wd]cx. r0, ptr 10688 // bne- loopMBB 10689 // fallthrough --> exitMBB 10690 10691 // For max/min... 10692 // loopMBB: 10693 // l[wd]arx dest, ptr 10694 // cmpl?[wd] incr, dest 10695 // bgt exitMBB 10696 // loop2MBB: 10697 // st[wd]cx. dest, ptr 10698 // bne- loopMBB 10699 // fallthrough --> exitMBB 10700 10701 BB = loopMBB; 10702 BuildMI(BB, dl, TII->get(LoadMnemonic), dest) 10703 .addReg(ptrA).addReg(ptrB); 10704 if (BinOpcode) 10705 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest); 10706 if (CmpOpcode) { 10707 // Signed comparisons of byte or halfword values must be sign-extended. 10708 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) { 10709 Register ExtReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 10710 BuildMI(BB, dl, TII->get(AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH), 10711 ExtReg).addReg(dest); 10712 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 10713 .addReg(incr).addReg(ExtReg); 10714 } else 10715 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 10716 .addReg(incr).addReg(dest); 10717 10718 BuildMI(BB, dl, TII->get(PPC::BCC)) 10719 .addImm(CmpPred).addReg(PPC::CR0).addMBB(exitMBB); 10720 BB->addSuccessor(loop2MBB); 10721 BB->addSuccessor(exitMBB); 10722 BB = loop2MBB; 10723 } 10724 BuildMI(BB, dl, TII->get(StoreMnemonic)) 10725 .addReg(TmpReg).addReg(ptrA).addReg(ptrB); 10726 BuildMI(BB, dl, TII->get(PPC::BCC)) 10727 .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); 10728 BB->addSuccessor(loopMBB); 10729 BB->addSuccessor(exitMBB); 10730 10731 // exitMBB: 10732 // ... 10733 BB = exitMBB; 10734 return BB; 10735} 10736 10737MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( 10738 MachineInstr &MI, MachineBasicBlock *BB, 10739 bool is8bit, // operation 10740 unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { 10741 // If we support part-word atomic mnemonics, just use them 10742 if (Subtarget.hasPartwordAtomics()) 10743 return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode, 10744 CmpPred); 10745 10746 // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. 10747 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10748 // In 64 bit mode we have to use 64 bits for addresses, even though the 10749 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address 10750 // registers without caring whether they're 32 or 64, but here we're 10751 // doing actual arithmetic on the addresses. 10752 bool is64bit = Subtarget.isPPC64(); 10753 bool isLittleEndian = Subtarget.isLittleEndian(); 10754 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 10755 10756 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 10757 MachineFunction *F = BB->getParent(); 10758 MachineFunction::iterator It = ++BB->getIterator(); 10759 10760 Register dest = MI.getOperand(0).getReg(); 10761 Register ptrA = MI.getOperand(1).getReg(); 10762 Register ptrB = MI.getOperand(2).getReg(); 10763 Register incr = MI.getOperand(3).getReg(); 10764 DebugLoc dl = MI.getDebugLoc(); 10765 10766 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); 10767 MachineBasicBlock *loop2MBB = 10768 CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; 10769 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 10770 F->insert(It, loopMBB); 10771 if (CmpOpcode) 10772 F->insert(It, loop2MBB); 10773 F->insert(It, exitMBB); 10774 exitMBB->splice(exitMBB->begin(), BB, 10775 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 10776 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 10777 10778 MachineRegisterInfo &RegInfo = F->getRegInfo(); 10779 const TargetRegisterClass *RC = 10780 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 10781 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; 10782 10783 Register PtrReg = RegInfo.createVirtualRegister(RC); 10784 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); 10785 Register ShiftReg = 10786 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); 10787 Register Incr2Reg = RegInfo.createVirtualRegister(GPRC); 10788 Register MaskReg = RegInfo.createVirtualRegister(GPRC); 10789 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); 10790 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); 10791 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); 10792 Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC); 10793 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); 10794 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); 10795 Register Ptr1Reg; 10796 Register TmpReg = 10797 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC); 10798 10799 // thisMBB: 10800 // ... 10801 // fallthrough --> loopMBB 10802 BB->addSuccessor(loopMBB); 10803 10804 // The 4-byte load must be aligned, while a char or short may be 10805 // anywhere in the word. Hence all this nasty bookkeeping code. 10806 // add ptr1, ptrA, ptrB [copy if ptrA==0] 10807 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 10808 // xori shift, shift1, 24 [16] 10809 // rlwinm ptr, ptr1, 0, 0, 29 10810 // slw incr2, incr, shift 10811 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 10812 // slw mask, mask2, shift 10813 // loopMBB: 10814 // lwarx tmpDest, ptr 10815 // add tmp, tmpDest, incr2 10816 // andc tmp2, tmpDest, mask 10817 // and tmp3, tmp, mask 10818 // or tmp4, tmp3, tmp2 10819 // stwcx. tmp4, ptr 10820 // bne- loopMBB 10821 // fallthrough --> exitMBB 10822 // srw dest, tmpDest, shift 10823 if (ptrA != ZeroReg) { 10824 Ptr1Reg = RegInfo.createVirtualRegister(RC); 10825 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 10826 .addReg(ptrA) 10827 .addReg(ptrB); 10828 } else { 10829 Ptr1Reg = ptrB; 10830 } 10831 // We need use 32-bit subregister to avoid mismatch register class in 64-bit 10832 // mode. 10833 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) 10834 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) 10835 .addImm(3) 10836 .addImm(27) 10837 .addImm(is8bit ? 28 : 27); 10838 if (!isLittleEndian) 10839 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) 10840 .addReg(Shift1Reg) 10841 .addImm(is8bit ? 24 : 16); 10842 if (is64bit) 10843 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 10844 .addReg(Ptr1Reg) 10845 .addImm(0) 10846 .addImm(61); 10847 else 10848 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 10849 .addReg(Ptr1Reg) 10850 .addImm(0) 10851 .addImm(0) 10852 .addImm(29); 10853 BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg).addReg(incr).addReg(ShiftReg); 10854 if (is8bit) 10855 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 10856 else { 10857 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 10858 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 10859 .addReg(Mask3Reg) 10860 .addImm(65535); 10861 } 10862 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 10863 .addReg(Mask2Reg) 10864 .addReg(ShiftReg); 10865 10866 BB = loopMBB; 10867 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 10868 .addReg(ZeroReg) 10869 .addReg(PtrReg); 10870 if (BinOpcode) 10871 BuildMI(BB, dl, TII->get(BinOpcode), TmpReg) 10872 .addReg(Incr2Reg) 10873 .addReg(TmpDestReg); 10874 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) 10875 .addReg(TmpDestReg) 10876 .addReg(MaskReg); 10877 BuildMI(BB, dl, TII->get(PPC::AND), Tmp3Reg).addReg(TmpReg).addReg(MaskReg); 10878 if (CmpOpcode) { 10879 // For unsigned comparisons, we can directly compare the shifted values. 10880 // For signed comparisons we shift and sign extend. 10881 Register SReg = RegInfo.createVirtualRegister(GPRC); 10882 BuildMI(BB, dl, TII->get(PPC::AND), SReg) 10883 .addReg(TmpDestReg) 10884 .addReg(MaskReg); 10885 unsigned ValueReg = SReg; 10886 unsigned CmpReg = Incr2Reg; 10887 if (CmpOpcode == PPC::CMPW) { 10888 ValueReg = RegInfo.createVirtualRegister(GPRC); 10889 BuildMI(BB, dl, TII->get(PPC::SRW), ValueReg) 10890 .addReg(SReg) 10891 .addReg(ShiftReg); 10892 Register ValueSReg = RegInfo.createVirtualRegister(GPRC); 10893 BuildMI(BB, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueSReg) 10894 .addReg(ValueReg); 10895 ValueReg = ValueSReg; 10896 CmpReg = incr; 10897 } 10898 BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) 10899 .addReg(CmpReg) 10900 .addReg(ValueReg); 10901 BuildMI(BB, dl, TII->get(PPC::BCC)) 10902 .addImm(CmpPred) 10903 .addReg(PPC::CR0) 10904 .addMBB(exitMBB); 10905 BB->addSuccessor(loop2MBB); 10906 BB->addSuccessor(exitMBB); 10907 BB = loop2MBB; 10908 } 10909 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg).addReg(Tmp3Reg).addReg(Tmp2Reg); 10910 BuildMI(BB, dl, TII->get(PPC::STWCX)) 10911 .addReg(Tmp4Reg) 10912 .addReg(ZeroReg) 10913 .addReg(PtrReg); 10914 BuildMI(BB, dl, TII->get(PPC::BCC)) 10915 .addImm(PPC::PRED_NE) 10916 .addReg(PPC::CR0) 10917 .addMBB(loopMBB); 10918 BB->addSuccessor(loopMBB); 10919 BB->addSuccessor(exitMBB); 10920 10921 // exitMBB: 10922 // ... 10923 BB = exitMBB; 10924 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) 10925 .addReg(TmpDestReg) 10926 .addReg(ShiftReg); 10927 return BB; 10928} 10929 10930llvm::MachineBasicBlock * 10931PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI, 10932 MachineBasicBlock *MBB) const { 10933 DebugLoc DL = MI.getDebugLoc(); 10934 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 10935 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 10936 10937 MachineFunction *MF = MBB->getParent(); 10938 MachineRegisterInfo &MRI = MF->getRegInfo(); 10939 10940 const BasicBlock *BB = MBB->getBasicBlock(); 10941 MachineFunction::iterator I = ++MBB->getIterator(); 10942 10943 Register DstReg = MI.getOperand(0).getReg(); 10944 const TargetRegisterClass *RC = MRI.getRegClass(DstReg); 10945 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"); 10946 Register mainDstReg = MRI.createVirtualRegister(RC); 10947 Register restoreDstReg = MRI.createVirtualRegister(RC); 10948 10949 MVT PVT = getPointerTy(MF->getDataLayout()); 10950 assert((PVT == MVT::i64 || PVT == MVT::i32) && 10951 "Invalid Pointer Size!"); 10952 // For v = setjmp(buf), we generate 10953 // 10954 // thisMBB: 10955 // SjLjSetup mainMBB 10956 // bl mainMBB 10957 // v_restore = 1 10958 // b sinkMBB 10959 // 10960 // mainMBB: 10961 // buf[LabelOffset] = LR 10962 // v_main = 0 10963 // 10964 // sinkMBB: 10965 // v = phi(main, restore) 10966 // 10967 10968 MachineBasicBlock *thisMBB = MBB; 10969 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); 10970 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); 10971 MF->insert(I, mainMBB); 10972 MF->insert(I, sinkMBB); 10973 10974 MachineInstrBuilder MIB; 10975 10976 // Transfer the remainder of BB and its successor edges to sinkMBB. 10977 sinkMBB->splice(sinkMBB->begin(), MBB, 10978 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 10979 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); 10980 10981 // Note that the structure of the jmp_buf used here is not compatible 10982 // with that used by libc, and is not designed to be. Specifically, it 10983 // stores only those 'reserved' registers that LLVM does not otherwise 10984 // understand how to spill. Also, by convention, by the time this 10985 // intrinsic is called, Clang has already stored the frame address in the 10986 // first slot of the buffer and stack address in the third. Following the 10987 // X86 target code, we'll store the jump address in the second slot. We also 10988 // need to save the TOC pointer (R2) to handle jumps between shared 10989 // libraries, and that will be stored in the fourth slot. The thread 10990 // identifier (R13) is not affected. 10991 10992 // thisMBB: 10993 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 10994 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 10995 const int64_t BPOffset = 4 * PVT.getStoreSize(); 10996 10997 // Prepare IP either in reg. 10998 const TargetRegisterClass *PtrRC = getRegClassFor(PVT); 10999 Register LabelReg = MRI.createVirtualRegister(PtrRC); 11000 Register BufReg = MI.getOperand(1).getReg(); 11001 11002 if (Subtarget.is64BitELFABI()) { 11003 setUsesTOCBasePtr(*MBB->getParent()); 11004 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD)) 11005 .addReg(PPC::X2) 11006 .addImm(TOCOffset) 11007 .addReg(BufReg) 11008 .cloneMemRefs(MI); 11009 } 11010 11011 // Naked functions never have a base pointer, and so we use r1. For all 11012 // other functions, this decision must be delayed until during PEI. 11013 unsigned BaseReg; 11014 if (MF->getFunction().hasFnAttribute(Attribute::Naked)) 11015 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1; 11016 else 11017 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP; 11018 11019 MIB = BuildMI(*thisMBB, MI, DL, 11020 TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW)) 11021 .addReg(BaseReg) 11022 .addImm(BPOffset) 11023 .addReg(BufReg) 11024 .cloneMemRefs(MI); 11025 11026 // Setup 11027 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB); 11028 MIB.addRegMask(TRI->getNoPreservedMask()); 11029 11030 BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1); 11031 11032 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup)) 11033 .addMBB(mainMBB); 11034 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); 11035 11036 thisMBB->addSuccessor(mainMBB, BranchProbability::getZero()); 11037 thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne()); 11038 11039 // mainMBB: 11040 // mainDstReg = 0 11041 MIB = 11042 BuildMI(mainMBB, DL, 11043 TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg); 11044 11045 // Store IP 11046 if (Subtarget.isPPC64()) { 11047 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD)) 11048 .addReg(LabelReg) 11049 .addImm(LabelOffset) 11050 .addReg(BufReg); 11051 } else { 11052 MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW)) 11053 .addReg(LabelReg) 11054 .addImm(LabelOffset) 11055 .addReg(BufReg); 11056 } 11057 MIB.cloneMemRefs(MI); 11058 11059 BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0); 11060 mainMBB->addSuccessor(sinkMBB); 11061 11062 // sinkMBB: 11063 BuildMI(*sinkMBB, sinkMBB->begin(), DL, 11064 TII->get(PPC::PHI), DstReg) 11065 .addReg(mainDstReg).addMBB(mainMBB) 11066 .addReg(restoreDstReg).addMBB(thisMBB); 11067 11068 MI.eraseFromParent(); 11069 return sinkMBB; 11070} 11071 11072MachineBasicBlock * 11073PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI, 11074 MachineBasicBlock *MBB) const { 11075 DebugLoc DL = MI.getDebugLoc(); 11076 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 11077 11078 MachineFunction *MF = MBB->getParent(); 11079 MachineRegisterInfo &MRI = MF->getRegInfo(); 11080 11081 MVT PVT = getPointerTy(MF->getDataLayout()); 11082 assert((PVT == MVT::i64 || PVT == MVT::i32) && 11083 "Invalid Pointer Size!"); 11084 11085 const TargetRegisterClass *RC = 11086 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 11087 Register Tmp = MRI.createVirtualRegister(RC); 11088 // Since FP is only updated here but NOT referenced, it's treated as GPR. 11089 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31; 11090 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1; 11091 unsigned BP = 11092 (PVT == MVT::i64) 11093 ? PPC::X30 11094 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29 11095 : PPC::R30); 11096 11097 MachineInstrBuilder MIB; 11098 11099 const int64_t LabelOffset = 1 * PVT.getStoreSize(); 11100 const int64_t SPOffset = 2 * PVT.getStoreSize(); 11101 const int64_t TOCOffset = 3 * PVT.getStoreSize(); 11102 const int64_t BPOffset = 4 * PVT.getStoreSize(); 11103 11104 Register BufReg = MI.getOperand(0).getReg(); 11105 11106 // Reload FP (the jumped-to function may not have had a 11107 // frame pointer, and if so, then its r31 will be restored 11108 // as necessary). 11109 if (PVT == MVT::i64) { 11110 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP) 11111 .addImm(0) 11112 .addReg(BufReg); 11113 } else { 11114 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP) 11115 .addImm(0) 11116 .addReg(BufReg); 11117 } 11118 MIB.cloneMemRefs(MI); 11119 11120 // Reload IP 11121 if (PVT == MVT::i64) { 11122 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp) 11123 .addImm(LabelOffset) 11124 .addReg(BufReg); 11125 } else { 11126 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp) 11127 .addImm(LabelOffset) 11128 .addReg(BufReg); 11129 } 11130 MIB.cloneMemRefs(MI); 11131 11132 // Reload SP 11133 if (PVT == MVT::i64) { 11134 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP) 11135 .addImm(SPOffset) 11136 .addReg(BufReg); 11137 } else { 11138 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP) 11139 .addImm(SPOffset) 11140 .addReg(BufReg); 11141 } 11142 MIB.cloneMemRefs(MI); 11143 11144 // Reload BP 11145 if (PVT == MVT::i64) { 11146 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP) 11147 .addImm(BPOffset) 11148 .addReg(BufReg); 11149 } else { 11150 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP) 11151 .addImm(BPOffset) 11152 .addReg(BufReg); 11153 } 11154 MIB.cloneMemRefs(MI); 11155 11156 // Reload TOC 11157 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) { 11158 setUsesTOCBasePtr(*MBB->getParent()); 11159 MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2) 11160 .addImm(TOCOffset) 11161 .addReg(BufReg) 11162 .cloneMemRefs(MI); 11163 } 11164 11165 // Jump 11166 BuildMI(*MBB, MI, DL, 11167 TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp); 11168 BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR)); 11169 11170 MI.eraseFromParent(); 11171 return MBB; 11172} 11173 11174MachineBasicBlock * 11175PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, 11176 MachineBasicBlock *BB) const { 11177 if (MI.getOpcode() == TargetOpcode::STACKMAP || 11178 MI.getOpcode() == TargetOpcode::PATCHPOINT) { 11179 if (Subtarget.is64BitELFABI() && 11180 MI.getOpcode() == TargetOpcode::PATCHPOINT) { 11181 // Call lowering should have added an r2 operand to indicate a dependence 11182 // on the TOC base pointer value. It can't however, because there is no 11183 // way to mark the dependence as implicit there, and so the stackmap code 11184 // will confuse it with a regular operand. Instead, add the dependence 11185 // here. 11186 MI.addOperand(MachineOperand::CreateReg(PPC::X2, false, true)); 11187 } 11188 11189 return emitPatchPoint(MI, BB); 11190 } 11191 11192 if (MI.getOpcode() == PPC::EH_SjLj_SetJmp32 || 11193 MI.getOpcode() == PPC::EH_SjLj_SetJmp64) { 11194 return emitEHSjLjSetJmp(MI, BB); 11195 } else if (MI.getOpcode() == PPC::EH_SjLj_LongJmp32 || 11196 MI.getOpcode() == PPC::EH_SjLj_LongJmp64) { 11197 return emitEHSjLjLongJmp(MI, BB); 11198 } 11199 11200 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 11201 11202 // To "insert" these instructions we actually have to insert their 11203 // control-flow patterns. 11204 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 11205 MachineFunction::iterator It = ++BB->getIterator(); 11206 11207 MachineFunction *F = BB->getParent(); 11208 11209 if (MI.getOpcode() == PPC::SELECT_CC_I4 || 11210 MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_I4 || 11211 MI.getOpcode() == PPC::SELECT_I8) { 11212 SmallVector<MachineOperand, 2> Cond; 11213 if (MI.getOpcode() == PPC::SELECT_CC_I4 || 11214 MI.getOpcode() == PPC::SELECT_CC_I8) 11215 Cond.push_back(MI.getOperand(4)); 11216 else 11217 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); 11218 Cond.push_back(MI.getOperand(1)); 11219 11220 DebugLoc dl = MI.getDebugLoc(); 11221 TII->insertSelect(*BB, MI, dl, MI.getOperand(0).getReg(), Cond, 11222 MI.getOperand(2).getReg(), MI.getOperand(3).getReg()); 11223 } else if (MI.getOpcode() == PPC::SELECT_CC_F4 || 11224 MI.getOpcode() == PPC::SELECT_CC_F8 || 11225 MI.getOpcode() == PPC::SELECT_CC_F16 || 11226 MI.getOpcode() == PPC::SELECT_CC_QFRC || 11227 MI.getOpcode() == PPC::SELECT_CC_QSRC || 11228 MI.getOpcode() == PPC::SELECT_CC_QBRC || 11229 MI.getOpcode() == PPC::SELECT_CC_VRRC || 11230 MI.getOpcode() == PPC::SELECT_CC_VSFRC || 11231 MI.getOpcode() == PPC::SELECT_CC_VSSRC || 11232 MI.getOpcode() == PPC::SELECT_CC_VSRC || 11233 MI.getOpcode() == PPC::SELECT_CC_SPE4 || 11234 MI.getOpcode() == PPC::SELECT_CC_SPE || 11235 MI.getOpcode() == PPC::SELECT_F4 || 11236 MI.getOpcode() == PPC::SELECT_F8 || 11237 MI.getOpcode() == PPC::SELECT_F16 || 11238 MI.getOpcode() == PPC::SELECT_QFRC || 11239 MI.getOpcode() == PPC::SELECT_QSRC || 11240 MI.getOpcode() == PPC::SELECT_QBRC || 11241 MI.getOpcode() == PPC::SELECT_SPE || 11242 MI.getOpcode() == PPC::SELECT_SPE4 || 11243 MI.getOpcode() == PPC::SELECT_VRRC || 11244 MI.getOpcode() == PPC::SELECT_VSFRC || 11245 MI.getOpcode() == PPC::SELECT_VSSRC || 11246 MI.getOpcode() == PPC::SELECT_VSRC) { 11247 // The incoming instruction knows the destination vreg to set, the 11248 // condition code register to branch on, the true/false values to 11249 // select between, and a branch opcode to use. 11250 11251 // thisMBB: 11252 // ... 11253 // TrueVal = ... 11254 // cmpTY ccX, r1, r2 11255 // bCC copy1MBB 11256 // fallthrough --> copy0MBB 11257 MachineBasicBlock *thisMBB = BB; 11258 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); 11259 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 11260 DebugLoc dl = MI.getDebugLoc(); 11261 F->insert(It, copy0MBB); 11262 F->insert(It, sinkMBB); 11263 11264 // Transfer the remainder of BB and its successor edges to sinkMBB. 11265 sinkMBB->splice(sinkMBB->begin(), BB, 11266 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11267 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 11268 11269 // Next, add the true and fallthrough blocks as its successors. 11270 BB->addSuccessor(copy0MBB); 11271 BB->addSuccessor(sinkMBB); 11272 11273 if (MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8 || 11274 MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || 11275 MI.getOpcode() == PPC::SELECT_F16 || 11276 MI.getOpcode() == PPC::SELECT_SPE4 || 11277 MI.getOpcode() == PPC::SELECT_SPE || 11278 MI.getOpcode() == PPC::SELECT_QFRC || 11279 MI.getOpcode() == PPC::SELECT_QSRC || 11280 MI.getOpcode() == PPC::SELECT_QBRC || 11281 MI.getOpcode() == PPC::SELECT_VRRC || 11282 MI.getOpcode() == PPC::SELECT_VSFRC || 11283 MI.getOpcode() == PPC::SELECT_VSSRC || 11284 MI.getOpcode() == PPC::SELECT_VSRC) { 11285 BuildMI(BB, dl, TII->get(PPC::BC)) 11286 .addReg(MI.getOperand(1).getReg()) 11287 .addMBB(sinkMBB); 11288 } else { 11289 unsigned SelectPred = MI.getOperand(4).getImm(); 11290 BuildMI(BB, dl, TII->get(PPC::BCC)) 11291 .addImm(SelectPred) 11292 .addReg(MI.getOperand(1).getReg()) 11293 .addMBB(sinkMBB); 11294 } 11295 11296 // copy0MBB: 11297 // %FalseValue = ... 11298 // # fallthrough to sinkMBB 11299 BB = copy0MBB; 11300 11301 // Update machine-CFG edges 11302 BB->addSuccessor(sinkMBB); 11303 11304 // sinkMBB: 11305 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] 11306 // ... 11307 BB = sinkMBB; 11308 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::PHI), MI.getOperand(0).getReg()) 11309 .addReg(MI.getOperand(3).getReg()) 11310 .addMBB(copy0MBB) 11311 .addReg(MI.getOperand(2).getReg()) 11312 .addMBB(thisMBB); 11313 } else if (MI.getOpcode() == PPC::ReadTB) { 11314 // To read the 64-bit time-base register on a 32-bit target, we read the 11315 // two halves. Should the counter have wrapped while it was being read, we 11316 // need to try again. 11317 // ... 11318 // readLoop: 11319 // mfspr Rx,TBU # load from TBU 11320 // mfspr Ry,TB # load from TB 11321 // mfspr Rz,TBU # load from TBU 11322 // cmpw crX,Rx,Rz # check if 'old'='new' 11323 // bne readLoop # branch if they're not equal 11324 // ... 11325 11326 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB); 11327 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); 11328 DebugLoc dl = MI.getDebugLoc(); 11329 F->insert(It, readMBB); 11330 F->insert(It, sinkMBB); 11331 11332 // Transfer the remainder of BB and its successor edges to sinkMBB. 11333 sinkMBB->splice(sinkMBB->begin(), BB, 11334 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11335 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); 11336 11337 BB->addSuccessor(readMBB); 11338 BB = readMBB; 11339 11340 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11341 Register ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); 11342 Register LoReg = MI.getOperand(0).getReg(); 11343 Register HiReg = MI.getOperand(1).getReg(); 11344 11345 BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269); 11346 BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268); 11347 BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269); 11348 11349 Register CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 11350 11351 BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg) 11352 .addReg(HiReg) 11353 .addReg(ReadAgainReg); 11354 BuildMI(BB, dl, TII->get(PPC::BCC)) 11355 .addImm(PPC::PRED_NE) 11356 .addReg(CmpReg) 11357 .addMBB(readMBB); 11358 11359 BB->addSuccessor(readMBB); 11360 BB->addSuccessor(sinkMBB); 11361 } else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I8) 11362 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4); 11363 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I16) 11364 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4); 11365 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I32) 11366 BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4); 11367 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I64) 11368 BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8); 11369 11370 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I8) 11371 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND); 11372 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I16) 11373 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND); 11374 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I32) 11375 BB = EmitAtomicBinary(MI, BB, 4, PPC::AND); 11376 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I64) 11377 BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8); 11378 11379 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I8) 11380 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR); 11381 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I16) 11382 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR); 11383 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I32) 11384 BB = EmitAtomicBinary(MI, BB, 4, PPC::OR); 11385 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I64) 11386 BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8); 11387 11388 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I8) 11389 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR); 11390 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I16) 11391 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR); 11392 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I32) 11393 BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR); 11394 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I64) 11395 BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8); 11396 11397 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I8) 11398 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND); 11399 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I16) 11400 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND); 11401 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I32) 11402 BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND); 11403 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I64) 11404 BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8); 11405 11406 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I8) 11407 BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF); 11408 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I16) 11409 BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF); 11410 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I32) 11411 BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF); 11412 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I64) 11413 BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8); 11414 11415 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I8) 11416 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_GE); 11417 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I16) 11418 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_GE); 11419 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I32) 11420 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_GE); 11421 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I64) 11422 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_GE); 11423 11424 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I8) 11425 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_LE); 11426 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I16) 11427 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_LE); 11428 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I32) 11429 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_LE); 11430 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I64) 11431 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_LE); 11432 11433 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I8) 11434 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_GE); 11435 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I16) 11436 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_GE); 11437 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I32) 11438 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_GE); 11439 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I64) 11440 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_GE); 11441 11442 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I8) 11443 BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_LE); 11444 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I16) 11445 BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_LE); 11446 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I32) 11447 BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_LE); 11448 else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I64) 11449 BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_LE); 11450 11451 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I8) 11452 BB = EmitPartwordAtomicBinary(MI, BB, true, 0); 11453 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I16) 11454 BB = EmitPartwordAtomicBinary(MI, BB, false, 0); 11455 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I32) 11456 BB = EmitAtomicBinary(MI, BB, 4, 0); 11457 else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I64) 11458 BB = EmitAtomicBinary(MI, BB, 8, 0); 11459 else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 || 11460 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 || 11461 (Subtarget.hasPartwordAtomics() && 11462 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) || 11463 (Subtarget.hasPartwordAtomics() && 11464 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) { 11465 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64; 11466 11467 auto LoadMnemonic = PPC::LDARX; 11468 auto StoreMnemonic = PPC::STDCX; 11469 switch (MI.getOpcode()) { 11470 default: 11471 llvm_unreachable("Compare and swap of unknown size"); 11472 case PPC::ATOMIC_CMP_SWAP_I8: 11473 LoadMnemonic = PPC::LBARX; 11474 StoreMnemonic = PPC::STBCX; 11475 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 11476 break; 11477 case PPC::ATOMIC_CMP_SWAP_I16: 11478 LoadMnemonic = PPC::LHARX; 11479 StoreMnemonic = PPC::STHCX; 11480 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); 11481 break; 11482 case PPC::ATOMIC_CMP_SWAP_I32: 11483 LoadMnemonic = PPC::LWARX; 11484 StoreMnemonic = PPC::STWCX; 11485 break; 11486 case PPC::ATOMIC_CMP_SWAP_I64: 11487 LoadMnemonic = PPC::LDARX; 11488 StoreMnemonic = PPC::STDCX; 11489 break; 11490 } 11491 Register dest = MI.getOperand(0).getReg(); 11492 Register ptrA = MI.getOperand(1).getReg(); 11493 Register ptrB = MI.getOperand(2).getReg(); 11494 Register oldval = MI.getOperand(3).getReg(); 11495 Register newval = MI.getOperand(4).getReg(); 11496 DebugLoc dl = MI.getDebugLoc(); 11497 11498 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 11499 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 11500 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 11501 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 11502 F->insert(It, loop1MBB); 11503 F->insert(It, loop2MBB); 11504 F->insert(It, midMBB); 11505 F->insert(It, exitMBB); 11506 exitMBB->splice(exitMBB->begin(), BB, 11507 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11508 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 11509 11510 // thisMBB: 11511 // ... 11512 // fallthrough --> loopMBB 11513 BB->addSuccessor(loop1MBB); 11514 11515 // loop1MBB: 11516 // l[bhwd]arx dest, ptr 11517 // cmp[wd] dest, oldval 11518 // bne- midMBB 11519 // loop2MBB: 11520 // st[bhwd]cx. newval, ptr 11521 // bne- loopMBB 11522 // b exitBB 11523 // midMBB: 11524 // st[bhwd]cx. dest, ptr 11525 // exitBB: 11526 BB = loop1MBB; 11527 BuildMI(BB, dl, TII->get(LoadMnemonic), dest).addReg(ptrA).addReg(ptrB); 11528 BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0) 11529 .addReg(oldval) 11530 .addReg(dest); 11531 BuildMI(BB, dl, TII->get(PPC::BCC)) 11532 .addImm(PPC::PRED_NE) 11533 .addReg(PPC::CR0) 11534 .addMBB(midMBB); 11535 BB->addSuccessor(loop2MBB); 11536 BB->addSuccessor(midMBB); 11537 11538 BB = loop2MBB; 11539 BuildMI(BB, dl, TII->get(StoreMnemonic)) 11540 .addReg(newval) 11541 .addReg(ptrA) 11542 .addReg(ptrB); 11543 BuildMI(BB, dl, TII->get(PPC::BCC)) 11544 .addImm(PPC::PRED_NE) 11545 .addReg(PPC::CR0) 11546 .addMBB(loop1MBB); 11547 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 11548 BB->addSuccessor(loop1MBB); 11549 BB->addSuccessor(exitMBB); 11550 11551 BB = midMBB; 11552 BuildMI(BB, dl, TII->get(StoreMnemonic)) 11553 .addReg(dest) 11554 .addReg(ptrA) 11555 .addReg(ptrB); 11556 BB->addSuccessor(exitMBB); 11557 11558 // exitMBB: 11559 // ... 11560 BB = exitMBB; 11561 } else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 || 11562 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) { 11563 // We must use 64-bit registers for addresses when targeting 64-bit, 11564 // since we're actually doing arithmetic on them. Other registers 11565 // can be 32-bit. 11566 bool is64bit = Subtarget.isPPC64(); 11567 bool isLittleEndian = Subtarget.isLittleEndian(); 11568 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8; 11569 11570 Register dest = MI.getOperand(0).getReg(); 11571 Register ptrA = MI.getOperand(1).getReg(); 11572 Register ptrB = MI.getOperand(2).getReg(); 11573 Register oldval = MI.getOperand(3).getReg(); 11574 Register newval = MI.getOperand(4).getReg(); 11575 DebugLoc dl = MI.getDebugLoc(); 11576 11577 MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); 11578 MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); 11579 MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); 11580 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); 11581 F->insert(It, loop1MBB); 11582 F->insert(It, loop2MBB); 11583 F->insert(It, midMBB); 11584 F->insert(It, exitMBB); 11585 exitMBB->splice(exitMBB->begin(), BB, 11586 std::next(MachineBasicBlock::iterator(MI)), BB->end()); 11587 exitMBB->transferSuccessorsAndUpdatePHIs(BB); 11588 11589 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11590 const TargetRegisterClass *RC = 11591 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; 11592 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; 11593 11594 Register PtrReg = RegInfo.createVirtualRegister(RC); 11595 Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); 11596 Register ShiftReg = 11597 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); 11598 Register NewVal2Reg = RegInfo.createVirtualRegister(GPRC); 11599 Register NewVal3Reg = RegInfo.createVirtualRegister(GPRC); 11600 Register OldVal2Reg = RegInfo.createVirtualRegister(GPRC); 11601 Register OldVal3Reg = RegInfo.createVirtualRegister(GPRC); 11602 Register MaskReg = RegInfo.createVirtualRegister(GPRC); 11603 Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); 11604 Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); 11605 Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); 11606 Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); 11607 Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); 11608 Register Ptr1Reg; 11609 Register TmpReg = RegInfo.createVirtualRegister(GPRC); 11610 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; 11611 // thisMBB: 11612 // ... 11613 // fallthrough --> loopMBB 11614 BB->addSuccessor(loop1MBB); 11615 11616 // The 4-byte load must be aligned, while a char or short may be 11617 // anywhere in the word. Hence all this nasty bookkeeping code. 11618 // add ptr1, ptrA, ptrB [copy if ptrA==0] 11619 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] 11620 // xori shift, shift1, 24 [16] 11621 // rlwinm ptr, ptr1, 0, 0, 29 11622 // slw newval2, newval, shift 11623 // slw oldval2, oldval,shift 11624 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] 11625 // slw mask, mask2, shift 11626 // and newval3, newval2, mask 11627 // and oldval3, oldval2, mask 11628 // loop1MBB: 11629 // lwarx tmpDest, ptr 11630 // and tmp, tmpDest, mask 11631 // cmpw tmp, oldval3 11632 // bne- midMBB 11633 // loop2MBB: 11634 // andc tmp2, tmpDest, mask 11635 // or tmp4, tmp2, newval3 11636 // stwcx. tmp4, ptr 11637 // bne- loop1MBB 11638 // b exitBB 11639 // midMBB: 11640 // stwcx. tmpDest, ptr 11641 // exitBB: 11642 // srw dest, tmpDest, shift 11643 if (ptrA != ZeroReg) { 11644 Ptr1Reg = RegInfo.createVirtualRegister(RC); 11645 BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) 11646 .addReg(ptrA) 11647 .addReg(ptrB); 11648 } else { 11649 Ptr1Reg = ptrB; 11650 } 11651 11652 // We need use 32-bit subregister to avoid mismatch register class in 64-bit 11653 // mode. 11654 BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) 11655 .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) 11656 .addImm(3) 11657 .addImm(27) 11658 .addImm(is8bit ? 28 : 27); 11659 if (!isLittleEndian) 11660 BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) 11661 .addReg(Shift1Reg) 11662 .addImm(is8bit ? 24 : 16); 11663 if (is64bit) 11664 BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) 11665 .addReg(Ptr1Reg) 11666 .addImm(0) 11667 .addImm(61); 11668 else 11669 BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) 11670 .addReg(Ptr1Reg) 11671 .addImm(0) 11672 .addImm(0) 11673 .addImm(29); 11674 BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg) 11675 .addReg(newval) 11676 .addReg(ShiftReg); 11677 BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg) 11678 .addReg(oldval) 11679 .addReg(ShiftReg); 11680 if (is8bit) 11681 BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); 11682 else { 11683 BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); 11684 BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) 11685 .addReg(Mask3Reg) 11686 .addImm(65535); 11687 } 11688 BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) 11689 .addReg(Mask2Reg) 11690 .addReg(ShiftReg); 11691 BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg) 11692 .addReg(NewVal2Reg) 11693 .addReg(MaskReg); 11694 BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg) 11695 .addReg(OldVal2Reg) 11696 .addReg(MaskReg); 11697 11698 BB = loop1MBB; 11699 BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) 11700 .addReg(ZeroReg) 11701 .addReg(PtrReg); 11702 BuildMI(BB, dl, TII->get(PPC::AND), TmpReg) 11703 .addReg(TmpDestReg) 11704 .addReg(MaskReg); 11705 BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0) 11706 .addReg(TmpReg) 11707 .addReg(OldVal3Reg); 11708 BuildMI(BB, dl, TII->get(PPC::BCC)) 11709 .addImm(PPC::PRED_NE) 11710 .addReg(PPC::CR0) 11711 .addMBB(midMBB); 11712 BB->addSuccessor(loop2MBB); 11713 BB->addSuccessor(midMBB); 11714 11715 BB = loop2MBB; 11716 BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) 11717 .addReg(TmpDestReg) 11718 .addReg(MaskReg); 11719 BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg) 11720 .addReg(Tmp2Reg) 11721 .addReg(NewVal3Reg); 11722 BuildMI(BB, dl, TII->get(PPC::STWCX)) 11723 .addReg(Tmp4Reg) 11724 .addReg(ZeroReg) 11725 .addReg(PtrReg); 11726 BuildMI(BB, dl, TII->get(PPC::BCC)) 11727 .addImm(PPC::PRED_NE) 11728 .addReg(PPC::CR0) 11729 .addMBB(loop1MBB); 11730 BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); 11731 BB->addSuccessor(loop1MBB); 11732 BB->addSuccessor(exitMBB); 11733 11734 BB = midMBB; 11735 BuildMI(BB, dl, TII->get(PPC::STWCX)) 11736 .addReg(TmpDestReg) 11737 .addReg(ZeroReg) 11738 .addReg(PtrReg); 11739 BB->addSuccessor(exitMBB); 11740 11741 // exitMBB: 11742 // ... 11743 BB = exitMBB; 11744 BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) 11745 .addReg(TmpReg) 11746 .addReg(ShiftReg); 11747 } else if (MI.getOpcode() == PPC::FADDrtz) { 11748 // This pseudo performs an FADD with rounding mode temporarily forced 11749 // to round-to-zero. We emit this via custom inserter since the FPSCR 11750 // is not modeled at the SelectionDAG level. 11751 Register Dest = MI.getOperand(0).getReg(); 11752 Register Src1 = MI.getOperand(1).getReg(); 11753 Register Src2 = MI.getOperand(2).getReg(); 11754 DebugLoc dl = MI.getDebugLoc(); 11755 11756 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11757 Register MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 11758 11759 // Save FPSCR value. 11760 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); 11761 11762 // Set rounding mode to round-to-zero. 11763 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31); 11764 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30); 11765 11766 // Perform addition. 11767 BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2); 11768 11769 // Restore FPSCR value. 11770 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg); 11771 } else if (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT || 11772 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT || 11773 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 || 11774 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8) { 11775 unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 || 11776 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8) 11777 ? PPC::ANDI8_rec 11778 : PPC::ANDI_rec; 11779 bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT || 11780 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8); 11781 11782 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11783 Register Dest = RegInfo.createVirtualRegister( 11784 Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass); 11785 11786 DebugLoc Dl = MI.getDebugLoc(); 11787 BuildMI(*BB, MI, Dl, TII->get(Opcode), Dest) 11788 .addReg(MI.getOperand(1).getReg()) 11789 .addImm(1); 11790 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 11791 MI.getOperand(0).getReg()) 11792 .addReg(IsEQ ? PPC::CR0EQ : PPC::CR0GT); 11793 } else if (MI.getOpcode() == PPC::TCHECK_RET) { 11794 DebugLoc Dl = MI.getDebugLoc(); 11795 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11796 Register CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); 11797 BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg); 11798 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 11799 MI.getOperand(0).getReg()) 11800 .addReg(CRReg); 11801 } else if (MI.getOpcode() == PPC::TBEGIN_RET) { 11802 DebugLoc Dl = MI.getDebugLoc(); 11803 unsigned Imm = MI.getOperand(1).getImm(); 11804 BuildMI(*BB, MI, Dl, TII->get(PPC::TBEGIN)).addImm(Imm); 11805 BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), 11806 MI.getOperand(0).getReg()) 11807 .addReg(PPC::CR0EQ); 11808 } else if (MI.getOpcode() == PPC::SETRNDi) { 11809 DebugLoc dl = MI.getDebugLoc(); 11810 Register OldFPSCRReg = MI.getOperand(0).getReg(); 11811 11812 // Save FPSCR value. 11813 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); 11814 11815 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has 11816 // the following settings: 11817 // 00 Round to nearest 11818 // 01 Round to 0 11819 // 10 Round to +inf 11820 // 11 Round to -inf 11821 11822 // When the operand is immediate, using the two least significant bits of 11823 // the immediate to set the bits 62:63 of FPSCR. 11824 unsigned Mode = MI.getOperand(1).getImm(); 11825 BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0)) 11826 .addImm(31); 11827 11828 BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0)) 11829 .addImm(30); 11830 } else if (MI.getOpcode() == PPC::SETRND) { 11831 DebugLoc dl = MI.getDebugLoc(); 11832 11833 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg 11834 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg. 11835 // If the target doesn't have DirectMove, we should use stack to do the 11836 // conversion, because the target doesn't have the instructions like mtvsrd 11837 // or mfvsrd to do this conversion directly. 11838 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) { 11839 if (Subtarget.hasDirectMove()) { 11840 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), DestReg) 11841 .addReg(SrcReg); 11842 } else { 11843 // Use stack to do the register copy. 11844 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD; 11845 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11846 const TargetRegisterClass *RC = RegInfo.getRegClass(SrcReg); 11847 if (RC == &PPC::F8RCRegClass) { 11848 // Copy register from F8RCRegClass to G8RCRegclass. 11849 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) && 11850 "Unsupported RegClass."); 11851 11852 StoreOp = PPC::STFD; 11853 LoadOp = PPC::LD; 11854 } else { 11855 // Copy register from G8RCRegClass to F8RCRegclass. 11856 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && 11857 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && 11858 "Unsupported RegClass."); 11859 } 11860 11861 MachineFrameInfo &MFI = F->getFrameInfo(); 11862 int FrameIdx = MFI.CreateStackObject(8, 8, false); 11863 11864 MachineMemOperand *MMOStore = F->getMachineMemOperand( 11865 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), 11866 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx), 11867 MFI.getObjectAlignment(FrameIdx)); 11868 11869 // Store the SrcReg into the stack. 11870 BuildMI(*BB, MI, dl, TII->get(StoreOp)) 11871 .addReg(SrcReg) 11872 .addImm(0) 11873 .addFrameIndex(FrameIdx) 11874 .addMemOperand(MMOStore); 11875 11876 MachineMemOperand *MMOLoad = F->getMachineMemOperand( 11877 MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), 11878 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx), 11879 MFI.getObjectAlignment(FrameIdx)); 11880 11881 // Load from the stack where SrcReg is stored, and save to DestReg, 11882 // so we have done the RegClass conversion from RegClass::SrcReg to 11883 // RegClass::DestReg. 11884 BuildMI(*BB, MI, dl, TII->get(LoadOp), DestReg) 11885 .addImm(0) 11886 .addFrameIndex(FrameIdx) 11887 .addMemOperand(MMOLoad); 11888 } 11889 }; 11890 11891 Register OldFPSCRReg = MI.getOperand(0).getReg(); 11892 11893 // Save FPSCR value. 11894 BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); 11895 11896 // When the operand is gprc register, use two least significant bits of the 11897 // register and mtfsf instruction to set the bits 62:63 of FPSCR. 11898 // 11899 // copy OldFPSCRTmpReg, OldFPSCRReg 11900 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1) 11901 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62 11902 // copy NewFPSCRReg, NewFPSCRTmpReg 11903 // mtfsf 255, NewFPSCRReg 11904 MachineOperand SrcOp = MI.getOperand(1); 11905 MachineRegisterInfo &RegInfo = F->getRegInfo(); 11906 Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11907 11908 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg); 11909 11910 Register ImDefReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11911 Register ExtSrcReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11912 11913 // The first operand of INSERT_SUBREG should be a register which has 11914 // subregisters, we only care about its RegClass, so we should use an 11915 // IMPLICIT_DEF register. 11916 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::IMPLICIT_DEF), ImDefReg); 11917 BuildMI(*BB, MI, dl, TII->get(PPC::INSERT_SUBREG), ExtSrcReg) 11918 .addReg(ImDefReg) 11919 .add(SrcOp) 11920 .addImm(1); 11921 11922 Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); 11923 BuildMI(*BB, MI, dl, TII->get(PPC::RLDIMI), NewFPSCRTmpReg) 11924 .addReg(OldFPSCRTmpReg) 11925 .addReg(ExtSrcReg) 11926 .addImm(0) 11927 .addImm(62); 11928 11929 Register NewFPSCRReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); 11930 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg); 11931 11932 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63 11933 // bits of FPSCR. 11934 BuildMI(*BB, MI, dl, TII->get(PPC::MTFSF)) 11935 .addImm(255) 11936 .addReg(NewFPSCRReg) 11937 .addImm(0) 11938 .addImm(0); 11939 } else { 11940 llvm_unreachable("Unexpected instr type to insert"); 11941 } 11942 11943 MI.eraseFromParent(); // The pseudo instruction is gone now. 11944 return BB; 11945} 11946 11947//===----------------------------------------------------------------------===// 11948// Target Optimization Hooks 11949//===----------------------------------------------------------------------===// 11950 11951static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) { 11952 // For the estimates, convergence is quadratic, so we essentially double the 11953 // number of digits correct after every iteration. For both FRE and FRSQRTE, 11954 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(), 11955 // this is 2^-14. IEEE float has 23 digits and double has 52 digits. 11956 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3; 11957 if (VT.getScalarType() == MVT::f64) 11958 RefinementSteps++; 11959 return RefinementSteps; 11960} 11961 11962SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, 11963 int Enabled, int &RefinementSteps, 11964 bool &UseOneConstNR, 11965 bool Reciprocal) const { 11966 EVT VT = Operand.getValueType(); 11967 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || 11968 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || 11969 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 11970 (VT == MVT::v2f64 && Subtarget.hasVSX()) || 11971 (VT == MVT::v4f32 && Subtarget.hasQPX()) || 11972 (VT == MVT::v4f64 && Subtarget.hasQPX())) { 11973 if (RefinementSteps == ReciprocalEstimate::Unspecified) 11974 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); 11975 11976 // The Newton-Raphson computation with a single constant does not provide 11977 // enough accuracy on some CPUs. 11978 UseOneConstNR = !Subtarget.needsTwoConstNR(); 11979 return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand); 11980 } 11981 return SDValue(); 11982} 11983 11984SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG, 11985 int Enabled, 11986 int &RefinementSteps) const { 11987 EVT VT = Operand.getValueType(); 11988 if ((VT == MVT::f32 && Subtarget.hasFRES()) || 11989 (VT == MVT::f64 && Subtarget.hasFRE()) || 11990 (VT == MVT::v4f32 && Subtarget.hasAltivec()) || 11991 (VT == MVT::v2f64 && Subtarget.hasVSX()) || 11992 (VT == MVT::v4f32 && Subtarget.hasQPX()) || 11993 (VT == MVT::v4f64 && Subtarget.hasQPX())) { 11994 if (RefinementSteps == ReciprocalEstimate::Unspecified) 11995 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); 11996 return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand); 11997 } 11998 return SDValue(); 11999} 12000 12001unsigned PPCTargetLowering::combineRepeatedFPDivisors() const { 12002 // Note: This functionality is used only when unsafe-fp-math is enabled, and 12003 // on cores with reciprocal estimates (which are used when unsafe-fp-math is 12004 // enabled for division), this functionality is redundant with the default 12005 // combiner logic (once the division -> reciprocal/multiply transformation 12006 // has taken place). As a result, this matters more for older cores than for 12007 // newer ones. 12008 12009 // Combine multiple FDIVs with the same divisor into multiple FMULs by the 12010 // reciprocal if there are two or more FDIVs (for embedded cores with only 12011 // one FP pipeline) for three or more FDIVs (for generic OOO cores). 12012 switch (Subtarget.getCPUDirective()) { 12013 default: 12014 return 3; 12015 case PPC::DIR_440: 12016 case PPC::DIR_A2: 12017 case PPC::DIR_E500: 12018 case PPC::DIR_E500mc: 12019 case PPC::DIR_E5500: 12020 return 2; 12021 } 12022} 12023 12024// isConsecutiveLSLoc needs to work even if all adds have not yet been 12025// collapsed, and so we need to look through chains of them. 12026static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base, 12027 int64_t& Offset, SelectionDAG &DAG) { 12028 if (DAG.isBaseWithConstantOffset(Loc)) { 12029 Base = Loc.getOperand(0); 12030 Offset += cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue(); 12031 12032 // The base might itself be a base plus an offset, and if so, accumulate 12033 // that as well. 12034 getBaseWithConstantOffset(Loc.getOperand(0), Base, Offset, DAG); 12035 } 12036} 12037 12038static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base, 12039 unsigned Bytes, int Dist, 12040 SelectionDAG &DAG) { 12041 if (VT.getSizeInBits() / 8 != Bytes) 12042 return false; 12043 12044 SDValue BaseLoc = Base->getBasePtr(); 12045 if (Loc.getOpcode() == ISD::FrameIndex) { 12046 if (BaseLoc.getOpcode() != ISD::FrameIndex) 12047 return false; 12048 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 12049 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 12050 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 12051 int FS = MFI.getObjectSize(FI); 12052 int BFS = MFI.getObjectSize(BFI); 12053 if (FS != BFS || FS != (int)Bytes) return false; 12054 return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes); 12055 } 12056 12057 SDValue Base1 = Loc, Base2 = BaseLoc; 12058 int64_t Offset1 = 0, Offset2 = 0; 12059 getBaseWithConstantOffset(Loc, Base1, Offset1, DAG); 12060 getBaseWithConstantOffset(BaseLoc, Base2, Offset2, DAG); 12061 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes)) 12062 return true; 12063 12064 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 12065 const GlobalValue *GV1 = nullptr; 12066 const GlobalValue *GV2 = nullptr; 12067 Offset1 = 0; 12068 Offset2 = 0; 12069 bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); 12070 bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 12071 if (isGA1 && isGA2 && GV1 == GV2) 12072 return Offset1 == (Offset2 + Dist*Bytes); 12073 return false; 12074} 12075 12076// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does 12077// not enforce equality of the chain operands. 12078static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, 12079 unsigned Bytes, int Dist, 12080 SelectionDAG &DAG) { 12081 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(N)) { 12082 EVT VT = LS->getMemoryVT(); 12083 SDValue Loc = LS->getBasePtr(); 12084 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG); 12085 } 12086 12087 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { 12088 EVT VT; 12089 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 12090 default: return false; 12091 case Intrinsic::ppc_qpx_qvlfd: 12092 case Intrinsic::ppc_qpx_qvlfda: 12093 VT = MVT::v4f64; 12094 break; 12095 case Intrinsic::ppc_qpx_qvlfs: 12096 case Intrinsic::ppc_qpx_qvlfsa: 12097 VT = MVT::v4f32; 12098 break; 12099 case Intrinsic::ppc_qpx_qvlfcd: 12100 case Intrinsic::ppc_qpx_qvlfcda: 12101 VT = MVT::v2f64; 12102 break; 12103 case Intrinsic::ppc_qpx_qvlfcs: 12104 case Intrinsic::ppc_qpx_qvlfcsa: 12105 VT = MVT::v2f32; 12106 break; 12107 case Intrinsic::ppc_qpx_qvlfiwa: 12108 case Intrinsic::ppc_qpx_qvlfiwz: 12109 case Intrinsic::ppc_altivec_lvx: 12110 case Intrinsic::ppc_altivec_lvxl: 12111 case Intrinsic::ppc_vsx_lxvw4x: 12112 case Intrinsic::ppc_vsx_lxvw4x_be: 12113 VT = MVT::v4i32; 12114 break; 12115 case Intrinsic::ppc_vsx_lxvd2x: 12116 case Intrinsic::ppc_vsx_lxvd2x_be: 12117 VT = MVT::v2f64; 12118 break; 12119 case Intrinsic::ppc_altivec_lvebx: 12120 VT = MVT::i8; 12121 break; 12122 case Intrinsic::ppc_altivec_lvehx: 12123 VT = MVT::i16; 12124 break; 12125 case Intrinsic::ppc_altivec_lvewx: 12126 VT = MVT::i32; 12127 break; 12128 } 12129 12130 return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG); 12131 } 12132 12133 if (N->getOpcode() == ISD::INTRINSIC_VOID) { 12134 EVT VT; 12135 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 12136 default: return false; 12137 case Intrinsic::ppc_qpx_qvstfd: 12138 case Intrinsic::ppc_qpx_qvstfda: 12139 VT = MVT::v4f64; 12140 break; 12141 case Intrinsic::ppc_qpx_qvstfs: 12142 case Intrinsic::ppc_qpx_qvstfsa: 12143 VT = MVT::v4f32; 12144 break; 12145 case Intrinsic::ppc_qpx_qvstfcd: 12146 case Intrinsic::ppc_qpx_qvstfcda: 12147 VT = MVT::v2f64; 12148 break; 12149 case Intrinsic::ppc_qpx_qvstfcs: 12150 case Intrinsic::ppc_qpx_qvstfcsa: 12151 VT = MVT::v2f32; 12152 break; 12153 case Intrinsic::ppc_qpx_qvstfiw: 12154 case Intrinsic::ppc_qpx_qvstfiwa: 12155 case Intrinsic::ppc_altivec_stvx: 12156 case Intrinsic::ppc_altivec_stvxl: 12157 case Intrinsic::ppc_vsx_stxvw4x: 12158 VT = MVT::v4i32; 12159 break; 12160 case Intrinsic::ppc_vsx_stxvd2x: 12161 VT = MVT::v2f64; 12162 break; 12163 case Intrinsic::ppc_vsx_stxvw4x_be: 12164 VT = MVT::v4i32; 12165 break; 12166 case Intrinsic::ppc_vsx_stxvd2x_be: 12167 VT = MVT::v2f64; 12168 break; 12169 case Intrinsic::ppc_altivec_stvebx: 12170 VT = MVT::i8; 12171 break; 12172 case Intrinsic::ppc_altivec_stvehx: 12173 VT = MVT::i16; 12174 break; 12175 case Intrinsic::ppc_altivec_stvewx: 12176 VT = MVT::i32; 12177 break; 12178 } 12179 12180 return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG); 12181 } 12182 12183 return false; 12184} 12185 12186// Return true is there is a nearyby consecutive load to the one provided 12187// (regardless of alignment). We search up and down the chain, looking though 12188// token factors and other loads (but nothing else). As a result, a true result 12189// indicates that it is safe to create a new consecutive load adjacent to the 12190// load provided. 12191static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) { 12192 SDValue Chain = LD->getChain(); 12193 EVT VT = LD->getMemoryVT(); 12194 12195 SmallSet<SDNode *, 16> LoadRoots; 12196 SmallVector<SDNode *, 8> Queue(1, Chain.getNode()); 12197 SmallSet<SDNode *, 16> Visited; 12198 12199 // First, search up the chain, branching to follow all token-factor operands. 12200 // If we find a consecutive load, then we're done, otherwise, record all 12201 // nodes just above the top-level loads and token factors. 12202 while (!Queue.empty()) { 12203 SDNode *ChainNext = Queue.pop_back_val(); 12204 if (!Visited.insert(ChainNext).second) 12205 continue; 12206 12207 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(ChainNext)) { 12208 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 12209 return true; 12210 12211 if (!Visited.count(ChainLD->getChain().getNode())) 12212 Queue.push_back(ChainLD->getChain().getNode()); 12213 } else if (ChainNext->getOpcode() == ISD::TokenFactor) { 12214 for (const SDUse &O : ChainNext->ops()) 12215 if (!Visited.count(O.getNode())) 12216 Queue.push_back(O.getNode()); 12217 } else 12218 LoadRoots.insert(ChainNext); 12219 } 12220 12221 // Second, search down the chain, starting from the top-level nodes recorded 12222 // in the first phase. These top-level nodes are the nodes just above all 12223 // loads and token factors. Starting with their uses, recursively look though 12224 // all loads (just the chain uses) and token factors to find a consecutive 12225 // load. 12226 Visited.clear(); 12227 Queue.clear(); 12228 12229 for (SmallSet<SDNode *, 16>::iterator I = LoadRoots.begin(), 12230 IE = LoadRoots.end(); I != IE; ++I) { 12231 Queue.push_back(*I); 12232 12233 while (!Queue.empty()) { 12234 SDNode *LoadRoot = Queue.pop_back_val(); 12235 if (!Visited.insert(LoadRoot).second) 12236 continue; 12237 12238 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(LoadRoot)) 12239 if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) 12240 return true; 12241 12242 for (SDNode::use_iterator UI = LoadRoot->use_begin(), 12243 UE = LoadRoot->use_end(); UI != UE; ++UI) 12244 if (((isa<MemSDNode>(*UI) && 12245 cast<MemSDNode>(*UI)->getChain().getNode() == LoadRoot) || 12246 UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI)) 12247 Queue.push_back(*UI); 12248 } 12249 } 12250 12251 return false; 12252} 12253 12254/// This function is called when we have proved that a SETCC node can be replaced 12255/// by subtraction (and other supporting instructions) so that the result of 12256/// comparison is kept in a GPR instead of CR. This function is purely for 12257/// codegen purposes and has some flags to guide the codegen process. 12258static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement, 12259 bool Swap, SDLoc &DL, SelectionDAG &DAG) { 12260 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); 12261 12262 // Zero extend the operands to the largest legal integer. Originally, they 12263 // must be of a strictly smaller size. 12264 auto Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(0), 12265 DAG.getConstant(Size, DL, MVT::i32)); 12266 auto Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1), 12267 DAG.getConstant(Size, DL, MVT::i32)); 12268 12269 // Swap if needed. Depends on the condition code. 12270 if (Swap) 12271 std::swap(Op0, Op1); 12272 12273 // Subtract extended integers. 12274 auto SubNode = DAG.getNode(ISD::SUB, DL, MVT::i64, Op0, Op1); 12275 12276 // Move the sign bit to the least significant position and zero out the rest. 12277 // Now the least significant bit carries the result of original comparison. 12278 auto Shifted = DAG.getNode(ISD::SRL, DL, MVT::i64, SubNode, 12279 DAG.getConstant(Size - 1, DL, MVT::i32)); 12280 auto Final = Shifted; 12281 12282 // Complement the result if needed. Based on the condition code. 12283 if (Complement) 12284 Final = DAG.getNode(ISD::XOR, DL, MVT::i64, Shifted, 12285 DAG.getConstant(1, DL, MVT::i64)); 12286 12287 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Final); 12288} 12289 12290SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N, 12291 DAGCombinerInfo &DCI) const { 12292 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); 12293 12294 SelectionDAG &DAG = DCI.DAG; 12295 SDLoc DL(N); 12296 12297 // Size of integers being compared has a critical role in the following 12298 // analysis, so we prefer to do this when all types are legal. 12299 if (!DCI.isAfterLegalizeDAG()) 12300 return SDValue(); 12301 12302 // If all users of SETCC extend its value to a legal integer type 12303 // then we replace SETCC with a subtraction 12304 for (SDNode::use_iterator UI = N->use_begin(), 12305 UE = N->use_end(); UI != UE; ++UI) { 12306 if (UI->getOpcode() != ISD::ZERO_EXTEND) 12307 return SDValue(); 12308 } 12309 12310 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 12311 auto OpSize = N->getOperand(0).getValueSizeInBits(); 12312 12313 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits(); 12314 12315 if (OpSize < Size) { 12316 switch (CC) { 12317 default: break; 12318 case ISD::SETULT: 12319 return generateEquivalentSub(N, Size, false, false, DL, DAG); 12320 case ISD::SETULE: 12321 return generateEquivalentSub(N, Size, true, true, DL, DAG); 12322 case ISD::SETUGT: 12323 return generateEquivalentSub(N, Size, false, true, DL, DAG); 12324 case ISD::SETUGE: 12325 return generateEquivalentSub(N, Size, true, false, DL, DAG); 12326 } 12327 } 12328 12329 return SDValue(); 12330} 12331 12332SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, 12333 DAGCombinerInfo &DCI) const { 12334 SelectionDAG &DAG = DCI.DAG; 12335 SDLoc dl(N); 12336 12337 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits"); 12338 // If we're tracking CR bits, we need to be careful that we don't have: 12339 // trunc(binary-ops(zext(x), zext(y))) 12340 // or 12341 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...) 12342 // such that we're unnecessarily moving things into GPRs when it would be 12343 // better to keep them in CR bits. 12344 12345 // Note that trunc here can be an actual i1 trunc, or can be the effective 12346 // truncation that comes from a setcc or select_cc. 12347 if (N->getOpcode() == ISD::TRUNCATE && 12348 N->getValueType(0) != MVT::i1) 12349 return SDValue(); 12350 12351 if (N->getOperand(0).getValueType() != MVT::i32 && 12352 N->getOperand(0).getValueType() != MVT::i64) 12353 return SDValue(); 12354 12355 if (N->getOpcode() == ISD::SETCC || 12356 N->getOpcode() == ISD::SELECT_CC) { 12357 // If we're looking at a comparison, then we need to make sure that the 12358 // high bits (all except for the first) don't matter the result. 12359 ISD::CondCode CC = 12360 cast<CondCodeSDNode>(N->getOperand( 12361 N->getOpcode() == ISD::SETCC ? 2 : 4))->get(); 12362 unsigned OpBits = N->getOperand(0).getValueSizeInBits(); 12363 12364 if (ISD::isSignedIntSetCC(CC)) { 12365 if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits || 12366 DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits) 12367 return SDValue(); 12368 } else if (ISD::isUnsignedIntSetCC(CC)) { 12369 if (!DAG.MaskedValueIsZero(N->getOperand(0), 12370 APInt::getHighBitsSet(OpBits, OpBits-1)) || 12371 !DAG.MaskedValueIsZero(N->getOperand(1), 12372 APInt::getHighBitsSet(OpBits, OpBits-1))) 12373 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI) 12374 : SDValue()); 12375 } else { 12376 // This is neither a signed nor an unsigned comparison, just make sure 12377 // that the high bits are equal. 12378 KnownBits Op1Known = DAG.computeKnownBits(N->getOperand(0)); 12379 KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1)); 12380 12381 // We don't really care about what is known about the first bit (if 12382 // anything), so clear it in all masks prior to comparing them. 12383 Op1Known.Zero.clearBit(0); Op1Known.One.clearBit(0); 12384 Op2Known.Zero.clearBit(0); Op2Known.One.clearBit(0); 12385 12386 if (Op1Known.Zero != Op2Known.Zero || Op1Known.One != Op2Known.One) 12387 return SDValue(); 12388 } 12389 } 12390 12391 // We now know that the higher-order bits are irrelevant, we just need to 12392 // make sure that all of the intermediate operations are bit operations, and 12393 // all inputs are extensions. 12394 if (N->getOperand(0).getOpcode() != ISD::AND && 12395 N->getOperand(0).getOpcode() != ISD::OR && 12396 N->getOperand(0).getOpcode() != ISD::XOR && 12397 N->getOperand(0).getOpcode() != ISD::SELECT && 12398 N->getOperand(0).getOpcode() != ISD::SELECT_CC && 12399 N->getOperand(0).getOpcode() != ISD::TRUNCATE && 12400 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND && 12401 N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && 12402 N->getOperand(0).getOpcode() != ISD::ANY_EXTEND) 12403 return SDValue(); 12404 12405 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) && 12406 N->getOperand(1).getOpcode() != ISD::AND && 12407 N->getOperand(1).getOpcode() != ISD::OR && 12408 N->getOperand(1).getOpcode() != ISD::XOR && 12409 N->getOperand(1).getOpcode() != ISD::SELECT && 12410 N->getOperand(1).getOpcode() != ISD::SELECT_CC && 12411 N->getOperand(1).getOpcode() != ISD::TRUNCATE && 12412 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND && 12413 N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && 12414 N->getOperand(1).getOpcode() != ISD::ANY_EXTEND) 12415 return SDValue(); 12416 12417 SmallVector<SDValue, 4> Inputs; 12418 SmallVector<SDValue, 8> BinOps, PromOps; 12419 SmallPtrSet<SDNode *, 16> Visited; 12420 12421 for (unsigned i = 0; i < 2; ++i) { 12422 if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 12423 N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 12424 N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 12425 N->getOperand(i).getOperand(0).getValueType() == MVT::i1) || 12426 isa<ConstantSDNode>(N->getOperand(i))) 12427 Inputs.push_back(N->getOperand(i)); 12428 else 12429 BinOps.push_back(N->getOperand(i)); 12430 12431 if (N->getOpcode() == ISD::TRUNCATE) 12432 break; 12433 } 12434 12435 // Visit all inputs, collect all binary operations (and, or, xor and 12436 // select) that are all fed by extensions. 12437 while (!BinOps.empty()) { 12438 SDValue BinOp = BinOps.back(); 12439 BinOps.pop_back(); 12440 12441 if (!Visited.insert(BinOp.getNode()).second) 12442 continue; 12443 12444 PromOps.push_back(BinOp); 12445 12446 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 12447 // The condition of the select is not promoted. 12448 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 12449 continue; 12450 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 12451 continue; 12452 12453 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 12454 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 12455 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) && 12456 BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) || 12457 isa<ConstantSDNode>(BinOp.getOperand(i))) { 12458 Inputs.push_back(BinOp.getOperand(i)); 12459 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 12460 BinOp.getOperand(i).getOpcode() == ISD::OR || 12461 BinOp.getOperand(i).getOpcode() == ISD::XOR || 12462 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 12463 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC || 12464 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 12465 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || 12466 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || 12467 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) { 12468 BinOps.push_back(BinOp.getOperand(i)); 12469 } else { 12470 // We have an input that is not an extension or another binary 12471 // operation; we'll abort this transformation. 12472 return SDValue(); 12473 } 12474 } 12475 } 12476 12477 // Make sure that this is a self-contained cluster of operations (which 12478 // is not quite the same thing as saying that everything has only one 12479 // use). 12480 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12481 if (isa<ConstantSDNode>(Inputs[i])) 12482 continue; 12483 12484 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 12485 UE = Inputs[i].getNode()->use_end(); 12486 UI != UE; ++UI) { 12487 SDNode *User = *UI; 12488 if (User != N && !Visited.count(User)) 12489 return SDValue(); 12490 12491 // Make sure that we're not going to promote the non-output-value 12492 // operand(s) or SELECT or SELECT_CC. 12493 // FIXME: Although we could sometimes handle this, and it does occur in 12494 // practice that one of the condition inputs to the select is also one of 12495 // the outputs, we currently can't deal with this. 12496 if (User->getOpcode() == ISD::SELECT) { 12497 if (User->getOperand(0) == Inputs[i]) 12498 return SDValue(); 12499 } else if (User->getOpcode() == ISD::SELECT_CC) { 12500 if (User->getOperand(0) == Inputs[i] || 12501 User->getOperand(1) == Inputs[i]) 12502 return SDValue(); 12503 } 12504 } 12505 } 12506 12507 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 12508 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 12509 UE = PromOps[i].getNode()->use_end(); 12510 UI != UE; ++UI) { 12511 SDNode *User = *UI; 12512 if (User != N && !Visited.count(User)) 12513 return SDValue(); 12514 12515 // Make sure that we're not going to promote the non-output-value 12516 // operand(s) or SELECT or SELECT_CC. 12517 // FIXME: Although we could sometimes handle this, and it does occur in 12518 // practice that one of the condition inputs to the select is also one of 12519 // the outputs, we currently can't deal with this. 12520 if (User->getOpcode() == ISD::SELECT) { 12521 if (User->getOperand(0) == PromOps[i]) 12522 return SDValue(); 12523 } else if (User->getOpcode() == ISD::SELECT_CC) { 12524 if (User->getOperand(0) == PromOps[i] || 12525 User->getOperand(1) == PromOps[i]) 12526 return SDValue(); 12527 } 12528 } 12529 } 12530 12531 // Replace all inputs with the extension operand. 12532 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12533 // Constants may have users outside the cluster of to-be-promoted nodes, 12534 // and so we need to replace those as we do the promotions. 12535 if (isa<ConstantSDNode>(Inputs[i])) 12536 continue; 12537 else 12538 DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0)); 12539 } 12540 12541 std::list<HandleSDNode> PromOpHandles; 12542 for (auto &PromOp : PromOps) 12543 PromOpHandles.emplace_back(PromOp); 12544 12545 // Replace all operations (these are all the same, but have a different 12546 // (i1) return type). DAG.getNode will validate that the types of 12547 // a binary operator match, so go through the list in reverse so that 12548 // we've likely promoted both operands first. Any intermediate truncations or 12549 // extensions disappear. 12550 while (!PromOpHandles.empty()) { 12551 SDValue PromOp = PromOpHandles.back().getValue(); 12552 PromOpHandles.pop_back(); 12553 12554 if (PromOp.getOpcode() == ISD::TRUNCATE || 12555 PromOp.getOpcode() == ISD::SIGN_EXTEND || 12556 PromOp.getOpcode() == ISD::ZERO_EXTEND || 12557 PromOp.getOpcode() == ISD::ANY_EXTEND) { 12558 if (!isa<ConstantSDNode>(PromOp.getOperand(0)) && 12559 PromOp.getOperand(0).getValueType() != MVT::i1) { 12560 // The operand is not yet ready (see comment below). 12561 PromOpHandles.emplace_front(PromOp); 12562 continue; 12563 } 12564 12565 SDValue RepValue = PromOp.getOperand(0); 12566 if (isa<ConstantSDNode>(RepValue)) 12567 RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue); 12568 12569 DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue); 12570 continue; 12571 } 12572 12573 unsigned C; 12574 switch (PromOp.getOpcode()) { 12575 default: C = 0; break; 12576 case ISD::SELECT: C = 1; break; 12577 case ISD::SELECT_CC: C = 2; break; 12578 } 12579 12580 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 12581 PromOp.getOperand(C).getValueType() != MVT::i1) || 12582 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 12583 PromOp.getOperand(C+1).getValueType() != MVT::i1)) { 12584 // The to-be-promoted operands of this node have not yet been 12585 // promoted (this should be rare because we're going through the 12586 // list backward, but if one of the operands has several users in 12587 // this cluster of to-be-promoted nodes, it is possible). 12588 PromOpHandles.emplace_front(PromOp); 12589 continue; 12590 } 12591 12592 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 12593 PromOp.getNode()->op_end()); 12594 12595 // If there are any constant inputs, make sure they're replaced now. 12596 for (unsigned i = 0; i < 2; ++i) 12597 if (isa<ConstantSDNode>(Ops[C+i])) 12598 Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]); 12599 12600 DAG.ReplaceAllUsesOfValueWith(PromOp, 12601 DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops)); 12602 } 12603 12604 // Now we're left with the initial truncation itself. 12605 if (N->getOpcode() == ISD::TRUNCATE) 12606 return N->getOperand(0); 12607 12608 // Otherwise, this is a comparison. The operands to be compared have just 12609 // changed type (to i1), but everything else is the same. 12610 return SDValue(N, 0); 12611} 12612 12613SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, 12614 DAGCombinerInfo &DCI) const { 12615 SelectionDAG &DAG = DCI.DAG; 12616 SDLoc dl(N); 12617 12618 // If we're tracking CR bits, we need to be careful that we don't have: 12619 // zext(binary-ops(trunc(x), trunc(y))) 12620 // or 12621 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...) 12622 // such that we're unnecessarily moving things into CR bits that can more 12623 // efficiently stay in GPRs. Note that if we're not certain that the high 12624 // bits are set as required by the final extension, we still may need to do 12625 // some masking to get the proper behavior. 12626 12627 // This same functionality is important on PPC64 when dealing with 12628 // 32-to-64-bit extensions; these occur often when 32-bit values are used as 12629 // the return values of functions. Because it is so similar, it is handled 12630 // here as well. 12631 12632 if (N->getValueType(0) != MVT::i32 && 12633 N->getValueType(0) != MVT::i64) 12634 return SDValue(); 12635 12636 if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) || 12637 (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64()))) 12638 return SDValue(); 12639 12640 if (N->getOperand(0).getOpcode() != ISD::AND && 12641 N->getOperand(0).getOpcode() != ISD::OR && 12642 N->getOperand(0).getOpcode() != ISD::XOR && 12643 N->getOperand(0).getOpcode() != ISD::SELECT && 12644 N->getOperand(0).getOpcode() != ISD::SELECT_CC) 12645 return SDValue(); 12646 12647 SmallVector<SDValue, 4> Inputs; 12648 SmallVector<SDValue, 8> BinOps(1, N->getOperand(0)), PromOps; 12649 SmallPtrSet<SDNode *, 16> Visited; 12650 12651 // Visit all inputs, collect all binary operations (and, or, xor and 12652 // select) that are all fed by truncations. 12653 while (!BinOps.empty()) { 12654 SDValue BinOp = BinOps.back(); 12655 BinOps.pop_back(); 12656 12657 if (!Visited.insert(BinOp.getNode()).second) 12658 continue; 12659 12660 PromOps.push_back(BinOp); 12661 12662 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { 12663 // The condition of the select is not promoted. 12664 if (BinOp.getOpcode() == ISD::SELECT && i == 0) 12665 continue; 12666 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) 12667 continue; 12668 12669 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || 12670 isa<ConstantSDNode>(BinOp.getOperand(i))) { 12671 Inputs.push_back(BinOp.getOperand(i)); 12672 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || 12673 BinOp.getOperand(i).getOpcode() == ISD::OR || 12674 BinOp.getOperand(i).getOpcode() == ISD::XOR || 12675 BinOp.getOperand(i).getOpcode() == ISD::SELECT || 12676 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) { 12677 BinOps.push_back(BinOp.getOperand(i)); 12678 } else { 12679 // We have an input that is not a truncation or another binary 12680 // operation; we'll abort this transformation. 12681 return SDValue(); 12682 } 12683 } 12684 } 12685 12686 // The operands of a select that must be truncated when the select is 12687 // promoted because the operand is actually part of the to-be-promoted set. 12688 DenseMap<SDNode *, EVT> SelectTruncOp[2]; 12689 12690 // Make sure that this is a self-contained cluster of operations (which 12691 // is not quite the same thing as saying that everything has only one 12692 // use). 12693 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12694 if (isa<ConstantSDNode>(Inputs[i])) 12695 continue; 12696 12697 for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), 12698 UE = Inputs[i].getNode()->use_end(); 12699 UI != UE; ++UI) { 12700 SDNode *User = *UI; 12701 if (User != N && !Visited.count(User)) 12702 return SDValue(); 12703 12704 // If we're going to promote the non-output-value operand(s) or SELECT or 12705 // SELECT_CC, record them for truncation. 12706 if (User->getOpcode() == ISD::SELECT) { 12707 if (User->getOperand(0) == Inputs[i]) 12708 SelectTruncOp[0].insert(std::make_pair(User, 12709 User->getOperand(0).getValueType())); 12710 } else if (User->getOpcode() == ISD::SELECT_CC) { 12711 if (User->getOperand(0) == Inputs[i]) 12712 SelectTruncOp[0].insert(std::make_pair(User, 12713 User->getOperand(0).getValueType())); 12714 if (User->getOperand(1) == Inputs[i]) 12715 SelectTruncOp[1].insert(std::make_pair(User, 12716 User->getOperand(1).getValueType())); 12717 } 12718 } 12719 } 12720 12721 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { 12722 for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), 12723 UE = PromOps[i].getNode()->use_end(); 12724 UI != UE; ++UI) { 12725 SDNode *User = *UI; 12726 if (User != N && !Visited.count(User)) 12727 return SDValue(); 12728 12729 // If we're going to promote the non-output-value operand(s) or SELECT or 12730 // SELECT_CC, record them for truncation. 12731 if (User->getOpcode() == ISD::SELECT) { 12732 if (User->getOperand(0) == PromOps[i]) 12733 SelectTruncOp[0].insert(std::make_pair(User, 12734 User->getOperand(0).getValueType())); 12735 } else if (User->getOpcode() == ISD::SELECT_CC) { 12736 if (User->getOperand(0) == PromOps[i]) 12737 SelectTruncOp[0].insert(std::make_pair(User, 12738 User->getOperand(0).getValueType())); 12739 if (User->getOperand(1) == PromOps[i]) 12740 SelectTruncOp[1].insert(std::make_pair(User, 12741 User->getOperand(1).getValueType())); 12742 } 12743 } 12744 } 12745 12746 unsigned PromBits = N->getOperand(0).getValueSizeInBits(); 12747 bool ReallyNeedsExt = false; 12748 if (N->getOpcode() != ISD::ANY_EXTEND) { 12749 // If all of the inputs are not already sign/zero extended, then 12750 // we'll still need to do that at the end. 12751 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12752 if (isa<ConstantSDNode>(Inputs[i])) 12753 continue; 12754 12755 unsigned OpBits = 12756 Inputs[i].getOperand(0).getValueSizeInBits(); 12757 assert(PromBits < OpBits && "Truncation not to a smaller bit count?"); 12758 12759 if ((N->getOpcode() == ISD::ZERO_EXTEND && 12760 !DAG.MaskedValueIsZero(Inputs[i].getOperand(0), 12761 APInt::getHighBitsSet(OpBits, 12762 OpBits-PromBits))) || 12763 (N->getOpcode() == ISD::SIGN_EXTEND && 12764 DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) < 12765 (OpBits-(PromBits-1)))) { 12766 ReallyNeedsExt = true; 12767 break; 12768 } 12769 } 12770 } 12771 12772 // Replace all inputs, either with the truncation operand, or a 12773 // truncation or extension to the final output type. 12774 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { 12775 // Constant inputs need to be replaced with the to-be-promoted nodes that 12776 // use them because they might have users outside of the cluster of 12777 // promoted nodes. 12778 if (isa<ConstantSDNode>(Inputs[i])) 12779 continue; 12780 12781 SDValue InSrc = Inputs[i].getOperand(0); 12782 if (Inputs[i].getValueType() == N->getValueType(0)) 12783 DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc); 12784 else if (N->getOpcode() == ISD::SIGN_EXTEND) 12785 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 12786 DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0))); 12787 else if (N->getOpcode() == ISD::ZERO_EXTEND) 12788 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 12789 DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0))); 12790 else 12791 DAG.ReplaceAllUsesOfValueWith(Inputs[i], 12792 DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); 12793 } 12794 12795 std::list<HandleSDNode> PromOpHandles; 12796 for (auto &PromOp : PromOps) 12797 PromOpHandles.emplace_back(PromOp); 12798 12799 // Replace all operations (these are all the same, but have a different 12800 // (promoted) return type). DAG.getNode will validate that the types of 12801 // a binary operator match, so go through the list in reverse so that 12802 // we've likely promoted both operands first. 12803 while (!PromOpHandles.empty()) { 12804 SDValue PromOp = PromOpHandles.back().getValue(); 12805 PromOpHandles.pop_back(); 12806 12807 unsigned C; 12808 switch (PromOp.getOpcode()) { 12809 default: C = 0; break; 12810 case ISD::SELECT: C = 1; break; 12811 case ISD::SELECT_CC: C = 2; break; 12812 } 12813 12814 if ((!isa<ConstantSDNode>(PromOp.getOperand(C)) && 12815 PromOp.getOperand(C).getValueType() != N->getValueType(0)) || 12816 (!isa<ConstantSDNode>(PromOp.getOperand(C+1)) && 12817 PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) { 12818 // The to-be-promoted operands of this node have not yet been 12819 // promoted (this should be rare because we're going through the 12820 // list backward, but if one of the operands has several users in 12821 // this cluster of to-be-promoted nodes, it is possible). 12822 PromOpHandles.emplace_front(PromOp); 12823 continue; 12824 } 12825 12826 // For SELECT and SELECT_CC nodes, we do a similar check for any 12827 // to-be-promoted comparison inputs. 12828 if (PromOp.getOpcode() == ISD::SELECT || 12829 PromOp.getOpcode() == ISD::SELECT_CC) { 12830 if ((SelectTruncOp[0].count(PromOp.getNode()) && 12831 PromOp.getOperand(0).getValueType() != N->getValueType(0)) || 12832 (SelectTruncOp[1].count(PromOp.getNode()) && 12833 PromOp.getOperand(1).getValueType() != N->getValueType(0))) { 12834 PromOpHandles.emplace_front(PromOp); 12835 continue; 12836 } 12837 } 12838 12839 SmallVector<SDValue, 3> Ops(PromOp.getNode()->op_begin(), 12840 PromOp.getNode()->op_end()); 12841 12842 // If this node has constant inputs, then they'll need to be promoted here. 12843 for (unsigned i = 0; i < 2; ++i) { 12844 if (!isa<ConstantSDNode>(Ops[C+i])) 12845 continue; 12846 if (Ops[C+i].getValueType() == N->getValueType(0)) 12847 continue; 12848 12849 if (N->getOpcode() == ISD::SIGN_EXTEND) 12850 Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 12851 else if (N->getOpcode() == ISD::ZERO_EXTEND) 12852 Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 12853 else 12854 Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); 12855 } 12856 12857 // If we've promoted the comparison inputs of a SELECT or SELECT_CC, 12858 // truncate them again to the original value type. 12859 if (PromOp.getOpcode() == ISD::SELECT || 12860 PromOp.getOpcode() == ISD::SELECT_CC) { 12861 auto SI0 = SelectTruncOp[0].find(PromOp.getNode()); 12862 if (SI0 != SelectTruncOp[0].end()) 12863 Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]); 12864 auto SI1 = SelectTruncOp[1].find(PromOp.getNode()); 12865 if (SI1 != SelectTruncOp[1].end()) 12866 Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]); 12867 } 12868 12869 DAG.ReplaceAllUsesOfValueWith(PromOp, 12870 DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops)); 12871 } 12872 12873 // Now we're left with the initial extension itself. 12874 if (!ReallyNeedsExt) 12875 return N->getOperand(0); 12876 12877 // To zero extend, just mask off everything except for the first bit (in the 12878 // i1 case). 12879 if (N->getOpcode() == ISD::ZERO_EXTEND) 12880 return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0), 12881 DAG.getConstant(APInt::getLowBitsSet( 12882 N->getValueSizeInBits(0), PromBits), 12883 dl, N->getValueType(0))); 12884 12885 assert(N->getOpcode() == ISD::SIGN_EXTEND && 12886 "Invalid extension type"); 12887 EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout()); 12888 SDValue ShiftCst = 12889 DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy); 12890 return DAG.getNode( 12891 ISD::SRA, dl, N->getValueType(0), 12892 DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst), 12893 ShiftCst); 12894} 12895 12896SDValue PPCTargetLowering::combineSetCC(SDNode *N, 12897 DAGCombinerInfo &DCI) const { 12898 assert(N->getOpcode() == ISD::SETCC && 12899 "Should be called with a SETCC node"); 12900 12901 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 12902 if (CC == ISD::SETNE || CC == ISD::SETEQ) { 12903 SDValue LHS = N->getOperand(0); 12904 SDValue RHS = N->getOperand(1); 12905 12906 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS. 12907 if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) && 12908 LHS.hasOneUse()) 12909 std::swap(LHS, RHS); 12910 12911 // x == 0-y --> x+y == 0 12912 // x != 0-y --> x+y != 0 12913 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) && 12914 RHS.hasOneUse()) { 12915 SDLoc DL(N); 12916 SelectionDAG &DAG = DCI.DAG; 12917 EVT VT = N->getValueType(0); 12918 EVT OpVT = LHS.getValueType(); 12919 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1)); 12920 return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC); 12921 } 12922 } 12923 12924 return DAGCombineTruncBoolExt(N, DCI); 12925} 12926 12927// Is this an extending load from an f32 to an f64? 12928static bool isFPExtLoad(SDValue Op) { 12929 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode())) 12930 return LD->getExtensionType() == ISD::EXTLOAD && 12931 Op.getValueType() == MVT::f64; 12932 return false; 12933} 12934 12935/// Reduces the number of fp-to-int conversion when building a vector. 12936/// 12937/// If this vector is built out of floating to integer conversions, 12938/// transform it to a vector built out of floating point values followed by a 12939/// single floating to integer conversion of the vector. 12940/// Namely (build_vector (fptosi $A), (fptosi $B), ...) 12941/// becomes (fptosi (build_vector ($A, $B, ...))) 12942SDValue PPCTargetLowering:: 12943combineElementTruncationToVectorTruncation(SDNode *N, 12944 DAGCombinerInfo &DCI) const { 12945 assert(N->getOpcode() == ISD::BUILD_VECTOR && 12946 "Should be called with a BUILD_VECTOR node"); 12947 12948 SelectionDAG &DAG = DCI.DAG; 12949 SDLoc dl(N); 12950 12951 SDValue FirstInput = N->getOperand(0); 12952 assert(FirstInput.getOpcode() == PPCISD::MFVSR && 12953 "The input operand must be an fp-to-int conversion."); 12954 12955 // This combine happens after legalization so the fp_to_[su]i nodes are 12956 // already converted to PPCSISD nodes. 12957 unsigned FirstConversion = FirstInput.getOperand(0).getOpcode(); 12958 if (FirstConversion == PPCISD::FCTIDZ || 12959 FirstConversion == PPCISD::FCTIDUZ || 12960 FirstConversion == PPCISD::FCTIWZ || 12961 FirstConversion == PPCISD::FCTIWUZ) { 12962 bool IsSplat = true; 12963 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ || 12964 FirstConversion == PPCISD::FCTIWUZ; 12965 EVT SrcVT = FirstInput.getOperand(0).getValueType(); 12966 SmallVector<SDValue, 4> Ops; 12967 EVT TargetVT = N->getValueType(0); 12968 for (int i = 0, e = N->getNumOperands(); i < e; ++i) { 12969 SDValue NextOp = N->getOperand(i); 12970 if (NextOp.getOpcode() != PPCISD::MFVSR) 12971 return SDValue(); 12972 unsigned NextConversion = NextOp.getOperand(0).getOpcode(); 12973 if (NextConversion != FirstConversion) 12974 return SDValue(); 12975 // If we are converting to 32-bit integers, we need to add an FP_ROUND. 12976 // This is not valid if the input was originally double precision. It is 12977 // also not profitable to do unless this is an extending load in which 12978 // case doing this combine will allow us to combine consecutive loads. 12979 if (Is32Bit && !isFPExtLoad(NextOp.getOperand(0).getOperand(0))) 12980 return SDValue(); 12981 if (N->getOperand(i) != FirstInput) 12982 IsSplat = false; 12983 } 12984 12985 // If this is a splat, we leave it as-is since there will be only a single 12986 // fp-to-int conversion followed by a splat of the integer. This is better 12987 // for 32-bit and smaller ints and neutral for 64-bit ints. 12988 if (IsSplat) 12989 return SDValue(); 12990 12991 // Now that we know we have the right type of node, get its operands 12992 for (int i = 0, e = N->getNumOperands(); i < e; ++i) { 12993 SDValue In = N->getOperand(i).getOperand(0); 12994 if (Is32Bit) { 12995 // For 32-bit values, we need to add an FP_ROUND node (if we made it 12996 // here, we know that all inputs are extending loads so this is safe). 12997 if (In.isUndef()) 12998 Ops.push_back(DAG.getUNDEF(SrcVT)); 12999 else { 13000 SDValue Trunc = DAG.getNode(ISD::FP_ROUND, dl, 13001 MVT::f32, In.getOperand(0), 13002 DAG.getIntPtrConstant(1, dl)); 13003 Ops.push_back(Trunc); 13004 } 13005 } else 13006 Ops.push_back(In.isUndef() ? DAG.getUNDEF(SrcVT) : In.getOperand(0)); 13007 } 13008 13009 unsigned Opcode; 13010 if (FirstConversion == PPCISD::FCTIDZ || 13011 FirstConversion == PPCISD::FCTIWZ) 13012 Opcode = ISD::FP_TO_SINT; 13013 else 13014 Opcode = ISD::FP_TO_UINT; 13015 13016 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32; 13017 SDValue BV = DAG.getBuildVector(NewVT, dl, Ops); 13018 return DAG.getNode(Opcode, dl, TargetVT, BV); 13019 } 13020 return SDValue(); 13021} 13022 13023/// Reduce the number of loads when building a vector. 13024/// 13025/// Building a vector out of multiple loads can be converted to a load 13026/// of the vector type if the loads are consecutive. If the loads are 13027/// consecutive but in descending order, a shuffle is added at the end 13028/// to reorder the vector. 13029static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) { 13030 assert(N->getOpcode() == ISD::BUILD_VECTOR && 13031 "Should be called with a BUILD_VECTOR node"); 13032 13033 SDLoc dl(N); 13034 13035 // Return early for non byte-sized type, as they can't be consecutive. 13036 if (!N->getValueType(0).getVectorElementType().isByteSized()) 13037 return SDValue(); 13038 13039 bool InputsAreConsecutiveLoads = true; 13040 bool InputsAreReverseConsecutive = true; 13041 unsigned ElemSize = N->getValueType(0).getScalarType().getStoreSize(); 13042 SDValue FirstInput = N->getOperand(0); 13043 bool IsRoundOfExtLoad = false; 13044 13045 if (FirstInput.getOpcode() == ISD::FP_ROUND && 13046 FirstInput.getOperand(0).getOpcode() == ISD::LOAD) { 13047 LoadSDNode *LD = dyn_cast<LoadSDNode>(FirstInput.getOperand(0)); 13048 IsRoundOfExtLoad = LD->getExtensionType() == ISD::EXTLOAD; 13049 } 13050 // Not a build vector of (possibly fp_rounded) loads. 13051 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) || 13052 N->getNumOperands() == 1) 13053 return SDValue(); 13054 13055 for (int i = 1, e = N->getNumOperands(); i < e; ++i) { 13056 // If any inputs are fp_round(extload), they all must be. 13057 if (IsRoundOfExtLoad && N->getOperand(i).getOpcode() != ISD::FP_ROUND) 13058 return SDValue(); 13059 13060 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(i).getOperand(0) : 13061 N->getOperand(i); 13062 if (NextInput.getOpcode() != ISD::LOAD) 13063 return SDValue(); 13064 13065 SDValue PreviousInput = 13066 IsRoundOfExtLoad ? N->getOperand(i-1).getOperand(0) : N->getOperand(i-1); 13067 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(PreviousInput); 13068 LoadSDNode *LD2 = dyn_cast<LoadSDNode>(NextInput); 13069 13070 // If any inputs are fp_round(extload), they all must be. 13071 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD) 13072 return SDValue(); 13073 13074 if (!isConsecutiveLS(LD2, LD1, ElemSize, 1, DAG)) 13075 InputsAreConsecutiveLoads = false; 13076 if (!isConsecutiveLS(LD1, LD2, ElemSize, 1, DAG)) 13077 InputsAreReverseConsecutive = false; 13078 13079 // Exit early if the loads are neither consecutive nor reverse consecutive. 13080 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive) 13081 return SDValue(); 13082 } 13083 13084 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) && 13085 "The loads cannot be both consecutive and reverse consecutive."); 13086 13087 SDValue FirstLoadOp = 13088 IsRoundOfExtLoad ? FirstInput.getOperand(0) : FirstInput; 13089 SDValue LastLoadOp = 13090 IsRoundOfExtLoad ? N->getOperand(N->getNumOperands()-1).getOperand(0) : 13091 N->getOperand(N->getNumOperands()-1); 13092 13093 LoadSDNode *LD1 = dyn_cast<LoadSDNode>(FirstLoadOp); 13094 LoadSDNode *LDL = dyn_cast<LoadSDNode>(LastLoadOp); 13095 if (InputsAreConsecutiveLoads) { 13096 assert(LD1 && "Input needs to be a LoadSDNode."); 13097 return DAG.getLoad(N->getValueType(0), dl, LD1->getChain(), 13098 LD1->getBasePtr(), LD1->getPointerInfo(), 13099 LD1->getAlignment()); 13100 } 13101 if (InputsAreReverseConsecutive) { 13102 assert(LDL && "Input needs to be a LoadSDNode."); 13103 SDValue Load = DAG.getLoad(N->getValueType(0), dl, LDL->getChain(), 13104 LDL->getBasePtr(), LDL->getPointerInfo(), 13105 LDL->getAlignment()); 13106 SmallVector<int, 16> Ops; 13107 for (int i = N->getNumOperands() - 1; i >= 0; i--) 13108 Ops.push_back(i); 13109 13110 return DAG.getVectorShuffle(N->getValueType(0), dl, Load, 13111 DAG.getUNDEF(N->getValueType(0)), Ops); 13112 } 13113 return SDValue(); 13114} 13115 13116// This function adds the required vector_shuffle needed to get 13117// the elements of the vector extract in the correct position 13118// as specified by the CorrectElems encoding. 13119static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG, 13120 SDValue Input, uint64_t Elems, 13121 uint64_t CorrectElems) { 13122 SDLoc dl(N); 13123 13124 unsigned NumElems = Input.getValueType().getVectorNumElements(); 13125 SmallVector<int, 16> ShuffleMask(NumElems, -1); 13126 13127 // Knowing the element indices being extracted from the original 13128 // vector and the order in which they're being inserted, just put 13129 // them at element indices required for the instruction. 13130 for (unsigned i = 0; i < N->getNumOperands(); i++) { 13131 if (DAG.getDataLayout().isLittleEndian()) 13132 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF; 13133 else 13134 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4; 13135 CorrectElems = CorrectElems >> 8; 13136 Elems = Elems >> 8; 13137 } 13138 13139 SDValue Shuffle = 13140 DAG.getVectorShuffle(Input.getValueType(), dl, Input, 13141 DAG.getUNDEF(Input.getValueType()), ShuffleMask); 13142 13143 EVT Ty = N->getValueType(0); 13144 SDValue BV = DAG.getNode(PPCISD::SExtVElems, dl, Ty, Shuffle); 13145 return BV; 13146} 13147 13148// Look for build vector patterns where input operands come from sign 13149// extended vector_extract elements of specific indices. If the correct indices 13150// aren't used, add a vector shuffle to fix up the indices and create a new 13151// PPCISD:SExtVElems node which selects the vector sign extend instructions 13152// during instruction selection. 13153static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) { 13154 // This array encodes the indices that the vector sign extend instructions 13155 // extract from when extending from one type to another for both BE and LE. 13156 // The right nibble of each byte corresponds to the LE incides. 13157 // and the left nibble of each byte corresponds to the BE incides. 13158 // For example: 0x3074B8FC byte->word 13159 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC 13160 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF 13161 // For example: 0x000070F8 byte->double word 13162 // For LE: the allowed indices are: 0x0,0x8 13163 // For BE: the allowed indices are: 0x7,0xF 13164 uint64_t TargetElems[] = { 13165 0x3074B8FC, // b->w 13166 0x000070F8, // b->d 13167 0x10325476, // h->w 13168 0x00003074, // h->d 13169 0x00001032, // w->d 13170 }; 13171 13172 uint64_t Elems = 0; 13173 int Index; 13174 SDValue Input; 13175 13176 auto isSExtOfVecExtract = [&](SDValue Op) -> bool { 13177 if (!Op) 13178 return false; 13179 if (Op.getOpcode() != ISD::SIGN_EXTEND && 13180 Op.getOpcode() != ISD::SIGN_EXTEND_INREG) 13181 return false; 13182 13183 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value 13184 // of the right width. 13185 SDValue Extract = Op.getOperand(0); 13186 if (Extract.getOpcode() == ISD::ANY_EXTEND) 13187 Extract = Extract.getOperand(0); 13188 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 13189 return false; 13190 13191 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Extract.getOperand(1)); 13192 if (!ExtOp) 13193 return false; 13194 13195 Index = ExtOp->getZExtValue(); 13196 if (Input && Input != Extract.getOperand(0)) 13197 return false; 13198 13199 if (!Input) 13200 Input = Extract.getOperand(0); 13201 13202 Elems = Elems << 8; 13203 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4; 13204 Elems |= Index; 13205 13206 return true; 13207 }; 13208 13209 // If the build vector operands aren't sign extended vector extracts, 13210 // of the same input vector, then return. 13211 for (unsigned i = 0; i < N->getNumOperands(); i++) { 13212 if (!isSExtOfVecExtract(N->getOperand(i))) { 13213 return SDValue(); 13214 } 13215 } 13216 13217 // If the vector extract indicies are not correct, add the appropriate 13218 // vector_shuffle. 13219 int TgtElemArrayIdx; 13220 int InputSize = Input.getValueType().getScalarSizeInBits(); 13221 int OutputSize = N->getValueType(0).getScalarSizeInBits(); 13222 if (InputSize + OutputSize == 40) 13223 TgtElemArrayIdx = 0; 13224 else if (InputSize + OutputSize == 72) 13225 TgtElemArrayIdx = 1; 13226 else if (InputSize + OutputSize == 48) 13227 TgtElemArrayIdx = 2; 13228 else if (InputSize + OutputSize == 80) 13229 TgtElemArrayIdx = 3; 13230 else if (InputSize + OutputSize == 96) 13231 TgtElemArrayIdx = 4; 13232 else 13233 return SDValue(); 13234 13235 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx]; 13236 CorrectElems = DAG.getDataLayout().isLittleEndian() 13237 ? CorrectElems & 0x0F0F0F0F0F0F0F0F 13238 : CorrectElems & 0xF0F0F0F0F0F0F0F0; 13239 if (Elems != CorrectElems) { 13240 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems); 13241 } 13242 13243 // Regular lowering will catch cases where a shuffle is not needed. 13244 return SDValue(); 13245} 13246 13247SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, 13248 DAGCombinerInfo &DCI) const { 13249 assert(N->getOpcode() == ISD::BUILD_VECTOR && 13250 "Should be called with a BUILD_VECTOR node"); 13251 13252 SelectionDAG &DAG = DCI.DAG; 13253 SDLoc dl(N); 13254 13255 if (!Subtarget.hasVSX()) 13256 return SDValue(); 13257 13258 // The target independent DAG combiner will leave a build_vector of 13259 // float-to-int conversions intact. We can generate MUCH better code for 13260 // a float-to-int conversion of a vector of floats. 13261 SDValue FirstInput = N->getOperand(0); 13262 if (FirstInput.getOpcode() == PPCISD::MFVSR) { 13263 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI); 13264 if (Reduced) 13265 return Reduced; 13266 } 13267 13268 // If we're building a vector out of consecutive loads, just load that 13269 // vector type. 13270 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG); 13271 if (Reduced) 13272 return Reduced; 13273 13274 // If we're building a vector out of extended elements from another vector 13275 // we have P9 vector integer extend instructions. The code assumes legal 13276 // input types (i.e. it can't handle things like v4i16) so do not run before 13277 // legalization. 13278 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) { 13279 Reduced = combineBVOfVecSExt(N, DAG); 13280 if (Reduced) 13281 return Reduced; 13282 } 13283 13284 13285 if (N->getValueType(0) != MVT::v2f64) 13286 return SDValue(); 13287 13288 // Looking for: 13289 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1)) 13290 if (FirstInput.getOpcode() != ISD::SINT_TO_FP && 13291 FirstInput.getOpcode() != ISD::UINT_TO_FP) 13292 return SDValue(); 13293 if (N->getOperand(1).getOpcode() != ISD::SINT_TO_FP && 13294 N->getOperand(1).getOpcode() != ISD::UINT_TO_FP) 13295 return SDValue(); 13296 if (FirstInput.getOpcode() != N->getOperand(1).getOpcode()) 13297 return SDValue(); 13298 13299 SDValue Ext1 = FirstInput.getOperand(0); 13300 SDValue Ext2 = N->getOperand(1).getOperand(0); 13301 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || 13302 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 13303 return SDValue(); 13304 13305 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Ext1.getOperand(1)); 13306 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Ext2.getOperand(1)); 13307 if (!Ext1Op || !Ext2Op) 13308 return SDValue(); 13309 if (Ext1.getOperand(0).getValueType() != MVT::v4i32 || 13310 Ext1.getOperand(0) != Ext2.getOperand(0)) 13311 return SDValue(); 13312 13313 int FirstElem = Ext1Op->getZExtValue(); 13314 int SecondElem = Ext2Op->getZExtValue(); 13315 int SubvecIdx; 13316 if (FirstElem == 0 && SecondElem == 1) 13317 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0; 13318 else if (FirstElem == 2 && SecondElem == 3) 13319 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1; 13320 else 13321 return SDValue(); 13322 13323 SDValue SrcVec = Ext1.getOperand(0); 13324 auto NodeType = (N->getOperand(1).getOpcode() == ISD::SINT_TO_FP) ? 13325 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP; 13326 return DAG.getNode(NodeType, dl, MVT::v2f64, 13327 SrcVec, DAG.getIntPtrConstant(SubvecIdx, dl)); 13328} 13329 13330SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N, 13331 DAGCombinerInfo &DCI) const { 13332 assert((N->getOpcode() == ISD::SINT_TO_FP || 13333 N->getOpcode() == ISD::UINT_TO_FP) && 13334 "Need an int -> FP conversion node here"); 13335 13336 if (useSoftFloat() || !Subtarget.has64BitSupport()) 13337 return SDValue(); 13338 13339 SelectionDAG &DAG = DCI.DAG; 13340 SDLoc dl(N); 13341 SDValue Op(N, 0); 13342 13343 // Don't handle ppc_fp128 here or conversions that are out-of-range capable 13344 // from the hardware. 13345 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) 13346 return SDValue(); 13347 if (Op.getOperand(0).getValueType().getSimpleVT() <= MVT(MVT::i1) || 13348 Op.getOperand(0).getValueType().getSimpleVT() > MVT(MVT::i64)) 13349 return SDValue(); 13350 13351 SDValue FirstOperand(Op.getOperand(0)); 13352 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD && 13353 (FirstOperand.getValueType() == MVT::i8 || 13354 FirstOperand.getValueType() == MVT::i16); 13355 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) { 13356 bool Signed = N->getOpcode() == ISD::SINT_TO_FP; 13357 bool DstDouble = Op.getValueType() == MVT::f64; 13358 unsigned ConvOp = Signed ? 13359 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) : 13360 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS); 13361 SDValue WidthConst = 13362 DAG.getIntPtrConstant(FirstOperand.getValueType() == MVT::i8 ? 1 : 2, 13363 dl, false); 13364 LoadSDNode *LDN = cast<LoadSDNode>(FirstOperand.getNode()); 13365 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst }; 13366 SDValue Ld = DAG.getMemIntrinsicNode(PPCISD::LXSIZX, dl, 13367 DAG.getVTList(MVT::f64, MVT::Other), 13368 Ops, MVT::i8, LDN->getMemOperand()); 13369 13370 // For signed conversion, we need to sign-extend the value in the VSR 13371 if (Signed) { 13372 SDValue ExtOps[] = { Ld, WidthConst }; 13373 SDValue Ext = DAG.getNode(PPCISD::VEXTS, dl, MVT::f64, ExtOps); 13374 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ext); 13375 } else 13376 return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ld); 13377 } 13378 13379 13380 // For i32 intermediate values, unfortunately, the conversion functions 13381 // leave the upper 32 bits of the value are undefined. Within the set of 13382 // scalar instructions, we have no method for zero- or sign-extending the 13383 // value. Thus, we cannot handle i32 intermediate values here. 13384 if (Op.getOperand(0).getValueType() == MVT::i32) 13385 return SDValue(); 13386 13387 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && 13388 "UINT_TO_FP is supported only with FPCVT"); 13389 13390 // If we have FCFIDS, then use it when converting to single-precision. 13391 // Otherwise, convert to double-precision and then round. 13392 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 13393 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS 13394 : PPCISD::FCFIDS) 13395 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU 13396 : PPCISD::FCFID); 13397 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) 13398 ? MVT::f32 13399 : MVT::f64; 13400 13401 // If we're converting from a float, to an int, and back to a float again, 13402 // then we don't need the store/load pair at all. 13403 if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT && 13404 Subtarget.hasFPCVT()) || 13405 (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) { 13406 SDValue Src = Op.getOperand(0).getOperand(0); 13407 if (Src.getValueType() == MVT::f32) { 13408 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); 13409 DCI.AddToWorklist(Src.getNode()); 13410 } else if (Src.getValueType() != MVT::f64) { 13411 // Make sure that we don't pick up a ppc_fp128 source value. 13412 return SDValue(); 13413 } 13414 13415 unsigned FCTOp = 13416 Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ : 13417 PPCISD::FCTIDUZ; 13418 13419 SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src); 13420 SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp); 13421 13422 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { 13423 FP = DAG.getNode(ISD::FP_ROUND, dl, 13424 MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); 13425 DCI.AddToWorklist(FP.getNode()); 13426 } 13427 13428 return FP; 13429 } 13430 13431 return SDValue(); 13432} 13433 13434// expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for 13435// builtins) into loads with swaps. 13436SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N, 13437 DAGCombinerInfo &DCI) const { 13438 SelectionDAG &DAG = DCI.DAG; 13439 SDLoc dl(N); 13440 SDValue Chain; 13441 SDValue Base; 13442 MachineMemOperand *MMO; 13443 13444 switch (N->getOpcode()) { 13445 default: 13446 llvm_unreachable("Unexpected opcode for little endian VSX load"); 13447 case ISD::LOAD: { 13448 LoadSDNode *LD = cast<LoadSDNode>(N); 13449 Chain = LD->getChain(); 13450 Base = LD->getBasePtr(); 13451 MMO = LD->getMemOperand(); 13452 // If the MMO suggests this isn't a load of a full vector, leave 13453 // things alone. For a built-in, we have to make the change for 13454 // correctness, so if there is a size problem that will be a bug. 13455 if (MMO->getSize() < 16) 13456 return SDValue(); 13457 break; 13458 } 13459 case ISD::INTRINSIC_W_CHAIN: { 13460 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 13461 Chain = Intrin->getChain(); 13462 // Similarly to the store case below, Intrin->getBasePtr() doesn't get 13463 // us what we want. Get operand 2 instead. 13464 Base = Intrin->getOperand(2); 13465 MMO = Intrin->getMemOperand(); 13466 break; 13467 } 13468 } 13469 13470 MVT VecTy = N->getValueType(0).getSimpleVT(); 13471 13472 // Do not expand to PPCISD::LXVD2X + PPCISD::XXSWAPD when the load is 13473 // aligned and the type is a vector with elements up to 4 bytes 13474 if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16) 13475 && VecTy.getScalarSizeInBits() <= 32 ) { 13476 return SDValue(); 13477 } 13478 13479 SDValue LoadOps[] = { Chain, Base }; 13480 SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl, 13481 DAG.getVTList(MVT::v2f64, MVT::Other), 13482 LoadOps, MVT::v2f64, MMO); 13483 13484 DCI.AddToWorklist(Load.getNode()); 13485 Chain = Load.getValue(1); 13486 SDValue Swap = DAG.getNode( 13487 PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Load); 13488 DCI.AddToWorklist(Swap.getNode()); 13489 13490 // Add a bitcast if the resulting load type doesn't match v2f64. 13491 if (VecTy != MVT::v2f64) { 13492 SDValue N = DAG.getNode(ISD::BITCAST, dl, VecTy, Swap); 13493 DCI.AddToWorklist(N.getNode()); 13494 // Package {bitcast value, swap's chain} to match Load's shape. 13495 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VecTy, MVT::Other), 13496 N, Swap.getValue(1)); 13497 } 13498 13499 return Swap; 13500} 13501 13502// expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for 13503// builtins) into stores with swaps. 13504SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N, 13505 DAGCombinerInfo &DCI) const { 13506 SelectionDAG &DAG = DCI.DAG; 13507 SDLoc dl(N); 13508 SDValue Chain; 13509 SDValue Base; 13510 unsigned SrcOpnd; 13511 MachineMemOperand *MMO; 13512 13513 switch (N->getOpcode()) { 13514 default: 13515 llvm_unreachable("Unexpected opcode for little endian VSX store"); 13516 case ISD::STORE: { 13517 StoreSDNode *ST = cast<StoreSDNode>(N); 13518 Chain = ST->getChain(); 13519 Base = ST->getBasePtr(); 13520 MMO = ST->getMemOperand(); 13521 SrcOpnd = 1; 13522 // If the MMO suggests this isn't a store of a full vector, leave 13523 // things alone. For a built-in, we have to make the change for 13524 // correctness, so if there is a size problem that will be a bug. 13525 if (MMO->getSize() < 16) 13526 return SDValue(); 13527 break; 13528 } 13529 case ISD::INTRINSIC_VOID: { 13530 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(N); 13531 Chain = Intrin->getChain(); 13532 // Intrin->getBasePtr() oddly does not get what we want. 13533 Base = Intrin->getOperand(3); 13534 MMO = Intrin->getMemOperand(); 13535 SrcOpnd = 2; 13536 break; 13537 } 13538 } 13539 13540 SDValue Src = N->getOperand(SrcOpnd); 13541 MVT VecTy = Src.getValueType().getSimpleVT(); 13542 13543 // Do not expand to PPCISD::XXSWAPD and PPCISD::STXVD2X when the load is 13544 // aligned and the type is a vector with elements up to 4 bytes 13545 if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16) 13546 && VecTy.getScalarSizeInBits() <= 32 ) { 13547 return SDValue(); 13548 } 13549 13550 // All stores are done as v2f64 and possible bit cast. 13551 if (VecTy != MVT::v2f64) { 13552 Src = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Src); 13553 DCI.AddToWorklist(Src.getNode()); 13554 } 13555 13556 SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl, 13557 DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Src); 13558 DCI.AddToWorklist(Swap.getNode()); 13559 Chain = Swap.getValue(1); 13560 SDValue StoreOps[] = { Chain, Swap, Base }; 13561 SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl, 13562 DAG.getVTList(MVT::Other), 13563 StoreOps, VecTy, MMO); 13564 DCI.AddToWorklist(Store.getNode()); 13565 return Store; 13566} 13567 13568// Handle DAG combine for STORE (FP_TO_INT F). 13569SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, 13570 DAGCombinerInfo &DCI) const { 13571 13572 SelectionDAG &DAG = DCI.DAG; 13573 SDLoc dl(N); 13574 unsigned Opcode = N->getOperand(1).getOpcode(); 13575 13576 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) 13577 && "Not a FP_TO_INT Instruction!"); 13578 13579 SDValue Val = N->getOperand(1).getOperand(0); 13580 EVT Op1VT = N->getOperand(1).getValueType(); 13581 EVT ResVT = Val.getValueType(); 13582 13583 // Floating point types smaller than 32 bits are not legal on Power. 13584 if (ResVT.getScalarSizeInBits() < 32) 13585 return SDValue(); 13586 13587 // Only perform combine for conversion to i64/i32 or power9 i16/i8. 13588 bool ValidTypeForStoreFltAsInt = 13589 (Op1VT == MVT::i32 || Op1VT == MVT::i64 || 13590 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8))); 13591 13592 if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Altivec() || 13593 cast<StoreSDNode>(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt) 13594 return SDValue(); 13595 13596 // Extend f32 values to f64 13597 if (ResVT.getScalarSizeInBits() == 32) { 13598 Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); 13599 DCI.AddToWorklist(Val.getNode()); 13600 } 13601 13602 // Set signed or unsigned conversion opcode. 13603 unsigned ConvOpcode = (Opcode == ISD::FP_TO_SINT) ? 13604 PPCISD::FP_TO_SINT_IN_VSR : 13605 PPCISD::FP_TO_UINT_IN_VSR; 13606 13607 Val = DAG.getNode(ConvOpcode, 13608 dl, ResVT == MVT::f128 ? MVT::f128 : MVT::f64, Val); 13609 DCI.AddToWorklist(Val.getNode()); 13610 13611 // Set number of bytes being converted. 13612 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8; 13613 SDValue Ops[] = { N->getOperand(0), Val, N->getOperand(2), 13614 DAG.getIntPtrConstant(ByteSize, dl, false), 13615 DAG.getValueType(Op1VT) }; 13616 13617 Val = DAG.getMemIntrinsicNode(PPCISD::ST_VSR_SCAL_INT, dl, 13618 DAG.getVTList(MVT::Other), Ops, 13619 cast<StoreSDNode>(N)->getMemoryVT(), 13620 cast<StoreSDNode>(N)->getMemOperand()); 13621 13622 DCI.AddToWorklist(Val.getNode()); 13623 return Val; 13624} 13625 13626SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN, 13627 LSBaseSDNode *LSBase, 13628 DAGCombinerInfo &DCI) const { 13629 assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) && 13630 "Not a reverse memop pattern!"); 13631 13632 auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool { 13633 auto Mask = SVN->getMask(); 13634 int i = 0; 13635 auto I = Mask.rbegin(); 13636 auto E = Mask.rend(); 13637 13638 for (; I != E; ++I) { 13639 if (*I != i) 13640 return false; 13641 i++; 13642 } 13643 return true; 13644 }; 13645 13646 SelectionDAG &DAG = DCI.DAG; 13647 EVT VT = SVN->getValueType(0); 13648 13649 if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX()) 13650 return SDValue(); 13651 13652 // Before P9, we have PPCVSXSwapRemoval pass to hack the element order. 13653 // See comment in PPCVSXSwapRemoval.cpp. 13654 // It is conflict with PPCVSXSwapRemoval opt. So we don't do it. 13655 if (!Subtarget.hasP9Vector()) 13656 return SDValue(); 13657 13658 if(!IsElementReverse(SVN)) 13659 return SDValue(); 13660 13661 if (LSBase->getOpcode() == ISD::LOAD) { 13662 SDLoc dl(SVN); 13663 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()}; 13664 return DAG.getMemIntrinsicNode( 13665 PPCISD::LOAD_VEC_BE, dl, DAG.getVTList(VT, MVT::Other), LoadOps, 13666 LSBase->getMemoryVT(), LSBase->getMemOperand()); 13667 } 13668 13669 if (LSBase->getOpcode() == ISD::STORE) { 13670 SDLoc dl(LSBase); 13671 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(0), 13672 LSBase->getBasePtr()}; 13673 return DAG.getMemIntrinsicNode( 13674 PPCISD::STORE_VEC_BE, dl, DAG.getVTList(MVT::Other), StoreOps, 13675 LSBase->getMemoryVT(), LSBase->getMemOperand()); 13676 } 13677 13678 llvm_unreachable("Expected a load or store node here"); 13679} 13680 13681SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, 13682 DAGCombinerInfo &DCI) const { 13683 SelectionDAG &DAG = DCI.DAG; 13684 SDLoc dl(N); 13685 switch (N->getOpcode()) { 13686 default: break; 13687 case ISD::ADD: 13688 return combineADD(N, DCI); 13689 case ISD::SHL: 13690 return combineSHL(N, DCI); 13691 case ISD::SRA: 13692 return combineSRA(N, DCI); 13693 case ISD::SRL: 13694 return combineSRL(N, DCI); 13695 case ISD::MUL: 13696 return combineMUL(N, DCI); 13697 case PPCISD::SHL: 13698 if (isNullConstant(N->getOperand(0))) // 0 << V -> 0. 13699 return N->getOperand(0); 13700 break; 13701 case PPCISD::SRL: 13702 if (isNullConstant(N->getOperand(0))) // 0 >>u V -> 0. 13703 return N->getOperand(0); 13704 break; 13705 case PPCISD::SRA: 13706 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 13707 if (C->isNullValue() || // 0 >>s V -> 0. 13708 C->isAllOnesValue()) // -1 >>s V -> -1. 13709 return N->getOperand(0); 13710 } 13711 break; 13712 case ISD::SIGN_EXTEND: 13713 case ISD::ZERO_EXTEND: 13714 case ISD::ANY_EXTEND: 13715 return DAGCombineExtBoolTrunc(N, DCI); 13716 case ISD::TRUNCATE: 13717 return combineTRUNCATE(N, DCI); 13718 case ISD::SETCC: 13719 if (SDValue CSCC = combineSetCC(N, DCI)) 13720 return CSCC; 13721 LLVM_FALLTHROUGH; 13722 case ISD::SELECT_CC: 13723 return DAGCombineTruncBoolExt(N, DCI); 13724 case ISD::SINT_TO_FP: 13725 case ISD::UINT_TO_FP: 13726 return combineFPToIntToFP(N, DCI); 13727 case ISD::VECTOR_SHUFFLE: 13728 if (ISD::isNormalLoad(N->getOperand(0).getNode())) { 13729 LSBaseSDNode* LSBase = cast<LSBaseSDNode>(N->getOperand(0)); 13730 return combineVReverseMemOP(cast<ShuffleVectorSDNode>(N), LSBase, DCI); 13731 } 13732 break; 13733 case ISD::STORE: { 13734 13735 EVT Op1VT = N->getOperand(1).getValueType(); 13736 unsigned Opcode = N->getOperand(1).getOpcode(); 13737 13738 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) { 13739 SDValue Val= combineStoreFPToInt(N, DCI); 13740 if (Val) 13741 return Val; 13742 } 13743 13744 if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) { 13745 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N->getOperand(1)); 13746 SDValue Val= combineVReverseMemOP(SVN, cast<LSBaseSDNode>(N), DCI); 13747 if (Val) 13748 return Val; 13749 } 13750 13751 // Turn STORE (BSWAP) -> sthbrx/stwbrx. 13752 if (cast<StoreSDNode>(N)->isUnindexed() && Opcode == ISD::BSWAP && 13753 N->getOperand(1).getNode()->hasOneUse() && 13754 (Op1VT == MVT::i32 || Op1VT == MVT::i16 || 13755 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) { 13756 13757 // STBRX can only handle simple types and it makes no sense to store less 13758 // two bytes in byte-reversed order. 13759 EVT mVT = cast<StoreSDNode>(N)->getMemoryVT(); 13760 if (mVT.isExtended() || mVT.getSizeInBits() < 16) 13761 break; 13762 13763 SDValue BSwapOp = N->getOperand(1).getOperand(0); 13764 // Do an any-extend to 32-bits if this is a half-word input. 13765 if (BSwapOp.getValueType() == MVT::i16) 13766 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp); 13767 13768 // If the type of BSWAP operand is wider than stored memory width 13769 // it need to be shifted to the right side before STBRX. 13770 if (Op1VT.bitsGT(mVT)) { 13771 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits(); 13772 BSwapOp = DAG.getNode(ISD::SRL, dl, Op1VT, BSwapOp, 13773 DAG.getConstant(Shift, dl, MVT::i32)); 13774 // Need to truncate if this is a bswap of i64 stored as i32/i16. 13775 if (Op1VT == MVT::i64) 13776 BSwapOp = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BSwapOp); 13777 } 13778 13779 SDValue Ops[] = { 13780 N->getOperand(0), BSwapOp, N->getOperand(2), DAG.getValueType(mVT) 13781 }; 13782 return 13783 DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other), 13784 Ops, cast<StoreSDNode>(N)->getMemoryVT(), 13785 cast<StoreSDNode>(N)->getMemOperand()); 13786 } 13787 13788 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0> 13789 // So it can increase the chance of CSE constant construction. 13790 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() && 13791 isa<ConstantSDNode>(N->getOperand(1)) && Op1VT == MVT::i32) { 13792 // Need to sign-extended to 64-bits to handle negative values. 13793 EVT MemVT = cast<StoreSDNode>(N)->getMemoryVT(); 13794 uint64_t Val64 = SignExtend64(N->getConstantOperandVal(1), 13795 MemVT.getSizeInBits()); 13796 SDValue Const64 = DAG.getConstant(Val64, dl, MVT::i64); 13797 13798 // DAG.getTruncStore() can't be used here because it doesn't accept 13799 // the general (base + offset) addressing mode. 13800 // So we use UpdateNodeOperands and setTruncatingStore instead. 13801 DAG.UpdateNodeOperands(N, N->getOperand(0), Const64, N->getOperand(2), 13802 N->getOperand(3)); 13803 cast<StoreSDNode>(N)->setTruncatingStore(true); 13804 return SDValue(N, 0); 13805 } 13806 13807 // For little endian, VSX stores require generating xxswapd/lxvd2x. 13808 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. 13809 if (Op1VT.isSimple()) { 13810 MVT StoreVT = Op1VT.getSimpleVT(); 13811 if (Subtarget.needsSwapsForVSXMemOps() && 13812 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 || 13813 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32)) 13814 return expandVSXStoreForLE(N, DCI); 13815 } 13816 break; 13817 } 13818 case ISD::LOAD: { 13819 LoadSDNode *LD = cast<LoadSDNode>(N); 13820 EVT VT = LD->getValueType(0); 13821 13822 // For little endian, VSX loads require generating lxvd2x/xxswapd. 13823 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. 13824 if (VT.isSimple()) { 13825 MVT LoadVT = VT.getSimpleVT(); 13826 if (Subtarget.needsSwapsForVSXMemOps() && 13827 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 || 13828 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32)) 13829 return expandVSXLoadForLE(N, DCI); 13830 } 13831 13832 // We sometimes end up with a 64-bit integer load, from which we extract 13833 // two single-precision floating-point numbers. This happens with 13834 // std::complex<float>, and other similar structures, because of the way we 13835 // canonicalize structure copies. However, if we lack direct moves, 13836 // then the final bitcasts from the extracted integer values to the 13837 // floating-point numbers turn into store/load pairs. Even with direct moves, 13838 // just loading the two floating-point numbers is likely better. 13839 auto ReplaceTwoFloatLoad = [&]() { 13840 if (VT != MVT::i64) 13841 return false; 13842 13843 if (LD->getExtensionType() != ISD::NON_EXTLOAD || 13844 LD->isVolatile()) 13845 return false; 13846 13847 // We're looking for a sequence like this: 13848 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64 13849 // t16: i64 = srl t13, Constant:i32<32> 13850 // t17: i32 = truncate t16 13851 // t18: f32 = bitcast t17 13852 // t19: i32 = truncate t13 13853 // t20: f32 = bitcast t19 13854 13855 if (!LD->hasNUsesOfValue(2, 0)) 13856 return false; 13857 13858 auto UI = LD->use_begin(); 13859 while (UI.getUse().getResNo() != 0) ++UI; 13860 SDNode *Trunc = *UI++; 13861 while (UI.getUse().getResNo() != 0) ++UI; 13862 SDNode *RightShift = *UI; 13863 if (Trunc->getOpcode() != ISD::TRUNCATE) 13864 std::swap(Trunc, RightShift); 13865 13866 if (Trunc->getOpcode() != ISD::TRUNCATE || 13867 Trunc->getValueType(0) != MVT::i32 || 13868 !Trunc->hasOneUse()) 13869 return false; 13870 if (RightShift->getOpcode() != ISD::SRL || 13871 !isa<ConstantSDNode>(RightShift->getOperand(1)) || 13872 RightShift->getConstantOperandVal(1) != 32 || 13873 !RightShift->hasOneUse()) 13874 return false; 13875 13876 SDNode *Trunc2 = *RightShift->use_begin(); 13877 if (Trunc2->getOpcode() != ISD::TRUNCATE || 13878 Trunc2->getValueType(0) != MVT::i32 || 13879 !Trunc2->hasOneUse()) 13880 return false; 13881 13882 SDNode *Bitcast = *Trunc->use_begin(); 13883 SDNode *Bitcast2 = *Trunc2->use_begin(); 13884 13885 if (Bitcast->getOpcode() != ISD::BITCAST || 13886 Bitcast->getValueType(0) != MVT::f32) 13887 return false; 13888 if (Bitcast2->getOpcode() != ISD::BITCAST || 13889 Bitcast2->getValueType(0) != MVT::f32) 13890 return false; 13891 13892 if (Subtarget.isLittleEndian()) 13893 std::swap(Bitcast, Bitcast2); 13894 13895 // Bitcast has the second float (in memory-layout order) and Bitcast2 13896 // has the first one. 13897 13898 SDValue BasePtr = LD->getBasePtr(); 13899 if (LD->isIndexed()) { 13900 assert(LD->getAddressingMode() == ISD::PRE_INC && 13901 "Non-pre-inc AM on PPC?"); 13902 BasePtr = 13903 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, 13904 LD->getOffset()); 13905 } 13906 13907 auto MMOFlags = 13908 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile; 13909 SDValue FloatLoad = DAG.getLoad(MVT::f32, dl, LD->getChain(), BasePtr, 13910 LD->getPointerInfo(), LD->getAlignment(), 13911 MMOFlags, LD->getAAInfo()); 13912 SDValue AddPtr = 13913 DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), 13914 BasePtr, DAG.getIntPtrConstant(4, dl)); 13915 SDValue FloatLoad2 = DAG.getLoad( 13916 MVT::f32, dl, SDValue(FloatLoad.getNode(), 1), AddPtr, 13917 LD->getPointerInfo().getWithOffset(4), 13918 MinAlign(LD->getAlignment(), 4), MMOFlags, LD->getAAInfo()); 13919 13920 if (LD->isIndexed()) { 13921 // Note that DAGCombine should re-form any pre-increment load(s) from 13922 // what is produced here if that makes sense. 13923 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), BasePtr); 13924 } 13925 13926 DCI.CombineTo(Bitcast2, FloatLoad); 13927 DCI.CombineTo(Bitcast, FloatLoad2); 13928 13929 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, LD->isIndexed() ? 2 : 1), 13930 SDValue(FloatLoad2.getNode(), 1)); 13931 return true; 13932 }; 13933 13934 if (ReplaceTwoFloatLoad()) 13935 return SDValue(N, 0); 13936 13937 EVT MemVT = LD->getMemoryVT(); 13938 Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); 13939 unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty); 13940 Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext()); 13941 unsigned ScalarABIAlignment = DAG.getDataLayout().getABITypeAlignment(STy); 13942 if (LD->isUnindexed() && VT.isVector() && 13943 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) && 13944 // P8 and later hardware should just use LOAD. 13945 !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 || 13946 VT == MVT::v4i32 || VT == MVT::v4f32)) || 13947 (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) && 13948 LD->getAlignment() >= ScalarABIAlignment)) && 13949 LD->getAlignment() < ABIAlignment) { 13950 // This is a type-legal unaligned Altivec or QPX load. 13951 SDValue Chain = LD->getChain(); 13952 SDValue Ptr = LD->getBasePtr(); 13953 bool isLittleEndian = Subtarget.isLittleEndian(); 13954 13955 // This implements the loading of unaligned vectors as described in 13956 // the venerable Apple Velocity Engine overview. Specifically: 13957 // https://developer.apple.com/hardwaredrivers/ve/alignment.html 13958 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html 13959 // 13960 // The general idea is to expand a sequence of one or more unaligned 13961 // loads into an alignment-based permutation-control instruction (lvsl 13962 // or lvsr), a series of regular vector loads (which always truncate 13963 // their input address to an aligned address), and a series of 13964 // permutations. The results of these permutations are the requested 13965 // loaded values. The trick is that the last "extra" load is not taken 13966 // from the address you might suspect (sizeof(vector) bytes after the 13967 // last requested load), but rather sizeof(vector) - 1 bytes after the 13968 // last requested vector. The point of this is to avoid a page fault if 13969 // the base address happened to be aligned. This works because if the 13970 // base address is aligned, then adding less than a full vector length 13971 // will cause the last vector in the sequence to be (re)loaded. 13972 // Otherwise, the next vector will be fetched as you might suspect was 13973 // necessary. 13974 13975 // We might be able to reuse the permutation generation from 13976 // a different base address offset from this one by an aligned amount. 13977 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this 13978 // optimization later. 13979 Intrinsic::ID Intr, IntrLD, IntrPerm; 13980 MVT PermCntlTy, PermTy, LDTy; 13981 if (Subtarget.hasAltivec()) { 13982 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr : 13983 Intrinsic::ppc_altivec_lvsl; 13984 IntrLD = Intrinsic::ppc_altivec_lvx; 13985 IntrPerm = Intrinsic::ppc_altivec_vperm; 13986 PermCntlTy = MVT::v16i8; 13987 PermTy = MVT::v4i32; 13988 LDTy = MVT::v4i32; 13989 } else { 13990 Intr = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld : 13991 Intrinsic::ppc_qpx_qvlpcls; 13992 IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd : 13993 Intrinsic::ppc_qpx_qvlfs; 13994 IntrPerm = Intrinsic::ppc_qpx_qvfperm; 13995 PermCntlTy = MVT::v4f64; 13996 PermTy = MVT::v4f64; 13997 LDTy = MemVT.getSimpleVT(); 13998 } 13999 14000 SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy); 14001 14002 // Create the new MMO for the new base load. It is like the original MMO, 14003 // but represents an area in memory almost twice the vector size centered 14004 // on the original address. If the address is unaligned, we might start 14005 // reading up to (sizeof(vector)-1) bytes below the address of the 14006 // original unaligned load. 14007 MachineFunction &MF = DAG.getMachineFunction(); 14008 MachineMemOperand *BaseMMO = 14009 MF.getMachineMemOperand(LD->getMemOperand(), 14010 -(long)MemVT.getStoreSize()+1, 14011 2*MemVT.getStoreSize()-1); 14012 14013 // Create the new base load. 14014 SDValue LDXIntID = 14015 DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout())); 14016 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr }; 14017 SDValue BaseLoad = 14018 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 14019 DAG.getVTList(PermTy, MVT::Other), 14020 BaseLoadOps, LDTy, BaseMMO); 14021 14022 // Note that the value of IncOffset (which is provided to the next 14023 // load's pointer info offset value, and thus used to calculate the 14024 // alignment), and the value of IncValue (which is actually used to 14025 // increment the pointer value) are different! This is because we 14026 // require the next load to appear to be aligned, even though it 14027 // is actually offset from the base pointer by a lesser amount. 14028 int IncOffset = VT.getSizeInBits() / 8; 14029 int IncValue = IncOffset; 14030 14031 // Walk (both up and down) the chain looking for another load at the real 14032 // (aligned) offset (the alignment of the other load does not matter in 14033 // this case). If found, then do not use the offset reduction trick, as 14034 // that will prevent the loads from being later combined (as they would 14035 // otherwise be duplicates). 14036 if (!findConsecutiveLoad(LD, DAG)) 14037 --IncValue; 14038 14039 SDValue Increment = 14040 DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout())); 14041 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); 14042 14043 MachineMemOperand *ExtraMMO = 14044 MF.getMachineMemOperand(LD->getMemOperand(), 14045 1, 2*MemVT.getStoreSize()-1); 14046 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr }; 14047 SDValue ExtraLoad = 14048 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, 14049 DAG.getVTList(PermTy, MVT::Other), 14050 ExtraLoadOps, LDTy, ExtraMMO); 14051 14052 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 14053 BaseLoad.getValue(1), ExtraLoad.getValue(1)); 14054 14055 // Because vperm has a big-endian bias, we must reverse the order 14056 // of the input vectors and complement the permute control vector 14057 // when generating little endian code. We have already handled the 14058 // latter by using lvsr instead of lvsl, so just reverse BaseLoad 14059 // and ExtraLoad here. 14060 SDValue Perm; 14061 if (isLittleEndian) 14062 Perm = BuildIntrinsicOp(IntrPerm, 14063 ExtraLoad, BaseLoad, PermCntl, DAG, dl); 14064 else 14065 Perm = BuildIntrinsicOp(IntrPerm, 14066 BaseLoad, ExtraLoad, PermCntl, DAG, dl); 14067 14068 if (VT != PermTy) 14069 Perm = Subtarget.hasAltivec() ? 14070 DAG.getNode(ISD::BITCAST, dl, VT, Perm) : 14071 DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX 14072 DAG.getTargetConstant(1, dl, MVT::i64)); 14073 // second argument is 1 because this rounding 14074 // is always exact. 14075 14076 // The output of the permutation is our loaded result, the TokenFactor is 14077 // our new chain. 14078 DCI.CombineTo(N, Perm, TF); 14079 return SDValue(N, 0); 14080 } 14081 } 14082 break; 14083 case ISD::INTRINSIC_WO_CHAIN: { 14084 bool isLittleEndian = Subtarget.isLittleEndian(); 14085 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 14086 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr 14087 : Intrinsic::ppc_altivec_lvsl); 14088 if ((IID == Intr || 14089 IID == Intrinsic::ppc_qpx_qvlpcld || 14090 IID == Intrinsic::ppc_qpx_qvlpcls) && 14091 N->getOperand(1)->getOpcode() == ISD::ADD) { 14092 SDValue Add = N->getOperand(1); 14093 14094 int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ? 14095 5 /* 32 byte alignment */ : 4 /* 16 byte alignment */; 14096 14097 if (DAG.MaskedValueIsZero(Add->getOperand(1), 14098 APInt::getAllOnesValue(Bits /* alignment */) 14099 .zext(Add.getScalarValueSizeInBits()))) { 14100 SDNode *BasePtr = Add->getOperand(0).getNode(); 14101 for (SDNode::use_iterator UI = BasePtr->use_begin(), 14102 UE = BasePtr->use_end(); 14103 UI != UE; ++UI) { 14104 if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 14105 cast<ConstantSDNode>(UI->getOperand(0))->getZExtValue() == IID) { 14106 // We've found another LVSL/LVSR, and this address is an aligned 14107 // multiple of that one. The results will be the same, so use the 14108 // one we've just found instead. 14109 14110 return SDValue(*UI, 0); 14111 } 14112 } 14113 } 14114 14115 if (isa<ConstantSDNode>(Add->getOperand(1))) { 14116 SDNode *BasePtr = Add->getOperand(0).getNode(); 14117 for (SDNode::use_iterator UI = BasePtr->use_begin(), 14118 UE = BasePtr->use_end(); UI != UE; ++UI) { 14119 if (UI->getOpcode() == ISD::ADD && 14120 isa<ConstantSDNode>(UI->getOperand(1)) && 14121 (cast<ConstantSDNode>(Add->getOperand(1))->getZExtValue() - 14122 cast<ConstantSDNode>(UI->getOperand(1))->getZExtValue()) % 14123 (1ULL << Bits) == 0) { 14124 SDNode *OtherAdd = *UI; 14125 for (SDNode::use_iterator VI = OtherAdd->use_begin(), 14126 VE = OtherAdd->use_end(); VI != VE; ++VI) { 14127 if (VI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && 14128 cast<ConstantSDNode>(VI->getOperand(0))->getZExtValue() == IID) { 14129 return SDValue(*VI, 0); 14130 } 14131 } 14132 } 14133 } 14134 } 14135 } 14136 14137 // Combine vmaxsw/h/b(a, a's negation) to abs(a) 14138 // Expose the vabsduw/h/b opportunity for down stream 14139 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() && 14140 (IID == Intrinsic::ppc_altivec_vmaxsw || 14141 IID == Intrinsic::ppc_altivec_vmaxsh || 14142 IID == Intrinsic::ppc_altivec_vmaxsb)) { 14143 SDValue V1 = N->getOperand(1); 14144 SDValue V2 = N->getOperand(2); 14145 if ((V1.getSimpleValueType() == MVT::v4i32 || 14146 V1.getSimpleValueType() == MVT::v8i16 || 14147 V1.getSimpleValueType() == MVT::v16i8) && 14148 V1.getSimpleValueType() == V2.getSimpleValueType()) { 14149 // (0-a, a) 14150 if (V1.getOpcode() == ISD::SUB && 14151 ISD::isBuildVectorAllZeros(V1.getOperand(0).getNode()) && 14152 V1.getOperand(1) == V2) { 14153 return DAG.getNode(ISD::ABS, dl, V2.getValueType(), V2); 14154 } 14155 // (a, 0-a) 14156 if (V2.getOpcode() == ISD::SUB && 14157 ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()) && 14158 V2.getOperand(1) == V1) { 14159 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); 14160 } 14161 // (x-y, y-x) 14162 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB && 14163 V1.getOperand(0) == V2.getOperand(1) && 14164 V1.getOperand(1) == V2.getOperand(0)) { 14165 return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); 14166 } 14167 } 14168 } 14169 } 14170 14171 break; 14172 case ISD::INTRINSIC_W_CHAIN: 14173 // For little endian, VSX loads require generating lxvd2x/xxswapd. 14174 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. 14175 if (Subtarget.needsSwapsForVSXMemOps()) { 14176 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 14177 default: 14178 break; 14179 case Intrinsic::ppc_vsx_lxvw4x: 14180 case Intrinsic::ppc_vsx_lxvd2x: 14181 return expandVSXLoadForLE(N, DCI); 14182 } 14183 } 14184 break; 14185 case ISD::INTRINSIC_VOID: 14186 // For little endian, VSX stores require generating xxswapd/stxvd2x. 14187 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. 14188 if (Subtarget.needsSwapsForVSXMemOps()) { 14189 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { 14190 default: 14191 break; 14192 case Intrinsic::ppc_vsx_stxvw4x: 14193 case Intrinsic::ppc_vsx_stxvd2x: 14194 return expandVSXStoreForLE(N, DCI); 14195 } 14196 } 14197 break; 14198 case ISD::BSWAP: 14199 // Turn BSWAP (LOAD) -> lhbrx/lwbrx. 14200 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && 14201 N->getOperand(0).hasOneUse() && 14202 (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || 14203 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && 14204 N->getValueType(0) == MVT::i64))) { 14205 SDValue Load = N->getOperand(0); 14206 LoadSDNode *LD = cast<LoadSDNode>(Load); 14207 // Create the byte-swapping load. 14208 SDValue Ops[] = { 14209 LD->getChain(), // Chain 14210 LD->getBasePtr(), // Ptr 14211 DAG.getValueType(N->getValueType(0)) // VT 14212 }; 14213 SDValue BSLoad = 14214 DAG.getMemIntrinsicNode(PPCISD::LBRX, dl, 14215 DAG.getVTList(N->getValueType(0) == MVT::i64 ? 14216 MVT::i64 : MVT::i32, MVT::Other), 14217 Ops, LD->getMemoryVT(), LD->getMemOperand()); 14218 14219 // If this is an i16 load, insert the truncate. 14220 SDValue ResVal = BSLoad; 14221 if (N->getValueType(0) == MVT::i16) 14222 ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad); 14223 14224 // First, combine the bswap away. This makes the value produced by the 14225 // load dead. 14226 DCI.CombineTo(N, ResVal); 14227 14228 // Next, combine the load away, we give it a bogus result value but a real 14229 // chain result. The result value is dead because the bswap is dead. 14230 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); 14231 14232 // Return N so it doesn't get rechecked! 14233 return SDValue(N, 0); 14234 } 14235 break; 14236 case PPCISD::VCMP: 14237 // If a VCMPo node already exists with exactly the same operands as this 14238 // node, use its result instead of this node (VCMPo computes both a CR6 and 14239 // a normal output). 14240 // 14241 if (!N->getOperand(0).hasOneUse() && 14242 !N->getOperand(1).hasOneUse() && 14243 !N->getOperand(2).hasOneUse()) { 14244 14245 // Scan all of the users of the LHS, looking for VCMPo's that match. 14246 SDNode *VCMPoNode = nullptr; 14247 14248 SDNode *LHSN = N->getOperand(0).getNode(); 14249 for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); 14250 UI != E; ++UI) 14251 if (UI->getOpcode() == PPCISD::VCMPo && 14252 UI->getOperand(1) == N->getOperand(1) && 14253 UI->getOperand(2) == N->getOperand(2) && 14254 UI->getOperand(0) == N->getOperand(0)) { 14255 VCMPoNode = *UI; 14256 break; 14257 } 14258 14259 // If there is no VCMPo node, or if the flag value has a single use, don't 14260 // transform this. 14261 if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1)) 14262 break; 14263 14264 // Look at the (necessarily single) use of the flag value. If it has a 14265 // chain, this transformation is more complex. Note that multiple things 14266 // could use the value result, which we should ignore. 14267 SDNode *FlagUser = nullptr; 14268 for (SDNode::use_iterator UI = VCMPoNode->use_begin(); 14269 FlagUser == nullptr; ++UI) { 14270 assert(UI != VCMPoNode->use_end() && "Didn't find user!"); 14271 SDNode *User = *UI; 14272 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { 14273 if (User->getOperand(i) == SDValue(VCMPoNode, 1)) { 14274 FlagUser = User; 14275 break; 14276 } 14277 } 14278 } 14279 14280 // If the user is a MFOCRF instruction, we know this is safe. 14281 // Otherwise we give up for right now. 14282 if (FlagUser->getOpcode() == PPCISD::MFOCRF) 14283 return SDValue(VCMPoNode, 0); 14284 } 14285 break; 14286 case ISD::BRCOND: { 14287 SDValue Cond = N->getOperand(1); 14288 SDValue Target = N->getOperand(2); 14289 14290 if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN && 14291 cast<ConstantSDNode>(Cond.getOperand(1))->getZExtValue() == 14292 Intrinsic::loop_decrement) { 14293 14294 // We now need to make the intrinsic dead (it cannot be instruction 14295 // selected). 14296 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0)); 14297 assert(Cond.getNode()->hasOneUse() && 14298 "Counter decrement has more than one use"); 14299 14300 return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other, 14301 N->getOperand(0), Target); 14302 } 14303 } 14304 break; 14305 case ISD::BR_CC: { 14306 // If this is a branch on an altivec predicate comparison, lower this so 14307 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This 14308 // lowering is done pre-legalize, because the legalizer lowers the predicate 14309 // compare down to code that is difficult to reassemble. 14310 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 14311 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); 14312 14313 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero 14314 // value. If so, pass-through the AND to get to the intrinsic. 14315 if (LHS.getOpcode() == ISD::AND && 14316 LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN && 14317 cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1))->getZExtValue() == 14318 Intrinsic::loop_decrement && 14319 isa<ConstantSDNode>(LHS.getOperand(1)) && 14320 !isNullConstant(LHS.getOperand(1))) 14321 LHS = LHS.getOperand(0); 14322 14323 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN && 14324 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() == 14325 Intrinsic::loop_decrement && 14326 isa<ConstantSDNode>(RHS)) { 14327 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && 14328 "Counter decrement comparison is not EQ or NE"); 14329 14330 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 14331 bool isBDNZ = (CC == ISD::SETEQ && Val) || 14332 (CC == ISD::SETNE && !Val); 14333 14334 // We now need to make the intrinsic dead (it cannot be instruction 14335 // selected). 14336 DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0)); 14337 assert(LHS.getNode()->hasOneUse() && 14338 "Counter decrement has more than one use"); 14339 14340 return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other, 14341 N->getOperand(0), N->getOperand(4)); 14342 } 14343 14344 int CompareOpc; 14345 bool isDot; 14346 14347 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN && 14348 isa<ConstantSDNode>(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) && 14349 getVectorCompareInfo(LHS, CompareOpc, isDot, Subtarget)) { 14350 assert(isDot && "Can't compare against a vector result!"); 14351 14352 // If this is a comparison against something other than 0/1, then we know 14353 // that the condition is never/always true. 14354 unsigned Val = cast<ConstantSDNode>(RHS)->getZExtValue(); 14355 if (Val != 0 && Val != 1) { 14356 if (CC == ISD::SETEQ) // Cond never true, remove branch. 14357 return N->getOperand(0); 14358 // Always !=, turn it into an unconditional branch. 14359 return DAG.getNode(ISD::BR, dl, MVT::Other, 14360 N->getOperand(0), N->getOperand(4)); 14361 } 14362 14363 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0); 14364 14365 // Create the PPCISD altivec 'dot' comparison node. 14366 SDValue Ops[] = { 14367 LHS.getOperand(2), // LHS of compare 14368 LHS.getOperand(3), // RHS of compare 14369 DAG.getConstant(CompareOpc, dl, MVT::i32) 14370 }; 14371 EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; 14372 SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); 14373 14374 // Unpack the result based on how the target uses it. 14375 PPC::Predicate CompOpc; 14376 switch (cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue()) { 14377 default: // Can't happen, don't crash on invalid number though. 14378 case 0: // Branch on the value of the EQ bit of CR6. 14379 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE; 14380 break; 14381 case 1: // Branch on the inverted value of the EQ bit of CR6. 14382 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ; 14383 break; 14384 case 2: // Branch on the value of the LT bit of CR6. 14385 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE; 14386 break; 14387 case 3: // Branch on the inverted value of the LT bit of CR6. 14388 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT; 14389 break; 14390 } 14391 14392 return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0), 14393 DAG.getConstant(CompOpc, dl, MVT::i32), 14394 DAG.getRegister(PPC::CR6, MVT::i32), 14395 N->getOperand(4), CompNode.getValue(1)); 14396 } 14397 break; 14398 } 14399 case ISD::BUILD_VECTOR: 14400 return DAGCombineBuildVector(N, DCI); 14401 case ISD::ABS: 14402 return combineABS(N, DCI); 14403 case ISD::VSELECT: 14404 return combineVSelect(N, DCI); 14405 } 14406 14407 return SDValue(); 14408} 14409 14410SDValue 14411PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, 14412 SelectionDAG &DAG, 14413 SmallVectorImpl<SDNode *> &Created) const { 14414 // fold (sdiv X, pow2) 14415 EVT VT = N->getValueType(0); 14416 if (VT == MVT::i64 && !Subtarget.isPPC64()) 14417 return SDValue(); 14418 if ((VT != MVT::i32 && VT != MVT::i64) || 14419 !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2())) 14420 return SDValue(); 14421 14422 SDLoc DL(N); 14423 SDValue N0 = N->getOperand(0); 14424 14425 bool IsNegPow2 = (-Divisor).isPowerOf2(); 14426 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros(); 14427 SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT); 14428 14429 SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt); 14430 Created.push_back(Op.getNode()); 14431 14432 if (IsNegPow2) { 14433 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); 14434 Created.push_back(Op.getNode()); 14435 } 14436 14437 return Op; 14438} 14439 14440//===----------------------------------------------------------------------===// 14441// Inline Assembly Support 14442//===----------------------------------------------------------------------===// 14443 14444void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, 14445 KnownBits &Known, 14446 const APInt &DemandedElts, 14447 const SelectionDAG &DAG, 14448 unsigned Depth) const { 14449 Known.resetAll(); 14450 switch (Op.getOpcode()) { 14451 default: break; 14452 case PPCISD::LBRX: { 14453 // lhbrx is known to have the top bits cleared out. 14454 if (cast<VTSDNode>(Op.getOperand(2))->getVT() == MVT::i16) 14455 Known.Zero = 0xFFFF0000; 14456 break; 14457 } 14458 case ISD::INTRINSIC_WO_CHAIN: { 14459 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) { 14460 default: break; 14461 case Intrinsic::ppc_altivec_vcmpbfp_p: 14462 case Intrinsic::ppc_altivec_vcmpeqfp_p: 14463 case Intrinsic::ppc_altivec_vcmpequb_p: 14464 case Intrinsic::ppc_altivec_vcmpequh_p: 14465 case Intrinsic::ppc_altivec_vcmpequw_p: 14466 case Intrinsic::ppc_altivec_vcmpequd_p: 14467 case Intrinsic::ppc_altivec_vcmpgefp_p: 14468 case Intrinsic::ppc_altivec_vcmpgtfp_p: 14469 case Intrinsic::ppc_altivec_vcmpgtsb_p: 14470 case Intrinsic::ppc_altivec_vcmpgtsh_p: 14471 case Intrinsic::ppc_altivec_vcmpgtsw_p: 14472 case Intrinsic::ppc_altivec_vcmpgtsd_p: 14473 case Intrinsic::ppc_altivec_vcmpgtub_p: 14474 case Intrinsic::ppc_altivec_vcmpgtuh_p: 14475 case Intrinsic::ppc_altivec_vcmpgtuw_p: 14476 case Intrinsic::ppc_altivec_vcmpgtud_p: 14477 Known.Zero = ~1U; // All bits but the low one are known to be zero. 14478 break; 14479 } 14480 } 14481 } 14482} 14483 14484Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { 14485 switch (Subtarget.getCPUDirective()) { 14486 default: break; 14487 case PPC::DIR_970: 14488 case PPC::DIR_PWR4: 14489 case PPC::DIR_PWR5: 14490 case PPC::DIR_PWR5X: 14491 case PPC::DIR_PWR6: 14492 case PPC::DIR_PWR6X: 14493 case PPC::DIR_PWR7: 14494 case PPC::DIR_PWR8: 14495 case PPC::DIR_PWR9: 14496 case PPC::DIR_PWR_FUTURE: { 14497 if (!ML) 14498 break; 14499 14500 if (!DisableInnermostLoopAlign32) { 14501 // If the nested loop is an innermost loop, prefer to a 32-byte alignment, 14502 // so that we can decrease cache misses and branch-prediction misses. 14503 // Actual alignment of the loop will depend on the hotness check and other 14504 // logic in alignBlocks. 14505 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty()) 14506 return Align(32); 14507 } 14508 14509 const PPCInstrInfo *TII = Subtarget.getInstrInfo(); 14510 14511 // For small loops (between 5 and 8 instructions), align to a 32-byte 14512 // boundary so that the entire loop fits in one instruction-cache line. 14513 uint64_t LoopSize = 0; 14514 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I) 14515 for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) { 14516 LoopSize += TII->getInstSizeInBytes(*J); 14517 if (LoopSize > 32) 14518 break; 14519 } 14520 14521 if (LoopSize > 16 && LoopSize <= 32) 14522 return Align(32); 14523 14524 break; 14525 } 14526 } 14527 14528 return TargetLowering::getPrefLoopAlignment(ML); 14529} 14530 14531/// getConstraintType - Given a constraint, return the type of 14532/// constraint it is for this target. 14533PPCTargetLowering::ConstraintType 14534PPCTargetLowering::getConstraintType(StringRef Constraint) const { 14535 if (Constraint.size() == 1) { 14536 switch (Constraint[0]) { 14537 default: break; 14538 case 'b': 14539 case 'r': 14540 case 'f': 14541 case 'd': 14542 case 'v': 14543 case 'y': 14544 return C_RegisterClass; 14545 case 'Z': 14546 // FIXME: While Z does indicate a memory constraint, it specifically 14547 // indicates an r+r address (used in conjunction with the 'y' modifier 14548 // in the replacement string). Currently, we're forcing the base 14549 // register to be r0 in the asm printer (which is interpreted as zero) 14550 // and forming the complete address in the second register. This is 14551 // suboptimal. 14552 return C_Memory; 14553 } 14554 } else if (Constraint == "wc") { // individual CR bits. 14555 return C_RegisterClass; 14556 } else if (Constraint == "wa" || Constraint == "wd" || 14557 Constraint == "wf" || Constraint == "ws" || 14558 Constraint == "wi" || Constraint == "ww") { 14559 return C_RegisterClass; // VSX registers. 14560 } 14561 return TargetLowering::getConstraintType(Constraint); 14562} 14563 14564/// Examine constraint type and operand type and determine a weight value. 14565/// This object must already have been set up with the operand type 14566/// and the current alternative constraint selected. 14567TargetLowering::ConstraintWeight 14568PPCTargetLowering::getSingleConstraintMatchWeight( 14569 AsmOperandInfo &info, const char *constraint) const { 14570 ConstraintWeight weight = CW_Invalid; 14571 Value *CallOperandVal = info.CallOperandVal; 14572 // If we don't have a value, we can't do a match, 14573 // but allow it at the lowest weight. 14574 if (!CallOperandVal) 14575 return CW_Default; 14576 Type *type = CallOperandVal->getType(); 14577 14578 // Look at the constraint type. 14579 if (StringRef(constraint) == "wc" && type->isIntegerTy(1)) 14580 return CW_Register; // an individual CR bit. 14581 else if ((StringRef(constraint) == "wa" || 14582 StringRef(constraint) == "wd" || 14583 StringRef(constraint) == "wf") && 14584 type->isVectorTy()) 14585 return CW_Register; 14586 else if (StringRef(constraint) == "wi" && type->isIntegerTy(64)) 14587 return CW_Register; // just hold 64-bit integers data. 14588 else if (StringRef(constraint) == "ws" && type->isDoubleTy()) 14589 return CW_Register; 14590 else if (StringRef(constraint) == "ww" && type->isFloatTy()) 14591 return CW_Register; 14592 14593 switch (*constraint) { 14594 default: 14595 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 14596 break; 14597 case 'b': 14598 if (type->isIntegerTy()) 14599 weight = CW_Register; 14600 break; 14601 case 'f': 14602 if (type->isFloatTy()) 14603 weight = CW_Register; 14604 break; 14605 case 'd': 14606 if (type->isDoubleTy()) 14607 weight = CW_Register; 14608 break; 14609 case 'v': 14610 if (type->isVectorTy()) 14611 weight = CW_Register; 14612 break; 14613 case 'y': 14614 weight = CW_Register; 14615 break; 14616 case 'Z': 14617 weight = CW_Memory; 14618 break; 14619 } 14620 return weight; 14621} 14622 14623std::pair<unsigned, const TargetRegisterClass *> 14624PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 14625 StringRef Constraint, 14626 MVT VT) const { 14627 if (Constraint.size() == 1) { 14628 // GCC RS6000 Constraint Letters 14629 switch (Constraint[0]) { 14630 case 'b': // R1-R31 14631 if (VT == MVT::i64 && Subtarget.isPPC64()) 14632 return std::make_pair(0U, &PPC::G8RC_NOX0RegClass); 14633 return std::make_pair(0U, &PPC::GPRC_NOR0RegClass); 14634 case 'r': // R0-R31 14635 if (VT == MVT::i64 && Subtarget.isPPC64()) 14636 return std::make_pair(0U, &PPC::G8RCRegClass); 14637 return std::make_pair(0U, &PPC::GPRCRegClass); 14638 // 'd' and 'f' constraints are both defined to be "the floating point 14639 // registers", where one is for 32-bit and the other for 64-bit. We don't 14640 // really care overly much here so just give them all the same reg classes. 14641 case 'd': 14642 case 'f': 14643 if (Subtarget.hasSPE()) { 14644 if (VT == MVT::f32 || VT == MVT::i32) 14645 return std::make_pair(0U, &PPC::GPRCRegClass); 14646 if (VT == MVT::f64 || VT == MVT::i64) 14647 return std::make_pair(0U, &PPC::SPERCRegClass); 14648 } else { 14649 if (VT == MVT::f32 || VT == MVT::i32) 14650 return std::make_pair(0U, &PPC::F4RCRegClass); 14651 if (VT == MVT::f64 || VT == MVT::i64) 14652 return std::make_pair(0U, &PPC::F8RCRegClass); 14653 if (VT == MVT::v4f64 && Subtarget.hasQPX()) 14654 return std::make_pair(0U, &PPC::QFRCRegClass); 14655 if (VT == MVT::v4f32 && Subtarget.hasQPX()) 14656 return std::make_pair(0U, &PPC::QSRCRegClass); 14657 } 14658 break; 14659 case 'v': 14660 if (VT == MVT::v4f64 && Subtarget.hasQPX()) 14661 return std::make_pair(0U, &PPC::QFRCRegClass); 14662 if (VT == MVT::v4f32 && Subtarget.hasQPX()) 14663 return std::make_pair(0U, &PPC::QSRCRegClass); 14664 if (Subtarget.hasAltivec()) 14665 return std::make_pair(0U, &PPC::VRRCRegClass); 14666 break; 14667 case 'y': // crrc 14668 return std::make_pair(0U, &PPC::CRRCRegClass); 14669 } 14670 } else if (Constraint == "wc" && Subtarget.useCRBits()) { 14671 // An individual CR bit. 14672 return std::make_pair(0U, &PPC::CRBITRCRegClass); 14673 } else if ((Constraint == "wa" || Constraint == "wd" || 14674 Constraint == "wf" || Constraint == "wi") && 14675 Subtarget.hasVSX()) { 14676 return std::make_pair(0U, &PPC::VSRCRegClass); 14677 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) { 14678 if (VT == MVT::f32 && Subtarget.hasP8Vector()) 14679 return std::make_pair(0U, &PPC::VSSRCRegClass); 14680 else 14681 return std::make_pair(0U, &PPC::VSFRCRegClass); 14682 } 14683 14684 // If we name a VSX register, we can't defer to the base class because it 14685 // will not recognize the correct register (their names will be VSL{0-31} 14686 // and V{0-31} so they won't match). So we match them here. 14687 if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') { 14688 int VSNum = atoi(Constraint.data() + 3); 14689 assert(VSNum >= 0 && VSNum <= 63 && 14690 "Attempted to access a vsr out of range"); 14691 if (VSNum < 32) 14692 return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass); 14693 return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass); 14694 } 14695 std::pair<unsigned, const TargetRegisterClass *> R = 14696 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 14697 14698 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers 14699 // (which we call X[0-9]+). If a 64-bit value has been requested, and a 14700 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent 14701 // register. 14702 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use 14703 // the AsmName field from *RegisterInfo.td, then this would not be necessary. 14704 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() && 14705 PPC::GPRCRegClass.contains(R.first)) 14706 return std::make_pair(TRI->getMatchingSuperReg(R.first, 14707 PPC::sub_32, &PPC::G8RCRegClass), 14708 &PPC::G8RCRegClass); 14709 14710 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same. 14711 if (!R.second && StringRef("{cc}").equals_lower(Constraint)) { 14712 R.first = PPC::CR0; 14713 R.second = &PPC::CRRCRegClass; 14714 } 14715 14716 return R; 14717} 14718 14719/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops 14720/// vector. If it is invalid, don't add anything to Ops. 14721void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, 14722 std::string &Constraint, 14723 std::vector<SDValue>&Ops, 14724 SelectionDAG &DAG) const { 14725 SDValue Result; 14726 14727 // Only support length 1 constraints. 14728 if (Constraint.length() > 1) return; 14729 14730 char Letter = Constraint[0]; 14731 switch (Letter) { 14732 default: break; 14733 case 'I': 14734 case 'J': 14735 case 'K': 14736 case 'L': 14737 case 'M': 14738 case 'N': 14739 case 'O': 14740 case 'P': { 14741 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op); 14742 if (!CST) return; // Must be an immediate to match. 14743 SDLoc dl(Op); 14744 int64_t Value = CST->getSExtValue(); 14745 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative 14746 // numbers are printed as such. 14747 switch (Letter) { 14748 default: llvm_unreachable("Unknown constraint letter!"); 14749 case 'I': // "I" is a signed 16-bit constant. 14750 if (isInt<16>(Value)) 14751 Result = DAG.getTargetConstant(Value, dl, TCVT); 14752 break; 14753 case 'J': // "J" is a constant with only the high-order 16 bits nonzero. 14754 if (isShiftedUInt<16, 16>(Value)) 14755 Result = DAG.getTargetConstant(Value, dl, TCVT); 14756 break; 14757 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits. 14758 if (isShiftedInt<16, 16>(Value)) 14759 Result = DAG.getTargetConstant(Value, dl, TCVT); 14760 break; 14761 case 'K': // "K" is a constant with only the low-order 16 bits nonzero. 14762 if (isUInt<16>(Value)) 14763 Result = DAG.getTargetConstant(Value, dl, TCVT); 14764 break; 14765 case 'M': // "M" is a constant that is greater than 31. 14766 if (Value > 31) 14767 Result = DAG.getTargetConstant(Value, dl, TCVT); 14768 break; 14769 case 'N': // "N" is a positive constant that is an exact power of two. 14770 if (Value > 0 && isPowerOf2_64(Value)) 14771 Result = DAG.getTargetConstant(Value, dl, TCVT); 14772 break; 14773 case 'O': // "O" is the constant zero. 14774 if (Value == 0) 14775 Result = DAG.getTargetConstant(Value, dl, TCVT); 14776 break; 14777 case 'P': // "P" is a constant whose negation is a signed 16-bit constant. 14778 if (isInt<16>(-Value)) 14779 Result = DAG.getTargetConstant(Value, dl, TCVT); 14780 break; 14781 } 14782 break; 14783 } 14784 } 14785 14786 if (Result.getNode()) { 14787 Ops.push_back(Result); 14788 return; 14789 } 14790 14791 // Handle standard constraint letters. 14792 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 14793} 14794 14795// isLegalAddressingMode - Return true if the addressing mode represented 14796// by AM is legal for this target, for a load/store of the specified type. 14797bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL, 14798 const AddrMode &AM, Type *Ty, 14799 unsigned AS, Instruction *I) const { 14800 // PPC does not allow r+i addressing modes for vectors! 14801 if (Ty->isVectorTy() && AM.BaseOffs != 0) 14802 return false; 14803 14804 // PPC allows a sign-extended 16-bit immediate field. 14805 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) 14806 return false; 14807 14808 // No global is ever allowed as a base. 14809 if (AM.BaseGV) 14810 return false; 14811 14812 // PPC only support r+r, 14813 switch (AM.Scale) { 14814 case 0: // "r+i" or just "i", depending on HasBaseReg. 14815 break; 14816 case 1: 14817 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. 14818 return false; 14819 // Otherwise we have r+r or r+i. 14820 break; 14821 case 2: 14822 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. 14823 return false; 14824 // Allow 2*r as r+r. 14825 break; 14826 default: 14827 // No other scales are supported. 14828 return false; 14829 } 14830 14831 return true; 14832} 14833 14834SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op, 14835 SelectionDAG &DAG) const { 14836 MachineFunction &MF = DAG.getMachineFunction(); 14837 MachineFrameInfo &MFI = MF.getFrameInfo(); 14838 MFI.setReturnAddressIsTaken(true); 14839 14840 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 14841 return SDValue(); 14842 14843 SDLoc dl(Op); 14844 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 14845 14846 // Make sure the function does not optimize away the store of the RA to 14847 // the stack. 14848 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 14849 FuncInfo->setLRStoreRequired(); 14850 bool isPPC64 = Subtarget.isPPC64(); 14851 auto PtrVT = getPointerTy(MF.getDataLayout()); 14852 14853 if (Depth > 0) { 14854 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); 14855 SDValue Offset = 14856 DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl, 14857 isPPC64 ? MVT::i64 : MVT::i32); 14858 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), 14859 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), 14860 MachinePointerInfo()); 14861 } 14862 14863 // Just load the return address off the stack. 14864 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG); 14865 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI, 14866 MachinePointerInfo()); 14867} 14868 14869SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op, 14870 SelectionDAG &DAG) const { 14871 SDLoc dl(Op); 14872 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 14873 14874 MachineFunction &MF = DAG.getMachineFunction(); 14875 MachineFrameInfo &MFI = MF.getFrameInfo(); 14876 MFI.setFrameAddressIsTaken(true); 14877 14878 EVT PtrVT = getPointerTy(MF.getDataLayout()); 14879 bool isPPC64 = PtrVT == MVT::i64; 14880 14881 // Naked functions never have a frame pointer, and so we use r1. For all 14882 // other functions, this decision must be delayed until during PEI. 14883 unsigned FrameReg; 14884 if (MF.getFunction().hasFnAttribute(Attribute::Naked)) 14885 FrameReg = isPPC64 ? PPC::X1 : PPC::R1; 14886 else 14887 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP; 14888 14889 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, 14890 PtrVT); 14891 while (Depth--) 14892 FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), 14893 FrameAddr, MachinePointerInfo()); 14894 return FrameAddr; 14895} 14896 14897// FIXME? Maybe this could be a TableGen attribute on some registers and 14898// this table could be generated automatically from RegInfo. 14899Register PPCTargetLowering::getRegisterByName(const char* RegName, LLT VT, 14900 const MachineFunction &MF) const { 14901 bool isPPC64 = Subtarget.isPPC64(); 14902 bool IsDarwinABI = Subtarget.isDarwinABI(); 14903 14904 bool is64Bit = isPPC64 && VT == LLT::scalar(64); 14905 if (!is64Bit && VT != LLT::scalar(32)) 14906 report_fatal_error("Invalid register global variable type"); 14907 14908 Register Reg = StringSwitch<Register>(RegName) 14909 .Case("r1", is64Bit ? PPC::X1 : PPC::R1) 14910 .Case("r2", (IsDarwinABI || isPPC64) ? Register() : PPC::R2) 14911 .Case("r13", (!isPPC64 && IsDarwinABI) ? Register() : 14912 (is64Bit ? PPC::X13 : PPC::R13)) 14913 .Default(Register()); 14914 14915 if (Reg) 14916 return Reg; 14917 report_fatal_error("Invalid register name global variable"); 14918} 14919 14920bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const { 14921 // 32-bit SVR4 ABI access everything as got-indirect. 14922 if (Subtarget.is32BitELFABI()) 14923 return true; 14924 14925 // AIX accesses everything indirectly through the TOC, which is similar to 14926 // the GOT. 14927 if (Subtarget.isAIXABI()) 14928 return true; 14929 14930 CodeModel::Model CModel = getTargetMachine().getCodeModel(); 14931 // If it is small or large code model, module locals are accessed 14932 // indirectly by loading their address from .toc/.got. 14933 if (CModel == CodeModel::Small || CModel == CodeModel::Large) 14934 return true; 14935 14936 // JumpTable and BlockAddress are accessed as got-indirect. 14937 if (isa<JumpTableSDNode>(GA) || isa<BlockAddressSDNode>(GA)) 14938 return true; 14939 14940 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA)) 14941 return Subtarget.isGVIndirectSymbol(G->getGlobal()); 14942 14943 return false; 14944} 14945 14946bool 14947PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { 14948 // The PowerPC target isn't yet aware of offsets. 14949 return false; 14950} 14951 14952bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 14953 const CallInst &I, 14954 MachineFunction &MF, 14955 unsigned Intrinsic) const { 14956 switch (Intrinsic) { 14957 case Intrinsic::ppc_qpx_qvlfd: 14958 case Intrinsic::ppc_qpx_qvlfs: 14959 case Intrinsic::ppc_qpx_qvlfcd: 14960 case Intrinsic::ppc_qpx_qvlfcs: 14961 case Intrinsic::ppc_qpx_qvlfiwa: 14962 case Intrinsic::ppc_qpx_qvlfiwz: 14963 case Intrinsic::ppc_altivec_lvx: 14964 case Intrinsic::ppc_altivec_lvxl: 14965 case Intrinsic::ppc_altivec_lvebx: 14966 case Intrinsic::ppc_altivec_lvehx: 14967 case Intrinsic::ppc_altivec_lvewx: 14968 case Intrinsic::ppc_vsx_lxvd2x: 14969 case Intrinsic::ppc_vsx_lxvw4x: { 14970 EVT VT; 14971 switch (Intrinsic) { 14972 case Intrinsic::ppc_altivec_lvebx: 14973 VT = MVT::i8; 14974 break; 14975 case Intrinsic::ppc_altivec_lvehx: 14976 VT = MVT::i16; 14977 break; 14978 case Intrinsic::ppc_altivec_lvewx: 14979 VT = MVT::i32; 14980 break; 14981 case Intrinsic::ppc_vsx_lxvd2x: 14982 VT = MVT::v2f64; 14983 break; 14984 case Intrinsic::ppc_qpx_qvlfd: 14985 VT = MVT::v4f64; 14986 break; 14987 case Intrinsic::ppc_qpx_qvlfs: 14988 VT = MVT::v4f32; 14989 break; 14990 case Intrinsic::ppc_qpx_qvlfcd: 14991 VT = MVT::v2f64; 14992 break; 14993 case Intrinsic::ppc_qpx_qvlfcs: 14994 VT = MVT::v2f32; 14995 break; 14996 default: 14997 VT = MVT::v4i32; 14998 break; 14999 } 15000 15001 Info.opc = ISD::INTRINSIC_W_CHAIN; 15002 Info.memVT = VT; 15003 Info.ptrVal = I.getArgOperand(0); 15004 Info.offset = -VT.getStoreSize()+1; 15005 Info.size = 2*VT.getStoreSize()-1; 15006 Info.align = Align::None(); 15007 Info.flags = MachineMemOperand::MOLoad; 15008 return true; 15009 } 15010 case Intrinsic::ppc_qpx_qvlfda: 15011 case Intrinsic::ppc_qpx_qvlfsa: 15012 case Intrinsic::ppc_qpx_qvlfcda: 15013 case Intrinsic::ppc_qpx_qvlfcsa: 15014 case Intrinsic::ppc_qpx_qvlfiwaa: 15015 case Intrinsic::ppc_qpx_qvlfiwza: { 15016 EVT VT; 15017 switch (Intrinsic) { 15018 case Intrinsic::ppc_qpx_qvlfda: 15019 VT = MVT::v4f64; 15020 break; 15021 case Intrinsic::ppc_qpx_qvlfsa: 15022 VT = MVT::v4f32; 15023 break; 15024 case Intrinsic::ppc_qpx_qvlfcda: 15025 VT = MVT::v2f64; 15026 break; 15027 case Intrinsic::ppc_qpx_qvlfcsa: 15028 VT = MVT::v2f32; 15029 break; 15030 default: 15031 VT = MVT::v4i32; 15032 break; 15033 } 15034 15035 Info.opc = ISD::INTRINSIC_W_CHAIN; 15036 Info.memVT = VT; 15037 Info.ptrVal = I.getArgOperand(0); 15038 Info.offset = 0; 15039 Info.size = VT.getStoreSize(); 15040 Info.align = Align::None(); 15041 Info.flags = MachineMemOperand::MOLoad; 15042 return true; 15043 } 15044 case Intrinsic::ppc_qpx_qvstfd: 15045 case Intrinsic::ppc_qpx_qvstfs: 15046 case Intrinsic::ppc_qpx_qvstfcd: 15047 case Intrinsic::ppc_qpx_qvstfcs: 15048 case Intrinsic::ppc_qpx_qvstfiw: 15049 case Intrinsic::ppc_altivec_stvx: 15050 case Intrinsic::ppc_altivec_stvxl: 15051 case Intrinsic::ppc_altivec_stvebx: 15052 case Intrinsic::ppc_altivec_stvehx: 15053 case Intrinsic::ppc_altivec_stvewx: 15054 case Intrinsic::ppc_vsx_stxvd2x: 15055 case Intrinsic::ppc_vsx_stxvw4x: { 15056 EVT VT; 15057 switch (Intrinsic) { 15058 case Intrinsic::ppc_altivec_stvebx: 15059 VT = MVT::i8; 15060 break; 15061 case Intrinsic::ppc_altivec_stvehx: 15062 VT = MVT::i16; 15063 break; 15064 case Intrinsic::ppc_altivec_stvewx: 15065 VT = MVT::i32; 15066 break; 15067 case Intrinsic::ppc_vsx_stxvd2x: 15068 VT = MVT::v2f64; 15069 break; 15070 case Intrinsic::ppc_qpx_qvstfd: 15071 VT = MVT::v4f64; 15072 break; 15073 case Intrinsic::ppc_qpx_qvstfs: 15074 VT = MVT::v4f32; 15075 break; 15076 case Intrinsic::ppc_qpx_qvstfcd: 15077 VT = MVT::v2f64; 15078 break; 15079 case Intrinsic::ppc_qpx_qvstfcs: 15080 VT = MVT::v2f32; 15081 break; 15082 default: 15083 VT = MVT::v4i32; 15084 break; 15085 } 15086 15087 Info.opc = ISD::INTRINSIC_VOID; 15088 Info.memVT = VT; 15089 Info.ptrVal = I.getArgOperand(1); 15090 Info.offset = -VT.getStoreSize()+1; 15091 Info.size = 2*VT.getStoreSize()-1; 15092 Info.align = Align::None(); 15093 Info.flags = MachineMemOperand::MOStore; 15094 return true; 15095 } 15096 case Intrinsic::ppc_qpx_qvstfda: 15097 case Intrinsic::ppc_qpx_qvstfsa: 15098 case Intrinsic::ppc_qpx_qvstfcda: 15099 case Intrinsic::ppc_qpx_qvstfcsa: 15100 case Intrinsic::ppc_qpx_qvstfiwa: { 15101 EVT VT; 15102 switch (Intrinsic) { 15103 case Intrinsic::ppc_qpx_qvstfda: 15104 VT = MVT::v4f64; 15105 break; 15106 case Intrinsic::ppc_qpx_qvstfsa: 15107 VT = MVT::v4f32; 15108 break; 15109 case Intrinsic::ppc_qpx_qvstfcda: 15110 VT = MVT::v2f64; 15111 break; 15112 case Intrinsic::ppc_qpx_qvstfcsa: 15113 VT = MVT::v2f32; 15114 break; 15115 default: 15116 VT = MVT::v4i32; 15117 break; 15118 } 15119 15120 Info.opc = ISD::INTRINSIC_VOID; 15121 Info.memVT = VT; 15122 Info.ptrVal = I.getArgOperand(1); 15123 Info.offset = 0; 15124 Info.size = VT.getStoreSize(); 15125 Info.align = Align::None(); 15126 Info.flags = MachineMemOperand::MOStore; 15127 return true; 15128 } 15129 default: 15130 break; 15131 } 15132 15133 return false; 15134} 15135 15136/// getOptimalMemOpType - Returns the target specific optimal type for load 15137/// and store operations as a result of memset, memcpy, and memmove 15138/// lowering. If DstAlign is zero that means it's safe to destination 15139/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it 15140/// means there isn't a need to check it against alignment requirement, 15141/// probably because the source does not need to be loaded. If 'IsMemset' is 15142/// true, that means it's expanding a memset. If 'ZeroMemset' is true, that 15143/// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy 15144/// source is constant so it does not need to be loaded. 15145/// It returns EVT::Other if the type should be determined using generic 15146/// target-independent logic. 15147EVT PPCTargetLowering::getOptimalMemOpType( 15148 uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset, 15149 bool ZeroMemset, bool MemcpyStrSrc, 15150 const AttributeList &FuncAttributes) const { 15151 if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { 15152 // When expanding a memset, require at least two QPX instructions to cover 15153 // the cost of loading the value to be stored from the constant pool. 15154 if (Subtarget.hasQPX() && Size >= 32 && (!IsMemset || Size >= 64) && 15155 (!SrcAlign || SrcAlign >= 32) && (!DstAlign || DstAlign >= 32) && 15156 !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { 15157 return MVT::v4f64; 15158 } 15159 15160 // We should use Altivec/VSX loads and stores when available. For unaligned 15161 // addresses, unaligned VSX loads are only fast starting with the P8. 15162 if (Subtarget.hasAltivec() && Size >= 16 && 15163 (((!SrcAlign || SrcAlign >= 16) && (!DstAlign || DstAlign >= 16)) || 15164 ((IsMemset && Subtarget.hasVSX()) || Subtarget.hasP8Vector()))) 15165 return MVT::v4i32; 15166 } 15167 15168 if (Subtarget.isPPC64()) { 15169 return MVT::i64; 15170 } 15171 15172 return MVT::i32; 15173} 15174 15175/// Returns true if it is beneficial to convert a load of a constant 15176/// to just the constant itself. 15177bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, 15178 Type *Ty) const { 15179 assert(Ty->isIntegerTy()); 15180 15181 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 15182 return !(BitSize == 0 || BitSize > 64); 15183} 15184 15185bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { 15186 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) 15187 return false; 15188 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); 15189 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); 15190 return NumBits1 == 64 && NumBits2 == 32; 15191} 15192 15193bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { 15194 if (!VT1.isInteger() || !VT2.isInteger()) 15195 return false; 15196 unsigned NumBits1 = VT1.getSizeInBits(); 15197 unsigned NumBits2 = VT2.getSizeInBits(); 15198 return NumBits1 == 64 && NumBits2 == 32; 15199} 15200 15201bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { 15202 // Generally speaking, zexts are not free, but they are free when they can be 15203 // folded with other operations. 15204 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) { 15205 EVT MemVT = LD->getMemoryVT(); 15206 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 || 15207 (Subtarget.isPPC64() && MemVT == MVT::i32)) && 15208 (LD->getExtensionType() == ISD::NON_EXTLOAD || 15209 LD->getExtensionType() == ISD::ZEXTLOAD)) 15210 return true; 15211 } 15212 15213 // FIXME: Add other cases... 15214 // - 32-bit shifts with a zext to i64 15215 // - zext after ctlz, bswap, etc. 15216 // - zext after and by a constant mask 15217 15218 return TargetLowering::isZExtFree(Val, VT2); 15219} 15220 15221bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const { 15222 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && 15223 "invalid fpext types"); 15224 // Extending to float128 is not free. 15225 if (DestVT == MVT::f128) 15226 return false; 15227 return true; 15228} 15229 15230bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 15231 return isInt<16>(Imm) || isUInt<16>(Imm); 15232} 15233 15234bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { 15235 return isInt<16>(Imm) || isUInt<16>(Imm); 15236} 15237 15238bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 15239 unsigned, 15240 unsigned, 15241 MachineMemOperand::Flags, 15242 bool *Fast) const { 15243 if (DisablePPCUnaligned) 15244 return false; 15245 15246 // PowerPC supports unaligned memory access for simple non-vector types. 15247 // Although accessing unaligned addresses is not as efficient as accessing 15248 // aligned addresses, it is generally more efficient than manual expansion, 15249 // and generally only traps for software emulation when crossing page 15250 // boundaries. 15251 15252 if (!VT.isSimple()) 15253 return false; 15254 15255 if (VT.getSimpleVT().isVector()) { 15256 if (Subtarget.hasVSX()) { 15257 if (VT != MVT::v2f64 && VT != MVT::v2i64 && 15258 VT != MVT::v4f32 && VT != MVT::v4i32) 15259 return false; 15260 } else { 15261 return false; 15262 } 15263 } 15264 15265 if (VT == MVT::ppcf128) 15266 return false; 15267 15268 if (Fast) 15269 *Fast = true; 15270 15271 return true; 15272} 15273 15274bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, 15275 EVT VT) const { 15276 VT = VT.getScalarType(); 15277 15278 if (!VT.isSimple()) 15279 return false; 15280 15281 switch (VT.getSimpleVT().SimpleTy) { 15282 case MVT::f32: 15283 case MVT::f64: 15284 return true; 15285 case MVT::f128: 15286 return (EnableQuadPrecision && Subtarget.hasP9Vector()); 15287 default: 15288 break; 15289 } 15290 15291 return false; 15292} 15293 15294const MCPhysReg * 15295PPCTargetLowering::getScratchRegisters(CallingConv::ID) const { 15296 // LR is a callee-save register, but we must treat it as clobbered by any call 15297 // site. Hence we include LR in the scratch registers, which are in turn added 15298 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies 15299 // to CTR, which is used by any indirect call. 15300 static const MCPhysReg ScratchRegs[] = { 15301 PPC::X12, PPC::LR8, PPC::CTR8, 0 15302 }; 15303 15304 return ScratchRegs; 15305} 15306 15307unsigned PPCTargetLowering::getExceptionPointerRegister( 15308 const Constant *PersonalityFn) const { 15309 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3; 15310} 15311 15312unsigned PPCTargetLowering::getExceptionSelectorRegister( 15313 const Constant *PersonalityFn) const { 15314 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4; 15315} 15316 15317bool 15318PPCTargetLowering::shouldExpandBuildVectorWithShuffles( 15319 EVT VT , unsigned DefinedValues) const { 15320 if (VT == MVT::v2i64) 15321 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves 15322 15323 if (Subtarget.hasVSX() || Subtarget.hasQPX()) 15324 return true; 15325 15326 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); 15327} 15328 15329Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const { 15330 if (DisableILPPref || Subtarget.enableMachineScheduler()) 15331 return TargetLowering::getSchedulingPreference(N); 15332 15333 return Sched::ILP; 15334} 15335 15336// Create a fast isel object. 15337FastISel * 15338PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo, 15339 const TargetLibraryInfo *LibInfo) const { 15340 return PPC::createFastISel(FuncInfo, LibInfo); 15341} 15342 15343void PPCTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { 15344 if (Subtarget.isDarwinABI()) return; 15345 if (!Subtarget.isPPC64()) return; 15346 15347 // Update IsSplitCSR in PPCFunctionInfo 15348 PPCFunctionInfo *PFI = Entry->getParent()->getInfo<PPCFunctionInfo>(); 15349 PFI->setIsSplitCSR(true); 15350} 15351 15352void PPCTargetLowering::insertCopiesSplitCSR( 15353 MachineBasicBlock *Entry, 15354 const SmallVectorImpl<MachineBasicBlock *> &Exits) const { 15355 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); 15356 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); 15357 if (!IStart) 15358 return; 15359 15360 const TargetInstrInfo *TII = Subtarget.getInstrInfo(); 15361 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); 15362 MachineBasicBlock::iterator MBBI = Entry->begin(); 15363 for (const MCPhysReg *I = IStart; *I; ++I) { 15364 const TargetRegisterClass *RC = nullptr; 15365 if (PPC::G8RCRegClass.contains(*I)) 15366 RC = &PPC::G8RCRegClass; 15367 else if (PPC::F8RCRegClass.contains(*I)) 15368 RC = &PPC::F8RCRegClass; 15369 else if (PPC::CRRCRegClass.contains(*I)) 15370 RC = &PPC::CRRCRegClass; 15371 else if (PPC::VRRCRegClass.contains(*I)) 15372 RC = &PPC::VRRCRegClass; 15373 else 15374 llvm_unreachable("Unexpected register class in CSRsViaCopy!"); 15375 15376 Register NewVR = MRI->createVirtualRegister(RC); 15377 // Create copy from CSR to a virtual register. 15378 // FIXME: this currently does not emit CFI pseudo-instructions, it works 15379 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be 15380 // nounwind. If we want to generalize this later, we may need to emit 15381 // CFI pseudo-instructions. 15382 assert(Entry->getParent()->getFunction().hasFnAttribute( 15383 Attribute::NoUnwind) && 15384 "Function should be nounwind in insertCopiesSplitCSR!"); 15385 Entry->addLiveIn(*I); 15386 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) 15387 .addReg(*I); 15388 15389 // Insert the copy-back instructions right before the terminator. 15390 for (auto *Exit : Exits) 15391 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), 15392 TII->get(TargetOpcode::COPY), *I) 15393 .addReg(NewVR); 15394 } 15395} 15396 15397// Override to enable LOAD_STACK_GUARD lowering on Linux. 15398bool PPCTargetLowering::useLoadStackGuardNode() const { 15399 if (!Subtarget.isTargetLinux()) 15400 return TargetLowering::useLoadStackGuardNode(); 15401 return true; 15402} 15403 15404// Override to disable global variable loading on Linux. 15405void PPCTargetLowering::insertSSPDeclarations(Module &M) const { 15406 if (!Subtarget.isTargetLinux()) 15407 return TargetLowering::insertSSPDeclarations(M); 15408} 15409 15410bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, 15411 bool ForCodeSize) const { 15412 if (!VT.isSimple() || !Subtarget.hasVSX()) 15413 return false; 15414 15415 switch(VT.getSimpleVT().SimpleTy) { 15416 default: 15417 // For FP types that are currently not supported by PPC backend, return 15418 // false. Examples: f16, f80. 15419 return false; 15420 case MVT::f32: 15421 case MVT::f64: 15422 case MVT::ppcf128: 15423 return Imm.isPosZero(); 15424 } 15425} 15426 15427// For vector shift operation op, fold 15428// (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y) 15429static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N, 15430 SelectionDAG &DAG) { 15431 SDValue N0 = N->getOperand(0); 15432 SDValue N1 = N->getOperand(1); 15433 EVT VT = N0.getValueType(); 15434 unsigned OpSizeInBits = VT.getScalarSizeInBits(); 15435 unsigned Opcode = N->getOpcode(); 15436 unsigned TargetOpcode; 15437 15438 switch (Opcode) { 15439 default: 15440 llvm_unreachable("Unexpected shift operation"); 15441 case ISD::SHL: 15442 TargetOpcode = PPCISD::SHL; 15443 break; 15444 case ISD::SRL: 15445 TargetOpcode = PPCISD::SRL; 15446 break; 15447 case ISD::SRA: 15448 TargetOpcode = PPCISD::SRA; 15449 break; 15450 } 15451 15452 if (VT.isVector() && TLI.isOperationLegal(Opcode, VT) && 15453 N1->getOpcode() == ISD::AND) 15454 if (ConstantSDNode *Mask = isConstOrConstSplat(N1->getOperand(1))) 15455 if (Mask->getZExtValue() == OpSizeInBits - 1) 15456 return DAG.getNode(TargetOpcode, SDLoc(N), VT, N0, N1->getOperand(0)); 15457 15458 return SDValue(); 15459} 15460 15461SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const { 15462 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 15463 return Value; 15464 15465 SDValue N0 = N->getOperand(0); 15466 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(N->getOperand(1)); 15467 if (!Subtarget.isISA3_0() || 15468 N0.getOpcode() != ISD::SIGN_EXTEND || 15469 N0.getOperand(0).getValueType() != MVT::i32 || 15470 CN1 == nullptr || N->getValueType(0) != MVT::i64) 15471 return SDValue(); 15472 15473 // We can't save an operation here if the value is already extended, and 15474 // the existing shift is easier to combine. 15475 SDValue ExtsSrc = N0.getOperand(0); 15476 if (ExtsSrc.getOpcode() == ISD::TRUNCATE && 15477 ExtsSrc.getOperand(0).getOpcode() == ISD::AssertSext) 15478 return SDValue(); 15479 15480 SDLoc DL(N0); 15481 SDValue ShiftBy = SDValue(CN1, 0); 15482 // We want the shift amount to be i32 on the extswli, but the shift could 15483 // have an i64. 15484 if (ShiftBy.getValueType() == MVT::i64) 15485 ShiftBy = DCI.DAG.getConstant(CN1->getZExtValue(), DL, MVT::i32); 15486 15487 return DCI.DAG.getNode(PPCISD::EXTSWSLI, DL, MVT::i64, N0->getOperand(0), 15488 ShiftBy); 15489} 15490 15491SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const { 15492 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 15493 return Value; 15494 15495 return SDValue(); 15496} 15497 15498SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const { 15499 if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) 15500 return Value; 15501 15502 return SDValue(); 15503} 15504 15505// Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1)) 15506// Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0)) 15507// When C is zero, the equation (addi Z, -C) can be simplified to Z 15508// Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types 15509static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG, 15510 const PPCSubtarget &Subtarget) { 15511 if (!Subtarget.isPPC64()) 15512 return SDValue(); 15513 15514 SDValue LHS = N->getOperand(0); 15515 SDValue RHS = N->getOperand(1); 15516 15517 auto isZextOfCompareWithConstant = [](SDValue Op) { 15518 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() || 15519 Op.getValueType() != MVT::i64) 15520 return false; 15521 15522 SDValue Cmp = Op.getOperand(0); 15523 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() || 15524 Cmp.getOperand(0).getValueType() != MVT::i64) 15525 return false; 15526 15527 if (auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1))) { 15528 int64_t NegConstant = 0 - Constant->getSExtValue(); 15529 // Due to the limitations of the addi instruction, 15530 // -C is required to be [-32768, 32767]. 15531 return isInt<16>(NegConstant); 15532 } 15533 15534 return false; 15535 }; 15536 15537 bool LHSHasPattern = isZextOfCompareWithConstant(LHS); 15538 bool RHSHasPattern = isZextOfCompareWithConstant(RHS); 15539 15540 // If there is a pattern, canonicalize a zext operand to the RHS. 15541 if (LHSHasPattern && !RHSHasPattern) 15542 std::swap(LHS, RHS); 15543 else if (!LHSHasPattern && !RHSHasPattern) 15544 return SDValue(); 15545 15546 SDLoc DL(N); 15547 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue); 15548 SDValue Cmp = RHS.getOperand(0); 15549 SDValue Z = Cmp.getOperand(0); 15550 auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1)); 15551 15552 assert(Constant && "Constant Should not be a null pointer."); 15553 int64_t NegConstant = 0 - Constant->getSExtValue(); 15554 15555 switch(cast<CondCodeSDNode>(Cmp.getOperand(2))->get()) { 15556 default: break; 15557 case ISD::SETNE: { 15558 // when C == 0 15559 // --> addze X, (addic Z, -1).carry 15560 // / 15561 // add X, (zext(setne Z, C))-- 15562 // \ when -32768 <= -C <= 32767 && C != 0 15563 // --> addze X, (addic (addi Z, -C), -1).carry 15564 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, 15565 DAG.getConstant(NegConstant, DL, MVT::i64)); 15566 SDValue AddOrZ = NegConstant != 0 ? Add : Z; 15567 SDValue Addc = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i64, MVT::Glue), 15568 AddOrZ, DAG.getConstant(-1ULL, DL, MVT::i64)); 15569 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), 15570 SDValue(Addc.getNode(), 1)); 15571 } 15572 case ISD::SETEQ: { 15573 // when C == 0 15574 // --> addze X, (subfic Z, 0).carry 15575 // / 15576 // add X, (zext(sete Z, C))-- 15577 // \ when -32768 <= -C <= 32767 && C != 0 15578 // --> addze X, (subfic (addi Z, -C), 0).carry 15579 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, 15580 DAG.getConstant(NegConstant, DL, MVT::i64)); 15581 SDValue AddOrZ = NegConstant != 0 ? Add : Z; 15582 SDValue Subc = DAG.getNode(ISD::SUBC, DL, DAG.getVTList(MVT::i64, MVT::Glue), 15583 DAG.getConstant(0, DL, MVT::i64), AddOrZ); 15584 return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), 15585 SDValue(Subc.getNode(), 1)); 15586 } 15587 } 15588 15589 return SDValue(); 15590} 15591 15592SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const { 15593 if (auto Value = combineADDToADDZE(N, DCI.DAG, Subtarget)) 15594 return Value; 15595 15596 return SDValue(); 15597} 15598 15599// Detect TRUNCATE operations on bitcasts of float128 values. 15600// What we are looking for here is the situtation where we extract a subset 15601// of bits from a 128 bit float. 15602// This can be of two forms: 15603// 1) BITCAST of f128 feeding TRUNCATE 15604// 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE 15605// The reason this is required is because we do not have a legal i128 type 15606// and so we want to prevent having to store the f128 and then reload part 15607// of it. 15608SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N, 15609 DAGCombinerInfo &DCI) const { 15610 // If we are using CRBits then try that first. 15611 if (Subtarget.useCRBits()) { 15612 // Check if CRBits did anything and return that if it did. 15613 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI)) 15614 return CRTruncValue; 15615 } 15616 15617 SDLoc dl(N); 15618 SDValue Op0 = N->getOperand(0); 15619 15620 // Looking for a truncate of i128 to i64. 15621 if (Op0.getValueType() != MVT::i128 || N->getValueType(0) != MVT::i64) 15622 return SDValue(); 15623 15624 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0; 15625 15626 // SRL feeding TRUNCATE. 15627 if (Op0.getOpcode() == ISD::SRL) { 15628 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Op0.getOperand(1)); 15629 // The right shift has to be by 64 bits. 15630 if (!ConstNode || ConstNode->getZExtValue() != 64) 15631 return SDValue(); 15632 15633 // Switch the element number to extract. 15634 EltToExtract = EltToExtract ? 0 : 1; 15635 // Update Op0 past the SRL. 15636 Op0 = Op0.getOperand(0); 15637 } 15638 15639 // BITCAST feeding a TRUNCATE possibly via SRL. 15640 if (Op0.getOpcode() == ISD::BITCAST && 15641 Op0.getValueType() == MVT::i128 && 15642 Op0.getOperand(0).getValueType() == MVT::f128) { 15643 SDValue Bitcast = DCI.DAG.getBitcast(MVT::v2i64, Op0.getOperand(0)); 15644 return DCI.DAG.getNode( 15645 ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Bitcast, 15646 DCI.DAG.getTargetConstant(EltToExtract, dl, MVT::i32)); 15647 } 15648 return SDValue(); 15649} 15650 15651SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const { 15652 SelectionDAG &DAG = DCI.DAG; 15653 15654 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N->getOperand(1)); 15655 if (!ConstOpOrElement) 15656 return SDValue(); 15657 15658 // An imul is usually smaller than the alternative sequence for legal type. 15659 if (DAG.getMachineFunction().getFunction().hasMinSize() && 15660 isOperationLegal(ISD::MUL, N->getValueType(0))) 15661 return SDValue(); 15662 15663 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool { 15664 switch (this->Subtarget.getCPUDirective()) { 15665 default: 15666 // TODO: enhance the condition for subtarget before pwr8 15667 return false; 15668 case PPC::DIR_PWR8: 15669 // type mul add shl 15670 // scalar 4 1 1 15671 // vector 7 2 2 15672 return true; 15673 case PPC::DIR_PWR9: 15674 case PPC::DIR_PWR_FUTURE: 15675 // type mul add shl 15676 // scalar 5 2 2 15677 // vector 7 2 2 15678 15679 // The cycle RATIO of related operations are showed as a table above. 15680 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both 15681 // scalar and vector type. For 2 instrs patterns, add/sub + shl 15682 // are 4, it is always profitable; but for 3 instrs patterns 15683 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6. 15684 // So we should only do it for vector type. 15685 return IsAddOne && IsNeg ? VT.isVector() : true; 15686 } 15687 }; 15688 15689 EVT VT = N->getValueType(0); 15690 SDLoc DL(N); 15691 15692 const APInt &MulAmt = ConstOpOrElement->getAPIntValue(); 15693 bool IsNeg = MulAmt.isNegative(); 15694 APInt MulAmtAbs = MulAmt.abs(); 15695 15696 if ((MulAmtAbs - 1).isPowerOf2()) { 15697 // (mul x, 2^N + 1) => (add (shl x, N), x) 15698 // (mul x, -(2^N + 1)) => -(add (shl x, N), x) 15699 15700 if (!IsProfitable(IsNeg, true, VT)) 15701 return SDValue(); 15702 15703 SDValue Op0 = N->getOperand(0); 15704 SDValue Op1 = 15705 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 15706 DAG.getConstant((MulAmtAbs - 1).logBase2(), DL, VT)); 15707 SDValue Res = DAG.getNode(ISD::ADD, DL, VT, Op0, Op1); 15708 15709 if (!IsNeg) 15710 return Res; 15711 15712 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); 15713 } else if ((MulAmtAbs + 1).isPowerOf2()) { 15714 // (mul x, 2^N - 1) => (sub (shl x, N), x) 15715 // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) 15716 15717 if (!IsProfitable(IsNeg, false, VT)) 15718 return SDValue(); 15719 15720 SDValue Op0 = N->getOperand(0); 15721 SDValue Op1 = 15722 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), 15723 DAG.getConstant((MulAmtAbs + 1).logBase2(), DL, VT)); 15724 15725 if (!IsNeg) 15726 return DAG.getNode(ISD::SUB, DL, VT, Op1, Op0); 15727 else 15728 return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1); 15729 15730 } else { 15731 return SDValue(); 15732 } 15733} 15734 15735bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 15736 // Only duplicate to increase tail-calls for the 64bit SysV ABIs. 15737 if (!Subtarget.is64BitELFABI()) 15738 return false; 15739 15740 // If not a tail call then no need to proceed. 15741 if (!CI->isTailCall()) 15742 return false; 15743 15744 // If sibling calls have been disabled and tail-calls aren't guaranteed 15745 // there is no reason to duplicate. 15746 auto &TM = getTargetMachine(); 15747 if (!TM.Options.GuaranteedTailCallOpt && DisableSCO) 15748 return false; 15749 15750 // Can't tail call a function called indirectly, or if it has variadic args. 15751 const Function *Callee = CI->getCalledFunction(); 15752 if (!Callee || Callee->isVarArg()) 15753 return false; 15754 15755 // Make sure the callee and caller calling conventions are eligible for tco. 15756 const Function *Caller = CI->getParent()->getParent(); 15757 if (!areCallingConvEligibleForTCO_64SVR4(Caller->getCallingConv(), 15758 CI->getCallingConv())) 15759 return false; 15760 15761 // If the function is local then we have a good chance at tail-calling it 15762 return getTargetMachine().shouldAssumeDSOLocal(*Caller->getParent(), Callee); 15763} 15764 15765bool PPCTargetLowering::hasBitPreservingFPLogic(EVT VT) const { 15766 if (!Subtarget.hasVSX()) 15767 return false; 15768 if (Subtarget.hasP9Vector() && VT == MVT::f128) 15769 return true; 15770 return VT == MVT::f32 || VT == MVT::f64 || 15771 VT == MVT::v4f32 || VT == MVT::v2f64; 15772} 15773 15774bool PPCTargetLowering:: 15775isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const { 15776 const Value *Mask = AndI.getOperand(1); 15777 // If the mask is suitable for andi. or andis. we should sink the and. 15778 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Mask)) { 15779 // Can't handle constants wider than 64-bits. 15780 if (CI->getBitWidth() > 64) 15781 return false; 15782 int64_t ConstVal = CI->getZExtValue(); 15783 return isUInt<16>(ConstVal) || 15784 (isUInt<16>(ConstVal >> 16) && !(ConstVal & 0xFFFF)); 15785 } 15786 15787 // For non-constant masks, we can always use the record-form and. 15788 return true; 15789} 15790 15791// Transform (abs (sub (zext a), (zext b))) to (vabsd a b 0) 15792// Transform (abs (sub (zext a), (zext_invec b))) to (vabsd a b 0) 15793// Transform (abs (sub (zext_invec a), (zext_invec b))) to (vabsd a b 0) 15794// Transform (abs (sub (zext_invec a), (zext b))) to (vabsd a b 0) 15795// Transform (abs (sub a, b) to (vabsd a b 1)) if a & b of type v4i32 15796SDValue PPCTargetLowering::combineABS(SDNode *N, DAGCombinerInfo &DCI) const { 15797 assert((N->getOpcode() == ISD::ABS) && "Need ABS node here"); 15798 assert(Subtarget.hasP9Altivec() && 15799 "Only combine this when P9 altivec supported!"); 15800 EVT VT = N->getValueType(0); 15801 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) 15802 return SDValue(); 15803 15804 SelectionDAG &DAG = DCI.DAG; 15805 SDLoc dl(N); 15806 if (N->getOperand(0).getOpcode() == ISD::SUB) { 15807 // Even for signed integers, if it's known to be positive (as signed 15808 // integer) due to zero-extended inputs. 15809 unsigned SubOpcd0 = N->getOperand(0)->getOperand(0).getOpcode(); 15810 unsigned SubOpcd1 = N->getOperand(0)->getOperand(1).getOpcode(); 15811 if ((SubOpcd0 == ISD::ZERO_EXTEND || 15812 SubOpcd0 == ISD::ZERO_EXTEND_VECTOR_INREG) && 15813 (SubOpcd1 == ISD::ZERO_EXTEND || 15814 SubOpcd1 == ISD::ZERO_EXTEND_VECTOR_INREG)) { 15815 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), 15816 N->getOperand(0)->getOperand(0), 15817 N->getOperand(0)->getOperand(1), 15818 DAG.getTargetConstant(0, dl, MVT::i32)); 15819 } 15820 15821 // For type v4i32, it can be optimized with xvnegsp + vabsduw 15822 if (N->getOperand(0).getValueType() == MVT::v4i32 && 15823 N->getOperand(0).hasOneUse()) { 15824 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), 15825 N->getOperand(0)->getOperand(0), 15826 N->getOperand(0)->getOperand(1), 15827 DAG.getTargetConstant(1, dl, MVT::i32)); 15828 } 15829 } 15830 15831 return SDValue(); 15832} 15833 15834// For type v4i32/v8ii16/v16i8, transform 15835// from (vselect (setcc a, b, setugt), (sub a, b), (sub b, a)) to (vabsd a, b) 15836// from (vselect (setcc a, b, setuge), (sub a, b), (sub b, a)) to (vabsd a, b) 15837// from (vselect (setcc a, b, setult), (sub b, a), (sub a, b)) to (vabsd a, b) 15838// from (vselect (setcc a, b, setule), (sub b, a), (sub a, b)) to (vabsd a, b) 15839SDValue PPCTargetLowering::combineVSelect(SDNode *N, 15840 DAGCombinerInfo &DCI) const { 15841 assert((N->getOpcode() == ISD::VSELECT) && "Need VSELECT node here"); 15842 assert(Subtarget.hasP9Altivec() && 15843 "Only combine this when P9 altivec supported!"); 15844 15845 SelectionDAG &DAG = DCI.DAG; 15846 SDLoc dl(N); 15847 SDValue Cond = N->getOperand(0); 15848 SDValue TrueOpnd = N->getOperand(1); 15849 SDValue FalseOpnd = N->getOperand(2); 15850 EVT VT = N->getOperand(1).getValueType(); 15851 15852 if (Cond.getOpcode() != ISD::SETCC || TrueOpnd.getOpcode() != ISD::SUB || 15853 FalseOpnd.getOpcode() != ISD::SUB) 15854 return SDValue(); 15855 15856 // ABSD only available for type v4i32/v8i16/v16i8 15857 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) 15858 return SDValue(); 15859 15860 // At least to save one more dependent computation 15861 if (!(Cond.hasOneUse() || TrueOpnd.hasOneUse() || FalseOpnd.hasOneUse())) 15862 return SDValue(); 15863 15864 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 15865 15866 // Can only handle unsigned comparison here 15867 switch (CC) { 15868 default: 15869 return SDValue(); 15870 case ISD::SETUGT: 15871 case ISD::SETUGE: 15872 break; 15873 case ISD::SETULT: 15874 case ISD::SETULE: 15875 std::swap(TrueOpnd, FalseOpnd); 15876 break; 15877 } 15878 15879 SDValue CmpOpnd1 = Cond.getOperand(0); 15880 SDValue CmpOpnd2 = Cond.getOperand(1); 15881 15882 // SETCC CmpOpnd1 CmpOpnd2 cond 15883 // TrueOpnd = CmpOpnd1 - CmpOpnd2 15884 // FalseOpnd = CmpOpnd2 - CmpOpnd1 15885 if (TrueOpnd.getOperand(0) == CmpOpnd1 && 15886 TrueOpnd.getOperand(1) == CmpOpnd2 && 15887 FalseOpnd.getOperand(0) == CmpOpnd2 && 15888 FalseOpnd.getOperand(1) == CmpOpnd1) { 15889 return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(1).getValueType(), 15890 CmpOpnd1, CmpOpnd2, 15891 DAG.getTargetConstant(0, dl, MVT::i32)); 15892 } 15893 15894 return SDValue(); 15895} 15896