1//===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// \file 9/// Interface for Targets to specify which operations they can successfully 10/// select and how the others should be expanded most efficiently. 11/// 12//===----------------------------------------------------------------------===// 13 14#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 15#define LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 16 17#include "llvm/ADT/SmallBitVector.h" 18#include "llvm/ADT/SmallVector.h" 19#include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h" 20#include "llvm/CodeGen/LowLevelType.h" 21#include "llvm/CodeGen/MachineMemOperand.h" 22#include "llvm/CodeGen/TargetOpcodes.h" 23#include "llvm/MC/MCInstrDesc.h" 24#include "llvm/Support/AtomicOrdering.h" 25#include "llvm/Support/CommandLine.h" 26#include <cassert> 27#include <cstdint> 28#include <tuple> 29#include <utility> 30 31namespace llvm { 32 33extern cl::opt<bool> DisableGISelLegalityCheck; 34 35class MachineFunction; 36class raw_ostream; 37class LegalizerHelper; 38class LostDebugLocObserver; 39class MachineInstr; 40class MachineRegisterInfo; 41class MCInstrInfo; 42 43namespace LegalizeActions { 44enum LegalizeAction : std::uint8_t { 45 /// The operation is expected to be selectable directly by the target, and 46 /// no transformation is necessary. 47 Legal, 48 49 /// The operation should be synthesized from multiple instructions acting on 50 /// a narrower scalar base-type. For example a 64-bit add might be 51 /// implemented in terms of 32-bit add-with-carry. 52 NarrowScalar, 53 54 /// The operation should be implemented in terms of a wider scalar 55 /// base-type. For example a <2 x s8> add could be implemented as a <2 56 /// x s32> add (ignoring the high bits). 57 WidenScalar, 58 59 /// The (vector) operation should be implemented by splitting it into 60 /// sub-vectors where the operation is legal. For example a <8 x s64> add 61 /// might be implemented as 4 separate <2 x s64> adds. There can be a leftover 62 /// if there are not enough elements for last sub-vector e.g. <7 x s64> add 63 /// will be implemented as 3 separate <2 x s64> adds and one s64 add. Leftover 64 /// types can be avoided by doing MoreElements first. 65 FewerElements, 66 67 /// The (vector) operation should be implemented by widening the input 68 /// vector and ignoring the lanes added by doing so. For example <2 x i8> is 69 /// rarely legal, but you might perform an <8 x i8> and then only look at 70 /// the first two results. 71 MoreElements, 72 73 /// Perform the operation on a different, but equivalently sized type. 74 Bitcast, 75 76 /// The operation itself must be expressed in terms of simpler actions on 77 /// this target. E.g. a SREM replaced by an SDIV and subtraction. 78 Lower, 79 80 /// The operation should be implemented as a call to some kind of runtime 81 /// support library. For example this usually happens on machines that don't 82 /// support floating-point operations natively. 83 Libcall, 84 85 /// The target wants to do something special with this combination of 86 /// operand and type. A callback will be issued when it is needed. 87 Custom, 88 89 /// This operation is completely unsupported on the target. A programming 90 /// error has occurred. 91 Unsupported, 92 93 /// Sentinel value for when no action was found in the specified table. 94 NotFound, 95 96 /// Fall back onto the old rules. 97 /// TODO: Remove this once we've migrated 98 UseLegacyRules, 99}; 100} // end namespace LegalizeActions 101raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action); 102 103using LegalizeActions::LegalizeAction; 104 105/// The LegalityQuery object bundles together all the information that's needed 106/// to decide whether a given operation is legal or not. 107/// For efficiency, it doesn't make a copy of Types so care must be taken not 108/// to free it before using the query. 109struct LegalityQuery { 110 unsigned Opcode; 111 ArrayRef<LLT> Types; 112 113 struct MemDesc { 114 LLT MemoryTy; 115 uint64_t AlignInBits; 116 AtomicOrdering Ordering; 117 118 MemDesc() = default; 119 MemDesc(LLT MemoryTy, uint64_t AlignInBits, AtomicOrdering Ordering) 120 : MemoryTy(MemoryTy), AlignInBits(AlignInBits), Ordering(Ordering) {} 121 MemDesc(const MachineMemOperand &MMO) 122 : MemoryTy(MMO.getMemoryType()), 123 AlignInBits(MMO.getAlign().value() * 8), 124 Ordering(MMO.getSuccessOrdering()) {} 125 }; 126 127 /// Operations which require memory can use this to place requirements on the 128 /// memory type for each MMO. 129 ArrayRef<MemDesc> MMODescrs; 130 131 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types, 132 const ArrayRef<MemDesc> MMODescrs) 133 : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {} 134 constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types) 135 : LegalityQuery(Opcode, Types, {}) {} 136 137 raw_ostream &print(raw_ostream &OS) const; 138}; 139 140/// The result of a query. It either indicates a final answer of Legal or 141/// Unsupported or describes an action that must be taken to make an operation 142/// more legal. 143struct LegalizeActionStep { 144 /// The action to take or the final answer. 145 LegalizeAction Action; 146 /// If describing an action, the type index to change. Otherwise zero. 147 unsigned TypeIdx; 148 /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. 149 LLT NewType; 150 151 LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, 152 const LLT NewType) 153 : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} 154 155 LegalizeActionStep(LegacyLegalizeActionStep Step) 156 : TypeIdx(Step.TypeIdx), NewType(Step.NewType) { 157 switch (Step.Action) { 158 case LegacyLegalizeActions::Legal: 159 Action = LegalizeActions::Legal; 160 break; 161 case LegacyLegalizeActions::NarrowScalar: 162 Action = LegalizeActions::NarrowScalar; 163 break; 164 case LegacyLegalizeActions::WidenScalar: 165 Action = LegalizeActions::WidenScalar; 166 break; 167 case LegacyLegalizeActions::FewerElements: 168 Action = LegalizeActions::FewerElements; 169 break; 170 case LegacyLegalizeActions::MoreElements: 171 Action = LegalizeActions::MoreElements; 172 break; 173 case LegacyLegalizeActions::Bitcast: 174 Action = LegalizeActions::Bitcast; 175 break; 176 case LegacyLegalizeActions::Lower: 177 Action = LegalizeActions::Lower; 178 break; 179 case LegacyLegalizeActions::Libcall: 180 Action = LegalizeActions::Libcall; 181 break; 182 case LegacyLegalizeActions::Custom: 183 Action = LegalizeActions::Custom; 184 break; 185 case LegacyLegalizeActions::Unsupported: 186 Action = LegalizeActions::Unsupported; 187 break; 188 case LegacyLegalizeActions::NotFound: 189 Action = LegalizeActions::NotFound; 190 break; 191 } 192 } 193 194 bool operator==(const LegalizeActionStep &RHS) const { 195 return std::tie(Action, TypeIdx, NewType) == 196 std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); 197 } 198}; 199 200using LegalityPredicate = std::function<bool (const LegalityQuery &)>; 201using LegalizeMutation = 202 std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; 203 204namespace LegalityPredicates { 205struct TypePairAndMemDesc { 206 LLT Type0; 207 LLT Type1; 208 LLT MemTy; 209 uint64_t Align; 210 211 bool operator==(const TypePairAndMemDesc &Other) const { 212 return Type0 == Other.Type0 && Type1 == Other.Type1 && 213 Align == Other.Align && MemTy == Other.MemTy; 214 } 215 216 /// \returns true if this memory access is legal with for the access described 217 /// by \p Other (The alignment is sufficient for the size and result type). 218 bool isCompatible(const TypePairAndMemDesc &Other) const { 219 return Type0 == Other.Type0 && Type1 == Other.Type1 && 220 Align >= Other.Align && 221 // FIXME: This perhaps should be stricter, but the current legality 222 // rules are written only considering the size. 223 MemTy.getSizeInBits() == Other.MemTy.getSizeInBits(); 224 } 225}; 226 227/// True iff P is false. 228template <typename Predicate> Predicate predNot(Predicate P) { 229 return [=](const LegalityQuery &Query) { return !P(Query); }; 230} 231 232/// True iff P0 and P1 are true. 233template<typename Predicate> 234Predicate all(Predicate P0, Predicate P1) { 235 return [=](const LegalityQuery &Query) { 236 return P0(Query) && P1(Query); 237 }; 238} 239/// True iff all given predicates are true. 240template<typename Predicate, typename... Args> 241Predicate all(Predicate P0, Predicate P1, Args... args) { 242 return all(all(P0, P1), args...); 243} 244 245/// True iff P0 or P1 are true. 246template<typename Predicate> 247Predicate any(Predicate P0, Predicate P1) { 248 return [=](const LegalityQuery &Query) { 249 return P0(Query) || P1(Query); 250 }; 251} 252/// True iff any given predicates are true. 253template<typename Predicate, typename... Args> 254Predicate any(Predicate P0, Predicate P1, Args... args) { 255 return any(any(P0, P1), args...); 256} 257 258/// True iff the given type index is the specified type. 259LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit); 260/// True iff the given type index is one of the specified types. 261LegalityPredicate typeInSet(unsigned TypeIdx, 262 std::initializer_list<LLT> TypesInit); 263 264/// True iff the given type index is not the specified type. 265inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) { 266 return [=](const LegalityQuery &Query) { 267 return Query.Types[TypeIdx] != Type; 268 }; 269} 270 271/// True iff the given types for the given pair of type indexes is one of the 272/// specified type pairs. 273LegalityPredicate 274typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, 275 std::initializer_list<std::pair<LLT, LLT>> TypesInit); 276/// True iff the given types for the given pair of type indexes is one of the 277/// specified type pairs. 278LegalityPredicate typePairAndMemDescInSet( 279 unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, 280 std::initializer_list<TypePairAndMemDesc> TypesAndMemDescInit); 281/// True iff the specified type index is a scalar. 282LegalityPredicate isScalar(unsigned TypeIdx); 283/// True iff the specified type index is a vector. 284LegalityPredicate isVector(unsigned TypeIdx); 285/// True iff the specified type index is a pointer (with any address space). 286LegalityPredicate isPointer(unsigned TypeIdx); 287/// True iff the specified type index is a pointer with the specified address 288/// space. 289LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace); 290 291/// True if the type index is a vector with element type \p EltTy 292LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy); 293 294/// True iff the specified type index is a scalar that's narrower than the given 295/// size. 296LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size); 297 298/// True iff the specified type index is a scalar that's wider than the given 299/// size. 300LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size); 301 302/// True iff the specified type index is a scalar or vector with an element type 303/// that's narrower than the given size. 304LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); 305 306/// True iff the specified type index is a scalar or a vector with an element 307/// type that's wider than the given size. 308LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); 309 310/// True iff the specified type index is a scalar whose size is not a multiple 311/// of Size. 312LegalityPredicate sizeNotMultipleOf(unsigned TypeIdx, unsigned Size); 313 314/// True iff the specified type index is a scalar whose size is not a power of 315/// 2. 316LegalityPredicate sizeNotPow2(unsigned TypeIdx); 317 318/// True iff the specified type index is a scalar or vector whose element size 319/// is not a power of 2. 320LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); 321 322/// True if the total bitwidth of the specified type index is \p Size bits. 323LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size); 324 325/// True iff the specified type indices are both the same bit size. 326LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); 327 328/// True iff the first type index has a larger total bit size than second type 329/// index. 330LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1); 331 332/// True iff the first type index has a smaller total bit size than second type 333/// index. 334LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1); 335 336/// True iff the specified MMO index has a size (rounded to bytes) that is not a 337/// power of 2. 338LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); 339 340/// True iff the specified MMO index has a size that is not an even byte size, 341/// or that even byte size is not a power of 2. 342LegalityPredicate memSizeNotByteSizePow2(unsigned MMOIdx); 343 344/// True iff the specified type index is a vector whose element count is not a 345/// power of 2. 346LegalityPredicate numElementsNotPow2(unsigned TypeIdx); 347/// True iff the specified MMO index has at an atomic ordering of at Ordering or 348/// stronger. 349LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, 350 AtomicOrdering Ordering); 351} // end namespace LegalityPredicates 352 353namespace LegalizeMutations { 354/// Select this specific type for the given type index. 355LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); 356 357/// Keep the same type as the given type index. 358LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); 359 360/// Keep the same scalar or element type as the given type index. 361LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); 362 363/// Keep the same scalar or element type as the given type. 364LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); 365 366/// Keep the same scalar or element type as \p TypeIdx, but take the number of 367/// elements from \p FromTypeIdx. 368LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx); 369 370/// Keep the same scalar or element type as \p TypeIdx, but take the number of 371/// elements from \p Ty. 372LegalizeMutation changeElementCountTo(unsigned TypeIdx, LLT Ty); 373 374/// Change the scalar size or element size to have the same scalar size as type 375/// index \p FromIndex. Unlike changeElementTo, this discards pointer types and 376/// only changes the size. 377LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx); 378 379/// Widen the scalar type or vector element type for the given type index to the 380/// next power of 2. 381LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); 382 383/// Widen the scalar type or vector element type for the given type index to 384/// next multiple of \p Size. 385LegalizeMutation widenScalarOrEltToNextMultipleOf(unsigned TypeIdx, 386 unsigned Size); 387 388/// Add more elements to the type for the given type index to the next power of 389/// 2. 390LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); 391/// Break up the vector type for the given type index into the element type. 392LegalizeMutation scalarize(unsigned TypeIdx); 393} // end namespace LegalizeMutations 394 395/// A single rule in a legalizer info ruleset. 396/// The specified action is chosen when the predicate is true. Where appropriate 397/// for the action (e.g. for WidenScalar) the new type is selected using the 398/// given mutator. 399class LegalizeRule { 400 LegalityPredicate Predicate; 401 LegalizeAction Action; 402 LegalizeMutation Mutation; 403 404public: 405 LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action, 406 LegalizeMutation Mutation = nullptr) 407 : Predicate(Predicate), Action(Action), Mutation(Mutation) {} 408 409 /// Test whether the LegalityQuery matches. 410 bool match(const LegalityQuery &Query) const { 411 return Predicate(Query); 412 } 413 414 LegalizeAction getAction() const { return Action; } 415 416 /// Determine the change to make. 417 std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const { 418 if (Mutation) 419 return Mutation(Query); 420 return std::make_pair(0, LLT{}); 421 } 422}; 423 424class LegalizeRuleSet { 425 /// When non-zero, the opcode we are an alias of 426 unsigned AliasOf = 0; 427 /// If true, there is another opcode that aliases this one 428 bool IsAliasedByAnother = false; 429 SmallVector<LegalizeRule, 2> Rules; 430 431#ifndef NDEBUG 432 /// If bit I is set, this rule set contains a rule that may handle (predicate 433 /// or perform an action upon (or both)) the type index I. The uncertainty 434 /// comes from free-form rules executing user-provided lambda functions. We 435 /// conservatively assume such rules do the right thing and cover all type 436 /// indices. The bitset is intentionally 1 bit wider than it absolutely needs 437 /// to be to distinguish such cases from the cases where all type indices are 438 /// individually handled. 439 SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC - 440 MCOI::OPERAND_FIRST_GENERIC + 2}; 441 SmallBitVector ImmIdxsCovered{MCOI::OPERAND_LAST_GENERIC_IMM - 442 MCOI::OPERAND_FIRST_GENERIC_IMM + 2}; 443#endif 444 445 unsigned typeIdx(unsigned TypeIdx) { 446 assert(TypeIdx <= 447 (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) && 448 "Type Index is out of bounds"); 449#ifndef NDEBUG 450 TypeIdxsCovered.set(TypeIdx); 451#endif 452 return TypeIdx; 453 } 454 455 void markAllIdxsAsCovered() { 456#ifndef NDEBUG 457 TypeIdxsCovered.set(); 458 ImmIdxsCovered.set(); 459#endif 460 } 461 462 void add(const LegalizeRule &Rule) { 463 assert(AliasOf == 0 && 464 "RuleSet is aliased, change the representative opcode instead"); 465 Rules.push_back(Rule); 466 } 467 468 static bool always(const LegalityQuery &) { return true; } 469 470 /// Use the given action when the predicate is true. 471 /// Action should not be an action that requires mutation. 472 LegalizeRuleSet &actionIf(LegalizeAction Action, 473 LegalityPredicate Predicate) { 474 add({Predicate, Action}); 475 return *this; 476 } 477 /// Use the given action when the predicate is true. 478 /// Action should be an action that requires mutation. 479 LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate, 480 LegalizeMutation Mutation) { 481 add({Predicate, Action, Mutation}); 482 return *this; 483 } 484 /// Use the given action when type index 0 is any type in the given list. 485 /// Action should not be an action that requires mutation. 486 LegalizeRuleSet &actionFor(LegalizeAction Action, 487 std::initializer_list<LLT> Types) { 488 using namespace LegalityPredicates; 489 return actionIf(Action, typeInSet(typeIdx(0), Types)); 490 } 491 /// Use the given action when type index 0 is any type in the given list. 492 /// Action should be an action that requires mutation. 493 LegalizeRuleSet &actionFor(LegalizeAction Action, 494 std::initializer_list<LLT> Types, 495 LegalizeMutation Mutation) { 496 using namespace LegalityPredicates; 497 return actionIf(Action, typeInSet(typeIdx(0), Types), Mutation); 498 } 499 /// Use the given action when type indexes 0 and 1 is any type pair in the 500 /// given list. 501 /// Action should not be an action that requires mutation. 502 LegalizeRuleSet &actionFor(LegalizeAction Action, 503 std::initializer_list<std::pair<LLT, LLT>> Types) { 504 using namespace LegalityPredicates; 505 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 506 } 507 /// Use the given action when type indexes 0 and 1 is any type pair in the 508 /// given list. 509 /// Action should be an action that requires mutation. 510 LegalizeRuleSet &actionFor(LegalizeAction Action, 511 std::initializer_list<std::pair<LLT, LLT>> Types, 512 LegalizeMutation Mutation) { 513 using namespace LegalityPredicates; 514 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types), 515 Mutation); 516 } 517 /// Use the given action when type index 0 is any type in the given list and 518 /// imm index 0 is anything. Action should not be an action that requires 519 /// mutation. 520 LegalizeRuleSet &actionForTypeWithAnyImm(LegalizeAction Action, 521 std::initializer_list<LLT> Types) { 522 using namespace LegalityPredicates; 523 immIdx(0); // Inform verifier imm idx 0 is handled. 524 return actionIf(Action, typeInSet(typeIdx(0), Types)); 525 } 526 527 LegalizeRuleSet &actionForTypeWithAnyImm( 528 LegalizeAction Action, std::initializer_list<std::pair<LLT, LLT>> Types) { 529 using namespace LegalityPredicates; 530 immIdx(0); // Inform verifier imm idx 0 is handled. 531 return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); 532 } 533 534 /// Use the given action when type indexes 0 and 1 are both in the given list. 535 /// That is, the type pair is in the cartesian product of the list. 536 /// Action should not be an action that requires mutation. 537 LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action, 538 std::initializer_list<LLT> Types) { 539 using namespace LegalityPredicates; 540 return actionIf(Action, all(typeInSet(typeIdx(0), Types), 541 typeInSet(typeIdx(1), Types))); 542 } 543 /// Use the given action when type indexes 0 and 1 are both in their 544 /// respective lists. 545 /// That is, the type pair is in the cartesian product of the lists 546 /// Action should not be an action that requires mutation. 547 LegalizeRuleSet & 548 actionForCartesianProduct(LegalizeAction Action, 549 std::initializer_list<LLT> Types0, 550 std::initializer_list<LLT> Types1) { 551 using namespace LegalityPredicates; 552 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 553 typeInSet(typeIdx(1), Types1))); 554 } 555 /// Use the given action when type indexes 0, 1, and 2 are all in their 556 /// respective lists. 557 /// That is, the type triple is in the cartesian product of the lists 558 /// Action should not be an action that requires mutation. 559 LegalizeRuleSet &actionForCartesianProduct( 560 LegalizeAction Action, std::initializer_list<LLT> Types0, 561 std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) { 562 using namespace LegalityPredicates; 563 return actionIf(Action, all(typeInSet(typeIdx(0), Types0), 564 all(typeInSet(typeIdx(1), Types1), 565 typeInSet(typeIdx(2), Types2)))); 566 } 567 568public: 569 LegalizeRuleSet() = default; 570 571 bool isAliasedByAnother() { return IsAliasedByAnother; } 572 void setIsAliasedByAnother() { IsAliasedByAnother = true; } 573 void aliasTo(unsigned Opcode) { 574 assert((AliasOf == 0 || AliasOf == Opcode) && 575 "Opcode is already aliased to another opcode"); 576 assert(Rules.empty() && "Aliasing will discard rules"); 577 AliasOf = Opcode; 578 } 579 unsigned getAlias() const { return AliasOf; } 580 581 unsigned immIdx(unsigned ImmIdx) { 582 assert(ImmIdx <= (MCOI::OPERAND_LAST_GENERIC_IMM - 583 MCOI::OPERAND_FIRST_GENERIC_IMM) && 584 "Imm Index is out of bounds"); 585#ifndef NDEBUG 586 ImmIdxsCovered.set(ImmIdx); 587#endif 588 return ImmIdx; 589 } 590 591 /// The instruction is legal if predicate is true. 592 LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { 593 // We have no choice but conservatively assume that the free-form 594 // user-provided Predicate properly handles all type indices: 595 markAllIdxsAsCovered(); 596 return actionIf(LegalizeAction::Legal, Predicate); 597 } 598 /// The instruction is legal when type index 0 is any type in the given list. 599 LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) { 600 return actionFor(LegalizeAction::Legal, Types); 601 } 602 /// The instruction is legal when type indexes 0 and 1 is any type pair in the 603 /// given list. 604 LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 605 return actionFor(LegalizeAction::Legal, Types); 606 } 607 /// The instruction is legal when type index 0 is any type in the given list 608 /// and imm index 0 is anything. 609 LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list<LLT> Types) { 610 markAllIdxsAsCovered(); 611 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 612 } 613 614 LegalizeRuleSet &legalForTypeWithAnyImm( 615 std::initializer_list<std::pair<LLT, LLT>> Types) { 616 markAllIdxsAsCovered(); 617 return actionForTypeWithAnyImm(LegalizeAction::Legal, Types); 618 } 619 620 /// The instruction is legal when type indexes 0 and 1 along with the memory 621 /// size and minimum alignment is any type and size tuple in the given list. 622 LegalizeRuleSet &legalForTypesWithMemDesc( 623 std::initializer_list<LegalityPredicates::TypePairAndMemDesc> 624 TypesAndMemDesc) { 625 return actionIf(LegalizeAction::Legal, 626 LegalityPredicates::typePairAndMemDescInSet( 627 typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc)); 628 } 629 /// The instruction is legal when type indexes 0 and 1 are both in the given 630 /// list. That is, the type pair is in the cartesian product of the list. 631 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) { 632 return actionForCartesianProduct(LegalizeAction::Legal, Types); 633 } 634 /// The instruction is legal when type indexes 0 and 1 are both their 635 /// respective lists. 636 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 637 std::initializer_list<LLT> Types1) { 638 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); 639 } 640 /// The instruction is legal when type indexes 0, 1, and 2 are both their 641 /// respective lists. 642 LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, 643 std::initializer_list<LLT> Types1, 644 std::initializer_list<LLT> Types2) { 645 return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1, 646 Types2); 647 } 648 649 LegalizeRuleSet &alwaysLegal() { 650 using namespace LegalizeMutations; 651 markAllIdxsAsCovered(); 652 return actionIf(LegalizeAction::Legal, always); 653 } 654 655 /// The specified type index is coerced if predicate is true. 656 LegalizeRuleSet &bitcastIf(LegalityPredicate Predicate, 657 LegalizeMutation Mutation) { 658 // We have no choice but conservatively assume that lowering with a 659 // free-form user provided Predicate properly handles all type indices: 660 markAllIdxsAsCovered(); 661 return actionIf(LegalizeAction::Bitcast, Predicate, Mutation); 662 } 663 664 /// The instruction is lowered. 665 LegalizeRuleSet &lower() { 666 using namespace LegalizeMutations; 667 // We have no choice but conservatively assume that predicate-less lowering 668 // properly handles all type indices by design: 669 markAllIdxsAsCovered(); 670 return actionIf(LegalizeAction::Lower, always); 671 } 672 /// The instruction is lowered if predicate is true. Keep type index 0 as the 673 /// same type. 674 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) { 675 using namespace LegalizeMutations; 676 // We have no choice but conservatively assume that lowering with a 677 // free-form user provided Predicate properly handles all type indices: 678 markAllIdxsAsCovered(); 679 return actionIf(LegalizeAction::Lower, Predicate); 680 } 681 /// The instruction is lowered if predicate is true. 682 LegalizeRuleSet &lowerIf(LegalityPredicate Predicate, 683 LegalizeMutation Mutation) { 684 // We have no choice but conservatively assume that lowering with a 685 // free-form user provided Predicate properly handles all type indices: 686 markAllIdxsAsCovered(); 687 return actionIf(LegalizeAction::Lower, Predicate, Mutation); 688 } 689 /// The instruction is lowered when type index 0 is any type in the given 690 /// list. Keep type index 0 as the same type. 691 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) { 692 return actionFor(LegalizeAction::Lower, Types); 693 } 694 /// The instruction is lowered when type index 0 is any type in the given 695 /// list. 696 LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types, 697 LegalizeMutation Mutation) { 698 return actionFor(LegalizeAction::Lower, Types, Mutation); 699 } 700 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 701 /// the given list. Keep type index 0 as the same type. 702 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 703 return actionFor(LegalizeAction::Lower, Types); 704 } 705 /// The instruction is lowered when type indexes 0 and 1 is any type pair in 706 /// the given list. 707 LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types, 708 LegalizeMutation Mutation) { 709 return actionFor(LegalizeAction::Lower, Types, Mutation); 710 } 711 /// The instruction is lowered when type indexes 0 and 1 are both in their 712 /// respective lists. 713 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 714 std::initializer_list<LLT> Types1) { 715 using namespace LegalityPredicates; 716 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1); 717 } 718 /// The instruction is lowered when type indexes 0, 1, and 2 are all in 719 /// their respective lists. 720 LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, 721 std::initializer_list<LLT> Types1, 722 std::initializer_list<LLT> Types2) { 723 using namespace LegalityPredicates; 724 return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1, 725 Types2); 726 } 727 728 /// The instruction is emitted as a library call. 729 LegalizeRuleSet &libcall() { 730 using namespace LegalizeMutations; 731 // We have no choice but conservatively assume that predicate-less lowering 732 // properly handles all type indices by design: 733 markAllIdxsAsCovered(); 734 return actionIf(LegalizeAction::Libcall, always); 735 } 736 737 /// Like legalIf, but for the Libcall action. 738 LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { 739 // We have no choice but conservatively assume that a libcall with a 740 // free-form user provided Predicate properly handles all type indices: 741 markAllIdxsAsCovered(); 742 return actionIf(LegalizeAction::Libcall, Predicate); 743 } 744 LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { 745 return actionFor(LegalizeAction::Libcall, Types); 746 } 747 LegalizeRuleSet & 748 libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 749 return actionFor(LegalizeAction::Libcall, Types); 750 } 751 LegalizeRuleSet & 752 libcallForCartesianProduct(std::initializer_list<LLT> Types) { 753 return actionForCartesianProduct(LegalizeAction::Libcall, Types); 754 } 755 LegalizeRuleSet & 756 libcallForCartesianProduct(std::initializer_list<LLT> Types0, 757 std::initializer_list<LLT> Types1) { 758 return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1); 759 } 760 761 /// Widen the scalar to the one selected by the mutation if the predicate is 762 /// true. 763 LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, 764 LegalizeMutation Mutation) { 765 // We have no choice but conservatively assume that an action with a 766 // free-form user provided Predicate properly handles all type indices: 767 markAllIdxsAsCovered(); 768 return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation); 769 } 770 /// Narrow the scalar to the one selected by the mutation if the predicate is 771 /// true. 772 LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, 773 LegalizeMutation Mutation) { 774 // We have no choice but conservatively assume that an action with a 775 // free-form user provided Predicate properly handles all type indices: 776 markAllIdxsAsCovered(); 777 return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation); 778 } 779 /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any 780 /// type pair in the given list. 781 LegalizeRuleSet & 782 narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types, 783 LegalizeMutation Mutation) { 784 return actionFor(LegalizeAction::NarrowScalar, Types, Mutation); 785 } 786 787 /// Add more elements to reach the type selected by the mutation if the 788 /// predicate is true. 789 LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, 790 LegalizeMutation Mutation) { 791 // We have no choice but conservatively assume that an action with a 792 // free-form user provided Predicate properly handles all type indices: 793 markAllIdxsAsCovered(); 794 return actionIf(LegalizeAction::MoreElements, Predicate, Mutation); 795 } 796 /// Remove elements to reach the type selected by the mutation if the 797 /// predicate is true. 798 LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, 799 LegalizeMutation Mutation) { 800 // We have no choice but conservatively assume that an action with a 801 // free-form user provided Predicate properly handles all type indices: 802 markAllIdxsAsCovered(); 803 return actionIf(LegalizeAction::FewerElements, Predicate, Mutation); 804 } 805 806 /// The instruction is unsupported. 807 LegalizeRuleSet &unsupported() { 808 markAllIdxsAsCovered(); 809 return actionIf(LegalizeAction::Unsupported, always); 810 } 811 LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { 812 return actionIf(LegalizeAction::Unsupported, Predicate); 813 } 814 815 LegalizeRuleSet &unsupportedFor(std::initializer_list<LLT> Types) { 816 return actionFor(LegalizeAction::Unsupported, Types); 817 } 818 819 LegalizeRuleSet &unsupportedIfMemSizeNotPow2() { 820 return actionIf(LegalizeAction::Unsupported, 821 LegalityPredicates::memSizeInBytesNotPow2(0)); 822 } 823 824 /// Lower a memory operation if the memory size, rounded to bytes, is not a 825 /// power of 2. For example, this will not trigger for s1 or s7, but will for 826 /// s24. 827 LegalizeRuleSet &lowerIfMemSizeNotPow2() { 828 return actionIf(LegalizeAction::Lower, 829 LegalityPredicates::memSizeInBytesNotPow2(0)); 830 } 831 832 /// Lower a memory operation if the memory access size is not a round power of 833 /// 2 byte size. This is stricter than lowerIfMemSizeNotPow2, and more likely 834 /// what you want (e.g. this will lower s1, s7 and s24). 835 LegalizeRuleSet &lowerIfMemSizeNotByteSizePow2() { 836 return actionIf(LegalizeAction::Lower, 837 LegalityPredicates::memSizeNotByteSizePow2(0)); 838 } 839 840 LegalizeRuleSet &customIf(LegalityPredicate Predicate) { 841 // We have no choice but conservatively assume that a custom action with a 842 // free-form user provided Predicate properly handles all type indices: 843 markAllIdxsAsCovered(); 844 return actionIf(LegalizeAction::Custom, Predicate); 845 } 846 LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { 847 return actionFor(LegalizeAction::Custom, Types); 848 } 849 850 /// The instruction is custom when type indexes 0 and 1 is any type pair in the 851 /// given list. 852 LegalizeRuleSet &customFor(std::initializer_list<std::pair<LLT, LLT>> Types) { 853 return actionFor(LegalizeAction::Custom, Types); 854 } 855 856 LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { 857 return actionForCartesianProduct(LegalizeAction::Custom, Types); 858 } 859 /// The instruction is custom when type indexes 0 and 1 are both in their 860 /// respective lists. 861 LegalizeRuleSet & 862 customForCartesianProduct(std::initializer_list<LLT> Types0, 863 std::initializer_list<LLT> Types1) { 864 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); 865 } 866 /// The instruction is custom when type indexes 0, 1, and 2 are all in 867 /// their respective lists. 868 LegalizeRuleSet & 869 customForCartesianProduct(std::initializer_list<LLT> Types0, 870 std::initializer_list<LLT> Types1, 871 std::initializer_list<LLT> Types2) { 872 return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1, 873 Types2); 874 } 875 876 /// Unconditionally custom lower. 877 LegalizeRuleSet &custom() { 878 return customIf(always); 879 } 880 881 /// Widen the scalar to the next power of two that is at least MinSize. 882 /// No effect if the type is not a scalar or is a power of two. 883 LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, 884 unsigned MinSize = 0) { 885 using namespace LegalityPredicates; 886 return actionIf( 887 LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), 888 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 889 } 890 891 /// Widen the scalar to the next multiple of Size. No effect if the 892 /// type is not a scalar or is a multiple of Size. 893 LegalizeRuleSet &widenScalarToNextMultipleOf(unsigned TypeIdx, 894 unsigned Size) { 895 using namespace LegalityPredicates; 896 return actionIf( 897 LegalizeAction::WidenScalar, sizeNotMultipleOf(typeIdx(TypeIdx), Size), 898 LegalizeMutations::widenScalarOrEltToNextMultipleOf(TypeIdx, Size)); 899 } 900 901 /// Widen the scalar or vector element type to the next power of two that is 902 /// at least MinSize. No effect if the scalar size is a power of two. 903 LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, 904 unsigned MinSize = 0) { 905 using namespace LegalityPredicates; 906 return actionIf( 907 LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)), 908 LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); 909 } 910 911 LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { 912 using namespace LegalityPredicates; 913 return actionIf(LegalizeAction::NarrowScalar, isScalar(typeIdx(TypeIdx)), 914 Mutation); 915 } 916 917 LegalizeRuleSet &scalarize(unsigned TypeIdx) { 918 using namespace LegalityPredicates; 919 return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)), 920 LegalizeMutations::scalarize(TypeIdx)); 921 } 922 923 LegalizeRuleSet &scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx) { 924 using namespace LegalityPredicates; 925 return actionIf(LegalizeAction::FewerElements, 926 all(Predicate, isVector(typeIdx(TypeIdx))), 927 LegalizeMutations::scalarize(TypeIdx)); 928 } 929 930 /// Ensure the scalar or element is at least as wide as Ty. 931 LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT Ty) { 932 using namespace LegalityPredicates; 933 using namespace LegalizeMutations; 934 return actionIf(LegalizeAction::WidenScalar, 935 scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()), 936 changeElementTo(typeIdx(TypeIdx), Ty)); 937 } 938 939 /// Ensure the scalar or element is at least as wide as Ty. 940 LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate, 941 unsigned TypeIdx, const LLT Ty) { 942 using namespace LegalityPredicates; 943 using namespace LegalizeMutations; 944 return actionIf(LegalizeAction::WidenScalar, 945 all(Predicate, scalarOrEltNarrowerThan( 946 TypeIdx, Ty.getScalarSizeInBits())), 947 changeElementTo(typeIdx(TypeIdx), Ty)); 948 } 949 950 /// Ensure the vector size is at least as wide as VectorSize by promoting the 951 /// element. 952 LegalizeRuleSet &widenVectorEltsToVectorMinSize(unsigned TypeIdx, 953 unsigned VectorSize) { 954 using namespace LegalityPredicates; 955 using namespace LegalizeMutations; 956 return actionIf( 957 LegalizeAction::WidenScalar, 958 [=](const LegalityQuery &Query) { 959 const LLT VecTy = Query.Types[TypeIdx]; 960 return VecTy.isVector() && !VecTy.isScalable() && 961 VecTy.getSizeInBits() < VectorSize; 962 }, 963 [=](const LegalityQuery &Query) { 964 const LLT VecTy = Query.Types[TypeIdx]; 965 unsigned NumElts = VecTy.getNumElements(); 966 unsigned MinSize = VectorSize / NumElts; 967 LLT NewTy = LLT::fixed_vector(NumElts, LLT::scalar(MinSize)); 968 return std::make_pair(TypeIdx, NewTy); 969 }); 970 } 971 972 /// Ensure the scalar is at least as wide as Ty. 973 LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT Ty) { 974 using namespace LegalityPredicates; 975 using namespace LegalizeMutations; 976 return actionIf(LegalizeAction::WidenScalar, 977 scalarNarrowerThan(TypeIdx, Ty.getSizeInBits()), 978 changeTo(typeIdx(TypeIdx), Ty)); 979 } 980 981 /// Ensure the scalar is at least as wide as Ty if condition is met. 982 LegalizeRuleSet &minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, 983 const LLT Ty) { 984 using namespace LegalityPredicates; 985 using namespace LegalizeMutations; 986 return actionIf( 987 LegalizeAction::WidenScalar, 988 [=](const LegalityQuery &Query) { 989 const LLT QueryTy = Query.Types[TypeIdx]; 990 return QueryTy.isScalar() && 991 QueryTy.getSizeInBits() < Ty.getSizeInBits() && 992 Predicate(Query); 993 }, 994 changeTo(typeIdx(TypeIdx), Ty)); 995 } 996 997 /// Ensure the scalar is at most as wide as Ty. 998 LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT Ty) { 999 using namespace LegalityPredicates; 1000 using namespace LegalizeMutations; 1001 return actionIf(LegalizeAction::NarrowScalar, 1002 scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()), 1003 changeElementTo(typeIdx(TypeIdx), Ty)); 1004 } 1005 1006 /// Ensure the scalar is at most as wide as Ty. 1007 LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT Ty) { 1008 using namespace LegalityPredicates; 1009 using namespace LegalizeMutations; 1010 return actionIf(LegalizeAction::NarrowScalar, 1011 scalarWiderThan(TypeIdx, Ty.getSizeInBits()), 1012 changeTo(typeIdx(TypeIdx), Ty)); 1013 } 1014 1015 /// Conditionally limit the maximum size of the scalar. 1016 /// For example, when the maximum size of one type depends on the size of 1017 /// another such as extracting N bits from an M bit container. 1018 LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, 1019 const LLT Ty) { 1020 using namespace LegalityPredicates; 1021 using namespace LegalizeMutations; 1022 return actionIf( 1023 LegalizeAction::NarrowScalar, 1024 [=](const LegalityQuery &Query) { 1025 const LLT QueryTy = Query.Types[TypeIdx]; 1026 return QueryTy.isScalar() && 1027 QueryTy.getSizeInBits() > Ty.getSizeInBits() && 1028 Predicate(Query); 1029 }, 1030 changeElementTo(typeIdx(TypeIdx), Ty)); 1031 } 1032 1033 /// Limit the range of scalar sizes to MinTy and MaxTy. 1034 LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT MinTy, 1035 const LLT MaxTy) { 1036 assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types"); 1037 return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); 1038 } 1039 1040 /// Limit the range of scalar sizes to MinTy and MaxTy. 1041 LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, 1042 const LLT MaxTy) { 1043 return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy); 1044 } 1045 1046 /// Widen the scalar to match the size of another. 1047 LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { 1048 typeIdx(TypeIdx); 1049 return widenScalarIf( 1050 [=](const LegalityQuery &Query) { 1051 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 1052 Query.Types[TypeIdx].getSizeInBits(); 1053 }, 1054 LegalizeMutations::changeElementSizeTo(TypeIdx, LargeTypeIdx)); 1055 } 1056 1057 /// Narrow the scalar to match the size of another. 1058 LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) { 1059 typeIdx(TypeIdx); 1060 return narrowScalarIf( 1061 [=](const LegalityQuery &Query) { 1062 return Query.Types[NarrowTypeIdx].getScalarSizeInBits() < 1063 Query.Types[TypeIdx].getSizeInBits(); 1064 }, 1065 LegalizeMutations::changeElementSizeTo(TypeIdx, NarrowTypeIdx)); 1066 } 1067 1068 /// Change the type \p TypeIdx to have the same scalar size as type \p 1069 /// SameSizeIdx. 1070 LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) { 1071 return minScalarSameAs(TypeIdx, SameSizeIdx) 1072 .maxScalarSameAs(TypeIdx, SameSizeIdx); 1073 } 1074 1075 /// Conditionally widen the scalar or elt to match the size of another. 1076 LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate, 1077 unsigned TypeIdx, unsigned LargeTypeIdx) { 1078 typeIdx(TypeIdx); 1079 return widenScalarIf( 1080 [=](const LegalityQuery &Query) { 1081 return Query.Types[LargeTypeIdx].getScalarSizeInBits() > 1082 Query.Types[TypeIdx].getScalarSizeInBits() && 1083 Predicate(Query); 1084 }, 1085 [=](const LegalityQuery &Query) { 1086 LLT T = Query.Types[LargeTypeIdx]; 1087 if (T.isVector() && T.getElementType().isPointer()) 1088 T = T.changeElementType(LLT::scalar(T.getScalarSizeInBits())); 1089 return std::make_pair(TypeIdx, T); 1090 }); 1091 } 1092 1093 /// Conditionally narrow the scalar or elt to match the size of another. 1094 LegalizeRuleSet &maxScalarEltSameAsIf(LegalityPredicate Predicate, 1095 unsigned TypeIdx, 1096 unsigned SmallTypeIdx) { 1097 typeIdx(TypeIdx); 1098 return narrowScalarIf( 1099 [=](const LegalityQuery &Query) { 1100 return Query.Types[SmallTypeIdx].getScalarSizeInBits() < 1101 Query.Types[TypeIdx].getScalarSizeInBits() && 1102 Predicate(Query); 1103 }, 1104 [=](const LegalityQuery &Query) { 1105 LLT T = Query.Types[SmallTypeIdx]; 1106 return std::make_pair(TypeIdx, T); 1107 }); 1108 } 1109 1110 /// Add more elements to the vector to reach the next power of two. 1111 /// No effect if the type is not a vector or the element count is a power of 1112 /// two. 1113 LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) { 1114 using namespace LegalityPredicates; 1115 return actionIf(LegalizeAction::MoreElements, 1116 numElementsNotPow2(typeIdx(TypeIdx)), 1117 LegalizeMutations::moreElementsToNextPow2(TypeIdx)); 1118 } 1119 1120 /// Limit the number of elements in EltTy vectors to at least MinElements. 1121 LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT EltTy, 1122 unsigned MinElements) { 1123 // Mark the type index as covered: 1124 typeIdx(TypeIdx); 1125 return actionIf( 1126 LegalizeAction::MoreElements, 1127 [=](const LegalityQuery &Query) { 1128 LLT VecTy = Query.Types[TypeIdx]; 1129 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1130 VecTy.getNumElements() < MinElements; 1131 }, 1132 [=](const LegalityQuery &Query) { 1133 LLT VecTy = Query.Types[TypeIdx]; 1134 return std::make_pair( 1135 TypeIdx, LLT::fixed_vector(MinElements, VecTy.getElementType())); 1136 }); 1137 } 1138 1139 /// Set number of elements to nearest larger multiple of NumElts. 1140 LegalizeRuleSet &alignNumElementsTo(unsigned TypeIdx, const LLT EltTy, 1141 unsigned NumElts) { 1142 typeIdx(TypeIdx); 1143 return actionIf( 1144 LegalizeAction::MoreElements, 1145 [=](const LegalityQuery &Query) { 1146 LLT VecTy = Query.Types[TypeIdx]; 1147 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1148 (VecTy.getNumElements() % NumElts != 0); 1149 }, 1150 [=](const LegalityQuery &Query) { 1151 LLT VecTy = Query.Types[TypeIdx]; 1152 unsigned NewSize = alignTo(VecTy.getNumElements(), NumElts); 1153 return std::make_pair( 1154 TypeIdx, LLT::fixed_vector(NewSize, VecTy.getElementType())); 1155 }); 1156 } 1157 1158 /// Limit the number of elements in EltTy vectors to at most MaxElements. 1159 LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, 1160 unsigned MaxElements) { 1161 // Mark the type index as covered: 1162 typeIdx(TypeIdx); 1163 return actionIf( 1164 LegalizeAction::FewerElements, 1165 [=](const LegalityQuery &Query) { 1166 LLT VecTy = Query.Types[TypeIdx]; 1167 return VecTy.isVector() && VecTy.getElementType() == EltTy && 1168 VecTy.getNumElements() > MaxElements; 1169 }, 1170 [=](const LegalityQuery &Query) { 1171 LLT VecTy = Query.Types[TypeIdx]; 1172 LLT NewTy = LLT::scalarOrVector(ElementCount::getFixed(MaxElements), 1173 VecTy.getElementType()); 1174 return std::make_pair(TypeIdx, NewTy); 1175 }); 1176 } 1177 /// Limit the number of elements for the given vectors to at least MinTy's 1178 /// number of elements and at most MaxTy's number of elements. 1179 /// 1180 /// No effect if the type is not a vector or does not have the same element 1181 /// type as the constraints. 1182 /// The element type of MinTy and MaxTy must match. 1183 LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT MinTy, 1184 const LLT MaxTy) { 1185 assert(MinTy.getElementType() == MaxTy.getElementType() && 1186 "Expected element types to agree"); 1187 1188 const LLT EltTy = MinTy.getElementType(); 1189 return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements()) 1190 .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements()); 1191 } 1192 1193 /// Express \p EltTy vectors strictly using vectors with \p NumElts elements 1194 /// (or scalars when \p NumElts equals 1). 1195 /// First pad with undef elements to nearest larger multiple of \p NumElts. 1196 /// Then perform split with all sub-instructions having the same type. 1197 /// Using clampMaxNumElements (non-strict) can result in leftover instruction 1198 /// with different type (fewer elements then \p NumElts or scalar). 1199 /// No effect if the type is not a vector. 1200 LegalizeRuleSet &clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, 1201 unsigned NumElts) { 1202 return alignNumElementsTo(TypeIdx, EltTy, NumElts) 1203 .clampMaxNumElements(TypeIdx, EltTy, NumElts); 1204 } 1205 1206 /// Fallback on the previous implementation. This should only be used while 1207 /// porting a rule. 1208 LegalizeRuleSet &fallback() { 1209 add({always, LegalizeAction::UseLegacyRules}); 1210 return *this; 1211 } 1212 1213 /// Check if there is no type index which is obviously not handled by the 1214 /// LegalizeRuleSet in any way at all. 1215 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1216 bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const; 1217 /// Check if there is no imm index which is obviously not handled by the 1218 /// LegalizeRuleSet in any way at all. 1219 /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. 1220 bool verifyImmIdxsCoverage(unsigned NumImmIdxs) const; 1221 1222 /// Apply the ruleset to the given LegalityQuery. 1223 LegalizeActionStep apply(const LegalityQuery &Query) const; 1224}; 1225 1226class LegalizerInfo { 1227public: 1228 virtual ~LegalizerInfo() = default; 1229 1230 const LegacyLegalizerInfo &getLegacyLegalizerInfo() const { 1231 return LegacyInfo; 1232 } 1233 LegacyLegalizerInfo &getLegacyLegalizerInfo() { return LegacyInfo; } 1234 1235 unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; 1236 unsigned getActionDefinitionsIdx(unsigned Opcode) const; 1237 1238 /// Perform simple self-diagnostic and assert if there is anything obviously 1239 /// wrong with the actions set up. 1240 void verify(const MCInstrInfo &MII) const; 1241 1242 /// Get the action definitions for the given opcode. Use this to run a 1243 /// LegalityQuery through the definitions. 1244 const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; 1245 1246 /// Get the action definition builder for the given opcode. Use this to define 1247 /// the action definitions. 1248 /// 1249 /// It is an error to request an opcode that has already been requested by the 1250 /// multiple-opcode variant. 1251 LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode); 1252 1253 /// Get the action definition builder for the given set of opcodes. Use this 1254 /// to define the action definitions for multiple opcodes at once. The first 1255 /// opcode given will be considered the representative opcode and will hold 1256 /// the definitions whereas the other opcodes will be configured to refer to 1257 /// the representative opcode. This lowers memory requirements and very 1258 /// slightly improves performance. 1259 /// 1260 /// It would be very easy to introduce unexpected side-effects as a result of 1261 /// this aliasing if it were permitted to request different but intersecting 1262 /// sets of opcodes but that is difficult to keep track of. It is therefore an 1263 /// error to request the same opcode twice using this API, to request an 1264 /// opcode that already has definitions, or to use the single-opcode API on an 1265 /// opcode that has already been requested by this API. 1266 LegalizeRuleSet & 1267 getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes); 1268 void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom); 1269 1270 /// Determine what action should be taken to legalize the described 1271 /// instruction. Requires computeTables to have been called. 1272 /// 1273 /// \returns a description of the next legalization step to perform. 1274 LegalizeActionStep getAction(const LegalityQuery &Query) const; 1275 1276 /// Determine what action should be taken to legalize the given generic 1277 /// instruction. 1278 /// 1279 /// \returns a description of the next legalization step to perform. 1280 LegalizeActionStep getAction(const MachineInstr &MI, 1281 const MachineRegisterInfo &MRI) const; 1282 1283 bool isLegal(const LegalityQuery &Query) const { 1284 return getAction(Query).Action == LegalizeAction::Legal; 1285 } 1286 1287 bool isLegalOrCustom(const LegalityQuery &Query) const { 1288 auto Action = getAction(Query).Action; 1289 return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom; 1290 } 1291 1292 bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; 1293 bool isLegalOrCustom(const MachineInstr &MI, 1294 const MachineRegisterInfo &MRI) const; 1295 1296 /// Called for instructions with the Custom LegalizationAction. 1297 virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, 1298 LostDebugLocObserver &LocObserver) const { 1299 llvm_unreachable("must implement this if custom action is used"); 1300 } 1301 1302 /// \returns true if MI is either legal or has been legalized and false if not 1303 /// legal. 1304 /// Return true if MI is either legal or has been legalized and false 1305 /// if not legal. 1306 virtual bool legalizeIntrinsic(LegalizerHelper &Helper, 1307 MachineInstr &MI) const { 1308 return true; 1309 } 1310 1311 /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while 1312 /// widening a constant of type SmallTy which targets can override. 1313 /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which 1314 /// will be the default. 1315 virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const; 1316 1317private: 1318 static const int FirstOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_START; 1319 static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; 1320 1321 LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; 1322 LegacyLegalizerInfo LegacyInfo; 1323}; 1324 1325#ifndef NDEBUG 1326/// Checks that MIR is fully legal, returns an illegal instruction if it's not, 1327/// nullptr otherwise 1328const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF); 1329#endif 1330 1331} // end namespace llvm. 1332 1333#endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H 1334