SpeculativeExecution.cpp revision 360784 
1//===- SpeculativeExecution.cpp ---------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass hoists instructions to enable speculative execution on
10// targets where branches are expensive. This is aimed at GPUs. It
11// currently works on simple if-then and if-then-else
12// patterns.
13//
14// Removing branches is not the only motivation for this
15// pass. E.g. consider this code and assume that there is no
16// addressing mode for multiplying by sizeof(*a):
17//
18//   if (b > 0)
19//     c = a[i + 1]
20//   if (d > 0)
21//     e = a[i + 2]
22//
23// turns into
24//
25//   p = &a[i + 1];
26//   if (b > 0)
27//     c = *p;
28//   q = &a[i + 2];
29//   if (d > 0)
30//     e = *q;
31//
32// which could later be optimized to
33//
34//   r = &a[i];
35//   if (b > 0)
36//     c = r[1];
37//   if (d > 0)
38//     e = r[2];
39//
40// Later passes sink back much of the speculated code that did not enable
41// further optimization.
42//
43// This pass is more aggressive than the function SpeculativeyExecuteBB in
44// SimplifyCFG. SimplifyCFG will not speculate if no selects are introduced and
45// it will speculate at most one instruction. It also will not speculate if
46// there is a value defined in the if-block that is only used in the then-block.
47// These restrictions make sense since the speculation in SimplifyCFG seems
48// aimed at introducing cheap selects, while this pass is intended to do more
49// aggressive speculation while counting on later passes to either capitalize on
50// that or clean it up.
51//
52// If the pass was created by calling
53// createSpeculativeExecutionIfHasBranchDivergencePass or the
54// -spec-exec-only-if-divergent-target option is present, this pass only has an
55// effect on targets where TargetTransformInfo::hasBranchDivergence() is true;
56// on other targets, it is a nop.
57//
58// This lets you include this pass unconditionally in the IR pass pipeline, but
59// only enable it for relevant targets.
60//
61//===----------------------------------------------------------------------===//
62
63#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
64#include "llvm/ADT/SmallPtrSet.h"
65#include "llvm/Analysis/GlobalsModRef.h"
66#include "llvm/Analysis/ValueTracking.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/Module.h"
69#include "llvm/IR/Operator.h"
70#include "llvm/InitializePasses.h"
71#include "llvm/Support/CommandLine.h"
72#include "llvm/Support/Debug.h"
73
74using namespace llvm;
75
76#define DEBUG_TYPE "speculative-execution"
77
78// The risk that speculation will not pay off increases with the
79// number of instructions speculated, so we put a limit on that.
80static cl::opt<unsigned> SpecExecMaxSpeculationCost(
81    "spec-exec-max-speculation-cost", cl::init(7), cl::Hidden,
82    cl::desc("Speculative execution is not applied to basic blocks where "
83             "the cost of the instructions to speculatively execute "
84             "exceeds this limit."));
85
86// Speculating just a few instructions from a larger block tends not
87// to be profitable and this limit prevents that. A reason for that is
88// that small basic blocks are more likely to be candidates for
89// further optimization.
90static cl::opt<unsigned> SpecExecMaxNotHoisted(
91    "spec-exec-max-not-hoisted", cl::init(5), cl::Hidden,
92    cl::desc("Speculative execution is not applied to basic blocks where the "
93             "number of instructions that would not be speculatively executed "
94             "exceeds this limit."));
95
96static cl::opt<bool> SpecExecOnlyIfDivergentTarget(
97    "spec-exec-only-if-divergent-target", cl::init(false), cl::Hidden,
98    cl::desc("Speculative execution is applied only to targets with divergent "
99             "branches, even if the pass was configured to apply only to all "
100             "targets."));
101
102namespace {
103
104class SpeculativeExecutionLegacyPass : public FunctionPass {
105public:
106  static char ID;
107  explicit SpeculativeExecutionLegacyPass(bool OnlyIfDivergentTarget = false)
108      : FunctionPass(ID), OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
109                                                SpecExecOnlyIfDivergentTarget),
110        Impl(OnlyIfDivergentTarget) {}
111
112  void getAnalysisUsage(AnalysisUsage &AU) const override;
113  bool runOnFunction(Function &F) override;
114
115  StringRef getPassName() const override {
116    if (OnlyIfDivergentTarget)
117      return "Speculatively execute instructions if target has divergent "
118             "branches";
119    return "Speculatively execute instructions";
120  }
121
122private:
123  // Variable preserved purely for correct name printing.
124  const bool OnlyIfDivergentTarget;
125
126  SpeculativeExecutionPass Impl;
127};
128} // namespace
129
130char SpeculativeExecutionLegacyPass::ID = 0;
131INITIALIZE_PASS_BEGIN(SpeculativeExecutionLegacyPass, "speculative-execution",
132                      "Speculatively execute instructions", false, false)
133INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
134INITIALIZE_PASS_END(SpeculativeExecutionLegacyPass, "speculative-execution",
135                    "Speculatively execute instructions", false, false)
136
137void SpeculativeExecutionLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const {
138  AU.addRequired<TargetTransformInfoWrapperPass>();
139  AU.addPreserved<GlobalsAAWrapperPass>();
140  AU.setPreservesCFG();
141}
142
143bool SpeculativeExecutionLegacyPass::runOnFunction(Function &F) {
144  if (skipFunction(F))
145    return false;
146
147  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
148  return Impl.runImpl(F, TTI);
149}
150
151namespace llvm {
152
153bool SpeculativeExecutionPass::runImpl(Function &F, TargetTransformInfo *TTI) {
154  if (OnlyIfDivergentTarget && !TTI->hasBranchDivergence()) {
155    LLVM_DEBUG(dbgs() << "Not running SpeculativeExecution because "
156                         "TTI->hasBranchDivergence() is false.\n");
157    return false;
158  }
159
160  this->TTI = TTI;
161  bool Changed = false;
162  for (auto& B : F) {
163    Changed |= runOnBasicBlock(B);
164  }
165  return Changed;
166}
167
168bool SpeculativeExecutionPass::runOnBasicBlock(BasicBlock &B) {
169  BranchInst *BI = dyn_cast<BranchInst>(B.getTerminator());
170  if (BI == nullptr)
171    return false;
172
173  if (BI->getNumSuccessors() != 2)
174    return false;
175  BasicBlock &Succ0 = *BI->getSuccessor(0);
176  BasicBlock &Succ1 = *BI->getSuccessor(1);
177
178  if (&B == &Succ0 || &B == &Succ1 || &Succ0 == &Succ1) {
179    return false;
180  }
181
182  // Hoist from if-then (triangle).
183  if (Succ0.getSinglePredecessor() != nullptr &&
184      Succ0.getSingleSuccessor() == &Succ1) {
185    return considerHoistingFromTo(Succ0, B);
186  }
187
188  // Hoist from if-else (triangle).
189  if (Succ1.getSinglePredecessor() != nullptr &&
190      Succ1.getSingleSuccessor() == &Succ0) {
191    return considerHoistingFromTo(Succ1, B);
192  }
193
194  // Hoist from if-then-else (diamond), but only if it is equivalent to
195  // an if-else or if-then due to one of the branches doing nothing.
196  if (Succ0.getSinglePredecessor() != nullptr &&
197      Succ1.getSinglePredecessor() != nullptr &&
198      Succ1.getSingleSuccessor() != nullptr &&
199      Succ1.getSingleSuccessor() != &B &&
200      Succ1.getSingleSuccessor() == Succ0.getSingleSuccessor()) {
201    // If a block has only one instruction, then that is a terminator
202    // instruction so that the block does nothing. This does happen.
203    if (Succ1.size() == 1) // equivalent to if-then
204      return considerHoistingFromTo(Succ0, B);
205    if (Succ0.size() == 1) // equivalent to if-else
206      return considerHoistingFromTo(Succ1, B);
207  }
208
209  return false;
210}
211
212static unsigned ComputeSpeculationCost(const Instruction *I,
213                                       const TargetTransformInfo &TTI) {
214  switch (Operator::getOpcode(I)) {
215    case Instruction::GetElementPtr:
216    case Instruction::Add:
217    case Instruction::Mul:
218    case Instruction::And:
219    case Instruction::Or:
220    case Instruction::Select:
221    case Instruction::Shl:
222    case Instruction::Sub:
223    case Instruction::LShr:
224    case Instruction::AShr:
225    case Instruction::Xor:
226    case Instruction::ZExt:
227    case Instruction::SExt:
228    case Instruction::Call:
229    case Instruction::BitCast:
230    case Instruction::PtrToInt:
231    case Instruction::IntToPtr:
232    case Instruction::AddrSpaceCast:
233    case Instruction::FPToUI:
234    case Instruction::FPToSI:
235    case Instruction::UIToFP:
236    case Instruction::SIToFP:
237    case Instruction::FPExt:
238    case Instruction::FPTrunc:
239    case Instruction::FAdd:
240    case Instruction::FSub:
241    case Instruction::FMul:
242    case Instruction::FDiv:
243    case Instruction::FRem:
244    case Instruction::FNeg:
245    case Instruction::ICmp:
246    case Instruction::FCmp:
247      return TTI.getUserCost(I);
248
249    default:
250      return UINT_MAX; // Disallow anything not whitelisted.
251  }
252}
253
254bool SpeculativeExecutionPass::considerHoistingFromTo(
255    BasicBlock &FromBlock, BasicBlock &ToBlock) {
256  SmallPtrSet<const Instruction *, 8> NotHoisted;
257  const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](User *U) {
258    for (Value* V : U->operand_values()) {
259      if (Instruction *I = dyn_cast<Instruction>(V)) {
260        if (NotHoisted.count(I) > 0)
261          return false;
262      }
263    }
264    return true;
265  };
266
267  unsigned TotalSpeculationCost = 0;
268  for (auto& I : FromBlock) {
269    const unsigned Cost = ComputeSpeculationCost(&I, *TTI);
270    if (Cost != UINT_MAX && isSafeToSpeculativelyExecute(&I) &&
271        AllPrecedingUsesFromBlockHoisted(&I)) {
272      TotalSpeculationCost += Cost;
273      if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
274        return false;  // too much to hoist
275    } else {
276      NotHoisted.insert(&I);
277      if (NotHoisted.size() > SpecExecMaxNotHoisted)
278        return false; // too much left behind
279    }
280  }
281
282  if (TotalSpeculationCost == 0)
283    return false; // nothing to hoist
284
285  for (auto I = FromBlock.begin(); I != FromBlock.end();) {
286    // We have to increment I before moving Current as moving Current
287    // changes the list that I is iterating through.
288    auto Current = I;
289    ++I;
290    if (!NotHoisted.count(&*Current)) {
291      Current->moveBefore(ToBlock.getTerminator());
292    }
293  }
294  return true;
295}
296
297FunctionPass *createSpeculativeExecutionPass() {
298  return new SpeculativeExecutionLegacyPass();
299}
300
301FunctionPass *createSpeculativeExecutionIfHasBranchDivergencePass() {
302  return new SpeculativeExecutionLegacyPass(/* OnlyIfDivergentTarget = */ true);
303}
304
305SpeculativeExecutionPass::SpeculativeExecutionPass(bool OnlyIfDivergentTarget)
306    : OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
307                            SpecExecOnlyIfDivergentTarget) {}
308
309PreservedAnalyses SpeculativeExecutionPass::run(Function &F,
310                                                FunctionAnalysisManager &AM) {
311  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
312
313  bool Changed = runImpl(F, TTI);
314
315  if (!Changed)
316    return PreservedAnalyses::all();
317  PreservedAnalyses PA;
318  PA.preserve<GlobalsAA>();
319  PA.preserveSet<CFGAnalyses>();
320  return PA;
321}
322}  // namespace llvm
323