CGStmt.cpp revision 360784 
1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Stmt nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGDebugInfo.h"
14#include "CodeGenFunction.h"
15#include "CodeGenModule.h"
16#include "TargetInfo.h"
17#include "clang/AST/Attr.h"
18#include "clang/AST/StmtVisitor.h"
19#include "clang/Basic/Builtins.h"
20#include "clang/Basic/PrettyStackTrace.h"
21#include "clang/Basic/TargetInfo.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/InlineAsm.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/MDBuilder.h"
27
28using namespace clang;
29using namespace CodeGen;
30
31//===----------------------------------------------------------------------===//
32//                              Statement Emission
33//===----------------------------------------------------------------------===//
34
35void CodeGenFunction::EmitStopPoint(const Stmt *S) {
36  if (CGDebugInfo *DI = getDebugInfo()) {
37    SourceLocation Loc;
38    Loc = S->getBeginLoc();
39    DI->EmitLocation(Builder, Loc);
40
41    LastStopPoint = Loc;
42  }
43}
44
45void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
46  assert(S && "Null statement?");
47  PGO.setCurrentStmt(S);
48
49  // These statements have their own debug info handling.
50  if (EmitSimpleStmt(S))
51    return;
52
53  // Check if we are generating unreachable code.
54  if (!HaveInsertPoint()) {
55    // If so, and the statement doesn't contain a label, then we do not need to
56    // generate actual code. This is safe because (1) the current point is
57    // unreachable, so we don't need to execute the code, and (2) we've already
58    // handled the statements which update internal data structures (like the
59    // local variable map) which could be used by subsequent statements.
60    if (!ContainsLabel(S)) {
61      // Verify that any decl statements were handled as simple, they may be in
62      // scope of subsequent reachable statements.
63      assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
64      return;
65    }
66
67    // Otherwise, make a new block to hold the code.
68    EnsureInsertPoint();
69  }
70
71  // Generate a stoppoint if we are emitting debug info.
72  EmitStopPoint(S);
73
74  // Ignore all OpenMP directives except for simd if OpenMP with Simd is
75  // enabled.
76  if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
77    if (const auto *D = dyn_cast<OMPExecutableDirective>(S)) {
78      EmitSimpleOMPExecutableDirective(*D);
79      return;
80    }
81  }
82
83  switch (S->getStmtClass()) {
84  case Stmt::NoStmtClass:
85  case Stmt::CXXCatchStmtClass:
86  case Stmt::SEHExceptStmtClass:
87  case Stmt::SEHFinallyStmtClass:
88  case Stmt::MSDependentExistsStmtClass:
89    llvm_unreachable("invalid statement class to emit generically");
90  case Stmt::NullStmtClass:
91  case Stmt::CompoundStmtClass:
92  case Stmt::DeclStmtClass:
93  case Stmt::LabelStmtClass:
94  case Stmt::AttributedStmtClass:
95  case Stmt::GotoStmtClass:
96  case Stmt::BreakStmtClass:
97  case Stmt::ContinueStmtClass:
98  case Stmt::DefaultStmtClass:
99  case Stmt::CaseStmtClass:
100  case Stmt::SEHLeaveStmtClass:
101    llvm_unreachable("should have emitted these statements as simple");
102
103#define STMT(Type, Base)
104#define ABSTRACT_STMT(Op)
105#define EXPR(Type, Base) \
106  case Stmt::Type##Class:
107#include "clang/AST/StmtNodes.inc"
108  {
109    // Remember the block we came in on.
110    llvm::BasicBlock *incoming = Builder.GetInsertBlock();
111    assert(incoming && "expression emission must have an insertion point");
112
113    EmitIgnoredExpr(cast<Expr>(S));
114
115    llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
116    assert(outgoing && "expression emission cleared block!");
117
118    // The expression emitters assume (reasonably!) that the insertion
119    // point is always set.  To maintain that, the call-emission code
120    // for noreturn functions has to enter a new block with no
121    // predecessors.  We want to kill that block and mark the current
122    // insertion point unreachable in the common case of a call like
123    // "exit();".  Since expression emission doesn't otherwise create
124    // blocks with no predecessors, we can just test for that.
125    // However, we must be careful not to do this to our incoming
126    // block, because *statement* emission does sometimes create
127    // reachable blocks which will have no predecessors until later in
128    // the function.  This occurs with, e.g., labels that are not
129    // reachable by fallthrough.
130    if (incoming != outgoing && outgoing->use_empty()) {
131      outgoing->eraseFromParent();
132      Builder.ClearInsertionPoint();
133    }
134    break;
135  }
136
137  case Stmt::IndirectGotoStmtClass:
138    EmitIndirectGotoStmt(cast<IndirectGotoStmt>(*S)); break;
139
140  case Stmt::IfStmtClass:      EmitIfStmt(cast<IfStmt>(*S));              break;
141  case Stmt::WhileStmtClass:   EmitWhileStmt(cast<WhileStmt>(*S), Attrs); break;
142  case Stmt::DoStmtClass:      EmitDoStmt(cast<DoStmt>(*S), Attrs);       break;
143  case Stmt::ForStmtClass:     EmitForStmt(cast<ForStmt>(*S), Attrs);     break;
144
145  case Stmt::ReturnStmtClass:  EmitReturnStmt(cast<ReturnStmt>(*S));      break;
146
147  case Stmt::SwitchStmtClass:  EmitSwitchStmt(cast<SwitchStmt>(*S));      break;
148  case Stmt::GCCAsmStmtClass:  // Intentional fall-through.
149  case Stmt::MSAsmStmtClass:   EmitAsmStmt(cast<AsmStmt>(*S));            break;
150  case Stmt::CoroutineBodyStmtClass:
151    EmitCoroutineBody(cast<CoroutineBodyStmt>(*S));
152    break;
153  case Stmt::CoreturnStmtClass:
154    EmitCoreturnStmt(cast<CoreturnStmt>(*S));
155    break;
156  case Stmt::CapturedStmtClass: {
157    const CapturedStmt *CS = cast<CapturedStmt>(S);
158    EmitCapturedStmt(*CS, CS->getCapturedRegionKind());
159    }
160    break;
161  case Stmt::ObjCAtTryStmtClass:
162    EmitObjCAtTryStmt(cast<ObjCAtTryStmt>(*S));
163    break;
164  case Stmt::ObjCAtCatchStmtClass:
165    llvm_unreachable(
166                    "@catch statements should be handled by EmitObjCAtTryStmt");
167  case Stmt::ObjCAtFinallyStmtClass:
168    llvm_unreachable(
169                  "@finally statements should be handled by EmitObjCAtTryStmt");
170  case Stmt::ObjCAtThrowStmtClass:
171    EmitObjCAtThrowStmt(cast<ObjCAtThrowStmt>(*S));
172    break;
173  case Stmt::ObjCAtSynchronizedStmtClass:
174    EmitObjCAtSynchronizedStmt(cast<ObjCAtSynchronizedStmt>(*S));
175    break;
176  case Stmt::ObjCForCollectionStmtClass:
177    EmitObjCForCollectionStmt(cast<ObjCForCollectionStmt>(*S));
178    break;
179  case Stmt::ObjCAutoreleasePoolStmtClass:
180    EmitObjCAutoreleasePoolStmt(cast<ObjCAutoreleasePoolStmt>(*S));
181    break;
182
183  case Stmt::CXXTryStmtClass:
184    EmitCXXTryStmt(cast<CXXTryStmt>(*S));
185    break;
186  case Stmt::CXXForRangeStmtClass:
187    EmitCXXForRangeStmt(cast<CXXForRangeStmt>(*S), Attrs);
188    break;
189  case Stmt::SEHTryStmtClass:
190    EmitSEHTryStmt(cast<SEHTryStmt>(*S));
191    break;
192  case Stmt::OMPParallelDirectiveClass:
193    EmitOMPParallelDirective(cast<OMPParallelDirective>(*S));
194    break;
195  case Stmt::OMPSimdDirectiveClass:
196    EmitOMPSimdDirective(cast<OMPSimdDirective>(*S));
197    break;
198  case Stmt::OMPForDirectiveClass:
199    EmitOMPForDirective(cast<OMPForDirective>(*S));
200    break;
201  case Stmt::OMPForSimdDirectiveClass:
202    EmitOMPForSimdDirective(cast<OMPForSimdDirective>(*S));
203    break;
204  case Stmt::OMPSectionsDirectiveClass:
205    EmitOMPSectionsDirective(cast<OMPSectionsDirective>(*S));
206    break;
207  case Stmt::OMPSectionDirectiveClass:
208    EmitOMPSectionDirective(cast<OMPSectionDirective>(*S));
209    break;
210  case Stmt::OMPSingleDirectiveClass:
211    EmitOMPSingleDirective(cast<OMPSingleDirective>(*S));
212    break;
213  case Stmt::OMPMasterDirectiveClass:
214    EmitOMPMasterDirective(cast<OMPMasterDirective>(*S));
215    break;
216  case Stmt::OMPCriticalDirectiveClass:
217    EmitOMPCriticalDirective(cast<OMPCriticalDirective>(*S));
218    break;
219  case Stmt::OMPParallelForDirectiveClass:
220    EmitOMPParallelForDirective(cast<OMPParallelForDirective>(*S));
221    break;
222  case Stmt::OMPParallelForSimdDirectiveClass:
223    EmitOMPParallelForSimdDirective(cast<OMPParallelForSimdDirective>(*S));
224    break;
225  case Stmt::OMPParallelMasterDirectiveClass:
226    EmitOMPParallelMasterDirective(cast<OMPParallelMasterDirective>(*S));
227    break;
228  case Stmt::OMPParallelSectionsDirectiveClass:
229    EmitOMPParallelSectionsDirective(cast<OMPParallelSectionsDirective>(*S));
230    break;
231  case Stmt::OMPTaskDirectiveClass:
232    EmitOMPTaskDirective(cast<OMPTaskDirective>(*S));
233    break;
234  case Stmt::OMPTaskyieldDirectiveClass:
235    EmitOMPTaskyieldDirective(cast<OMPTaskyieldDirective>(*S));
236    break;
237  case Stmt::OMPBarrierDirectiveClass:
238    EmitOMPBarrierDirective(cast<OMPBarrierDirective>(*S));
239    break;
240  case Stmt::OMPTaskwaitDirectiveClass:
241    EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
242    break;
243  case Stmt::OMPTaskgroupDirectiveClass:
244    EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));
245    break;
246  case Stmt::OMPFlushDirectiveClass:
247    EmitOMPFlushDirective(cast<OMPFlushDirective>(*S));
248    break;
249  case Stmt::OMPOrderedDirectiveClass:
250    EmitOMPOrderedDirective(cast<OMPOrderedDirective>(*S));
251    break;
252  case Stmt::OMPAtomicDirectiveClass:
253    EmitOMPAtomicDirective(cast<OMPAtomicDirective>(*S));
254    break;
255  case Stmt::OMPTargetDirectiveClass:
256    EmitOMPTargetDirective(cast<OMPTargetDirective>(*S));
257    break;
258  case Stmt::OMPTeamsDirectiveClass:
259    EmitOMPTeamsDirective(cast<OMPTeamsDirective>(*S));
260    break;
261  case Stmt::OMPCancellationPointDirectiveClass:
262    EmitOMPCancellationPointDirective(cast<OMPCancellationPointDirective>(*S));
263    break;
264  case Stmt::OMPCancelDirectiveClass:
265    EmitOMPCancelDirective(cast<OMPCancelDirective>(*S));
266    break;
267  case Stmt::OMPTargetDataDirectiveClass:
268    EmitOMPTargetDataDirective(cast<OMPTargetDataDirective>(*S));
269    break;
270  case Stmt::OMPTargetEnterDataDirectiveClass:
271    EmitOMPTargetEnterDataDirective(cast<OMPTargetEnterDataDirective>(*S));
272    break;
273  case Stmt::OMPTargetExitDataDirectiveClass:
274    EmitOMPTargetExitDataDirective(cast<OMPTargetExitDataDirective>(*S));
275    break;
276  case Stmt::OMPTargetParallelDirectiveClass:
277    EmitOMPTargetParallelDirective(cast<OMPTargetParallelDirective>(*S));
278    break;
279  case Stmt::OMPTargetParallelForDirectiveClass:
280    EmitOMPTargetParallelForDirective(cast<OMPTargetParallelForDirective>(*S));
281    break;
282  case Stmt::OMPTaskLoopDirectiveClass:
283    EmitOMPTaskLoopDirective(cast<OMPTaskLoopDirective>(*S));
284    break;
285  case Stmt::OMPTaskLoopSimdDirectiveClass:
286    EmitOMPTaskLoopSimdDirective(cast<OMPTaskLoopSimdDirective>(*S));
287    break;
288  case Stmt::OMPMasterTaskLoopDirectiveClass:
289    EmitOMPMasterTaskLoopDirective(cast<OMPMasterTaskLoopDirective>(*S));
290    break;
291  case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
292    EmitOMPMasterTaskLoopSimdDirective(
293        cast<OMPMasterTaskLoopSimdDirective>(*S));
294    break;
295  case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
296    EmitOMPParallelMasterTaskLoopDirective(
297        cast<OMPParallelMasterTaskLoopDirective>(*S));
298    break;
299  case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
300    EmitOMPParallelMasterTaskLoopSimdDirective(
301        cast<OMPParallelMasterTaskLoopSimdDirective>(*S));
302    break;
303  case Stmt::OMPDistributeDirectiveClass:
304    EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S));
305    break;
306  case Stmt::OMPTargetUpdateDirectiveClass:
307    EmitOMPTargetUpdateDirective(cast<OMPTargetUpdateDirective>(*S));
308    break;
309  case Stmt::OMPDistributeParallelForDirectiveClass:
310    EmitOMPDistributeParallelForDirective(
311        cast<OMPDistributeParallelForDirective>(*S));
312    break;
313  case Stmt::OMPDistributeParallelForSimdDirectiveClass:
314    EmitOMPDistributeParallelForSimdDirective(
315        cast<OMPDistributeParallelForSimdDirective>(*S));
316    break;
317  case Stmt::OMPDistributeSimdDirectiveClass:
318    EmitOMPDistributeSimdDirective(cast<OMPDistributeSimdDirective>(*S));
319    break;
320  case Stmt::OMPTargetParallelForSimdDirectiveClass:
321    EmitOMPTargetParallelForSimdDirective(
322        cast<OMPTargetParallelForSimdDirective>(*S));
323    break;
324  case Stmt::OMPTargetSimdDirectiveClass:
325    EmitOMPTargetSimdDirective(cast<OMPTargetSimdDirective>(*S));
326    break;
327  case Stmt::OMPTeamsDistributeDirectiveClass:
328    EmitOMPTeamsDistributeDirective(cast<OMPTeamsDistributeDirective>(*S));
329    break;
330  case Stmt::OMPTeamsDistributeSimdDirectiveClass:
331    EmitOMPTeamsDistributeSimdDirective(
332        cast<OMPTeamsDistributeSimdDirective>(*S));
333    break;
334  case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
335    EmitOMPTeamsDistributeParallelForSimdDirective(
336        cast<OMPTeamsDistributeParallelForSimdDirective>(*S));
337    break;
338  case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
339    EmitOMPTeamsDistributeParallelForDirective(
340        cast<OMPTeamsDistributeParallelForDirective>(*S));
341    break;
342  case Stmt::OMPTargetTeamsDirectiveClass:
343    EmitOMPTargetTeamsDirective(cast<OMPTargetTeamsDirective>(*S));
344    break;
345  case Stmt::OMPTargetTeamsDistributeDirectiveClass:
346    EmitOMPTargetTeamsDistributeDirective(
347        cast<OMPTargetTeamsDistributeDirective>(*S));
348    break;
349  case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
350    EmitOMPTargetTeamsDistributeParallelForDirective(
351        cast<OMPTargetTeamsDistributeParallelForDirective>(*S));
352    break;
353  case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
354    EmitOMPTargetTeamsDistributeParallelForSimdDirective(
355        cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S));
356    break;
357  case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
358    EmitOMPTargetTeamsDistributeSimdDirective(
359        cast<OMPTargetTeamsDistributeSimdDirective>(*S));
360    break;
361  }
362}
363
364bool CodeGenFunction::EmitSimpleStmt(const Stmt *S) {
365  switch (S->getStmtClass()) {
366  default: return false;
367  case Stmt::NullStmtClass: break;
368  case Stmt::CompoundStmtClass: EmitCompoundStmt(cast<CompoundStmt>(*S)); break;
369  case Stmt::DeclStmtClass:     EmitDeclStmt(cast<DeclStmt>(*S));         break;
370  case Stmt::LabelStmtClass:    EmitLabelStmt(cast<LabelStmt>(*S));       break;
371  case Stmt::AttributedStmtClass:
372                            EmitAttributedStmt(cast<AttributedStmt>(*S)); break;
373  case Stmt::GotoStmtClass:     EmitGotoStmt(cast<GotoStmt>(*S));         break;
374  case Stmt::BreakStmtClass:    EmitBreakStmt(cast<BreakStmt>(*S));       break;
375  case Stmt::ContinueStmtClass: EmitContinueStmt(cast<ContinueStmt>(*S)); break;
376  case Stmt::DefaultStmtClass:  EmitDefaultStmt(cast<DefaultStmt>(*S));   break;
377  case Stmt::CaseStmtClass:     EmitCaseStmt(cast<CaseStmt>(*S));         break;
378  case Stmt::SEHLeaveStmtClass: EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S)); break;
379  }
380
381  return true;
382}
383
384/// EmitCompoundStmt - Emit a compound statement {..} node.  If GetLast is true,
385/// this captures the expression result of the last sub-statement and returns it
386/// (for use by the statement expression extension).
387Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
388                                          AggValueSlot AggSlot) {
389  PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
390                             "LLVM IR generation of compound statement ('{}')");
391
392  // Keep track of the current cleanup stack depth, including debug scopes.
393  LexicalScope Scope(*this, S.getSourceRange());
394
395  return EmitCompoundStmtWithoutScope(S, GetLast, AggSlot);
396}
397
398Address
399CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
400                                              bool GetLast,
401                                              AggValueSlot AggSlot) {
402
403  const Stmt *ExprResult = S.getStmtExprResult();
404  assert((!GetLast || (GetLast && ExprResult)) &&
405         "If GetLast is true then the CompoundStmt must have a StmtExprResult");
406
407  Address RetAlloca = Address::invalid();
408
409  for (auto *CurStmt : S.body()) {
410    if (GetLast && ExprResult == CurStmt) {
411      // We have to special case labels here.  They are statements, but when put
412      // at the end of a statement expression, they yield the value of their
413      // subexpression.  Handle this by walking through all labels we encounter,
414      // emitting them before we evaluate the subexpr.
415      // Similar issues arise for attributed statements.
416      while (!isa<Expr>(ExprResult)) {
417        if (const auto *LS = dyn_cast<LabelStmt>(ExprResult)) {
418          EmitLabel(LS->getDecl());
419          ExprResult = LS->getSubStmt();
420        } else if (const auto *AS = dyn_cast<AttributedStmt>(ExprResult)) {
421          // FIXME: Update this if we ever have attributes that affect the
422          // semantics of an expression.
423          ExprResult = AS->getSubStmt();
424        } else {
425          llvm_unreachable("unknown value statement");
426        }
427      }
428
429      EnsureInsertPoint();
430
431      const Expr *E = cast<Expr>(ExprResult);
432      QualType ExprTy = E->getType();
433      if (hasAggregateEvaluationKind(ExprTy)) {
434        EmitAggExpr(E, AggSlot);
435      } else {
436        // We can't return an RValue here because there might be cleanups at
437        // the end of the StmtExpr.  Because of that, we have to emit the result
438        // here into a temporary alloca.
439        RetAlloca = CreateMemTemp(ExprTy);
440        EmitAnyExprToMem(E, RetAlloca, Qualifiers(),
441                         /*IsInit*/ false);
442      }
443    } else {
444      EmitStmt(CurStmt);
445    }
446  }
447
448  return RetAlloca;
449}
450
451void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
452  llvm::BranchInst *BI = dyn_cast<llvm::BranchInst>(BB->getTerminator());
453
454  // If there is a cleanup stack, then we it isn't worth trying to
455  // simplify this block (we would need to remove it from the scope map
456  // and cleanup entry).
457  if (!EHStack.empty())
458    return;
459
460  // Can only simplify direct branches.
461  if (!BI || !BI->isUnconditional())
462    return;
463
464  // Can only simplify empty blocks.
465  if (BI->getIterator() != BB->begin())
466    return;
467
468  BB->replaceAllUsesWith(BI->getSuccessor(0));
469  BI->eraseFromParent();
470  BB->eraseFromParent();
471}
472
473void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
474  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
475
476  // Fall out of the current block (if necessary).
477  EmitBranch(BB);
478
479  if (IsFinished && BB->use_empty()) {
480    delete BB;
481    return;
482  }
483
484  // Place the block after the current block, if possible, or else at
485  // the end of the function.
486  if (CurBB && CurBB->getParent())
487    CurFn->getBasicBlockList().insertAfter(CurBB->getIterator(), BB);
488  else
489    CurFn->getBasicBlockList().push_back(BB);
490  Builder.SetInsertPoint(BB);
491}
492
493void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
494  // Emit a branch from the current block to the target one if this
495  // was a real block.  If this was just a fall-through block after a
496  // terminator, don't emit it.
497  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
498
499  if (!CurBB || CurBB->getTerminator()) {
500    // If there is no insert point or the previous block is already
501    // terminated, don't touch it.
502  } else {
503    // Otherwise, create a fall-through branch.
504    Builder.CreateBr(Target);
505  }
506
507  Builder.ClearInsertionPoint();
508}
509
510void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
511  bool inserted = false;
512  for (llvm::User *u : block->users()) {
513    if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) {
514      CurFn->getBasicBlockList().insertAfter(insn->getParent()->getIterator(),
515                                             block);
516      inserted = true;
517      break;
518    }
519  }
520
521  if (!inserted)
522    CurFn->getBasicBlockList().push_back(block);
523
524  Builder.SetInsertPoint(block);
525}
526
527CodeGenFunction::JumpDest
528CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
529  JumpDest &Dest = LabelMap[D];
530  if (Dest.isValid()) return Dest;
531
532  // Create, but don't insert, the new block.
533  Dest = JumpDest(createBasicBlock(D->getName()),
534                  EHScopeStack::stable_iterator::invalid(),
535                  NextCleanupDestIndex++);
536  return Dest;
537}
538
539void CodeGenFunction::EmitLabel(const LabelDecl *D) {
540  // Add this label to the current lexical scope if we're within any
541  // normal cleanups.  Jumps "in" to this label --- when permitted by
542  // the language --- may need to be routed around such cleanups.
543  if (EHStack.hasNormalCleanups() && CurLexicalScope)
544    CurLexicalScope->addLabel(D);
545
546  JumpDest &Dest = LabelMap[D];
547
548  // If we didn't need a forward reference to this label, just go
549  // ahead and create a destination at the current scope.
550  if (!Dest.isValid()) {
551    Dest = getJumpDestInCurrentScope(D->getName());
552
553  // Otherwise, we need to give this label a target depth and remove
554  // it from the branch-fixups list.
555  } else {
556    assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
557    Dest.setScopeDepth(EHStack.stable_begin());
558    ResolveBranchFixups(Dest.getBlock());
559  }
560
561  EmitBlock(Dest.getBlock());
562
563  // Emit debug info for labels.
564  if (CGDebugInfo *DI = getDebugInfo()) {
565    if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
566      DI->setLocation(D->getLocation());
567      DI->EmitLabel(D, Builder);
568    }
569  }
570
571  incrementProfileCounter(D->getStmt());
572}
573
574/// Change the cleanup scope of the labels in this lexical scope to
575/// match the scope of the enclosing context.
576void CodeGenFunction::LexicalScope::rescopeLabels() {
577  assert(!Labels.empty());
578  EHScopeStack::stable_iterator innermostScope
579    = CGF.EHStack.getInnermostNormalCleanup();
580
581  // Change the scope depth of all the labels.
582  for (SmallVectorImpl<const LabelDecl*>::const_iterator
583         i = Labels.begin(), e = Labels.end(); i != e; ++i) {
584    assert(CGF.LabelMap.count(*i));
585    JumpDest &dest = CGF.LabelMap.find(*i)->second;
586    assert(dest.getScopeDepth().isValid());
587    assert(innermostScope.encloses(dest.getScopeDepth()));
588    dest.setScopeDepth(innermostScope);
589  }
590
591  // Reparent the labels if the new scope also has cleanups.
592  if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
593    ParentScope->Labels.append(Labels.begin(), Labels.end());
594  }
595}
596
597
598void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
599  EmitLabel(S.getDecl());
600  EmitStmt(S.getSubStmt());
601}
602
603void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
604  EmitStmt(S.getSubStmt(), S.getAttrs());
605}
606
607void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
608  // If this code is reachable then emit a stop point (if generating
609  // debug info). We have to do this ourselves because we are on the
610  // "simple" statement path.
611  if (HaveInsertPoint())
612    EmitStopPoint(&S);
613
614  EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel()));
615}
616
617
618void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
619  if (const LabelDecl *Target = S.getConstantTarget()) {
620    EmitBranchThroughCleanup(getJumpDestForLabel(Target));
621    return;
622  }
623
624  // Ensure that we have an i8* for our PHI node.
625  llvm::Value *V = Builder.CreateBitCast(EmitScalarExpr(S.getTarget()),
626                                         Int8PtrTy, "addr");
627  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
628
629  // Get the basic block for the indirect goto.
630  llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
631
632  // The first instruction in the block has to be the PHI for the switch dest,
633  // add an entry for this branch.
634  cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB);
635
636  EmitBranch(IndGotoBB);
637}
638
639void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
640  // C99 6.8.4.1: The first substatement is executed if the expression compares
641  // unequal to 0.  The condition must be a scalar type.
642  LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
643
644  if (S.getInit())
645    EmitStmt(S.getInit());
646
647  if (S.getConditionVariable())
648    EmitDecl(*S.getConditionVariable());
649
650  // If the condition constant folds and can be elided, try to avoid emitting
651  // the condition and the dead arm of the if/else.
652  bool CondConstant;
653  if (ConstantFoldsToSimpleInteger(S.getCond(), CondConstant,
654                                   S.isConstexpr())) {
655    // Figure out which block (then or else) is executed.
656    const Stmt *Executed = S.getThen();
657    const Stmt *Skipped  = S.getElse();
658    if (!CondConstant)  // Condition false?
659      std::swap(Executed, Skipped);
660
661    // If the skipped block has no labels in it, just emit the executed block.
662    // This avoids emitting dead code and simplifies the CFG substantially.
663    if (S.isConstexpr() || !ContainsLabel(Skipped)) {
664      if (CondConstant)
665        incrementProfileCounter(&S);
666      if (Executed) {
667        RunCleanupsScope ExecutedScope(*this);
668        EmitStmt(Executed);
669      }
670      return;
671    }
672  }
673
674  // Otherwise, the condition did not fold, or we couldn't elide it.  Just emit
675  // the conditional branch.
676  llvm::BasicBlock *ThenBlock = createBasicBlock("if.then");
677  llvm::BasicBlock *ContBlock = createBasicBlock("if.end");
678  llvm::BasicBlock *ElseBlock = ContBlock;
679  if (S.getElse())
680    ElseBlock = createBasicBlock("if.else");
681
682  EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock,
683                       getProfileCount(S.getThen()));
684
685  // Emit the 'then' code.
686  EmitBlock(ThenBlock);
687  incrementProfileCounter(&S);
688  {
689    RunCleanupsScope ThenScope(*this);
690    EmitStmt(S.getThen());
691  }
692  EmitBranch(ContBlock);
693
694  // Emit the 'else' code if present.
695  if (const Stmt *Else = S.getElse()) {
696    {
697      // There is no need to emit line number for an unconditional branch.
698      auto NL = ApplyDebugLocation::CreateEmpty(*this);
699      EmitBlock(ElseBlock);
700    }
701    {
702      RunCleanupsScope ElseScope(*this);
703      EmitStmt(Else);
704    }
705    {
706      // There is no need to emit line number for an unconditional branch.
707      auto NL = ApplyDebugLocation::CreateEmpty(*this);
708      EmitBranch(ContBlock);
709    }
710  }
711
712  // Emit the continuation block for code after the if.
713  EmitBlock(ContBlock, true);
714}
715
716void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
717                                    ArrayRef<const Attr *> WhileAttrs) {
718  // Emit the header for the loop, which will also become
719  // the continue target.
720  JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
721  EmitBlock(LoopHeader.getBlock());
722
723  const SourceRange &R = S.getSourceRange();
724  LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), WhileAttrs,
725                 SourceLocToDebugLoc(R.getBegin()),
726                 SourceLocToDebugLoc(R.getEnd()));
727
728  // Create an exit block for when the condition fails, which will
729  // also become the break target.
730  JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
731
732  // Store the blocks to use for break and continue.
733  BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader));
734
735  // C++ [stmt.while]p2:
736  //   When the condition of a while statement is a declaration, the
737  //   scope of the variable that is declared extends from its point
738  //   of declaration (3.3.2) to the end of the while statement.
739  //   [...]
740  //   The object created in a condition is destroyed and created
741  //   with each iteration of the loop.
742  RunCleanupsScope ConditionScope(*this);
743
744  if (S.getConditionVariable())
745    EmitDecl(*S.getConditionVariable());
746
747  // Evaluate the conditional in the while header.  C99 6.8.5.1: The
748  // evaluation of the controlling expression takes place before each
749  // execution of the loop body.
750  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
751
752  // while(1) is common, avoid extra exit blocks.  Be sure
753  // to correctly handle break/continue though.
754  bool EmitBoolCondBranch = true;
755  if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
756    if (C->isOne())
757      EmitBoolCondBranch = false;
758
759  // As long as the condition is true, go to the loop body.
760  llvm::BasicBlock *LoopBody = createBasicBlock("while.body");
761  if (EmitBoolCondBranch) {
762    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
763    if (ConditionScope.requiresCleanups())
764      ExitBlock = createBasicBlock("while.exit");
765    Builder.CreateCondBr(
766        BoolCondVal, LoopBody, ExitBlock,
767        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())));
768
769    if (ExitBlock != LoopExit.getBlock()) {
770      EmitBlock(ExitBlock);
771      EmitBranchThroughCleanup(LoopExit);
772    }
773  }
774
775  // Emit the loop body.  We have to emit this in a cleanup scope
776  // because it might be a singleton DeclStmt.
777  {
778    RunCleanupsScope BodyScope(*this);
779    EmitBlock(LoopBody);
780    incrementProfileCounter(&S);
781    EmitStmt(S.getBody());
782  }
783
784  BreakContinueStack.pop_back();
785
786  // Immediately force cleanup.
787  ConditionScope.ForceCleanup();
788
789  EmitStopPoint(&S);
790  // Branch to the loop header again.
791  EmitBranch(LoopHeader.getBlock());
792
793  LoopStack.pop();
794
795  // Emit the exit block.
796  EmitBlock(LoopExit.getBlock(), true);
797
798  // The LoopHeader typically is just a branch if we skipped emitting
799  // a branch, try to erase it.
800  if (!EmitBoolCondBranch)
801    SimplifyForwardingBlocks(LoopHeader.getBlock());
802}
803
804void CodeGenFunction::EmitDoStmt(const DoStmt &S,
805                                 ArrayRef<const Attr *> DoAttrs) {
806  JumpDest LoopExit = getJumpDestInCurrentScope("do.end");
807  JumpDest LoopCond = getJumpDestInCurrentScope("do.cond");
808
809  uint64_t ParentCount = getCurrentProfileCount();
810
811  // Store the blocks to use for break and continue.
812  BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond));
813
814  // Emit the body of the loop.
815  llvm::BasicBlock *LoopBody = createBasicBlock("do.body");
816
817  EmitBlockWithFallThrough(LoopBody, &S);
818  {
819    RunCleanupsScope BodyScope(*this);
820    EmitStmt(S.getBody());
821  }
822
823  EmitBlock(LoopCond.getBlock());
824
825  const SourceRange &R = S.getSourceRange();
826  LoopStack.push(LoopBody, CGM.getContext(), DoAttrs,
827                 SourceLocToDebugLoc(R.getBegin()),
828                 SourceLocToDebugLoc(R.getEnd()));
829
830  // C99 6.8.5.2: "The evaluation of the controlling expression takes place
831  // after each execution of the loop body."
832
833  // Evaluate the conditional in the while header.
834  // C99 6.8.5p2/p4: The first substatement is executed if the expression
835  // compares unequal to 0.  The condition must be a scalar type.
836  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
837
838  BreakContinueStack.pop_back();
839
840  // "do {} while (0)" is common in macros, avoid extra blocks.  Be sure
841  // to correctly handle break/continue though.
842  bool EmitBoolCondBranch = true;
843  if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
844    if (C->isZero())
845      EmitBoolCondBranch = false;
846
847  // As long as the condition is true, iterate the loop.
848  if (EmitBoolCondBranch) {
849    uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount;
850    Builder.CreateCondBr(
851        BoolCondVal, LoopBody, LoopExit.getBlock(),
852        createProfileWeightsForLoop(S.getCond(), BackedgeCount));
853  }
854
855  LoopStack.pop();
856
857  // Emit the exit block.
858  EmitBlock(LoopExit.getBlock());
859
860  // The DoCond block typically is just a branch if we skipped
861  // emitting a branch, try to erase it.
862  if (!EmitBoolCondBranch)
863    SimplifyForwardingBlocks(LoopCond.getBlock());
864}
865
866void CodeGenFunction::EmitForStmt(const ForStmt &S,
867                                  ArrayRef<const Attr *> ForAttrs) {
868  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
869
870  LexicalScope ForScope(*this, S.getSourceRange());
871
872  // Evaluate the first part before the loop.
873  if (S.getInit())
874    EmitStmt(S.getInit());
875
876  // Start the loop with a block that tests the condition.
877  // If there's an increment, the continue scope will be overwritten
878  // later.
879  JumpDest Continue = getJumpDestInCurrentScope("for.cond");
880  llvm::BasicBlock *CondBlock = Continue.getBlock();
881  EmitBlock(CondBlock);
882
883  const SourceRange &R = S.getSourceRange();
884  LoopStack.push(CondBlock, CGM.getContext(), ForAttrs,
885                 SourceLocToDebugLoc(R.getBegin()),
886                 SourceLocToDebugLoc(R.getEnd()));
887
888  // If the for loop doesn't have an increment we can just use the
889  // condition as the continue block.  Otherwise we'll need to create
890  // a block for it (in the current scope, i.e. in the scope of the
891  // condition), and that we will become our continue block.
892  if (S.getInc())
893    Continue = getJumpDestInCurrentScope("for.inc");
894
895  // Store the blocks to use for break and continue.
896  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
897
898  // Create a cleanup scope for the condition variable cleanups.
899  LexicalScope ConditionScope(*this, S.getSourceRange());
900
901  if (S.getCond()) {
902    // If the for statement has a condition scope, emit the local variable
903    // declaration.
904    if (S.getConditionVariable()) {
905      EmitDecl(*S.getConditionVariable());
906    }
907
908    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
909    // If there are any cleanups between here and the loop-exit scope,
910    // create a block to stage a loop exit along.
911    if (ForScope.requiresCleanups())
912      ExitBlock = createBasicBlock("for.cond.cleanup");
913
914    // As long as the condition is true, iterate the loop.
915    llvm::BasicBlock *ForBody = createBasicBlock("for.body");
916
917    // C99 6.8.5p2/p4: The first substatement is executed if the expression
918    // compares unequal to 0.  The condition must be a scalar type.
919    llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
920    Builder.CreateCondBr(
921        BoolCondVal, ForBody, ExitBlock,
922        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())));
923
924    if (ExitBlock != LoopExit.getBlock()) {
925      EmitBlock(ExitBlock);
926      EmitBranchThroughCleanup(LoopExit);
927    }
928
929    EmitBlock(ForBody);
930  } else {
931    // Treat it as a non-zero constant.  Don't even create a new block for the
932    // body, just fall into it.
933  }
934  incrementProfileCounter(&S);
935
936  {
937    // Create a separate cleanup scope for the body, in case it is not
938    // a compound statement.
939    RunCleanupsScope BodyScope(*this);
940    EmitStmt(S.getBody());
941  }
942
943  // If there is an increment, emit it next.
944  if (S.getInc()) {
945    EmitBlock(Continue.getBlock());
946    EmitStmt(S.getInc());
947  }
948
949  BreakContinueStack.pop_back();
950
951  ConditionScope.ForceCleanup();
952
953  EmitStopPoint(&S);
954  EmitBranch(CondBlock);
955
956  ForScope.ForceCleanup();
957
958  LoopStack.pop();
959
960  // Emit the fall-through block.
961  EmitBlock(LoopExit.getBlock(), true);
962}
963
964void
965CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
966                                     ArrayRef<const Attr *> ForAttrs) {
967  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
968
969  LexicalScope ForScope(*this, S.getSourceRange());
970
971  // Evaluate the first pieces before the loop.
972  if (S.getInit())
973    EmitStmt(S.getInit());
974  EmitStmt(S.getRangeStmt());
975  EmitStmt(S.getBeginStmt());
976  EmitStmt(S.getEndStmt());
977
978  // Start the loop with a block that tests the condition.
979  // If there's an increment, the continue scope will be overwritten
980  // later.
981  llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
982  EmitBlock(CondBlock);
983
984  const SourceRange &R = S.getSourceRange();
985  LoopStack.push(CondBlock, CGM.getContext(), ForAttrs,
986                 SourceLocToDebugLoc(R.getBegin()),
987                 SourceLocToDebugLoc(R.getEnd()));
988
989  // If there are any cleanups between here and the loop-exit scope,
990  // create a block to stage a loop exit along.
991  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
992  if (ForScope.requiresCleanups())
993    ExitBlock = createBasicBlock("for.cond.cleanup");
994
995  // The loop body, consisting of the specified body and the loop variable.
996  llvm::BasicBlock *ForBody = createBasicBlock("for.body");
997
998  // The body is executed if the expression, contextually converted
999  // to bool, is true.
1000  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
1001  Builder.CreateCondBr(
1002      BoolCondVal, ForBody, ExitBlock,
1003      createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody())));
1004
1005  if (ExitBlock != LoopExit.getBlock()) {
1006    EmitBlock(ExitBlock);
1007    EmitBranchThroughCleanup(LoopExit);
1008  }
1009
1010  EmitBlock(ForBody);
1011  incrementProfileCounter(&S);
1012
1013  // Create a block for the increment. In case of a 'continue', we jump there.
1014  JumpDest Continue = getJumpDestInCurrentScope("for.inc");
1015
1016  // Store the blocks to use for break and continue.
1017  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1018
1019  {
1020    // Create a separate cleanup scope for the loop variable and body.
1021    LexicalScope BodyScope(*this, S.getSourceRange());
1022    EmitStmt(S.getLoopVarStmt());
1023    EmitStmt(S.getBody());
1024  }
1025
1026  EmitStopPoint(&S);
1027  // If there is an increment, emit it next.
1028  EmitBlock(Continue.getBlock());
1029  EmitStmt(S.getInc());
1030
1031  BreakContinueStack.pop_back();
1032
1033  EmitBranch(CondBlock);
1034
1035  ForScope.ForceCleanup();
1036
1037  LoopStack.pop();
1038
1039  // Emit the fall-through block.
1040  EmitBlock(LoopExit.getBlock(), true);
1041}
1042
1043void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1044  if (RV.isScalar()) {
1045    Builder.CreateStore(RV.getScalarVal(), ReturnValue);
1046  } else if (RV.isAggregate()) {
1047    LValue Dest = MakeAddrLValue(ReturnValue, Ty);
1048    LValue Src = MakeAddrLValue(RV.getAggregateAddress(), Ty);
1049    EmitAggregateCopy(Dest, Src, Ty, getOverlapForReturnValue());
1050  } else {
1051    EmitStoreOfComplex(RV.getComplexVal(), MakeAddrLValue(ReturnValue, Ty),
1052                       /*init*/ true);
1053  }
1054  EmitBranchThroughCleanup(ReturnBlock);
1055}
1056
1057/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1058/// if the function returns void, or may be missing one if the function returns
1059/// non-void.  Fun stuff :).
1060void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1061  if (requiresReturnValueCheck()) {
1062    llvm::Constant *SLoc = EmitCheckSourceLocation(S.getBeginLoc());
1063    auto *SLocPtr =
1064        new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1065                                 llvm::GlobalVariable::PrivateLinkage, SLoc);
1066    SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1067    CGM.getSanitizerMetadata()->disableSanitizerForGlobal(SLocPtr);
1068    assert(ReturnLocation.isValid() && "No valid return location");
1069    Builder.CreateStore(Builder.CreateBitCast(SLocPtr, Int8PtrTy),
1070                        ReturnLocation);
1071  }
1072
1073  // Returning from an outlined SEH helper is UB, and we already warn on it.
1074  if (IsOutlinedSEHHelper) {
1075    Builder.CreateUnreachable();
1076    Builder.ClearInsertionPoint();
1077  }
1078
1079  // Emit the result value, even if unused, to evaluate the side effects.
1080  const Expr *RV = S.getRetValue();
1081
1082  // Treat block literals in a return expression as if they appeared
1083  // in their own scope.  This permits a small, easily-implemented
1084  // exception to our over-conservative rules about not jumping to
1085  // statements following block literals with non-trivial cleanups.
1086  RunCleanupsScope cleanupScope(*this);
1087  if (const FullExpr *fe = dyn_cast_or_null<FullExpr>(RV)) {
1088    enterFullExpression(fe);
1089    RV = fe->getSubExpr();
1090  }
1091
1092  // FIXME: Clean this up by using an LValue for ReturnTemp,
1093  // EmitStoreThroughLValue, and EmitAnyExpr.
1094  if (getLangOpts().ElideConstructors &&
1095      S.getNRVOCandidate() && S.getNRVOCandidate()->isNRVOVariable()) {
1096    // Apply the named return value optimization for this return statement,
1097    // which means doing nothing: the appropriate result has already been
1098    // constructed into the NRVO variable.
1099
1100    // If there is an NRVO flag for this variable, set it to 1 into indicate
1101    // that the cleanup code should not destroy the variable.
1102    if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1103      Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag);
1104  } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1105    // Make sure not to return anything, but evaluate the expression
1106    // for side effects.
1107    if (RV)
1108      EmitAnyExpr(RV);
1109  } else if (!RV) {
1110    // Do nothing (return value is left uninitialized)
1111  } else if (FnRetTy->isReferenceType()) {
1112    // If this function returns a reference, take the address of the expression
1113    // rather than the value.
1114    RValue Result = EmitReferenceBindingToExpr(RV);
1115    Builder.CreateStore(Result.getScalarVal(), ReturnValue);
1116  } else {
1117    switch (getEvaluationKind(RV->getType())) {
1118    case TEK_Scalar:
1119      Builder.CreateStore(EmitScalarExpr(RV), ReturnValue);
1120      break;
1121    case TEK_Complex:
1122      EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()),
1123                                /*isInit*/ true);
1124      break;
1125    case TEK_Aggregate:
1126      EmitAggExpr(RV, AggValueSlot::forAddr(
1127                          ReturnValue, Qualifiers(),
1128                          AggValueSlot::IsDestructed,
1129                          AggValueSlot::DoesNotNeedGCBarriers,
1130                          AggValueSlot::IsNotAliased,
1131                          getOverlapForReturnValue()));
1132      break;
1133    }
1134  }
1135
1136  ++NumReturnExprs;
1137  if (!RV || RV->isEvaluatable(getContext()))
1138    ++NumSimpleReturnExprs;
1139
1140  cleanupScope.ForceCleanup();
1141  EmitBranchThroughCleanup(ReturnBlock);
1142}
1143
1144void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1145  // As long as debug info is modeled with instructions, we have to ensure we
1146  // have a place to insert here and write the stop point here.
1147  if (HaveInsertPoint())
1148    EmitStopPoint(&S);
1149
1150  for (const auto *I : S.decls())
1151    EmitDecl(*I);
1152}
1153
1154void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1155  assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1156
1157  // If this code is reachable then emit a stop point (if generating
1158  // debug info). We have to do this ourselves because we are on the
1159  // "simple" statement path.
1160  if (HaveInsertPoint())
1161    EmitStopPoint(&S);
1162
1163  EmitBranchThroughCleanup(BreakContinueStack.back().BreakBlock);
1164}
1165
1166void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1167  assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1168
1169  // If this code is reachable then emit a stop point (if generating
1170  // debug info). We have to do this ourselves because we are on the
1171  // "simple" statement path.
1172  if (HaveInsertPoint())
1173    EmitStopPoint(&S);
1174
1175  EmitBranchThroughCleanup(BreakContinueStack.back().ContinueBlock);
1176}
1177
1178/// EmitCaseStmtRange - If case statement range is not too big then
1179/// add multiple cases to switch instruction, one for each value within
1180/// the range. If range is too big then emit "if" condition check.
1181void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
1182  assert(S.getRHS() && "Expected RHS value in CaseStmt");
1183
1184  llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());
1185  llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());
1186
1187  // Emit the code for this case. We do this first to make sure it is
1188  // properly chained from our predecessor before generating the
1189  // switch machinery to enter this block.
1190  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1191  EmitBlockWithFallThrough(CaseDest, &S);
1192  EmitStmt(S.getSubStmt());
1193
1194  // If range is empty, do nothing.
1195  if (LHS.isSigned() ? RHS.slt(LHS) : RHS.ult(LHS))
1196    return;
1197
1198  llvm::APInt Range = RHS - LHS;
1199  // FIXME: parameters such as this should not be hardcoded.
1200  if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
1201    // Range is small enough to add multiple switch instruction cases.
1202    uint64_t Total = getProfileCount(&S);
1203    unsigned NCases = Range.getZExtValue() + 1;
1204    // We only have one region counter for the entire set of cases here, so we
1205    // need to divide the weights evenly between the generated cases, ensuring
1206    // that the total weight is preserved. E.g., a weight of 5 over three cases
1207    // will be distributed as weights of 2, 2, and 1.
1208    uint64_t Weight = Total / NCases, Rem = Total % NCases;
1209    for (unsigned I = 0; I != NCases; ++I) {
1210      if (SwitchWeights)
1211        SwitchWeights->push_back(Weight + (Rem ? 1 : 0));
1212      if (Rem)
1213        Rem--;
1214      SwitchInsn->addCase(Builder.getInt(LHS), CaseDest);
1215      ++LHS;
1216    }
1217    return;
1218  }
1219
1220  // The range is too big. Emit "if" condition into a new block,
1221  // making sure to save and restore the current insertion point.
1222  llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1223
1224  // Push this test onto the chain of range checks (which terminates
1225  // in the default basic block). The switch's default will be changed
1226  // to the top of this chain after switch emission is complete.
1227  llvm::BasicBlock *FalseDest = CaseRangeBlock;
1228  CaseRangeBlock = createBasicBlock("sw.caserange");
1229
1230  CurFn->getBasicBlockList().push_back(CaseRangeBlock);
1231  Builder.SetInsertPoint(CaseRangeBlock);
1232
1233  // Emit range check.
1234  llvm::Value *Diff =
1235    Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS));
1236  llvm::Value *Cond =
1237    Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds");
1238
1239  llvm::MDNode *Weights = nullptr;
1240  if (SwitchWeights) {
1241    uint64_t ThisCount = getProfileCount(&S);
1242    uint64_t DefaultCount = (*SwitchWeights)[0];
1243    Weights = createProfileWeights(ThisCount, DefaultCount);
1244
1245    // Since we're chaining the switch default through each large case range, we
1246    // need to update the weight for the default, ie, the first case, to include
1247    // this case.
1248    (*SwitchWeights)[0] += ThisCount;
1249  }
1250  Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
1251
1252  // Restore the appropriate insertion point.
1253  if (RestoreBB)
1254    Builder.SetInsertPoint(RestoreBB);
1255  else
1256    Builder.ClearInsertionPoint();
1257}
1258
1259void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
1260  // If there is no enclosing switch instance that we're aware of, then this
1261  // case statement and its block can be elided.  This situation only happens
1262  // when we've constant-folded the switch, are emitting the constant case,
1263  // and part of the constant case includes another case statement.  For
1264  // instance: switch (4) { case 4: do { case 5: } while (1); }
1265  if (!SwitchInsn) {
1266    EmitStmt(S.getSubStmt());
1267    return;
1268  }
1269
1270  // Handle case ranges.
1271  if (S.getRHS()) {
1272    EmitCaseStmtRange(S);
1273    return;
1274  }
1275
1276  llvm::ConstantInt *CaseVal =
1277    Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));
1278
1279  // If the body of the case is just a 'break', try to not emit an empty block.
1280  // If we're profiling or we're not optimizing, leave the block in for better
1281  // debug and coverage analysis.
1282  if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1283      CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1284      isa<BreakStmt>(S.getSubStmt())) {
1285    JumpDest Block = BreakContinueStack.back().BreakBlock;
1286
1287    // Only do this optimization if there are no cleanups that need emitting.
1288    if (isObviouslyBranchWithoutCleanups(Block)) {
1289      if (SwitchWeights)
1290        SwitchWeights->push_back(getProfileCount(&S));
1291      SwitchInsn->addCase(CaseVal, Block.getBlock());
1292
1293      // If there was a fallthrough into this case, make sure to redirect it to
1294      // the end of the switch as well.
1295      if (Builder.GetInsertBlock()) {
1296        Builder.CreateBr(Block.getBlock());
1297        Builder.ClearInsertionPoint();
1298      }
1299      return;
1300    }
1301  }
1302
1303  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1304  EmitBlockWithFallThrough(CaseDest, &S);
1305  if (SwitchWeights)
1306    SwitchWeights->push_back(getProfileCount(&S));
1307  SwitchInsn->addCase(CaseVal, CaseDest);
1308
1309  // Recursively emitting the statement is acceptable, but is not wonderful for
1310  // code where we have many case statements nested together, i.e.:
1311  //  case 1:
1312  //    case 2:
1313  //      case 3: etc.
1314  // Handling this recursively will create a new block for each case statement
1315  // that falls through to the next case which is IR intensive.  It also causes
1316  // deep recursion which can run into stack depth limitations.  Handle
1317  // sequential non-range case statements specially.
1318  const CaseStmt *CurCase = &S;
1319  const CaseStmt *NextCase = dyn_cast<CaseStmt>(S.getSubStmt());
1320
1321  // Otherwise, iteratively add consecutive cases to this switch stmt.
1322  while (NextCase && NextCase->getRHS() == nullptr) {
1323    CurCase = NextCase;
1324    llvm::ConstantInt *CaseVal =
1325      Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));
1326
1327    if (SwitchWeights)
1328      SwitchWeights->push_back(getProfileCount(NextCase));
1329    if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1330      CaseDest = createBasicBlock("sw.bb");
1331      EmitBlockWithFallThrough(CaseDest, &S);
1332    }
1333
1334    SwitchInsn->addCase(CaseVal, CaseDest);
1335    NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
1336  }
1337
1338  // Normal default recursion for non-cases.
1339  EmitStmt(CurCase->getSubStmt());
1340}
1341
1342void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S) {
1343  // If there is no enclosing switch instance that we're aware of, then this
1344  // default statement can be elided. This situation only happens when we've
1345  // constant-folded the switch.
1346  if (!SwitchInsn) {
1347    EmitStmt(S.getSubStmt());
1348    return;
1349  }
1350
1351  llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1352  assert(DefaultBlock->empty() &&
1353         "EmitDefaultStmt: Default block already defined?");
1354
1355  EmitBlockWithFallThrough(DefaultBlock, &S);
1356
1357  EmitStmt(S.getSubStmt());
1358}
1359
1360/// CollectStatementsForCase - Given the body of a 'switch' statement and a
1361/// constant value that is being switched on, see if we can dead code eliminate
1362/// the body of the switch to a simple series of statements to emit.  Basically,
1363/// on a switch (5) we want to find these statements:
1364///    case 5:
1365///      printf(...);    <--
1366///      ++i;            <--
1367///      break;
1368///
1369/// and add them to the ResultStmts vector.  If it is unsafe to do this
1370/// transformation (for example, one of the elided statements contains a label
1371/// that might be jumped to), return CSFC_Failure.  If we handled it and 'S'
1372/// should include statements after it (e.g. the printf() line is a substmt of
1373/// the case) then return CSFC_FallThrough.  If we handled it and found a break
1374/// statement, then return CSFC_Success.
1375///
1376/// If Case is non-null, then we are looking for the specified case, checking
1377/// that nothing we jump over contains labels.  If Case is null, then we found
1378/// the case and are looking for the break.
1379///
1380/// If the recursive walk actually finds our Case, then we set FoundCase to
1381/// true.
1382///
1383enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
1384static CSFC_Result CollectStatementsForCase(const Stmt *S,
1385                                            const SwitchCase *Case,
1386                                            bool &FoundCase,
1387                              SmallVectorImpl<const Stmt*> &ResultStmts) {
1388  // If this is a null statement, just succeed.
1389  if (!S)
1390    return Case ? CSFC_Success : CSFC_FallThrough;
1391
1392  // If this is the switchcase (case 4: or default) that we're looking for, then
1393  // we're in business.  Just add the substatement.
1394  if (const SwitchCase *SC = dyn_cast<SwitchCase>(S)) {
1395    if (S == Case) {
1396      FoundCase = true;
1397      return CollectStatementsForCase(SC->getSubStmt(), nullptr, FoundCase,
1398                                      ResultStmts);
1399    }
1400
1401    // Otherwise, this is some other case or default statement, just ignore it.
1402    return CollectStatementsForCase(SC->getSubStmt(), Case, FoundCase,
1403                                    ResultStmts);
1404  }
1405
1406  // If we are in the live part of the code and we found our break statement,
1407  // return a success!
1408  if (!Case && isa<BreakStmt>(S))
1409    return CSFC_Success;
1410
1411  // If this is a switch statement, then it might contain the SwitchCase, the
1412  // break, or neither.
1413  if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(S)) {
1414    // Handle this as two cases: we might be looking for the SwitchCase (if so
1415    // the skipped statements must be skippable) or we might already have it.
1416    CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
1417    bool StartedInLiveCode = FoundCase;
1418    unsigned StartSize = ResultStmts.size();
1419
1420    // If we've not found the case yet, scan through looking for it.
1421    if (Case) {
1422      // Keep track of whether we see a skipped declaration.  The code could be
1423      // using the declaration even if it is skipped, so we can't optimize out
1424      // the decl if the kept statements might refer to it.
1425      bool HadSkippedDecl = false;
1426
1427      // If we're looking for the case, just see if we can skip each of the
1428      // substatements.
1429      for (; Case && I != E; ++I) {
1430        HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(*I);
1431
1432        switch (CollectStatementsForCase(*I, Case, FoundCase, ResultStmts)) {
1433        case CSFC_Failure: return CSFC_Failure;
1434        case CSFC_Success:
1435          // A successful result means that either 1) that the statement doesn't
1436          // have the case and is skippable, or 2) does contain the case value
1437          // and also contains the break to exit the switch.  In the later case,
1438          // we just verify the rest of the statements are elidable.
1439          if (FoundCase) {
1440            // If we found the case and skipped declarations, we can't do the
1441            // optimization.
1442            if (HadSkippedDecl)
1443              return CSFC_Failure;
1444
1445            for (++I; I != E; ++I)
1446              if (CodeGenFunction::ContainsLabel(*I, true))
1447                return CSFC_Failure;
1448            return CSFC_Success;
1449          }
1450          break;
1451        case CSFC_FallThrough:
1452          // If we have a fallthrough condition, then we must have found the
1453          // case started to include statements.  Consider the rest of the
1454          // statements in the compound statement as candidates for inclusion.
1455          assert(FoundCase && "Didn't find case but returned fallthrough?");
1456          // We recursively found Case, so we're not looking for it anymore.
1457          Case = nullptr;
1458
1459          // If we found the case and skipped declarations, we can't do the
1460          // optimization.
1461          if (HadSkippedDecl)
1462            return CSFC_Failure;
1463          break;
1464        }
1465      }
1466
1467      if (!FoundCase)
1468        return CSFC_Success;
1469
1470      assert(!HadSkippedDecl && "fallthrough after skipping decl");
1471    }
1472
1473    // If we have statements in our range, then we know that the statements are
1474    // live and need to be added to the set of statements we're tracking.
1475    bool AnyDecls = false;
1476    for (; I != E; ++I) {
1477      AnyDecls |= CodeGenFunction::mightAddDeclToScope(*I);
1478
1479      switch (CollectStatementsForCase(*I, nullptr, FoundCase, ResultStmts)) {
1480      case CSFC_Failure: return CSFC_Failure;
1481      case CSFC_FallThrough:
1482        // A fallthrough result means that the statement was simple and just
1483        // included in ResultStmt, keep adding them afterwards.
1484        break;
1485      case CSFC_Success:
1486        // A successful result means that we found the break statement and
1487        // stopped statement inclusion.  We just ensure that any leftover stmts
1488        // are skippable and return success ourselves.
1489        for (++I; I != E; ++I)
1490          if (CodeGenFunction::ContainsLabel(*I, true))
1491            return CSFC_Failure;
1492        return CSFC_Success;
1493      }
1494    }
1495
1496    // If we're about to fall out of a scope without hitting a 'break;', we
1497    // can't perform the optimization if there were any decls in that scope
1498    // (we'd lose their end-of-lifetime).
1499    if (AnyDecls) {
1500      // If the entire compound statement was live, there's one more thing we
1501      // can try before giving up: emit the whole thing as a single statement.
1502      // We can do that unless the statement contains a 'break;'.
1503      // FIXME: Such a break must be at the end of a construct within this one.
1504      // We could emit this by just ignoring the BreakStmts entirely.
1505      if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
1506        ResultStmts.resize(StartSize);
1507        ResultStmts.push_back(S);
1508      } else {
1509        return CSFC_Failure;
1510      }
1511    }
1512
1513    return CSFC_FallThrough;
1514  }
1515
1516  // Okay, this is some other statement that we don't handle explicitly, like a
1517  // for statement or increment etc.  If we are skipping over this statement,
1518  // just verify it doesn't have labels, which would make it invalid to elide.
1519  if (Case) {
1520    if (CodeGenFunction::ContainsLabel(S, true))
1521      return CSFC_Failure;
1522    return CSFC_Success;
1523  }
1524
1525  // Otherwise, we want to include this statement.  Everything is cool with that
1526  // so long as it doesn't contain a break out of the switch we're in.
1527  if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
1528
1529  // Otherwise, everything is great.  Include the statement and tell the caller
1530  // that we fall through and include the next statement as well.
1531  ResultStmts.push_back(S);
1532  return CSFC_FallThrough;
1533}
1534
1535/// FindCaseStatementsForValue - Find the case statement being jumped to and
1536/// then invoke CollectStatementsForCase to find the list of statements to emit
1537/// for a switch on constant.  See the comment above CollectStatementsForCase
1538/// for more details.
1539static bool FindCaseStatementsForValue(const SwitchStmt &S,
1540                                       const llvm::APSInt &ConstantCondValue,
1541                                SmallVectorImpl<const Stmt*> &ResultStmts,
1542                                       ASTContext &C,
1543                                       const SwitchCase *&ResultCase) {
1544  // First step, find the switch case that is being branched to.  We can do this
1545  // efficiently by scanning the SwitchCase list.
1546  const SwitchCase *Case = S.getSwitchCaseList();
1547  const DefaultStmt *DefaultCase = nullptr;
1548
1549  for (; Case; Case = Case->getNextSwitchCase()) {
1550    // It's either a default or case.  Just remember the default statement in
1551    // case we're not jumping to any numbered cases.
1552    if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Case)) {
1553      DefaultCase = DS;
1554      continue;
1555    }
1556
1557    // Check to see if this case is the one we're looking for.
1558    const CaseStmt *CS = cast<CaseStmt>(Case);
1559    // Don't handle case ranges yet.
1560    if (CS->getRHS()) return false;
1561
1562    // If we found our case, remember it as 'case'.
1563    if (CS->getLHS()->EvaluateKnownConstInt(C) == ConstantCondValue)
1564      break;
1565  }
1566
1567  // If we didn't find a matching case, we use a default if it exists, or we
1568  // elide the whole switch body!
1569  if (!Case) {
1570    // It is safe to elide the body of the switch if it doesn't contain labels
1571    // etc.  If it is safe, return successfully with an empty ResultStmts list.
1572    if (!DefaultCase)
1573      return !CodeGenFunction::ContainsLabel(&S);
1574    Case = DefaultCase;
1575  }
1576
1577  // Ok, we know which case is being jumped to, try to collect all the
1578  // statements that follow it.  This can fail for a variety of reasons.  Also,
1579  // check to see that the recursive walk actually found our case statement.
1580  // Insane cases like this can fail to find it in the recursive walk since we
1581  // don't handle every stmt kind:
1582  // switch (4) {
1583  //   while (1) {
1584  //     case 4: ...
1585  bool FoundCase = false;
1586  ResultCase = Case;
1587  return CollectStatementsForCase(S.getBody(), Case, FoundCase,
1588                                  ResultStmts) != CSFC_Failure &&
1589         FoundCase;
1590}
1591
1592void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
1593  // Handle nested switch statements.
1594  llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
1595  SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
1596  llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
1597
1598  // See if we can constant fold the condition of the switch and therefore only
1599  // emit the live case statement (if any) of the switch.
1600  llvm::APSInt ConstantCondValue;
1601  if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) {
1602    SmallVector<const Stmt*, 4> CaseStmts;
1603    const SwitchCase *Case = nullptr;
1604    if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,
1605                                   getContext(), Case)) {
1606      if (Case)
1607        incrementProfileCounter(Case);
1608      RunCleanupsScope ExecutedScope(*this);
1609
1610      if (S.getInit())
1611        EmitStmt(S.getInit());
1612
1613      // Emit the condition variable if needed inside the entire cleanup scope
1614      // used by this special case for constant folded switches.
1615      if (S.getConditionVariable())
1616        EmitDecl(*S.getConditionVariable());
1617
1618      // At this point, we are no longer "within" a switch instance, so
1619      // we can temporarily enforce this to ensure that any embedded case
1620      // statements are not emitted.
1621      SwitchInsn = nullptr;
1622
1623      // Okay, we can dead code eliminate everything except this case.  Emit the
1624      // specified series of statements and we're good.
1625      for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i)
1626        EmitStmt(CaseStmts[i]);
1627      incrementProfileCounter(&S);
1628
1629      // Now we want to restore the saved switch instance so that nested
1630      // switches continue to function properly
1631      SwitchInsn = SavedSwitchInsn;
1632
1633      return;
1634    }
1635  }
1636
1637  JumpDest SwitchExit = getJumpDestInCurrentScope("sw.epilog");
1638
1639  RunCleanupsScope ConditionScope(*this);
1640
1641  if (S.getInit())
1642    EmitStmt(S.getInit());
1643
1644  if (S.getConditionVariable())
1645    EmitDecl(*S.getConditionVariable());
1646  llvm::Value *CondV = EmitScalarExpr(S.getCond());
1647
1648  // Create basic block to hold stuff that comes after switch
1649  // statement. We also need to create a default block now so that
1650  // explicit case ranges tests can have a place to jump to on
1651  // failure.
1652  llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default");
1653  SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock);
1654  if (PGO.haveRegionCounts()) {
1655    // Walk the SwitchCase list to find how many there are.
1656    uint64_t DefaultCount = 0;
1657    unsigned NumCases = 0;
1658    for (const SwitchCase *Case = S.getSwitchCaseList();
1659         Case;
1660         Case = Case->getNextSwitchCase()) {
1661      if (isa<DefaultStmt>(Case))
1662        DefaultCount = getProfileCount(Case);
1663      NumCases += 1;
1664    }
1665    SwitchWeights = new SmallVector<uint64_t, 16>();
1666    SwitchWeights->reserve(NumCases);
1667    // The default needs to be first. We store the edge count, so we already
1668    // know the right weight.
1669    SwitchWeights->push_back(DefaultCount);
1670  }
1671  CaseRangeBlock = DefaultBlock;
1672
1673  // Clear the insertion point to indicate we are in unreachable code.
1674  Builder.ClearInsertionPoint();
1675
1676  // All break statements jump to NextBlock. If BreakContinueStack is non-empty
1677  // then reuse last ContinueBlock.
1678  JumpDest OuterContinue;
1679  if (!BreakContinueStack.empty())
1680    OuterContinue = BreakContinueStack.back().ContinueBlock;
1681
1682  BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue));
1683
1684  // Emit switch body.
1685  EmitStmt(S.getBody());
1686
1687  BreakContinueStack.pop_back();
1688
1689  // Update the default block in case explicit case range tests have
1690  // been chained on top.
1691  SwitchInsn->setDefaultDest(CaseRangeBlock);
1692
1693  // If a default was never emitted:
1694  if (!DefaultBlock->getParent()) {
1695    // If we have cleanups, emit the default block so that there's a
1696    // place to jump through the cleanups from.
1697    if (ConditionScope.requiresCleanups()) {
1698      EmitBlock(DefaultBlock);
1699
1700    // Otherwise, just forward the default block to the switch end.
1701    } else {
1702      DefaultBlock->replaceAllUsesWith(SwitchExit.getBlock());
1703      delete DefaultBlock;
1704    }
1705  }
1706
1707  ConditionScope.ForceCleanup();
1708
1709  // Emit continuation.
1710  EmitBlock(SwitchExit.getBlock(), true);
1711  incrementProfileCounter(&S);
1712
1713  // If the switch has a condition wrapped by __builtin_unpredictable,
1714  // create metadata that specifies that the switch is unpredictable.
1715  // Don't bother if not optimizing because that metadata would not be used.
1716  auto *Call = dyn_cast<CallExpr>(S.getCond());
1717  if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
1718    auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1719    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1720      llvm::MDBuilder MDHelper(getLLVMContext());
1721      SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,
1722                              MDHelper.createUnpredictable());
1723    }
1724  }
1725
1726  if (SwitchWeights) {
1727    assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
1728           "switch weights do not match switch cases");
1729    // If there's only one jump destination there's no sense weighting it.
1730    if (SwitchWeights->size() > 1)
1731      SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
1732                              createProfileWeights(*SwitchWeights));
1733    delete SwitchWeights;
1734  }
1735  SwitchInsn = SavedSwitchInsn;
1736  SwitchWeights = SavedSwitchWeights;
1737  CaseRangeBlock = SavedCRBlock;
1738}
1739
1740static std::string
1741SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
1742                 SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=nullptr) {
1743  std::string Result;
1744
1745  while (*Constraint) {
1746    switch (*Constraint) {
1747    default:
1748      Result += Target.convertConstraint(Constraint);
1749      break;
1750    // Ignore these
1751    case '*':
1752    case '?':
1753    case '!':
1754    case '=': // Will see this and the following in mult-alt constraints.
1755    case '+':
1756      break;
1757    case '#': // Ignore the rest of the constraint alternative.
1758      while (Constraint[1] && Constraint[1] != ',')
1759        Constraint++;
1760      break;
1761    case '&':
1762    case '%':
1763      Result += *Constraint;
1764      while (Constraint[1] && Constraint[1] == *Constraint)
1765        Constraint++;
1766      break;
1767    case ',':
1768      Result += "|";
1769      break;
1770    case 'g':
1771      Result += "imr";
1772      break;
1773    case '[': {
1774      assert(OutCons &&
1775             "Must pass output names to constraints with a symbolic name");
1776      unsigned Index;
1777      bool result = Target.resolveSymbolicName(Constraint, *OutCons, Index);
1778      assert(result && "Could not resolve symbolic name"); (void)result;
1779      Result += llvm::utostr(Index);
1780      break;
1781    }
1782    }
1783
1784    Constraint++;
1785  }
1786
1787  return Result;
1788}
1789
1790/// AddVariableConstraints - Look at AsmExpr and if it is a variable declared
1791/// as using a particular register add that as a constraint that will be used
1792/// in this asm stmt.
1793static std::string
1794AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
1795                       const TargetInfo &Target, CodeGenModule &CGM,
1796                       const AsmStmt &Stmt, const bool EarlyClobber) {
1797  const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr);
1798  if (!AsmDeclRef)
1799    return Constraint;
1800  const ValueDecl &Value = *AsmDeclRef->getDecl();
1801  const VarDecl *Variable = dyn_cast<VarDecl>(&Value);
1802  if (!Variable)
1803    return Constraint;
1804  if (Variable->getStorageClass() != SC_Register)
1805    return Constraint;
1806  AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();
1807  if (!Attr)
1808    return Constraint;
1809  StringRef Register = Attr->getLabel();
1810  assert(Target.isValidGCCRegisterName(Register));
1811  // We're using validateOutputConstraint here because we only care if
1812  // this is a register constraint.
1813  TargetInfo::ConstraintInfo Info(Constraint, "");
1814  if (Target.validateOutputConstraint(Info) &&
1815      !Info.allowsRegister()) {
1816    CGM.ErrorUnsupported(&Stmt, "__asm__");
1817    return Constraint;
1818  }
1819  // Canonicalize the register here before returning it.
1820  Register = Target.getNormalizedGCCRegisterName(Register);
1821  return (EarlyClobber ? "&{" : "{") + Register.str() + "}";
1822}
1823
1824llvm::Value*
1825CodeGenFunction::EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info,
1826                                    LValue InputValue, QualType InputType,
1827                                    std::string &ConstraintStr,
1828                                    SourceLocation Loc) {
1829  llvm::Value *Arg;
1830  if (Info.allowsRegister() || !Info.allowsMemory()) {
1831    if (CodeGenFunction::hasScalarEvaluationKind(InputType)) {
1832      Arg = EmitLoadOfLValue(InputValue, Loc).getScalarVal();
1833    } else {
1834      llvm::Type *Ty = ConvertType(InputType);
1835      uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
1836      if (Size <= 64 && llvm::isPowerOf2_64(Size)) {
1837        Ty = llvm::IntegerType::get(getLLVMContext(), Size);
1838        Ty = llvm::PointerType::getUnqual(Ty);
1839
1840        Arg = Builder.CreateLoad(
1841            Builder.CreateBitCast(InputValue.getAddress(*this), Ty));
1842      } else {
1843        Arg = InputValue.getPointer(*this);
1844        ConstraintStr += '*';
1845      }
1846    }
1847  } else {
1848    Arg = InputValue.getPointer(*this);
1849    ConstraintStr += '*';
1850  }
1851
1852  return Arg;
1853}
1854
1855llvm::Value* CodeGenFunction::EmitAsmInput(
1856                                         const TargetInfo::ConstraintInfo &Info,
1857                                           const Expr *InputExpr,
1858                                           std::string &ConstraintStr) {
1859  // If this can't be a register or memory, i.e., has to be a constant
1860  // (immediate or symbolic), try to emit it as such.
1861  if (!Info.allowsRegister() && !Info.allowsMemory()) {
1862    if (Info.requiresImmediateConstant()) {
1863      Expr::EvalResult EVResult;
1864      InputExpr->EvaluateAsRValue(EVResult, getContext(), true);
1865
1866      llvm::APSInt IntResult;
1867      if (EVResult.Val.toIntegralConstant(IntResult, InputExpr->getType(),
1868                                          getContext()))
1869        return llvm::ConstantInt::get(getLLVMContext(), IntResult);
1870    }
1871
1872    Expr::EvalResult Result;
1873    if (InputExpr->EvaluateAsInt(Result, getContext()))
1874      return llvm::ConstantInt::get(getLLVMContext(), Result.Val.getInt());
1875  }
1876
1877  if (Info.allowsRegister() || !Info.allowsMemory())
1878    if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType()))
1879      return EmitScalarExpr(InputExpr);
1880  if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
1881    return EmitScalarExpr(InputExpr);
1882  InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
1883  LValue Dest = EmitLValue(InputExpr);
1884  return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr,
1885                            InputExpr->getExprLoc());
1886}
1887
1888/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
1889/// asm call instruction.  The !srcloc MDNode contains a list of constant
1890/// integers which are the source locations of the start of each line in the
1891/// asm.
1892static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
1893                                      CodeGenFunction &CGF) {
1894  SmallVector<llvm::Metadata *, 8> Locs;
1895  // Add the location of the first line to the MDNode.
1896  Locs.push_back(llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
1897      CGF.Int32Ty, Str->getBeginLoc().getRawEncoding())));
1898  StringRef StrVal = Str->getString();
1899  if (!StrVal.empty()) {
1900    const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
1901    const LangOptions &LangOpts = CGF.CGM.getLangOpts();
1902    unsigned StartToken = 0;
1903    unsigned ByteOffset = 0;
1904
1905    // Add the location of the start of each subsequent line of the asm to the
1906    // MDNode.
1907    for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
1908      if (StrVal[i] != '\n') continue;
1909      SourceLocation LineLoc = Str->getLocationOfByte(
1910          i + 1, SM, LangOpts, CGF.getTarget(), &StartToken, &ByteOffset);
1911      Locs.push_back(llvm::ConstantAsMetadata::get(
1912          llvm::ConstantInt::get(CGF.Int32Ty, LineLoc.getRawEncoding())));
1913    }
1914  }
1915
1916  return llvm::MDNode::get(CGF.getLLVMContext(), Locs);
1917}
1918
1919static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
1920                              bool ReadOnly, bool ReadNone, const AsmStmt &S,
1921                              const std::vector<llvm::Type *> &ResultRegTypes,
1922                              CodeGenFunction &CGF,
1923                              std::vector<llvm::Value *> &RegResults) {
1924  Result.addAttribute(llvm::AttributeList::FunctionIndex,
1925                      llvm::Attribute::NoUnwind);
1926  // Attach readnone and readonly attributes.
1927  if (!HasSideEffect) {
1928    if (ReadNone)
1929      Result.addAttribute(llvm::AttributeList::FunctionIndex,
1930                          llvm::Attribute::ReadNone);
1931    else if (ReadOnly)
1932      Result.addAttribute(llvm::AttributeList::FunctionIndex,
1933                          llvm::Attribute::ReadOnly);
1934  }
1935
1936  // Slap the source location of the inline asm into a !srcloc metadata on the
1937  // call.
1938  if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S))
1939    Result.setMetadata("srcloc",
1940                       getAsmSrcLocInfo(gccAsmStmt->getAsmString(), CGF));
1941  else {
1942    // At least put the line number on MS inline asm blobs.
1943    llvm::Constant *Loc = llvm::ConstantInt::get(CGF.Int32Ty,
1944                                        S.getAsmLoc().getRawEncoding());
1945    Result.setMetadata("srcloc",
1946                       llvm::MDNode::get(CGF.getLLVMContext(),
1947                                         llvm::ConstantAsMetadata::get(Loc)));
1948  }
1949
1950  if (CGF.getLangOpts().assumeFunctionsAreConvergent())
1951    // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
1952    // convergent (meaning, they may call an intrinsically convergent op, such
1953    // as bar.sync, and so can't have certain optimizations applied around
1954    // them).
1955    Result.addAttribute(llvm::AttributeList::FunctionIndex,
1956                        llvm::Attribute::Convergent);
1957  // Extract all of the register value results from the asm.
1958  if (ResultRegTypes.size() == 1) {
1959    RegResults.push_back(&Result);
1960  } else {
1961    for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
1962      llvm::Value *Tmp = CGF.Builder.CreateExtractValue(&Result, i, "asmresult");
1963      RegResults.push_back(Tmp);
1964    }
1965  }
1966}
1967
1968void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
1969  // Assemble the final asm string.
1970  std::string AsmString = S.generateAsmString(getContext());
1971
1972  // Get all the output and input constraints together.
1973  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
1974  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
1975
1976  for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
1977    StringRef Name;
1978    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
1979      Name = GAS->getOutputName(i);
1980    TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
1981    bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
1982    assert(IsValid && "Failed to parse output constraint");
1983    OutputConstraintInfos.push_back(Info);
1984  }
1985
1986  for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
1987    StringRef Name;
1988    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
1989      Name = GAS->getInputName(i);
1990    TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
1991    bool IsValid =
1992      getTarget().validateInputConstraint(OutputConstraintInfos, Info);
1993    assert(IsValid && "Failed to parse input constraint"); (void)IsValid;
1994    InputConstraintInfos.push_back(Info);
1995  }
1996
1997  std::string Constraints;
1998
1999  std::vector<LValue> ResultRegDests;
2000  std::vector<QualType> ResultRegQualTys;
2001  std::vector<llvm::Type *> ResultRegTypes;
2002  std::vector<llvm::Type *> ResultTruncRegTypes;
2003  std::vector<llvm::Type *> ArgTypes;
2004  std::vector<llvm::Value*> Args;
2005  llvm::BitVector ResultTypeRequiresCast;
2006
2007  // Keep track of inout constraints.
2008  std::string InOutConstraints;
2009  std::vector<llvm::Value*> InOutArgs;
2010  std::vector<llvm::Type*> InOutArgTypes;
2011
2012  // Keep track of out constraints for tied input operand.
2013  std::vector<std::string> OutputConstraints;
2014
2015  // An inline asm can be marked readonly if it meets the following conditions:
2016  //  - it doesn't have any sideeffects
2017  //  - it doesn't clobber memory
2018  //  - it doesn't return a value by-reference
2019  // It can be marked readnone if it doesn't have any input memory constraints
2020  // in addition to meeting the conditions listed above.
2021  bool ReadOnly = true, ReadNone = true;
2022
2023  for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
2024    TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
2025
2026    // Simplify the output constraint.
2027    std::string OutputConstraint(S.getOutputConstraint(i));
2028    OutputConstraint = SimplifyConstraint(OutputConstraint.c_str() + 1,
2029                                          getTarget(), &OutputConstraintInfos);
2030
2031    const Expr *OutExpr = S.getOutputExpr(i);
2032    OutExpr = OutExpr->IgnoreParenNoopCasts(getContext());
2033
2034    OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr,
2035                                              getTarget(), CGM, S,
2036                                              Info.earlyClobber());
2037    OutputConstraints.push_back(OutputConstraint);
2038    LValue Dest = EmitLValue(OutExpr);
2039    if (!Constraints.empty())
2040      Constraints += ',';
2041
2042    // If this is a register output, then make the inline asm return it
2043    // by-value.  If this is a memory result, return the value by-reference.
2044    bool isScalarizableAggregate =
2045        hasAggregateEvaluationKind(OutExpr->getType());
2046    if (!Info.allowsMemory() && (hasScalarEvaluationKind(OutExpr->getType()) ||
2047                                 isScalarizableAggregate)) {
2048      Constraints += "=" + OutputConstraint;
2049      ResultRegQualTys.push_back(OutExpr->getType());
2050      ResultRegDests.push_back(Dest);
2051      ResultTruncRegTypes.push_back(ConvertTypeForMem(OutExpr->getType()));
2052      if (Info.allowsRegister() && isScalarizableAggregate) {
2053        ResultTypeRequiresCast.push_back(true);
2054        unsigned Size = getContext().getTypeSize(OutExpr->getType());
2055        llvm::Type *ConvTy = llvm::IntegerType::get(getLLVMContext(), Size);
2056        ResultRegTypes.push_back(ConvTy);
2057      } else {
2058        ResultTypeRequiresCast.push_back(false);
2059        ResultRegTypes.push_back(ResultTruncRegTypes.back());
2060      }
2061      // If this output is tied to an input, and if the input is larger, then
2062      // we need to set the actual result type of the inline asm node to be the
2063      // same as the input type.
2064      if (Info.hasMatchingInput()) {
2065        unsigned InputNo;
2066        for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2067          TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2068          if (Input.hasTiedOperand() && Input.getTiedOperand() == i)
2069            break;
2070        }
2071        assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2072
2073        QualType InputTy = S.getInputExpr(InputNo)->getType();
2074        QualType OutputType = OutExpr->getType();
2075
2076        uint64_t InputSize = getContext().getTypeSize(InputTy);
2077        if (getContext().getTypeSize(OutputType) < InputSize) {
2078          // Form the asm to return the value as a larger integer or fp type.
2079          ResultRegTypes.back() = ConvertType(InputTy);
2080        }
2081      }
2082      if (llvm::Type* AdjTy =
2083            getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2084                                                 ResultRegTypes.back()))
2085        ResultRegTypes.back() = AdjTy;
2086      else {
2087        CGM.getDiags().Report(S.getAsmLoc(),
2088                              diag::err_asm_invalid_type_in_input)
2089            << OutExpr->getType() << OutputConstraint;
2090      }
2091
2092      // Update largest vector width for any vector types.
2093      if (auto *VT = dyn_cast<llvm::VectorType>(ResultRegTypes.back()))
2094        LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
2095                                   VT->getPrimitiveSizeInBits().getFixedSize());
2096    } else {
2097      ArgTypes.push_back(Dest.getAddress(*this).getType());
2098      Args.push_back(Dest.getPointer(*this));
2099      Constraints += "=*";
2100      Constraints += OutputConstraint;
2101      ReadOnly = ReadNone = false;
2102    }
2103
2104    if (Info.isReadWrite()) {
2105      InOutConstraints += ',';
2106
2107      const Expr *InputExpr = S.getOutputExpr(i);
2108      llvm::Value *Arg = EmitAsmInputLValue(Info, Dest, InputExpr->getType(),
2109                                            InOutConstraints,
2110                                            InputExpr->getExprLoc());
2111
2112      if (llvm::Type* AdjTy =
2113          getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2114                                               Arg->getType()))
2115        Arg = Builder.CreateBitCast(Arg, AdjTy);
2116
2117      // Update largest vector width for any vector types.
2118      if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2119        LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
2120                                   VT->getPrimitiveSizeInBits().getFixedSize());
2121      if (Info.allowsRegister())
2122        InOutConstraints += llvm::utostr(i);
2123      else
2124        InOutConstraints += OutputConstraint;
2125
2126      InOutArgTypes.push_back(Arg->getType());
2127      InOutArgs.push_back(Arg);
2128    }
2129  }
2130
2131  // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2132  // to the return value slot. Only do this when returning in registers.
2133  if (isa<MSAsmStmt>(&S)) {
2134    const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();
2135    if (RetAI.isDirect() || RetAI.isExtend()) {
2136      // Make a fake lvalue for the return value slot.
2137      LValue ReturnSlot = MakeAddrLValue(ReturnValue, FnRetTy);
2138      CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
2139          *this, ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
2140          ResultRegDests, AsmString, S.getNumOutputs());
2141      SawAsmBlock = true;
2142    }
2143  }
2144
2145  for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
2146    const Expr *InputExpr = S.getInputExpr(i);
2147
2148    TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
2149
2150    if (Info.allowsMemory())
2151      ReadNone = false;
2152
2153    if (!Constraints.empty())
2154      Constraints += ',';
2155
2156    // Simplify the input constraint.
2157    std::string InputConstraint(S.getInputConstraint(i));
2158    InputConstraint = SimplifyConstraint(InputConstraint.c_str(), getTarget(),
2159                                         &OutputConstraintInfos);
2160
2161    InputConstraint = AddVariableConstraints(
2162        InputConstraint, *InputExpr->IgnoreParenNoopCasts(getContext()),
2163        getTarget(), CGM, S, false /* No EarlyClobber */);
2164
2165    std::string ReplaceConstraint (InputConstraint);
2166    llvm::Value *Arg = EmitAsmInput(Info, InputExpr, Constraints);
2167
2168    // If this input argument is tied to a larger output result, extend the
2169    // input to be the same size as the output.  The LLVM backend wants to see
2170    // the input and output of a matching constraint be the same size.  Note
2171    // that GCC does not define what the top bits are here.  We use zext because
2172    // that is usually cheaper, but LLVM IR should really get an anyext someday.
2173    if (Info.hasTiedOperand()) {
2174      unsigned Output = Info.getTiedOperand();
2175      QualType OutputType = S.getOutputExpr(Output)->getType();
2176      QualType InputTy = InputExpr->getType();
2177
2178      if (getContext().getTypeSize(OutputType) >
2179          getContext().getTypeSize(InputTy)) {
2180        // Use ptrtoint as appropriate so that we can do our extension.
2181        if (isa<llvm::PointerType>(Arg->getType()))
2182          Arg = Builder.CreatePtrToInt(Arg, IntPtrTy);
2183        llvm::Type *OutputTy = ConvertType(OutputType);
2184        if (isa<llvm::IntegerType>(OutputTy))
2185          Arg = Builder.CreateZExt(Arg, OutputTy);
2186        else if (isa<llvm::PointerType>(OutputTy))
2187          Arg = Builder.CreateZExt(Arg, IntPtrTy);
2188        else {
2189          assert(OutputTy->isFloatingPointTy() && "Unexpected output type");
2190          Arg = Builder.CreateFPExt(Arg, OutputTy);
2191        }
2192      }
2193      // Deal with the tied operands' constraint code in adjustInlineAsmType.
2194      ReplaceConstraint = OutputConstraints[Output];
2195    }
2196    if (llvm::Type* AdjTy =
2197          getTargetHooks().adjustInlineAsmType(*this, ReplaceConstraint,
2198                                                   Arg->getType()))
2199      Arg = Builder.CreateBitCast(Arg, AdjTy);
2200    else
2201      CGM.getDiags().Report(S.getAsmLoc(), diag::err_asm_invalid_type_in_input)
2202          << InputExpr->getType() << InputConstraint;
2203
2204    // Update largest vector width for any vector types.
2205    if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2206      LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
2207                                   VT->getPrimitiveSizeInBits().getFixedSize());
2208
2209    ArgTypes.push_back(Arg->getType());
2210    Args.push_back(Arg);
2211    Constraints += InputConstraint;
2212  }
2213
2214  // Append the "input" part of inout constraints last.
2215  for (unsigned i = 0, e = InOutArgs.size(); i != e; i++) {
2216    ArgTypes.push_back(InOutArgTypes[i]);
2217    Args.push_back(InOutArgs[i]);
2218  }
2219  Constraints += InOutConstraints;
2220
2221  // Labels
2222  SmallVector<llvm::BasicBlock *, 16> Transfer;
2223  llvm::BasicBlock *Fallthrough = nullptr;
2224  bool IsGCCAsmGoto = false;
2225  if (const auto *GS =  dyn_cast<GCCAsmStmt>(&S)) {
2226    IsGCCAsmGoto = GS->isAsmGoto();
2227    if (IsGCCAsmGoto) {
2228      for (auto *E : GS->labels()) {
2229        JumpDest Dest = getJumpDestForLabel(E->getLabel());
2230        Transfer.push_back(Dest.getBlock());
2231        llvm::BlockAddress *BA =
2232            llvm::BlockAddress::get(CurFn, Dest.getBlock());
2233        Args.push_back(BA);
2234        ArgTypes.push_back(BA->getType());
2235        if (!Constraints.empty())
2236          Constraints += ',';
2237        Constraints += 'X';
2238      }
2239      StringRef Name = "asm.fallthrough";
2240      Fallthrough = createBasicBlock(Name);
2241    }
2242  }
2243
2244  // Clobbers
2245  for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
2246    StringRef Clobber = S.getClobber(i);
2247
2248    if (Clobber == "memory")
2249      ReadOnly = ReadNone = false;
2250    else if (Clobber != "cc")
2251      Clobber = getTarget().getNormalizedGCCRegisterName(Clobber);
2252
2253    if (!Constraints.empty())
2254      Constraints += ',';
2255
2256    Constraints += "~{";
2257    Constraints += Clobber;
2258    Constraints += '}';
2259  }
2260
2261  // Add machine specific clobbers
2262  std::string MachineClobbers = getTarget().getClobbers();
2263  if (!MachineClobbers.empty()) {
2264    if (!Constraints.empty())
2265      Constraints += ',';
2266    Constraints += MachineClobbers;
2267  }
2268
2269  llvm::Type *ResultType;
2270  if (ResultRegTypes.empty())
2271    ResultType = VoidTy;
2272  else if (ResultRegTypes.size() == 1)
2273    ResultType = ResultRegTypes[0];
2274  else
2275    ResultType = llvm::StructType::get(getLLVMContext(), ResultRegTypes);
2276
2277  llvm::FunctionType *FTy =
2278    llvm::FunctionType::get(ResultType, ArgTypes, false);
2279
2280  bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
2281  llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(&S) ?
2282    llvm::InlineAsm::AD_Intel : llvm::InlineAsm::AD_ATT;
2283  llvm::InlineAsm *IA =
2284    llvm::InlineAsm::get(FTy, AsmString, Constraints, HasSideEffect,
2285                         /* IsAlignStack */ false, AsmDialect);
2286  std::vector<llvm::Value*> RegResults;
2287  if (IsGCCAsmGoto) {
2288    llvm::CallBrInst *Result =
2289        Builder.CreateCallBr(IA, Fallthrough, Transfer, Args);
2290    UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, ReadOnly,
2291                      ReadNone, S, ResultRegTypes, *this, RegResults);
2292    EmitBlock(Fallthrough);
2293  } else {
2294    llvm::CallInst *Result =
2295        Builder.CreateCall(IA, Args, getBundlesForFunclet(IA));
2296    UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, ReadOnly,
2297                      ReadNone, S, ResultRegTypes, *this, RegResults);
2298  }
2299
2300  assert(RegResults.size() == ResultRegTypes.size());
2301  assert(RegResults.size() == ResultTruncRegTypes.size());
2302  assert(RegResults.size() == ResultRegDests.size());
2303  // ResultRegDests can be also populated by addReturnRegisterOutputs() above,
2304  // in which case its size may grow.
2305  assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
2306  for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
2307    llvm::Value *Tmp = RegResults[i];
2308
2309    // If the result type of the LLVM IR asm doesn't match the result type of
2310    // the expression, do the conversion.
2311    if (ResultRegTypes[i] != ResultTruncRegTypes[i]) {
2312      llvm::Type *TruncTy = ResultTruncRegTypes[i];
2313
2314      // Truncate the integer result to the right size, note that TruncTy can be
2315      // a pointer.
2316      if (TruncTy->isFloatingPointTy())
2317        Tmp = Builder.CreateFPTrunc(Tmp, TruncTy);
2318      else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
2319        uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(TruncTy);
2320        Tmp = Builder.CreateTrunc(Tmp,
2321                   llvm::IntegerType::get(getLLVMContext(), (unsigned)ResSize));
2322        Tmp = Builder.CreateIntToPtr(Tmp, TruncTy);
2323      } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
2324        uint64_t TmpSize =CGM.getDataLayout().getTypeSizeInBits(Tmp->getType());
2325        Tmp = Builder.CreatePtrToInt(Tmp,
2326                   llvm::IntegerType::get(getLLVMContext(), (unsigned)TmpSize));
2327        Tmp = Builder.CreateTrunc(Tmp, TruncTy);
2328      } else if (TruncTy->isIntegerTy()) {
2329        Tmp = Builder.CreateZExtOrTrunc(Tmp, TruncTy);
2330      } else if (TruncTy->isVectorTy()) {
2331        Tmp = Builder.CreateBitCast(Tmp, TruncTy);
2332      }
2333    }
2334
2335    LValue Dest = ResultRegDests[i];
2336    // ResultTypeRequiresCast elements correspond to the first
2337    // ResultTypeRequiresCast.size() elements of RegResults.
2338    if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) {
2339      unsigned Size = getContext().getTypeSize(ResultRegQualTys[i]);
2340      Address A = Builder.CreateBitCast(Dest.getAddress(*this),
2341                                        ResultRegTypes[i]->getPointerTo());
2342      QualType Ty = getContext().getIntTypeForBitwidth(Size, /*Signed*/ false);
2343      if (Ty.isNull()) {
2344        const Expr *OutExpr = S.getOutputExpr(i);
2345        CGM.Error(
2346            OutExpr->getExprLoc(),
2347            "impossible constraint in asm: can't store value into a register");
2348        return;
2349      }
2350      Dest = MakeAddrLValue(A, Ty);
2351    }
2352    EmitStoreThroughLValue(RValue::get(Tmp), Dest);
2353  }
2354}
2355
2356LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
2357  const RecordDecl *RD = S.getCapturedRecordDecl();
2358  QualType RecordTy = getContext().getRecordType(RD);
2359
2360  // Initialize the captured struct.
2361  LValue SlotLV =
2362    MakeAddrLValue(CreateMemTemp(RecordTy, "agg.captured"), RecordTy);
2363
2364  RecordDecl::field_iterator CurField = RD->field_begin();
2365  for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
2366                                                 E = S.capture_init_end();
2367       I != E; ++I, ++CurField) {
2368    LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField);
2369    if (CurField->hasCapturedVLAType()) {
2370      auto VAT = CurField->getCapturedVLAType();
2371      EmitStoreThroughLValue(RValue::get(VLASizeMap[VAT->getSizeExpr()]), LV);
2372    } else {
2373      EmitInitializerForField(*CurField, LV, *I);
2374    }
2375  }
2376
2377  return SlotLV;
2378}
2379
2380/// Generate an outlined function for the body of a CapturedStmt, store any
2381/// captured variables into the captured struct, and call the outlined function.
2382llvm::Function *
2383CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
2384  LValue CapStruct = InitCapturedStruct(S);
2385
2386  // Emit the CapturedDecl
2387  CodeGenFunction CGF(CGM, true);
2388  CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
2389  llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
2390  delete CGF.CapturedStmtInfo;
2391
2392  // Emit call to the helper function.
2393  EmitCallOrInvoke(F, CapStruct.getPointer(*this));
2394
2395  return F;
2396}
2397
2398Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
2399  LValue CapStruct = InitCapturedStruct(S);
2400  return CapStruct.getAddress(*this);
2401}
2402
2403/// Creates the outlined function for a CapturedStmt.
2404llvm::Function *
2405CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
2406  assert(CapturedStmtInfo &&
2407    "CapturedStmtInfo should be set when generating the captured function");
2408  const CapturedDecl *CD = S.getCapturedDecl();
2409  const RecordDecl *RD = S.getCapturedRecordDecl();
2410  SourceLocation Loc = S.getBeginLoc();
2411  assert(CD->hasBody() && "missing CapturedDecl body");
2412
2413  // Build the argument list.
2414  ASTContext &Ctx = CGM.getContext();
2415  FunctionArgList Args;
2416  Args.append(CD->param_begin(), CD->param_end());
2417
2418  // Create the function declaration.
2419  const CGFunctionInfo &FuncInfo =
2420    CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
2421  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
2422
2423  llvm::Function *F =
2424    llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
2425                           CapturedStmtInfo->getHelperName(), &CGM.getModule());
2426  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
2427  if (CD->isNothrow())
2428    F->addFnAttr(llvm::Attribute::NoUnwind);
2429
2430  // Generate the function.
2431  StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
2432                CD->getBody()->getBeginLoc());
2433  // Set the context parameter in CapturedStmtInfo.
2434  Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());
2435  CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
2436
2437  // Initialize variable-length arrays.
2438  LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(),
2439                                           Ctx.getTagDeclType(RD));
2440  for (auto *FD : RD->fields()) {
2441    if (FD->hasCapturedVLAType()) {
2442      auto *ExprArg =
2443          EmitLoadOfLValue(EmitLValueForField(Base, FD), S.getBeginLoc())
2444              .getScalarVal();
2445      auto VAT = FD->getCapturedVLAType();
2446      VLASizeMap[VAT->getSizeExpr()] = ExprArg;
2447    }
2448  }
2449
2450  // If 'this' is captured, load it into CXXThisValue.
2451  if (CapturedStmtInfo->isCXXThisExprCaptured()) {
2452    FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
2453    LValue ThisLValue = EmitLValueForField(Base, FD);
2454    CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();
2455  }
2456
2457  PGO.assignRegionCounters(GlobalDecl(CD), F);
2458  CapturedStmtInfo->EmitBody(*this, CD->getBody());
2459  FinishFunction(CD->getBodyRBrace());
2460
2461  return F;
2462}
2463