SampleProfile.cpp revision 360784
1177633Sdfr//===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===// 2177633Sdfr// 3261046Smav// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4261046Smav// See https://llvm.org/LICENSE.txt for license information. 5261046Smav// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6261046Smav// 7261046Smav//===----------------------------------------------------------------------===// 8261046Smav// 9261046Smav// This file implements the SampleProfileLoader transformation. This pass 10261046Smav// reads a profile file generated by a sampling profiler (e.g. Linux Perf - 11261046Smav// http://perf.wiki.kernel.org/) and generates IR metadata to reflect the 12261046Smav// profile information in the given profile. 13261046Smav// 14261046Smav// This pass generates branch weight annotations on the IR: 15261046Smav// 16261046Smav// - prof: Represents branch weights. This annotation is added to branches 17177633Sdfr// to indicate the weights of each edge coming out of the branch. 18261046Smav// The weight of each edge is the weight of the target block for 19261046Smav// that edge. The weight of a block B is computed as the maximum 20261046Smav// number of samples found in B. 21261046Smav// 22261046Smav//===----------------------------------------------------------------------===// 23261046Smav 24261046Smav#include "llvm/Transforms/IPO/SampleProfile.h" 25261046Smav#include "llvm/ADT/ArrayRef.h" 26261046Smav#include "llvm/ADT/DenseMap.h" 27261046Smav#include "llvm/ADT/DenseSet.h" 28261046Smav#include "llvm/ADT/None.h" 29177633Sdfr#include "llvm/ADT/SCCIterator.h" 30177633Sdfr#include "llvm/ADT/SmallPtrSet.h" 31177633Sdfr#include "llvm/ADT/SmallSet.h" 32177633Sdfr#include "llvm/ADT/SmallVector.h" 33177633Sdfr#include "llvm/ADT/Statistic.h" 34177633Sdfr#include "llvm/ADT/StringMap.h" 35177633Sdfr#include "llvm/ADT/StringRef.h" 36177633Sdfr#include "llvm/ADT/Twine.h" 37177633Sdfr#include "llvm/Analysis/AssumptionCache.h" 38177633Sdfr#include "llvm/Analysis/CallGraph.h" 39177633Sdfr#include "llvm/Analysis/CallGraphSCCPass.h" 40177633Sdfr#include "llvm/Analysis/InlineCost.h" 41177633Sdfr#include "llvm/Analysis/LoopInfo.h" 42177633Sdfr#include "llvm/Analysis/OptimizationRemarkEmitter.h" 43177633Sdfr#include "llvm/Analysis/PostDominators.h" 44177633Sdfr#include "llvm/Analysis/ProfileSummaryInfo.h" 45177633Sdfr#include "llvm/Analysis/TargetTransformInfo.h" 46177633Sdfr#include "llvm/IR/BasicBlock.h" 47177633Sdfr#include "llvm/IR/CFG.h" 48177633Sdfr#include "llvm/IR/CallSite.h" 49177633Sdfr#include "llvm/IR/DebugInfoMetadata.h" 50177633Sdfr#include "llvm/IR/DebugLoc.h" 51177633Sdfr#include "llvm/IR/DiagnosticInfo.h" 52177633Sdfr#include "llvm/IR/Dominators.h" 53177633Sdfr#include "llvm/IR/Function.h" 54177633Sdfr#include "llvm/IR/GlobalValue.h" 55184588Sdfr#include "llvm/IR/InstrTypes.h" 56177633Sdfr#include "llvm/IR/Instruction.h" 57177633Sdfr#include "llvm/IR/Instructions.h" 58177633Sdfr#include "llvm/IR/IntrinsicInst.h" 59196503Szec#include "llvm/IR/LLVMContext.h" 60196503Szec#include "llvm/IR/MDBuilder.h" 61177633Sdfr#include "llvm/IR/Module.h" 62177633Sdfr#include "llvm/IR/PassManager.h" 63177685Sdfr#include "llvm/IR/ValueSymbolTable.h" 64177633Sdfr#include "llvm/InitializePasses.h" 65177633Sdfr#include "llvm/Pass.h" 66184588Sdfr#include "llvm/ProfileData/InstrProf.h" 67184588Sdfr#include "llvm/ProfileData/SampleProf.h" 68184588Sdfr#include "llvm/ProfileData/SampleProfReader.h" 69261055Smav#include "llvm/Support/Casting.h" 70177633Sdfr#include "llvm/Support/CommandLine.h" 71177633Sdfr#include "llvm/Support/Debug.h" 72193272Sjhb#include "llvm/Support/ErrorHandling.h" 73177633Sdfr#include "llvm/Support/ErrorOr.h" 74177633Sdfr#include "llvm/Support/GenericDomTree.h" 75177633Sdfr#include "llvm/Support/raw_ostream.h" 76177633Sdfr#include "llvm/Transforms/IPO.h" 77177633Sdfr#include "llvm/Transforms/Instrumentation.h" 78177633Sdfr#include "llvm/Transforms/Utils/CallPromotionUtils.h" 79177633Sdfr#include "llvm/Transforms/Utils/Cloning.h" 80177633Sdfr#include "llvm/Transforms/Utils/MisExpect.h" 81177633Sdfr#include <algorithm> 82177633Sdfr#include <cassert> 83177633Sdfr#include <cstdint> 84177633Sdfr#include <functional> 85177633Sdfr#include <limits> 86177633Sdfr#include <map> 87177633Sdfr#include <memory> 88177633Sdfr#include <queue> 89177633Sdfr#include <string> 90177633Sdfr#include <system_error> 91177633Sdfr#include <utility> 92177633Sdfr#include <vector> 93177633Sdfr 94177633Sdfrusing namespace llvm; 95177633Sdfrusing namespace sampleprof; 96177633Sdfrusing ProfileCount = Function::ProfileCount; 97177633Sdfr#define DEBUG_TYPE "sample-profile" 98177633Sdfr#define CSINLINE_DEBUG DEBUG_TYPE "-inline" 99177633Sdfr 100177633SdfrSTATISTIC(NumCSInlined, 101177633Sdfr "Number of functions inlined with context sensitive profile"); 102177633SdfrSTATISTIC(NumCSNotInlined, 103177633Sdfr "Number of functions not inlined with context sensitive profile"); 104177633Sdfr 105177633Sdfr// Command line option to specify the file to read samples from. This is 106177633Sdfr// mainly used for debugging. 107177633Sdfrstatic cl::opt<std::string> SampleProfileFile( 108177633Sdfr "sample-profile-file", cl::init(""), cl::value_desc("filename"), 109177633Sdfr cl::desc("Profile file loaded by -sample-profile"), cl::Hidden); 110177633Sdfr 111177633Sdfr// The named file contains a set of transformations that may have been applied 112177633Sdfr// to the symbol names between the program from which the sample data was 113177633Sdfr// collected and the current program's symbols. 114177633Sdfrstatic cl::opt<std::string> SampleProfileRemappingFile( 115177633Sdfr "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), 116177633Sdfr cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden); 117177633Sdfr 118177633Sdfrstatic cl::opt<unsigned> SampleProfileMaxPropagateIterations( 119184588Sdfr "sample-profile-max-propagate-iterations", cl::init(100), 120184588Sdfr cl::desc("Maximum number of iterations to go through when propagating " 121177633Sdfr "sample block/edge weights through the CFG.")); 122177633Sdfr 123177633Sdfrstatic cl::opt<unsigned> SampleProfileRecordCoverage( 124177633Sdfr "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"), 125177633Sdfr cl::desc("Emit a warning if less than N% of records in the input profile " 126177633Sdfr "are matched to the IR.")); 127218757Sbz 128177633Sdfrstatic cl::opt<unsigned> SampleProfileSampleCoverage( 129196503Szec "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"), 130177633Sdfr cl::desc("Emit a warning if less than N% of samples in the input profile " 131177633Sdfr "are matched to the IR.")); 132177633Sdfr 133184588Sdfrstatic cl::opt<bool> NoWarnSampleUnused( 134177633Sdfr "no-warn-sample-unused", cl::init(false), cl::Hidden, 135177633Sdfr cl::desc("Use this option to turn off/on warnings about function with " 136177633Sdfr "samples but without debug information to use those samples. ")); 137177633Sdfr 138177633Sdfrstatic cl::opt<bool> ProfileSampleAccurate( 139193272Sjhb "profile-sample-accurate", cl::Hidden, cl::init(false), 140177633Sdfr cl::desc("If the sample profile is accurate, we will mark all un-sampled " 141177633Sdfr "callsite and function as having 0 samples. Otherwise, treat " 142177633Sdfr "un-sampled callsites and functions conservatively as unknown. ")); 143177633Sdfr 144177633Sdfrstatic cl::opt<bool> ProfileAccurateForSymsInList( 145303692Sngie "profile-accurate-for-symsinlist", cl::Hidden, cl::ZeroOrMore, 146303692Sngie cl::init(true), 147177633Sdfr cl::desc("For symbols in profile symbol list, regard their profiles to " 148177633Sdfr "be accurate. It may be overriden by profile-sample-accurate. ")); 149177633Sdfr 150177633Sdfrstatic cl::opt<bool> ProfileMergeInlinee( 151177633Sdfr "sample-profile-merge-inlinee", cl::Hidden, cl::init(false), 152177633Sdfr cl::desc("Merge past inlinee's profile to outline version if sample " 153177633Sdfr "profile loader decided not to inline a call site.")); 154177633Sdfr 155184588Sdfrstatic cl::opt<bool> ProfileTopDownLoad( 156184588Sdfr "sample-profile-top-down-load", cl::Hidden, cl::init(false), 157184588Sdfr cl::desc("Do profile annotation and inlining for functions in top-down " 158177633Sdfr "order of call graph during sample profile loading.")); 159177633Sdfr 160177633Sdfrstatic cl::opt<bool> ProfileSizeInline( 161177633Sdfr "sample-profile-inline-size", cl::Hidden, cl::init(false), 162184588Sdfr cl::desc("Inline cold call sites in profile loader if it's beneficial " 163184588Sdfr "for code size.")); 164177633Sdfr 165177633Sdfrstatic cl::opt<int> SampleColdCallSiteThreshold( 166177633Sdfr "sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45), 167177633Sdfr cl::desc("Threshold for inlining cold callsites")); 168184588Sdfr 169177633Sdfrnamespace { 170177633Sdfr 171177633Sdfrusing BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>; 172184588Sdfrusing EquivalenceClassMap = DenseMap<const BasicBlock *, const BasicBlock *>; 173184588Sdfrusing Edge = std::pair<const BasicBlock *, const BasicBlock *>; 174184588Sdfrusing EdgeWeightMap = DenseMap<Edge, uint64_t>; 175184588Sdfrusing BlockEdgeMap = 176184588Sdfr DenseMap<const BasicBlock *, SmallVector<const BasicBlock *, 8>>; 177177633Sdfr 178177633Sdfrclass SampleProfileLoader; 179177633Sdfr 180177633Sdfrclass SampleCoverageTracker { 181177633Sdfrpublic: 182177633Sdfr SampleCoverageTracker(SampleProfileLoader &SPL) : SPLoader(SPL){}; 183177633Sdfr 184177633Sdfr bool markSamplesUsed(const FunctionSamples *FS, uint32_t LineOffset, 185177633Sdfr uint32_t Discriminator, uint64_t Samples); 186177633Sdfr unsigned computeCoverage(unsigned Used, unsigned Total) const; 187177633Sdfr unsigned countUsedRecords(const FunctionSamples *FS, 188177633Sdfr ProfileSummaryInfo *PSI) const; 189177633Sdfr unsigned countBodyRecords(const FunctionSamples *FS, 190184588Sdfr ProfileSummaryInfo *PSI) const; 191184588Sdfr uint64_t getTotalUsedSamples() const { return TotalUsedSamples; } 192184588Sdfr uint64_t countBodySamples(const FunctionSamples *FS, 193184588Sdfr ProfileSummaryInfo *PSI) const; 194184588Sdfr 195184588Sdfr void clear() { 196184588Sdfr SampleCoverage.clear(); 197184588Sdfr TotalUsedSamples = 0; 198261053Smav } 199184588Sdfr 200261053Smavprivate: 201261053Smav using BodySampleCoverageMap = std::map<LineLocation, unsigned>; 202184588Sdfr using FunctionSamplesCoverageMap = 203177633Sdfr DenseMap<const FunctionSamples *, BodySampleCoverageMap>; 204177633Sdfr 205177633Sdfr /// Coverage map for sampling records. 206177633Sdfr /// 207177633Sdfr /// This map keeps a record of sampling records that have been matched to 208193272Sjhb /// an IR instruction. This is used to detect some form of staleness in 209177633Sdfr /// profiles (see flag -sample-profile-check-coverage). 210261053Smav /// 211184588Sdfr /// Each entry in the map corresponds to a FunctionSamples instance. This is 212177633Sdfr /// another map that counts how many times the sample record at the 213177633Sdfr /// given location has been used. 214177633Sdfr FunctionSamplesCoverageMap SampleCoverage; 215184588Sdfr 216177633Sdfr /// Number of samples used from the profile. 217184588Sdfr /// 218184588Sdfr /// When a sampling record is used for the first time, the samples from 219184588Sdfr /// that record are added to this accumulator. Coverage is later computed 220177633Sdfr /// based on the total number of samples available in this function and 221177633Sdfr /// its callsites. 222177633Sdfr /// 223184588Sdfr /// Note that this accumulator tracks samples used from a single function 224184588Sdfr /// and all the inlined callsites. Strictly, we should have a map of counters 225184588Sdfr /// keyed by FunctionSamples pointers, but these stats are cleared after 226184588Sdfr /// every function, so we just need to keep a single counter. 227177633Sdfr uint64_t TotalUsedSamples = 0; 228177633Sdfr 229177633Sdfr SampleProfileLoader &SPLoader; 230177633Sdfr}; 231184588Sdfr 232261055Smavclass GUIDToFuncNameMapper { 233177633Sdfrpublic: 234184588Sdfr GUIDToFuncNameMapper(Module &M, SampleProfileReader &Reader, 235177633Sdfr DenseMap<uint64_t, StringRef> &GUIDToFuncNameMap) 236184588Sdfr : CurrentReader(Reader), CurrentModule(M), 237177633Sdfr CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) { 238177633Sdfr if (CurrentReader.getFormat() != SPF_Compact_Binary) 239248195Sglebius return; 240177633Sdfr 241184588Sdfr for (const auto &F : CurrentModule) { 242184588Sdfr StringRef OrigName = F.getName(); 243184588Sdfr CurrentGUIDToFuncNameMap.insert( 244184588Sdfr {Function::getGUID(OrigName), OrigName}); 245184588Sdfr 246184588Sdfr // Local to global var promotion used by optimization like thinlto 247184588Sdfr // will rename the var and add suffix like ".llvm.xxx" to the 248184588Sdfr // original local name. In sample profile, the suffixes of function 249184588Sdfr // names are all stripped. Since it is possible that the mapper is 250184588Sdfr // built in post-thin-link phase and var promotion has been done, 251184588Sdfr // we need to add the substring of function name without the suffix 252184588Sdfr // into the GUIDToFuncNameMap. 253184588Sdfr StringRef CanonName = FunctionSamples::getCanonicalFnName(F); 254177633Sdfr if (CanonName != OrigName) 255184588Sdfr CurrentGUIDToFuncNameMap.insert( 256177633Sdfr {Function::getGUID(CanonName), CanonName}); 257177633Sdfr } 258177633Sdfr 259177633Sdfr // Update GUIDToFuncNameMap for each function including inlinees. 260177633Sdfr SetGUIDToFuncNameMapForAll(&CurrentGUIDToFuncNameMap); 261177633Sdfr } 262177633Sdfr 263177633Sdfr ~GUIDToFuncNameMapper() { 264184588Sdfr if (CurrentReader.getFormat() != SPF_Compact_Binary) 265177633Sdfr return; 266177633Sdfr 267177633Sdfr CurrentGUIDToFuncNameMap.clear(); 268177633Sdfr 269177633Sdfr // Reset GUIDToFuncNameMap for of each function as they're no 270177633Sdfr // longer valid at this point. 271177633Sdfr SetGUIDToFuncNameMapForAll(nullptr); 272177633Sdfr } 273184588Sdfr 274177633Sdfrprivate: 275193272Sjhb void SetGUIDToFuncNameMapForAll(DenseMap<uint64_t, StringRef> *Map) { 276177633Sdfr std::queue<FunctionSamples *> FSToUpdate; 277177633Sdfr for (auto &IFS : CurrentReader.getProfiles()) { 278184588Sdfr FSToUpdate.push(&IFS.second); 279177633Sdfr } 280177633Sdfr 281177633Sdfr while (!FSToUpdate.empty()) { 282184588Sdfr FunctionSamples *FS = FSToUpdate.front(); 283184588Sdfr FSToUpdate.pop(); 284184588Sdfr FS->GUIDToFuncNameMap = Map; 285177633Sdfr for (const auto &ICS : FS->getCallsiteSamples()) { 286177633Sdfr const FunctionSamplesMap &FSMap = ICS.second; 287177633Sdfr for (auto &IFS : FSMap) { 288177633Sdfr FunctionSamples &FS = const_cast<FunctionSamples &>(IFS.second); 289177633Sdfr FSToUpdate.push(&FS); 290177633Sdfr } 291177633Sdfr } 292177633Sdfr } 293177633Sdfr } 294177633Sdfr 295177633Sdfr SampleProfileReader &CurrentReader; 296177633Sdfr Module &CurrentModule; 297177633Sdfr DenseMap<uint64_t, StringRef> &CurrentGUIDToFuncNameMap; 298193272Sjhb}; 299177633Sdfr 300177633Sdfr/// Sample profile pass. 301177633Sdfr/// 302177633Sdfr/// This pass reads profile data from the file specified by 303177633Sdfr/// -sample-profile-file and annotates every affected function with the 304193272Sjhb/// profile information found in that file. 305177633Sdfrclass SampleProfileLoader { 306public: 307 SampleProfileLoader( 308 StringRef Name, StringRef RemapName, bool IsThinLTOPreLink, 309 std::function<AssumptionCache &(Function &)> GetAssumptionCache, 310 std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo) 311 : GetAC(std::move(GetAssumptionCache)), 312 GetTTI(std::move(GetTargetTransformInfo)), CoverageTracker(*this), 313 Filename(Name), RemappingFilename(RemapName), 314 IsThinLTOPreLink(IsThinLTOPreLink) {} 315 316 bool doInitialization(Module &M); 317 bool runOnModule(Module &M, ModuleAnalysisManager *AM, 318 ProfileSummaryInfo *_PSI, CallGraph *CG); 319 320 void dump() { Reader->dump(); } 321 322protected: 323 friend class SampleCoverageTracker; 324 325 bool runOnFunction(Function &F, ModuleAnalysisManager *AM); 326 unsigned getFunctionLoc(Function &F); 327 bool emitAnnotations(Function &F); 328 ErrorOr<uint64_t> getInstWeight(const Instruction &I); 329 ErrorOr<uint64_t> getBlockWeight(const BasicBlock *BB); 330 const FunctionSamples *findCalleeFunctionSamples(const Instruction &I) const; 331 std::vector<const FunctionSamples *> 332 findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const; 333 mutable DenseMap<const DILocation *, const FunctionSamples *> DILocation2SampleMap; 334 const FunctionSamples *findFunctionSamples(const Instruction &I) const; 335 bool inlineCallInstruction(Instruction *I); 336 bool inlineHotFunctions(Function &F, 337 DenseSet<GlobalValue::GUID> &InlinedGUIDs); 338 // Inline cold/small functions in addition to hot ones 339 bool shouldInlineColdCallee(Instruction &CallInst); 340 void emitOptimizationRemarksForInlineCandidates( 341 const SmallVector<Instruction *, 10> &Candidates, const Function &F, bool Hot); 342 void printEdgeWeight(raw_ostream &OS, Edge E); 343 void printBlockWeight(raw_ostream &OS, const BasicBlock *BB) const; 344 void printBlockEquivalence(raw_ostream &OS, const BasicBlock *BB); 345 bool computeBlockWeights(Function &F); 346 void findEquivalenceClasses(Function &F); 347 template <bool IsPostDom> 348 void findEquivalencesFor(BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants, 349 DominatorTreeBase<BasicBlock, IsPostDom> *DomTree); 350 351 void propagateWeights(Function &F); 352 uint64_t visitEdge(Edge E, unsigned *NumUnknownEdges, Edge *UnknownEdge); 353 void buildEdges(Function &F); 354 std::vector<Function *> buildFunctionOrder(Module &M, CallGraph *CG); 355 bool propagateThroughEdges(Function &F, bool UpdateBlockCount); 356 void computeDominanceAndLoopInfo(Function &F); 357 void clearFunctionData(); 358 bool callsiteIsHot(const FunctionSamples *CallsiteFS, 359 ProfileSummaryInfo *PSI); 360 361 /// Map basic blocks to their computed weights. 362 /// 363 /// The weight of a basic block is defined to be the maximum 364 /// of all the instruction weights in that block. 365 BlockWeightMap BlockWeights; 366 367 /// Map edges to their computed weights. 368 /// 369 /// Edge weights are computed by propagating basic block weights in 370 /// SampleProfile::propagateWeights. 371 EdgeWeightMap EdgeWeights; 372 373 /// Set of visited blocks during propagation. 374 SmallPtrSet<const BasicBlock *, 32> VisitedBlocks; 375 376 /// Set of visited edges during propagation. 377 SmallSet<Edge, 32> VisitedEdges; 378 379 /// Equivalence classes for block weights. 380 /// 381 /// Two blocks BB1 and BB2 are in the same equivalence class if they 382 /// dominate and post-dominate each other, and they are in the same loop 383 /// nest. When this happens, the two blocks are guaranteed to execute 384 /// the same number of times. 385 EquivalenceClassMap EquivalenceClass; 386 387 /// Map from function name to Function *. Used to find the function from 388 /// the function name. If the function name contains suffix, additional 389 /// entry is added to map from the stripped name to the function if there 390 /// is one-to-one mapping. 391 StringMap<Function *> SymbolMap; 392 393 /// Dominance, post-dominance and loop information. 394 std::unique_ptr<DominatorTree> DT; 395 std::unique_ptr<PostDominatorTree> PDT; 396 std::unique_ptr<LoopInfo> LI; 397 398 std::function<AssumptionCache &(Function &)> GetAC; 399 std::function<TargetTransformInfo &(Function &)> GetTTI; 400 401 /// Predecessors for each basic block in the CFG. 402 BlockEdgeMap Predecessors; 403 404 /// Successors for each basic block in the CFG. 405 BlockEdgeMap Successors; 406 407 SampleCoverageTracker CoverageTracker; 408 409 /// Profile reader object. 410 std::unique_ptr<SampleProfileReader> Reader; 411 412 /// Samples collected for the body of this function. 413 FunctionSamples *Samples = nullptr; 414 415 /// Name of the profile file to load. 416 std::string Filename; 417 418 /// Name of the profile remapping file to load. 419 std::string RemappingFilename; 420 421 /// Flag indicating whether the profile input loaded successfully. 422 bool ProfileIsValid = false; 423 424 /// Flag indicating if the pass is invoked in ThinLTO compile phase. 425 /// 426 /// In this phase, in annotation, we should not promote indirect calls. 427 /// Instead, we will mark GUIDs that needs to be annotated to the function. 428 bool IsThinLTOPreLink; 429 430 /// Profile Summary Info computed from sample profile. 431 ProfileSummaryInfo *PSI = nullptr; 432 433 /// Profle Symbol list tells whether a function name appears in the binary 434 /// used to generate the current profile. 435 std::unique_ptr<ProfileSymbolList> PSL; 436 437 /// Total number of samples collected in this profile. 438 /// 439 /// This is the sum of all the samples collected in all the functions executed 440 /// at runtime. 441 uint64_t TotalCollectedSamples = 0; 442 443 /// Optimization Remark Emitter used to emit diagnostic remarks. 444 OptimizationRemarkEmitter *ORE = nullptr; 445 446 // Information recorded when we declined to inline a call site 447 // because we have determined it is too cold is accumulated for 448 // each callee function. Initially this is just the entry count. 449 struct NotInlinedProfileInfo { 450 uint64_t entryCount; 451 }; 452 DenseMap<Function *, NotInlinedProfileInfo> notInlinedCallInfo; 453 454 // GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for 455 // all the function symbols defined or declared in current module. 456 DenseMap<uint64_t, StringRef> GUIDToFuncNameMap; 457 458 // All the Names used in FunctionSamples including outline function 459 // names, inline instance names and call target names. 460 StringSet<> NamesInProfile; 461 462 // For symbol in profile symbol list, whether to regard their profiles 463 // to be accurate. It is mainly decided by existance of profile symbol 464 // list and -profile-accurate-for-symsinlist flag, but it can be 465 // overriden by -profile-sample-accurate or profile-sample-accurate 466 // attribute. 467 bool ProfAccForSymsInList; 468}; 469 470class SampleProfileLoaderLegacyPass : public ModulePass { 471public: 472 // Class identification, replacement for typeinfo 473 static char ID; 474 475 SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile, 476 bool IsThinLTOPreLink = false) 477 : ModulePass(ID), 478 SampleLoader(Name, SampleProfileRemappingFile, IsThinLTOPreLink, 479 [&](Function &F) -> AssumptionCache & { 480 return ACT->getAssumptionCache(F); 481 }, 482 [&](Function &F) -> TargetTransformInfo & { 483 return TTIWP->getTTI(F); 484 }) { 485 initializeSampleProfileLoaderLegacyPassPass( 486 *PassRegistry::getPassRegistry()); 487 } 488 489 void dump() { SampleLoader.dump(); } 490 491 bool doInitialization(Module &M) override { 492 return SampleLoader.doInitialization(M); 493 } 494 495 StringRef getPassName() const override { return "Sample profile pass"; } 496 bool runOnModule(Module &M) override; 497 498 void getAnalysisUsage(AnalysisUsage &AU) const override { 499 AU.addRequired<AssumptionCacheTracker>(); 500 AU.addRequired<TargetTransformInfoWrapperPass>(); 501 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 502 } 503 504private: 505 SampleProfileLoader SampleLoader; 506 AssumptionCacheTracker *ACT = nullptr; 507 TargetTransformInfoWrapperPass *TTIWP = nullptr; 508}; 509 510} // end anonymous namespace 511 512/// Return true if the given callsite is hot wrt to hot cutoff threshold. 513/// 514/// Functions that were inlined in the original binary will be represented 515/// in the inline stack in the sample profile. If the profile shows that 516/// the original inline decision was "good" (i.e., the callsite is executed 517/// frequently), then we will recreate the inline decision and apply the 518/// profile from the inlined callsite. 519/// 520/// To decide whether an inlined callsite is hot, we compare the callsite 521/// sample count with the hot cutoff computed by ProfileSummaryInfo, it is 522/// regarded as hot if the count is above the cutoff value. 523/// 524/// When ProfileAccurateForSymsInList is enabled and profile symbol list 525/// is present, functions in the profile symbol list but without profile will 526/// be regarded as cold and much less inlining will happen in CGSCC inlining 527/// pass, so we tend to lower the hot criteria here to allow more early 528/// inlining to happen for warm callsites and it is helpful for performance. 529bool SampleProfileLoader::callsiteIsHot(const FunctionSamples *CallsiteFS, 530 ProfileSummaryInfo *PSI) { 531 if (!CallsiteFS) 532 return false; // The callsite was not inlined in the original binary. 533 534 assert(PSI && "PSI is expected to be non null"); 535 uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples(); 536 if (ProfAccForSymsInList) 537 return !PSI->isColdCount(CallsiteTotalSamples); 538 else 539 return PSI->isHotCount(CallsiteTotalSamples); 540} 541 542/// Mark as used the sample record for the given function samples at 543/// (LineOffset, Discriminator). 544/// 545/// \returns true if this is the first time we mark the given record. 546bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS, 547 uint32_t LineOffset, 548 uint32_t Discriminator, 549 uint64_t Samples) { 550 LineLocation Loc(LineOffset, Discriminator); 551 unsigned &Count = SampleCoverage[FS][Loc]; 552 bool FirstTime = (++Count == 1); 553 if (FirstTime) 554 TotalUsedSamples += Samples; 555 return FirstTime; 556} 557 558/// Return the number of sample records that were applied from this profile. 559/// 560/// This count does not include records from cold inlined callsites. 561unsigned 562SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS, 563 ProfileSummaryInfo *PSI) const { 564 auto I = SampleCoverage.find(FS); 565 566 // The size of the coverage map for FS represents the number of records 567 // that were marked used at least once. 568 unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0; 569 570 // If there are inlined callsites in this function, count the samples found 571 // in the respective bodies. However, do not bother counting callees with 0 572 // total samples, these are callees that were never invoked at runtime. 573 for (const auto &I : FS->getCallsiteSamples()) 574 for (const auto &J : I.second) { 575 const FunctionSamples *CalleeSamples = &J.second; 576 if (SPLoader.callsiteIsHot(CalleeSamples, PSI)) 577 Count += countUsedRecords(CalleeSamples, PSI); 578 } 579 580 return Count; 581} 582 583/// Return the number of sample records in the body of this profile. 584/// 585/// This count does not include records from cold inlined callsites. 586unsigned 587SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS, 588 ProfileSummaryInfo *PSI) const { 589 unsigned Count = FS->getBodySamples().size(); 590 591 // Only count records in hot callsites. 592 for (const auto &I : FS->getCallsiteSamples()) 593 for (const auto &J : I.second) { 594 const FunctionSamples *CalleeSamples = &J.second; 595 if (SPLoader.callsiteIsHot(CalleeSamples, PSI)) 596 Count += countBodyRecords(CalleeSamples, PSI); 597 } 598 599 return Count; 600} 601 602/// Return the number of samples collected in the body of this profile. 603/// 604/// This count does not include samples from cold inlined callsites. 605uint64_t 606SampleCoverageTracker::countBodySamples(const FunctionSamples *FS, 607 ProfileSummaryInfo *PSI) const { 608 uint64_t Total = 0; 609 for (const auto &I : FS->getBodySamples()) 610 Total += I.second.getSamples(); 611 612 // Only count samples in hot callsites. 613 for (const auto &I : FS->getCallsiteSamples()) 614 for (const auto &J : I.second) { 615 const FunctionSamples *CalleeSamples = &J.second; 616 if (SPLoader.callsiteIsHot(CalleeSamples, PSI)) 617 Total += countBodySamples(CalleeSamples, PSI); 618 } 619 620 return Total; 621} 622 623/// Return the fraction of sample records used in this profile. 624/// 625/// The returned value is an unsigned integer in the range 0-100 indicating 626/// the percentage of sample records that were used while applying this 627/// profile to the associated function. 628unsigned SampleCoverageTracker::computeCoverage(unsigned Used, 629 unsigned Total) const { 630 assert(Used <= Total && 631 "number of used records cannot exceed the total number of records"); 632 return Total > 0 ? Used * 100 / Total : 100; 633} 634 635/// Clear all the per-function data used to load samples and propagate weights. 636void SampleProfileLoader::clearFunctionData() { 637 BlockWeights.clear(); 638 EdgeWeights.clear(); 639 VisitedBlocks.clear(); 640 VisitedEdges.clear(); 641 EquivalenceClass.clear(); 642 DT = nullptr; 643 PDT = nullptr; 644 LI = nullptr; 645 Predecessors.clear(); 646 Successors.clear(); 647 CoverageTracker.clear(); 648} 649 650#ifndef NDEBUG 651/// Print the weight of edge \p E on stream \p OS. 652/// 653/// \param OS Stream to emit the output to. 654/// \param E Edge to print. 655void SampleProfileLoader::printEdgeWeight(raw_ostream &OS, Edge E) { 656 OS << "weight[" << E.first->getName() << "->" << E.second->getName() 657 << "]: " << EdgeWeights[E] << "\n"; 658} 659 660/// Print the equivalence class of block \p BB on stream \p OS. 661/// 662/// \param OS Stream to emit the output to. 663/// \param BB Block to print. 664void SampleProfileLoader::printBlockEquivalence(raw_ostream &OS, 665 const BasicBlock *BB) { 666 const BasicBlock *Equiv = EquivalenceClass[BB]; 667 OS << "equivalence[" << BB->getName() 668 << "]: " << ((Equiv) ? EquivalenceClass[BB]->getName() : "NONE") << "\n"; 669} 670 671/// Print the weight of block \p BB on stream \p OS. 672/// 673/// \param OS Stream to emit the output to. 674/// \param BB Block to print. 675void SampleProfileLoader::printBlockWeight(raw_ostream &OS, 676 const BasicBlock *BB) const { 677 const auto &I = BlockWeights.find(BB); 678 uint64_t W = (I == BlockWeights.end() ? 0 : I->second); 679 OS << "weight[" << BB->getName() << "]: " << W << "\n"; 680} 681#endif 682 683/// Get the weight for an instruction. 684/// 685/// The "weight" of an instruction \p Inst is the number of samples 686/// collected on that instruction at runtime. To retrieve it, we 687/// need to compute the line number of \p Inst relative to the start of its 688/// function. We use HeaderLineno to compute the offset. We then 689/// look up the samples collected for \p Inst using BodySamples. 690/// 691/// \param Inst Instruction to query. 692/// 693/// \returns the weight of \p Inst. 694ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) { 695 const DebugLoc &DLoc = Inst.getDebugLoc(); 696 if (!DLoc) 697 return std::error_code(); 698 699 const FunctionSamples *FS = findFunctionSamples(Inst); 700 if (!FS) 701 return std::error_code(); 702 703 // Ignore all intrinsics, phinodes and branch instructions. 704 // Branch and phinodes instruction usually contains debug info from sources outside of 705 // the residing basic block, thus we ignore them during annotation. 706 if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst) || isa<PHINode>(Inst)) 707 return std::error_code(); 708 709 // If a direct call/invoke instruction is inlined in profile 710 // (findCalleeFunctionSamples returns non-empty result), but not inlined here, 711 // it means that the inlined callsite has no sample, thus the call 712 // instruction should have 0 count. 713 if ((isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) && 714 !ImmutableCallSite(&Inst).isIndirectCall() && 715 findCalleeFunctionSamples(Inst)) 716 return 0; 717 718 const DILocation *DIL = DLoc; 719 uint32_t LineOffset = FunctionSamples::getOffset(DIL); 720 uint32_t Discriminator = DIL->getBaseDiscriminator(); 721 ErrorOr<uint64_t> R = FS->findSamplesAt(LineOffset, Discriminator); 722 if (R) { 723 bool FirstMark = 724 CoverageTracker.markSamplesUsed(FS, LineOffset, Discriminator, R.get()); 725 if (FirstMark) { 726 ORE->emit([&]() { 727 OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "AppliedSamples", &Inst); 728 Remark << "Applied " << ore::NV("NumSamples", *R); 729 Remark << " samples from profile (offset: "; 730 Remark << ore::NV("LineOffset", LineOffset); 731 if (Discriminator) { 732 Remark << "."; 733 Remark << ore::NV("Discriminator", Discriminator); 734 } 735 Remark << ")"; 736 return Remark; 737 }); 738 } 739 LLVM_DEBUG(dbgs() << " " << DLoc.getLine() << "." 740 << DIL->getBaseDiscriminator() << ":" << Inst 741 << " (line offset: " << LineOffset << "." 742 << DIL->getBaseDiscriminator() << " - weight: " << R.get() 743 << ")\n"); 744 } 745 return R; 746} 747 748/// Compute the weight of a basic block. 749/// 750/// The weight of basic block \p BB is the maximum weight of all the 751/// instructions in BB. 752/// 753/// \param BB The basic block to query. 754/// 755/// \returns the weight for \p BB. 756ErrorOr<uint64_t> SampleProfileLoader::getBlockWeight(const BasicBlock *BB) { 757 uint64_t Max = 0; 758 bool HasWeight = false; 759 for (auto &I : BB->getInstList()) { 760 const ErrorOr<uint64_t> &R = getInstWeight(I); 761 if (R) { 762 Max = std::max(Max, R.get()); 763 HasWeight = true; 764 } 765 } 766 return HasWeight ? ErrorOr<uint64_t>(Max) : std::error_code(); 767} 768 769/// Compute and store the weights of every basic block. 770/// 771/// This populates the BlockWeights map by computing 772/// the weights of every basic block in the CFG. 773/// 774/// \param F The function to query. 775bool SampleProfileLoader::computeBlockWeights(Function &F) { 776 bool Changed = false; 777 LLVM_DEBUG(dbgs() << "Block weights\n"); 778 for (const auto &BB : F) { 779 ErrorOr<uint64_t> Weight = getBlockWeight(&BB); 780 if (Weight) { 781 BlockWeights[&BB] = Weight.get(); 782 VisitedBlocks.insert(&BB); 783 Changed = true; 784 } 785 LLVM_DEBUG(printBlockWeight(dbgs(), &BB)); 786 } 787 788 return Changed; 789} 790 791/// Get the FunctionSamples for a call instruction. 792/// 793/// The FunctionSamples of a call/invoke instruction \p Inst is the inlined 794/// instance in which that call instruction is calling to. It contains 795/// all samples that resides in the inlined instance. We first find the 796/// inlined instance in which the call instruction is from, then we 797/// traverse its children to find the callsite with the matching 798/// location. 799/// 800/// \param Inst Call/Invoke instruction to query. 801/// 802/// \returns The FunctionSamples pointer to the inlined instance. 803const FunctionSamples * 804SampleProfileLoader::findCalleeFunctionSamples(const Instruction &Inst) const { 805 const DILocation *DIL = Inst.getDebugLoc(); 806 if (!DIL) { 807 return nullptr; 808 } 809 810 StringRef CalleeName; 811 if (const CallInst *CI = dyn_cast<CallInst>(&Inst)) 812 if (Function *Callee = CI->getCalledFunction()) 813 CalleeName = Callee->getName(); 814 815 const FunctionSamples *FS = findFunctionSamples(Inst); 816 if (FS == nullptr) 817 return nullptr; 818 819 return FS->findFunctionSamplesAt(LineLocation(FunctionSamples::getOffset(DIL), 820 DIL->getBaseDiscriminator()), 821 CalleeName); 822} 823 824/// Returns a vector of FunctionSamples that are the indirect call targets 825/// of \p Inst. The vector is sorted by the total number of samples. Stores 826/// the total call count of the indirect call in \p Sum. 827std::vector<const FunctionSamples *> 828SampleProfileLoader::findIndirectCallFunctionSamples( 829 const Instruction &Inst, uint64_t &Sum) const { 830 const DILocation *DIL = Inst.getDebugLoc(); 831 std::vector<const FunctionSamples *> R; 832 833 if (!DIL) { 834 return R; 835 } 836 837 const FunctionSamples *FS = findFunctionSamples(Inst); 838 if (FS == nullptr) 839 return R; 840 841 uint32_t LineOffset = FunctionSamples::getOffset(DIL); 842 uint32_t Discriminator = DIL->getBaseDiscriminator(); 843 844 auto T = FS->findCallTargetMapAt(LineOffset, Discriminator); 845 Sum = 0; 846 if (T) 847 for (const auto &T_C : T.get()) 848 Sum += T_C.second; 849 if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(LineLocation( 850 FunctionSamples::getOffset(DIL), DIL->getBaseDiscriminator()))) { 851 if (M->empty()) 852 return R; 853 for (const auto &NameFS : *M) { 854 Sum += NameFS.second.getEntrySamples(); 855 R.push_back(&NameFS.second); 856 } 857 llvm::sort(R, [](const FunctionSamples *L, const FunctionSamples *R) { 858 if (L->getEntrySamples() != R->getEntrySamples()) 859 return L->getEntrySamples() > R->getEntrySamples(); 860 return FunctionSamples::getGUID(L->getName()) < 861 FunctionSamples::getGUID(R->getName()); 862 }); 863 } 864 return R; 865} 866 867/// Get the FunctionSamples for an instruction. 868/// 869/// The FunctionSamples of an instruction \p Inst is the inlined instance 870/// in which that instruction is coming from. We traverse the inline stack 871/// of that instruction, and match it with the tree nodes in the profile. 872/// 873/// \param Inst Instruction to query. 874/// 875/// \returns the FunctionSamples pointer to the inlined instance. 876const FunctionSamples * 877SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const { 878 const DILocation *DIL = Inst.getDebugLoc(); 879 if (!DIL) 880 return Samples; 881 882 auto it = DILocation2SampleMap.try_emplace(DIL,nullptr); 883 if (it.second) 884 it.first->second = Samples->findFunctionSamples(DIL); 885 return it.first->second; 886} 887 888bool SampleProfileLoader::inlineCallInstruction(Instruction *I) { 889 assert(isa<CallInst>(I) || isa<InvokeInst>(I)); 890 CallSite CS(I); 891 Function *CalledFunction = CS.getCalledFunction(); 892 assert(CalledFunction); 893 DebugLoc DLoc = I->getDebugLoc(); 894 BasicBlock *BB = I->getParent(); 895 InlineParams Params = getInlineParams(); 896 Params.ComputeFullInlineCost = true; 897 // Checks if there is anything in the reachable portion of the callee at 898 // this callsite that makes this inlining potentially illegal. Need to 899 // set ComputeFullInlineCost, otherwise getInlineCost may return early 900 // when cost exceeds threshold without checking all IRs in the callee. 901 // The acutal cost does not matter because we only checks isNever() to 902 // see if it is legal to inline the callsite. 903 InlineCost Cost = 904 getInlineCost(cast<CallBase>(*I), Params, GetTTI(*CalledFunction), GetAC, 905 None, nullptr, nullptr); 906 if (Cost.isNever()) { 907 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG, "InlineFail", DLoc, BB) 908 << "incompatible inlining"); 909 return false; 910 } 911 InlineFunctionInfo IFI(nullptr, &GetAC); 912 if (InlineFunction(CS, IFI)) { 913 // The call to InlineFunction erases I, so we can't pass it here. 914 ORE->emit(OptimizationRemark(CSINLINE_DEBUG, "InlineSuccess", DLoc, BB) 915 << "inlined callee '" << ore::NV("Callee", CalledFunction) 916 << "' into '" << ore::NV("Caller", BB->getParent()) << "'"); 917 return true; 918 } 919 return false; 920} 921 922bool SampleProfileLoader::shouldInlineColdCallee(Instruction &CallInst) { 923 if (!ProfileSizeInline) 924 return false; 925 926 Function *Callee = CallSite(&CallInst).getCalledFunction(); 927 if (Callee == nullptr) 928 return false; 929 930 InlineCost Cost = 931 getInlineCost(cast<CallBase>(CallInst), getInlineParams(), 932 GetTTI(*Callee), GetAC, None, nullptr, nullptr); 933 934 return Cost.getCost() <= SampleColdCallSiteThreshold; 935} 936 937void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates( 938 const SmallVector<Instruction *, 10> &Candidates, const Function &F, 939 bool Hot) { 940 for (auto I : Candidates) { 941 Function *CalledFunction = CallSite(I).getCalledFunction(); 942 if (CalledFunction) { 943 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG, "InlineAttempt", 944 I->getDebugLoc(), I->getParent()) 945 << "previous inlining reattempted for " 946 << (Hot ? "hotness: '" : "size: '") 947 << ore::NV("Callee", CalledFunction) << "' into '" 948 << ore::NV("Caller", &F) << "'"); 949 } 950 } 951} 952 953/// Iteratively inline hot callsites of a function. 954/// 955/// Iteratively traverse all callsites of the function \p F, and find if 956/// the corresponding inlined instance exists and is hot in profile. If 957/// it is hot enough, inline the callsites and adds new callsites of the 958/// callee into the caller. If the call is an indirect call, first promote 959/// it to direct call. Each indirect call is limited with a single target. 960/// 961/// \param F function to perform iterative inlining. 962/// \param InlinedGUIDs a set to be updated to include all GUIDs that are 963/// inlined in the profiled binary. 964/// 965/// \returns True if there is any inline happened. 966bool SampleProfileLoader::inlineHotFunctions( 967 Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) { 968 DenseSet<Instruction *> PromotedInsns; 969 970 // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure 971 // Profile symbol list is ignored when profile-sample-accurate is on. 972 assert((!ProfAccForSymsInList || 973 (!ProfileSampleAccurate && 974 !F.hasFnAttribute("profile-sample-accurate"))) && 975 "ProfAccForSymsInList should be false when profile-sample-accurate " 976 "is enabled"); 977 978 DenseMap<Instruction *, const FunctionSamples *> localNotInlinedCallSites; 979 bool Changed = false; 980 while (true) { 981 bool LocalChanged = false; 982 SmallVector<Instruction *, 10> CIS; 983 for (auto &BB : F) { 984 bool Hot = false; 985 SmallVector<Instruction *, 10> AllCandidates; 986 SmallVector<Instruction *, 10> ColdCandidates; 987 for (auto &I : BB.getInstList()) { 988 const FunctionSamples *FS = nullptr; 989 if ((isa<CallInst>(I) || isa<InvokeInst>(I)) && 990 !isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(I))) { 991 AllCandidates.push_back(&I); 992 if (FS->getEntrySamples() > 0) 993 localNotInlinedCallSites.try_emplace(&I, FS); 994 if (callsiteIsHot(FS, PSI)) 995 Hot = true; 996 else if (shouldInlineColdCallee(I)) 997 ColdCandidates.push_back(&I); 998 } 999 } 1000 if (Hot) { 1001 CIS.insert(CIS.begin(), AllCandidates.begin(), AllCandidates.end()); 1002 emitOptimizationRemarksForInlineCandidates(AllCandidates, F, true); 1003 } 1004 else { 1005 CIS.insert(CIS.begin(), ColdCandidates.begin(), ColdCandidates.end()); 1006 emitOptimizationRemarksForInlineCandidates(ColdCandidates, F, false); 1007 } 1008 } 1009 for (auto I : CIS) { 1010 Function *CalledFunction = CallSite(I).getCalledFunction(); 1011 // Do not inline recursive calls. 1012 if (CalledFunction == &F) 1013 continue; 1014 if (CallSite(I).isIndirectCall()) { 1015 if (PromotedInsns.count(I)) 1016 continue; 1017 uint64_t Sum; 1018 for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) { 1019 if (IsThinLTOPreLink) { 1020 FS->findInlinedFunctions(InlinedGUIDs, F.getParent(), 1021 PSI->getOrCompHotCountThreshold()); 1022 continue; 1023 } 1024 auto CalleeFunctionName = FS->getFuncNameInModule(F.getParent()); 1025 // If it is a recursive call, we do not inline it as it could bloat 1026 // the code exponentially. There is way to better handle this, e.g. 1027 // clone the caller first, and inline the cloned caller if it is 1028 // recursive. As llvm does not inline recursive calls, we will 1029 // simply ignore it instead of handling it explicitly. 1030 if (CalleeFunctionName == F.getName()) 1031 continue; 1032 1033 if (!callsiteIsHot(FS, PSI)) 1034 continue; 1035 1036 const char *Reason = "Callee function not available"; 1037 auto R = SymbolMap.find(CalleeFunctionName); 1038 if (R != SymbolMap.end() && R->getValue() && 1039 !R->getValue()->isDeclaration() && 1040 R->getValue()->getSubprogram() && 1041 isLegalToPromote(CallSite(I), R->getValue(), &Reason)) { 1042 uint64_t C = FS->getEntrySamples(); 1043 Instruction *DI = 1044 pgo::promoteIndirectCall(I, R->getValue(), C, Sum, false, ORE); 1045 Sum -= C; 1046 PromotedInsns.insert(I); 1047 // If profile mismatches, we should not attempt to inline DI. 1048 if ((isa<CallInst>(DI) || isa<InvokeInst>(DI)) && 1049 inlineCallInstruction(DI)) { 1050 localNotInlinedCallSites.erase(I); 1051 LocalChanged = true; 1052 ++NumCSInlined; 1053 } 1054 } else { 1055 LLVM_DEBUG(dbgs() 1056 << "\nFailed to promote indirect call to " 1057 << CalleeFunctionName << " because " << Reason << "\n"); 1058 } 1059 } 1060 } else if (CalledFunction && CalledFunction->getSubprogram() && 1061 !CalledFunction->isDeclaration()) { 1062 if (inlineCallInstruction(I)) { 1063 localNotInlinedCallSites.erase(I); 1064 LocalChanged = true; 1065 ++NumCSInlined; 1066 } 1067 } else if (IsThinLTOPreLink) { 1068 findCalleeFunctionSamples(*I)->findInlinedFunctions( 1069 InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); 1070 } 1071 } 1072 if (LocalChanged) { 1073 Changed = true; 1074 } else { 1075 break; 1076 } 1077 } 1078 1079 // Accumulate not inlined callsite information into notInlinedSamples 1080 for (const auto &Pair : localNotInlinedCallSites) { 1081 Instruction *I = Pair.getFirst(); 1082 Function *Callee = CallSite(I).getCalledFunction(); 1083 if (!Callee || Callee->isDeclaration()) 1084 continue; 1085 1086 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG, "NotInline", 1087 I->getDebugLoc(), I->getParent()) 1088 << "previous inlining not repeated: '" 1089 << ore::NV("Callee", Callee) << "' into '" 1090 << ore::NV("Caller", &F) << "'"); 1091 1092 ++NumCSNotInlined; 1093 const FunctionSamples *FS = Pair.getSecond(); 1094 if (FS->getTotalSamples() == 0 && FS->getEntrySamples() == 0) { 1095 continue; 1096 } 1097 1098 if (ProfileMergeInlinee) { 1099 // Use entry samples as head samples during the merge, as inlinees 1100 // don't have head samples. 1101 assert(FS->getHeadSamples() == 0 && "Expect 0 head sample for inlinee"); 1102 const_cast<FunctionSamples *>(FS)->addHeadSamples(FS->getEntrySamples()); 1103 1104 // Note that we have to do the merge right after processing function. 1105 // This allows OutlineFS's profile to be used for annotation during 1106 // top-down processing of functions' annotation. 1107 FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee); 1108 OutlineFS->merge(*FS); 1109 } else { 1110 auto pair = 1111 notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0}); 1112 pair.first->second.entryCount += FS->getEntrySamples(); 1113 } 1114 } 1115 return Changed; 1116} 1117 1118/// Find equivalence classes for the given block. 1119/// 1120/// This finds all the blocks that are guaranteed to execute the same 1121/// number of times as \p BB1. To do this, it traverses all the 1122/// descendants of \p BB1 in the dominator or post-dominator tree. 1123/// 1124/// A block BB2 will be in the same equivalence class as \p BB1 if 1125/// the following holds: 1126/// 1127/// 1- \p BB1 is a descendant of BB2 in the opposite tree. So, if BB2 1128/// is a descendant of \p BB1 in the dominator tree, then BB2 should 1129/// dominate BB1 in the post-dominator tree. 1130/// 1131/// 2- Both BB2 and \p BB1 must be in the same loop. 1132/// 1133/// For every block BB2 that meets those two requirements, we set BB2's 1134/// equivalence class to \p BB1. 1135/// 1136/// \param BB1 Block to check. 1137/// \param Descendants Descendants of \p BB1 in either the dom or pdom tree. 1138/// \param DomTree Opposite dominator tree. If \p Descendants is filled 1139/// with blocks from \p BB1's dominator tree, then 1140/// this is the post-dominator tree, and vice versa. 1141template <bool IsPostDom> 1142void SampleProfileLoader::findEquivalencesFor( 1143 BasicBlock *BB1, ArrayRef<BasicBlock *> Descendants, 1144 DominatorTreeBase<BasicBlock, IsPostDom> *DomTree) { 1145 const BasicBlock *EC = EquivalenceClass[BB1]; 1146 uint64_t Weight = BlockWeights[EC]; 1147 for (const auto *BB2 : Descendants) { 1148 bool IsDomParent = DomTree->dominates(BB2, BB1); 1149 bool IsInSameLoop = LI->getLoopFor(BB1) == LI->getLoopFor(BB2); 1150 if (BB1 != BB2 && IsDomParent && IsInSameLoop) { 1151 EquivalenceClass[BB2] = EC; 1152 // If BB2 is visited, then the entire EC should be marked as visited. 1153 if (VisitedBlocks.count(BB2)) { 1154 VisitedBlocks.insert(EC); 1155 } 1156 1157 // If BB2 is heavier than BB1, make BB2 have the same weight 1158 // as BB1. 1159 // 1160 // Note that we don't worry about the opposite situation here 1161 // (when BB2 is lighter than BB1). We will deal with this 1162 // during the propagation phase. Right now, we just want to 1163 // make sure that BB1 has the largest weight of all the 1164 // members of its equivalence set. 1165 Weight = std::max(Weight, BlockWeights[BB2]); 1166 } 1167 } 1168 if (EC == &EC->getParent()->getEntryBlock()) { 1169 BlockWeights[EC] = Samples->getHeadSamples() + 1; 1170 } else { 1171 BlockWeights[EC] = Weight; 1172 } 1173} 1174 1175/// Find equivalence classes. 1176/// 1177/// Since samples may be missing from blocks, we can fill in the gaps by setting 1178/// the weights of all the blocks in the same equivalence class to the same 1179/// weight. To compute the concept of equivalence, we use dominance and loop 1180/// information. Two blocks B1 and B2 are in the same equivalence class if B1 1181/// dominates B2, B2 post-dominates B1 and both are in the same loop. 1182/// 1183/// \param F The function to query. 1184void SampleProfileLoader::findEquivalenceClasses(Function &F) { 1185 SmallVector<BasicBlock *, 8> DominatedBBs; 1186 LLVM_DEBUG(dbgs() << "\nBlock equivalence classes\n"); 1187 // Find equivalence sets based on dominance and post-dominance information. 1188 for (auto &BB : F) { 1189 BasicBlock *BB1 = &BB; 1190 1191 // Compute BB1's equivalence class once. 1192 if (EquivalenceClass.count(BB1)) { 1193 LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1)); 1194 continue; 1195 } 1196 1197 // By default, blocks are in their own equivalence class. 1198 EquivalenceClass[BB1] = BB1; 1199 1200 // Traverse all the blocks dominated by BB1. We are looking for 1201 // every basic block BB2 such that: 1202 // 1203 // 1- BB1 dominates BB2. 1204 // 2- BB2 post-dominates BB1. 1205 // 3- BB1 and BB2 are in the same loop nest. 1206 // 1207 // If all those conditions hold, it means that BB2 is executed 1208 // as many times as BB1, so they are placed in the same equivalence 1209 // class by making BB2's equivalence class be BB1. 1210 DominatedBBs.clear(); 1211 DT->getDescendants(BB1, DominatedBBs); 1212 findEquivalencesFor(BB1, DominatedBBs, PDT.get()); 1213 1214 LLVM_DEBUG(printBlockEquivalence(dbgs(), BB1)); 1215 } 1216 1217 // Assign weights to equivalence classes. 1218 // 1219 // All the basic blocks in the same equivalence class will execute 1220 // the same number of times. Since we know that the head block in 1221 // each equivalence class has the largest weight, assign that weight 1222 // to all the blocks in that equivalence class. 1223 LLVM_DEBUG( 1224 dbgs() << "\nAssign the same weight to all blocks in the same class\n"); 1225 for (auto &BI : F) { 1226 const BasicBlock *BB = &BI; 1227 const BasicBlock *EquivBB = EquivalenceClass[BB]; 1228 if (BB != EquivBB) 1229 BlockWeights[BB] = BlockWeights[EquivBB]; 1230 LLVM_DEBUG(printBlockWeight(dbgs(), BB)); 1231 } 1232} 1233 1234/// Visit the given edge to decide if it has a valid weight. 1235/// 1236/// If \p E has not been visited before, we copy to \p UnknownEdge 1237/// and increment the count of unknown edges. 1238/// 1239/// \param E Edge to visit. 1240/// \param NumUnknownEdges Current number of unknown edges. 1241/// \param UnknownEdge Set if E has not been visited before. 1242/// 1243/// \returns E's weight, if known. Otherwise, return 0. 1244uint64_t SampleProfileLoader::visitEdge(Edge E, unsigned *NumUnknownEdges, 1245 Edge *UnknownEdge) { 1246 if (!VisitedEdges.count(E)) { 1247 (*NumUnknownEdges)++; 1248 *UnknownEdge = E; 1249 return 0; 1250 } 1251 1252 return EdgeWeights[E]; 1253} 1254 1255/// Propagate weights through incoming/outgoing edges. 1256/// 1257/// If the weight of a basic block is known, and there is only one edge 1258/// with an unknown weight, we can calculate the weight of that edge. 1259/// 1260/// Similarly, if all the edges have a known count, we can calculate the 1261/// count of the basic block, if needed. 1262/// 1263/// \param F Function to process. 1264/// \param UpdateBlockCount Whether we should update basic block counts that 1265/// has already been annotated. 1266/// 1267/// \returns True if new weights were assigned to edges or blocks. 1268bool SampleProfileLoader::propagateThroughEdges(Function &F, 1269 bool UpdateBlockCount) { 1270 bool Changed = false; 1271 LLVM_DEBUG(dbgs() << "\nPropagation through edges\n"); 1272 for (const auto &BI : F) { 1273 const BasicBlock *BB = &BI; 1274 const BasicBlock *EC = EquivalenceClass[BB]; 1275 1276 // Visit all the predecessor and successor edges to determine 1277 // which ones have a weight assigned already. Note that it doesn't 1278 // matter that we only keep track of a single unknown edge. The 1279 // only case we are interested in handling is when only a single 1280 // edge is unknown (see setEdgeOrBlockWeight). 1281 for (unsigned i = 0; i < 2; i++) { 1282 uint64_t TotalWeight = 0; 1283 unsigned NumUnknownEdges = 0, NumTotalEdges = 0; 1284 Edge UnknownEdge, SelfReferentialEdge, SingleEdge; 1285 1286 if (i == 0) { 1287 // First, visit all predecessor edges. 1288 NumTotalEdges = Predecessors[BB].size(); 1289 for (auto *Pred : Predecessors[BB]) { 1290 Edge E = std::make_pair(Pred, BB); 1291 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge); 1292 if (E.first == E.second) 1293 SelfReferentialEdge = E; 1294 } 1295 if (NumTotalEdges == 1) { 1296 SingleEdge = std::make_pair(Predecessors[BB][0], BB); 1297 } 1298 } else { 1299 // On the second round, visit all successor edges. 1300 NumTotalEdges = Successors[BB].size(); 1301 for (auto *Succ : Successors[BB]) { 1302 Edge E = std::make_pair(BB, Succ); 1303 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge); 1304 } 1305 if (NumTotalEdges == 1) { 1306 SingleEdge = std::make_pair(BB, Successors[BB][0]); 1307 } 1308 } 1309 1310 // After visiting all the edges, there are three cases that we 1311 // can handle immediately: 1312 // 1313 // - All the edge weights are known (i.e., NumUnknownEdges == 0). 1314 // In this case, we simply check that the sum of all the edges 1315 // is the same as BB's weight. If not, we change BB's weight 1316 // to match. Additionally, if BB had not been visited before, 1317 // we mark it visited. 1318 // 1319 // - Only one edge is unknown and BB has already been visited. 1320 // In this case, we can compute the weight of the edge by 1321 // subtracting the total block weight from all the known 1322 // edge weights. If the edges weight more than BB, then the 1323 // edge of the last remaining edge is set to zero. 1324 // 1325 // - There exists a self-referential edge and the weight of BB is 1326 // known. In this case, this edge can be based on BB's weight. 1327 // We add up all the other known edges and set the weight on 1328 // the self-referential edge as we did in the previous case. 1329 // 1330 // In any other case, we must continue iterating. Eventually, 1331 // all edges will get a weight, or iteration will stop when 1332 // it reaches SampleProfileMaxPropagateIterations. 1333 if (NumUnknownEdges <= 1) { 1334 uint64_t &BBWeight = BlockWeights[EC]; 1335 if (NumUnknownEdges == 0) { 1336 if (!VisitedBlocks.count(EC)) { 1337 // If we already know the weight of all edges, the weight of the 1338 // basic block can be computed. It should be no larger than the sum 1339 // of all edge weights. 1340 if (TotalWeight > BBWeight) { 1341 BBWeight = TotalWeight; 1342 Changed = true; 1343 LLVM_DEBUG(dbgs() << "All edge weights for " << BB->getName() 1344 << " known. Set weight for block: "; 1345 printBlockWeight(dbgs(), BB);); 1346 } 1347 } else if (NumTotalEdges == 1 && 1348 EdgeWeights[SingleEdge] < BlockWeights[EC]) { 1349 // If there is only one edge for the visited basic block, use the 1350 // block weight to adjust edge weight if edge weight is smaller. 1351 EdgeWeights[SingleEdge] = BlockWeights[EC]; 1352 Changed = true; 1353 } 1354 } else if (NumUnknownEdges == 1 && VisitedBlocks.count(EC)) { 1355 // If there is a single unknown edge and the block has been 1356 // visited, then we can compute E's weight. 1357 if (BBWeight >= TotalWeight) 1358 EdgeWeights[UnknownEdge] = BBWeight - TotalWeight; 1359 else 1360 EdgeWeights[UnknownEdge] = 0; 1361 const BasicBlock *OtherEC; 1362 if (i == 0) 1363 OtherEC = EquivalenceClass[UnknownEdge.first]; 1364 else 1365 OtherEC = EquivalenceClass[UnknownEdge.second]; 1366 // Edge weights should never exceed the BB weights it connects. 1367 if (VisitedBlocks.count(OtherEC) && 1368 EdgeWeights[UnknownEdge] > BlockWeights[OtherEC]) 1369 EdgeWeights[UnknownEdge] = BlockWeights[OtherEC]; 1370 VisitedEdges.insert(UnknownEdge); 1371 Changed = true; 1372 LLVM_DEBUG(dbgs() << "Set weight for edge: "; 1373 printEdgeWeight(dbgs(), UnknownEdge)); 1374 } 1375 } else if (VisitedBlocks.count(EC) && BlockWeights[EC] == 0) { 1376 // If a block Weights 0, all its in/out edges should weight 0. 1377 if (i == 0) { 1378 for (auto *Pred : Predecessors[BB]) { 1379 Edge E = std::make_pair(Pred, BB); 1380 EdgeWeights[E] = 0; 1381 VisitedEdges.insert(E); 1382 } 1383 } else { 1384 for (auto *Succ : Successors[BB]) { 1385 Edge E = std::make_pair(BB, Succ); 1386 EdgeWeights[E] = 0; 1387 VisitedEdges.insert(E); 1388 } 1389 } 1390 } else if (SelfReferentialEdge.first && VisitedBlocks.count(EC)) { 1391 uint64_t &BBWeight = BlockWeights[BB]; 1392 // We have a self-referential edge and the weight of BB is known. 1393 if (BBWeight >= TotalWeight) 1394 EdgeWeights[SelfReferentialEdge] = BBWeight - TotalWeight; 1395 else 1396 EdgeWeights[SelfReferentialEdge] = 0; 1397 VisitedEdges.insert(SelfReferentialEdge); 1398 Changed = true; 1399 LLVM_DEBUG(dbgs() << "Set self-referential edge weight to: "; 1400 printEdgeWeight(dbgs(), SelfReferentialEdge)); 1401 } 1402 if (UpdateBlockCount && !VisitedBlocks.count(EC) && TotalWeight > 0) { 1403 BlockWeights[EC] = TotalWeight; 1404 VisitedBlocks.insert(EC); 1405 Changed = true; 1406 } 1407 } 1408 } 1409 1410 return Changed; 1411} 1412 1413/// Build in/out edge lists for each basic block in the CFG. 1414/// 1415/// We are interested in unique edges. If a block B1 has multiple 1416/// edges to another block B2, we only add a single B1->B2 edge. 1417void SampleProfileLoader::buildEdges(Function &F) { 1418 for (auto &BI : F) { 1419 BasicBlock *B1 = &BI; 1420 1421 // Add predecessors for B1. 1422 SmallPtrSet<BasicBlock *, 16> Visited; 1423 if (!Predecessors[B1].empty()) 1424 llvm_unreachable("Found a stale predecessors list in a basic block."); 1425 for (pred_iterator PI = pred_begin(B1), PE = pred_end(B1); PI != PE; ++PI) { 1426 BasicBlock *B2 = *PI; 1427 if (Visited.insert(B2).second) 1428 Predecessors[B1].push_back(B2); 1429 } 1430 1431 // Add successors for B1. 1432 Visited.clear(); 1433 if (!Successors[B1].empty()) 1434 llvm_unreachable("Found a stale successors list in a basic block."); 1435 for (succ_iterator SI = succ_begin(B1), SE = succ_end(B1); SI != SE; ++SI) { 1436 BasicBlock *B2 = *SI; 1437 if (Visited.insert(B2).second) 1438 Successors[B1].push_back(B2); 1439 } 1440 } 1441} 1442 1443/// Returns the sorted CallTargetMap \p M by count in descending order. 1444static SmallVector<InstrProfValueData, 2> GetSortedValueDataFromCallTargets( 1445 const SampleRecord::CallTargetMap & M) { 1446 SmallVector<InstrProfValueData, 2> R; 1447 for (const auto &I : SampleRecord::SortCallTargets(M)) { 1448 R.emplace_back(InstrProfValueData{FunctionSamples::getGUID(I.first), I.second}); 1449 } 1450 return R; 1451} 1452 1453/// Propagate weights into edges 1454/// 1455/// The following rules are applied to every block BB in the CFG: 1456/// 1457/// - If BB has a single predecessor/successor, then the weight 1458/// of that edge is the weight of the block. 1459/// 1460/// - If all incoming or outgoing edges are known except one, and the 1461/// weight of the block is already known, the weight of the unknown 1462/// edge will be the weight of the block minus the sum of all the known 1463/// edges. If the sum of all the known edges is larger than BB's weight, 1464/// we set the unknown edge weight to zero. 1465/// 1466/// - If there is a self-referential edge, and the weight of the block is 1467/// known, the weight for that edge is set to the weight of the block 1468/// minus the weight of the other incoming edges to that block (if 1469/// known). 1470void SampleProfileLoader::propagateWeights(Function &F) { 1471 bool Changed = true; 1472 unsigned I = 0; 1473 1474 // If BB weight is larger than its corresponding loop's header BB weight, 1475 // use the BB weight to replace the loop header BB weight. 1476 for (auto &BI : F) { 1477 BasicBlock *BB = &BI; 1478 Loop *L = LI->getLoopFor(BB); 1479 if (!L) { 1480 continue; 1481 } 1482 BasicBlock *Header = L->getHeader(); 1483 if (Header && BlockWeights[BB] > BlockWeights[Header]) { 1484 BlockWeights[Header] = BlockWeights[BB]; 1485 } 1486 } 1487 1488 // Before propagation starts, build, for each block, a list of 1489 // unique predecessors and successors. This is necessary to handle 1490 // identical edges in multiway branches. Since we visit all blocks and all 1491 // edges of the CFG, it is cleaner to build these lists once at the start 1492 // of the pass. 1493 buildEdges(F); 1494 1495 // Propagate until we converge or we go past the iteration limit. 1496 while (Changed && I++ < SampleProfileMaxPropagateIterations) { 1497 Changed = propagateThroughEdges(F, false); 1498 } 1499 1500 // The first propagation propagates BB counts from annotated BBs to unknown 1501 // BBs. The 2nd propagation pass resets edges weights, and use all BB weights 1502 // to propagate edge weights. 1503 VisitedEdges.clear(); 1504 Changed = true; 1505 while (Changed && I++ < SampleProfileMaxPropagateIterations) { 1506 Changed = propagateThroughEdges(F, false); 1507 } 1508 1509 // The 3rd propagation pass allows adjust annotated BB weights that are 1510 // obviously wrong. 1511 Changed = true; 1512 while (Changed && I++ < SampleProfileMaxPropagateIterations) { 1513 Changed = propagateThroughEdges(F, true); 1514 } 1515 1516 // Generate MD_prof metadata for every branch instruction using the 1517 // edge weights computed during propagation. 1518 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch weights\n"); 1519 LLVMContext &Ctx = F.getContext(); 1520 MDBuilder MDB(Ctx); 1521 for (auto &BI : F) { 1522 BasicBlock *BB = &BI; 1523 1524 if (BlockWeights[BB]) { 1525 for (auto &I : BB->getInstList()) { 1526 if (!isa<CallInst>(I) && !isa<InvokeInst>(I)) 1527 continue; 1528 CallSite CS(&I); 1529 if (!CS.getCalledFunction()) { 1530 const DebugLoc &DLoc = I.getDebugLoc(); 1531 if (!DLoc) 1532 continue; 1533 const DILocation *DIL = DLoc; 1534 uint32_t LineOffset = FunctionSamples::getOffset(DIL); 1535 uint32_t Discriminator = DIL->getBaseDiscriminator(); 1536 1537 const FunctionSamples *FS = findFunctionSamples(I); 1538 if (!FS) 1539 continue; 1540 auto T = FS->findCallTargetMapAt(LineOffset, Discriminator); 1541 if (!T || T.get().empty()) 1542 continue; 1543 SmallVector<InstrProfValueData, 2> SortedCallTargets = 1544 GetSortedValueDataFromCallTargets(T.get()); 1545 uint64_t Sum; 1546 findIndirectCallFunctionSamples(I, Sum); 1547 annotateValueSite(*I.getParent()->getParent()->getParent(), I, 1548 SortedCallTargets, Sum, IPVK_IndirectCallTarget, 1549 SortedCallTargets.size()); 1550 } else if (!isa<IntrinsicInst>(&I)) { 1551 I.setMetadata(LLVMContext::MD_prof, 1552 MDB.createBranchWeights( 1553 {static_cast<uint32_t>(BlockWeights[BB])})); 1554 } 1555 } 1556 } 1557 Instruction *TI = BB->getTerminator(); 1558 if (TI->getNumSuccessors() == 1) 1559 continue; 1560 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI)) 1561 continue; 1562 1563 DebugLoc BranchLoc = TI->getDebugLoc(); 1564 LLVM_DEBUG(dbgs() << "\nGetting weights for branch at line " 1565 << ((BranchLoc) ? Twine(BranchLoc.getLine()) 1566 : Twine("<UNKNOWN LOCATION>")) 1567 << ".\n"); 1568 SmallVector<uint32_t, 4> Weights; 1569 uint32_t MaxWeight = 0; 1570 Instruction *MaxDestInst; 1571 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) { 1572 BasicBlock *Succ = TI->getSuccessor(I); 1573 Edge E = std::make_pair(BB, Succ); 1574 uint64_t Weight = EdgeWeights[E]; 1575 LLVM_DEBUG(dbgs() << "\t"; printEdgeWeight(dbgs(), E)); 1576 // Use uint32_t saturated arithmetic to adjust the incoming weights, 1577 // if needed. Sample counts in profiles are 64-bit unsigned values, 1578 // but internally branch weights are expressed as 32-bit values. 1579 if (Weight > std::numeric_limits<uint32_t>::max()) { 1580 LLVM_DEBUG(dbgs() << " (saturated due to uint32_t overflow)"); 1581 Weight = std::numeric_limits<uint32_t>::max(); 1582 } 1583 // Weight is added by one to avoid propagation errors introduced by 1584 // 0 weights. 1585 Weights.push_back(static_cast<uint32_t>(Weight + 1)); 1586 if (Weight != 0) { 1587 if (Weight > MaxWeight) { 1588 MaxWeight = Weight; 1589 MaxDestInst = Succ->getFirstNonPHIOrDbgOrLifetime(); 1590 } 1591 } 1592 } 1593 1594 misexpect::verifyMisExpect(TI, Weights, TI->getContext()); 1595 1596 uint64_t TempWeight; 1597 // Only set weights if there is at least one non-zero weight. 1598 // In any other case, let the analyzer set weights. 1599 // Do not set weights if the weights are present. In ThinLTO, the profile 1600 // annotation is done twice. If the first annotation already set the 1601 // weights, the second pass does not need to set it. 1602 if (MaxWeight > 0 && !TI->extractProfTotalWeight(TempWeight)) { 1603 LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n"); 1604 TI->setMetadata(LLVMContext::MD_prof, 1605 MDB.createBranchWeights(Weights)); 1606 ORE->emit([&]() { 1607 return OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst) 1608 << "most popular destination for conditional branches at " 1609 << ore::NV("CondBranchesLoc", BranchLoc); 1610 }); 1611 } else { 1612 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n"); 1613 } 1614 } 1615} 1616 1617/// Get the line number for the function header. 1618/// 1619/// This looks up function \p F in the current compilation unit and 1620/// retrieves the line number where the function is defined. This is 1621/// line 0 for all the samples read from the profile file. Every line 1622/// number is relative to this line. 1623/// 1624/// \param F Function object to query. 1625/// 1626/// \returns the line number where \p F is defined. If it returns 0, 1627/// it means that there is no debug information available for \p F. 1628unsigned SampleProfileLoader::getFunctionLoc(Function &F) { 1629 if (DISubprogram *S = F.getSubprogram()) 1630 return S->getLine(); 1631 1632 if (NoWarnSampleUnused) 1633 return 0; 1634 1635 // If the start of \p F is missing, emit a diagnostic to inform the user 1636 // about the missed opportunity. 1637 F.getContext().diagnose(DiagnosticInfoSampleProfile( 1638 "No debug information found in function " + F.getName() + 1639 ": Function profile not used", 1640 DS_Warning)); 1641 return 0; 1642} 1643 1644void SampleProfileLoader::computeDominanceAndLoopInfo(Function &F) { 1645 DT.reset(new DominatorTree); 1646 DT->recalculate(F); 1647 1648 PDT.reset(new PostDominatorTree(F)); 1649 1650 LI.reset(new LoopInfo); 1651 LI->analyze(*DT); 1652} 1653 1654/// Generate branch weight metadata for all branches in \p F. 1655/// 1656/// Branch weights are computed out of instruction samples using a 1657/// propagation heuristic. Propagation proceeds in 3 phases: 1658/// 1659/// 1- Assignment of block weights. All the basic blocks in the function 1660/// are initial assigned the same weight as their most frequently 1661/// executed instruction. 1662/// 1663/// 2- Creation of equivalence classes. Since samples may be missing from 1664/// blocks, we can fill in the gaps by setting the weights of all the 1665/// blocks in the same equivalence class to the same weight. To compute 1666/// the concept of equivalence, we use dominance and loop information. 1667/// Two blocks B1 and B2 are in the same equivalence class if B1 1668/// dominates B2, B2 post-dominates B1 and both are in the same loop. 1669/// 1670/// 3- Propagation of block weights into edges. This uses a simple 1671/// propagation heuristic. The following rules are applied to every 1672/// block BB in the CFG: 1673/// 1674/// - If BB has a single predecessor/successor, then the weight 1675/// of that edge is the weight of the block. 1676/// 1677/// - If all the edges are known except one, and the weight of the 1678/// block is already known, the weight of the unknown edge will 1679/// be the weight of the block minus the sum of all the known 1680/// edges. If the sum of all the known edges is larger than BB's weight, 1681/// we set the unknown edge weight to zero. 1682/// 1683/// - If there is a self-referential edge, and the weight of the block is 1684/// known, the weight for that edge is set to the weight of the block 1685/// minus the weight of the other incoming edges to that block (if 1686/// known). 1687/// 1688/// Since this propagation is not guaranteed to finalize for every CFG, we 1689/// only allow it to proceed for a limited number of iterations (controlled 1690/// by -sample-profile-max-propagate-iterations). 1691/// 1692/// FIXME: Try to replace this propagation heuristic with a scheme 1693/// that is guaranteed to finalize. A work-list approach similar to 1694/// the standard value propagation algorithm used by SSA-CCP might 1695/// work here. 1696/// 1697/// Once all the branch weights are computed, we emit the MD_prof 1698/// metadata on BB using the computed values for each of its branches. 1699/// 1700/// \param F The function to query. 1701/// 1702/// \returns true if \p F was modified. Returns false, otherwise. 1703bool SampleProfileLoader::emitAnnotations(Function &F) { 1704 bool Changed = false; 1705 1706 if (getFunctionLoc(F) == 0) 1707 return false; 1708 1709 LLVM_DEBUG(dbgs() << "Line number for the first instruction in " 1710 << F.getName() << ": " << getFunctionLoc(F) << "\n"); 1711 1712 DenseSet<GlobalValue::GUID> InlinedGUIDs; 1713 Changed |= inlineHotFunctions(F, InlinedGUIDs); 1714 1715 // Compute basic block weights. 1716 Changed |= computeBlockWeights(F); 1717 1718 if (Changed) { 1719 // Add an entry count to the function using the samples gathered at the 1720 // function entry. 1721 // Sets the GUIDs that are inlined in the profiled binary. This is used 1722 // for ThinLink to make correct liveness analysis, and also make the IR 1723 // match the profiled binary before annotation. 1724 F.setEntryCount( 1725 ProfileCount(Samples->getHeadSamples() + 1, Function::PCT_Real), 1726 &InlinedGUIDs); 1727 1728 // Compute dominance and loop info needed for propagation. 1729 computeDominanceAndLoopInfo(F); 1730 1731 // Find equivalence classes. 1732 findEquivalenceClasses(F); 1733 1734 // Propagate weights to all edges. 1735 propagateWeights(F); 1736 } 1737 1738 // If coverage checking was requested, compute it now. 1739 if (SampleProfileRecordCoverage) { 1740 unsigned Used = CoverageTracker.countUsedRecords(Samples, PSI); 1741 unsigned Total = CoverageTracker.countBodyRecords(Samples, PSI); 1742 unsigned Coverage = CoverageTracker.computeCoverage(Used, Total); 1743 if (Coverage < SampleProfileRecordCoverage) { 1744 F.getContext().diagnose(DiagnosticInfoSampleProfile( 1745 F.getSubprogram()->getFilename(), getFunctionLoc(F), 1746 Twine(Used) + " of " + Twine(Total) + " available profile records (" + 1747 Twine(Coverage) + "%) were applied", 1748 DS_Warning)); 1749 } 1750 } 1751 1752 if (SampleProfileSampleCoverage) { 1753 uint64_t Used = CoverageTracker.getTotalUsedSamples(); 1754 uint64_t Total = CoverageTracker.countBodySamples(Samples, PSI); 1755 unsigned Coverage = CoverageTracker.computeCoverage(Used, Total); 1756 if (Coverage < SampleProfileSampleCoverage) { 1757 F.getContext().diagnose(DiagnosticInfoSampleProfile( 1758 F.getSubprogram()->getFilename(), getFunctionLoc(F), 1759 Twine(Used) + " of " + Twine(Total) + " available profile samples (" + 1760 Twine(Coverage) + "%) were applied", 1761 DS_Warning)); 1762 } 1763 } 1764 return Changed; 1765} 1766 1767char SampleProfileLoaderLegacyPass::ID = 0; 1768 1769INITIALIZE_PASS_BEGIN(SampleProfileLoaderLegacyPass, "sample-profile", 1770 "Sample Profile loader", false, false) 1771INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 1772INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 1773INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 1774INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass, "sample-profile", 1775 "Sample Profile loader", false, false) 1776 1777std::vector<Function *> 1778SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) { 1779 std::vector<Function *> FunctionOrderList; 1780 FunctionOrderList.reserve(M.size()); 1781 1782 if (!ProfileTopDownLoad || CG == nullptr) { 1783 for (Function &F : M) 1784 if (!F.isDeclaration()) 1785 FunctionOrderList.push_back(&F); 1786 return FunctionOrderList; 1787 } 1788 1789 assert(&CG->getModule() == &M); 1790 scc_iterator<CallGraph *> CGI = scc_begin(CG); 1791 while (!CGI.isAtEnd()) { 1792 for (CallGraphNode *node : *CGI) { 1793 auto F = node->getFunction(); 1794 if (F && !F->isDeclaration()) 1795 FunctionOrderList.push_back(F); 1796 } 1797 ++CGI; 1798 } 1799 1800 std::reverse(FunctionOrderList.begin(), FunctionOrderList.end()); 1801 return FunctionOrderList; 1802} 1803 1804bool SampleProfileLoader::doInitialization(Module &M) { 1805 auto &Ctx = M.getContext(); 1806 1807 std::unique_ptr<SampleProfileReaderItaniumRemapper> RemapReader; 1808 auto ReaderOrErr = 1809 SampleProfileReader::create(Filename, Ctx, RemappingFilename); 1810 if (std::error_code EC = ReaderOrErr.getError()) { 1811 std::string Msg = "Could not open profile: " + EC.message(); 1812 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); 1813 return false; 1814 } 1815 Reader = std::move(ReaderOrErr.get()); 1816 Reader->collectFuncsFrom(M); 1817 ProfileIsValid = (Reader->read() == sampleprof_error::success); 1818 PSL = Reader->getProfileSymbolList(); 1819 1820 // While profile-sample-accurate is on, ignore symbol list. 1821 ProfAccForSymsInList = 1822 ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate; 1823 if (ProfAccForSymsInList) { 1824 NamesInProfile.clear(); 1825 if (auto NameTable = Reader->getNameTable()) 1826 NamesInProfile.insert(NameTable->begin(), NameTable->end()); 1827 } 1828 1829 return true; 1830} 1831 1832ModulePass *llvm::createSampleProfileLoaderPass() { 1833 return new SampleProfileLoaderLegacyPass(); 1834} 1835 1836ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) { 1837 return new SampleProfileLoaderLegacyPass(Name); 1838} 1839 1840bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, 1841 ProfileSummaryInfo *_PSI, CallGraph *CG) { 1842 GUIDToFuncNameMapper Mapper(M, *Reader, GUIDToFuncNameMap); 1843 if (!ProfileIsValid) 1844 return false; 1845 1846 PSI = _PSI; 1847 if (M.getProfileSummary(/* IsCS */ false) == nullptr) 1848 M.setProfileSummary(Reader->getSummary().getMD(M.getContext()), 1849 ProfileSummary::PSK_Sample); 1850 1851 // Compute the total number of samples collected in this profile. 1852 for (const auto &I : Reader->getProfiles()) 1853 TotalCollectedSamples += I.second.getTotalSamples(); 1854 1855 // Populate the symbol map. 1856 for (const auto &N_F : M.getValueSymbolTable()) { 1857 StringRef OrigName = N_F.getKey(); 1858 Function *F = dyn_cast<Function>(N_F.getValue()); 1859 if (F == nullptr) 1860 continue; 1861 SymbolMap[OrigName] = F; 1862 auto pos = OrigName.find('.'); 1863 if (pos != StringRef::npos) { 1864 StringRef NewName = OrigName.substr(0, pos); 1865 auto r = SymbolMap.insert(std::make_pair(NewName, F)); 1866 // Failiing to insert means there is already an entry in SymbolMap, 1867 // thus there are multiple functions that are mapped to the same 1868 // stripped name. In this case of name conflicting, set the value 1869 // to nullptr to avoid confusion. 1870 if (!r.second) 1871 r.first->second = nullptr; 1872 } 1873 } 1874 1875 bool retval = false; 1876 for (auto F : buildFunctionOrder(M, CG)) { 1877 assert(!F->isDeclaration()); 1878 clearFunctionData(); 1879 retval |= runOnFunction(*F, AM); 1880 } 1881 1882 // Account for cold calls not inlined.... 1883 for (const std::pair<Function *, NotInlinedProfileInfo> &pair : 1884 notInlinedCallInfo) 1885 updateProfileCallee(pair.first, pair.second.entryCount); 1886 1887 return retval; 1888} 1889 1890bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) { 1891 ACT = &getAnalysis<AssumptionCacheTracker>(); 1892 TTIWP = &getAnalysis<TargetTransformInfoWrapperPass>(); 1893 ProfileSummaryInfo *PSI = 1894 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 1895 return SampleLoader.runOnModule(M, nullptr, PSI, nullptr); 1896} 1897 1898bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) { 1899 1900 DILocation2SampleMap.clear(); 1901 // By default the entry count is initialized to -1, which will be treated 1902 // conservatively by getEntryCount as the same as unknown (None). This is 1903 // to avoid newly added code to be treated as cold. If we have samples 1904 // this will be overwritten in emitAnnotations. 1905 uint64_t initialEntryCount = -1; 1906 1907 ProfAccForSymsInList = ProfileAccurateForSymsInList && PSL; 1908 if (ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate")) { 1909 // initialize all the function entry counts to 0. It means all the 1910 // functions without profile will be regarded as cold. 1911 initialEntryCount = 0; 1912 // profile-sample-accurate is a user assertion which has a higher precedence 1913 // than symbol list. When profile-sample-accurate is on, ignore symbol list. 1914 ProfAccForSymsInList = false; 1915 } 1916 1917 // PSL -- profile symbol list include all the symbols in sampled binary. 1918 // If ProfileAccurateForSymsInList is enabled, PSL is used to treat 1919 // old functions without samples being cold, without having to worry 1920 // about new and hot functions being mistakenly treated as cold. 1921 if (ProfAccForSymsInList) { 1922 // Initialize the entry count to 0 for functions in the list. 1923 if (PSL->contains(F.getName())) 1924 initialEntryCount = 0; 1925 1926 // Function in the symbol list but without sample will be regarded as 1927 // cold. To minimize the potential negative performance impact it could 1928 // have, we want to be a little conservative here saying if a function 1929 // shows up in the profile, no matter as outline function, inline instance 1930 // or call targets, treat the function as not being cold. This will handle 1931 // the cases such as most callsites of a function are inlined in sampled 1932 // binary but not inlined in current build (because of source code drift, 1933 // imprecise debug information, or the callsites are all cold individually 1934 // but not cold accumulatively...), so the outline function showing up as 1935 // cold in sampled binary will actually not be cold after current build. 1936 StringRef CanonName = FunctionSamples::getCanonicalFnName(F); 1937 if (NamesInProfile.count(CanonName)) 1938 initialEntryCount = -1; 1939 } 1940 1941 F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real)); 1942 std::unique_ptr<OptimizationRemarkEmitter> OwnedORE; 1943 if (AM) { 1944 auto &FAM = 1945 AM->getResult<FunctionAnalysisManagerModuleProxy>(*F.getParent()) 1946 .getManager(); 1947 ORE = &FAM.getResult<OptimizationRemarkEmitterAnalysis>(F); 1948 } else { 1949 OwnedORE = std::make_unique<OptimizationRemarkEmitter>(&F); 1950 ORE = OwnedORE.get(); 1951 } 1952 Samples = Reader->getSamplesFor(F); 1953 if (Samples && !Samples->empty()) 1954 return emitAnnotations(F); 1955 return false; 1956} 1957 1958PreservedAnalyses SampleProfileLoaderPass::run(Module &M, 1959 ModuleAnalysisManager &AM) { 1960 FunctionAnalysisManager &FAM = 1961 AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 1962 1963 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 1964 return FAM.getResult<AssumptionAnalysis>(F); 1965 }; 1966 auto GetTTI = [&](Function &F) -> TargetTransformInfo & { 1967 return FAM.getResult<TargetIRAnalysis>(F); 1968 }; 1969 1970 SampleProfileLoader SampleLoader( 1971 ProfileFileName.empty() ? SampleProfileFile : ProfileFileName, 1972 ProfileRemappingFileName.empty() ? SampleProfileRemappingFile 1973 : ProfileRemappingFileName, 1974 IsThinLTOPreLink, GetAssumptionCache, GetTTI); 1975 1976 if (!SampleLoader.doInitialization(M)) 1977 return PreservedAnalyses::all(); 1978 1979 ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M); 1980 CallGraph &CG = AM.getResult<CallGraphAnalysis>(M); 1981 if (!SampleLoader.runOnModule(M, &AM, PSI, &CG)) 1982 return PreservedAnalyses::all(); 1983 1984 return PreservedAnalyses::none(); 1985} 1986