1241470Sgrehan//===- ChainedIncludesSource.cpp - Chained PCHs in Memory -------*- C++ -*-===// 2252707Sbryanv// 3241470Sgrehan// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4241470Sgrehan// See https://llvm.org/LICENSE.txt for license information. 5241470Sgrehan// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6241470Sgrehan// 7241470Sgrehan//===----------------------------------------------------------------------===// 8241470Sgrehan// 9241470Sgrehan// This file defines the ChainedIncludesSource class, which converts headers 10241470Sgrehan// to chained PCHs in memory, mainly used for testing. 11241470Sgrehan// 12241470Sgrehan//===----------------------------------------------------------------------===// 13241470Sgrehan 14241470Sgrehan#include "clang/Basic/Builtins.h" 15241470Sgrehan#include "clang/Basic/TargetInfo.h" 16241470Sgrehan#include "clang/Frontend/ASTUnit.h" 17241470Sgrehan#include "clang/Frontend/CompilerInstance.h" 18241470Sgrehan#include "clang/Frontend/TextDiagnosticPrinter.h" 19241470Sgrehan#include "clang/Lex/Preprocessor.h" 20241470Sgrehan#include "clang/Lex/PreprocessorOptions.h" 21241470Sgrehan#include "clang/Parse/ParseAST.h" 22241470Sgrehan#include "clang/Sema/MultiplexExternalSemaSource.h" 23241470Sgrehan#include "clang/Serialization/ASTReader.h" 24241470Sgrehan#include "clang/Serialization/ASTWriter.h" 25241470Sgrehan#include "llvm/Support/MemoryBuffer.h" 26241470Sgrehan 27241470Sgrehanusing namespace clang; 28241470Sgrehan 29241470Sgrehannamespace { 30241470Sgrehanclass ChainedIncludesSourceImpl : public ExternalSemaSource { 31241470Sgrehanpublic: 32241470Sgrehan ChainedIncludesSourceImpl(std::vector<std::unique_ptr<CompilerInstance>> CIs) 33241470Sgrehan : CIs(std::move(CIs)) {} 34241470Sgrehan 35241470Sgrehanprotected: 36241470Sgrehan //===----------------------------------------------------------------------===// 37241470Sgrehan // ExternalASTSource interface. 38241470Sgrehan //===----------------------------------------------------------------------===// 39241470Sgrehan 40241470Sgrehan /// Return the amount of memory used by memory buffers, breaking down 41241470Sgrehan /// by heap-backed versus mmap'ed memory. 42241470Sgrehan void getMemoryBufferSizes(MemoryBufferSizes &sizes) const override { 43241470Sgrehan for (unsigned i = 0, e = CIs.size(); i != e; ++i) { 44241470Sgrehan if (const ExternalASTSource *eSrc = 45241470Sgrehan CIs[i]->getASTContext().getExternalSource()) { 46241470Sgrehan eSrc->getMemoryBufferSizes(sizes); 47241470Sgrehan } 48241470Sgrehan } 49241470Sgrehan } 50241470Sgrehan 51241470Sgrehanprivate: 52241470Sgrehan std::vector<std::unique_ptr<CompilerInstance>> CIs; 53241470Sgrehan}; 54241470Sgrehan 55241470Sgrehan/// Members of ChainedIncludesSource, factored out so we can initialize 56241470Sgrehan/// them before we initialize the ExternalSemaSource base class. 57241470Sgrehanstruct ChainedIncludesSourceMembers { 58241470Sgrehan ChainedIncludesSourceMembers( 59241470Sgrehan std::vector<std::unique_ptr<CompilerInstance>> CIs, 60241470Sgrehan IntrusiveRefCntPtr<ExternalSemaSource> FinalReader) 61241470Sgrehan : Impl(std::move(CIs)), FinalReader(std::move(FinalReader)) {} 62241470Sgrehan ChainedIncludesSourceImpl Impl; 63241470Sgrehan IntrusiveRefCntPtr<ExternalSemaSource> FinalReader; 64241470Sgrehan}; 65241470Sgrehan 66241470Sgrehan/// Use MultiplexExternalSemaSource to dispatch all ExternalSemaSource 67241470Sgrehan/// calls to the final reader. 68241470Sgrehanclass ChainedIncludesSource 69241470Sgrehan : private ChainedIncludesSourceMembers, 70241470Sgrehan public MultiplexExternalSemaSource { 71241470Sgrehanpublic: 72241470Sgrehan ChainedIncludesSource(std::vector<std::unique_ptr<CompilerInstance>> CIs, 73241470Sgrehan IntrusiveRefCntPtr<ExternalSemaSource> FinalReader) 74241470Sgrehan : ChainedIncludesSourceMembers(std::move(CIs), std::move(FinalReader)), 75241470Sgrehan MultiplexExternalSemaSource(Impl, *this->FinalReader) {} 76241470Sgrehan}; 77241470Sgrehan} 78241470Sgrehan 79241470Sgrehanstatic ASTReader * 80241470SgrehancreateASTReader(CompilerInstance &CI, StringRef pchFile, 81241470Sgrehan SmallVectorImpl<std::unique_ptr<llvm::MemoryBuffer>> &MemBufs, 82241470Sgrehan SmallVectorImpl<std::string> &bufNames, 83241470Sgrehan ASTDeserializationListener *deserialListener = nullptr) { 84241470Sgrehan Preprocessor &PP = CI.getPreprocessor(); 85241470Sgrehan std::unique_ptr<ASTReader> Reader; 86241470Sgrehan Reader.reset(new ASTReader( 87241470Sgrehan PP, CI.getModuleCache(), &CI.getASTContext(), CI.getPCHContainerReader(), 88241470Sgrehan /*Extensions=*/{}, 89241470Sgrehan /*isysroot=*/"", DisableValidationForModuleKind::PCH)); 90241470Sgrehan for (unsigned ti = 0; ti < bufNames.size(); ++ti) { 91241470Sgrehan StringRef sr(bufNames[ti]); 92241470Sgrehan Reader->addInMemoryBuffer(sr, std::move(MemBufs[ti])); 93241470Sgrehan } 94241470Sgrehan Reader->setDeserializationListener(deserialListener); 95241470Sgrehan switch (Reader->ReadAST(pchFile, serialization::MK_PCH, SourceLocation(), 96241470Sgrehan ASTReader::ARR_None)) { 97241470Sgrehan case ASTReader::Success: 98241470Sgrehan // Set the predefines buffer as suggested by the PCH reader. 99241470Sgrehan PP.setPredefines(Reader->getSuggestedPredefines()); 100241470Sgrehan return Reader.release(); 101241470Sgrehan 102241470Sgrehan case ASTReader::Failure: 103241470Sgrehan case ASTReader::Missing: 104241470Sgrehan case ASTReader::OutOfDate: 105241470Sgrehan case ASTReader::VersionMismatch: 106241470Sgrehan case ASTReader::ConfigurationMismatch: 107241470Sgrehan case ASTReader::HadErrors: 108241470Sgrehan break; 109241470Sgrehan } 110241470Sgrehan return nullptr; 111241470Sgrehan} 112241470Sgrehan 113241470SgrehanIntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource( 114241470Sgrehan CompilerInstance &CI, IntrusiveRefCntPtr<ExternalSemaSource> &Reader) { 115241470Sgrehan 116241470Sgrehan std::vector<std::string> &includes = CI.getPreprocessorOpts().ChainedIncludes; 117241470Sgrehan assert(!includes.empty() && "No '-chain-include' in options!"); 118241470Sgrehan 119241470Sgrehan std::vector<std::unique_ptr<CompilerInstance>> CIs; 120241470Sgrehan InputKind IK = CI.getFrontendOpts().Inputs[0].getKind(); 121241470Sgrehan 122241470Sgrehan SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 4> SerialBufs; 123241470Sgrehan SmallVector<std::string, 4> serialBufNames; 124241470Sgrehan 125241470Sgrehan for (unsigned i = 0, e = includes.size(); i != e; ++i) { 126241470Sgrehan bool firstInclude = (i == 0); 127241470Sgrehan std::unique_ptr<CompilerInvocation> CInvok; 128241470Sgrehan CInvok.reset(new CompilerInvocation(CI.getInvocation())); 129241470Sgrehan 130241470Sgrehan CInvok->getPreprocessorOpts().ChainedIncludes.clear(); 131241470Sgrehan CInvok->getPreprocessorOpts().ImplicitPCHInclude.clear(); 132241470Sgrehan CInvok->getPreprocessorOpts().DisablePCHOrModuleValidation = 133241470Sgrehan DisableValidationForModuleKind::PCH; 134241470Sgrehan CInvok->getPreprocessorOpts().Includes.clear(); 135241470Sgrehan CInvok->getPreprocessorOpts().MacroIncludes.clear(); 136241470Sgrehan CInvok->getPreprocessorOpts().Macros.clear(); 137241470Sgrehan 138241470Sgrehan CInvok->getFrontendOpts().Inputs.clear(); 139241470Sgrehan FrontendInputFile InputFile(includes[i], IK); 140241470Sgrehan CInvok->getFrontendOpts().Inputs.push_back(InputFile); 141241470Sgrehan 142241470Sgrehan TextDiagnosticPrinter *DiagClient = 143241470Sgrehan new TextDiagnosticPrinter(llvm::errs(), new DiagnosticOptions()); 144241470Sgrehan IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs()); 145241470Sgrehan IntrusiveRefCntPtr<DiagnosticsEngine> Diags( 146241470Sgrehan new DiagnosticsEngine(DiagID, &CI.getDiagnosticOpts(), DiagClient)); 147241470Sgrehan 148241470Sgrehan std::unique_ptr<CompilerInstance> Clang( 149241470Sgrehan new CompilerInstance(CI.getPCHContainerOperations())); 150241470Sgrehan Clang->setInvocation(std::move(CInvok)); 151241470Sgrehan Clang->setDiagnostics(Diags.get()); 152241470Sgrehan Clang->setTarget(TargetInfo::CreateTargetInfo( 153241470Sgrehan Clang->getDiagnostics(), Clang->getInvocation().TargetOpts)); 154241470Sgrehan Clang->createFileManager(); 155241470Sgrehan Clang->createSourceManager(Clang->getFileManager()); 156241470Sgrehan Clang->createPreprocessor(TU_Prefix); 157241470Sgrehan Clang->getDiagnosticClient().BeginSourceFile(Clang->getLangOpts(), 158241470Sgrehan &Clang->getPreprocessor()); 159241470Sgrehan Clang->createASTContext(); 160241470Sgrehan 161241470Sgrehan auto Buffer = std::make_shared<PCHBuffer>(); 162241470Sgrehan ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions; 163241470Sgrehan auto consumer = std::make_unique<PCHGenerator>( 164241470Sgrehan Clang->getPreprocessor(), Clang->getModuleCache(), "-", /*isysroot=*/"", 165241470Sgrehan Buffer, Extensions, /*AllowASTWithErrors=*/true); 166241470Sgrehan Clang->getASTContext().setASTMutationListener( 167241470Sgrehan consumer->GetASTMutationListener()); 168241470Sgrehan Clang->setASTConsumer(std::move(consumer)); 169241470Sgrehan Clang->createSema(TU_Prefix, nullptr); 170241470Sgrehan 171241470Sgrehan if (firstInclude) { 172241470Sgrehan Preprocessor &PP = Clang->getPreprocessor(); 173241470Sgrehan PP.getBuiltinInfo().initializeBuiltins(PP.getIdentifierTable(), 174241470Sgrehan PP.getLangOpts()); 175241470Sgrehan } else { 176241470Sgrehan assert(!SerialBufs.empty()); 177241470Sgrehan SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 4> Bufs; 178241470Sgrehan // TODO: Pass through the existing MemoryBuffer instances instead of 179241470Sgrehan // allocating new ones. 180241470Sgrehan for (auto &SB : SerialBufs) 181241470Sgrehan Bufs.push_back(llvm::MemoryBuffer::getMemBuffer(SB->getBuffer())); 182241470Sgrehan std::string pchName = includes[i-1]; 183241470Sgrehan llvm::raw_string_ostream os(pchName); 184241470Sgrehan os << ".pch" << i-1; 185241470Sgrehan serialBufNames.push_back(os.str()); 186241470Sgrehan 187241470Sgrehan IntrusiveRefCntPtr<ASTReader> Reader; 188241470Sgrehan Reader = createASTReader( 189241470Sgrehan *Clang, pchName, Bufs, serialBufNames, 190241470Sgrehan Clang->getASTConsumer().GetASTDeserializationListener()); 191241470Sgrehan if (!Reader) 192241470Sgrehan return nullptr; 193241470Sgrehan Clang->setASTReader(Reader); 194241470Sgrehan Clang->getASTContext().setExternalSource(Reader); 195241470Sgrehan } 196241470Sgrehan 197241470Sgrehan if (!Clang->InitializeSourceManager(InputFile)) 198241470Sgrehan return nullptr; 199241470Sgrehan 200241470Sgrehan ParseAST(Clang->getSema()); 201241470Sgrehan Clang->getDiagnosticClient().EndSourceFile(); 202241470Sgrehan assert(Buffer->IsComplete && "serialization did not complete"); 203241470Sgrehan auto &serialAST = Buffer->Data; 204241470Sgrehan SerialBufs.push_back(llvm::MemoryBuffer::getMemBufferCopy( 205241470Sgrehan StringRef(serialAST.data(), serialAST.size()))); 206241470Sgrehan serialAST.clear(); 207241470Sgrehan CIs.push_back(std::move(Clang)); 208241470Sgrehan } 209241470Sgrehan 210241470Sgrehan assert(!SerialBufs.empty()); 211241470Sgrehan std::string pchName = includes.back() + ".pch-final"; 212241470Sgrehan serialBufNames.push_back(pchName); 213241470Sgrehan Reader = createASTReader(CI, pchName, SerialBufs, serialBufNames); 214241470Sgrehan if (!Reader) 215241470Sgrehan return nullptr; 216241470Sgrehan 217 return IntrusiveRefCntPtr<ChainedIncludesSource>( 218 new ChainedIncludesSource(std::move(CIs), Reader)); 219} 220