Preprocessor.cpp revision 263508 
1//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10//  This file implements the Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13//
14// Options to support:
15//   -H       - Print the name of each header file used.
16//   -d[DNI] - Dump various things.
17//   -fworking-directory - #line's with preprocessor's working dir.
18//   -fpreprocessed
19//   -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
20//   -W*
21//   -w
22//
23// Messages to emit:
24//   "Multiple include guards may be useful for:\n"
25//
26//===----------------------------------------------------------------------===//
27
28#include "clang/Lex/Preprocessor.h"
29#include "clang/Lex/MacroArgs.h"
30#include "clang/Basic/FileManager.h"
31#include "clang/Basic/SourceManager.h"
32#include "clang/Basic/TargetInfo.h"
33#include "clang/Lex/CodeCompletionHandler.h"
34#include "clang/Lex/ExternalPreprocessorSource.h"
35#include "clang/Lex/HeaderSearch.h"
36#include "clang/Lex/LexDiagnostic.h"
37#include "clang/Lex/LiteralSupport.h"
38#include "clang/Lex/MacroInfo.h"
39#include "clang/Lex/ModuleLoader.h"
40#include "clang/Lex/Pragma.h"
41#include "clang/Lex/PreprocessingRecord.h"
42#include "clang/Lex/PreprocessorOptions.h"
43#include "clang/Lex/ScratchBuffer.h"
44#include "llvm/ADT/APFloat.h"
45#include "llvm/ADT/SmallString.h"
46#include "llvm/ADT/STLExtras.h"
47#include "llvm/ADT/StringExtras.h"
48#include "llvm/Support/Capacity.h"
49#include "llvm/Support/ConvertUTF.h"
50#include "llvm/Support/MemoryBuffer.h"
51#include "llvm/Support/raw_ostream.h"
52using namespace clang;
53
54//===----------------------------------------------------------------------===//
55ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
56
57Preprocessor::Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
58                           DiagnosticsEngine &diags, LangOptions &opts,
59                           const TargetInfo *target, SourceManager &SM,
60                           HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
61                           IdentifierInfoLookup *IILookup, bool OwnsHeaders,
62                           bool DelayInitialization, bool IncrProcessing)
63    : PPOpts(PPOpts), Diags(&diags), LangOpts(opts), Target(target),
64      FileMgr(Headers.getFileMgr()), SourceMgr(SM), HeaderInfo(Headers),
65      TheModuleLoader(TheModuleLoader), ExternalSource(0),
66      Identifiers(opts, IILookup), IncrementalProcessing(IncrProcessing),
67      CodeComplete(0), CodeCompletionFile(0), CodeCompletionOffset(0),
68      LastTokenWasAt(false), ModuleImportExpectsIdentifier(false),
69      CodeCompletionReached(0), SkipMainFilePreamble(0, true), CurPPLexer(0),
70      CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0),
71      MacroArgCache(0), Record(0), MIChainHead(0), MICache(0),
72      DeserialMIChainHead(0) {
73  OwnsHeaderSearch = OwnsHeaders;
74
75  ScratchBuf = new ScratchBuffer(SourceMgr);
76  CounterValue = 0; // __COUNTER__ starts at 0.
77
78  // Clear stats.
79  NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
80  NumIf = NumElse = NumEndif = 0;
81  NumEnteredSourceFiles = 0;
82  NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
83  NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
84  MaxIncludeStackDepth = 0;
85  NumSkipped = 0;
86
87  // Default to discarding comments.
88  KeepComments = false;
89  KeepMacroComments = false;
90  SuppressIncludeNotFoundError = false;
91
92  // Macro expansion is enabled.
93  DisableMacroExpansion = false;
94  MacroExpansionInDirectivesOverride = false;
95  InMacroArgs = false;
96  InMacroArgPreExpansion = false;
97  NumCachedTokenLexers = 0;
98  PragmasEnabled = true;
99  ParsingIfOrElifDirective = false;
100  PreprocessedOutput = false;
101
102  CachedLexPos = 0;
103
104  // We haven't read anything from the external source.
105  ReadMacrosFromExternalSource = false;
106
107  // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
108  // This gets unpoisoned where it is allowed.
109  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
110  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
111
112  // Initialize the pragma handlers.
113  PragmaHandlers = new PragmaNamespace(StringRef());
114  RegisterBuiltinPragmas();
115
116  // Initialize builtin macros like __LINE__ and friends.
117  RegisterBuiltinMacros();
118
119  if(LangOpts.Borland) {
120    Ident__exception_info        = getIdentifierInfo("_exception_info");
121    Ident___exception_info       = getIdentifierInfo("__exception_info");
122    Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
123    Ident__exception_code        = getIdentifierInfo("_exception_code");
124    Ident___exception_code       = getIdentifierInfo("__exception_code");
125    Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
126    Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
127    Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
128    Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
129  } else {
130    Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0;
131    Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0;
132    Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0;
133  }
134
135  if (!DelayInitialization) {
136    assert(Target && "Must provide target information for PP initialization");
137    Initialize(*Target);
138  }
139}
140
141Preprocessor::~Preprocessor() {
142  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
143
144  while (!IncludeMacroStack.empty()) {
145    delete IncludeMacroStack.back().TheLexer;
146    delete IncludeMacroStack.back().TheTokenLexer;
147    IncludeMacroStack.pop_back();
148  }
149
150  // Free any macro definitions.
151  for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next)
152    I->MI.Destroy();
153
154  // Free any cached macro expanders.
155  for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
156    delete TokenLexerCache[i];
157
158  for (DeserializedMacroInfoChain *I = DeserialMIChainHead ; I ; I = I->Next)
159    I->MI.Destroy();
160
161  // Free any cached MacroArgs.
162  for (MacroArgs *ArgList = MacroArgCache; ArgList; )
163    ArgList = ArgList->deallocate();
164
165  // Release pragma information.
166  delete PragmaHandlers;
167
168  // Delete the scratch buffer info.
169  delete ScratchBuf;
170
171  // Delete the header search info, if we own it.
172  if (OwnsHeaderSearch)
173    delete &HeaderInfo;
174
175  delete Callbacks;
176}
177
178void Preprocessor::Initialize(const TargetInfo &Target) {
179  assert((!this->Target || this->Target == &Target) &&
180         "Invalid override of target information");
181  this->Target = &Target;
182
183  // Initialize information about built-ins.
184  BuiltinInfo.InitializeTarget(Target);
185  HeaderInfo.setTarget(Target);
186}
187
188void Preprocessor::setPTHManager(PTHManager* pm) {
189  PTH.reset(pm);
190  FileMgr.addStatCache(PTH->createStatCache());
191}
192
193void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
194  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
195               << getSpelling(Tok) << "'";
196
197  if (!DumpFlags) return;
198
199  llvm::errs() << "\t";
200  if (Tok.isAtStartOfLine())
201    llvm::errs() << " [StartOfLine]";
202  if (Tok.hasLeadingSpace())
203    llvm::errs() << " [LeadingSpace]";
204  if (Tok.isExpandDisabled())
205    llvm::errs() << " [ExpandDisabled]";
206  if (Tok.needsCleaning()) {
207    const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
208    llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
209                 << "']";
210  }
211
212  llvm::errs() << "\tLoc=<";
213  DumpLocation(Tok.getLocation());
214  llvm::errs() << ">";
215}
216
217void Preprocessor::DumpLocation(SourceLocation Loc) const {
218  Loc.dump(SourceMgr);
219}
220
221void Preprocessor::DumpMacro(const MacroInfo &MI) const {
222  llvm::errs() << "MACRO: ";
223  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
224    DumpToken(MI.getReplacementToken(i));
225    llvm::errs() << "  ";
226  }
227  llvm::errs() << "\n";
228}
229
230void Preprocessor::PrintStats() {
231  llvm::errs() << "\n*** Preprocessor Stats:\n";
232  llvm::errs() << NumDirectives << " directives found:\n";
233  llvm::errs() << "  " << NumDefined << " #define.\n";
234  llvm::errs() << "  " << NumUndefined << " #undef.\n";
235  llvm::errs() << "  #include/#include_next/#import:\n";
236  llvm::errs() << "    " << NumEnteredSourceFiles << " source files entered.\n";
237  llvm::errs() << "    " << MaxIncludeStackDepth << " max include stack depth\n";
238  llvm::errs() << "  " << NumIf << " #if/#ifndef/#ifdef.\n";
239  llvm::errs() << "  " << NumElse << " #else/#elif.\n";
240  llvm::errs() << "  " << NumEndif << " #endif.\n";
241  llvm::errs() << "  " << NumPragma << " #pragma.\n";
242  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
243
244  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
245             << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
246             << NumFastMacroExpanded << " on the fast path.\n";
247  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
248             << " token paste (##) operations performed, "
249             << NumFastTokenPaste << " on the fast path.\n";
250
251  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
252
253  llvm::errs() << "\n  BumpPtr: " << BP.getTotalMemory();
254  llvm::errs() << "\n  Macro Expanded Tokens: "
255               << llvm::capacity_in_bytes(MacroExpandedTokens);
256  llvm::errs() << "\n  Predefines Buffer: " << Predefines.capacity();
257  llvm::errs() << "\n  Macros: " << llvm::capacity_in_bytes(Macros);
258  llvm::errs() << "\n  #pragma push_macro Info: "
259               << llvm::capacity_in_bytes(PragmaPushMacroInfo);
260  llvm::errs() << "\n  Poison Reasons: "
261               << llvm::capacity_in_bytes(PoisonReasons);
262  llvm::errs() << "\n  Comment Handlers: "
263               << llvm::capacity_in_bytes(CommentHandlers) << "\n";
264}
265
266Preprocessor::macro_iterator
267Preprocessor::macro_begin(bool IncludeExternalMacros) const {
268  if (IncludeExternalMacros && ExternalSource &&
269      !ReadMacrosFromExternalSource) {
270    ReadMacrosFromExternalSource = true;
271    ExternalSource->ReadDefinedMacros();
272  }
273
274  return Macros.begin();
275}
276
277size_t Preprocessor::getTotalMemory() const {
278  return BP.getTotalMemory()
279    + llvm::capacity_in_bytes(MacroExpandedTokens)
280    + Predefines.capacity() /* Predefines buffer. */
281    + llvm::capacity_in_bytes(Macros)
282    + llvm::capacity_in_bytes(PragmaPushMacroInfo)
283    + llvm::capacity_in_bytes(PoisonReasons)
284    + llvm::capacity_in_bytes(CommentHandlers);
285}
286
287Preprocessor::macro_iterator
288Preprocessor::macro_end(bool IncludeExternalMacros) const {
289  if (IncludeExternalMacros && ExternalSource &&
290      !ReadMacrosFromExternalSource) {
291    ReadMacrosFromExternalSource = true;
292    ExternalSource->ReadDefinedMacros();
293  }
294
295  return Macros.end();
296}
297
298/// \brief Compares macro tokens with a specified token value sequence.
299static bool MacroDefinitionEquals(const MacroInfo *MI,
300                                  ArrayRef<TokenValue> Tokens) {
301  return Tokens.size() == MI->getNumTokens() &&
302      std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
303}
304
305StringRef Preprocessor::getLastMacroWithSpelling(
306                                    SourceLocation Loc,
307                                    ArrayRef<TokenValue> Tokens) const {
308  SourceLocation BestLocation;
309  StringRef BestSpelling;
310  for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
311       I != E; ++I) {
312    if (!I->second->getMacroInfo()->isObjectLike())
313      continue;
314    const MacroDirective::DefInfo
315      Def = I->second->findDirectiveAtLoc(Loc, SourceMgr);
316    if (!Def)
317      continue;
318    if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
319      continue;
320    SourceLocation Location = Def.getLocation();
321    // Choose the macro defined latest.
322    if (BestLocation.isInvalid() ||
323        (Location.isValid() &&
324         SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
325      BestLocation = Location;
326      BestSpelling = I->first->getName();
327    }
328  }
329  return BestSpelling;
330}
331
332void Preprocessor::recomputeCurLexerKind() {
333  if (CurLexer)
334    CurLexerKind = CLK_Lexer;
335  else if (CurPTHLexer)
336    CurLexerKind = CLK_PTHLexer;
337  else if (CurTokenLexer)
338    CurLexerKind = CLK_TokenLexer;
339  else
340    CurLexerKind = CLK_CachingLexer;
341}
342
343bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
344                                          unsigned CompleteLine,
345                                          unsigned CompleteColumn) {
346  assert(File);
347  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
348  assert(!CodeCompletionFile && "Already set");
349
350  using llvm::MemoryBuffer;
351
352  // Load the actual file's contents.
353  bool Invalid = false;
354  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
355  if (Invalid)
356    return true;
357
358  // Find the byte position of the truncation point.
359  const char *Position = Buffer->getBufferStart();
360  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
361    for (; *Position; ++Position) {
362      if (*Position != '\r' && *Position != '\n')
363        continue;
364
365      // Eat \r\n or \n\r as a single line.
366      if ((Position[1] == '\r' || Position[1] == '\n') &&
367          Position[0] != Position[1])
368        ++Position;
369      ++Position;
370      break;
371    }
372  }
373
374  Position += CompleteColumn - 1;
375
376  // Insert '\0' at the code-completion point.
377  if (Position < Buffer->getBufferEnd()) {
378    CodeCompletionFile = File;
379    CodeCompletionOffset = Position - Buffer->getBufferStart();
380
381    MemoryBuffer *NewBuffer =
382        MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
383                                            Buffer->getBufferIdentifier());
384    char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
385    char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
386    *NewPos = '\0';
387    std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
388    SourceMgr.overrideFileContents(File, NewBuffer);
389  }
390
391  return false;
392}
393
394void Preprocessor::CodeCompleteNaturalLanguage() {
395  if (CodeComplete)
396    CodeComplete->CodeCompleteNaturalLanguage();
397  setCodeCompletionReached();
398}
399
400/// getSpelling - This method is used to get the spelling of a token into a
401/// SmallVector. Note that the returned StringRef may not point to the
402/// supplied buffer if a copy can be avoided.
403StringRef Preprocessor::getSpelling(const Token &Tok,
404                                          SmallVectorImpl<char> &Buffer,
405                                          bool *Invalid) const {
406  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
407  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
408    // Try the fast path.
409    if (const IdentifierInfo *II = Tok.getIdentifierInfo())
410      return II->getName();
411  }
412
413  // Resize the buffer if we need to copy into it.
414  if (Tok.needsCleaning())
415    Buffer.resize(Tok.getLength());
416
417  const char *Ptr = Buffer.data();
418  unsigned Len = getSpelling(Tok, Ptr, Invalid);
419  return StringRef(Ptr, Len);
420}
421
422/// CreateString - Plop the specified string into a scratch buffer and return a
423/// location for it.  If specified, the source location provides a source
424/// location for the token.
425void Preprocessor::CreateString(StringRef Str, Token &Tok,
426                                SourceLocation ExpansionLocStart,
427                                SourceLocation ExpansionLocEnd) {
428  Tok.setLength(Str.size());
429
430  const char *DestPtr;
431  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
432
433  if (ExpansionLocStart.isValid())
434    Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
435                                       ExpansionLocEnd, Str.size());
436  Tok.setLocation(Loc);
437
438  // If this is a raw identifier or a literal token, set the pointer data.
439  if (Tok.is(tok::raw_identifier))
440    Tok.setRawIdentifierData(DestPtr);
441  else if (Tok.isLiteral())
442    Tok.setLiteralData(DestPtr);
443}
444
445Module *Preprocessor::getCurrentModule() {
446  if (getLangOpts().CurrentModule.empty())
447    return 0;
448
449  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
450}
451
452//===----------------------------------------------------------------------===//
453// Preprocessor Initialization Methods
454//===----------------------------------------------------------------------===//
455
456
457/// EnterMainSourceFile - Enter the specified FileID as the main source file,
458/// which implicitly adds the builtin defines etc.
459void Preprocessor::EnterMainSourceFile() {
460  // We do not allow the preprocessor to reenter the main file.  Doing so will
461  // cause FileID's to accumulate information from both runs (e.g. #line
462  // information) and predefined macros aren't guaranteed to be set properly.
463  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
464  FileID MainFileID = SourceMgr.getMainFileID();
465
466  // If MainFileID is loaded it means we loaded an AST file, no need to enter
467  // a main file.
468  if (!SourceMgr.isLoadedFileID(MainFileID)) {
469    // Enter the main file source buffer.
470    EnterSourceFile(MainFileID, 0, SourceLocation());
471
472    // If we've been asked to skip bytes in the main file (e.g., as part of a
473    // precompiled preamble), do so now.
474    if (SkipMainFilePreamble.first > 0)
475      CurLexer->SkipBytes(SkipMainFilePreamble.first,
476                          SkipMainFilePreamble.second);
477
478    // Tell the header info that the main file was entered.  If the file is later
479    // #imported, it won't be re-entered.
480    if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
481      HeaderInfo.IncrementIncludeCount(FE);
482  }
483
484  // Preprocess Predefines to populate the initial preprocessor state.
485  llvm::MemoryBuffer *SB =
486    llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
487  assert(SB && "Cannot create predefined source buffer");
488  FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
489  assert(!FID.isInvalid() && "Could not create FileID for predefines?");
490  setPredefinesFileID(FID);
491
492  // Start parsing the predefines.
493  EnterSourceFile(FID, 0, SourceLocation());
494}
495
496void Preprocessor::EndSourceFile() {
497  // Notify the client that we reached the end of the source file.
498  if (Callbacks)
499    Callbacks->EndOfMainFile();
500}
501
502//===----------------------------------------------------------------------===//
503// Lexer Event Handling.
504//===----------------------------------------------------------------------===//
505
506static void appendCodePoint(unsigned Codepoint,
507                            llvm::SmallVectorImpl<char> &Str) {
508  char ResultBuf[4];
509  char *ResultPtr = ResultBuf;
510  bool Res = llvm::ConvertCodePointToUTF8(Codepoint, ResultPtr);
511  (void)Res;
512  assert(Res && "Unexpected conversion failure");
513  Str.append(ResultBuf, ResultPtr);
514}
515
516static void expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
517  for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) {
518    if (*I != '\\') {
519      Buf.push_back(*I);
520      continue;
521    }
522
523    ++I;
524    assert(*I == 'u' || *I == 'U');
525
526    unsigned NumHexDigits;
527    if (*I == 'u')
528      NumHexDigits = 4;
529    else
530      NumHexDigits = 8;
531
532    assert(I + NumHexDigits <= E);
533
534    uint32_t CodePoint = 0;
535    for (++I; NumHexDigits != 0; ++I, --NumHexDigits) {
536      unsigned Value = llvm::hexDigitValue(*I);
537      assert(Value != -1U);
538
539      CodePoint <<= 4;
540      CodePoint += Value;
541    }
542
543    appendCodePoint(CodePoint, Buf);
544    --I;
545  }
546}
547
548/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
549/// identifier information for the token and install it into the token,
550/// updating the token kind accordingly.
551IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
552  assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!");
553
554  // Look up this token, see if it is a macro, or if it is a language keyword.
555  IdentifierInfo *II;
556  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
557    // No cleaning needed, just use the characters from the lexed buffer.
558    II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(),
559                                     Identifier.getLength()));
560  } else {
561    // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
562    SmallString<64> IdentifierBuffer;
563    StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
564
565    if (Identifier.hasUCN()) {
566      SmallString<64> UCNIdentifierBuffer;
567      expandUCNs(UCNIdentifierBuffer, CleanedStr);
568      II = getIdentifierInfo(UCNIdentifierBuffer);
569    } else {
570      II = getIdentifierInfo(CleanedStr);
571    }
572  }
573
574  // Update the token info (identifier info and appropriate token kind).
575  Identifier.setIdentifierInfo(II);
576  Identifier.setKind(II->getTokenID());
577
578  return II;
579}
580
581void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
582  PoisonReasons[II] = DiagID;
583}
584
585void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
586  assert(Ident__exception_code && Ident__exception_info);
587  assert(Ident___exception_code && Ident___exception_info);
588  Ident__exception_code->setIsPoisoned(Poison);
589  Ident___exception_code->setIsPoisoned(Poison);
590  Ident_GetExceptionCode->setIsPoisoned(Poison);
591  Ident__exception_info->setIsPoisoned(Poison);
592  Ident___exception_info->setIsPoisoned(Poison);
593  Ident_GetExceptionInfo->setIsPoisoned(Poison);
594  Ident__abnormal_termination->setIsPoisoned(Poison);
595  Ident___abnormal_termination->setIsPoisoned(Poison);
596  Ident_AbnormalTermination->setIsPoisoned(Poison);
597}
598
599void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
600  assert(Identifier.getIdentifierInfo() &&
601         "Can't handle identifiers without identifier info!");
602  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
603    PoisonReasons.find(Identifier.getIdentifierInfo());
604  if(it == PoisonReasons.end())
605    Diag(Identifier, diag::err_pp_used_poisoned_id);
606  else
607    Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
608}
609
610/// HandleIdentifier - This callback is invoked when the lexer reads an
611/// identifier.  This callback looks up the identifier in the map and/or
612/// potentially macro expands it or turns it into a named token (like 'for').
613///
614/// Note that callers of this method are guarded by checking the
615/// IdentifierInfo's 'isHandleIdentifierCase' bit.  If this method changes, the
616/// IdentifierInfo methods that compute these properties will need to change to
617/// match.
618bool Preprocessor::HandleIdentifier(Token &Identifier) {
619  assert(Identifier.getIdentifierInfo() &&
620         "Can't handle identifiers without identifier info!");
621
622  IdentifierInfo &II = *Identifier.getIdentifierInfo();
623
624  // If the information about this identifier is out of date, update it from
625  // the external source.
626  // We have to treat __VA_ARGS__ in a special way, since it gets
627  // serialized with isPoisoned = true, but our preprocessor may have
628  // unpoisoned it if we're defining a C99 macro.
629  if (II.isOutOfDate()) {
630    bool CurrentIsPoisoned = false;
631    if (&II == Ident__VA_ARGS__)
632      CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned();
633
634    ExternalSource->updateOutOfDateIdentifier(II);
635    Identifier.setKind(II.getTokenID());
636
637    if (&II == Ident__VA_ARGS__)
638      II.setIsPoisoned(CurrentIsPoisoned);
639  }
640
641  // If this identifier was poisoned, and if it was not produced from a macro
642  // expansion, emit an error.
643  if (II.isPoisoned() && CurPPLexer) {
644    HandlePoisonedIdentifier(Identifier);
645  }
646
647  // If this is a macro to be expanded, do it.
648  if (MacroDirective *MD = getMacroDirective(&II)) {
649    MacroInfo *MI = MD->getMacroInfo();
650    if (!DisableMacroExpansion) {
651      if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
652        // C99 6.10.3p10: If the preprocessing token immediately after the
653        // macro name isn't a '(', this macro should not be expanded.
654        if (!MI->isFunctionLike() || isNextPPTokenLParen())
655          return HandleMacroExpandedIdentifier(Identifier, MD);
656      } else {
657        // C99 6.10.3.4p2 says that a disabled macro may never again be
658        // expanded, even if it's in a context where it could be expanded in the
659        // future.
660        Identifier.setFlag(Token::DisableExpand);
661        if (MI->isObjectLike() || isNextPPTokenLParen())
662          Diag(Identifier, diag::pp_disabled_macro_expansion);
663      }
664    }
665  }
666
667  // If this identifier is a keyword in C++11, produce a warning. Don't warn if
668  // we're not considering macro expansion, since this identifier might be the
669  // name of a macro.
670  // FIXME: This warning is disabled in cases where it shouldn't be, like
671  //   "#define constexpr constexpr", "int constexpr;"
672  if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) {
673    Diag(Identifier, diag::warn_cxx11_keyword) << II.getName();
674    // Don't diagnose this keyword again in this translation unit.
675    II.setIsCXX11CompatKeyword(false);
676  }
677
678  // C++ 2.11p2: If this is an alternative representation of a C++ operator,
679  // then we act as if it is the actual operator and not the textual
680  // representation of it.
681  if (II.isCPlusPlusOperatorKeyword())
682    Identifier.setIdentifierInfo(0);
683
684  // If this is an extension token, diagnose its use.
685  // We avoid diagnosing tokens that originate from macro definitions.
686  // FIXME: This warning is disabled in cases where it shouldn't be,
687  // like "#define TY typeof", "TY(1) x".
688  if (II.isExtensionToken() && !DisableMacroExpansion)
689    Diag(Identifier, diag::ext_token_used);
690
691  // If this is the 'import' contextual keyword following an '@', note
692  // that the next token indicates a module name.
693  //
694  // Note that we do not treat 'import' as a contextual
695  // keyword when we're in a caching lexer, because caching lexers only get
696  // used in contexts where import declarations are disallowed.
697  if (LastTokenWasAt && II.isModulesImport() && !InMacroArgs &&
698      !DisableMacroExpansion && getLangOpts().Modules &&
699      CurLexerKind != CLK_CachingLexer) {
700    ModuleImportLoc = Identifier.getLocation();
701    ModuleImportPath.clear();
702    ModuleImportExpectsIdentifier = true;
703    CurLexerKind = CLK_LexAfterModuleImport;
704  }
705  return true;
706}
707
708void Preprocessor::Lex(Token &Result) {
709  // We loop here until a lex function retuns a token; this avoids recursion.
710  bool ReturnedToken;
711  do {
712    switch (CurLexerKind) {
713    case CLK_Lexer:
714      ReturnedToken = CurLexer->Lex(Result);
715      break;
716    case CLK_PTHLexer:
717      ReturnedToken = CurPTHLexer->Lex(Result);
718      break;
719    case CLK_TokenLexer:
720      ReturnedToken = CurTokenLexer->Lex(Result);
721      break;
722    case CLK_CachingLexer:
723      CachingLex(Result);
724      ReturnedToken = true;
725      break;
726    case CLK_LexAfterModuleImport:
727      LexAfterModuleImport(Result);
728      ReturnedToken = true;
729      break;
730    }
731  } while (!ReturnedToken);
732
733  LastTokenWasAt = Result.is(tok::at);
734}
735
736
737/// \brief Lex a token following the 'import' contextual keyword.
738///
739void Preprocessor::LexAfterModuleImport(Token &Result) {
740  // Figure out what kind of lexer we actually have.
741  recomputeCurLexerKind();
742
743  // Lex the next token.
744  Lex(Result);
745
746  // The token sequence
747  //
748  //   import identifier (. identifier)*
749  //
750  // indicates a module import directive. We already saw the 'import'
751  // contextual keyword, so now we're looking for the identifiers.
752  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
753    // We expected to see an identifier here, and we did; continue handling
754    // identifiers.
755    ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
756                                              Result.getLocation()));
757    ModuleImportExpectsIdentifier = false;
758    CurLexerKind = CLK_LexAfterModuleImport;
759    return;
760  }
761
762  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
763  // see the next identifier.
764  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
765    ModuleImportExpectsIdentifier = true;
766    CurLexerKind = CLK_LexAfterModuleImport;
767    return;
768  }
769
770  // If we have a non-empty module path, load the named module.
771  if (!ModuleImportPath.empty() && getLangOpts().Modules) {
772    Module *Imported = TheModuleLoader.loadModule(ModuleImportLoc,
773                                                  ModuleImportPath,
774                                                  Module::MacrosVisible,
775                                                  /*IsIncludeDirective=*/false);
776    if (Callbacks)
777      Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
778  }
779}
780
781bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
782                                          const char *DiagnosticTag,
783                                          bool AllowMacroExpansion) {
784  // We need at least one string literal.
785  if (Result.isNot(tok::string_literal)) {
786    Diag(Result, diag::err_expected_string_literal)
787      << /*Source='in...'*/0 << DiagnosticTag;
788    return false;
789  }
790
791  // Lex string literal tokens, optionally with macro expansion.
792  SmallVector<Token, 4> StrToks;
793  do {
794    StrToks.push_back(Result);
795
796    if (Result.hasUDSuffix())
797      Diag(Result, diag::err_invalid_string_udl);
798
799    if (AllowMacroExpansion)
800      Lex(Result);
801    else
802      LexUnexpandedToken(Result);
803  } while (Result.is(tok::string_literal));
804
805  // Concatenate and parse the strings.
806  StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this);
807  assert(Literal.isAscii() && "Didn't allow wide strings in");
808
809  if (Literal.hadError)
810    return false;
811
812  if (Literal.Pascal) {
813    Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
814      << /*Source='in...'*/0 << DiagnosticTag;
815    return false;
816  }
817
818  String = Literal.GetString();
819  return true;
820}
821
822void Preprocessor::addCommentHandler(CommentHandler *Handler) {
823  assert(Handler && "NULL comment handler");
824  assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
825         CommentHandlers.end() && "Comment handler already registered");
826  CommentHandlers.push_back(Handler);
827}
828
829void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
830  std::vector<CommentHandler *>::iterator Pos
831  = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
832  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
833  CommentHandlers.erase(Pos);
834}
835
836bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
837  bool AnyPendingTokens = false;
838  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
839       HEnd = CommentHandlers.end();
840       H != HEnd; ++H) {
841    if ((*H)->HandleComment(*this, Comment))
842      AnyPendingTokens = true;
843  }
844  if (!AnyPendingTokens || getCommentRetentionState())
845    return false;
846  Lex(result);
847  return true;
848}
849
850ModuleLoader::~ModuleLoader() { }
851
852CommentHandler::~CommentHandler() { }
853
854CodeCompletionHandler::~CodeCompletionHandler() { }
855
856void Preprocessor::createPreprocessingRecord() {
857  if (Record)
858    return;
859
860  Record = new PreprocessingRecord(getSourceManager());
861  addPPCallbacks(Record);
862}
863