CommentLexer.cpp revision 263508
1#include "clang/AST/CommentLexer.h" 2#include "clang/AST/CommentCommandTraits.h" 3#include "clang/AST/CommentDiagnostic.h" 4#include "clang/Basic/CharInfo.h" 5#include "llvm/ADT/StringExtras.h" 6#include "llvm/ADT/StringSwitch.h" 7#include "llvm/Support/ConvertUTF.h" 8#include "llvm/Support/ErrorHandling.h" 9 10namespace clang { 11namespace comments { 12 13void Token::dump(const Lexer &L, const SourceManager &SM) const { 14 llvm::errs() << "comments::Token Kind=" << Kind << " "; 15 Loc.dump(SM); 16 llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n"; 17} 18 19static inline bool isHTMLNamedCharacterReferenceCharacter(char C) { 20 return isLetter(C); 21} 22 23static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) { 24 return isDigit(C); 25} 26 27static inline bool isHTMLHexCharacterReferenceCharacter(char C) { 28 return isHexDigit(C); 29} 30 31static inline StringRef convertCodePointToUTF8( 32 llvm::BumpPtrAllocator &Allocator, 33 unsigned CodePoint) { 34 char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT); 35 char *ResolvedPtr = Resolved; 36 if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr)) 37 return StringRef(Resolved, ResolvedPtr - Resolved); 38 else 39 return StringRef(); 40} 41 42namespace { 43 44#include "clang/AST/CommentHTMLTags.inc" 45#include "clang/AST/CommentHTMLNamedCharacterReferences.inc" 46 47} // unnamed namespace 48 49StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const { 50 // Fast path, first check a few most widely used named character references. 51 return llvm::StringSwitch<StringRef>(Name) 52 .Case("amp", "&") 53 .Case("lt", "<") 54 .Case("gt", ">") 55 .Case("quot", "\"") 56 .Case("apos", "\'") 57 // Slow path. 58 .Default(translateHTMLNamedCharacterReferenceToUTF8(Name)); 59} 60 61StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const { 62 unsigned CodePoint = 0; 63 for (unsigned i = 0, e = Name.size(); i != e; ++i) { 64 assert(isHTMLDecimalCharacterReferenceCharacter(Name[i])); 65 CodePoint *= 10; 66 CodePoint += Name[i] - '0'; 67 } 68 return convertCodePointToUTF8(Allocator, CodePoint); 69} 70 71StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const { 72 unsigned CodePoint = 0; 73 for (unsigned i = 0, e = Name.size(); i != e; ++i) { 74 CodePoint *= 16; 75 const char C = Name[i]; 76 assert(isHTMLHexCharacterReferenceCharacter(C)); 77 CodePoint += llvm::hexDigitValue(C); 78 } 79 return convertCodePointToUTF8(Allocator, CodePoint); 80} 81 82void Lexer::skipLineStartingDecorations() { 83 // This function should be called only for C comments 84 assert(CommentState == LCS_InsideCComment); 85 86 if (BufferPtr == CommentEnd) 87 return; 88 89 switch (*BufferPtr) { 90 case ' ': 91 case '\t': 92 case '\f': 93 case '\v': { 94 const char *NewBufferPtr = BufferPtr; 95 NewBufferPtr++; 96 if (NewBufferPtr == CommentEnd) 97 return; 98 99 char C = *NewBufferPtr; 100 while (isHorizontalWhitespace(C)) { 101 NewBufferPtr++; 102 if (NewBufferPtr == CommentEnd) 103 return; 104 C = *NewBufferPtr; 105 } 106 if (C == '*') 107 BufferPtr = NewBufferPtr + 1; 108 break; 109 } 110 case '*': 111 BufferPtr++; 112 break; 113 } 114} 115 116namespace { 117/// Returns pointer to the first newline character in the string. 118const char *findNewline(const char *BufferPtr, const char *BufferEnd) { 119 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 120 if (isVerticalWhitespace(*BufferPtr)) 121 return BufferPtr; 122 } 123 return BufferEnd; 124} 125 126const char *skipNewline(const char *BufferPtr, const char *BufferEnd) { 127 if (BufferPtr == BufferEnd) 128 return BufferPtr; 129 130 if (*BufferPtr == '\n') 131 BufferPtr++; 132 else { 133 assert(*BufferPtr == '\r'); 134 BufferPtr++; 135 if (BufferPtr != BufferEnd && *BufferPtr == '\n') 136 BufferPtr++; 137 } 138 return BufferPtr; 139} 140 141const char *skipNamedCharacterReference(const char *BufferPtr, 142 const char *BufferEnd) { 143 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 144 if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr)) 145 return BufferPtr; 146 } 147 return BufferEnd; 148} 149 150const char *skipDecimalCharacterReference(const char *BufferPtr, 151 const char *BufferEnd) { 152 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 153 if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr)) 154 return BufferPtr; 155 } 156 return BufferEnd; 157} 158 159const char *skipHexCharacterReference(const char *BufferPtr, 160 const char *BufferEnd) { 161 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 162 if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr)) 163 return BufferPtr; 164 } 165 return BufferEnd; 166} 167 168bool isHTMLIdentifierStartingCharacter(char C) { 169 return isLetter(C); 170} 171 172bool isHTMLIdentifierCharacter(char C) { 173 return isAlphanumeric(C); 174} 175 176const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) { 177 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 178 if (!isHTMLIdentifierCharacter(*BufferPtr)) 179 return BufferPtr; 180 } 181 return BufferEnd; 182} 183 184/// Skip HTML string quoted in single or double quotes. Escaping quotes inside 185/// string allowed. 186/// 187/// Returns pointer to closing quote. 188const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd) 189{ 190 const char Quote = *BufferPtr; 191 assert(Quote == '\"' || Quote == '\''); 192 193 BufferPtr++; 194 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 195 const char C = *BufferPtr; 196 if (C == Quote && BufferPtr[-1] != '\\') 197 return BufferPtr; 198 } 199 return BufferEnd; 200} 201 202const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) { 203 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 204 if (!isWhitespace(*BufferPtr)) 205 return BufferPtr; 206 } 207 return BufferEnd; 208} 209 210bool isWhitespace(const char *BufferPtr, const char *BufferEnd) { 211 return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd; 212} 213 214bool isCommandNameStartCharacter(char C) { 215 return isLetter(C); 216} 217 218bool isCommandNameCharacter(char C) { 219 return isAlphanumeric(C); 220} 221 222const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) { 223 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 224 if (!isCommandNameCharacter(*BufferPtr)) 225 return BufferPtr; 226 } 227 return BufferEnd; 228} 229 230/// Return the one past end pointer for BCPL comments. 231/// Handles newlines escaped with backslash or trigraph for backslahs. 232const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) { 233 const char *CurPtr = BufferPtr; 234 while (CurPtr != BufferEnd) { 235 while (!isVerticalWhitespace(*CurPtr)) { 236 CurPtr++; 237 if (CurPtr == BufferEnd) 238 return BufferEnd; 239 } 240 // We found a newline, check if it is escaped. 241 const char *EscapePtr = CurPtr - 1; 242 while(isHorizontalWhitespace(*EscapePtr)) 243 EscapePtr--; 244 245 if (*EscapePtr == '\\' || 246 (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' && 247 EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) { 248 // We found an escaped newline. 249 CurPtr = skipNewline(CurPtr, BufferEnd); 250 } else 251 return CurPtr; // Not an escaped newline. 252 } 253 return BufferEnd; 254} 255 256/// Return the one past end pointer for C comments. 257/// Very dumb, does not handle escaped newlines or trigraphs. 258const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) { 259 for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 260 if (*BufferPtr == '*') { 261 assert(BufferPtr + 1 != BufferEnd); 262 if (*(BufferPtr + 1) == '/') 263 return BufferPtr; 264 } 265 } 266 llvm_unreachable("buffer end hit before '*/' was seen"); 267} 268 269} // unnamed namespace 270 271void Lexer::lexCommentText(Token &T) { 272 assert(CommentState == LCS_InsideBCPLComment || 273 CommentState == LCS_InsideCComment); 274 275 switch (State) { 276 case LS_Normal: 277 break; 278 case LS_VerbatimBlockFirstLine: 279 lexVerbatimBlockFirstLine(T); 280 return; 281 case LS_VerbatimBlockBody: 282 lexVerbatimBlockBody(T); 283 return; 284 case LS_VerbatimLineText: 285 lexVerbatimLineText(T); 286 return; 287 case LS_HTMLStartTag: 288 lexHTMLStartTag(T); 289 return; 290 case LS_HTMLEndTag: 291 lexHTMLEndTag(T); 292 return; 293 } 294 295 assert(State == LS_Normal); 296 297 const char *TokenPtr = BufferPtr; 298 assert(TokenPtr < CommentEnd); 299 while (TokenPtr != CommentEnd) { 300 switch(*TokenPtr) { 301 case '\\': 302 case '@': { 303 // Commands that start with a backslash and commands that start with 304 // 'at' have equivalent semantics. But we keep information about the 305 // exact syntax in AST for comments. 306 tok::TokenKind CommandKind = 307 (*TokenPtr == '@') ? tok::at_command : tok::backslash_command; 308 TokenPtr++; 309 if (TokenPtr == CommentEnd) { 310 formTextToken(T, TokenPtr); 311 return; 312 } 313 char C = *TokenPtr; 314 switch (C) { 315 default: 316 break; 317 318 case '\\': case '@': case '&': case '$': 319 case '#': case '<': case '>': case '%': 320 case '\"': case '.': case ':': 321 // This is one of \\ \@ \& \$ etc escape sequences. 322 TokenPtr++; 323 if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') { 324 // This is the \:: escape sequence. 325 TokenPtr++; 326 } 327 StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1)); 328 formTokenWithChars(T, TokenPtr, tok::text); 329 T.setText(UnescapedText); 330 return; 331 } 332 333 // Don't make zero-length commands. 334 if (!isCommandNameStartCharacter(*TokenPtr)) { 335 formTextToken(T, TokenPtr); 336 return; 337 } 338 339 TokenPtr = skipCommandName(TokenPtr, CommentEnd); 340 unsigned Length = TokenPtr - (BufferPtr + 1); 341 342 // Hardcoded support for lexing LaTeX formula commands 343 // \f$ \f[ \f] \f{ \f} as a single command. 344 if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) { 345 C = *TokenPtr; 346 if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') { 347 TokenPtr++; 348 Length++; 349 } 350 } 351 352 const StringRef CommandName(BufferPtr + 1, Length); 353 354 const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName); 355 if (!Info) { 356 if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) { 357 StringRef CorrectedName = Info->Name; 358 SourceLocation Loc = getSourceLocation(BufferPtr); 359 SourceRange CommandRange(Loc.getLocWithOffset(1), 360 getSourceLocation(TokenPtr)); 361 Diag(Loc, diag::warn_correct_comment_command_name) 362 << CommandName << CorrectedName 363 << FixItHint::CreateReplacement(CommandRange, CorrectedName); 364 } else { 365 formTokenWithChars(T, TokenPtr, tok::unknown_command); 366 T.setUnknownCommandName(CommandName); 367 Diag(T.getLocation(), diag::warn_unknown_comment_command_name); 368 return; 369 } 370 } 371 if (Info->IsVerbatimBlockCommand) { 372 setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info); 373 return; 374 } 375 if (Info->IsVerbatimLineCommand) { 376 setupAndLexVerbatimLine(T, TokenPtr, Info); 377 return; 378 } 379 formTokenWithChars(T, TokenPtr, CommandKind); 380 T.setCommandID(Info->getID()); 381 return; 382 } 383 384 case '&': 385 lexHTMLCharacterReference(T); 386 return; 387 388 case '<': { 389 TokenPtr++; 390 if (TokenPtr == CommentEnd) { 391 formTextToken(T, TokenPtr); 392 return; 393 } 394 const char C = *TokenPtr; 395 if (isHTMLIdentifierStartingCharacter(C)) 396 setupAndLexHTMLStartTag(T); 397 else if (C == '/') 398 setupAndLexHTMLEndTag(T); 399 else 400 formTextToken(T, TokenPtr); 401 402 return; 403 } 404 405 case '\n': 406 case '\r': 407 TokenPtr = skipNewline(TokenPtr, CommentEnd); 408 formTokenWithChars(T, TokenPtr, tok::newline); 409 410 if (CommentState == LCS_InsideCComment) 411 skipLineStartingDecorations(); 412 return; 413 414 default: { 415 size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr). 416 find_first_of("\n\r\\@&<"); 417 if (End != StringRef::npos) 418 TokenPtr += End; 419 else 420 TokenPtr = CommentEnd; 421 formTextToken(T, TokenPtr); 422 return; 423 } 424 } 425 } 426} 427 428void Lexer::setupAndLexVerbatimBlock(Token &T, 429 const char *TextBegin, 430 char Marker, const CommandInfo *Info) { 431 assert(Info->IsVerbatimBlockCommand); 432 433 VerbatimBlockEndCommandName.clear(); 434 VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@"); 435 VerbatimBlockEndCommandName.append(Info->EndCommandName); 436 437 formTokenWithChars(T, TextBegin, tok::verbatim_block_begin); 438 T.setVerbatimBlockID(Info->getID()); 439 440 // If there is a newline following the verbatim opening command, skip the 441 // newline so that we don't create an tok::verbatim_block_line with empty 442 // text content. 443 if (BufferPtr != CommentEnd && 444 isVerticalWhitespace(*BufferPtr)) { 445 BufferPtr = skipNewline(BufferPtr, CommentEnd); 446 State = LS_VerbatimBlockBody; 447 return; 448 } 449 450 State = LS_VerbatimBlockFirstLine; 451} 452 453void Lexer::lexVerbatimBlockFirstLine(Token &T) { 454again: 455 assert(BufferPtr < CommentEnd); 456 457 // FIXME: It would be better to scan the text once, finding either the block 458 // end command or newline. 459 // 460 // Extract current line. 461 const char *Newline = findNewline(BufferPtr, CommentEnd); 462 StringRef Line(BufferPtr, Newline - BufferPtr); 463 464 // Look for end command in current line. 465 size_t Pos = Line.find(VerbatimBlockEndCommandName); 466 const char *TextEnd; 467 const char *NextLine; 468 if (Pos == StringRef::npos) { 469 // Current line is completely verbatim. 470 TextEnd = Newline; 471 NextLine = skipNewline(Newline, CommentEnd); 472 } else if (Pos == 0) { 473 // Current line contains just an end command. 474 const char *End = BufferPtr + VerbatimBlockEndCommandName.size(); 475 StringRef Name(BufferPtr + 1, End - (BufferPtr + 1)); 476 formTokenWithChars(T, End, tok::verbatim_block_end); 477 T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID()); 478 State = LS_Normal; 479 return; 480 } else { 481 // There is some text, followed by end command. Extract text first. 482 TextEnd = BufferPtr + Pos; 483 NextLine = TextEnd; 484 // If there is only whitespace before end command, skip whitespace. 485 if (isWhitespace(BufferPtr, TextEnd)) { 486 BufferPtr = TextEnd; 487 goto again; 488 } 489 } 490 491 StringRef Text(BufferPtr, TextEnd - BufferPtr); 492 formTokenWithChars(T, NextLine, tok::verbatim_block_line); 493 T.setVerbatimBlockText(Text); 494 495 State = LS_VerbatimBlockBody; 496} 497 498void Lexer::lexVerbatimBlockBody(Token &T) { 499 assert(State == LS_VerbatimBlockBody); 500 501 if (CommentState == LCS_InsideCComment) 502 skipLineStartingDecorations(); 503 504 lexVerbatimBlockFirstLine(T); 505} 506 507void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin, 508 const CommandInfo *Info) { 509 assert(Info->IsVerbatimLineCommand); 510 formTokenWithChars(T, TextBegin, tok::verbatim_line_name); 511 T.setVerbatimLineID(Info->getID()); 512 513 State = LS_VerbatimLineText; 514} 515 516void Lexer::lexVerbatimLineText(Token &T) { 517 assert(State == LS_VerbatimLineText); 518 519 // Extract current line. 520 const char *Newline = findNewline(BufferPtr, CommentEnd); 521 const StringRef Text(BufferPtr, Newline - BufferPtr); 522 formTokenWithChars(T, Newline, tok::verbatim_line_text); 523 T.setVerbatimLineText(Text); 524 525 State = LS_Normal; 526} 527 528void Lexer::lexHTMLCharacterReference(Token &T) { 529 const char *TokenPtr = BufferPtr; 530 assert(*TokenPtr == '&'); 531 TokenPtr++; 532 if (TokenPtr == CommentEnd) { 533 formTextToken(T, TokenPtr); 534 return; 535 } 536 const char *NamePtr; 537 bool isNamed = false; 538 bool isDecimal = false; 539 char C = *TokenPtr; 540 if (isHTMLNamedCharacterReferenceCharacter(C)) { 541 NamePtr = TokenPtr; 542 TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd); 543 isNamed = true; 544 } else if (C == '#') { 545 TokenPtr++; 546 if (TokenPtr == CommentEnd) { 547 formTextToken(T, TokenPtr); 548 return; 549 } 550 C = *TokenPtr; 551 if (isHTMLDecimalCharacterReferenceCharacter(C)) { 552 NamePtr = TokenPtr; 553 TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd); 554 isDecimal = true; 555 } else if (C == 'x' || C == 'X') { 556 TokenPtr++; 557 NamePtr = TokenPtr; 558 TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd); 559 } else { 560 formTextToken(T, TokenPtr); 561 return; 562 } 563 } else { 564 formTextToken(T, TokenPtr); 565 return; 566 } 567 if (NamePtr == TokenPtr || TokenPtr == CommentEnd || 568 *TokenPtr != ';') { 569 formTextToken(T, TokenPtr); 570 return; 571 } 572 StringRef Name(NamePtr, TokenPtr - NamePtr); 573 TokenPtr++; // Skip semicolon. 574 StringRef Resolved; 575 if (isNamed) 576 Resolved = resolveHTMLNamedCharacterReference(Name); 577 else if (isDecimal) 578 Resolved = resolveHTMLDecimalCharacterReference(Name); 579 else 580 Resolved = resolveHTMLHexCharacterReference(Name); 581 582 if (Resolved.empty()) { 583 formTextToken(T, TokenPtr); 584 return; 585 } 586 formTokenWithChars(T, TokenPtr, tok::text); 587 T.setText(Resolved); 588 return; 589} 590 591void Lexer::setupAndLexHTMLStartTag(Token &T) { 592 assert(BufferPtr[0] == '<' && 593 isHTMLIdentifierStartingCharacter(BufferPtr[1])); 594 const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd); 595 StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1)); 596 if (!isHTMLTagName(Name)) { 597 formTextToken(T, TagNameEnd); 598 return; 599 } 600 601 formTokenWithChars(T, TagNameEnd, tok::html_start_tag); 602 T.setHTMLTagStartName(Name); 603 604 BufferPtr = skipWhitespace(BufferPtr, CommentEnd); 605 606 const char C = *BufferPtr; 607 if (BufferPtr != CommentEnd && 608 (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C))) 609 State = LS_HTMLStartTag; 610} 611 612void Lexer::lexHTMLStartTag(Token &T) { 613 assert(State == LS_HTMLStartTag); 614 615 const char *TokenPtr = BufferPtr; 616 char C = *TokenPtr; 617 if (isHTMLIdentifierCharacter(C)) { 618 TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd); 619 StringRef Ident(BufferPtr, TokenPtr - BufferPtr); 620 formTokenWithChars(T, TokenPtr, tok::html_ident); 621 T.setHTMLIdent(Ident); 622 } else { 623 switch (C) { 624 case '=': 625 TokenPtr++; 626 formTokenWithChars(T, TokenPtr, tok::html_equals); 627 break; 628 case '\"': 629 case '\'': { 630 const char *OpenQuote = TokenPtr; 631 TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd); 632 const char *ClosingQuote = TokenPtr; 633 if (TokenPtr != CommentEnd) // Skip closing quote. 634 TokenPtr++; 635 formTokenWithChars(T, TokenPtr, tok::html_quoted_string); 636 T.setHTMLQuotedString(StringRef(OpenQuote + 1, 637 ClosingQuote - (OpenQuote + 1))); 638 break; 639 } 640 case '>': 641 TokenPtr++; 642 formTokenWithChars(T, TokenPtr, tok::html_greater); 643 State = LS_Normal; 644 return; 645 case '/': 646 TokenPtr++; 647 if (TokenPtr != CommentEnd && *TokenPtr == '>') { 648 TokenPtr++; 649 formTokenWithChars(T, TokenPtr, tok::html_slash_greater); 650 } else 651 formTextToken(T, TokenPtr); 652 653 State = LS_Normal; 654 return; 655 } 656 } 657 658 // Now look ahead and return to normal state if we don't see any HTML tokens 659 // ahead. 660 BufferPtr = skipWhitespace(BufferPtr, CommentEnd); 661 if (BufferPtr == CommentEnd) { 662 State = LS_Normal; 663 return; 664 } 665 666 C = *BufferPtr; 667 if (!isHTMLIdentifierStartingCharacter(C) && 668 C != '=' && C != '\"' && C != '\'' && C != '>') { 669 State = LS_Normal; 670 return; 671 } 672} 673 674void Lexer::setupAndLexHTMLEndTag(Token &T) { 675 assert(BufferPtr[0] == '<' && BufferPtr[1] == '/'); 676 677 const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd); 678 const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd); 679 StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin); 680 if (!isHTMLTagName(Name)) { 681 formTextToken(T, TagNameEnd); 682 return; 683 } 684 685 const char *End = skipWhitespace(TagNameEnd, CommentEnd); 686 687 formTokenWithChars(T, End, tok::html_end_tag); 688 T.setHTMLTagEndName(Name); 689 690 if (BufferPtr != CommentEnd && *BufferPtr == '>') 691 State = LS_HTMLEndTag; 692} 693 694void Lexer::lexHTMLEndTag(Token &T) { 695 assert(BufferPtr != CommentEnd && *BufferPtr == '>'); 696 697 formTokenWithChars(T, BufferPtr + 1, tok::html_greater); 698 State = LS_Normal; 699} 700 701Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags, 702 const CommandTraits &Traits, 703 SourceLocation FileLoc, 704 const char *BufferStart, const char *BufferEnd): 705 Allocator(Allocator), Diags(Diags), Traits(Traits), 706 BufferStart(BufferStart), BufferEnd(BufferEnd), 707 FileLoc(FileLoc), BufferPtr(BufferStart), 708 CommentState(LCS_BeforeComment), State(LS_Normal) { 709} 710 711void Lexer::lex(Token &T) { 712again: 713 switch (CommentState) { 714 case LCS_BeforeComment: 715 if (BufferPtr == BufferEnd) { 716 formTokenWithChars(T, BufferPtr, tok::eof); 717 return; 718 } 719 720 assert(*BufferPtr == '/'); 721 BufferPtr++; // Skip first slash. 722 switch(*BufferPtr) { 723 case '/': { // BCPL comment. 724 BufferPtr++; // Skip second slash. 725 726 if (BufferPtr != BufferEnd) { 727 // Skip Doxygen magic marker, if it is present. 728 // It might be missing because of a typo //< or /*<, or because we 729 // merged this non-Doxygen comment into a bunch of Doxygen comments 730 // around it: /** ... */ /* ... */ /** ... */ 731 const char C = *BufferPtr; 732 if (C == '/' || C == '!') 733 BufferPtr++; 734 } 735 736 // Skip less-than symbol that marks trailing comments. 737 // Skip it even if the comment is not a Doxygen one, because //< and /*< 738 // are frequent typos. 739 if (BufferPtr != BufferEnd && *BufferPtr == '<') 740 BufferPtr++; 741 742 CommentState = LCS_InsideBCPLComment; 743 if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine) 744 State = LS_Normal; 745 CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd); 746 goto again; 747 } 748 case '*': { // C comment. 749 BufferPtr++; // Skip star. 750 751 // Skip Doxygen magic marker. 752 const char C = *BufferPtr; 753 if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!') 754 BufferPtr++; 755 756 // Skip less-than symbol that marks trailing comments. 757 if (BufferPtr != BufferEnd && *BufferPtr == '<') 758 BufferPtr++; 759 760 CommentState = LCS_InsideCComment; 761 State = LS_Normal; 762 CommentEnd = findCCommentEnd(BufferPtr, BufferEnd); 763 goto again; 764 } 765 default: 766 llvm_unreachable("second character of comment should be '/' or '*'"); 767 } 768 769 case LCS_BetweenComments: { 770 // Consecutive comments are extracted only if there is only whitespace 771 // between them. So we can search for the start of the next comment. 772 const char *EndWhitespace = BufferPtr; 773 while(EndWhitespace != BufferEnd && *EndWhitespace != '/') 774 EndWhitespace++; 775 776 // Turn any whitespace between comments (and there is only whitespace 777 // between them -- guaranteed by comment extraction) into a newline. We 778 // have two newlines between C comments in total (first one was synthesized 779 // after a comment). 780 formTokenWithChars(T, EndWhitespace, tok::newline); 781 782 CommentState = LCS_BeforeComment; 783 break; 784 } 785 786 case LCS_InsideBCPLComment: 787 case LCS_InsideCComment: 788 if (BufferPtr != CommentEnd) { 789 lexCommentText(T); 790 break; 791 } else { 792 // Skip C comment closing sequence. 793 if (CommentState == LCS_InsideCComment) { 794 assert(BufferPtr[0] == '*' && BufferPtr[1] == '/'); 795 BufferPtr += 2; 796 assert(BufferPtr <= BufferEnd); 797 798 // Synthenize newline just after the C comment, regardless if there is 799 // actually a newline. 800 formTokenWithChars(T, BufferPtr, tok::newline); 801 802 CommentState = LCS_BetweenComments; 803 break; 804 } else { 805 // Don't synthesized a newline after BCPL comment. 806 CommentState = LCS_BetweenComments; 807 goto again; 808 } 809 } 810 } 811} 812 813StringRef Lexer::getSpelling(const Token &Tok, 814 const SourceManager &SourceMgr, 815 bool *Invalid) const { 816 SourceLocation Loc = Tok.getLocation(); 817 std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc); 818 819 bool InvalidTemp = false; 820 StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp); 821 if (InvalidTemp) { 822 *Invalid = true; 823 return StringRef(); 824 } 825 826 const char *Begin = File.data() + LocInfo.second; 827 return StringRef(Begin, Tok.getLength()); 828} 829 830} // end namespace comments 831} // end namespace clang 832 833