1//===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "clang/AST/CommentParser.h"
11#include "clang/AST/CommentCommandTraits.h"
12#include "clang/AST/CommentDiagnostic.h"
13#include "clang/AST/CommentSema.h"
14#include "clang/Basic/CharInfo.h"
15#include "clang/Basic/SourceManager.h"
16#include "llvm/Support/ErrorHandling.h"
17
18namespace clang {
19namespace comments {
20
21/// Re-lexes a sequence of tok::text tokens.
22class TextTokenRetokenizer {
23  llvm::BumpPtrAllocator &Allocator;
24  Parser &P;
25
26  /// This flag is set when there are no more tokens we can fetch from lexer.
27  bool NoMoreInterestingTokens;
28
29  /// Token buffer: tokens we have processed and lookahead.
30  SmallVector<Token, 16> Toks;
31
32  /// A position in \c Toks.
33  struct Position {
34    unsigned CurToken;
35    const char *BufferStart;
36    const char *BufferEnd;
37    const char *BufferPtr;
38    SourceLocation BufferStartLoc;
39  };
40
41  /// Current position in Toks.
42  Position Pos;
43
44  bool isEnd() const {
45    return Pos.CurToken >= Toks.size();
46  }
47
48  /// Sets up the buffer pointers to point to current token.
49  void setupBuffer() {
50    assert(!isEnd());
51    const Token &Tok = Toks[Pos.CurToken];
52
53    Pos.BufferStart = Tok.getText().begin();
54    Pos.BufferEnd = Tok.getText().end();
55    Pos.BufferPtr = Pos.BufferStart;
56    Pos.BufferStartLoc = Tok.getLocation();
57  }
58
59  SourceLocation getSourceLocation() const {
60    const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
61    return Pos.BufferStartLoc.getLocWithOffset(CharNo);
62  }
63
64  char peek() const {
65    assert(!isEnd());
66    assert(Pos.BufferPtr != Pos.BufferEnd);
67    return *Pos.BufferPtr;
68  }
69
70  void consumeChar() {
71    assert(!isEnd());
72    assert(Pos.BufferPtr != Pos.BufferEnd);
73    Pos.BufferPtr++;
74    if (Pos.BufferPtr == Pos.BufferEnd) {
75      Pos.CurToken++;
76      if (isEnd() && !addToken())
77        return;
78
79      assert(!isEnd());
80      setupBuffer();
81    }
82  }
83
84  /// Add a token.
85  /// Returns true on success, false if there are no interesting tokens to
86  /// fetch from lexer.
87  bool addToken() {
88    if (NoMoreInterestingTokens)
89      return false;
90
91    if (P.Tok.is(tok::newline)) {
92      // If we see a single newline token between text tokens, skip it.
93      Token Newline = P.Tok;
94      P.consumeToken();
95      if (P.Tok.isNot(tok::text)) {
96        P.putBack(Newline);
97        NoMoreInterestingTokens = true;
98        return false;
99      }
100    }
101    if (P.Tok.isNot(tok::text)) {
102      NoMoreInterestingTokens = true;
103      return false;
104    }
105
106    Toks.push_back(P.Tok);
107    P.consumeToken();
108    if (Toks.size() == 1)
109      setupBuffer();
110    return true;
111  }
112
113  void consumeWhitespace() {
114    while (!isEnd()) {
115      if (isWhitespace(peek()))
116        consumeChar();
117      else
118        break;
119    }
120  }
121
122  void formTokenWithChars(Token &Result,
123                          SourceLocation Loc,
124                          const char *TokBegin,
125                          unsigned TokLength,
126                          StringRef Text) {
127    Result.setLocation(Loc);
128    Result.setKind(tok::text);
129    Result.setLength(TokLength);
130#ifndef NDEBUG
131    Result.TextPtr = "<UNSET>";
132    Result.IntVal = 7;
133#endif
134    Result.setText(Text);
135  }
136
137public:
138  TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
139      Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
140    Pos.CurToken = 0;
141    addToken();
142  }
143
144  /// Extract a word -- sequence of non-whitespace characters.
145  bool lexWord(Token &Tok) {
146    if (isEnd())
147      return false;
148
149    Position SavedPos = Pos;
150
151    consumeWhitespace();
152    SmallString<32> WordText;
153    const char *WordBegin = Pos.BufferPtr;
154    SourceLocation Loc = getSourceLocation();
155    while (!isEnd()) {
156      const char C = peek();
157      if (!isWhitespace(C)) {
158        WordText.push_back(C);
159        consumeChar();
160      } else
161        break;
162    }
163    const unsigned Length = WordText.size();
164    if (Length == 0) {
165      Pos = SavedPos;
166      return false;
167    }
168
169    char *TextPtr = Allocator.Allocate<char>(Length + 1);
170
171    memcpy(TextPtr, WordText.c_str(), Length + 1);
172    StringRef Text = StringRef(TextPtr, Length);
173
174    formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
175    return true;
176  }
177
178  bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
179    if (isEnd())
180      return false;
181
182    Position SavedPos = Pos;
183
184    consumeWhitespace();
185    SmallString<32> WordText;
186    const char *WordBegin = Pos.BufferPtr;
187    SourceLocation Loc = getSourceLocation();
188    bool Error = false;
189    if (!isEnd()) {
190      const char C = peek();
191      if (C == OpenDelim) {
192        WordText.push_back(C);
193        consumeChar();
194      } else
195        Error = true;
196    }
197    char C = '\0';
198    while (!Error && !isEnd()) {
199      C = peek();
200      WordText.push_back(C);
201      consumeChar();
202      if (C == CloseDelim)
203        break;
204    }
205    if (!Error && C != CloseDelim)
206      Error = true;
207
208    if (Error) {
209      Pos = SavedPos;
210      return false;
211    }
212
213    const unsigned Length = WordText.size();
214    char *TextPtr = Allocator.Allocate<char>(Length + 1);
215
216    memcpy(TextPtr, WordText.c_str(), Length + 1);
217    StringRef Text = StringRef(TextPtr, Length);
218
219    formTokenWithChars(Tok, Loc, WordBegin,
220                       Pos.BufferPtr - WordBegin, Text);
221    return true;
222  }
223
224  /// Put back tokens that we didn't consume.
225  void putBackLeftoverTokens() {
226    if (isEnd())
227      return;
228
229    bool HavePartialTok = false;
230    Token PartialTok;
231    if (Pos.BufferPtr != Pos.BufferStart) {
232      formTokenWithChars(PartialTok, getSourceLocation(),
233                         Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
234                         StringRef(Pos.BufferPtr,
235                                   Pos.BufferEnd - Pos.BufferPtr));
236      HavePartialTok = true;
237      Pos.CurToken++;
238    }
239
240    P.putBack(llvm::makeArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
241    Pos.CurToken = Toks.size();
242
243    if (HavePartialTok)
244      P.putBack(PartialTok);
245  }
246};
247
248Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
249               const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
250               const CommandTraits &Traits):
251    L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
252    Traits(Traits) {
253  consumeToken();
254}
255
256void Parser::parseParamCommandArgs(ParamCommandComment *PC,
257                                   TextTokenRetokenizer &Retokenizer) {
258  Token Arg;
259  // Check if argument looks like direction specification: [dir]
260  // e.g., [in], [out], [in,out]
261  if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
262    S.actOnParamCommandDirectionArg(PC,
263                                    Arg.getLocation(),
264                                    Arg.getEndLocation(),
265                                    Arg.getText());
266
267  if (Retokenizer.lexWord(Arg))
268    S.actOnParamCommandParamNameArg(PC,
269                                    Arg.getLocation(),
270                                    Arg.getEndLocation(),
271                                    Arg.getText());
272}
273
274void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
275                                    TextTokenRetokenizer &Retokenizer) {
276  Token Arg;
277  if (Retokenizer.lexWord(Arg))
278    S.actOnTParamCommandParamNameArg(TPC,
279                                     Arg.getLocation(),
280                                     Arg.getEndLocation(),
281                                     Arg.getText());
282}
283
284void Parser::parseBlockCommandArgs(BlockCommandComment *BC,
285                                   TextTokenRetokenizer &Retokenizer,
286                                   unsigned NumArgs) {
287  typedef BlockCommandComment::Argument Argument;
288  Argument *Args =
289      new (Allocator.Allocate<Argument>(NumArgs)) Argument[NumArgs];
290  unsigned ParsedArgs = 0;
291  Token Arg;
292  while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
293    Args[ParsedArgs] = Argument(SourceRange(Arg.getLocation(),
294                                            Arg.getEndLocation()),
295                                Arg.getText());
296    ParsedArgs++;
297  }
298
299  S.actOnBlockCommandArgs(BC, llvm::makeArrayRef(Args, ParsedArgs));
300}
301
302BlockCommandComment *Parser::parseBlockCommand() {
303  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
304
305  ParamCommandComment *PC = 0;
306  TParamCommandComment *TPC = 0;
307  BlockCommandComment *BC = 0;
308  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
309  CommandMarkerKind CommandMarker =
310      Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
311  if (Info->IsParamCommand) {
312    PC = S.actOnParamCommandStart(Tok.getLocation(),
313                                  Tok.getEndLocation(),
314                                  Tok.getCommandID(),
315                                  CommandMarker);
316  } else if (Info->IsTParamCommand) {
317    TPC = S.actOnTParamCommandStart(Tok.getLocation(),
318                                    Tok.getEndLocation(),
319                                    Tok.getCommandID(),
320                                    CommandMarker);
321  } else {
322    BC = S.actOnBlockCommandStart(Tok.getLocation(),
323                                  Tok.getEndLocation(),
324                                  Tok.getCommandID(),
325                                  CommandMarker);
326  }
327  consumeToken();
328
329  if (isTokBlockCommand()) {
330    // Block command ahead.  We can't nest block commands, so pretend that this
331    // command has an empty argument.
332    ParagraphComment *Paragraph = S.actOnParagraphComment(None);
333    if (PC) {
334      S.actOnParamCommandFinish(PC, Paragraph);
335      return PC;
336    } else if (TPC) {
337      S.actOnTParamCommandFinish(TPC, Paragraph);
338      return TPC;
339    } else {
340      S.actOnBlockCommandFinish(BC, Paragraph);
341      return BC;
342    }
343  }
344
345  if (PC || TPC || Info->NumArgs > 0) {
346    // In order to parse command arguments we need to retokenize a few
347    // following text tokens.
348    TextTokenRetokenizer Retokenizer(Allocator, *this);
349
350    if (PC)
351      parseParamCommandArgs(PC, Retokenizer);
352    else if (TPC)
353      parseTParamCommandArgs(TPC, Retokenizer);
354    else
355      parseBlockCommandArgs(BC, Retokenizer, Info->NumArgs);
356
357    Retokenizer.putBackLeftoverTokens();
358  }
359
360  // If there's a block command ahead, we will attach an empty paragraph to
361  // this command.
362  bool EmptyParagraph = false;
363  if (isTokBlockCommand())
364    EmptyParagraph = true;
365  else if (Tok.is(tok::newline)) {
366    Token PrevTok = Tok;
367    consumeToken();
368    EmptyParagraph = isTokBlockCommand();
369    putBack(PrevTok);
370  }
371
372  ParagraphComment *Paragraph;
373  if (EmptyParagraph)
374    Paragraph = S.actOnParagraphComment(None);
375  else {
376    BlockContentComment *Block = parseParagraphOrBlockCommand();
377    // Since we have checked for a block command, we should have parsed a
378    // paragraph.
379    Paragraph = cast<ParagraphComment>(Block);
380  }
381
382  if (PC) {
383    S.actOnParamCommandFinish(PC, Paragraph);
384    return PC;
385  } else if (TPC) {
386    S.actOnTParamCommandFinish(TPC, Paragraph);
387    return TPC;
388  } else {
389    S.actOnBlockCommandFinish(BC, Paragraph);
390    return BC;
391  }
392}
393
394InlineCommandComment *Parser::parseInlineCommand() {
395  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
396
397  const Token CommandTok = Tok;
398  consumeToken();
399
400  TextTokenRetokenizer Retokenizer(Allocator, *this);
401
402  Token ArgTok;
403  bool ArgTokValid = Retokenizer.lexWord(ArgTok);
404
405  InlineCommandComment *IC;
406  if (ArgTokValid) {
407    IC = S.actOnInlineCommand(CommandTok.getLocation(),
408                              CommandTok.getEndLocation(),
409                              CommandTok.getCommandID(),
410                              ArgTok.getLocation(),
411                              ArgTok.getEndLocation(),
412                              ArgTok.getText());
413  } else {
414    IC = S.actOnInlineCommand(CommandTok.getLocation(),
415                              CommandTok.getEndLocation(),
416                              CommandTok.getCommandID());
417  }
418
419  Retokenizer.putBackLeftoverTokens();
420
421  return IC;
422}
423
424HTMLStartTagComment *Parser::parseHTMLStartTag() {
425  assert(Tok.is(tok::html_start_tag));
426  HTMLStartTagComment *HST =
427      S.actOnHTMLStartTagStart(Tok.getLocation(),
428                               Tok.getHTMLTagStartName());
429  consumeToken();
430
431  SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
432  while (true) {
433    switch (Tok.getKind()) {
434    case tok::html_ident: {
435      Token Ident = Tok;
436      consumeToken();
437      if (Tok.isNot(tok::html_equals)) {
438        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
439                                                       Ident.getHTMLIdent()));
440        continue;
441      }
442      Token Equals = Tok;
443      consumeToken();
444      if (Tok.isNot(tok::html_quoted_string)) {
445        Diag(Tok.getLocation(),
446             diag::warn_doc_html_start_tag_expected_quoted_string)
447          << SourceRange(Equals.getLocation());
448        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
449                                                       Ident.getHTMLIdent()));
450        while (Tok.is(tok::html_equals) ||
451               Tok.is(tok::html_quoted_string))
452          consumeToken();
453        continue;
454      }
455      Attrs.push_back(HTMLStartTagComment::Attribute(
456                              Ident.getLocation(),
457                              Ident.getHTMLIdent(),
458                              Equals.getLocation(),
459                              SourceRange(Tok.getLocation(),
460                                          Tok.getEndLocation()),
461                              Tok.getHTMLQuotedString()));
462      consumeToken();
463      continue;
464    }
465
466    case tok::html_greater:
467      S.actOnHTMLStartTagFinish(HST,
468                                S.copyArray(llvm::makeArrayRef(Attrs)),
469                                Tok.getLocation(),
470                                /* IsSelfClosing = */ false);
471      consumeToken();
472      return HST;
473
474    case tok::html_slash_greater:
475      S.actOnHTMLStartTagFinish(HST,
476                                S.copyArray(llvm::makeArrayRef(Attrs)),
477                                Tok.getLocation(),
478                                /* IsSelfClosing = */ true);
479      consumeToken();
480      return HST;
481
482    case tok::html_equals:
483    case tok::html_quoted_string:
484      Diag(Tok.getLocation(),
485           diag::warn_doc_html_start_tag_expected_ident_or_greater);
486      while (Tok.is(tok::html_equals) ||
487             Tok.is(tok::html_quoted_string))
488        consumeToken();
489      if (Tok.is(tok::html_ident) ||
490          Tok.is(tok::html_greater) ||
491          Tok.is(tok::html_slash_greater))
492        continue;
493
494      S.actOnHTMLStartTagFinish(HST,
495                                S.copyArray(llvm::makeArrayRef(Attrs)),
496                                SourceLocation(),
497                                /* IsSelfClosing = */ false);
498      return HST;
499
500    default:
501      // Not a token from an HTML start tag.  Thus HTML tag prematurely ended.
502      S.actOnHTMLStartTagFinish(HST,
503                                S.copyArray(llvm::makeArrayRef(Attrs)),
504                                SourceLocation(),
505                                /* IsSelfClosing = */ false);
506      bool StartLineInvalid;
507      const unsigned StartLine = SourceMgr.getPresumedLineNumber(
508                                                  HST->getLocation(),
509                                                  &StartLineInvalid);
510      bool EndLineInvalid;
511      const unsigned EndLine = SourceMgr.getPresumedLineNumber(
512                                                  Tok.getLocation(),
513                                                  &EndLineInvalid);
514      if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
515        Diag(Tok.getLocation(),
516             diag::warn_doc_html_start_tag_expected_ident_or_greater)
517          << HST->getSourceRange();
518      else {
519        Diag(Tok.getLocation(),
520             diag::warn_doc_html_start_tag_expected_ident_or_greater);
521        Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
522          << HST->getSourceRange();
523      }
524      return HST;
525    }
526  }
527}
528
529HTMLEndTagComment *Parser::parseHTMLEndTag() {
530  assert(Tok.is(tok::html_end_tag));
531  Token TokEndTag = Tok;
532  consumeToken();
533  SourceLocation Loc;
534  if (Tok.is(tok::html_greater)) {
535    Loc = Tok.getLocation();
536    consumeToken();
537  }
538
539  return S.actOnHTMLEndTag(TokEndTag.getLocation(),
540                           Loc,
541                           TokEndTag.getHTMLTagEndName());
542}
543
544BlockContentComment *Parser::parseParagraphOrBlockCommand() {
545  SmallVector<InlineContentComment *, 8> Content;
546
547  while (true) {
548    switch (Tok.getKind()) {
549    case tok::verbatim_block_begin:
550    case tok::verbatim_line_name:
551    case tok::eof:
552      assert(Content.size() != 0);
553      break; // Block content or EOF ahead, finish this parapgaph.
554
555    case tok::unknown_command:
556      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
557                                              Tok.getEndLocation(),
558                                              Tok.getUnknownCommandName()));
559      consumeToken();
560      continue;
561
562    case tok::backslash_command:
563    case tok::at_command: {
564      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
565      if (Info->IsBlockCommand) {
566        if (Content.size() == 0)
567          return parseBlockCommand();
568        break; // Block command ahead, finish this parapgaph.
569      }
570      if (Info->IsVerbatimBlockEndCommand) {
571        Diag(Tok.getLocation(),
572             diag::warn_verbatim_block_end_without_start)
573          << Tok.is(tok::at_command)
574          << Info->Name
575          << SourceRange(Tok.getLocation(), Tok.getEndLocation());
576        consumeToken();
577        continue;
578      }
579      if (Info->IsUnknownCommand) {
580        Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
581                                                Tok.getEndLocation(),
582                                                Info->getID()));
583        consumeToken();
584        continue;
585      }
586      assert(Info->IsInlineCommand);
587      Content.push_back(parseInlineCommand());
588      continue;
589    }
590
591    case tok::newline: {
592      consumeToken();
593      if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
594        consumeToken();
595        break; // Two newlines -- end of paragraph.
596      }
597      if (Content.size() > 0)
598        Content.back()->addTrailingNewline();
599      continue;
600    }
601
602    // Don't deal with HTML tag soup now.
603    case tok::html_start_tag:
604      Content.push_back(parseHTMLStartTag());
605      continue;
606
607    case tok::html_end_tag:
608      Content.push_back(parseHTMLEndTag());
609      continue;
610
611    case tok::text:
612      Content.push_back(S.actOnText(Tok.getLocation(),
613                                    Tok.getEndLocation(),
614                                    Tok.getText()));
615      consumeToken();
616      continue;
617
618    case tok::verbatim_block_line:
619    case tok::verbatim_block_end:
620    case tok::verbatim_line_text:
621    case tok::html_ident:
622    case tok::html_equals:
623    case tok::html_quoted_string:
624    case tok::html_greater:
625    case tok::html_slash_greater:
626      llvm_unreachable("should not see this token");
627    }
628    break;
629  }
630
631  return S.actOnParagraphComment(S.copyArray(llvm::makeArrayRef(Content)));
632}
633
634VerbatimBlockComment *Parser::parseVerbatimBlock() {
635  assert(Tok.is(tok::verbatim_block_begin));
636
637  VerbatimBlockComment *VB =
638      S.actOnVerbatimBlockStart(Tok.getLocation(),
639                                Tok.getVerbatimBlockID());
640  consumeToken();
641
642  // Don't create an empty line if verbatim opening command is followed
643  // by a newline.
644  if (Tok.is(tok::newline))
645    consumeToken();
646
647  SmallVector<VerbatimBlockLineComment *, 8> Lines;
648  while (Tok.is(tok::verbatim_block_line) ||
649         Tok.is(tok::newline)) {
650    VerbatimBlockLineComment *Line;
651    if (Tok.is(tok::verbatim_block_line)) {
652      Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
653                                      Tok.getVerbatimBlockText());
654      consumeToken();
655      if (Tok.is(tok::newline)) {
656        consumeToken();
657      }
658    } else {
659      // Empty line, just a tok::newline.
660      Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
661      consumeToken();
662    }
663    Lines.push_back(Line);
664  }
665
666  if (Tok.is(tok::verbatim_block_end)) {
667    const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
668    S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
669                               Info->Name,
670                               S.copyArray(llvm::makeArrayRef(Lines)));
671    consumeToken();
672  } else {
673    // Unterminated \\verbatim block
674    S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
675                               S.copyArray(llvm::makeArrayRef(Lines)));
676  }
677
678  return VB;
679}
680
681VerbatimLineComment *Parser::parseVerbatimLine() {
682  assert(Tok.is(tok::verbatim_line_name));
683
684  Token NameTok = Tok;
685  consumeToken();
686
687  SourceLocation TextBegin;
688  StringRef Text;
689  // Next token might not be a tok::verbatim_line_text if verbatim line
690  // starting command comes just before a newline or comment end.
691  if (Tok.is(tok::verbatim_line_text)) {
692    TextBegin = Tok.getLocation();
693    Text = Tok.getVerbatimLineText();
694  } else {
695    TextBegin = NameTok.getEndLocation();
696    Text = "";
697  }
698
699  VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
700                                                NameTok.getVerbatimLineID(),
701                                                TextBegin,
702                                                Text);
703  consumeToken();
704  return VL;
705}
706
707BlockContentComment *Parser::parseBlockContent() {
708  switch (Tok.getKind()) {
709  case tok::text:
710  case tok::unknown_command:
711  case tok::backslash_command:
712  case tok::at_command:
713  case tok::html_start_tag:
714  case tok::html_end_tag:
715    return parseParagraphOrBlockCommand();
716
717  case tok::verbatim_block_begin:
718    return parseVerbatimBlock();
719
720  case tok::verbatim_line_name:
721    return parseVerbatimLine();
722
723  case tok::eof:
724  case tok::newline:
725  case tok::verbatim_block_line:
726  case tok::verbatim_block_end:
727  case tok::verbatim_line_text:
728  case tok::html_ident:
729  case tok::html_equals:
730  case tok::html_quoted_string:
731  case tok::html_greater:
732  case tok::html_slash_greater:
733    llvm_unreachable("should not see this token");
734  }
735  llvm_unreachable("bogus token kind");
736}
737
738FullComment *Parser::parseFullComment() {
739  // Skip newlines at the beginning of the comment.
740  while (Tok.is(tok::newline))
741    consumeToken();
742
743  SmallVector<BlockContentComment *, 8> Blocks;
744  while (Tok.isNot(tok::eof)) {
745    Blocks.push_back(parseBlockContent());
746
747    // Skip extra newlines after paragraph end.
748    while (Tok.is(tok::newline))
749      consumeToken();
750  }
751  return S.actOnFullComment(S.copyArray(llvm::makeArrayRef(Blocks)));
752}
753
754} // end namespace comments
755} // end namespace clang
756