/usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/lib/Lex/Lexer.cpp

Bug Summary

File:	src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/lib/Lex/Lexer.cpp
Warning:	line 2440, column 26 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name Lexer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libclangLex/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/include -I /usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libclangLex/../include -I /usr/src/gnu/usr.bin/clang/libclangLex/obj -I /usr/src/gnu/usr.bin/clang/libclangLex/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libclangLex/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/lib/Lex/Lexer.cpp

/usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/lib/Lex/Lexer.cpp

→

1//===- Lexer.cpp - C Language Family Lexer --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9//  This file implements the Lexer and Token interfaces.
10//
11//===----------------------------------------------------------------------===//

13#include "clang/Lex/Lexer.h"
14#include "UnicodeCharSets.h"
15#include "clang/Basic/CharInfo.h"
16#include "clang/Basic/Diagnostic.h"
17#include "clang/Basic/IdentifierTable.h"
18#include "clang/Basic/LLVM.h"
19#include "clang/Basic/LangOptions.h"
20#include "clang/Basic/SourceLocation.h"
21#include "clang/Basic/SourceManager.h"
22#include "clang/Basic/TokenKinds.h"
23#include "clang/Lex/LexDiagnostic.h"
24#include "clang/Lex/LiteralSupport.h"
25#include "clang/Lex/MultipleIncludeOpt.h"
26#include "clang/Lex/Preprocessor.h"
27#include "clang/Lex/PreprocessorOptions.h"
28#include "clang/Lex/Token.h"
29#include "llvm/ADT/None.h"
30#include "llvm/ADT/Optional.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/ADT/StringRef.h"
34#include "llvm/ADT/StringSwitch.h"
35#include "llvm/Support/Compiler.h"
36#include "llvm/Support/ConvertUTF.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Support/MemoryBufferRef.h"
39#include "llvm/Support/NativeFormatting.h"
40#include "llvm/Support/UnicodeCharRanges.h"
41#include <algorithm>
42#include <cassert>
43#include <cstddef>
44#include <cstdint>
45#include <cstring>
46#include <string>
47#include <tuple>
48#include <utility>

50using namespace clang;

52//===----------------------------------------------------------------------===//
53// Token Class Implementation
54//===----------------------------------------------------------------------===//

56/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
57bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
if (isAnnotation())
  return false;
if (IdentifierInfo *II = getIdentifierInfo())
  return II->getObjCKeywordID() == objcKey;
return false;
63}

65/// getObjCKeywordID - Return the ObjC keyword kind.
66tok::ObjCKeywordKind Token::getObjCKeywordID() const {
if (isAnnotation())
  return tok::objc_not_keyword;
IdentifierInfo *specId = getIdentifierInfo();
return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
71}

73//===----------------------------------------------------------------------===//
74// Lexer Class Implementation
75//===----------------------------------------------------------------------===//

77void Lexer::anchor() {}

79void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
                    const char *BufEnd) {
BufferStart = BufStart;
BufferPtr = BufPtr;
BufferEnd = BufEnd;

assert(BufEnd[0] == 0 &&((void)0)
       "We assume that the input buffer has a null character at the end"((void)0)
       " to simplify lexing!")((void)0);

// Check whether we have a BOM in the beginning of the buffer. If yes - act
// accordingly. Right now we support only UTF-8 with and without BOM, so, just
// skip the UTF-8 BOM if it's present.
if (BufferStart == BufferPtr) {
  // Determine the size of the BOM.
  StringRef Buf(BufferStart, BufferEnd - BufferStart);
  size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
    .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM
    .Default(0);

  // Skip the BOM.
  BufferPtr += BOMLength;
}

Is_PragmaLexer = false;
CurrentConflictMarkerState = CMK_None;

// Start of the file is a start of line.
IsAtStartOfLine = true;
IsAtPhysicalStartOfLine = true;

HasLeadingSpace = false;
HasLeadingEmptyMacro = false;

// We are not after parsing a #.
ParsingPreprocessorDirective = false;

// We are not after parsing #include.
ParsingFilename = false;

// We are not in raw mode.  Raw mode disables diagnostics and interpretation
// of tokens (e.g. identifiers, thus disabling macro expansion).  It is used
// to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
// or otherwise skipping over tokens.
LexingRawMode = false;

// Default to not keeping comments.
ExtendedTokenMode = 0;

NewLinePtr = nullptr;
129}

131/// Lexer constructor - Create a new lexer object for the specified buffer
132/// with the specified preprocessor managing the lexing process.  This lexer
133/// assumes that the associated file buffer and Preprocessor objects will
134/// outlive it, so it doesn't take ownership of either of them.
135Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
           Preprocessor &PP)
  : PreprocessorLexer(&PP, FID),
    FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
    LangOpts(PP.getLangOpts()) {
InitLexer(InputFile.getBufferStart(), InputFile.getBufferStart(),
          InputFile.getBufferEnd());

resetExtendedTokenMode();
144}

146/// Lexer constructor - Create a new raw lexer object.  This object is only
147/// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the text
148/// range will outlive it, so it doesn't take ownership of it.
149Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
           const char *BufStart, const char *BufPtr, const char *BufEnd)
  : FileLoc(fileloc), LangOpts(langOpts) {
InitLexer(BufStart, BufPtr, BufEnd);

// We *are* in raw mode.
LexingRawMode = true;
156}

158/// Lexer constructor - Create a new raw lexer object.  This object is only
159/// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the text
160/// range will outlive it, so it doesn't take ownership of it.
161Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
           const SourceManager &SM, const LangOptions &langOpts)
  : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
          FromFile.getBufferStart(), FromFile.getBufferEnd()) {}

166void Lexer::resetExtendedTokenMode() {
assert(PP && "Cannot reset token mode without a preprocessor")((void)0);
if (LangOpts.TraditionalCPP)
  SetKeepWhitespaceMode(true);
else
  SetCommentRetentionState(PP->getCommentRetentionState());
172}

174/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
175/// _Pragma expansion.  This has a variety of magic semantics that this method
176/// sets up.  It returns a new'd Lexer that must be delete'd when done.
177///
178/// On entrance to this routine, TokStartLoc is a macro location which has a
179/// spelling loc that indicates the bytes to be lexed for the token and an
180/// expansion location that indicates where all lexed tokens should be
181/// "expanded from".
182///
183/// TODO: It would really be nice to make _Pragma just be a wrapper around a
184/// normal lexer that remaps tokens as they fly by.  This would require making
185/// Preprocessor::Lex virtual.  Given that, we could just dump in a magic lexer
186/// interface that could handle this stuff.  This would pull GetMappedTokenLoc
187/// out of the critical path of the lexer!
188///
189Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
                               SourceLocation ExpansionLocStart,
                               SourceLocation ExpansionLocEnd,
                               unsigned TokLen, Preprocessor &PP) {
SourceManager &SM = PP.getSourceManager();

// Create the lexer as if we were going to lex the file normally.
FileID SpellingFID = SM.getFileID(SpellingLoc);
llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(SpellingFID);
Lexer *L = new Lexer(SpellingFID, InputFile, PP);

// Now that the lexer is created, change the start/end locations so that we
// just lex the subsection of the file that we want.  This is lexing from a
// scratch buffer.
const char *StrData = SM.getCharacterData(SpellingLoc);

L->BufferPtr = StrData;
L->BufferEnd = StrData+TokLen;
assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!")((void)0);

// Set the SourceLocation with the remapping information.  This ensures that
// GetMappedTokenLoc will remap the tokens as they are lexed.
L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID),
                                   ExpansionLocStart,
                                   ExpansionLocEnd, TokLen);

// Ensure that the lexer thinks it is inside a directive, so that end \n will
// return an EOD token.
L->ParsingPreprocessorDirective = true;

// This lexer really is for _Pragma.
L->Is_PragmaLexer = true;
return L;
222}

224bool Lexer::skipOver(unsigned NumBytes) {
IsAtPhysicalStartOfLine = true;
IsAtStartOfLine = true;
if ((BufferPtr + NumBytes) > BufferEnd)
  return true;
BufferPtr += NumBytes;
return false;
231}

233template <typename T> static void StringifyImpl(T &Str, char Quote) {
typename T::size_type i = 0, e = Str.size();
while (i < e) {
  if (Str[i] == '\\' || Str[i] == Quote) {
    Str.insert(Str.begin() + i, '\\');
    i += 2;
    ++e;
  } else if (Str[i] == '\n' || Str[i] == '\r') {
    // Replace '\r\n' and '\n\r' to '\\' followed by 'n'.
    if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') &&
        Str[i] != Str[i + 1]) {
      Str[i] = '\\';
      Str[i + 1] = 'n';
    } else {
      // Replace '\n' and '\r' to '\\' followed by 'n'.
      Str[i] = '\\';
      Str.insert(Str.begin() + i + 1, 'n');
      ++e;
    }
    i += 2;
  } else
    ++i;
}
256}

258std::string Lexer::Stringify(StringRef Str, bool Charify) {
std::string Result = std::string(Str);
char Quote = Charify ? '\'' : '"';
StringifyImpl(Result, Quote);
return Result;
263}

265void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, '"'); }

267//===----------------------------------------------------------------------===//
268// Token Spelling
269//===----------------------------------------------------------------------===//

271/// Slow case of getSpelling. Extract the characters comprising the
272/// spelling of this token from the provided input buffer.
273static size_t getSpellingSlow(const Token &Tok, const char *BufPtr,
                            const LangOptions &LangOpts, char *Spelling) {
assert(Tok.needsCleaning() && "getSpellingSlow called on simple token")((void)0);

size_t Length = 0;
const char *BufEnd = BufPtr + Tok.getLength();

if (tok::isStringLiteral(Tok.getKind())) {
  // Munch the encoding-prefix and opening double-quote.
  while (BufPtr < BufEnd) {
    unsigned Size;
    Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
    BufPtr += Size;

    if (Spelling[Length - 1] == '"')
      break;
  }

  // Raw string literals need special handling; trigraph expansion and line
  // splicing do not occur within their d-char-sequence nor within their
  // r-char-sequence.
  if (Length >= 2 &&
      Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') {
    // Search backwards from the end of the token to find the matching closing
    // quote.
    const char *RawEnd = BufEnd;
    do --RawEnd; while (*RawEnd != '"');
    size_t RawLength = RawEnd - BufPtr + 1;

    // Everything between the quotes is included verbatim in the spelling.
    memcpy(Spelling + Length, BufPtr, RawLength);
    Length += RawLength;
    BufPtr += RawLength;

    // The rest of the token is lexed normally.
  }
}

while (BufPtr < BufEnd) {
  unsigned Size;
  Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts);
  BufPtr += Size;
}

assert(Length < Tok.getLength() &&((void)0)
       "NeedsCleaning flag set on token that didn't need cleaning!")((void)0);
return Length;
320}

322/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
323/// token are the characters used to represent the token in the source file
324/// after trigraph expansion and escaped-newline folding.  In particular, this
325/// wants to get the true, uncanonicalized, spelling of things like digraphs
326/// UCNs, etc.
327StringRef Lexer::getSpelling(SourceLocation loc,
                           SmallVectorImpl<char> &buffer,
                           const SourceManager &SM,
                           const LangOptions &options,
                           bool *invalid) {
// Break down the source location.
std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);

// Try to the load the file buffer.
bool invalidTemp = false;
StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
if (invalidTemp) {
  if (invalid) *invalid = true;
  return {};
}

const char *tokenBegin = file.data() + locInfo.second;

// Lex from the start of the given location.
Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options,
            file.begin(), tokenBegin, file.end());
Token token;
lexer.LexFromRawLexer(token);

unsigned length = token.getLength();

// Common case:  no need for cleaning.
if (!token.needsCleaning())
  return StringRef(tokenBegin, length);

// Hard case, we need to relex the characters into the string.
buffer.resize(length);
buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data()));
return StringRef(buffer.data(), buffer.size());
361}

363/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
364/// token are the characters used to represent the token in the source file
365/// after trigraph expansion and escaped-newline folding.  In particular, this
366/// wants to get the true, uncanonicalized, spelling of things like digraphs
367/// UCNs, etc.
368std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
                             const LangOptions &LangOpts, bool *Invalid) {
assert((int)Tok.getLength() >= 0 && "Token character range is bogus!")((void)0);

bool CharDataInvalid = false;
const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
                                                  &CharDataInvalid);
if (Invalid)
  *Invalid = CharDataInvalid;
if (CharDataInvalid)
  return {};

// If this token contains nothing interesting, return it directly.
if (!Tok.needsCleaning())
  return std::string(TokStart, TokStart + Tok.getLength());

std::string Result;
Result.resize(Tok.getLength());
Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin()));
return Result;
388}

390/// getSpelling - This method is used to get the spelling of a token into a
391/// preallocated buffer, instead of as an std::string.  The caller is required
392/// to allocate enough space for the token, which is guaranteed to be at least
393/// Tok.getLength() bytes long.  The actual length of the token is returned.
394///
395/// Note that this method may do two possible things: it may either fill in
396/// the buffer specified with characters, or it may *change the input pointer*
397/// to point to a constant buffer with the data already in it (avoiding a
398/// copy).  The caller is not allowed to modify the returned buffer pointer
399/// if an internal buffer is returned.
400unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
                          const SourceManager &SourceMgr,
                          const LangOptions &LangOpts, bool *Invalid) {
assert((int)Tok.getLength() >= 0 && "Token character range is bogus!")((void)0);

const char *TokStart = nullptr;
// NOTE: this has to be checked *before* testing for an IdentifierInfo.
if (Tok.is(tok::raw_identifier))
  TokStart = Tok.getRawIdentifier().data();
else if (!Tok.hasUCN()) {
  if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
    // Just return the string from the identifier table, which is very quick.
    Buffer = II->getNameStart();
    return II->getLength();
  }
}

// NOTE: this can be checked even after testing for an IdentifierInfo.
if (Tok.isLiteral())
  TokStart = Tok.getLiteralData();

if (!TokStart) {
  // Compute the start of the token in the input lexer buffer.
  bool CharDataInvalid = false;
  TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
  if (Invalid)
    *Invalid = CharDataInvalid;
  if (CharDataInvalid) {
    Buffer = "";
    return 0;
  }
}

// If this token contains nothing interesting, return it directly.
if (!Tok.needsCleaning()) {
  Buffer = TokStart;
  return Tok.getLength();
}

// Otherwise, hard case, relex the characters into the string.
return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer));
441}

443/// MeasureTokenLength - Relex the token at the specified location and return
444/// its length in bytes in the input file.  If the token needs cleaning (e.g.
445/// includes a trigraph or an escaped newline) then this count includes bytes
446/// that are part of that.
447unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
                                 const SourceManager &SM,
                                 const LangOptions &LangOpts) {
Token TheTok;
if (getRawToken(Loc, TheTok, SM, LangOpts))
  return 0;
return TheTok.getLength();
454}

456/// Relex the token at the specified location.
457/// \returns true if there was a failure, false on success.
458bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
                      const SourceManager &SM,
                      const LangOptions &LangOpts,
                      bool IgnoreWhiteSpace) {
// TODO: this could be special cased for common tokens like identifiers, ')',
// etc to make this faster, if it mattered.  Just look at StrData[0] to handle
// all obviously single-char tokens.  This could use
// Lexer::isObviouslySimpleCharacter for example to handle identifiers or
// something.

// If this comes from a macro expansion, we really do want the macro name, not
// the token this macro expanded to.
Loc = SM.getExpansionLoc(Loc);
std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
bool Invalid = false;
StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
if (Invalid)
  return true;

const char *StrData = Buffer.data()+LocInfo.second;

if (!IgnoreWhiteSpace && isWhitespace(StrData[0]))
  return true;

// Create a lexer starting at the beginning of this token.
Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
               Buffer.begin(), StrData, Buffer.end());
TheLexer.SetCommentRetentionState(true);
TheLexer.LexFromRawLexer(Result);
return false;
488}

490/// Returns the pointer that points to the beginning of line that contains
491/// the given offset, or null if the offset if invalid.
492static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) {
const char *BufStart = Buffer.data();
if (Offset >= Buffer.size())
  return nullptr;

const char *LexStart = BufStart + Offset;
for (; LexStart != BufStart; --LexStart) {
  if (isVerticalWhitespace(LexStart[0]) &&
      !Lexer::isNewLineEscaped(BufStart, LexStart)) {
    // LexStart should point at first character of logical line.
    ++LexStart;
    break;
  }
}
return LexStart;
507}

509static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
                                            const SourceManager &SM,
                                            const LangOptions &LangOpts) {
assert(Loc.isFileID())((void)0);
std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
if (LocInfo.first.isInvalid())
  return Loc;

bool Invalid = false;
StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
if (Invalid)
  return Loc;

// Back up from the current location until we hit the beginning of a line
// (or the buffer). We'll relex from that point.
const char *StrData = Buffer.data() + LocInfo.second;
const char *LexStart = findBeginningOfLine(Buffer, LocInfo.second);
if (!LexStart || LexStart == StrData)
  return Loc;

// Create a lexer starting at the beginning of this token.
SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second);
Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart,
               Buffer.end());
TheLexer.SetCommentRetentionState(true);

// Lex tokens until we find the token that contains the source location.
Token TheTok;
do {
  TheLexer.LexFromRawLexer(TheTok);

  if (TheLexer.getBufferLocation() > StrData) {
    // Lexing this token has taken the lexer past the source location we're
    // looking for. If the current token encompasses our source location,
    // return the beginning of that token.
    if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
      return TheTok.getLocation();

    // We ended up skipping over the source location entirely, which means
    // that it points into whitespace. We're done here.
    break;
  }
} while (TheTok.getKind() != tok::eof);

// We've passed our source location; just return the original source location.
return Loc;
555}

557SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
                                        const SourceManager &SM,
                                        const LangOptions &LangOpts) {
if (Loc.isFileID())
  return getBeginningOfFileToken(Loc, SM, LangOpts);

if (!SM.isMacroArgExpansion(Loc))
  return Loc;

SourceLocation FileLoc = SM.getSpellingLoc(Loc);
SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts);
std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc);
std::pair<FileID, unsigned> BeginFileLocInfo =
    SM.getDecomposedLoc(BeginFileLoc);
assert(FileLocInfo.first == BeginFileLocInfo.first &&((void)0)
       FileLocInfo.second >= BeginFileLocInfo.second)((void)0);
return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second);
574}

576namespace {

578enum PreambleDirectiveKind {
PDK_Skipped,
PDK_Unknown
581};

583} // namespace

585PreambleBounds Lexer::ComputePreamble(StringRef Buffer,
                                    const LangOptions &LangOpts,
                                    unsigned MaxLines) {
// Create a lexer starting at the beginning of the file. Note that we use a
// "fake" file source location at offset 1 so that the lexer will track our
// position within the file.
const SourceLocation::UIntTy StartOffset = 1;
SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset);
Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(),
               Buffer.end());
TheLexer.SetCommentRetentionState(true);

bool InPreprocessorDirective = false;
Token TheTok;
SourceLocation ActiveCommentLoc;

unsigned MaxLineOffset = 0;
if (MaxLines) {
  const char *CurPtr = Buffer.begin();
  unsigned CurLine = 0;
  while (CurPtr != Buffer.end()) {
    char ch = *CurPtr++;
    if (ch == '\n') {
      ++CurLine;
      if (CurLine == MaxLines)
        break;
    }
  }
  if (CurPtr != Buffer.end())
    MaxLineOffset = CurPtr - Buffer.begin();
}

do {
  TheLexer.LexFromRawLexer(TheTok);

  if (InPreprocessorDirective) {
    // If we've hit the end of the file, we're done.
    if (TheTok.getKind() == tok::eof) {
      break;
    }

    // If we haven't hit the end of the preprocessor directive, skip this
    // token.
    if (!TheTok.isAtStartOfLine())
      continue;

    // We've passed the end of the preprocessor directive, and will look
    // at this token again below.
    InPreprocessorDirective = false;
  }

  // Keep track of the # of lines in the preamble.
  if (TheTok.isAtStartOfLine()) {
    unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;

    // If we were asked to limit the number of lines in the preamble,
    // and we're about to exceed that limit, we're done.
    if (MaxLineOffset && TokOffset >= MaxLineOffset)
      break;
  }

  // Comments are okay; skip over them.
  if (TheTok.getKind() == tok::comment) {
    if (ActiveCommentLoc.isInvalid())
      ActiveCommentLoc = TheTok.getLocation();
    continue;
  }

  if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
    // This is the start of a preprocessor directive.
    Token HashTok = TheTok;
    InPreprocessorDirective = true;
    ActiveCommentLoc = SourceLocation();

    // Figure out which directive this is. Since we're lexing raw tokens,
    // we don't have an identifier table available. Instead, just look at
    // the raw identifier to recognize and categorize preprocessor directives.
    TheLexer.LexFromRawLexer(TheTok);
    if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
      StringRef Keyword = TheTok.getRawIdentifier();
      PreambleDirectiveKind PDK
        = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
            .Case("include", PDK_Skipped)
            .Case("__include_macros", PDK_Skipped)
            .Case("define", PDK_Skipped)
            .Case("undef", PDK_Skipped)
            .Case("line", PDK_Skipped)
            .Case("error", PDK_Skipped)
            .Case("pragma", PDK_Skipped)
            .Case("import", PDK_Skipped)
            .Case("include_next", PDK_Skipped)
            .Case("warning", PDK_Skipped)
            .Case("ident", PDK_Skipped)
            .Case("sccs", PDK_Skipped)
            .Case("assert", PDK_Skipped)
            .Case("unassert", PDK_Skipped)
            .Case("if", PDK_Skipped)
            .Case("ifdef", PDK_Skipped)
            .Case("ifndef", PDK_Skipped)
            .Case("elif", PDK_Skipped)
            .Case("elifdef", PDK_Skipped)
            .Case("elifndef", PDK_Skipped)
            .Case("else", PDK_Skipped)
            .Case("endif", PDK_Skipped)
            .Default(PDK_Unknown);

      switch (PDK) {
      case PDK_Skipped:
        continue;

      case PDK_Unknown:
        // We don't know what this directive is; stop at the '#'.
        break;
      }
    }

    // We only end up here if we didn't recognize the preprocessor
    // directive or it was one that can't occur in the preamble at this
    // point. Roll back the current token to the location of the '#'.
    TheTok = HashTok;
  }

  // We hit a token that we don't recognize as being in the
  // "preprocessing only" part of the file, so we're no longer in
  // the preamble.
  break;
} while (true);

SourceLocation End;
if (ActiveCommentLoc.isValid())
  End = ActiveCommentLoc; // don't truncate a decl comment.
else
  End = TheTok.getLocation();

return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(),
                      TheTok.isAtStartOfLine());
721}

723unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo,
                                   const SourceManager &SM,
                                   const LangOptions &LangOpts) {
// Figure out how many physical characters away the specified expansion
// character is.  This needs to take into consideration newlines and
// trigraphs.
bool Invalid = false;
const char *TokPtr = SM.getCharacterData(TokStart, &Invalid);

// If they request the first char of the token, we're trivially done.
if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
  return 0;

unsigned PhysOffset = 0;

// The usual case is that tokens don't contain anything interesting.  Skip
// over the uninteresting characters.  If a token only consists of simple
// chars, this method is extremely fast.
while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
  if (CharNo == 0)
    return PhysOffset;
  ++TokPtr;
  --CharNo;
  ++PhysOffset;
}

// If we have a character that may be a trigraph or escaped newline, use a
// lexer to parse it correctly.
for (; CharNo; --CharNo) {
  unsigned Size;
  Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts);
  TokPtr += Size;
  PhysOffset += Size;
}

// Final detail: if we end up on an escaped newline, we want to return the
// location of the actual byte of the token.  For example foo\<newline>bar
// advanced by 3 should return the location of b, not of \\.  One compounding
// detail of this is that the escape may be made by a trigraph.
if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
  PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;

return PhysOffset;
766}

768/// Computes the source location just past the end of the
769/// token at this source location.
770///
771/// This routine can be used to produce a source location that
772/// points just past the end of the token referenced by \p Loc, and
773/// is generally used when a diagnostic needs to point just after a
774/// token where it expected something different that it received. If
775/// the returned source location would not be meaningful (e.g., if
776/// it points into a macro), this routine returns an invalid
777/// source location.
778///
779/// \param Offset an offset from the end of the token, where the source
780/// location should refer to. The default offset (0) produces a source
781/// location pointing just past the end of the token; an offset of 1 produces
782/// a source location pointing to the last character in the token, etc.
783SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
                                        const SourceManager &SM,
                                        const LangOptions &LangOpts) {
if (Loc.isInvalid())
  return {};

if (Loc.isMacroID()) {
  if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
    return {}; // Points inside the macro expansion.
}

unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
if (Len > Offset)
  Len = Len - Offset;
else
  return Loc;

return Loc.getLocWithOffset(Len);
801}

803/// Returns true if the given MacroID location points at the first
804/// token of the macro expansion.
805bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
                                    const SourceManager &SM,
                                    const LangOptions &LangOpts,
                                    SourceLocation *MacroBegin) {
assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc")((void)0);

SourceLocation expansionLoc;
if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc))
  return false;

if (expansionLoc.isFileID()) {
  // No other macro expansions, this is the first.
  if (MacroBegin)
    *MacroBegin = expansionLoc;
  return true;
}

return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin);
823}

825/// Returns true if the given MacroID location points at the last
826/// token of the macro expansion.
827bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
                                  const SourceManager &SM,
                                  const LangOptions &LangOpts,
                                  SourceLocation *MacroEnd) {
assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc")((void)0);

SourceLocation spellLoc = SM.getSpellingLoc(loc);
unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts);
if (tokLen == 0)
  return false;

SourceLocation afterLoc = loc.getLocWithOffset(tokLen);
SourceLocation expansionLoc;
if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc))
  return false;

if (expansionLoc.isFileID()) {
  // No other macro expansions.
  if (MacroEnd)
    *MacroEnd = expansionLoc;
  return true;
}

return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd);
851}

853static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
                                           const SourceManager &SM,
                                           const LangOptions &LangOpts) {
SourceLocation Begin = Range.getBegin();
SourceLocation End = Range.getEnd();
assert(Begin.isFileID() && End.isFileID())((void)0);
if (Range.isTokenRange()) {
  End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts);
  if (End.isInvalid())
    return {};
}

// Break down the source locations.
FileID FID;
unsigned BeginOffs;
std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
if (FID.isInvalid())
  return {};

unsigned EndOffs;
if (!SM.isInFileID(End, FID, &EndOffs) ||
    BeginOffs > EndOffs)
  return {};

return CharSourceRange::getCharRange(Begin, End);
878}

880// Assumes that `Loc` is in an expansion.
881static bool isInExpansionTokenRange(const SourceLocation Loc,
                                  const SourceManager &SM) {
return SM.getSLocEntry(SM.getFileID(Loc))
    .getExpansion()
    .isExpansionTokenRange();
886}

888CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
                                       const SourceManager &SM,
                                       const LangOptions &LangOpts) {
SourceLocation Begin = Range.getBegin();
SourceLocation End = Range.getEnd();
if (Begin.isInvalid() || End.isInvalid())
  return {};

if (Begin.isFileID() && End.isFileID())
  return makeRangeFromFileLocs(Range, SM, LangOpts);

if (Begin.isMacroID() && End.isFileID()) {
  if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin))
    return {};
  Range.setBegin(Begin);
  return makeRangeFromFileLocs(Range, SM, LangOpts);
}

if (Begin.isFileID() && End.isMacroID()) {
  if (Range.isTokenRange()) {
    if (!isAtEndOfMacroExpansion(End, SM, LangOpts, &End))
      return {};
    // Use the *original* end, not the expanded one in `End`.
    Range.setTokenRange(isInExpansionTokenRange(Range.getEnd(), SM));
  } else if (!isAtStartOfMacroExpansion(End, SM, LangOpts, &End))
    return {};
  Range.setEnd(End);
  return makeRangeFromFileLocs(Range, SM, LangOpts);
}

assert(Begin.isMacroID() && End.isMacroID())((void)0);
SourceLocation MacroBegin, MacroEnd;
if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) &&
    ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts,
                                                      &MacroEnd)) ||
     (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts,
                                                       &MacroEnd)))) {
  Range.setBegin(MacroBegin);
  Range.setEnd(MacroEnd);
  // Use the *original* `End`, not the expanded one in `MacroEnd`.
  if (Range.isTokenRange())
    Range.setTokenRange(isInExpansionTokenRange(End, SM));
  return makeRangeFromFileLocs(Range, SM, LangOpts);
}

bool Invalid = false;
const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin),
                                                      &Invalid);
if (Invalid)
  return {};

if (BeginEntry.getExpansion().isMacroArgExpansion()) {
  const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End),
                                                      &Invalid);
  if (Invalid)
    return {};

  if (EndEntry.getExpansion().isMacroArgExpansion() &&
      BeginEntry.getExpansion().getExpansionLocStart() ==
          EndEntry.getExpansion().getExpansionLocStart()) {
    Range.setBegin(SM.getImmediateSpellingLoc(Begin));
    Range.setEnd(SM.getImmediateSpellingLoc(End));
    return makeFileCharRange(Range, SM, LangOpts);
  }
}

return {};
955}

957StringRef Lexer::getSourceText(CharSourceRange Range,
                             const SourceManager &SM,
                             const LangOptions &LangOpts,
                             bool *Invalid) {
Range = makeFileCharRange(Range, SM, LangOpts);
if (Range.isInvalid()) {
  if (Invalid) *Invalid = true;
  return {};
}

// Break down the source location.
std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin());
if (beginInfo.first.isInvalid()) {
  if (Invalid) *Invalid = true;
  return {};
}

unsigned EndOffs;
if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) ||
    beginInfo.second > EndOffs) {
  if (Invalid) *Invalid = true;
  return {};
}

// Try to the load the file buffer.
bool invalidTemp = false;
StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp);
if (invalidTemp) {
  if (Invalid) *Invalid = true;
  return {};
}

if (Invalid) *Invalid = false;
return file.substr(beginInfo.second, EndOffs - beginInfo.second);
991}

993StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
                                     const SourceManager &SM,
                                     const LangOptions &LangOpts) {
assert(Loc.isMacroID() && "Only reasonable to call this on macros")((void)0);

// Find the location of the immediate macro expansion.
while (true) {
  FileID FID = SM.getFileID(Loc);
  const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
  const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
  Loc = Expansion.getExpansionLocStart();
  if (!Expansion.isMacroArgExpansion())
    break;

  // For macro arguments we need to check that the argument did not come
  // from an inner macro, e.g: "MAC1( MAC2(foo) )"

  // Loc points to the argument id of the macro definition, move to the
  // macro expansion.
  Loc = SM.getImmediateExpansionRange(Loc).getBegin();
  SourceLocation SpellLoc = Expansion.getSpellingLoc();
  if (SpellLoc.isFileID())
    break; // No inner macro.

  // If spelling location resides in the same FileID as macro expansion
  // location, it means there is no inner macro.
  FileID MacroFID = SM.getFileID(Loc);
  if (SM.isInFileID(SpellLoc, MacroFID))
    break;

  // Argument came from inner macro.
  Loc = SpellLoc;
}

// Find the spelling location of the start of the non-argument expansion
// range. This is where the macro name was spelled in order to begin
// expanding this macro.
Loc = SM.getSpellingLoc(Loc);

// Dig out the buffer where the macro name was spelled and the extents of the
// name so that we can render it into the expansion note.
std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
1038}

1040StringRef Lexer::getImmediateMacroNameForDiagnostics(
  SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) {
assert(Loc.isMacroID() && "Only reasonable to call this on macros")((void)0);
// Walk past macro argument expansions.
while (SM.isMacroArgExpansion(Loc))
  Loc = SM.getImmediateExpansionRange(Loc).getBegin();

// If the macro's spelling has no FileID, then it's actually a token paste
// or stringization (or similar) and not a macro at all.
if (!SM.getFileEntryForID(SM.getFileID(SM.getSpellingLoc(Loc))))
  return {};

// Find the spelling location of the start of the non-argument expansion
// range. This is where the macro name was spelled in order to begin
// expanding this macro.
Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).getBegin());

// Dig out the buffer where the macro name was spelled and the extents of the
// name so that we can render it into the expansion note.
std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
1063}

1065bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) {
return isIdentifierBody(c, LangOpts.DollarIdents);
1067}

1069bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) {
assert(isVerticalWhitespace(Str[0]))((void)0);
if (Str - 1 < BufferStart)
  return false;

if ((Str[0] == '\n' && Str[-1] == '\r') ||
    (Str[0] == '\r' && Str[-1] == '\n')) {
  if (Str - 2 < BufferStart)
    return false;
  --Str;
}
--Str;

// Rewind to first non-space character:
while (Str > BufferStart && isHorizontalWhitespace(*Str))
  --Str;

return *Str == '\\';
1087}

1089StringRef Lexer::getIndentationForLine(SourceLocation Loc,
                                     const SourceManager &SM) {
if (Loc.isInvalid() || Loc.isMacroID())
  return {};
std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
if (LocInfo.first.isInvalid())
  return {};
bool Invalid = false;
StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
if (Invalid)
  return {};
const char *Line = findBeginningOfLine(Buffer, LocInfo.second);
if (!Line)
  return {};
StringRef Rest = Buffer.substr(Line - Buffer.data());
size_t NumWhitespaceChars = Rest.find_first_not_of(" \t");
return NumWhitespaceChars == StringRef::npos
           ? ""
           : Rest.take_front(NumWhitespaceChars);
1108}

1110//===----------------------------------------------------------------------===//
1111// Diagnostics forwarding code.
1112//===----------------------------------------------------------------------===//

1114/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
1115/// lexer buffer was all expanded at a single point, perform the mapping.
1116/// This is currently only used for _Pragma implementation, so it is the slow
1117/// path of the hot getSourceLocation method.  Do not allow it to be inlined.
1118static LLVM_ATTRIBUTE_NOINLINE__attribute__((noinline)) SourceLocation GetMappedTokenLoc(
  Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
1120static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
                                      SourceLocation FileLoc,
                                      unsigned CharNo, unsigned TokLen) {
assert(FileLoc.isMacroID() && "Must be a macro expansion")((void)0);

// Otherwise, we're lexing "mapped tokens".  This is used for things like
// _Pragma handling.  Combine the expansion location of FileLoc with the
// spelling location.
SourceManager &SM = PP.getSourceManager();

// Create a new SLoc which is expanded from Expansion(FileLoc) but whose
// characters come from spelling(FileLoc)+Offset.
SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc);
SpellingLoc = SpellingLoc.getLocWithOffset(CharNo);

// Figure out the expansion loc range, which is the range covered by the
// original _Pragma(...) sequence.
CharSourceRange II = SM.getImmediateExpansionRange(FileLoc);

return SM.createExpansionLoc(SpellingLoc, II.getBegin(), II.getEnd(), TokLen);
1140}

1142/// getSourceLocation - Return a source location identifier for the specified
1143/// offset in the current file.
1144SourceLocation Lexer::getSourceLocation(const char *Loc,
                                      unsigned TokLen) const {
assert(Loc >= BufferStart && Loc <= BufferEnd &&((void)0)
       "Location out of range for this buffer!")((void)0);

// In the normal case, we're just lexing from a simple file buffer, return
// the file id from FileLoc with the offset specified.
unsigned CharNo = Loc-BufferStart;
if (FileLoc.isFileID())
  return FileLoc.getLocWithOffset(CharNo);

// Otherwise, this is the _Pragma lexer case, which pretends that all of the
// tokens are lexed from where the _Pragma was defined.
assert(PP && "This doesn't work on raw lexers")((void)0);
return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen);
1159}

1161/// Diag - Forwarding function for diagnostics.  This translate a source
1162/// position in the current buffer into a SourceLocation object for rendering.
1163DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
return PP->Diag(getSourceLocation(Loc), DiagID);
1165}

1167//===----------------------------------------------------------------------===//
1168// Trigraph and Escaped Newline Handling Code.
1169//===----------------------------------------------------------------------===//

1171/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
1172/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
1173static char GetTrigraphCharForLetter(char Letter) {
switch (Letter) {
default:   return 0;
case '=':  return '#';
case ')':  return ']';
case '(':  return '[';
case '!':  return '|';
case '\'': return '^';
case '>':  return '}';
case '/':  return '\\';
case '<':  return '{';
case '-':  return '~';
}
1186}

1188/// DecodeTrigraphChar - If the specified character is a legal trigraph when
1189/// prefixed with ??, emit a trigraph warning.  If trigraphs are enabled,
1190/// return the result character.  Finally, emit a warning about trigraph use
1191/// whether trigraphs are enabled or not.
1192static char DecodeTrigraphChar(const char *CP, Lexer *L) {
char Res = GetTrigraphCharForLetter(*CP);
if (!Res || !L) return Res;

if (!L->getLangOpts().Trigraphs) {
  if (!L->isLexingRawMode())
    L->Diag(CP-2, diag::trigraph_ignored);
  return 0;
}

if (!L->isLexingRawMode())
  L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1);
return Res;
1205}

1207/// getEscapedNewLineSize - Return the size of the specified escaped newline,
1208/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
1209/// trigraph equivalent on entry to this function.
1210unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
unsigned Size = 0;
while (isWhitespace(Ptr[Size])) {
  ++Size;

  if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
    continue;

  // If this is a \r\n or \n\r, skip the other half.
  if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
      Ptr[Size-1] != Ptr[Size])
    ++Size;

  return Size;
}

// Not an escaped newline, must be a \t or something else.
return 0;
1228}

1230/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
1231/// them), skip over them and return the first non-escaped-newline found,
1232/// otherwise return P.
1233const char *Lexer::SkipEscapedNewLines(const char *P) {
while (true) {
  const char *AfterEscape;
  if (*P == '\\') {
    AfterEscape = P+1;
  } else if (*P == '?') {
    // If not a trigraph for escape, bail out.
    if (P[1] != '?' || P[2] != '/')
      return P;
    // FIXME: Take LangOpts into account; the language might not
    // support trigraphs.
    AfterEscape = P+3;
  } else {
    return P;
  }

  unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
  if (NewLineSize == 0) return P;
  P = AfterEscape+NewLineSize;
}
1253}

1255Optional<Token> Lexer::findNextToken(SourceLocation Loc,
                                   const SourceManager &SM,
                                   const LangOptions &LangOpts) {
if (Loc.isMacroID()) {
  if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
    return None;
}
Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);

// Break down the source location.
std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);

// Try to load the file buffer.
bool InvalidTemp = false;
StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp);
if (InvalidTemp)
  return None;

const char *TokenBegin = File.data() + LocInfo.second;

// Lex from the start of the given location.
Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
                                    TokenBegin, File.end());
// Find the token.
Token Tok;
lexer.LexFromRawLexer(Tok);
return Tok;
1282}

1284/// Checks that the given token is the first token that occurs after the
1285/// given location (this excludes comments and whitespace). Returns the location
1286/// immediately after the specified token. If the token is not found or the
1287/// location is inside a macro, the returned source location will be invalid.
1288SourceLocation Lexer::findLocationAfterToken(
  SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM,
  const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) {
Optional<Token> Tok = findNextToken(Loc, SM, LangOpts);
if (!Tok || Tok->isNot(TKind))
  return {};
SourceLocation TokenLoc = Tok->getLocation();

// Calculate how much whitespace needs to be skipped if any.
unsigned NumWhitespaceChars = 0;
if (SkipTrailingWhitespaceAndNewLine) {
  const char *TokenEnd = SM.getCharacterData(TokenLoc) + Tok->getLength();
  unsigned char C = *TokenEnd;
  while (isHorizontalWhitespace(C)) {
    C = *(++TokenEnd);
    NumWhitespaceChars++;
  }

  // Skip \r, \n, \r\n, or \n\r
  if (C == '\n' || C == '\r') {
    char PrevC = C;
    C = *(++TokenEnd);
    NumWhitespaceChars++;
    if ((C == '\n' || C == '\r') && C != PrevC)
      NumWhitespaceChars++;
  }
}

return TokenLoc.getLocWithOffset(Tok->getLength() + NumWhitespaceChars);
1317}

1319/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
1320/// get its size, and return it.  This is tricky in several cases:
1321///   1. If currently at the start of a trigraph, we warn about the trigraph,
1322///      then either return the trigraph (skipping 3 chars) or the '?',
1323///      depending on whether trigraphs are enabled or not.
1324///   2. If this is an escaped newline (potentially with whitespace between
1325///      the backslash and newline), implicitly skip the newline and return
1326///      the char after it.
1327///
1328/// This handles the slow/uncommon case of the getCharAndSize method.  Here we
1329/// know that we can accumulate into Size, and that we have already incremented
1330/// Ptr by Size bytes.
1331///
1332/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
1333/// be updated to match.
1334char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
                             Token *Tok) {
// If we have a slash, look for an escaped newline.
if (Ptr[0] == '\\') {
  ++Size;
  ++Ptr;
1340Slash:
  // Common case, backslash-char where the char is not whitespace.
  if (!isWhitespace(Ptr[0])) return '\\';

  // See if we have optional whitespace characters between the slash and
  // newline.
  if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
    // Remember that this token needs to be cleaned.
    if (Tok) Tok->setFlag(Token::NeedsCleaning);

    // Warn if there was whitespace between the backslash and newline.
    if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
      Diag(Ptr, diag::backslash_newline_space);

    // Found backslash<whitespace><newline>.  Parse the char after it.
    Size += EscapedNewLineSize;
    Ptr  += EscapedNewLineSize;

    // Use slow version to accumulate a correct size field.
    return getCharAndSizeSlow(Ptr, Size, Tok);
  }

  // Otherwise, this is not an escaped newline, just return the slash.
  return '\\';
}

// If this is a trigraph, process it.
if (Ptr[0] == '?' && Ptr[1] == '?') {
  // If this is actually a legal trigraph (not something like "??x"), emit
  // a trigraph warning.  If so, and if trigraphs are enabled, return it.
  if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) {
    // Remember that this token needs to be cleaned.
    if (Tok) Tok->setFlag(Token::NeedsCleaning);

    Ptr += 3;
    Size += 3;
    if (C == '\\') goto Slash;
    return C;
  }
}

// If this is neither, return a single character.
++Size;
return *Ptr;
1384}

1386/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
1387/// getCharAndSizeNoWarn method.  Here we know that we can accumulate into Size,
1388/// and that we have already incremented Ptr by Size bytes.
1389///
1390/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
1391/// be updated to match.
1392char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
                                   const LangOptions &LangOpts) {
// If we have a slash, look for an escaped newline.
if (Ptr[0] == '\\') {
  ++Size;
  ++Ptr;
1398Slash:
  // Common case, backslash-char where the char is not whitespace.
  if (!isWhitespace(Ptr[0])) return '\\';

  // See if we have optional whitespace characters followed by a newline.
  if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
    // Found backslash<whitespace><newline>.  Parse the char after it.
    Size += EscapedNewLineSize;
    Ptr  += EscapedNewLineSize;

    // Use slow version to accumulate a correct size field.
    return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts);
  }

  // Otherwise, this is not an escaped newline, just return the slash.
  return '\\';
}

// If this is a trigraph, process it.
if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
  // If this is actually a legal trigraph (not something like "??x"), return
  // it.
  if (char C = GetTrigraphCharForLetter(Ptr[2])) {
    Ptr += 3;
    Size += 3;
    if (C == '\\') goto Slash;
    return C;
  }
}

// If this is neither, return a single character.
++Size;
return *Ptr;
1431}

1433//===----------------------------------------------------------------------===//
1434// Helper methods for lexing.
1435//===----------------------------------------------------------------------===//

1437/// Routine that indiscriminately sets the offset into the source file.
1438void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) {
BufferPtr = BufferStart + Offset;
if (BufferPtr > BufferEnd)
  BufferPtr = BufferEnd;
// FIXME: What exactly does the StartOfLine bit mean?  There are two
// possible meanings for the "start" of the line: the first token on the
// unexpanded line, or the first token on the expanded line.
IsAtStartOfLine = StartOfLine;
IsAtPhysicalStartOfLine = StartOfLine;
1447}

1449static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) {
if (LangOpts.AsmPreprocessor) {
  return false;
} else if (LangOpts.DollarIdents && '$' == C) {
  return true;
} else if (LangOpts.CPlusPlus11 || LangOpts.C11) {
  static const llvm::sys::UnicodeCharSet C11AllowedIDChars(
      C11AllowedIDCharRanges);
  return C11AllowedIDChars.contains(C);
} else if (LangOpts.CPlusPlus) {
  static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars(
      CXX03AllowedIDCharRanges);
  return CXX03AllowedIDChars.contains(C);
} else {
  static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
      C99AllowedIDCharRanges);
  return C99AllowedIDChars.contains(C);
}
1467}

1469static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) {
assert(isAllowedIDChar(C, LangOpts))((void)0);
if (LangOpts.AsmPreprocessor) {
  return false;
} else if (LangOpts.CPlusPlus11 || LangOpts.C11) {
  static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars(
      C11DisallowedInitialIDCharRanges);
  return !C11DisallowedInitialIDChars.contains(C);
} else if (LangOpts.CPlusPlus) {
  return true;
} else {
  static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
      C99DisallowedInitialIDCharRanges);
  return !C99DisallowedInitialIDChars.contains(C);
}
1484}

1486static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin,
                                          const char *End) {
return CharSourceRange::getCharRange(L.getSourceLocation(Begin),
                                     L.getSourceLocation(End));
1490}

1492static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C,
                                    CharSourceRange Range, bool IsFirst) {
// Check C99 compatibility.
if (!Diags.isIgnored(diag::warn_c99_compat_unicode_id, Range.getBegin())) {
  enum {
    CannotAppearInIdentifier = 0,
    CannotStartIdentifier
  };

  static const llvm::sys::UnicodeCharSet C99AllowedIDChars(
      C99AllowedIDCharRanges);
  static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars(
      C99DisallowedInitialIDCharRanges);
  if (!C99AllowedIDChars.contains(C)) {
    Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id)
      << Range
      << CannotAppearInIdentifier;
  } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) {
    Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id)
      << Range
      << CannotStartIdentifier;
  }
}

// Check C++98 compatibility.
if (!Diags.isIgnored(diag::warn_cxx98_compat_unicode_id, Range.getBegin())) {
  static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars(
      CXX03AllowedIDCharRanges);
  if (!CXX03AllowedIDChars.contains(C)) {
    Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id)
      << Range;
  }
}
1525}

1527/// After encountering UTF-8 character C and interpreting it as an identifier
1528/// character, check whether it's a homoglyph for a common non-identifier
1529/// source character that is unlikely to be an intentional identifier
1530/// character and warn if so.
1531static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C,
                                     CharSourceRange Range) {
// FIXME: Handle Unicode quotation marks (smart quotes, fullwidth quotes).
struct HomoglyphPair {
  uint32_t Character;
  char LooksLike;
  bool operator<(HomoglyphPair R) const { return Character < R.Character; }
};
static constexpr HomoglyphPair SortedHomoglyphs[] = {
  {U'\u00ad', 0},   // SOFT HYPHEN
  {U'\u01c3', '!'}, // LATIN LETTER RETROFLEX CLICK
  {U'\u037e', ';'}, // GREEK QUESTION MARK
  {U'\u200b', 0},   // ZERO WIDTH SPACE
  {U'\u200c', 0},   // ZERO WIDTH NON-JOINER
  {U'\u200d', 0},   // ZERO WIDTH JOINER
  {U'\u2060', 0},   // WORD JOINER
  {U'\u2061', 0},   // FUNCTION APPLICATION
  {U'\u2062', 0},   // INVISIBLE TIMES
  {U'\u2063', 0},   // INVISIBLE SEPARATOR
  {U'\u2064', 0},   // INVISIBLE PLUS
  {U'\u2212', '-'}, // MINUS SIGN
  {U'\u2215', '/'}, // DIVISION SLASH
  {U'\u2216', '\\'}, // SET MINUS
  {U'\u2217', '*'}, // ASTERISK OPERATOR
  {U'\u2223', '|'}, // DIVIDES
  {U'\u2227', '^'}, // LOGICAL AND
  {U'\u2236', ':'}, // RATIO
  {U'\u223c', '~'}, // TILDE OPERATOR
  {U'\ua789', ':'}, // MODIFIER LETTER COLON
  {U'\ufeff', 0},   // ZERO WIDTH NO-BREAK SPACE
  {U'\uff01', '!'}, // FULLWIDTH EXCLAMATION MARK
  {U'\uff03', '#'}, // FULLWIDTH NUMBER SIGN
  {U'\uff04', '$'}, // FULLWIDTH DOLLAR SIGN
  {U'\uff05', '%'}, // FULLWIDTH PERCENT SIGN
  {U'\uff06', '&'}, // FULLWIDTH AMPERSAND
  {U'\uff08', '('}, // FULLWIDTH LEFT PARENTHESIS
  {U'\uff09', ')'}, // FULLWIDTH RIGHT PARENTHESIS
  {U'\uff0a', '*'}, // FULLWIDTH ASTERISK
  {U'\uff0b', '+'}, // FULLWIDTH ASTERISK
  {U'\uff0c', ','}, // FULLWIDTH COMMA
  {U'\uff0d', '-'}, // FULLWIDTH HYPHEN-MINUS
  {U'\uff0e', '.'}, // FULLWIDTH FULL STOP
  {U'\uff0f', '/'}, // FULLWIDTH SOLIDUS
  {U'\uff1a', ':'}, // FULLWIDTH COLON
  {U'\uff1b', ';'}, // FULLWIDTH SEMICOLON
  {U'\uff1c', '<'}, // FULLWIDTH LESS-THAN SIGN
  {U'\uff1d', '='}, // FULLWIDTH EQUALS SIGN
  {U'\uff1e', '>'}, // FULLWIDTH GREATER-THAN SIGN
  {U'\uff1f', '?'}, // FULLWIDTH QUESTION MARK
  {U'\uff20', '@'}, // FULLWIDTH COMMERCIAL AT
  {U'\uff3b', '['}, // FULLWIDTH LEFT SQUARE BRACKET
  {U'\uff3c', '\\'}, // FULLWIDTH REVERSE SOLIDUS
  {U'\uff3d', ']'}, // FULLWIDTH RIGHT SQUARE BRACKET
  {U'\uff3e', '^'}, // FULLWIDTH CIRCUMFLEX ACCENT
  {U'\uff5b', '{'}, // FULLWIDTH LEFT CURLY BRACKET
  {U'\uff5c', '|'}, // FULLWIDTH VERTICAL LINE
  {U'\uff5d', '}'}, // FULLWIDTH RIGHT CURLY BRACKET
  {U'\uff5e', '~'}, // FULLWIDTH TILDE
  {0, 0}
};
auto Homoglyph =
    std::lower_bound(std::begin(SortedHomoglyphs),
                     std::end(SortedHomoglyphs) - 1, HomoglyphPair{C, '\0'});
if (Homoglyph->Character == C) {
  llvm::SmallString<5> CharBuf;
  {
    llvm::raw_svector_ostream CharOS(CharBuf);
    llvm::write_hex(CharOS, C, llvm::HexPrintStyle::Upper, 4);
  }
  if (Homoglyph->LooksLike) {
    const char LooksLikeStr[] = {Homoglyph->LooksLike, 0};
    Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_homoglyph)
        << Range << CharBuf << LooksLikeStr;
  } else {
    Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_zero_width)
        << Range << CharBuf;
  }
}
1609}

1611bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
                                  Token &Result) {
const char *UCNPtr = CurPtr + Size;
uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/nullptr);
if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts))
  return false;

if (!isLexingRawMode())
  maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,
                            makeCharRange(*this, CurPtr, UCNPtr),
                            /*IsFirst=*/false);

Result.setFlag(Token::HasUCN);
if ((UCNPtr - CurPtr ==  6 && CurPtr[1] == 'u') ||
    (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U'))
  CurPtr = UCNPtr;
else
  while (CurPtr != UCNPtr)
    (void)getAndAdvanceChar(CurPtr, Result);
return true;
1631}

1633bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) {
const char *UnicodePtr = CurPtr;
llvm::UTF32 CodePoint;
llvm::ConversionResult Result =
    llvm::convertUTF8Sequence((const llvm::UTF8 **)&UnicodePtr,
                              (const llvm::UTF8 *)BufferEnd,
                              &CodePoint,
                              llvm::strictConversion);
if (Result != llvm::conversionOK ||
    !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts))
  return false;

if (!isLexingRawMode()) {
  maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint,
                            makeCharRange(*this, CurPtr, UnicodePtr),
                            /*IsFirst=*/false);
  maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), CodePoint,
                             makeCharRange(*this, CurPtr, UnicodePtr));
}

CurPtr = UnicodePtr;
return true;
1655}

1657bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
// Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
unsigned Size;
unsigned char C = *CurPtr++;
while (isIdentifierBody(C))
  C = *CurPtr++;

--CurPtr;   // Back up over the skipped character.

// Fast path, no $,\,? in identifier found.  '\' might be an escaped newline
// or UCN, and ? might be a trigraph for '\', an escaped newline or UCN.
//
// TODO: Could merge these checks into an InfoTable flag to make the
// comparison cheaper
if (isASCII(C) && C != '\\' && C != '?' &&
    (C != '$' || !LangOpts.DollarIdents)) {
1673FinishIdentifier:
  const char *IdStart = BufferPtr;
  FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
  Result.setRawIdentifierData(IdStart);

  // If we are in raw mode, return this identifier raw.  There is no need to
  // look up identifier information or attempt to macro expand it.
  if (LexingRawMode)
    return true;

  // Fill in Result.IdentifierInfo and update the token kind,
  // looking up the identifier in the identifier table.
  IdentifierInfo *II = PP->LookUpIdentifierInfo(Result);
  // Note that we have to call PP->LookUpIdentifierInfo() even for code
  // completion, it writes IdentifierInfo into Result, and callers rely on it.

  // If the completion point is at the end of an identifier, we want to treat
  // the identifier as incomplete even if it resolves to a macro or a keyword.
  // This allows e.g. 'class^' to complete to 'classifier'.
  if (isCodeCompletionPoint(CurPtr)) {
    // Return the code-completion token.
    Result.setKind(tok::code_completion);
    // Skip the code-completion char and all immediate identifier characters.
    // This ensures we get consistent behavior when completing at any point in
    // an identifier (i.e. at the start, in the middle, at the end). Note that
    // only simple cases (i.e. [a-zA-Z0-9_]) are supported to keep the code
    // simpler.
    assert(*CurPtr == 0 && "Completion character must be 0")((void)0);
    ++CurPtr;
    // Note that code completion token is not added as a separate character
    // when the completion point is at the end of the buffer. Therefore, we need
    // to check if the buffer has ended.
    if (CurPtr < BufferEnd) {
      while (isIdentifierBody(*CurPtr))
        ++CurPtr;
    }
    BufferPtr = CurPtr;
    return true;
  }

  // Finally, now that we know we have an identifier, pass this off to the
  // preprocessor, which may macro expand it or something.
  if (II->isHandleIdentifierCase())
    return PP->HandleIdentifier(Result);

  return true;
}

// Otherwise, $,\,? in identifier found.  Enter slower path.

C = getCharAndSize(CurPtr, Size);
while (true) {
  if (C == '$') {
    // If we hit a $ and they are not supported in identifiers, we are done.
    if (!LangOpts.DollarIdents) goto FinishIdentifier;

    // Otherwise, emit a diagnostic and continue.
    if (!isLexingRawMode())
      Diag(CurPtr, diag::ext_dollar_in_identifier);
    CurPtr = ConsumeChar(CurPtr, Size, Result);
    C = getCharAndSize(CurPtr, Size);
    continue;
  } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {
    C = getCharAndSize(CurPtr, Size);
    continue;
  } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {
    C = getCharAndSize(CurPtr, Size);
    continue;
  } else if (!isIdentifierBody(C)) {
    goto FinishIdentifier;
  }

  // Otherwise, this character is good, consume it.
  CurPtr = ConsumeChar(CurPtr, Size, Result);

  C = getCharAndSize(CurPtr, Size);
  while (isIdentifierBody(C)) {
    CurPtr = ConsumeChar(CurPtr, Size, Result);
    C = getCharAndSize(CurPtr, Size);
  }
}
1754}

1756/// isHexaLiteral - Return true if Start points to a hex constant.
1757/// in microsoft mode (where this is supposed to be several different tokens).
1758bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
unsigned Size;
char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts);
if (C1 != '0')
  return false;
char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts);
return (C2 == 'x' || C2 == 'X');
1765}

1767/// LexNumericConstant - Lex the remainder of a integer or floating point
1768/// constant. From[-1] is the first character lexed.  Return the end of the
1769/// constant.
1770bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
unsigned Size;
char C = getCharAndSize(CurPtr, Size);
char PrevCh = 0;
while (isPreprocessingNumberBody(C)) {
  CurPtr = ConsumeChar(CurPtr, Size, Result);
  PrevCh = C;
  C = getCharAndSize(CurPtr, Size);
}

// If we fell out, check for a sign, due to 1e+12.  If we have one, continue.
if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
  // If we are in Microsoft mode, don't continue if the constant is hex.
  // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
  if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts))
    return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
}

// If we have a hex FP constant, continue.
if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) {
  // Outside C99 and C++17, we accept hexadecimal floating point numbers as a
  // not-quite-conforming extension. Only do so if this looks like it's
  // actually meant to be a hexfloat, and not if it has a ud-suffix.
  bool IsHexFloat = true;
  if (!LangOpts.C99) {
    if (!isHexaLiteral(BufferPtr, LangOpts))
      IsHexFloat = false;
    else if (!getLangOpts().CPlusPlus17 &&
             std::find(BufferPtr, CurPtr, '_') != CurPtr)
      IsHexFloat = false;
  }
  if (IsHexFloat)
    return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
}

// If we have a digit separator, continue.
if (C == '\'' && (getLangOpts().CPlusPlus14 || getLangOpts().C2x)) {
  unsigned NextSize;
  char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts());
  if (isIdentifierBody(Next)) {
    if (!isLexingRawMode())
      Diag(CurPtr, getLangOpts().CPlusPlus
                       ? diag::warn_cxx11_compat_digit_separator
                       : diag::warn_c2x_compat_digit_separator);
    CurPtr = ConsumeChar(CurPtr, Size, Result);
    CurPtr = ConsumeChar(CurPtr, NextSize, Result);
    return LexNumericConstant(Result, CurPtr);
  }
}

// If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
  return LexNumericConstant(Result, CurPtr);
if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr))
  return LexNumericConstant(Result, CurPtr);

// Update the location of token as well as BufferPtr.
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
Result.setLiteralData(TokStart);
return true;
1831}

1833/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
1834/// in C++11, or warn on a ud-suffix in C++98.
1835const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr,
                             bool IsStringLiteral) {
assert(getLangOpts().CPlusPlus)((void)0);

// Maximally munch an identifier.
unsigned Size;
char C = getCharAndSize(CurPtr, Size);
bool Consumed = false;

if (!isIdentifierHead(C)) {
  if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result))
    Consumed = true;
  else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr))
    Consumed = true;
  else
    return CurPtr;
}

if (!getLangOpts().CPlusPlus11) {
  if (!isLexingRawMode())
    Diag(CurPtr,
         C == '_' ? diag::warn_cxx11_compat_user_defined_literal
                  : diag::warn_cxx11_compat_reserved_user_defined_literal)
      << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " ");
  return CurPtr;
}

// C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix
// that does not start with an underscore is ill-formed. As a conforming
// extension, we treat all such suffixes as if they had whitespace before
// them. We assume a suffix beginning with a UCN or UTF-8 character is more
// likely to be a ud-suffix than a macro, however, and accept that.
if (!Consumed) {
  bool IsUDSuffix = false;
  if (C == '_')
    IsUDSuffix = true;
  else if (IsStringLiteral && getLangOpts().CPlusPlus14) {
    // In C++1y, we need to look ahead a few characters to see if this is a
    // valid suffix for a string literal or a numeric literal (this could be
    // the 'operator""if' defining a numeric literal operator).
    const unsigned MaxStandardSuffixLength = 3;
    char Buffer[MaxStandardSuffixLength] = { C };
    unsigned Consumed = Size;
    unsigned Chars = 1;
    while (true) {
      unsigned NextSize;
      char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize,
                                       getLangOpts());
      if (!isIdentifierBody(Next)) {
        // End of suffix. Check whether this is on the allowed list.
        const StringRef CompleteSuffix(Buffer, Chars);
        IsUDSuffix = StringLiteralParser::isValidUDSuffix(getLangOpts(),
                                                          CompleteSuffix);
        break;
      }

      if (Chars == MaxStandardSuffixLength)
        // Too long: can't be a standard suffix.
        break;

      Buffer[Chars++] = Next;
      Consumed += NextSize;
    }
  }

  if (!IsUDSuffix) {
    if (!isLexingRawMode())
      Diag(CurPtr, getLangOpts().MSVCCompat
                       ? diag::ext_ms_reserved_user_defined_literal
                       : diag::ext_reserved_user_defined_literal)
        << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " ");
    return CurPtr;
  }

  CurPtr = ConsumeChar(CurPtr, Size, Result);
}

Result.setFlag(Token::HasUDSuffix);
while (true) {
  C = getCharAndSize(CurPtr, Size);
  if (isIdentifierBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); }
  else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {}
  else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {}
  else break;
}

return CurPtr;
1922}

1924/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
1925/// either " or L" or u8" or u" or U".
1926bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
                           tok::TokenKind Kind) {
const char *AfterQuote = CurPtr;
// Does this string contain the \0 character?
const char *NulCharacter = nullptr;

if (!isLexingRawMode() &&
    (Kind == tok::utf8_string_literal ||
     Kind == tok::utf16_string_literal ||
     Kind == tok::utf32_string_literal))
  Diag(BufferPtr, getLangOpts().CPlusPlus
         ? diag::warn_cxx98_compat_unicode_literal
         : diag::warn_c99_compat_unicode_literal);

char C = getAndAdvanceChar(CurPtr, Result);
while (C != '"') {
  // Skip escaped characters.  Escaped newlines will already be processed by
  // getAndAdvanceChar.
  if (C == '\\')
    C = getAndAdvanceChar(CurPtr, Result);

  if (C == '\n' || C == '\r' ||             // Newline.
      (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
    if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
      Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1;
    FormTokenWithChars(Result, CurPtr-1, tok::unknown);
    return true;
  }

  if (C == 0) {
    if (isCodeCompletionPoint(CurPtr-1)) {
      if (ParsingFilename)
        codeCompleteIncludedFile(AfterQuote, CurPtr - 1, /*IsAngled=*/false);
      else
        PP->CodeCompleteNaturalLanguage();
      FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
      cutOffLexing();
      return true;
    }

    NulCharacter = CurPtr-1;
  }
  C = getAndAdvanceChar(CurPtr, Result);
}

// If we are in C++11, lex the optional ud-suffix.
if (getLangOpts().CPlusPlus)
  CurPtr = LexUDSuffix(Result, CurPtr, true);

// If a nul character existed in the string, warn about it.
if (NulCharacter && !isLexingRawMode())
  Diag(NulCharacter, diag::null_in_char_or_string) << 1;

// Update the location of the token as well as the BufferPtr instance var.
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, Kind);
Result.setLiteralData(TokStart);
return true;
1984}

1986/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
1987/// having lexed R", LR", u8R", uR", or UR".
1988bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
                              tok::TokenKind Kind) {
// This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
//  Between the initial and final double quote characters of the raw string,
//  any transformations performed in phases 1 and 2 (trigraphs,
//  universal-character-names, and line splicing) are reverted.

if (!isLexingRawMode())
  Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal);

unsigned PrefixLen = 0;

while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen]))
  ++PrefixLen;

// If the last character was not a '(', then we didn't lex a valid delimiter.
if (CurPtr[PrefixLen] != '(') {
  if (!isLexingRawMode()) {
    const char *PrefixEnd = &CurPtr[PrefixLen];
    if (PrefixLen == 16) {
      Diag(PrefixEnd, diag::err_raw_delim_too_long);
    } else {
      Diag(PrefixEnd, diag::err_invalid_char_raw_delim)
        << StringRef(PrefixEnd, 1);
    }
  }

  // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
  // it's possible the '"' was intended to be part of the raw string, but
  // there's not much we can do about that.
  while (true) {
    char C = *CurPtr++;

    if (C == '"')
      break;
    if (C == 0 && CurPtr-1 == BufferEnd) {
      --CurPtr;
      break;
    }
  }

  FormTokenWithChars(Result, CurPtr, tok::unknown);
  return true;
}

// Save prefix and move CurPtr past it
const char *Prefix = CurPtr;
CurPtr += PrefixLen + 1; // skip over prefix and '('

while (true) {
  char C = *CurPtr++;

  if (C == ')') {
    // Check for prefix match and closing quote.
    if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
      CurPtr += PrefixLen + 1; // skip over prefix and '"'
      break;
    }
  } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
    if (!isLexingRawMode())
      Diag(BufferPtr, diag::err_unterminated_raw_string)
        << StringRef(Prefix, PrefixLen);
    FormTokenWithChars(Result, CurPtr-1, tok::unknown);
    return true;
  }
}

// If we are in C++11, lex the optional ud-suffix.
if (getLangOpts().CPlusPlus)
  CurPtr = LexUDSuffix(Result, CurPtr, true);

// Update the location of token as well as BufferPtr.
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, Kind);
Result.setLiteralData(TokStart);
return true;
2064}

2066/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
2067/// after having lexed the '<' character.  This is used for #include filenames.
2068bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
// Does this string contain the \0 character?
const char *NulCharacter = nullptr;
const char *AfterLessPos = CurPtr;
char C = getAndAdvanceChar(CurPtr, Result);
while (C != '>') {
  // Skip escaped characters.  Escaped newlines will already be processed by
  // getAndAdvanceChar.
  if (C == '\\')
    C = getAndAdvanceChar(CurPtr, Result);

  if (isVerticalWhitespace(C) ||               // Newline.
      (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file.
    // If the filename is unterminated, then it must just be a lone <
    // character.  Return this as such.
    FormTokenWithChars(Result, AfterLessPos, tok::less);
    return true;
  }

  if (C == 0) {
    if (isCodeCompletionPoint(CurPtr - 1)) {
      codeCompleteIncludedFile(AfterLessPos, CurPtr - 1, /*IsAngled=*/true);
      cutOffLexing();
      FormTokenWithChars(Result, CurPtr - 1, tok::unknown);
      return true;
    }
    NulCharacter = CurPtr-1;
  }
  C = getAndAdvanceChar(CurPtr, Result);
}

// If a nul character existed in the string, warn about it.
if (NulCharacter && !isLexingRawMode())
  Diag(NulCharacter, diag::null_in_char_or_string) << 1;

// Update the location of token as well as BufferPtr.
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, tok::header_name);
Result.setLiteralData(TokStart);
return true;
2108}

2110void Lexer::codeCompleteIncludedFile(const char *PathStart,
                                   const char *CompletionPoint,
                                   bool IsAngled) {
// Completion only applies to the filename, after the last slash.
StringRef PartialPath(PathStart, CompletionPoint - PathStart);
llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/";
auto Slash = PartialPath.find_last_of(SlashChars);
StringRef Dir =
    (Slash == StringRef::npos) ? "" : PartialPath.take_front(Slash);
const char *StartOfFilename =
    (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1;
// Code completion filter range is the filename only, up to completion point.
PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
    StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
// We should replace the characters up to the closing quote or closest slash,
// if any.
while (CompletionPoint < BufferEnd) {
  char Next = *(CompletionPoint + 1);
  if (Next == 0 || Next == '\r' || Next == '\n')
    break;
  ++CompletionPoint;
  if (Next == (IsAngled ? '>' : '"'))
    break;
  if (llvm::is_contained(SlashChars, Next))
    break;
}

PP->setCodeCompletionTokenRange(
    FileLoc.getLocWithOffset(StartOfFilename - BufferStart),
    FileLoc.getLocWithOffset(CompletionPoint - BufferStart));
PP->CodeCompleteIncludedFile(Dir, IsAngled);
2141}

2143/// LexCharConstant - Lex the remainder of a character constant, after having
2144/// lexed either ' or L' or u8' or u' or U'.
2145bool Lexer::LexCharConstant(Token &Result, const char *CurPtr,
                          tok::TokenKind Kind) {
// Does this character contain the \0 character?
const char *NulCharacter = nullptr;

if (!isLexingRawMode()) {
  if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)
    Diag(BufferPtr, getLangOpts().CPlusPlus
                        ? diag::warn_cxx98_compat_unicode_literal
                        : diag::warn_c99_compat_unicode_literal);
  else if (Kind == tok::utf8_char_constant)
    Diag(BufferPtr, diag::warn_cxx14_compat_u8_character_literal);
}

char C = getAndAdvanceChar(CurPtr, Result);
if (C == '\'') {
  if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
    Diag(BufferPtr, diag::ext_empty_character);
  FormTokenWithChars(Result, CurPtr, tok::unknown);
  return true;
}

while (C != '\'') {
  // Skip escaped characters.
  if (C == '\\')
    C = getAndAdvanceChar(CurPtr, Result);

  if (C == '\n' || C == '\r' ||             // Newline.
      (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
    if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
      Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 0;
    FormTokenWithChars(Result, CurPtr-1, tok::unknown);
    return true;
  }

  if (C == 0) {
    if (isCodeCompletionPoint(CurPtr-1)) {
      PP->CodeCompleteNaturalLanguage();
      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
      cutOffLexing();
      return true;
    }

    NulCharacter = CurPtr-1;
  }
  C = getAndAdvanceChar(CurPtr, Result);
}

// If we are in C++11, lex the optional ud-suffix.
if (getLangOpts().CPlusPlus)
  CurPtr = LexUDSuffix(Result, CurPtr, false);

// If a nul character existed in the character, warn about it.
if (NulCharacter && !isLexingRawMode())
  Diag(NulCharacter, diag::null_in_char_or_string) << 0;

// Update the location of token as well as BufferPtr.
const char *TokStart = BufferPtr;
FormTokenWithChars(Result, CurPtr, Kind);
Result.setLiteralData(TokStart);
return true;
2206}

2208/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
2209/// Update BufferPtr to point to the next non-whitespace character and return.
2210///
2211/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
2212bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr,
                         bool &TokAtPhysicalStartOfLine) {
// Whitespace - Skip it, then return the token after the whitespace.
bool SawNewline = isVerticalWhitespace(CurPtr[-1]);

unsigned char Char = *CurPtr;

const char *lastNewLine = nullptr;
auto setLastNewLine = [&](const char *Ptr) {
  lastNewLine = Ptr;
  if (!NewLinePtr)
    NewLinePtr = Ptr;
};
if (SawNewline)
  setLastNewLine(CurPtr - 1);

// Skip consecutive spaces efficiently.
while (true) {
  // Skip horizontal whitespace very aggressively.
  while (isHorizontalWhitespace(Char))
    Char = *++CurPtr;

  // Otherwise if we have something other than whitespace, we're done.
  if (!isVerticalWhitespace(Char))
    break;

  if (ParsingPreprocessorDirective) {
    // End of preprocessor directive line, let LexTokenInternal handle this.
    BufferPtr = CurPtr;
    return false;
  }

  // OK, but handle newline.
  if (*CurPtr == '\n')
    setLastNewLine(CurPtr);
  SawNewline = true;
  Char = *++CurPtr;
}

// If the client wants us to return whitespace, return it now.
if (isKeepWhitespaceMode()) {
  FormTokenWithChars(Result, CurPtr, tok::unknown);
  if (SawNewline) {
    IsAtStartOfLine = true;
    IsAtPhysicalStartOfLine = true;
  }
  // FIXME: The next token will not have LeadingSpace set.
  return true;
}

// If this isn't immediately after a newline, there is leading space.
char PrevChar = CurPtr[-1];
bool HasLeadingSpace = !isVerticalWhitespace(PrevChar);

Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
if (SawNewline) {
  Result.setFlag(Token::StartOfLine);
  TokAtPhysicalStartOfLine = true;

  if (NewLinePtr && lastNewLine && NewLinePtr != lastNewLine && PP) {
    if (auto *Handler = PP->getEmptylineHandler())
      Handler->HandleEmptyline(SourceRange(getSourceLocation(NewLinePtr + 1),
                                           getSourceLocation(lastNewLine)));
  }
}

BufferPtr = CurPtr;
return false;
2280}

2282/// We have just read the // characters from input.  Skip until we find the
2283/// newline character that terminates the comment.  Then update BufferPtr and
2284/// return.
2285///
2286/// If we're in KeepCommentMode or any CommentHandler has inserted
2287/// some tokens, this will store the first token and return true.
2288bool Lexer::SkipLineComment(Token &Result, const char *CurPtr,
                          bool &TokAtPhysicalStartOfLine) {
// If Line comments aren't explicitly enabled for this language, emit an
// extension warning.
if (!LangOpts.LineComment13.1
Field 'LineComment' is not equal to 0
1
Field 'LineComment' is not equal to 0
 && !isLexingRawMode()) {
  Diag(BufferPtr, diag::ext_line_comment);

  // Mark them enabled so we only emit one warning for this translation
  // unit.
  LangOpts.LineComment = true;
}

// Scan over the body of the comment.  The common case, when scanning, is that
// the comment contains normal ascii characters with nothing interesting in
// them.  As such, optimize for this case with the inner loop.
//
// This loop terminates with CurPtr pointing at the newline (or end of buffer)
// character that ends the line comment.
char C;
while (true) {
14
←
Loop condition is true.  Entering loop body→
  C = *CurPtr;
  // Skip over characters in the fast loop.
  while (C != 0 &&                // Potentially EOF.
15
←
Assuming 'C' is equal to 0→
         C != '\n' && C != '\r')  // Newline or DOS-style newline.
    C = *++CurPtr;

  const char *NextLine = CurPtr;
  if (C15.1
'C' is equal to 0
1
'C' is equal to 0
 != 0) {
16
←
Taking false branch→
    // We found a newline, see if it's escaped.
    const char *EscapePtr = CurPtr-1;
    bool HasSpace = false;
    while (isHorizontalWhitespace(*EscapePtr)) { // Skip whitespace.
      --EscapePtr;
      HasSpace = true;
    }

    if (*EscapePtr == '\\')
      // Escaped newline.
      CurPtr = EscapePtr;
    else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
             EscapePtr[-2] == '?' && LangOpts.Trigraphs)
      // Trigraph-escaped newline.
      CurPtr = EscapePtr-2;
    else
      break; // This is a newline, we're done.

    // If there was space between the backslash and newline, warn about it.
    if (HasSpace && !isLexingRawMode())
      Diag(EscapePtr, diag::backslash_newline_space);
  }

  // Otherwise, this is a hard case.  Fall back on getAndAdvanceChar to
  // properly decode the character.  Read it in raw mode to avoid emitting
  // diagnostics about things like trigraphs.  If we see an escaped newline,
  // we'll handle it below.
  const char *OldPtr = CurPtr;
  bool OldRawMode = isLexingRawMode();
  LexingRawMode = true;
  C = getAndAdvanceChar(CurPtr, Result);
17
←
Calling 'Lexer::getAndAdvanceChar'→
20
←
Returning from 'Lexer::getAndAdvanceChar'→
  LexingRawMode = OldRawMode;

  // If we only read only one character, then no special handling is needed.
  // We're done and can skip forward to the newline.
  if (C != 0 && CurPtr == OldPtr+1) {
21
←
Assuming 'C' is equal to 0→
    CurPtr = NextLine;
    break;
  }

  // If we read multiple characters, and one of those characters was a \r or
  // \n, then we had an escaped newline within the comment.  Emit diagnostic
  // unless the next line is also a // comment.
  if (CurPtr != OldPtr + 1 && C != '/' &&
22
←
Assuming the condition is false→
      (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) {
    for (; OldPtr != CurPtr; ++OldPtr)
      if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
        // Okay, we found a // comment that ends in a newline, if the next
        // line is also a // comment, but has spaces, don't emit a diagnostic.
        if (isWhitespace(C)) {
          const char *ForwardPtr = CurPtr;
          while (isWhitespace(*ForwardPtr))  // Skip whitespace.
            ++ForwardPtr;
          if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
            break;
        }

        if (!isLexingRawMode())
          Diag(OldPtr-1, diag::ext_multi_line_line_comment);
        break;
      }
  }

  if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) {
23
←
Assuming the condition is true→
24
←
Taking true branch→
    --CurPtr;
    break;
25
←
 Execution continues on line 2393→
  }

  if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
    PP->CodeCompleteNaturalLanguage();
    cutOffLexing();
    return false;
  }
}

// Found but did not consume the newline.  Notify comment handlers about the
// comment unless we're in a #if 0 block.
if (PP && !isLexingRawMode() &&
26
←
Assuming field 'PP' is null→
27
←
Taking false branch→
    PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
                                          getSourceLocation(CurPtr)))) {
  BufferPtr = CurPtr;
  return true; // A token has to be returned.
}

// If we are returning comments as tokens, return this comment as a token.
if (inKeepCommentMode())
28
←
Taking true branch→
  return SaveLineComment(Result, CurPtr);
29
←
Calling 'Lexer::SaveLineComment'→

// If we are inside a preprocessor directive and we see the end of line,
// return immediately, so that the lexer can return this as an EOD token.
if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
  BufferPtr = CurPtr;
  return false;
}

// Otherwise, eat the \n character.  We don't care if this is a \n\r or
// \r\n sequence.  This is an efficiency hack (because we know the \n can't
// contribute to another token), it isn't needed for correctness.  Note that
// this is ok even in KeepWhitespaceMode, because we would have returned the
/// comment above in that mode.
NewLinePtr = CurPtr++;

// The next returned token is at the start of the line.
Result.setFlag(Token::StartOfLine);
TokAtPhysicalStartOfLine = true;
// No leading whitespace seen so far.
Result.clearFlag(Token::LeadingSpace);
BufferPtr = CurPtr;
return false;
2425}

2427/// If in save-comment mode, package up this Line comment in an appropriate
2428/// way and return it.
2429bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
// If we're not in a preprocessor directive, just return the // comment
// directly.
FormTokenWithChars(Result, CurPtr, tok::comment);

if (!ParsingPreprocessorDirective || LexingRawMode)
30
←
Assuming field 'ParsingPreprocessorDirective' is true→
31
←
Assuming field 'LexingRawMode' is false→
32
←
Taking false branch→
  return true;

// If this Line-style comment is in a macro definition, transmogrify it into
// a C-style block comment.
bool Invalid = false;
std::string Spelling = PP->getSpelling(Result, &Invalid);
33
←
Called C++ object pointer is null
if (Invalid)
  return true;

assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?")((void)0);
Spelling[1] = '*';   // Change prefix to "/*".
Spelling += "*/";    // add suffix.

Result.setKind(tok::comment);
PP->CreateString(Spelling, Result,
                 Result.getLocation(), Result.getLocation());
return true;
2452}

2454/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
2455/// character (either \\n or \\r) is part of an escaped newline sequence.  Issue
2456/// a diagnostic if so.  We know that the newline is inside of a block comment.
2457static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
                                                Lexer *L) {
assert(CurPtr[0] == '\n' || CurPtr[0] == '\r')((void)0);

// Position of the first trigraph in the ending sequence.
const char *TrigraphPos = 0;
// Position of the first whitespace after a '\' in the ending sequence.
const char *SpacePos = 0;

while (true) {
  // Back up off the newline.
  --CurPtr;

  // If this is a two-character newline sequence, skip the other character.
  if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
    // \n\n or \r\r -> not escaped newline.
    if (CurPtr[0] == CurPtr[1])
      return false;
    // \n\r or \r\n -> skip the newline.
    --CurPtr;
  }

  // If we have horizontal whitespace, skip over it.  We allow whitespace
  // between the slash and newline.
  while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) {
    SpacePos = CurPtr;
    --CurPtr;
  }

  // If we have a slash, this is an escaped newline.
  if (*CurPtr == '\\') {
    --CurPtr;
  } else if (CurPtr[0] == '/' && CurPtr[-1] == '?' && CurPtr[-2] == '?') {
    // This is a trigraph encoding of a slash.
    TrigraphPos = CurPtr - 2;
    CurPtr -= 3;
  } else {
    return false;
  }

  // If the character preceding the escaped newline is a '*', then after line
  // splicing we have a '*/' ending the comment.
  if (*CurPtr == '*')
    break;

  if (*CurPtr != '\n' && *CurPtr != '\r')
    return false;
}

if (TrigraphPos) {
  // If no trigraphs are enabled, warn that we ignored this trigraph and
  // ignore this * character.
  if (!L->getLangOpts().Trigraphs) {
    if (!L->isLexingRawMode())
      L->Diag(TrigraphPos, diag::trigraph_ignored_block_comment);
    return false;
  }
  if (!L->isLexingRawMode())
    L->Diag(TrigraphPos, diag::trigraph_ends_block_comment);
}

// Warn about having an escaped newline between the */ characters.
if (!L->isLexingRawMode())
  L->Diag(CurPtr + 1, diag::escaped_newline_block_comment_end);

// If there was space between the backslash and newline, warn about it.
if (SpacePos && !L->isLexingRawMode())
  L->Diag(SpacePos, diag::backslash_newline_space);

return true;
2527}

2529#ifdef __SSE2__1
2530#include <emmintrin.h>
2531#elif __ALTIVEC__
2532#include <altivec.h>
2533#undef bool
2534#endif

2536/// We have just read from input the / and * characters that started a comment.
2537/// Read until we find the * and / characters that terminate the comment.
2538/// Note that we don't bother decoding trigraphs or escaped newlines in block
2539/// comments, because they cannot cause the comment to end.  The only thing
2540/// that can happen is the comment could end with an escaped newline between
2541/// the terminating * and /.
2542///
2543/// If we're in KeepCommentMode or any CommentHandler has inserted
2544/// some tokens, this will store the first token and return true.
2545bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr,
                           bool &TokAtPhysicalStartOfLine) {
// Scan one character past where we should, looking for a '/' character.  Once
// we find it, check to see if it was preceded by a *.  This common
// optimization helps people who like to put a lot of * characters in their
// comments.

// The first character we get with newlines and trigraphs skipped to handle
// the degenerate /*/ case below correctly if the * has an escaped newline
// after it.
unsigned CharSize;
unsigned char C = getCharAndSize(CurPtr, CharSize);
CurPtr += CharSize;
if (C == 0 && CurPtr == BufferEnd+1) {
  if (!isLexingRawMode())
    Diag(BufferPtr, diag::err_unterminated_block_comment);
  --CurPtr;

  // KeepWhitespaceMode should return this broken comment as a token.  Since
  // it isn't a well formed comment, just return it as an 'unknown' token.
  if (isKeepWhitespaceMode()) {
    FormTokenWithChars(Result, CurPtr, tok::unknown);
    return true;
  }

  BufferPtr = CurPtr;
  return false;
}

// Check to see if the first character after the '/*' is another /.  If so,
// then this slash does not end the block comment, it is part of it.
if (C == '/')
  C = *CurPtr++;

while (true) {
  // Skip over all non-interesting characters until we find end of buffer or a
  // (probably ending) '/' character.
  if (CurPtr + 24 < BufferEnd &&
      // If there is a code-completion point avoid the fast scan because it
      // doesn't check for '\0'.
      !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
    // While not aligned to a 16-byte boundary.
    while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0)
      C = *CurPtr++;

    if (C == '/') goto FoundSlash;

2592#ifdef __SSE2__1
    __m128i Slashes = _mm_set1_epi8('/');
    while (CurPtr+16 <= BufferEnd) {
      int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr,
                                  Slashes));
      if (cmp != 0) {
        // Adjust the pointer to point directly after the first slash. It's
        // not necessary to set C here, it will be overwritten at the end of
        // the outer loop.
        CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1;
        goto FoundSlash;
      }
      CurPtr += 16;
    }
2606#elif __ALTIVEC__
    __vector unsigned char Slashes = {
      '/', '/', '/', '/',  '/', '/', '/', '/',
      '/', '/', '/', '/',  '/', '/', '/', '/'
    };
    while (CurPtr + 16 <= BufferEnd &&
           !vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes))
      CurPtr += 16;
2614#else
    // Scan for '/' quickly.  Many block comments are very large.
    while (CurPtr[0] != '/' &&
           CurPtr[1] != '/' &&
           CurPtr[2] != '/' &&
           CurPtr[3] != '/' &&
           CurPtr+4 < BufferEnd) {
      CurPtr += 4;
    }
2623#endif

    // It has to be one of the bytes scanned, increment to it and read one.
    C = *CurPtr++;
  }

  // Loop to scan the remainder.
  while (C != '/' && C != '\0')
    C = *CurPtr++;

  if (C == '/') {
FoundSlash:
    if (CurPtr[-2] == '*')  // We found the final */.  We're done!
      break;

    if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
      if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) {
        // We found the final */, though it had an escaped newline between the
        // * and /.  We're done!
        break;
      }
    }
    if (CurPtr[0] == '*' && CurPtr[1] != '/') {
      // If this is a /* inside of the comment, emit a warning.  Don't do this
      // if this is a /*/, which will end the comment.  This misses cases with
      // embedded escaped newlines, but oh well.
      if (!isLexingRawMode())
        Diag(CurPtr-1, diag::warn_nested_block_comment);
    }
  } else if (C == 0 && CurPtr == BufferEnd+1) {
    if (!isLexingRawMode())
      Diag(BufferPtr, diag::err_unterminated_block_comment);
    // Note: the user probably forgot a */.  We could continue immediately
    // after the /*, but this would involve lexing a lot of what really is the
    // comment, which surely would confuse the parser.
    --CurPtr;

    // KeepWhitespaceMode should return this broken comment as a token.  Since
    // it isn't a well formed comment, just return it as an 'unknown' token.
    if (isKeepWhitespaceMode()) {
      FormTokenWithChars(Result, CurPtr, tok::unknown);
      return true;
    }

    BufferPtr = CurPtr;
    return false;
  } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
    PP->CodeCompleteNaturalLanguage();
    cutOffLexing();
    return false;
  }

  C = *CurPtr++;
}

// Notify comment handlers about the comment unless we're in a #if 0 block.
if (PP && !isLexingRawMode() &&
    PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
                                          getSourceLocation(CurPtr)))) {
  BufferPtr = CurPtr;
  return true; // A token has to be returned.
}

// If we are returning comments as tokens, return this comment as a token.
if (inKeepCommentMode()) {
  FormTokenWithChars(Result, CurPtr, tok::comment);
  return true;
}

// It is common for the tokens immediately after a /**/ comment to be
// whitespace.  Instead of going through the big switch, handle it
// efficiently now.  This is safe even in KeepWhitespaceMode because we would
// have already returned above with the comment as a token.
if (isHorizontalWhitespace(*CurPtr)) {
  SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine);
  return false;
}

// Otherwise, just return so that the next character will be lexed as a token.
BufferPtr = CurPtr;
Result.setFlag(Token::LeadingSpace);
return false;
2705}

2707//===----------------------------------------------------------------------===//
2708// Primary Lexing Entry Points
2709//===----------------------------------------------------------------------===//

2711/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
2712/// uninterpreted string.  This switches the lexer out of directive mode.
2713void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
assert(ParsingPreprocessorDirective && ParsingFilename == false &&((void)0)
       "Must be in a preprocessing directive!")((void)0);
Token Tmp;
Tmp.startToken();

// CurPtr - Cache BufferPtr in an automatic variable.
const char *CurPtr = BufferPtr;
while (true) {
  char Char = getAndAdvanceChar(CurPtr, Tmp);
  switch (Char) {
  default:
    if (Result)
      Result->push_back(Char);
    break;
  case 0:  // Null.
    // Found end of file?
    if (CurPtr-1 != BufferEnd) {
      if (isCodeCompletionPoint(CurPtr-1)) {
        PP->CodeCompleteNaturalLanguage();
        cutOffLexing();
        return;
      }

      // Nope, normal character, continue.
      if (Result)
        Result->push_back(Char);
      break;
    }
    // FALL THROUGH.
    LLVM_FALLTHROUGH[[gnu::fallthrough]];
  case '\r':
  case '\n':
    // Okay, we found the end of the line. First, back up past the \0, \r, \n.
    assert(CurPtr[-1] == Char && "Trigraphs for newline?")((void)0);
    BufferPtr = CurPtr-1;

    // Next, lex the character, which should handle the EOD transition.
    Lex(Tmp);
    if (Tmp.is(tok::code_completion)) {
      if (PP)
        PP->CodeCompleteNaturalLanguage();
      Lex(Tmp);
    }
    assert(Tmp.is(tok::eod) && "Unexpected token!")((void)0);

    // Finally, we're done;
    return;
  }
}
2763}

2765/// LexEndOfFile - CurPtr points to the end of this file.  Handle this
2766/// condition, reporting diagnostics and handling other edge cases as required.
2767/// This returns true if Result contains a token, false if PP.Lex should be
2768/// called again.
2769bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
// If we hit the end of the file while parsing a preprocessor directive,
// end the preprocessor directive first.  The next token returned will
// then be the end of file.
if (ParsingPreprocessorDirective) {
  // Done parsing the "line".
  ParsingPreprocessorDirective = false;
  // Update the location of token as well as BufferPtr.
  FormTokenWithChars(Result, CurPtr, tok::eod);

  // Restore comment saving mode, in case it was disabled for directive.
  if (PP)
    resetExtendedTokenMode();
  return true;  // Have a token.
}

// If we are in raw mode, return this event as an EOF token.  Let the caller
// that put us in raw mode handle the event.
if (isLexingRawMode()) {
  Result.startToken();
  BufferPtr = BufferEnd;
  FormTokenWithChars(Result, BufferEnd, tok::eof);
  return true;
}

if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) {
  PP->setRecordedPreambleConditionalStack(ConditionalStack);
  // If the preamble cuts off the end of a header guard, consider it guarded.
  // The guard is valid for the preamble content itself, and for tools the
  // most useful answer is "yes, this file has a header guard".
  if (!ConditionalStack.empty())
    MIOpt.ExitTopLevelConditional();
  ConditionalStack.clear();
}

// Issue diagnostics for unterminated #if and missing newline.

// If we are in a #if directive, emit an error.
while (!ConditionalStack.empty()) {
  if (PP->getCodeCompletionFileLoc() != FileLoc)
    PP->Diag(ConditionalStack.back().IfLoc,
             diag::err_pp_unterminated_conditional);
  ConditionalStack.pop_back();
}

SourceLocation EndLoc = getSourceLocation(BufferEnd);
// C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue
// a pedwarn.
if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) {
  DiagnosticsEngine &Diags = PP->getDiagnostics();
  unsigned DiagID;

  if (LangOpts.CPlusPlus11) {
    // C++11 [lex.phases] 2.2 p2
    // Prefer the C++98 pedantic compatibility warning over the generic,
    // non-extension, user-requested "missing newline at EOF" warning.
    if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) {
      DiagID = diag::warn_cxx98_compat_no_newline_eof;
    } else {
      DiagID = diag::warn_no_newline_eof;
    }
  } else {
    DiagID = diag::ext_no_newline_eof;
  }

  Diag(BufferEnd, DiagID)
    << FixItHint::CreateInsertion(EndLoc, "\n");
}

BufferPtr = CurPtr;

// Finally, let the preprocessor handle this.
return PP->HandleEndOfFile(Result, EndLoc, isPragmaLexer());
2842}

2844/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
2845/// the specified lexer will return a tok::l_paren token, 0 if it is something
2846/// else and 2 if there are no more tokens in the buffer controlled by the
2847/// lexer.
2848unsigned Lexer::isNextPPTokenLParen() {
assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?")((void)0);

// Switch to 'skipping' mode.  This will ensure that we can lex a token
// without emitting diagnostics, disables macro expansion, and will cause EOF
// to return an EOF token instead of popping the include stack.
LexingRawMode = true;

// Save state that can be changed while lexing so that we can restore it.
const char *TmpBufferPtr = BufferPtr;
bool inPPDirectiveMode = ParsingPreprocessorDirective;
bool atStartOfLine = IsAtStartOfLine;
bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
bool leadingSpace = HasLeadingSpace;

Token Tok;
Lex(Tok);

// Restore state that may have changed.
BufferPtr = TmpBufferPtr;
ParsingPreprocessorDirective = inPPDirectiveMode;
HasLeadingSpace = leadingSpace;
IsAtStartOfLine = atStartOfLine;
IsAtPhysicalStartOfLine = atPhysicalStartOfLine;

// Restore the lexer back to non-skipping mode.
LexingRawMode = false;

if (Tok.is(tok::eof))
  return 2;
return Tok.is(tok::l_paren);
2879}

2881/// Find the end of a version control conflict marker.
2882static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
                                 ConflictMarkerKind CMK) {
const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(TermLen);
size_t Pos = RestOfBuffer.find(Terminator);
while (Pos != StringRef::npos) {
  // Must occur at start of line.
  if (Pos == 0 ||
      (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) {
    RestOfBuffer = RestOfBuffer.substr(Pos+TermLen);
    Pos = RestOfBuffer.find(Terminator);
    continue;
  }
  return RestOfBuffer.data()+Pos;
}
return nullptr;
2899}

2901/// IsStartOfConflictMarker - If the specified pointer is the start of a version
2902/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
2903/// and recover nicely.  This returns true if it is a conflict marker and false
2904/// if not.
2905bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
// Only a conflict marker if it starts at the beginning of a line.
if (CurPtr != BufferStart &&
    CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
  return false;

// Check to see if we have <<<<<<< or >>>>.
if (!StringRef(CurPtr, BufferEnd - CurPtr).startswith("<<<<<<<") &&
    !StringRef(CurPtr, BufferEnd - CurPtr).startswith(">>>> "))
  return false;

// If we have a situation where we don't care about conflict markers, ignore
// it.
if (CurrentConflictMarkerState || isLexingRawMode())
  return false;

ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;

// Check to see if there is an ending marker somewhere in the buffer at the
// start of a line to terminate this conflict marker.
if (FindConflictEnd(CurPtr, BufferEnd, Kind)) {
  // We found a match.  We are really in a conflict marker.
  // Diagnose this, and ignore to the end of line.
  Diag(CurPtr, diag::err_conflict_marker);
  CurrentConflictMarkerState = Kind;

  // Skip ahead to the end of line.  We know this exists because the
  // end-of-conflict marker starts with \r or \n.
  while (*CurPtr != '\r' && *CurPtr != '\n') {
    assert(CurPtr != BufferEnd && "Didn't find end of line")((void)0);
    ++CurPtr;
  }
  BufferPtr = CurPtr;
  return true;
}

// No end of conflict marker found.
return false;
2943}

2945/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
2946/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
2947/// is the end of a conflict marker.  Handle it by ignoring up until the end of
2948/// the line.  This returns true if it is a conflict marker and false if not.
2949bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
// Only a conflict marker if it starts at the beginning of a line.
if (CurPtr != BufferStart &&
    CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
  return false;

// If we have a situation where we don't care about conflict markers, ignore
// it.
if (!CurrentConflictMarkerState || isLexingRawMode())
  return false;

// Check to see if we have the marker (4 characters in a row).
for (unsigned i = 1; i != 4; ++i)
  if (CurPtr[i] != CurPtr[0])
    return false;

// If we do have it, search for the end of the conflict marker.  This could
// fail if it got skipped with a '#if 0' or something.  Note that CurPtr might
// be the end of conflict marker.
if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
                                      CurrentConflictMarkerState)) {
  CurPtr = End;

  // Skip ahead to the end of line.
  while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
    ++CurPtr;

  BufferPtr = CurPtr;

  // No longer in the conflict marker.
  CurrentConflictMarkerState = CMK_None;
  return true;
}

return false;
2984}

2986static const char *findPlaceholderEnd(const char *CurPtr,
                                    const char *BufferEnd) {
if (CurPtr == BufferEnd)
  return nullptr;
BufferEnd -= 1; // Scan until the second last character.
for (; CurPtr != BufferEnd; ++CurPtr) {
  if (CurPtr[0] == '#' && CurPtr[1] == '>')
    return CurPtr + 2;
}
return nullptr;
2996}

2998bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) {
assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!")((void)0);
if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode)
  return false;
const char *End = findPlaceholderEnd(CurPtr + 1, BufferEnd);
if (!End)
  return false;
const char *Start = CurPtr - 1;
if (!LangOpts.AllowEditorPlaceholders)
  Diag(Start, diag::err_placeholder_in_source);
Result.startToken();
FormTokenWithChars(Result, End, tok::raw_identifier);
Result.setRawIdentifierData(Start);
PP->LookUpIdentifierInfo(Result);
Result.setFlag(Token::IsEditorPlaceholder);
BufferPtr = End;
return true;
3015}

3017bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
if (PP && PP->isCodeCompletionEnabled()) {
  SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart);
  return Loc == PP->getCodeCompletionLoc();
}

return false;
3024}

3026uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc,
                         Token *Result) {
unsigned CharSize;
char Kind = getCharAndSize(StartPtr, CharSize);

unsigned NumHexDigits;
if (Kind == 'u')
  NumHexDigits = 4;
else if (Kind == 'U')
  NumHexDigits = 8;
else
  return 0;

if (!LangOpts.CPlusPlus && !LangOpts.C99) {
  if (Result && !isLexingRawMode())
    Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89);
  return 0;
}

const char *CurPtr = StartPtr + CharSize;
const char *KindLoc = &CurPtr[-1];

uint32_t CodePoint = 0;
for (unsigned i = 0; i < NumHexDigits; ++i) {
  char C = getCharAndSize(CurPtr, CharSize);

  unsigned Value = llvm::hexDigitValue(C);
  if (Value == -1U) {
    if (Result && !isLexingRawMode()) {
      if (i == 0) {
        Diag(BufferPtr, diag::warn_ucn_escape_no_digits)
          << StringRef(KindLoc, 1);
      } else {
        Diag(BufferPtr, diag::warn_ucn_escape_incomplete);

        // If the user wrote \U1234, suggest a fixit to \u.
        if (i == 4 && NumHexDigits == 8) {
          CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1);
          Diag(KindLoc, diag::note_ucn_four_not_eight)
            << FixItHint::CreateReplacement(URange, "u");
        }
      }
    }

    return 0;
  }

  CodePoint <<= 4;
  CodePoint += Value;

  CurPtr += CharSize;
}

if (Result) {
  Result->setFlag(Token::HasUCN);
  if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2)
    StartPtr = CurPtr;
  else
    while (StartPtr != CurPtr)
      (void)getAndAdvanceChar(StartPtr, *Result);
} else {
  StartPtr = CurPtr;
}

// Don't apply C family restrictions to UCNs in assembly mode
if (LangOpts.AsmPreprocessor)
  return CodePoint;

// C99 6.4.3p2: A universal character name shall not specify a character whose
//   short identifier is less than 00A0 other than 0024 ($), 0040 (@), or
//   0060 (`), nor one in the range D800 through DFFF inclusive.)
// C++11 [lex.charset]p2: If the hexadecimal value for a
//   universal-character-name corresponds to a surrogate code point (in the
//   range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally,
//   if the hexadecimal value for a universal-character-name outside the
//   c-char-sequence, s-char-sequence, or r-char-sequence of a character or
//   string literal corresponds to a control character (in either of the
//   ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the
//   basic source character set, the program is ill-formed.
if (CodePoint < 0xA0) {
  if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60)
    return CodePoint;

  // We don't use isLexingRawMode() here because we need to warn about bad
  // UCNs even when skipping preprocessing tokens in a #if block.
  if (Result && PP) {
    if (CodePoint < 0x20 || CodePoint >= 0x7F)
      Diag(BufferPtr, diag::err_ucn_control_character);
    else {
      char C = static_cast<char>(CodePoint);
      Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1);
    }
  }

  return 0;
} else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) {
  // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't.
  // We don't use isLexingRawMode() here because we need to diagnose bad
  // UCNs even when skipping preprocessing tokens in a #if block.
  if (Result && PP) {
    if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11)
      Diag(BufferPtr, diag::warn_ucn_escape_surrogate);
    else
      Diag(BufferPtr, diag::err_ucn_escape_invalid);
  }
  return 0;
}

return CodePoint;
3135}

3137bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C,
                                 const char *CurPtr) {
static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars(
    UnicodeWhitespaceCharRanges);
if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
    UnicodeWhitespaceChars.contains(C)) {
  Diag(BufferPtr, diag::ext_unicode_whitespace)
    << makeCharRange(*this, BufferPtr, CurPtr);

  Result.setFlag(Token::LeadingSpace);
  return true;
}
return false;
3150}

3152bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) {
  if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
      !PP->isPreprocessedOutput()) {
    maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C,
                              makeCharRange(*this, BufferPtr, CurPtr),
                              /*IsFirst=*/true);
    maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), C,
                               makeCharRange(*this, BufferPtr, CurPtr));
  }

  MIOpt.ReadToken();
  return LexIdentifier(Result, CurPtr);
}

if (!isLexingRawMode() && !ParsingPreprocessorDirective &&
    !PP->isPreprocessedOutput() &&
    !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) {
  // Non-ASCII characters tend to creep into source code unintentionally.
  // Instead of letting the parser complain about the unknown token,
  // just drop the character.
  // Note that we can /only/ do this when the non-ASCII character is actually
  // spelled as Unicode, not written as a UCN. The standard requires that
  // we not throw away any possible preprocessor tokens, but there's a
  // loophole in the mapping of Unicode characters to basic character set
  // characters that allows us to map these particular characters to, say,
  // whitespace.
  Diag(BufferPtr, diag::err_non_ascii)
    << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr));

  BufferPtr = CurPtr;
  return false;
}

// Otherwise, we have an explicit UCN or a character that's unlikely to show
// up by accident.
MIOpt.ReadToken();
FormTokenWithChars(Result, CurPtr, tok::unknown);
return true;
3191}

3193void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
IsAtStartOfLine = Result.isAtStartOfLine();
HasLeadingSpace = Result.hasLeadingSpace();
HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro();
// Note that this doesn't affect IsAtPhysicalStartOfLine.
3198}

3200bool Lexer::Lex(Token &Result) {
// Start a new token.
Result.startToken();

// Set up misc whitespace flags for LexTokenInternal.
if (IsAtStartOfLine) {
  Result.setFlag(Token::StartOfLine);
  IsAtStartOfLine = false;
}

if (HasLeadingSpace) {
  Result.setFlag(Token::LeadingSpace);
  HasLeadingSpace = false;
}

if (HasLeadingEmptyMacro) {
  Result.setFlag(Token::LeadingEmptyMacro);
  HasLeadingEmptyMacro = false;
}

bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
IsAtPhysicalStartOfLine = false;
bool isRawLex = isLexingRawMode();
(void) isRawLex;
bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine);
// (After the LexTokenInternal call, the lexer might be destroyed.)
assert((returnedToken || !isRawLex) && "Raw lex must succeed")((void)0);
return returnedToken;
3228}

3230/// LexTokenInternal - This implements a simple C family lexer.  It is an
3231/// extremely performance critical piece of code.  This assumes that the buffer
3232/// has a null character at the end of the file.  This returns a preprocessing
3233/// token, not a normal token, as such, it is an internal interface.  It assumes
3234/// that the Flags of result have been cleared before calling this.
3235bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
3236LexNextToken:
// New token, can't need cleaning yet.
Result.clearFlag(Token::NeedsCleaning);
Result.setIdentifierInfo(nullptr);

// CurPtr - Cache BufferPtr in an automatic variable.
const char *CurPtr = BufferPtr;

// Small amounts of horizontal whitespace is very common between tokens.
if (isHorizontalWhitespace(*CurPtr)) {
1
Assuming the condition is false→
2
←
Taking false branch→
  do {
    ++CurPtr;
  } while (isHorizontalWhitespace(*CurPtr));

  // If we are keeping whitespace and other tokens, just return what we just
  // skipped.  The next lexer invocation will return the token after the
  // whitespace.
  if (isKeepWhitespaceMode()) {
    FormTokenWithChars(Result, CurPtr, tok::unknown);
    // FIXME: The next token will not have LeadingSpace set.
    return true;
  }

  BufferPtr = CurPtr;
  Result.setFlag(Token::LeadingSpace);
}

unsigned SizeTmp, SizeTmp2;   // Temporaries for use in cases below.

// Read a character, advancing over it.
char Char = getAndAdvanceChar(CurPtr, Result);
tok::TokenKind Kind;

if (!isVerticalWhitespace(Char))
3
←
Assuming the condition is false→
4
←
Taking false branch→
  NewLinePtr = nullptr;

switch (Char) {
5
←
Control jumps to 'case 32:'  at line 3341→
case 0:  // Null.
  // Found end of file?
  if (CurPtr-1 == BufferEnd)
    return LexEndOfFile(Result, CurPtr-1);

  // Check if we are performing code completion.
  if (isCodeCompletionPoint(CurPtr-1)) {
    // Return the code-completion token.
    Result.startToken();
    FormTokenWithChars(Result, CurPtr, tok::code_completion);
    return true;
  }

  if (!isLexingRawMode())
    Diag(CurPtr-1, diag::null_in_file);
  Result.setFlag(Token::LeadingSpace);
  if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
    return true; // KeepWhitespaceMode

  // We know the lexer hasn't changed, so just try again with this lexer.
  // (We manually eliminate the tail call to avoid recursion.)
  goto LexNextToken;

case 26:  // DOS & CP/M EOF: "^Z".
  // If we're in Microsoft extensions mode, treat this as end of file.
  if (LangOpts.MicrosoftExt) {
    if (!isLexingRawMode())
      Diag(CurPtr-1, diag::ext_ctrl_z_eof_microsoft);
    return LexEndOfFile(Result, CurPtr-1);
  }

  // If Microsoft extensions are disabled, this is just random garbage.
  Kind = tok::unknown;
  break;

case '\r':
  if (CurPtr[0] == '\n')
    (void)getAndAdvanceChar(CurPtr, Result);
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case '\n':
  // If we are inside a preprocessor directive and we see the end of line,
  // we know we are done with the directive, so return an EOD token.
  if (ParsingPreprocessorDirective) {
    // Done parsing the "line".
    ParsingPreprocessorDirective = false;

    // Restore comment saving mode, in case it was disabled for directive.
    if (PP)
      resetExtendedTokenMode();

    // Since we consumed a newline, we are back at the start of a line.
    IsAtStartOfLine = true;
    IsAtPhysicalStartOfLine = true;
    NewLinePtr = CurPtr - 1;

    Kind = tok::eod;
    break;
  }

  // No leading whitespace seen so far.
  Result.clearFlag(Token::LeadingSpace);

  if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
    return true; // KeepWhitespaceMode

  // We only saw whitespace, so just try again with this lexer.
  // (We manually eliminate the tail call to avoid recursion.)
  goto LexNextToken;
case ' ':
case '\t':
case '\f':
case '\v':
SkipHorizontalWhitespace:
  Result.setFlag(Token::LeadingSpace);
  if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
6
←
Taking false branch→
    return true; // KeepWhitespaceMode

SkipIgnoredUnits:
  CurPtr = BufferPtr;

  // If the next token is obviously a // or /* */ comment, skip it efficiently
  // too (without going through the big switch stmt).
  if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
7
←
Assuming the condition is true→
8
←
Assuming the condition is true→
12
←
Taking true branch→
      LangOpts.LineComment &&
9
←
Assuming field 'LineComment' is not equal to 0→
      (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) {
10
←
Assuming field 'CPlusPlus' is 0→
11
←
Assuming field 'TraditionalCPP' is 0→
    if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine))
13
←
Calling 'Lexer::SkipLineComment'→
      return true; // There is a token to return.
    goto SkipIgnoredUnits;
  } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
    if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine))
      return true; // There is a token to return.
    goto SkipIgnoredUnits;
  } else if (isHorizontalWhitespace(*CurPtr)) {
    goto SkipHorizontalWhitespace;
  }
  // We only saw whitespace, so just try again with this lexer.
  // (We manually eliminate the tail call to avoid recursion.)
  goto LexNextToken;

// C99 6.4.4.1: Integer Constants.
// C99 6.4.4.2: Floating Constants.
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();
  return LexNumericConstant(Result, CurPtr);

case 'u':   // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();

  if (LangOpts.CPlusPlus11 || LangOpts.C11) {
    Char = getCharAndSize(CurPtr, SizeTmp);

    // UTF-16 string literal
    if (Char == '"')
      return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                              tok::utf16_string_literal);

    // UTF-16 character constant
    if (Char == '\'')
      return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                             tok::utf16_char_constant);

    // UTF-16 raw string literal
    if (Char == 'R' && LangOpts.CPlusPlus11 &&
        getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
      return LexRawStringLiteral(Result,
                             ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                                         SizeTmp2, Result),
                             tok::utf16_string_literal);

    if (Char == '8') {
      char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2);

      // UTF-8 string literal
      if (Char2 == '"')
        return LexStringLiteral(Result,
                             ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                                         SizeTmp2, Result),
                             tok::utf8_string_literal);
      if (Char2 == '\'' && LangOpts.CPlusPlus17)
        return LexCharConstant(
            Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                                SizeTmp2, Result),
            tok::utf8_char_constant);

      if (Char2 == 'R' && LangOpts.CPlusPlus11) {
        unsigned SizeTmp3;
        char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
        // UTF-8 raw string literal
        if (Char3 == '"') {
          return LexRawStringLiteral(Result,
                 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                                         SizeTmp2, Result),
                             SizeTmp3, Result),
                 tok::utf8_string_literal);
        }
      }
    }
  }

  // treat u like the start of an identifier.
  return LexIdentifier(Result, CurPtr);

case 'U':   // Identifier (Uber) or C11/C++11 UTF-32 string literal
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();

  if (LangOpts.CPlusPlus11 || LangOpts.C11) {
    Char = getCharAndSize(CurPtr, SizeTmp);

    // UTF-32 string literal
    if (Char == '"')
      return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                              tok::utf32_string_literal);

    // UTF-32 character constant
    if (Char == '\'')
      return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                             tok::utf32_char_constant);

    // UTF-32 raw string literal
    if (Char == 'R' && LangOpts.CPlusPlus11 &&
        getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
      return LexRawStringLiteral(Result,
                             ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                                         SizeTmp2, Result),
                             tok::utf32_string_literal);
  }

  // treat U like the start of an identifier.
  return LexIdentifier(Result, CurPtr);

case 'R': // Identifier or C++0x raw string literal
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();

  if (LangOpts.CPlusPlus11) {
    Char = getCharAndSize(CurPtr, SizeTmp);

    if (Char == '"')
      return LexRawStringLiteral(Result,
                                 ConsumeChar(CurPtr, SizeTmp, Result),
                                 tok::string_literal);
  }

  // treat R like the start of an identifier.
  return LexIdentifier(Result, CurPtr);

case 'L':   // Identifier (Loony) or wide literal (L'x' or L"xyz").
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();
  Char = getCharAndSize(CurPtr, SizeTmp);

  // Wide string literal.
  if (Char == '"')
    return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                            tok::wide_string_literal);

  // Wide raw string literal.
  if (LangOpts.CPlusPlus11 && Char == 'R' &&
      getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
    return LexRawStringLiteral(Result,
                             ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                                         SizeTmp2, Result),
                             tok::wide_string_literal);

  // Wide character constant.
  if (Char == '\'')
    return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                           tok::wide_char_constant);
  // FALL THROUGH, treating L like the start of an identifier.
  LLVM_FALLTHROUGH[[gnu::fallthrough]];

// C99 6.4.2: Identifiers.
case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
case 'H': case 'I': case 'J': case 'K':    /*'L'*/case 'M': case 'N':
case 'O': case 'P': case 'Q':    /*'R'*/case 'S': case 'T':    /*'U'*/
case 'V': case 'W': case 'X': case 'Y': case 'Z':
case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
case 'o': case 'p': case 'q': case 'r': case 's': case 't':    /*'u'*/
case 'v': case 'w': case 'x': case 'y': case 'z':
case '_':
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();
  return LexIdentifier(Result, CurPtr);

case '$':   // $ in identifiers.
  if (LangOpts.DollarIdents) {
    if (!isLexingRawMode())
      Diag(CurPtr-1, diag::ext_dollar_in_identifier);
    // Notify MIOpt that we read a non-whitespace/non-comment token.
    MIOpt.ReadToken();
    return LexIdentifier(Result, CurPtr);
  }

  Kind = tok::unknown;
  break;

// C99 6.4.4: Character Constants.
case '\'':
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();
  return LexCharConstant(Result, CurPtr, tok::char_constant);

// C99 6.4.5: String Literals.
case '"':
  // Notify MIOpt that we read a non-whitespace/non-comment token.
  MIOpt.ReadToken();
  return LexStringLiteral(Result, CurPtr,
                          ParsingFilename ? tok::header_name
                                          : tok::string_literal);

// C99 6.4.6: Punctuators.
case '?':
  Kind = tok::question;
  break;
case '[':
  Kind = tok::l_square;
  break;
case ']':
  Kind = tok::r_square;
  break;
case '(':
  Kind = tok::l_paren;
  break;
case ')':
  Kind = tok::r_paren;
  break;
case '{':
  Kind = tok::l_brace;
  break;
case '}':
  Kind = tok::r_brace;
  break;
case '.':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char >= '0' && Char <= '9') {
    // Notify MIOpt that we read a non-whitespace/non-comment token.
    MIOpt.ReadToken();

    return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
  } else if (LangOpts.CPlusPlus && Char == '*') {
    Kind = tok::periodstar;
    CurPtr += SizeTmp;
  } else if (Char == '.' &&
             getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') {
    Kind = tok::ellipsis;
    CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                         SizeTmp2, Result);
  } else {
    Kind = tok::period;
  }
  break;
case '&':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '&') {
    Kind = tok::ampamp;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else if (Char == '=') {
    Kind = tok::ampequal;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else {
    Kind = tok::amp;
  }
  break;
case '*':
  if (getCharAndSize(CurPtr, SizeTmp) == '=') {
    Kind = tok::starequal;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else {
    Kind = tok::star;
  }
  break;
case '+':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '+') {
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::plusplus;
  } else if (Char == '=') {
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::plusequal;
  } else {
    Kind = tok::plus;
  }
  break;
case '-':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '-') {      // --
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::minusminus;
  } else if (Char == '>' && LangOpts.CPlusPlus &&
             getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') {  // C++ ->*
    CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                         SizeTmp2, Result);
    Kind = tok::arrowstar;
  } else if (Char == '>') {   // ->
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::arrow;
  } else if (Char == '=') {   // -=
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::minusequal;
  } else {
    Kind = tok::minus;
  }
  break;
case '~':
  Kind = tok::tilde;
  break;
case '!':
  if (getCharAndSize(CurPtr, SizeTmp) == '=') {
    Kind = tok::exclaimequal;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else {
    Kind = tok::exclaim;
  }
  break;
case '/':
  // 6.4.9: Comments
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '/') {         // Line comment.
    // Even if Line comments are disabled (e.g. in C89 mode), we generally
    // want to lex this as a comment.  There is one problem with this though,
    // that in one particular corner case, this can change the behavior of the
    // resultant program.  For example, In  "foo //**/ bar", C89 would lex
    // this as "foo / bar" and languages with Line comments would lex it as
    // "foo".  Check to see if the character after the second slash is a '*'.
    // If so, we will lex that as a "/" instead of the start of a comment.
    // However, we never do this if we are just preprocessing.
    bool TreatAsComment = LangOpts.LineComment &&
                          (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP);
    if (!TreatAsComment)
      if (!(PP && PP->isPreprocessedOutput()))
        TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*';

    if (TreatAsComment) {
      if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                          TokAtPhysicalStartOfLine))
        return true; // There is a token to return.

      // It is common for the tokens immediately after a // comment to be
      // whitespace (indentation for the next line).  Instead of going through
      // the big switch, handle it efficiently now.
      goto SkipIgnoredUnits;
    }
  }

  if (Char == '*') {  // /**/ comment.
    if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result),
                         TokAtPhysicalStartOfLine))
      return true; // There is a token to return.

    // We only saw whitespace, so just try again with this lexer.
    // (We manually eliminate the tail call to avoid recursion.)
    goto LexNextToken;
  }

  if (Char == '=') {
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::slashequal;
  } else {
    Kind = tok::slash;
  }
  break;
case '%':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '=') {
    Kind = tok::percentequal;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else if (LangOpts.Digraphs && Char == '>') {
    Kind = tok::r_brace;                             // '%>' -> '}'
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else if (LangOpts.Digraphs && Char == ':') {
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Char = getCharAndSize(CurPtr, SizeTmp);
    if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') {
      Kind = tok::hashhash;                          // '%:%:' -> '##'
      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                           SizeTmp2, Result);
    } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
      if (!isLexingRawMode())
        Diag(BufferPtr, diag::ext_charize_microsoft);
      Kind = tok::hashat;
    } else {                                         // '%:' -> '#'
      // We parsed a # character.  If this occurs at the start of the line,
      // it's actually the start of a preprocessing directive.  Callback to
      // the preprocessor to handle it.
      // TODO: -fpreprocessed mode??
      if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer)
        goto HandleDirective;

      Kind = tok::hash;
    }
  } else {
    Kind = tok::percent;
  }
  break;
case '<':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (ParsingFilename) {
    return LexAngledStringLiteral(Result, CurPtr);
  } else if (Char == '<') {
    char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
    if (After == '=') {
      Kind = tok::lesslessequal;
      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                           SizeTmp2, Result);
    } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) {
      // If this is actually a '<<<<<<<' version control conflict marker,
      // recognize it as such and recover nicely.
      goto LexNextToken;
    } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) {
      // If this is '<<<<' and we're in a Perforce-style conflict marker,
      // ignore it.
      goto LexNextToken;
    } else if (LangOpts.CUDA && After == '<') {
      Kind = tok::lesslessless;
      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                           SizeTmp2, Result);
    } else {
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
      Kind = tok::lessless;
    }
  } else if (Char == '=') {
    char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
    if (After == '>') {
      if (getLangOpts().CPlusPlus20) {
        if (!isLexingRawMode())
          Diag(BufferPtr, diag::warn_cxx17_compat_spaceship);
        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                             SizeTmp2, Result);
        Kind = tok::spaceship;
        break;
      }
      // Suggest adding a space between the '<=' and the '>' to avoid a
      // change in semantics if this turns up in C++ <=17 mode.
      if (getLangOpts().CPlusPlus && !isLexingRawMode()) {
        Diag(BufferPtr, diag::warn_cxx20_compat_spaceship)
          << FixItHint::CreateInsertion(
                 getSourceLocation(CurPtr + SizeTmp, SizeTmp2), " ");
      }
    }
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::lessequal;
  } else if (LangOpts.Digraphs && Char == ':') {     // '<:' -> '['
    if (LangOpts.CPlusPlus11 &&
        getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') {
      // C++0x [lex.pptoken]p3:
      //  Otherwise, if the next three characters are <:: and the subsequent
      //  character is neither : nor >, the < is treated as a preprocessor
      //  token by itself and not as the first character of the alternative
      //  token <:.
      unsigned SizeTmp3;
      char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
      if (After != ':' && After != '>') {
        Kind = tok::less;
        if (!isLexingRawMode())
          Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon);
        break;
      }
    }

    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::l_square;
  } else if (LangOpts.Digraphs && Char == '%') {     // '<%' -> '{'
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::l_brace;
  } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 &&
             lexEditorPlaceholder(Result, CurPtr)) {
    return true;
  } else {
    Kind = tok::less;
  }
  break;
case '>':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '=') {
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::greaterequal;
  } else if (Char == '>') {
    char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
    if (After == '=') {
      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                           SizeTmp2, Result);
      Kind = tok::greatergreaterequal;
    } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) {
      // If this is actually a '>>>>' conflict marker, recognize it as such
      // and recover nicely.
      goto LexNextToken;
    } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) {
      // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
      goto LexNextToken;
    } else if (LangOpts.CUDA && After == '>') {
      Kind = tok::greatergreatergreater;
      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
                           SizeTmp2, Result);
    } else {
      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
      Kind = tok::greatergreater;
    }
  } else {
    Kind = tok::greater;
  }
  break;
case '^':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '=') {
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::caretequal;
  } else if (LangOpts.OpenCL && Char == '^') {
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
    Kind = tok::caretcaret;
  } else {
    Kind = tok::caret;
  }
  break;
case '|':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '=') {
    Kind = tok::pipeequal;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else if (Char == '|') {
    // If this is '|||||||' and we're in a conflict marker, ignore it.
    if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
      goto LexNextToken;
    Kind = tok::pipepipe;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else {
    Kind = tok::pipe;
  }
  break;
case ':':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (LangOpts.Digraphs && Char == '>') {
    Kind = tok::r_square; // ':>' -> ']'
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else if ((LangOpts.CPlusPlus ||
              LangOpts.DoubleSquareBracketAttributes) &&
             Char == ':') {
    Kind = tok::coloncolon;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else {
    Kind = tok::colon;
  }
  break;
case ';':
  Kind = tok::semi;
  break;
case '=':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '=') {
    // If this is '====' and we're in a conflict marker, ignore it.
    if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1))
      goto LexNextToken;

    Kind = tok::equalequal;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else {
    Kind = tok::equal;
  }
  break;
case ',':
  Kind = tok::comma;
  break;
case '#':
  Char = getCharAndSize(CurPtr, SizeTmp);
  if (Char == '#') {
    Kind = tok::hashhash;
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else if (Char == '@' && LangOpts.MicrosoftExt) {  // #@ -> Charize
    Kind = tok::hashat;
    if (!isLexingRawMode())
      Diag(BufferPtr, diag::ext_charize_microsoft);
    CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
  } else {
    // We parsed a # character.  If this occurs at the start of the line,
    // it's actually the start of a preprocessing directive.  Callback to
    // the preprocessor to handle it.
    // TODO: -fpreprocessed mode??
    if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer)
      goto HandleDirective;

    Kind = tok::hash;
  }
  break;

case '@':
  // Objective C support.
  if (CurPtr[-1] == '@' && LangOpts.ObjC)
    Kind = tok::at;
  else
    Kind = tok::unknown;
  break;

// UCNs (C99 6.4.3, C++11 [lex.charset]p2)
case '\\':
  if (!LangOpts.AsmPreprocessor) {
    if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) {
      if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) {
        if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
          return true; // KeepWhitespaceMode

        // We only saw whitespace, so just try again with this lexer.
        // (We manually eliminate the tail call to avoid recursion.)
        goto LexNextToken;
      }

      return LexUnicode(Result, CodePoint, CurPtr);
    }
  }

  Kind = tok::unknown;
  break;

default: {
  if (isASCII(Char)) {
    Kind = tok::unknown;
    break;
  }

  llvm::UTF32 CodePoint;

  // We can't just reset CurPtr to BufferPtr because BufferPtr may point to
  // an escaped newline.
  --CurPtr;
  llvm::ConversionResult Status =
      llvm::convertUTF8Sequence((const llvm::UTF8 **)&CurPtr,
                                (const llvm::UTF8 *)BufferEnd,
                                &CodePoint,
                                llvm::strictConversion);
  if (Status == llvm::conversionOK) {
    if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) {
      if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine))
        return true; // KeepWhitespaceMode

      // We only saw whitespace, so just try again with this lexer.
      // (We manually eliminate the tail call to avoid recursion.)
      goto LexNextToken;
    }
    return LexUnicode(Result, CodePoint, CurPtr);
  }

  if (isLexingRawMode() || ParsingPreprocessorDirective ||
      PP->isPreprocessedOutput()) {
    ++CurPtr;
    Kind = tok::unknown;
    break;
  }

  // Non-ASCII characters tend to creep into source code unintentionally.
  // Instead of letting the parser complain about the unknown token,
  // just diagnose the invalid UTF-8, then drop the character.
  Diag(CurPtr, diag::err_invalid_utf8);

  BufferPtr = CurPtr+1;
  // We're pretending the character didn't exist, so just try again with
  // this lexer.
  // (We manually eliminate the tail call to avoid recursion.)
  goto LexNextToken;
}
}

// Notify MIOpt that we read a non-whitespace/non-comment token.
MIOpt.ReadToken();

// Update the location of token as well as BufferPtr.
FormTokenWithChars(Result, CurPtr, Kind);
return true;

4005HandleDirective:
// We parsed a # character and it's the start of a preprocessing directive.

FormTokenWithChars(Result, CurPtr, tok::hash);
PP->HandleDirective(Result);

if (PP->hadModuleLoaderFatalFailure()) {
  // With a fatal failure in the module loader, we abort parsing.
  assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof")((void)0);
  return true;
}

// We parsed the directive; lex a token with the new state.
return false;
4019}

←

/usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/include/clang/Lex/Lexer.h

1//===- Lexer.h - C Language Family Lexer ------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9//  This file defines the Lexer interface.
10//
11//===----------------------------------------------------------------------===//

13#ifndef LLVM_CLANG_LEX_LEXER_H
14#define LLVM_CLANG_LEX_LEXER_H

16#include "clang/Basic/LangOptions.h"
17#include "clang/Basic/SourceLocation.h"
18#include "clang/Basic/TokenKinds.h"
19#include "clang/Lex/PreprocessorLexer.h"
20#include "clang/Lex/Token.h"
21#include "llvm/ADT/Optional.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
24#include <cassert>
25#include <cstdint>
26#include <string>

28namespace llvm {

30class MemoryBufferRef;

32} // namespace llvm

34namespace clang {

36class DiagnosticBuilder;
37class Preprocessor;
38class SourceManager;

40/// ConflictMarkerKind - Kinds of conflict marker which the lexer might be
41/// recovering from.
42enum ConflictMarkerKind {
/// Not within a conflict marker.
CMK_None,

/// A normal or diff3 conflict marker, initiated by at least 7 "<"s,
/// separated by at least 7 "="s or "|"s, and terminated by at least 7 ">"s.
CMK_Normal,

/// A Perforce-style conflict marker, initiated by 4 ">"s,
/// separated by 4 "="s, and terminated by 4 "<"s.
CMK_Perforce
53};

55/// Describes the bounds (start, size) of the preamble and a flag required by
56/// PreprocessorOptions::PrecompiledPreambleBytes.
57/// The preamble includes the BOM, if any.
58struct PreambleBounds {
/// Size of the preamble in bytes.
unsigned Size;

/// Whether the preamble ends at the start of a new line.
///
/// Used to inform the lexer as to whether it's starting at the beginning of
/// a line after skipping the preamble.
bool PreambleEndsAtStartOfLine;

PreambleBounds(unsigned Size, bool PreambleEndsAtStartOfLine)
    : Size(Size), PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {}
70};

72/// Lexer - This provides a simple interface that turns a text buffer into a
73/// stream of tokens.  This provides no support for file reading or buffering,
74/// or buffering/seeking of tokens, only forward lexing is supported.  It relies
75/// on the specified Preprocessor object to handle preprocessor directives, etc.
76class Lexer : public PreprocessorLexer {
friend class Preprocessor;

void anchor() override;

//===--------------------------------------------------------------------===//
// Constant configuration values for this lexer.

// Start of the buffer.
const char *BufferStart;

// End of the buffer.
const char *BufferEnd;

// Location for start of file.
SourceLocation FileLoc;

// LangOpts enabled by this language (cache).
LangOptions LangOpts;

// True if lexer for _Pragma handling.
bool Is_PragmaLexer;

//===--------------------------------------------------------------------===//
// Context-specific lexing flags set by the preprocessor.
//

/// ExtendedTokenMode - The lexer can optionally keep comments and whitespace
/// and return them as tokens.  This is used for -C and -CC modes, and
/// whitespace preservation can be useful for some clients that want to lex
/// the file in raw mode and get every character from the file.
///
/// When this is set to 2 it returns comments and whitespace.  When set to 1
/// it returns comments, when it is set to 0 it returns normal tokens only.
unsigned char ExtendedTokenMode;

//===--------------------------------------------------------------------===//
// Context that changes as the file is lexed.
// NOTE: any state that mutates when in raw mode must have save/restore code
// in Lexer::isNextPPTokenLParen.

// BufferPtr - Current pointer into the buffer.  This is the next character
// to be lexed.
const char *BufferPtr;

// IsAtStartOfLine - True if the next lexed token should get the "start of
// line" flag set on it.
bool IsAtStartOfLine;

bool IsAtPhysicalStartOfLine;

bool HasLeadingSpace;

bool HasLeadingEmptyMacro;

// NewLinePtr - A pointer to new line character '\n' being lexed. For '\r\n',
// it also points to '\n.'
const char *NewLinePtr;

// CurrentConflictMarkerState - The kind of conflict marker we are handling.
ConflictMarkerKind CurrentConflictMarkerState;

void InitLexer(const char *BufStart, const char *BufPtr, const char *BufEnd);

140public:
/// Lexer constructor - Create a new lexer object for the specified buffer
/// with the specified preprocessor managing the lexing process.  This lexer
/// assumes that the associated file buffer and Preprocessor objects will
/// outlive it, so it doesn't take ownership of either of them.
Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile, Preprocessor &PP);

/// Lexer constructor - Create a new raw lexer object.  This object is only
/// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the
/// text range will outlive it, so it doesn't take ownership of it.
Lexer(SourceLocation FileLoc, const LangOptions &LangOpts,
      const char *BufStart, const char *BufPtr, const char *BufEnd);

/// Lexer constructor - Create a new raw lexer object.  This object is only
/// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the
/// text range will outlive it, so it doesn't take ownership of it.
Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
      const SourceManager &SM, const LangOptions &LangOpts);

Lexer(const Lexer &) = delete;
Lexer &operator=(const Lexer &) = delete;

/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
/// _Pragma expansion.  This has a variety of magic semantics that this method
/// sets up.  It returns a new'd Lexer that must be delete'd when done.
static Lexer *Create_PragmaLexer(SourceLocation SpellingLoc,
                                 SourceLocation ExpansionLocStart,
                                 SourceLocation ExpansionLocEnd,
                                 unsigned TokLen, Preprocessor &PP);

/// getLangOpts - Return the language features currently enabled.
/// NOTE: this lexer modifies features as a file is parsed!
const LangOptions &getLangOpts() const { return LangOpts; }

/// getFileLoc - Return the File Location for the file we are lexing out of.
/// The physical location encodes the location where the characters come from,
/// the virtual location encodes where we should *claim* the characters came
/// from.  Currently this is only used by _Pragma handling.
SourceLocation getFileLoc() const { return FileLoc; }

180private:
/// Lex - Return the next token in the file.  If this is the end of file, it
/// return the tok::eof token.  This implicitly involves the preprocessor.
bool Lex(Token &Result);

185public:
/// isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
bool isPragmaLexer() const { return Is_PragmaLexer; }

189private:
/// IndirectLex - An indirect call to 'Lex' that can be invoked via
///  the PreprocessorLexer interface.
void IndirectLex(Token &Result) override { Lex(Result); }

194public:
/// LexFromRawLexer - Lex a token from a designated raw lexer (one with no
/// associated preprocessor object.  Return true if the 'next character to
/// read' pointer points at the end of the lexer buffer, false otherwise.
bool LexFromRawLexer(Token &Result) {
  assert(LexingRawMode && "Not already in raw mode!")((void)0);
  Lex(Result);
  // Note that lexing to the end of the buffer doesn't implicitly delete the
  // lexer when in raw mode.
  return BufferPtr == BufferEnd;
}

/// isKeepWhitespaceMode - Return true if the lexer should return tokens for
/// every character in the file, including whitespace and comments.  This
/// should only be used in raw mode, as the preprocessor is not prepared to
/// deal with the excess tokens.
bool isKeepWhitespaceMode() const {
  return ExtendedTokenMode > 1;
}

/// SetKeepWhitespaceMode - This method lets clients enable or disable
/// whitespace retention mode.
void SetKeepWhitespaceMode(bool Val) {
  assert((!Val || LexingRawMode || LangOpts.TraditionalCPP) &&((void)0)
         "Can only retain whitespace in raw mode or -traditional-cpp")((void)0);
  ExtendedTokenMode = Val ? 2 : 0;
}

/// inKeepCommentMode - Return true if the lexer should return comments as
/// tokens.
bool inKeepCommentMode() const {
  return ExtendedTokenMode > 0;
}

/// SetCommentRetentionMode - Change the comment retention mode of the lexer
/// to the specified mode.  This is really only useful when lexing in raw
/// mode, because otherwise the lexer needs to manage this.
void SetCommentRetentionState(bool Mode) {
  assert(!isKeepWhitespaceMode() &&((void)0)
         "Can't play with comment retention state when retaining whitespace")((void)0);
  ExtendedTokenMode = Mode ? 1 : 0;
}

/// Sets the extended token mode back to its initial value, according to the
/// language options and preprocessor. This controls whether the lexer
/// produces comment and whitespace tokens.
///
/// This requires the lexer to have an associated preprocessor. A standalone
/// lexer has nothing to reset to.
void resetExtendedTokenMode();

/// Gets source code buffer.
StringRef getBuffer() const {
  return StringRef(BufferStart, BufferEnd - BufferStart);
}

/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
/// uninterpreted string.  This switches the lexer out of directive mode.
void ReadToEndOfLine(SmallVectorImpl<char> *Result = nullptr);


/// Diag - Forwarding function for diagnostics.  This translate a source
/// position in the current buffer into a SourceLocation object for rendering.
DiagnosticBuilder Diag(const char *Loc, unsigned DiagID) const;

/// getSourceLocation - Return a source location identifier for the specified
/// offset in the current file.
SourceLocation getSourceLocation(const char *Loc, unsigned TokLen = 1) const;

/// getSourceLocation - Return a source location for the next character in
/// the current file.
SourceLocation getSourceLocation() override {
  return getSourceLocation(BufferPtr);
}

/// Return the current location in the buffer.
const char *getBufferLocation() const { return BufferPtr; }

/// Returns the current lexing offset.
unsigned getCurrentBufferOffset() {
  assert(BufferPtr >= BufferStart && "Invalid buffer state")((void)0);
  return BufferPtr - BufferStart;
}

/// Skip over \p NumBytes bytes.
///
/// If the skip is successful, the next token will be lexed from the new
/// offset. The lexer also assumes that we skipped to the start of the line.
///
/// \returns true if the skip failed (new offset would have been past the
/// end of the buffer), false otherwise.
bool skipOver(unsigned NumBytes);

/// Stringify - Convert the specified string into a C string by i) escaping
/// '\\' and " characters and ii) replacing newline character(s) with "\\n".
/// If Charify is true, this escapes the ' character instead of ".
static std::string Stringify(StringRef Str, bool Charify = false);

/// Stringify - Convert the specified string into a C string by i) escaping
/// '\\' and " characters and ii) replacing newline character(s) with "\\n".
static void Stringify(SmallVectorImpl<char> &Str);

/// getSpelling - This method is used to get the spelling of a token into a
/// preallocated buffer, instead of as an std::string.  The caller is required
/// to allocate enough space for the token, which is guaranteed to be at least
/// Tok.getLength() bytes long.  The length of the actual result is returned.
///
/// Note that this method may do two possible things: it may either fill in
/// the buffer specified with characters, or it may *change the input pointer*
/// to point to a constant buffer with the data already in it (avoiding a
/// copy).  The caller is not allowed to modify the returned buffer pointer
/// if an internal buffer is returned.
static unsigned getSpelling(const Token &Tok, const char *&Buffer,
                            const SourceManager &SourceMgr,
                            const LangOptions &LangOpts,
                            bool *Invalid = nullptr);

/// getSpelling() - Return the 'spelling' of the Tok token.  The spelling of a
/// token is the characters used to represent the token in the source file
/// after trigraph expansion and escaped-newline folding.  In particular, this
/// wants to get the true, uncanonicalized, spelling of things like digraphs
/// UCNs, etc.
static std::string getSpelling(const Token &Tok,
                               const SourceManager &SourceMgr,
                               const LangOptions &LangOpts,
                               bool *Invalid = nullptr);

/// getSpelling - This method is used to get the spelling of the
/// token at the given source location.  If, as is usually true, it
/// is not necessary to copy any data, then the returned string may
/// not point into the provided buffer.
///
/// This method lexes at the expansion depth of the given
/// location and does not jump to the expansion or spelling
/// location.
static StringRef getSpelling(SourceLocation loc,
                             SmallVectorImpl<char> &buffer,
                             const SourceManager &SM,
                             const LangOptions &options,
                             bool *invalid = nullptr);

/// MeasureTokenLength - Relex the token at the specified location and return
/// its length in bytes in the input file.  If the token needs cleaning (e.g.
/// includes a trigraph or an escaped newline) then this count includes bytes
/// that are part of that.
static unsigned MeasureTokenLength(SourceLocation Loc,
                                   const SourceManager &SM,
                                   const LangOptions &LangOpts);

/// Relex the token at the specified location.
/// \returns true if there was a failure, false on success.
static bool getRawToken(SourceLocation Loc, Token &Result,
                        const SourceManager &SM,
                        const LangOptions &LangOpts,
                        bool IgnoreWhiteSpace = false);

/// Given a location any where in a source buffer, find the location
/// that corresponds to the beginning of the token in which the original
/// source location lands.
static SourceLocation GetBeginningOfToken(SourceLocation Loc,
                                          const SourceManager &SM,
                                          const LangOptions &LangOpts);

/// Get the physical length (including trigraphs and escaped newlines) of the
/// first \p Characters characters of the token starting at TokStart.
static unsigned getTokenPrefixLength(SourceLocation TokStart,
                                     unsigned CharNo,
                                     const SourceManager &SM,
                                     const LangOptions &LangOpts);

/// AdvanceToTokenCharacter - If the current SourceLocation specifies a
/// location at the start of a token, return a new location that specifies a
/// character within the token.  This handles trigraphs and escaped newlines.
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
                                              unsigned Characters,
                                              const SourceManager &SM,
                                              const LangOptions &LangOpts) {
  return TokStart.getLocWithOffset(
      getTokenPrefixLength(TokStart, Characters, SM, LangOpts));
}

/// Computes the source location just past the end of the
/// token at this source location.
///
/// This routine can be used to produce a source location that
/// points just past the end of the token referenced by \p Loc, and
/// is generally used when a diagnostic needs to point just after a
/// token where it expected something different that it received. If
/// the returned source location would not be meaningful (e.g., if
/// it points into a macro), this routine returns an invalid
/// source location.
///
/// \param Offset an offset from the end of the token, where the source
/// location should refer to. The default offset (0) produces a source
/// location pointing just past the end of the token; an offset of 1 produces
/// a source location pointing to the last character in the token, etc.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
                                          const SourceManager &SM,
                                          const LangOptions &LangOpts);

/// Given a token range, produce a corresponding CharSourceRange that
/// is not a token range. This allows the source range to be used by
/// components that don't have access to the lexer and thus can't find the
/// end of the range for themselves.
static CharSourceRange getAsCharRange(SourceRange Range,
                                      const SourceManager &SM,
                                      const LangOptions &LangOpts) {
  SourceLocation End = getLocForEndOfToken(Range.getEnd(), 0, SM, LangOpts);
  return End.isInvalid() ? CharSourceRange()
                         : CharSourceRange::getCharRange(
                               Range.getBegin(), End);
}
static CharSourceRange getAsCharRange(CharSourceRange Range,
                                      const SourceManager &SM,
                                      const LangOptions &LangOpts) {
  return Range.isTokenRange()
             ? getAsCharRange(Range.getAsRange(), SM, LangOpts)
             : Range;
}

/// Returns true if the given MacroID location points at the first
/// token of the macro expansion.
///
/// \param MacroBegin If non-null and function returns true, it is set to
/// begin location of the macro.
static bool isAtStartOfMacroExpansion(SourceLocation loc,
                                      const SourceManager &SM,
                                      const LangOptions &LangOpts,
                                      SourceLocation *MacroBegin = nullptr);

/// Returns true if the given MacroID location points at the last
/// token of the macro expansion.
///
/// \param MacroEnd If non-null and function returns true, it is set to
/// end location of the macro.
static bool isAtEndOfMacroExpansion(SourceLocation loc,
                                    const SourceManager &SM,
                                    const LangOptions &LangOpts,
                                    SourceLocation *MacroEnd = nullptr);

/// Accepts a range and returns a character range with file locations.
///
/// Returns a null range if a part of the range resides inside a macro
/// expansion or the range does not reside on the same FileID.
///
/// This function is trying to deal with macros and return a range based on
/// file locations. The cases where it can successfully handle macros are:
///
/// -begin or end range lies at the start or end of a macro expansion, in
///  which case the location will be set to the expansion point, e.g:
///    \#define M 1 2
///    a M
/// If you have a range [a, 2] (where 2 came from the macro), the function
/// will return a range for "a M"
/// if you have range [a, 1], the function will fail because the range
/// overlaps with only a part of the macro
///
/// -The macro is a function macro and the range can be mapped to the macro
///  arguments, e.g:
///    \#define M 1 2
///    \#define FM(x) x
///    FM(a b M)
/// if you have range [b, 2], the function will return the file range "b M"
/// inside the macro arguments.
/// if you have range [a, 2], the function will return the file range
/// "FM(a b M)" since the range includes all of the macro expansion.
static CharSourceRange makeFileCharRange(CharSourceRange Range,
                                         const SourceManager &SM,
                                         const LangOptions &LangOpts);

/// Returns a string for the source that the range encompasses.
static StringRef getSourceText(CharSourceRange Range,
                               const SourceManager &SM,
                               const LangOptions &LangOpts,
                               bool *Invalid = nullptr);

/// Retrieve the name of the immediate macro expansion.
///
/// This routine starts from a source location, and finds the name of the macro
/// responsible for its immediate expansion. It looks through any intervening
/// macro argument expansions to compute this. It returns a StringRef which
/// refers to the SourceManager-owned buffer of the source where that macro
/// name is spelled. Thus, the result shouldn't out-live that SourceManager.
static StringRef getImmediateMacroName(SourceLocation Loc,
                                       const SourceManager &SM,
                                       const LangOptions &LangOpts);

/// Retrieve the name of the immediate macro expansion.
///
/// This routine starts from a source location, and finds the name of the
/// macro responsible for its immediate expansion. It looks through any
/// intervening macro argument expansions to compute this. It returns a
/// StringRef which refers to the SourceManager-owned buffer of the source
/// where that macro name is spelled. Thus, the result shouldn't out-live
/// that SourceManager.
///
/// This differs from Lexer::getImmediateMacroName in that any macro argument
/// location will result in the topmost function macro that accepted it.
/// e.g.
/// \code
///   MAC1( MAC2(foo) )
/// \endcode
/// for location of 'foo' token, this function will return "MAC1" while
/// Lexer::getImmediateMacroName will return "MAC2".
static StringRef getImmediateMacroNameForDiagnostics(
    SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts);

/// Compute the preamble of the given file.
///
/// The preamble of a file contains the initial comments, include directives,
/// and other preprocessor directives that occur before the code in this
/// particular file actually begins. The preamble of the main source file is
/// a potential prefix header.
///
/// \param Buffer The memory buffer containing the file's contents.
///
/// \param MaxLines If non-zero, restrict the length of the preamble
/// to fewer than this number of lines.
///
/// \returns The offset into the file where the preamble ends and the rest
/// of the file begins along with a boolean value indicating whether
/// the preamble ends at the beginning of a new line.
static PreambleBounds ComputePreamble(StringRef Buffer,
                                      const LangOptions &LangOpts,
                                      unsigned MaxLines = 0);

/// Finds the token that comes right after the given location.
///
/// Returns the next token, or none if the location is inside a macro.
static Optional<Token> findNextToken(SourceLocation Loc,
                                     const SourceManager &SM,
                                     const LangOptions &LangOpts);

/// Checks that the given token is the first token that occurs after
/// the given location (this excludes comments and whitespace). Returns the
/// location immediately after the specified token. If the token is not found
/// or the location is inside a macro, the returned source location will be
/// invalid.
static SourceLocation findLocationAfterToken(SourceLocation loc,
                                       tok::TokenKind TKind,
                                       const SourceManager &SM,
                                       const LangOptions &LangOpts,
                                       bool SkipTrailingWhitespaceAndNewLine);

/// Returns true if the given character could appear in an identifier.
static bool isIdentifierBodyChar(char c, const LangOptions &LangOpts);

/// Checks whether new line pointed by Str is preceded by escape
/// sequence.
static bool isNewLineEscaped(const char *BufferStart, const char *Str);

/// getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever
/// emit a warning.
static inline char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size,
                                        const LangOptions &LangOpts) {
  // If this is not a trigraph and not a UCN or escaped newline, return
  // quickly.
  if (isObviouslySimpleCharacter(Ptr[0])) {
    Size = 1;
    return *Ptr;
  }

  Size = 0;
  return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts);
}

/// Returns the leading whitespace for line that corresponds to the given
/// location \p Loc.
static StringRef getIndentationForLine(SourceLocation Loc,
                                       const SourceManager &SM);

565private:
//===--------------------------------------------------------------------===//
// Internal implementation interfaces.

/// LexTokenInternal - Internal interface to lex a preprocessing token. Called
/// by Lex.
///
bool LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine);

bool CheckUnicodeWhitespace(Token &Result, uint32_t C, const char *CurPtr);

/// Given that a token begins with the Unicode character \p C, figure out
/// what kind of token it is and dispatch to the appropriate lexing helper
/// function.
bool LexUnicode(Token &Result, uint32_t C, const char *CurPtr);

/// FormTokenWithChars - When we lex a token, we have identified a span
/// starting at BufferPtr, going to TokEnd that forms the token.  This method
/// takes that range and assigns it to the token as its location and size.  In
/// addition, since tokens cannot overlap, this also updates BufferPtr to be
/// TokEnd.
void FormTokenWithChars(Token &Result, const char *TokEnd,
                        tok::TokenKind Kind) {
  unsigned TokLen = TokEnd-BufferPtr;
  Result.setLength(TokLen);
  Result.setLocation(getSourceLocation(BufferPtr, TokLen));
  Result.setKind(Kind);
  BufferPtr = TokEnd;
}

/// isNextPPTokenLParen - Return 1 if the next unexpanded token will return a
/// tok::l_paren token, 0 if it is something else and 2 if there are no more
/// tokens in the buffer controlled by this lexer.
unsigned isNextPPTokenLParen();

//===--------------------------------------------------------------------===//
// Lexer character reading interfaces.

// This lexer is built on two interfaces for reading characters, both of which
// automatically provide phase 1/2 translation.  getAndAdvanceChar is used
// when we know that we will be reading a character from the input buffer and
// that this character will be part of the result token. This occurs in (f.e.)
// string processing, because we know we need to read until we find the
// closing '"' character.
//
// The second interface is the combination of getCharAndSize with
// ConsumeChar.  getCharAndSize reads a phase 1/2 translated character,
// returning it and its size.  If the lexer decides that this character is
// part of the current token, it calls ConsumeChar on it.  This two stage
// approach allows us to emit diagnostics for characters (e.g. warnings about
// trigraphs), knowing that they only are emitted if the character is
// consumed.

/// isObviouslySimpleCharacter - Return true if the specified character is
/// obviously the same in translation phase 1 and translation phase 3.  This
/// can return false for characters that end up being the same, but it will
/// never return true for something that needs to be mapped.
static bool isObviouslySimpleCharacter(char C) {
  return C != '?' && C != '\\';
}

/// getAndAdvanceChar - Read a single 'character' from the specified buffer,
/// advance over it, and return it.  This is tricky in several cases.  Here we
/// just handle the trivial case and fall-back to the non-inlined
/// getCharAndSizeSlow method to handle the hard case.
inline char getAndAdvanceChar(const char *&Ptr, Token &Tok) {
  // If this is not a trigraph and not a UCN or escaped newline, return
  // quickly.
  if (isObviouslySimpleCharacter(Ptr[0])) return *Ptr++;
18
←
Taking false branch→

  unsigned Size = 0;
  char C = getCharAndSizeSlow(Ptr, Size, &Tok);
19
←
Value assigned to field 'PP'→
  Ptr += Size;
  return C;
}

/// ConsumeChar - When a character (identified by getCharAndSize) is consumed
/// and added to a given token, check to see if there are diagnostics that
/// need to be emitted or flags that need to be set on the token.  If so, do
/// it.
const char *ConsumeChar(const char *Ptr, unsigned Size, Token &Tok) {
  // Normal case, we consumed exactly one token.  Just return it.
  if (Size == 1)
    return Ptr+Size;

  // Otherwise, re-lex the character with a current token, allowing
  // diagnostics to be emitted and flags to be set.
  Size = 0;
  getCharAndSizeSlow(Ptr, Size, &Tok);
  return Ptr+Size;
}

/// getCharAndSize - Peek a single 'character' from the specified buffer,
/// get its size, and return it.  This is tricky in several cases.  Here we
/// just handle the trivial case and fall-back to the non-inlined
/// getCharAndSizeSlow method to handle the hard case.
inline char getCharAndSize(const char *Ptr, unsigned &Size) {
  // If this is not a trigraph and not a UCN or escaped newline, return
  // quickly.
  if (isObviouslySimpleCharacter(Ptr[0])) {
    Size = 1;
    return *Ptr;
  }

  Size = 0;
  return getCharAndSizeSlow(Ptr, Size);
}

/// getCharAndSizeSlow - Handle the slow/uncommon case of the getCharAndSize
/// method.
char getCharAndSizeSlow(const char *Ptr, unsigned &Size,
                        Token *Tok = nullptr);

/// getEscapedNewLineSize - Return the size of the specified escaped newline,
/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" on entry
/// to this function.
static unsigned getEscapedNewLineSize(const char *P);

/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
/// them), skip over them and return the first non-escaped-newline found,
/// otherwise return P.
static const char *SkipEscapedNewLines(const char *P);

/// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a
/// diagnostic.
static char getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
                                     const LangOptions &LangOpts);

//===--------------------------------------------------------------------===//
// Other lexer functions.

void SetByteOffset(unsigned Offset, bool StartOfLine);

void PropagateLineStartLeadingSpaceInfo(Token &Result);

const char *LexUDSuffix(Token &Result, const char *CurPtr,
                        bool IsStringLiteral);

// Helper functions to lex the remainder of a token of the specific type.
bool LexIdentifier         (Token &Result, const char *CurPtr);
bool LexNumericConstant    (Token &Result, const char *CurPtr);
bool LexStringLiteral      (Token &Result, const char *CurPtr,
                            tok::TokenKind Kind);
bool LexRawStringLiteral   (Token &Result, const char *CurPtr,
                            tok::TokenKind Kind);
bool LexAngledStringLiteral(Token &Result, const char *CurPtr);
bool LexCharConstant       (Token &Result, const char *CurPtr,
                            tok::TokenKind Kind);
bool LexEndOfFile          (Token &Result, const char *CurPtr);
bool SkipWhitespace        (Token &Result, const char *CurPtr,
                            bool &TokAtPhysicalStartOfLine);
bool SkipLineComment       (Token &Result, const char *CurPtr,
                            bool &TokAtPhysicalStartOfLine);
bool SkipBlockComment      (Token &Result, const char *CurPtr,
                            bool &TokAtPhysicalStartOfLine);
bool SaveLineComment       (Token &Result, const char *CurPtr);

bool IsStartOfConflictMarker(const char *CurPtr);
bool HandleEndOfConflictMarker(const char *CurPtr);

bool lexEditorPlaceholder(Token &Result, const char *CurPtr);

bool isCodeCompletionPoint(const char *CurPtr) const;
void cutOffLexing() { BufferPtr = BufferEnd; }

bool isHexaLiteral(const char *Start, const LangOptions &LangOpts);

void codeCompleteIncludedFile(const char *PathStart,
                              const char *CompletionPoint, bool IsAngled);

/// Read a universal character name.
///
/// \param StartPtr The position in the source buffer after the initial '\'.
///                 If the UCN is syntactically well-formed (but not 
///                 necessarily valid), this parameter will be updated to
///                 point to the character after the UCN.
/// \param SlashLoc The position in the source buffer of the '\'.
/// \param Result   The token being formed. Pass \c nullptr to suppress
///                 diagnostics and handle token formation in the caller.
///
/// \return The Unicode codepoint specified by the UCN, or 0 if the UCN is
///         invalid.
uint32_t tryReadUCN(const char *&StartPtr, const char *SlashLoc, Token *Result);

/// Try to consume a UCN as part of an identifier at the current
/// location.
/// \param CurPtr Initially points to the range of characters in the source
///               buffer containing the '\'. Updated to point past the end of
///               the UCN on success.
/// \param Size The number of characters occupied by the '\' (including
///             trigraphs and escaped newlines).
/// \param Result The token being produced. Marked as containing a UCN on
///               success.
/// \return \c true if a UCN was lexed and it produced an acceptable
///         identifier character, \c false otherwise.
bool tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size,
                             Token &Result);

/// Try to consume an identifier character encoded in UTF-8.
/// \param CurPtr Points to the start of the (potential) UTF-8 code unit
///        sequence. On success, updated to point past the end of it.
/// \return \c true if a UTF-8 sequence mapping to an acceptable identifier
///         character was lexed, \c false otherwise.
bool tryConsumeIdentifierUTF8Char(const char *&CurPtr);
769};

771} // namespace clang

773#endif // LLVM_CLANG_LEX_LEXER_H