clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name Lexer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/gnu/usr.bin/clang/libclangLex/obj -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/include -I /usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/llvm/include -I /usr/src/gnu/usr.bin/clang/libclangLex/../include -I /usr/src/gnu/usr.bin/clang/libclangLex/obj -I /usr/src/gnu/usr.bin/clang/libclangLex/obj/../include -D NDEBUG -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D LLVM_PREFIX="/usr" -internal-isystem /usr/include/c++/v1 -internal-isystem /usr/local/lib/clang/13.0.0/include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/usr/src/gnu/usr.bin/clang/libclangLex/obj -ferror-limit 19 -fvisibility-inlines-hidden -fwrapv -stack-protector 2 -fno-rtti -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /home/ben/Projects/vmm/scan-build/2022-01-12-194120-40624-1 -x c++ /usr/src/gnu/usr.bin/clang/libclangLex/../../../llvm/clang/lib/Lex/Lexer.cpp
1 | |
2 | |
3 | |
4 | |
5 | |
6 | |
7 | |
8 | |
9 | |
10 | |
11 | |
12 | |
13 | #include "clang/Lex/Lexer.h" |
14 | #include "UnicodeCharSets.h" |
15 | #include "clang/Basic/CharInfo.h" |
16 | #include "clang/Basic/Diagnostic.h" |
17 | #include "clang/Basic/IdentifierTable.h" |
18 | #include "clang/Basic/LLVM.h" |
19 | #include "clang/Basic/LangOptions.h" |
20 | #include "clang/Basic/SourceLocation.h" |
21 | #include "clang/Basic/SourceManager.h" |
22 | #include "clang/Basic/TokenKinds.h" |
23 | #include "clang/Lex/LexDiagnostic.h" |
24 | #include "clang/Lex/LiteralSupport.h" |
25 | #include "clang/Lex/MultipleIncludeOpt.h" |
26 | #include "clang/Lex/Preprocessor.h" |
27 | #include "clang/Lex/PreprocessorOptions.h" |
28 | #include "clang/Lex/Token.h" |
29 | #include "llvm/ADT/None.h" |
30 | #include "llvm/ADT/Optional.h" |
31 | #include "llvm/ADT/STLExtras.h" |
32 | #include "llvm/ADT/StringExtras.h" |
33 | #include "llvm/ADT/StringRef.h" |
34 | #include "llvm/ADT/StringSwitch.h" |
35 | #include "llvm/Support/Compiler.h" |
36 | #include "llvm/Support/ConvertUTF.h" |
37 | #include "llvm/Support/MathExtras.h" |
38 | #include "llvm/Support/MemoryBufferRef.h" |
39 | #include "llvm/Support/NativeFormatting.h" |
40 | #include "llvm/Support/UnicodeCharRanges.h" |
41 | #include <algorithm> |
42 | #include <cassert> |
43 | #include <cstddef> |
44 | #include <cstdint> |
45 | #include <cstring> |
46 | #include <string> |
47 | #include <tuple> |
48 | #include <utility> |
49 | |
50 | using namespace clang; |
51 | |
52 | |
53 | |
54 | |
55 | |
56 | |
57 | bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { |
58 | if (isAnnotation()) |
59 | return false; |
60 | if (IdentifierInfo *II = getIdentifierInfo()) |
61 | return II->getObjCKeywordID() == objcKey; |
62 | return false; |
63 | } |
64 | |
65 | |
66 | tok::ObjCKeywordKind Token::getObjCKeywordID() const { |
67 | if (isAnnotation()) |
68 | return tok::objc_not_keyword; |
69 | IdentifierInfo *specId = getIdentifierInfo(); |
70 | return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; |
71 | } |
72 | |
73 | |
74 | |
75 | |
76 | |
77 | void Lexer::anchor() {} |
78 | |
79 | void Lexer::InitLexer(const char *BufStart, const char *BufPtr, |
80 | const char *BufEnd) { |
81 | BufferStart = BufStart; |
82 | BufferPtr = BufPtr; |
83 | BufferEnd = BufEnd; |
84 | |
85 | assert(BufEnd[0] == 0 && |
86 | "We assume that the input buffer has a null character at the end" |
87 | " to simplify lexing!"); |
88 | |
89 | |
90 | |
91 | |
92 | if (BufferStart == BufferPtr) { |
93 | |
94 | StringRef Buf(BufferStart, BufferEnd - BufferStart); |
95 | size_t BOMLength = llvm::StringSwitch<size_t>(Buf) |
96 | .StartsWith("\xEF\xBB\xBF", 3) |
97 | .Default(0); |
98 | |
99 | |
100 | BufferPtr += BOMLength; |
101 | } |
102 | |
103 | Is_PragmaLexer = false; |
104 | CurrentConflictMarkerState = CMK_None; |
105 | |
106 | |
107 | IsAtStartOfLine = true; |
108 | IsAtPhysicalStartOfLine = true; |
109 | |
110 | HasLeadingSpace = false; |
111 | HasLeadingEmptyMacro = false; |
112 | |
113 | |
114 | ParsingPreprocessorDirective = false; |
115 | |
116 | |
117 | ParsingFilename = false; |
118 | |
119 | |
120 | |
121 | |
122 | |
123 | LexingRawMode = false; |
124 | |
125 | |
126 | ExtendedTokenMode = 0; |
127 | |
128 | NewLinePtr = nullptr; |
129 | } |
130 | |
131 | |
132 | |
133 | |
134 | |
135 | Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile, |
136 | Preprocessor &PP) |
137 | : PreprocessorLexer(&PP, FID), |
138 | FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), |
139 | LangOpts(PP.getLangOpts()) { |
140 | InitLexer(InputFile.getBufferStart(), InputFile.getBufferStart(), |
141 | InputFile.getBufferEnd()); |
142 | |
143 | resetExtendedTokenMode(); |
144 | } |
145 | |
146 | |
147 | |
148 | |
149 | Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, |
150 | const char *BufStart, const char *BufPtr, const char *BufEnd) |
151 | : FileLoc(fileloc), LangOpts(langOpts) { |
152 | InitLexer(BufStart, BufPtr, BufEnd); |
153 | |
154 | |
155 | LexingRawMode = true; |
156 | } |
157 | |
158 | |
159 | |
160 | |
161 | Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile, |
162 | const SourceManager &SM, const LangOptions &langOpts) |
163 | : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(), |
164 | FromFile.getBufferStart(), FromFile.getBufferEnd()) {} |
165 | |
166 | void Lexer::resetExtendedTokenMode() { |
167 | assert(PP && "Cannot reset token mode without a preprocessor"); |
168 | if (LangOpts.TraditionalCPP) |
169 | SetKeepWhitespaceMode(true); |
170 | else |
171 | SetCommentRetentionState(PP->getCommentRetentionState()); |
172 | } |
173 | |
174 | |
175 | |
176 | |
177 | |
178 | |
179 | |
180 | |
181 | |
182 | |
183 | |
184 | |
185 | |
186 | |
187 | |
188 | |
189 | Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, |
190 | SourceLocation ExpansionLocStart, |
191 | SourceLocation ExpansionLocEnd, |
192 | unsigned TokLen, Preprocessor &PP) { |
193 | SourceManager &SM = PP.getSourceManager(); |
194 | |
195 | |
196 | FileID SpellingFID = SM.getFileID(SpellingLoc); |
197 | llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(SpellingFID); |
198 | Lexer *L = new Lexer(SpellingFID, InputFile, PP); |
199 | |
200 | |
201 | |
202 | |
203 | const char *StrData = SM.getCharacterData(SpellingLoc); |
204 | |
205 | L->BufferPtr = StrData; |
206 | L->BufferEnd = StrData+TokLen; |
207 | assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); |
208 | |
209 | |
210 | |
211 | L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), |
212 | ExpansionLocStart, |
213 | ExpansionLocEnd, TokLen); |
214 | |
215 | |
216 | |
217 | L->ParsingPreprocessorDirective = true; |
218 | |
219 | |
220 | L->Is_PragmaLexer = true; |
221 | return L; |
222 | } |
223 | |
224 | bool Lexer::skipOver(unsigned NumBytes) { |
225 | IsAtPhysicalStartOfLine = true; |
226 | IsAtStartOfLine = true; |
227 | if ((BufferPtr + NumBytes) > BufferEnd) |
228 | return true; |
229 | BufferPtr += NumBytes; |
230 | return false; |
231 | } |
232 | |
233 | template <typename T> static void StringifyImpl(T &Str, char Quote) { |
234 | typename T::size_type i = 0, e = Str.size(); |
235 | while (i < e) { |
236 | if (Str[i] == '\\' || Str[i] == Quote) { |
237 | Str.insert(Str.begin() + i, '\\'); |
238 | i += 2; |
239 | ++e; |
240 | } else if (Str[i] == '\n' || Str[i] == '\r') { |
241 | |
242 | if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') && |
243 | Str[i] != Str[i + 1]) { |
244 | Str[i] = '\\'; |
245 | Str[i + 1] = 'n'; |
246 | } else { |
247 | |
248 | Str[i] = '\\'; |
249 | Str.insert(Str.begin() + i + 1, 'n'); |
250 | ++e; |
251 | } |
252 | i += 2; |
253 | } else |
254 | ++i; |
255 | } |
256 | } |
257 | |
258 | std::string Lexer::Stringify(StringRef Str, bool Charify) { |
259 | std::string Result = std::string(Str); |
260 | char Quote = Charify ? '\'' : '"'; |
261 | StringifyImpl(Result, Quote); |
262 | return Result; |
263 | } |
264 | |
265 | void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, '"'); } |
266 | |
267 | |
268 | |
269 | |
270 | |
271 | |
272 | |
273 | static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, |
274 | const LangOptions &LangOpts, char *Spelling) { |
275 | assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); |
276 | |
277 | size_t Length = 0; |
278 | const char *BufEnd = BufPtr + Tok.getLength(); |
279 | |
280 | if (tok::isStringLiteral(Tok.getKind())) { |
281 | |
282 | while (BufPtr < BufEnd) { |
283 | unsigned Size; |
284 | Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); |
285 | BufPtr += Size; |
286 | |
287 | if (Spelling[Length - 1] == '"') |
288 | break; |
289 | } |
290 | |
291 | |
292 | |
293 | |
294 | if (Length >= 2 && |
295 | Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { |
296 | |
297 | |
298 | const char *RawEnd = BufEnd; |
299 | do --RawEnd; while (*RawEnd != '"'); |
300 | size_t RawLength = RawEnd - BufPtr + 1; |
301 | |
302 | |
303 | memcpy(Spelling + Length, BufPtr, RawLength); |
304 | Length += RawLength; |
305 | BufPtr += RawLength; |
306 | |
307 | |
308 | } |
309 | } |
310 | |
311 | while (BufPtr < BufEnd) { |
312 | unsigned Size; |
313 | Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); |
314 | BufPtr += Size; |
315 | } |
316 | |
317 | assert(Length < Tok.getLength() && |
318 | "NeedsCleaning flag set on token that didn't need cleaning!"); |
319 | return Length; |
320 | } |
321 | |
322 | |
323 | |
324 | |
325 | |
326 | |
327 | StringRef Lexer::getSpelling(SourceLocation loc, |
328 | SmallVectorImpl<char> &buffer, |
329 | const SourceManager &SM, |
330 | const LangOptions &options, |
331 | bool *invalid) { |
332 | |
333 | std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); |
334 | |
335 | |
336 | bool invalidTemp = false; |
337 | StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); |
338 | if (invalidTemp) { |
339 | if (invalid) *invalid = true; |
340 | return {}; |
341 | } |
342 | |
343 | const char *tokenBegin = file.data() + locInfo.second; |
344 | |
345 | |
346 | Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, |
347 | file.begin(), tokenBegin, file.end()); |
348 | Token token; |
349 | lexer.LexFromRawLexer(token); |
350 | |
351 | unsigned length = token.getLength(); |
352 | |
353 | |
354 | if (!token.needsCleaning()) |
355 | return StringRef(tokenBegin, length); |
356 | |
357 | |
358 | buffer.resize(length); |
359 | buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); |
360 | return StringRef(buffer.data(), buffer.size()); |
361 | } |
362 | |
363 | |
364 | |
365 | |
366 | |
367 | |
368 | std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, |
369 | const LangOptions &LangOpts, bool *Invalid) { |
370 | assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); |
371 | |
372 | bool CharDataInvalid = false; |
373 | const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), |
374 | &CharDataInvalid); |
375 | if (Invalid) |
376 | *Invalid = CharDataInvalid; |
377 | if (CharDataInvalid) |
378 | return {}; |
379 | |
380 | |
381 | if (!Tok.needsCleaning()) |
382 | return std::string(TokStart, TokStart + Tok.getLength()); |
383 | |
384 | std::string Result; |
385 | Result.resize(Tok.getLength()); |
386 | Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); |
387 | return Result; |
388 | } |
389 | |
390 | |
391 | |
392 | |
393 | |
394 | |
395 | |
396 | |
397 | |
398 | |
399 | |
400 | unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, |
401 | const SourceManager &SourceMgr, |
402 | const LangOptions &LangOpts, bool *Invalid) { |
403 | assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); |
404 | |
405 | const char *TokStart = nullptr; |
406 | |
407 | if (Tok.is(tok::raw_identifier)) |
408 | TokStart = Tok.getRawIdentifier().data(); |
409 | else if (!Tok.hasUCN()) { |
410 | if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { |
411 | |
412 | Buffer = II->getNameStart(); |
413 | return II->getLength(); |
414 | } |
415 | } |
416 | |
417 | |
418 | if (Tok.isLiteral()) |
419 | TokStart = Tok.getLiteralData(); |
420 | |
421 | if (!TokStart) { |
422 | |
423 | bool CharDataInvalid = false; |
424 | TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); |
425 | if (Invalid) |
426 | *Invalid = CharDataInvalid; |
427 | if (CharDataInvalid) { |
428 | Buffer = ""; |
429 | return 0; |
430 | } |
431 | } |
432 | |
433 | |
434 | if (!Tok.needsCleaning()) { |
435 | Buffer = TokStart; |
436 | return Tok.getLength(); |
437 | } |
438 | |
439 | |
440 | return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); |
441 | } |
442 | |
443 | |
444 | |
445 | |
446 | |
447 | unsigned Lexer::MeasureTokenLength(SourceLocation Loc, |
448 | const SourceManager &SM, |
449 | const LangOptions &LangOpts) { |
450 | Token TheTok; |
451 | if (getRawToken(Loc, TheTok, SM, LangOpts)) |
452 | return 0; |
453 | return TheTok.getLength(); |
454 | } |
455 | |
456 | |
457 | |
458 | bool Lexer::getRawToken(SourceLocation Loc, Token &Result, |
459 | const SourceManager &SM, |
460 | const LangOptions &LangOpts, |
461 | bool IgnoreWhiteSpace) { |
462 | |
463 | |
464 | |
465 | |
466 | |
467 | |
468 | |
469 | |
470 | Loc = SM.getExpansionLoc(Loc); |
471 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
472 | bool Invalid = false; |
473 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
474 | if (Invalid) |
475 | return true; |
476 | |
477 | const char *StrData = Buffer.data()+LocInfo.second; |
478 | |
479 | if (!IgnoreWhiteSpace && isWhitespace(StrData[0])) |
480 | return true; |
481 | |
482 | |
483 | Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, |
484 | Buffer.begin(), StrData, Buffer.end()); |
485 | TheLexer.SetCommentRetentionState(true); |
486 | TheLexer.LexFromRawLexer(Result); |
487 | return false; |
488 | } |
489 | |
490 | |
491 | |
492 | static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) { |
493 | const char *BufStart = Buffer.data(); |
494 | if (Offset >= Buffer.size()) |
495 | return nullptr; |
496 | |
497 | const char *LexStart = BufStart + Offset; |
498 | for (; LexStart != BufStart; --LexStart) { |
499 | if (isVerticalWhitespace(LexStart[0]) && |
500 | !Lexer::isNewLineEscaped(BufStart, LexStart)) { |
501 | |
502 | ++LexStart; |
503 | break; |
504 | } |
505 | } |
506 | return LexStart; |
507 | } |
508 | |
509 | static SourceLocation getBeginningOfFileToken(SourceLocation Loc, |
510 | const SourceManager &SM, |
511 | const LangOptions &LangOpts) { |
512 | assert(Loc.isFileID()); |
513 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
514 | if (LocInfo.first.isInvalid()) |
515 | return Loc; |
516 | |
517 | bool Invalid = false; |
518 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
519 | if (Invalid) |
520 | return Loc; |
521 | |
522 | |
523 | |
524 | const char *StrData = Buffer.data() + LocInfo.second; |
525 | const char *LexStart = findBeginningOfLine(Buffer, LocInfo.second); |
526 | if (!LexStart || LexStart == StrData) |
527 | return Loc; |
528 | |
529 | |
530 | SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); |
531 | Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart, |
532 | Buffer.end()); |
533 | TheLexer.SetCommentRetentionState(true); |
534 | |
535 | |
536 | Token TheTok; |
537 | do { |
538 | TheLexer.LexFromRawLexer(TheTok); |
539 | |
540 | if (TheLexer.getBufferLocation() > StrData) { |
541 | |
542 | |
543 | |
544 | if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) |
545 | return TheTok.getLocation(); |
546 | |
547 | |
548 | |
549 | break; |
550 | } |
551 | } while (TheTok.getKind() != tok::eof); |
552 | |
553 | |
554 | return Loc; |
555 | } |
556 | |
557 | SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, |
558 | const SourceManager &SM, |
559 | const LangOptions &LangOpts) { |
560 | if (Loc.isFileID()) |
561 | return getBeginningOfFileToken(Loc, SM, LangOpts); |
562 | |
563 | if (!SM.isMacroArgExpansion(Loc)) |
564 | return Loc; |
565 | |
566 | SourceLocation FileLoc = SM.getSpellingLoc(Loc); |
567 | SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); |
568 | std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); |
569 | std::pair<FileID, unsigned> BeginFileLocInfo = |
570 | SM.getDecomposedLoc(BeginFileLoc); |
571 | assert(FileLocInfo.first == BeginFileLocInfo.first && |
572 | FileLocInfo.second >= BeginFileLocInfo.second); |
573 | return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); |
574 | } |
575 | |
576 | namespace { |
577 | |
578 | enum PreambleDirectiveKind { |
579 | PDK_Skipped, |
580 | PDK_Unknown |
581 | }; |
582 | |
583 | } |
584 | |
585 | PreambleBounds Lexer::ComputePreamble(StringRef Buffer, |
586 | const LangOptions &LangOpts, |
587 | unsigned MaxLines) { |
588 | |
589 | |
590 | |
591 | const SourceLocation::UIntTy StartOffset = 1; |
592 | SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); |
593 | Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(), |
594 | Buffer.end()); |
595 | TheLexer.SetCommentRetentionState(true); |
596 | |
597 | bool InPreprocessorDirective = false; |
598 | Token TheTok; |
599 | SourceLocation ActiveCommentLoc; |
600 | |
601 | unsigned MaxLineOffset = 0; |
602 | if (MaxLines) { |
603 | const char *CurPtr = Buffer.begin(); |
604 | unsigned CurLine = 0; |
605 | while (CurPtr != Buffer.end()) { |
606 | char ch = *CurPtr++; |
607 | if (ch == '\n') { |
608 | ++CurLine; |
609 | if (CurLine == MaxLines) |
610 | break; |
611 | } |
612 | } |
613 | if (CurPtr != Buffer.end()) |
614 | MaxLineOffset = CurPtr - Buffer.begin(); |
615 | } |
616 | |
617 | do { |
618 | TheLexer.LexFromRawLexer(TheTok); |
619 | |
620 | if (InPreprocessorDirective) { |
621 | |
622 | if (TheTok.getKind() == tok::eof) { |
623 | break; |
624 | } |
625 | |
626 | |
627 | |
628 | if (!TheTok.isAtStartOfLine()) |
629 | continue; |
630 | |
631 | |
632 | |
633 | InPreprocessorDirective = false; |
634 | } |
635 | |
636 | |
637 | if (TheTok.isAtStartOfLine()) { |
638 | unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; |
639 | |
640 | |
641 | |
642 | if (MaxLineOffset && TokOffset >= MaxLineOffset) |
643 | break; |
644 | } |
645 | |
646 | |
647 | if (TheTok.getKind() == tok::comment) { |
648 | if (ActiveCommentLoc.isInvalid()) |
649 | ActiveCommentLoc = TheTok.getLocation(); |
650 | continue; |
651 | } |
652 | |
653 | if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { |
654 | |
655 | Token HashTok = TheTok; |
656 | InPreprocessorDirective = true; |
657 | ActiveCommentLoc = SourceLocation(); |
658 | |
659 | |
660 | |
661 | |
662 | TheLexer.LexFromRawLexer(TheTok); |
663 | if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { |
664 | StringRef Keyword = TheTok.getRawIdentifier(); |
665 | PreambleDirectiveKind PDK |
666 | = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) |
667 | .Case("include", PDK_Skipped) |
668 | .Case("__include_macros", PDK_Skipped) |
669 | .Case("define", PDK_Skipped) |
670 | .Case("undef", PDK_Skipped) |
671 | .Case("line", PDK_Skipped) |
672 | .Case("error", PDK_Skipped) |
673 | .Case("pragma", PDK_Skipped) |
674 | .Case("import", PDK_Skipped) |
675 | .Case("include_next", PDK_Skipped) |
676 | .Case("warning", PDK_Skipped) |
677 | .Case("ident", PDK_Skipped) |
678 | .Case("sccs", PDK_Skipped) |
679 | .Case("assert", PDK_Skipped) |
680 | .Case("unassert", PDK_Skipped) |
681 | .Case("if", PDK_Skipped) |
682 | .Case("ifdef", PDK_Skipped) |
683 | .Case("ifndef", PDK_Skipped) |
684 | .Case("elif", PDK_Skipped) |
685 | .Case("elifdef", PDK_Skipped) |
686 | .Case("elifndef", PDK_Skipped) |
687 | .Case("else", PDK_Skipped) |
688 | .Case("endif", PDK_Skipped) |
689 | .Default(PDK_Unknown); |
690 | |
691 | switch (PDK) { |
692 | case PDK_Skipped: |
693 | continue; |
694 | |
695 | case PDK_Unknown: |
696 | |
697 | break; |
698 | } |
699 | } |
700 | |
701 | |
702 | |
703 | |
704 | TheTok = HashTok; |
705 | } |
706 | |
707 | |
708 | |
709 | |
710 | break; |
711 | } while (true); |
712 | |
713 | SourceLocation End; |
714 | if (ActiveCommentLoc.isValid()) |
715 | End = ActiveCommentLoc; |
716 | else |
717 | End = TheTok.getLocation(); |
718 | |
719 | return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(), |
720 | TheTok.isAtStartOfLine()); |
721 | } |
722 | |
723 | unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo, |
724 | const SourceManager &SM, |
725 | const LangOptions &LangOpts) { |
726 | |
727 | |
728 | |
729 | bool Invalid = false; |
730 | const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); |
731 | |
732 | |
733 | if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) |
734 | return 0; |
735 | |
736 | unsigned PhysOffset = 0; |
737 | |
738 | |
739 | |
740 | |
741 | while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { |
742 | if (CharNo == 0) |
743 | return PhysOffset; |
744 | ++TokPtr; |
745 | --CharNo; |
746 | ++PhysOffset; |
747 | } |
748 | |
749 | |
750 | |
751 | for (; CharNo; --CharNo) { |
752 | unsigned Size; |
753 | Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); |
754 | TokPtr += Size; |
755 | PhysOffset += Size; |
756 | } |
757 | |
758 | |
759 | |
760 | |
761 | |
762 | if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) |
763 | PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; |
764 | |
765 | return PhysOffset; |
766 | } |
767 | |
768 | |
769 | |
770 | |
771 | |
772 | |
773 | |
774 | |
775 | |
776 | |
777 | |
778 | |
779 | |
780 | |
781 | |
782 | |
783 | SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, |
784 | const SourceManager &SM, |
785 | const LangOptions &LangOpts) { |
786 | if (Loc.isInvalid()) |
787 | return {}; |
788 | |
789 | if (Loc.isMacroID()) { |
790 | if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) |
791 | return {}; |
792 | } |
793 | |
794 | unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
795 | if (Len > Offset) |
796 | Len = Len - Offset; |
797 | else |
798 | return Loc; |
799 | |
800 | return Loc.getLocWithOffset(Len); |
801 | } |
802 | |
803 | |
804 | |
805 | bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, |
806 | const SourceManager &SM, |
807 | const LangOptions &LangOpts, |
808 | SourceLocation *MacroBegin) { |
809 | assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); |
810 | |
811 | SourceLocation expansionLoc; |
812 | if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) |
813 | return false; |
814 | |
815 | if (expansionLoc.isFileID()) { |
816 | |
817 | if (MacroBegin) |
818 | *MacroBegin = expansionLoc; |
819 | return true; |
820 | } |
821 | |
822 | return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); |
823 | } |
824 | |
825 | |
826 | |
827 | bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, |
828 | const SourceManager &SM, |
829 | const LangOptions &LangOpts, |
830 | SourceLocation *MacroEnd) { |
831 | assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); |
832 | |
833 | SourceLocation spellLoc = SM.getSpellingLoc(loc); |
834 | unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); |
835 | if (tokLen == 0) |
836 | return false; |
837 | |
838 | SourceLocation afterLoc = loc.getLocWithOffset(tokLen); |
839 | SourceLocation expansionLoc; |
840 | if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) |
841 | return false; |
842 | |
843 | if (expansionLoc.isFileID()) { |
844 | |
845 | if (MacroEnd) |
846 | *MacroEnd = expansionLoc; |
847 | return true; |
848 | } |
849 | |
850 | return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); |
851 | } |
852 | |
853 | static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, |
854 | const SourceManager &SM, |
855 | const LangOptions &LangOpts) { |
856 | SourceLocation Begin = Range.getBegin(); |
857 | SourceLocation End = Range.getEnd(); |
858 | assert(Begin.isFileID() && End.isFileID()); |
859 | if (Range.isTokenRange()) { |
860 | End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); |
861 | if (End.isInvalid()) |
862 | return {}; |
863 | } |
864 | |
865 | |
866 | FileID FID; |
867 | unsigned BeginOffs; |
868 | std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); |
869 | if (FID.isInvalid()) |
870 | return {}; |
871 | |
872 | unsigned EndOffs; |
873 | if (!SM.isInFileID(End, FID, &EndOffs) || |
874 | BeginOffs > EndOffs) |
875 | return {}; |
876 | |
877 | return CharSourceRange::getCharRange(Begin, End); |
878 | } |
879 | |
880 | |
881 | static bool isInExpansionTokenRange(const SourceLocation Loc, |
882 | const SourceManager &SM) { |
883 | return SM.getSLocEntry(SM.getFileID(Loc)) |
884 | .getExpansion() |
885 | .isExpansionTokenRange(); |
886 | } |
887 | |
888 | CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, |
889 | const SourceManager &SM, |
890 | const LangOptions &LangOpts) { |
891 | SourceLocation Begin = Range.getBegin(); |
892 | SourceLocation End = Range.getEnd(); |
893 | if (Begin.isInvalid() || End.isInvalid()) |
894 | return {}; |
895 | |
896 | if (Begin.isFileID() && End.isFileID()) |
897 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
898 | |
899 | if (Begin.isMacroID() && End.isFileID()) { |
900 | if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) |
901 | return {}; |
902 | Range.setBegin(Begin); |
903 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
904 | } |
905 | |
906 | if (Begin.isFileID() && End.isMacroID()) { |
907 | if (Range.isTokenRange()) { |
908 | if (!isAtEndOfMacroExpansion(End, SM, LangOpts, &End)) |
909 | return {}; |
910 | |
911 | Range.setTokenRange(isInExpansionTokenRange(Range.getEnd(), SM)); |
912 | } else if (!isAtStartOfMacroExpansion(End, SM, LangOpts, &End)) |
913 | return {}; |
914 | Range.setEnd(End); |
915 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
916 | } |
917 | |
918 | assert(Begin.isMacroID() && End.isMacroID()); |
919 | SourceLocation MacroBegin, MacroEnd; |
920 | if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && |
921 | ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, |
922 | &MacroEnd)) || |
923 | (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, |
924 | &MacroEnd)))) { |
925 | Range.setBegin(MacroBegin); |
926 | Range.setEnd(MacroEnd); |
927 | |
928 | if (Range.isTokenRange()) |
929 | Range.setTokenRange(isInExpansionTokenRange(End, SM)); |
930 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
931 | } |
932 | |
933 | bool Invalid = false; |
934 | const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), |
935 | &Invalid); |
936 | if (Invalid) |
937 | return {}; |
938 | |
939 | if (BeginEntry.getExpansion().isMacroArgExpansion()) { |
940 | const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), |
941 | &Invalid); |
942 | if (Invalid) |
943 | return {}; |
944 | |
945 | if (EndEntry.getExpansion().isMacroArgExpansion() && |
946 | BeginEntry.getExpansion().getExpansionLocStart() == |
947 | EndEntry.getExpansion().getExpansionLocStart()) { |
948 | Range.setBegin(SM.getImmediateSpellingLoc(Begin)); |
949 | Range.setEnd(SM.getImmediateSpellingLoc(End)); |
950 | return makeFileCharRange(Range, SM, LangOpts); |
951 | } |
952 | } |
953 | |
954 | return {}; |
955 | } |
956 | |
957 | StringRef Lexer::getSourceText(CharSourceRange Range, |
958 | const SourceManager &SM, |
959 | const LangOptions &LangOpts, |
960 | bool *Invalid) { |
961 | Range = makeFileCharRange(Range, SM, LangOpts); |
962 | if (Range.isInvalid()) { |
963 | if (Invalid) *Invalid = true; |
964 | return {}; |
965 | } |
966 | |
967 | |
968 | std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); |
969 | if (beginInfo.first.isInvalid()) { |
970 | if (Invalid) *Invalid = true; |
971 | return {}; |
972 | } |
973 | |
974 | unsigned EndOffs; |
975 | if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || |
976 | beginInfo.second > EndOffs) { |
977 | if (Invalid) *Invalid = true; |
978 | return {}; |
979 | } |
980 | |
981 | |
982 | bool invalidTemp = false; |
983 | StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); |
984 | if (invalidTemp) { |
985 | if (Invalid) *Invalid = true; |
986 | return {}; |
987 | } |
988 | |
989 | if (Invalid) *Invalid = false; |
990 | return file.substr(beginInfo.second, EndOffs - beginInfo.second); |
991 | } |
992 | |
993 | StringRef Lexer::getImmediateMacroName(SourceLocation Loc, |
994 | const SourceManager &SM, |
995 | const LangOptions &LangOpts) { |
996 | assert(Loc.isMacroID() && "Only reasonable to call this on macros"); |
997 | |
998 | |
999 | while (true) { |
1000 | FileID FID = SM.getFileID(Loc); |
1001 | const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); |
1002 | const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); |
1003 | Loc = Expansion.getExpansionLocStart(); |
1004 | if (!Expansion.isMacroArgExpansion()) |
1005 | break; |
1006 | |
1007 | |
1008 | |
1009 | |
1010 | |
1011 | |
1012 | Loc = SM.getImmediateExpansionRange(Loc).getBegin(); |
1013 | SourceLocation SpellLoc = Expansion.getSpellingLoc(); |
1014 | if (SpellLoc.isFileID()) |
1015 | break; |
1016 | |
1017 | |
1018 | |
1019 | FileID MacroFID = SM.getFileID(Loc); |
1020 | if (SM.isInFileID(SpellLoc, MacroFID)) |
1021 | break; |
1022 | |
1023 | |
1024 | Loc = SpellLoc; |
1025 | } |
1026 | |
1027 | |
1028 | |
1029 | |
1030 | Loc = SM.getSpellingLoc(Loc); |
1031 | |
1032 | |
1033 | |
1034 | std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); |
1035 | unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
1036 | StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); |
1037 | return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); |
1038 | } |
1039 | |
1040 | StringRef Lexer::getImmediateMacroNameForDiagnostics( |
1041 | SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) { |
1042 | assert(Loc.isMacroID() && "Only reasonable to call this on macros"); |
1043 | |
1044 | while (SM.isMacroArgExpansion(Loc)) |
1045 | Loc = SM.getImmediateExpansionRange(Loc).getBegin(); |
1046 | |
1047 | |
1048 | |
1049 | if (!SM.getFileEntryForID(SM.getFileID(SM.getSpellingLoc(Loc)))) |
1050 | return {}; |
1051 | |
1052 | |
1053 | |
1054 | |
1055 | Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).getBegin()); |
1056 | |
1057 | |
1058 | |
1059 | std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); |
1060 | unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
1061 | StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); |
1062 | return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); |
1063 | } |
1064 | |
1065 | bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { |
1066 | return isIdentifierBody(c, LangOpts.DollarIdents); |
1067 | } |
1068 | |
1069 | bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) { |
1070 | assert(isVerticalWhitespace(Str[0])); |
1071 | if (Str - 1 < BufferStart) |
1072 | return false; |
1073 | |
1074 | if ((Str[0] == '\n' && Str[-1] == '\r') || |
1075 | (Str[0] == '\r' && Str[-1] == '\n')) { |
1076 | if (Str - 2 < BufferStart) |
1077 | return false; |
1078 | --Str; |
1079 | } |
1080 | --Str; |
1081 | |
1082 | |
1083 | while (Str > BufferStart && isHorizontalWhitespace(*Str)) |
1084 | --Str; |
1085 | |
1086 | return *Str == '\\'; |
1087 | } |
1088 | |
1089 | StringRef Lexer::getIndentationForLine(SourceLocation Loc, |
1090 | const SourceManager &SM) { |
1091 | if (Loc.isInvalid() || Loc.isMacroID()) |
1092 | return {}; |
1093 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
1094 | if (LocInfo.first.isInvalid()) |
1095 | return {}; |
1096 | bool Invalid = false; |
1097 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
1098 | if (Invalid) |
1099 | return {}; |
1100 | const char *Line = findBeginningOfLine(Buffer, LocInfo.second); |
1101 | if (!Line) |
1102 | return {}; |
1103 | StringRef Rest = Buffer.substr(Line - Buffer.data()); |
1104 | size_t NumWhitespaceChars = Rest.find_first_not_of(" \t"); |
1105 | return NumWhitespaceChars == StringRef::npos |
1106 | ? "" |
1107 | : Rest.take_front(NumWhitespaceChars); |
1108 | } |
1109 | |
1110 | |
1111 | |
1112 | |
1113 | |
1114 | |
1115 | |
1116 | |
1117 | |
1118 | static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( |
1119 | Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); |
1120 | static SourceLocation GetMappedTokenLoc(Preprocessor &PP, |
1121 | SourceLocation FileLoc, |
1122 | unsigned CharNo, unsigned TokLen) { |
1123 | assert(FileLoc.isMacroID() && "Must be a macro expansion"); |
1124 | |
1125 | |
1126 | |
1127 | |
1128 | SourceManager &SM = PP.getSourceManager(); |
1129 | |
1130 | |
1131 | |
1132 | SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); |
1133 | SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); |
1134 | |
1135 | |
1136 | |
1137 | CharSourceRange II = SM.getImmediateExpansionRange(FileLoc); |
1138 | |
1139 | return SM.createExpansionLoc(SpellingLoc, II.getBegin(), II.getEnd(), TokLen); |
1140 | } |
1141 | |
1142 | |
1143 | |
1144 | SourceLocation Lexer::getSourceLocation(const char *Loc, |
1145 | unsigned TokLen) const { |
1146 | assert(Loc >= BufferStart && Loc <= BufferEnd && |
1147 | "Location out of range for this buffer!"); |
1148 | |
1149 | |
1150 | |
1151 | unsigned CharNo = Loc-BufferStart; |
1152 | if (FileLoc.isFileID()) |
1153 | return FileLoc.getLocWithOffset(CharNo); |
1154 | |
1155 | |
1156 | |
1157 | assert(PP && "This doesn't work on raw lexers"); |
1158 | return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); |
1159 | } |
1160 | |
1161 | |
1162 | |
1163 | DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { |
1164 | return PP->Diag(getSourceLocation(Loc), DiagID); |
| 40 | | Called C++ object pointer is null |
|
1165 | } |
1166 | |
1167 | |
1168 | |
1169 | |
1170 | |
1171 | |
1172 | |
1173 | static char GetTrigraphCharForLetter(char Letter) { |
1174 | switch (Letter) { |
1175 | default: return 0; |
1176 | case '=': return '#'; |
1177 | case ')': return ']'; |
1178 | case '(': return '['; |
1179 | case '!': return '|'; |
1180 | case '\'': return '^'; |
1181 | case '>': return '}'; |
1182 | case '/': return '\\'; |
1183 | case '<': return '{'; |
1184 | case '-': return '~'; |
1185 | } |
1186 | } |
1187 | |
1188 | |
1189 | |
1190 | |
1191 | |
1192 | static char DecodeTrigraphChar(const char *CP, Lexer *L) { |
1193 | char Res = GetTrigraphCharForLetter(*CP); |
1194 | if (!Res || !L) return Res; |
1195 | |
1196 | if (!L->getLangOpts().Trigraphs) { |
1197 | if (!L->isLexingRawMode()) |
1198 | L->Diag(CP-2, diag::trigraph_ignored); |
1199 | return 0; |
1200 | } |
1201 | |
1202 | if (!L->isLexingRawMode()) |
1203 | L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); |
1204 | return Res; |
1205 | } |
1206 | |
1207 | |
1208 | |
1209 | |
1210 | unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { |
1211 | unsigned Size = 0; |
1212 | while (isWhitespace(Ptr[Size])) { |
1213 | ++Size; |
1214 | |
1215 | if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') |
1216 | continue; |
1217 | |
1218 | |
1219 | if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && |
1220 | Ptr[Size-1] != Ptr[Size]) |
1221 | ++Size; |
1222 | |
1223 | return Size; |
1224 | } |
1225 | |
1226 | |
1227 | return 0; |
1228 | } |
1229 | |
1230 | |
1231 | |
1232 | |
1233 | const char *Lexer::SkipEscapedNewLines(const char *P) { |
1234 | while (true) { |
1235 | const char *AfterEscape; |
1236 | if (*P == '\\') { |
1237 | AfterEscape = P+1; |
1238 | } else if (*P == '?') { |
1239 | |
1240 | if (P[1] != '?' || P[2] != '/') |
1241 | return P; |
1242 | |
1243 | |
1244 | AfterEscape = P+3; |
1245 | } else { |
1246 | return P; |
1247 | } |
1248 | |
1249 | unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); |
1250 | if (NewLineSize == 0) return P; |
1251 | P = AfterEscape+NewLineSize; |
1252 | } |
1253 | } |
1254 | |
1255 | Optional<Token> Lexer::findNextToken(SourceLocation Loc, |
1256 | const SourceManager &SM, |
1257 | const LangOptions &LangOpts) { |
1258 | if (Loc.isMacroID()) { |
1259 | if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) |
1260 | return None; |
1261 | } |
1262 | Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); |
1263 | |
1264 | |
1265 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
1266 | |
1267 | |
1268 | bool InvalidTemp = false; |
1269 | StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); |
1270 | if (InvalidTemp) |
1271 | return None; |
1272 | |
1273 | const char *TokenBegin = File.data() + LocInfo.second; |
1274 | |
1275 | |
1276 | Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), |
1277 | TokenBegin, File.end()); |
1278 | |
1279 | Token Tok; |
1280 | lexer.LexFromRawLexer(Tok); |
1281 | return Tok; |
1282 | } |
1283 | |
1284 | |
1285 | |
1286 | |
1287 | |
1288 | SourceLocation Lexer::findLocationAfterToken( |
1289 | SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM, |
1290 | const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) { |
1291 | Optional<Token> Tok = findNextToken(Loc, SM, LangOpts); |
1292 | if (!Tok || Tok->isNot(TKind)) |
1293 | return {}; |
1294 | SourceLocation TokenLoc = Tok->getLocation(); |
1295 | |
1296 | |
1297 | unsigned NumWhitespaceChars = 0; |
1298 | if (SkipTrailingWhitespaceAndNewLine) { |
1299 | const char *TokenEnd = SM.getCharacterData(TokenLoc) + Tok->getLength(); |
1300 | unsigned char C = *TokenEnd; |
1301 | while (isHorizontalWhitespace(C)) { |
1302 | C = *(++TokenEnd); |
1303 | NumWhitespaceChars++; |
1304 | } |
1305 | |
1306 | |
1307 | if (C == '\n' || C == '\r') { |
1308 | char PrevC = C; |
1309 | C = *(++TokenEnd); |
1310 | NumWhitespaceChars++; |
1311 | if ((C == '\n' || C == '\r') && C != PrevC) |
1312 | NumWhitespaceChars++; |
1313 | } |
1314 | } |
1315 | |
1316 | return TokenLoc.getLocWithOffset(Tok->getLength() + NumWhitespaceChars); |
1317 | } |
1318 | |
1319 | |
1320 | |
1321 | |
1322 | |
1323 | |
1324 | |
1325 | |
1326 | |
1327 | |
1328 | |
1329 | |
1330 | |
1331 | |
1332 | |
1333 | |
1334 | char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, |
1335 | Token *Tok) { |
1336 | |
1337 | if (Ptr[0] == '\\') { |
1338 | ++Size; |
1339 | ++Ptr; |
1340 | Slash: |
1341 | |
1342 | if (!isWhitespace(Ptr[0])) return '\\'; |
1343 | |
1344 | |
1345 | |
1346 | if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { |
1347 | |
1348 | if (Tok) Tok->setFlag(Token::NeedsCleaning); |
1349 | |
1350 | |
1351 | if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) |
1352 | Diag(Ptr, diag::backslash_newline_space); |
1353 | |
1354 | |
1355 | Size += EscapedNewLineSize; |
1356 | Ptr += EscapedNewLineSize; |
1357 | |
1358 | |
1359 | return getCharAndSizeSlow(Ptr, Size, Tok); |
1360 | } |
1361 | |
1362 | |
1363 | return '\\'; |
1364 | } |
1365 | |
1366 | |
1367 | if (Ptr[0] == '?' && Ptr[1] == '?') { |
1368 | |
1369 | |
1370 | if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) { |
1371 | |
1372 | if (Tok) Tok->setFlag(Token::NeedsCleaning); |
1373 | |
1374 | Ptr += 3; |
1375 | Size += 3; |
1376 | if (C == '\\') goto Slash; |
1377 | return C; |
1378 | } |
1379 | } |
1380 | |
1381 | |
1382 | ++Size; |
1383 | return *Ptr; |
1384 | } |
1385 | |
1386 | |
1387 | |
1388 | |
1389 | |
1390 | |
1391 | |
1392 | char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, |
1393 | const LangOptions &LangOpts) { |
1394 | |
1395 | if (Ptr[0] == '\\') { |
1396 | ++Size; |
1397 | ++Ptr; |
1398 | Slash: |
1399 | |
1400 | if (!isWhitespace(Ptr[0])) return '\\'; |
1401 | |
1402 | |
1403 | if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { |
1404 | |
1405 | Size += EscapedNewLineSize; |
1406 | Ptr += EscapedNewLineSize; |
1407 | |
1408 | |
1409 | return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); |
1410 | } |
1411 | |
1412 | |
1413 | return '\\'; |
1414 | } |
1415 | |
1416 | |
1417 | if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { |
1418 | |
1419 | |
1420 | if (char C = GetTrigraphCharForLetter(Ptr[2])) { |
1421 | Ptr += 3; |
1422 | Size += 3; |
1423 | if (C == '\\') goto Slash; |
1424 | return C; |
1425 | } |
1426 | } |
1427 | |
1428 | |
1429 | ++Size; |
1430 | return *Ptr; |
1431 | } |
1432 | |
1433 | |
1434 | |
1435 | |
1436 | |
1437 | |
1438 | void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) { |
1439 | BufferPtr = BufferStart + Offset; |
1440 | if (BufferPtr > BufferEnd) |
1441 | BufferPtr = BufferEnd; |
1442 | |
1443 | |
1444 | |
1445 | IsAtStartOfLine = StartOfLine; |
1446 | IsAtPhysicalStartOfLine = StartOfLine; |
1447 | } |
1448 | |
1449 | static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { |
1450 | if (LangOpts.AsmPreprocessor) { |
1451 | return false; |
1452 | } else if (LangOpts.DollarIdents && '$' == C) { |
1453 | return true; |
1454 | } else if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
1455 | static const llvm::sys::UnicodeCharSet C11AllowedIDChars( |
1456 | C11AllowedIDCharRanges); |
1457 | return C11AllowedIDChars.contains(C); |
1458 | } else if (LangOpts.CPlusPlus) { |
1459 | static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( |
1460 | CXX03AllowedIDCharRanges); |
1461 | return CXX03AllowedIDChars.contains(C); |
1462 | } else { |
1463 | static const llvm::sys::UnicodeCharSet C99AllowedIDChars( |
1464 | C99AllowedIDCharRanges); |
1465 | return C99AllowedIDChars.contains(C); |
1466 | } |
1467 | } |
1468 | |
1469 | static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { |
1470 | assert(isAllowedIDChar(C, LangOpts)); |
1471 | if (LangOpts.AsmPreprocessor) { |
1472 | return false; |
1473 | } else if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
1474 | static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars( |
1475 | C11DisallowedInitialIDCharRanges); |
1476 | return !C11DisallowedInitialIDChars.contains(C); |
1477 | } else if (LangOpts.CPlusPlus) { |
1478 | return true; |
1479 | } else { |
1480 | static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( |
1481 | C99DisallowedInitialIDCharRanges); |
1482 | return !C99DisallowedInitialIDChars.contains(C); |
1483 | } |
1484 | } |
1485 | |
1486 | static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, |
1487 | const char *End) { |
1488 | return CharSourceRange::getCharRange(L.getSourceLocation(Begin), |
1489 | L.getSourceLocation(End)); |
1490 | } |
1491 | |
1492 | static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, |
1493 | CharSourceRange Range, bool IsFirst) { |
1494 | |
1495 | if (!Diags.isIgnored(diag::warn_c99_compat_unicode_id, Range.getBegin())) { |
1496 | enum { |
1497 | CannotAppearInIdentifier = 0, |
1498 | CannotStartIdentifier |
1499 | }; |
1500 | |
1501 | static const llvm::sys::UnicodeCharSet C99AllowedIDChars( |
1502 | C99AllowedIDCharRanges); |
1503 | static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( |
1504 | C99DisallowedInitialIDCharRanges); |
1505 | if (!C99AllowedIDChars.contains(C)) { |
1506 | Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) |
1507 | << Range |
1508 | << CannotAppearInIdentifier; |
1509 | } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) { |
1510 | Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) |
1511 | << Range |
1512 | << CannotStartIdentifier; |
1513 | } |
1514 | } |
1515 | |
1516 | |
1517 | if (!Diags.isIgnored(diag::warn_cxx98_compat_unicode_id, Range.getBegin())) { |
1518 | static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( |
1519 | CXX03AllowedIDCharRanges); |
1520 | if (!CXX03AllowedIDChars.contains(C)) { |
1521 | Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) |
1522 | << Range; |
1523 | } |
1524 | } |
1525 | } |
1526 | |
1527 | |
1528 | |
1529 | |
1530 | |
1531 | static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C, |
1532 | CharSourceRange Range) { |
1533 | |
1534 | struct HomoglyphPair { |
1535 | uint32_t Character; |
1536 | char LooksLike; |
1537 | bool operator<(HomoglyphPair R) const { return Character < R.Character; } |
1538 | }; |
1539 | static constexpr HomoglyphPair SortedHomoglyphs[] = { |
1540 | {U'\u00ad', 0}, |
1541 | {U'\u01c3', '!'}, |
1542 | {U'\u037e', ';'}, |
1543 | {U'\u200b', 0}, |
1544 | {U'\u200c', 0}, |
1545 | {U'\u200d', 0}, |
1546 | {U'\u2060', 0}, |
1547 | {U'\u2061', 0}, |
1548 | {U'\u2062', 0}, |
1549 | {U'\u2063', 0}, |
1550 | {U'\u2064', 0}, |
1551 | {U'\u2212', '-'}, |
1552 | {U'\u2215', '/'}, |
1553 | {U'\u2216', '\\'}, |
1554 | {U'\u2217', '*'}, |
1555 | {U'\u2223', '|'}, |
1556 | {U'\u2227', '^'}, |
1557 | {U'\u2236', ':'}, |
1558 | {U'\u223c', '~'}, |
1559 | {U'\ua789', ':'}, |
1560 | {U'\ufeff', 0}, |
1561 | {U'\uff01', '!'}, |
1562 | {U'\uff03', '#'}, |
1563 | {U'\uff04', '$'}, |
1564 | {U'\uff05', '%'}, |
1565 | {U'\uff06', '&'}, |
1566 | {U'\uff08', '('}, |
1567 | {U'\uff09', ')'}, |
1568 | {U'\uff0a', '*'}, |
1569 | {U'\uff0b', '+'}, |
1570 | {U'\uff0c', ','}, |
1571 | {U'\uff0d', '-'}, |
1572 | {U'\uff0e', '.'}, |
1573 | {U'\uff0f', '/'}, |
1574 | {U'\uff1a', ':'}, |
1575 | {U'\uff1b', ';'}, |
1576 | {U'\uff1c', '<'}, |
1577 | {U'\uff1d', '='}, |
1578 | {U'\uff1e', '>'}, |
1579 | {U'\uff1f', '?'}, |
1580 | {U'\uff20', '@'}, |
1581 | {U'\uff3b', '['}, |
1582 | {U'\uff3c', '\\'}, |
1583 | {U'\uff3d', ']'}, |
1584 | {U'\uff3e', '^'}, |
1585 | {U'\uff5b', '{'}, |
1586 | {U'\uff5c', '|'}, |
1587 | {U'\uff5d', '}'}, |
1588 | {U'\uff5e', '~'}, |
1589 | {0, 0} |
1590 | }; |
1591 | auto Homoglyph = |
1592 | std::lower_bound(std::begin(SortedHomoglyphs), |
1593 | std::end(SortedHomoglyphs) - 1, HomoglyphPair{C, '\0'}); |
1594 | if (Homoglyph->Character == C) { |
1595 | llvm::SmallString<5> CharBuf; |
1596 | { |
1597 | llvm::raw_svector_ostream CharOS(CharBuf); |
1598 | llvm::write_hex(CharOS, C, llvm::HexPrintStyle::Upper, 4); |
1599 | } |
1600 | if (Homoglyph->LooksLike) { |
1601 | const char LooksLikeStr[] = {Homoglyph->LooksLike, 0}; |
1602 | Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_homoglyph) |
1603 | << Range << CharBuf << LooksLikeStr; |
1604 | } else { |
1605 | Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_zero_width) |
1606 | << Range << CharBuf; |
1607 | } |
1608 | } |
1609 | } |
1610 | |
1611 | bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size, |
1612 | Token &Result) { |
1613 | const char *UCNPtr = CurPtr + Size; |
1614 | uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, nullptr); |
1615 | if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) |
1616 | return false; |
1617 | |
1618 | if (!isLexingRawMode()) |
1619 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, |
1620 | makeCharRange(*this, CurPtr, UCNPtr), |
1621 | false); |
1622 | |
1623 | Result.setFlag(Token::HasUCN); |
1624 | if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || |
1625 | (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) |
1626 | CurPtr = UCNPtr; |
1627 | else |
1628 | while (CurPtr != UCNPtr) |
1629 | (void)getAndAdvanceChar(CurPtr, Result); |
1630 | return true; |
1631 | } |
1632 | |
1633 | bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) { |
1634 | const char *UnicodePtr = CurPtr; |
1635 | llvm::UTF32 CodePoint; |
1636 | llvm::ConversionResult Result = |
1637 | llvm::convertUTF8Sequence((const llvm::UTF8 **)&UnicodePtr, |
1638 | (const llvm::UTF8 *)BufferEnd, |
1639 | &CodePoint, |
1640 | llvm::strictConversion); |
1641 | if (Result != llvm::conversionOK || |
1642 | !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) |
1643 | return false; |
1644 | |
1645 | if (!isLexingRawMode()) { |
1646 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, |
1647 | makeCharRange(*this, CurPtr, UnicodePtr), |
1648 | false); |
1649 | maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), CodePoint, |
1650 | makeCharRange(*this, CurPtr, UnicodePtr)); |
1651 | } |
1652 | |
1653 | CurPtr = UnicodePtr; |
1654 | return true; |
1655 | } |
1656 | |
1657 | bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) { |
1658 | |
1659 | unsigned Size; |
1660 | unsigned char C = *CurPtr++; |
1661 | while (isIdentifierBody(C)) |
1662 | C = *CurPtr++; |
1663 | |
1664 | --CurPtr; |
1665 | |
1666 | |
1667 | |
1668 | |
1669 | |
1670 | |
1671 | if (isASCII(C) && C != '\\' && C != '?' && |
1672 | (C != '$' || !LangOpts.DollarIdents)) { |
1673 | FinishIdentifier: |
1674 | const char *IdStart = BufferPtr; |
1675 | FormTokenWithChars(Result, CurPtr, tok::raw_identifier); |
1676 | Result.setRawIdentifierData(IdStart); |
1677 | |
1678 | |
1679 | |
1680 | if (LexingRawMode) |
1681 | return true; |
1682 | |
1683 | |
1684 | |
1685 | IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); |
1686 | |
1687 | |
1688 | |
1689 | |
1690 | |
1691 | |
1692 | if (isCodeCompletionPoint(CurPtr)) { |
1693 | |
1694 | Result.setKind(tok::code_completion); |
1695 | |
1696 | |
1697 | |
1698 | |
1699 | |
1700 | assert(*CurPtr == 0 && "Completion character must be 0"); |
1701 | ++CurPtr; |
1702 | |
1703 | |
1704 | |
1705 | if (CurPtr < BufferEnd) { |
1706 | while (isIdentifierBody(*CurPtr)) |
1707 | ++CurPtr; |
1708 | } |
1709 | BufferPtr = CurPtr; |
1710 | return true; |
1711 | } |
1712 | |
1713 | |
1714 | |
1715 | if (II->isHandleIdentifierCase()) |
1716 | return PP->HandleIdentifier(Result); |
1717 | |
1718 | return true; |
1719 | } |
1720 | |
1721 | |
1722 | |
1723 | C = getCharAndSize(CurPtr, Size); |
1724 | while (true) { |
1725 | if (C == '$') { |
1726 | |
1727 | if (!LangOpts.DollarIdents) goto FinishIdentifier; |
1728 | |
1729 | |
1730 | if (!isLexingRawMode()) |
1731 | Diag(CurPtr, diag::ext_dollar_in_identifier); |
1732 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
1733 | C = getCharAndSize(CurPtr, Size); |
1734 | continue; |
1735 | } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) { |
1736 | C = getCharAndSize(CurPtr, Size); |
1737 | continue; |
1738 | } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) { |
1739 | C = getCharAndSize(CurPtr, Size); |
1740 | continue; |
1741 | } else if (!isIdentifierBody(C)) { |
1742 | goto FinishIdentifier; |
1743 | } |
1744 | |
1745 | |
1746 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
1747 | |
1748 | C = getCharAndSize(CurPtr, Size); |
1749 | while (isIdentifierBody(C)) { |
1750 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
1751 | C = getCharAndSize(CurPtr, Size); |
1752 | } |
1753 | } |
1754 | } |
1755 | |
1756 | |
1757 | |
1758 | bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { |
1759 | unsigned Size; |
1760 | char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); |
1761 | if (C1 != '0') |
1762 | return false; |
1763 | char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); |
1764 | return (C2 == 'x' || C2 == 'X'); |
1765 | } |
1766 | |
1767 | |
1768 | |
1769 | |
1770 | bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { |
1771 | unsigned Size; |
1772 | char C = getCharAndSize(CurPtr, Size); |
1773 | char PrevCh = 0; |
1774 | while (isPreprocessingNumberBody(C)) { |
1775 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
1776 | PrevCh = C; |
1777 | C = getCharAndSize(CurPtr, Size); |
1778 | } |
1779 | |
1780 | |
1781 | if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { |
1782 | |
1783 | |
1784 | if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) |
1785 | return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); |
1786 | } |
1787 | |
1788 | |
1789 | if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { |
1790 | |
1791 | |
1792 | |
1793 | bool IsHexFloat = true; |
1794 | if (!LangOpts.C99) { |
1795 | if (!isHexaLiteral(BufferPtr, LangOpts)) |
1796 | IsHexFloat = false; |
1797 | else if (!getLangOpts().CPlusPlus17 && |
1798 | std::find(BufferPtr, CurPtr, '_') != CurPtr) |
1799 | IsHexFloat = false; |
1800 | } |
1801 | if (IsHexFloat) |
1802 | return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); |
1803 | } |
1804 | |
1805 | |
1806 | if (C == '\'' && (getLangOpts().CPlusPlus14 || getLangOpts().C2x)) { |
1807 | unsigned NextSize; |
1808 | char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts()); |
1809 | if (isIdentifierBody(Next)) { |
1810 | if (!isLexingRawMode()) |
1811 | Diag(CurPtr, getLangOpts().CPlusPlus |
1812 | ? diag::warn_cxx11_compat_digit_separator |
1813 | : diag::warn_c2x_compat_digit_separator); |
1814 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
1815 | CurPtr = ConsumeChar(CurPtr, NextSize, Result); |
1816 | return LexNumericConstant(Result, CurPtr); |
1817 | } |
1818 | } |
1819 | |
1820 | |
1821 | if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) |
1822 | return LexNumericConstant(Result, CurPtr); |
1823 | if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) |
1824 | return LexNumericConstant(Result, CurPtr); |
1825 | |
1826 | |
1827 | const char *TokStart = BufferPtr; |
1828 | FormTokenWithChars(Result, CurPtr, tok::numeric_constant); |
1829 | Result.setLiteralData(TokStart); |
1830 | return true; |
1831 | } |
1832 | |
1833 | |
1834 | |
1835 | const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr, |
1836 | bool IsStringLiteral) { |
1837 | assert(getLangOpts().CPlusPlus); |
1838 | |
1839 | |
1840 | unsigned Size; |
1841 | char C = getCharAndSize(CurPtr, Size); |
1842 | bool Consumed = false; |
1843 | |
1844 | if (!isIdentifierHead(C)) { |
1845 | if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) |
1846 | Consumed = true; |
1847 | else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) |
1848 | Consumed = true; |
1849 | else |
1850 | return CurPtr; |
1851 | } |
1852 | |
1853 | if (!getLangOpts().CPlusPlus11) { |
1854 | if (!isLexingRawMode()) |
1855 | Diag(CurPtr, |
1856 | C == '_' ? diag::warn_cxx11_compat_user_defined_literal |
1857 | : diag::warn_cxx11_compat_reserved_user_defined_literal) |
1858 | << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); |
1859 | return CurPtr; |
1860 | } |
1861 | |
1862 | |
1863 | |
1864 | |
1865 | |
1866 | |
1867 | if (!Consumed) { |
1868 | bool IsUDSuffix = false; |
1869 | if (C == '_') |
1870 | IsUDSuffix = true; |
1871 | else if (IsStringLiteral && getLangOpts().CPlusPlus14) { |
1872 | |
1873 | |
1874 | |
1875 | const unsigned MaxStandardSuffixLength = 3; |
1876 | char Buffer[MaxStandardSuffixLength] = { C }; |
1877 | unsigned Consumed = Size; |
1878 | unsigned Chars = 1; |
1879 | while (true) { |
1880 | unsigned NextSize; |
1881 | char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize, |
1882 | getLangOpts()); |
1883 | if (!isIdentifierBody(Next)) { |
1884 | |
1885 | const StringRef CompleteSuffix(Buffer, Chars); |
1886 | IsUDSuffix = StringLiteralParser::isValidUDSuffix(getLangOpts(), |
1887 | CompleteSuffix); |
1888 | break; |
1889 | } |
1890 | |
1891 | if (Chars == MaxStandardSuffixLength) |
1892 | |
1893 | break; |
1894 | |
1895 | Buffer[Chars++] = Next; |
1896 | Consumed += NextSize; |
1897 | } |
1898 | } |
1899 | |
1900 | if (!IsUDSuffix) { |
1901 | if (!isLexingRawMode()) |
1902 | Diag(CurPtr, getLangOpts().MSVCCompat |
1903 | ? diag::ext_ms_reserved_user_defined_literal |
1904 | : diag::ext_reserved_user_defined_literal) |
1905 | << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); |
1906 | return CurPtr; |
1907 | } |
1908 | |
1909 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
1910 | } |
1911 | |
1912 | Result.setFlag(Token::HasUDSuffix); |
1913 | while (true) { |
1914 | C = getCharAndSize(CurPtr, Size); |
1915 | if (isIdentifierBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); } |
1916 | else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {} |
1917 | else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {} |
1918 | else break; |
1919 | } |
1920 | |
1921 | return CurPtr; |
1922 | } |
1923 | |
1924 | |
1925 | |
1926 | bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr, |
1927 | tok::TokenKind Kind) { |
1928 | const char *AfterQuote = CurPtr; |
1929 | |
1930 | const char *NulCharacter = nullptr; |
1931 | |
1932 | if (!isLexingRawMode() && |
1933 | (Kind == tok::utf8_string_literal || |
1934 | Kind == tok::utf16_string_literal || |
1935 | Kind == tok::utf32_string_literal)) |
1936 | Diag(BufferPtr, getLangOpts().CPlusPlus |
1937 | ? diag::warn_cxx98_compat_unicode_literal |
1938 | : diag::warn_c99_compat_unicode_literal); |
1939 | |
1940 | char C = getAndAdvanceChar(CurPtr, Result); |
1941 | while (C != '"') { |
1942 | |
1943 | |
1944 | if (C == '\\') |
1945 | C = getAndAdvanceChar(CurPtr, Result); |
1946 | |
1947 | if (C == '\n' || C == '\r' || |
1948 | (C == 0 && CurPtr-1 == BufferEnd)) { |
1949 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
1950 | Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; |
1951 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
1952 | return true; |
1953 | } |
1954 | |
1955 | if (C == 0) { |
1956 | if (isCodeCompletionPoint(CurPtr-1)) { |
1957 | if (ParsingFilename) |
1958 | codeCompleteIncludedFile(AfterQuote, CurPtr - 1, false); |
1959 | else |
1960 | PP->CodeCompleteNaturalLanguage(); |
1961 | FormTokenWithChars(Result, CurPtr - 1, tok::unknown); |
1962 | cutOffLexing(); |
1963 | return true; |
1964 | } |
1965 | |
1966 | NulCharacter = CurPtr-1; |
1967 | } |
1968 | C = getAndAdvanceChar(CurPtr, Result); |
1969 | } |
1970 | |
1971 | |
1972 | if (getLangOpts().CPlusPlus) |
1973 | CurPtr = LexUDSuffix(Result, CurPtr, true); |
1974 | |
1975 | |
1976 | if (NulCharacter && !isLexingRawMode()) |
1977 | Diag(NulCharacter, diag::null_in_char_or_string) << 1; |
1978 | |
1979 | |
1980 | const char *TokStart = BufferPtr; |
1981 | FormTokenWithChars(Result, CurPtr, Kind); |
1982 | Result.setLiteralData(TokStart); |
1983 | return true; |
1984 | } |
1985 | |
1986 | |
1987 | |
1988 | bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, |
1989 | tok::TokenKind Kind) { |
1990 | |
1991 | |
1992 | |
1993 | |
1994 | |
1995 | if (!isLexingRawMode()) |
1996 | Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); |
1997 | |
1998 | unsigned PrefixLen = 0; |
1999 | |
2000 | while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) |
2001 | ++PrefixLen; |
2002 | |
2003 | |
2004 | if (CurPtr[PrefixLen] != '(') { |
2005 | if (!isLexingRawMode()) { |
2006 | const char *PrefixEnd = &CurPtr[PrefixLen]; |
2007 | if (PrefixLen == 16) { |
2008 | Diag(PrefixEnd, diag::err_raw_delim_too_long); |
2009 | } else { |
2010 | Diag(PrefixEnd, diag::err_invalid_char_raw_delim) |
2011 | << StringRef(PrefixEnd, 1); |
2012 | } |
2013 | } |
2014 | |
2015 | |
2016 | |
2017 | |
2018 | while (true) { |
2019 | char C = *CurPtr++; |
2020 | |
2021 | if (C == '"') |
2022 | break; |
2023 | if (C == 0 && CurPtr-1 == BufferEnd) { |
2024 | --CurPtr; |
2025 | break; |
2026 | } |
2027 | } |
2028 | |
2029 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
2030 | return true; |
2031 | } |
2032 | |
2033 | |
2034 | const char *Prefix = CurPtr; |
2035 | CurPtr += PrefixLen + 1; |
2036 | |
2037 | while (true) { |
2038 | char C = *CurPtr++; |
2039 | |
2040 | if (C == ')') { |
2041 | |
2042 | if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { |
2043 | CurPtr += PrefixLen + 1; |
2044 | break; |
2045 | } |
2046 | } else if (C == 0 && CurPtr-1 == BufferEnd) { |
2047 | if (!isLexingRawMode()) |
2048 | Diag(BufferPtr, diag::err_unterminated_raw_string) |
2049 | << StringRef(Prefix, PrefixLen); |
2050 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
2051 | return true; |
2052 | } |
2053 | } |
2054 | |
2055 | |
2056 | if (getLangOpts().CPlusPlus) |
2057 | CurPtr = LexUDSuffix(Result, CurPtr, true); |
2058 | |
2059 | |
2060 | const char *TokStart = BufferPtr; |
2061 | FormTokenWithChars(Result, CurPtr, Kind); |
2062 | Result.setLiteralData(TokStart); |
2063 | return true; |
2064 | } |
2065 | |
2066 | |
2067 | |
2068 | bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { |
2069 | |
2070 | const char *NulCharacter = nullptr; |
2071 | const char *AfterLessPos = CurPtr; |
2072 | char C = getAndAdvanceChar(CurPtr, Result); |
2073 | while (C != '>') { |
2074 | |
2075 | |
2076 | if (C == '\\') |
2077 | C = getAndAdvanceChar(CurPtr, Result); |
2078 | |
2079 | if (isVerticalWhitespace(C) || |
2080 | (C == 0 && (CurPtr - 1 == BufferEnd))) { |
2081 | |
2082 | |
2083 | FormTokenWithChars(Result, AfterLessPos, tok::less); |
2084 | return true; |
2085 | } |
2086 | |
2087 | if (C == 0) { |
2088 | if (isCodeCompletionPoint(CurPtr - 1)) { |
2089 | codeCompleteIncludedFile(AfterLessPos, CurPtr - 1, true); |
2090 | cutOffLexing(); |
2091 | FormTokenWithChars(Result, CurPtr - 1, tok::unknown); |
2092 | return true; |
2093 | } |
2094 | NulCharacter = CurPtr-1; |
2095 | } |
2096 | C = getAndAdvanceChar(CurPtr, Result); |
2097 | } |
2098 | |
2099 | |
2100 | if (NulCharacter && !isLexingRawMode()) |
2101 | Diag(NulCharacter, diag::null_in_char_or_string) << 1; |
2102 | |
2103 | |
2104 | const char *TokStart = BufferPtr; |
2105 | FormTokenWithChars(Result, CurPtr, tok::header_name); |
2106 | Result.setLiteralData(TokStart); |
2107 | return true; |
2108 | } |
2109 | |
2110 | void Lexer::codeCompleteIncludedFile(const char *PathStart, |
2111 | const char *CompletionPoint, |
2112 | bool IsAngled) { |
2113 | |
2114 | StringRef PartialPath(PathStart, CompletionPoint - PathStart); |
2115 | llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/"; |
2116 | auto Slash = PartialPath.find_last_of(SlashChars); |
2117 | StringRef Dir = |
2118 | (Slash == StringRef::npos) ? "" : PartialPath.take_front(Slash); |
2119 | const char *StartOfFilename = |
2120 | (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1; |
2121 | |
2122 | PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get( |
2123 | StringRef(StartOfFilename, CompletionPoint - StartOfFilename))); |
2124 | |
2125 | |
2126 | while (CompletionPoint < BufferEnd) { |
2127 | char Next = *(CompletionPoint + 1); |
2128 | if (Next == 0 || Next == '\r' || Next == '\n') |
2129 | break; |
2130 | ++CompletionPoint; |
2131 | if (Next == (IsAngled ? '>' : '"')) |
2132 | break; |
2133 | if (llvm::is_contained(SlashChars, Next)) |
2134 | break; |
2135 | } |
2136 | |
2137 | PP->setCodeCompletionTokenRange( |
2138 | FileLoc.getLocWithOffset(StartOfFilename - BufferStart), |
2139 | FileLoc.getLocWithOffset(CompletionPoint - BufferStart)); |
2140 | PP->CodeCompleteIncludedFile(Dir, IsAngled); |
2141 | } |
2142 | |
2143 | |
2144 | |
2145 | bool Lexer::LexCharConstant(Token &Result, const char *CurPtr, |
2146 | tok::TokenKind Kind) { |
2147 | |
2148 | const char *NulCharacter = nullptr; |
2149 | |
2150 | if (!isLexingRawMode()) { |
2151 | if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant) |
2152 | Diag(BufferPtr, getLangOpts().CPlusPlus |
2153 | ? diag::warn_cxx98_compat_unicode_literal |
2154 | : diag::warn_c99_compat_unicode_literal); |
2155 | else if (Kind == tok::utf8_char_constant) |
2156 | Diag(BufferPtr, diag::warn_cxx14_compat_u8_character_literal); |
2157 | } |
2158 | |
2159 | char C = getAndAdvanceChar(CurPtr, Result); |
2160 | if (C == '\'') { |
2161 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
2162 | Diag(BufferPtr, diag::ext_empty_character); |
2163 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
2164 | return true; |
2165 | } |
2166 | |
2167 | while (C != '\'') { |
2168 | |
2169 | if (C == '\\') |
2170 | C = getAndAdvanceChar(CurPtr, Result); |
2171 | |
2172 | if (C == '\n' || C == '\r' || |
2173 | (C == 0 && CurPtr-1 == BufferEnd)) { |
2174 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
2175 | Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 0; |
2176 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
2177 | return true; |
2178 | } |
2179 | |
2180 | if (C == 0) { |
2181 | if (isCodeCompletionPoint(CurPtr-1)) { |
2182 | PP->CodeCompleteNaturalLanguage(); |
2183 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
2184 | cutOffLexing(); |
2185 | return true; |
2186 | } |
2187 | |
2188 | NulCharacter = CurPtr-1; |
2189 | } |
2190 | C = getAndAdvanceChar(CurPtr, Result); |
2191 | } |
2192 | |
2193 | |
2194 | if (getLangOpts().CPlusPlus) |
2195 | CurPtr = LexUDSuffix(Result, CurPtr, false); |
2196 | |
2197 | |
2198 | if (NulCharacter && !isLexingRawMode()) |
2199 | Diag(NulCharacter, diag::null_in_char_or_string) << 0; |
2200 | |
2201 | |
2202 | const char *TokStart = BufferPtr; |
2203 | FormTokenWithChars(Result, CurPtr, Kind); |
2204 | Result.setLiteralData(TokStart); |
2205 | return true; |
2206 | } |
2207 | |
2208 | |
2209 | |
2210 | |
2211 | |
2212 | bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr, |
2213 | bool &TokAtPhysicalStartOfLine) { |
2214 | |
2215 | bool SawNewline = isVerticalWhitespace(CurPtr[-1]); |
2216 | |
2217 | unsigned char Char = *CurPtr; |
2218 | |
2219 | const char *lastNewLine = nullptr; |
2220 | auto setLastNewLine = [&](const char *Ptr) { |
2221 | lastNewLine = Ptr; |
2222 | if (!NewLinePtr) |
2223 | NewLinePtr = Ptr; |
2224 | }; |
2225 | if (SawNewline) |
2226 | setLastNewLine(CurPtr - 1); |
2227 | |
2228 | |
2229 | while (true) { |
2230 | |
2231 | while (isHorizontalWhitespace(Char)) |
2232 | Char = *++CurPtr; |
2233 | |
2234 | |
2235 | if (!isVerticalWhitespace(Char)) |
2236 | break; |
2237 | |
2238 | if (ParsingPreprocessorDirective) { |
2239 | |
2240 | BufferPtr = CurPtr; |
2241 | return false; |
2242 | } |
2243 | |
2244 | |
2245 | if (*CurPtr == '\n') |
2246 | setLastNewLine(CurPtr); |
2247 | SawNewline = true; |
2248 | Char = *++CurPtr; |
2249 | } |
2250 | |
2251 | |
2252 | if (isKeepWhitespaceMode()) { |
2253 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
2254 | if (SawNewline) { |
2255 | IsAtStartOfLine = true; |
2256 | IsAtPhysicalStartOfLine = true; |
2257 | } |
2258 | |
2259 | return true; |
2260 | } |
2261 | |
2262 | |
2263 | char PrevChar = CurPtr[-1]; |
2264 | bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); |
2265 | |
2266 | Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); |
2267 | if (SawNewline) { |
2268 | Result.setFlag(Token::StartOfLine); |
2269 | TokAtPhysicalStartOfLine = true; |
2270 | |
2271 | if (NewLinePtr && lastNewLine && NewLinePtr != lastNewLine && PP) { |
2272 | if (auto *Handler = PP->getEmptylineHandler()) |
2273 | Handler->HandleEmptyline(SourceRange(getSourceLocation(NewLinePtr + 1), |
2274 | getSourceLocation(lastNewLine))); |
2275 | } |
2276 | } |
2277 | |
2278 | BufferPtr = CurPtr; |
2279 | return false; |
2280 | } |
2281 | |
2282 | |
2283 | |
2284 | |
2285 | |
2286 | |
2287 | |
2288 | bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, |
2289 | bool &TokAtPhysicalStartOfLine) { |
2290 | |
2291 | |
2292 | if (!LangOpts.LineComment && !isLexingRawMode()) { |
2293 | Diag(BufferPtr, diag::ext_line_comment); |
2294 | |
2295 | |
2296 | |
2297 | LangOpts.LineComment = true; |
2298 | } |
2299 | |
2300 | |
2301 | |
2302 | |
2303 | |
2304 | |
2305 | |
2306 | char C; |
2307 | while (true) { |
2308 | C = *CurPtr; |
2309 | |
2310 | while (C != 0 && |
2311 | C != '\n' && C != '\r') |
2312 | C = *++CurPtr; |
2313 | |
2314 | const char *NextLine = CurPtr; |
2315 | if (C != 0) { |
2316 | |
2317 | const char *EscapePtr = CurPtr-1; |
2318 | bool HasSpace = false; |
2319 | while (isHorizontalWhitespace(*EscapePtr)) { |
2320 | --EscapePtr; |
2321 | HasSpace = true; |
2322 | } |
2323 | |
2324 | if (*EscapePtr == '\\') |
2325 | |
2326 | CurPtr = EscapePtr; |
2327 | else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && |
2328 | EscapePtr[-2] == '?' && LangOpts.Trigraphs) |
2329 | |
2330 | CurPtr = EscapePtr-2; |
2331 | else |
2332 | break; |
2333 | |
2334 | |
2335 | if (HasSpace && !isLexingRawMode()) |
2336 | Diag(EscapePtr, diag::backslash_newline_space); |
2337 | } |
2338 | |
2339 | |
2340 | |
2341 | |
2342 | |
2343 | const char *OldPtr = CurPtr; |
2344 | bool OldRawMode = isLexingRawMode(); |
2345 | LexingRawMode = true; |
2346 | C = getAndAdvanceChar(CurPtr, Result); |
2347 | LexingRawMode = OldRawMode; |
2348 | |
2349 | |
2350 | |
2351 | if (C != 0 && CurPtr == OldPtr+1) { |
2352 | CurPtr = NextLine; |
2353 | break; |
2354 | } |
2355 | |
2356 | |
2357 | |
2358 | |
2359 | if (CurPtr != OldPtr + 1 && C != '/' && |
2360 | (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) { |
2361 | for (; OldPtr != CurPtr; ++OldPtr) |
2362 | if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { |
2363 | |
2364 | |
2365 | if (isWhitespace(C)) { |
2366 | const char *ForwardPtr = CurPtr; |
2367 | while (isWhitespace(*ForwardPtr)) |
2368 | ++ForwardPtr; |
2369 | if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') |
2370 | break; |
2371 | } |
2372 | |
2373 | if (!isLexingRawMode()) |
2374 | Diag(OldPtr-1, diag::ext_multi_line_line_comment); |
2375 | break; |
2376 | } |
2377 | } |
2378 | |
2379 | if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) { |
2380 | --CurPtr; |
2381 | break; |
2382 | } |
2383 | |
2384 | if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { |
2385 | PP->CodeCompleteNaturalLanguage(); |
2386 | cutOffLexing(); |
2387 | return false; |
2388 | } |
2389 | } |
2390 | |
2391 | |
2392 | |
2393 | if (PP && !isLexingRawMode() && |
2394 | PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), |
2395 | getSourceLocation(CurPtr)))) { |
2396 | BufferPtr = CurPtr; |
2397 | return true; |
2398 | } |
2399 | |
2400 | |
2401 | if (inKeepCommentMode()) |
2402 | return SaveLineComment(Result, CurPtr); |
2403 | |
2404 | |
2405 | |
2406 | if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { |
2407 | BufferPtr = CurPtr; |
2408 | return false; |
2409 | } |
2410 | |
2411 | |
2412 | |
2413 | |
2414 | |
2415 | |
2416 | NewLinePtr = CurPtr++; |
2417 | |
2418 | |
2419 | Result.setFlag(Token::StartOfLine); |
2420 | TokAtPhysicalStartOfLine = true; |
2421 | |
2422 | Result.clearFlag(Token::LeadingSpace); |
2423 | BufferPtr = CurPtr; |
2424 | return false; |
2425 | } |
2426 | |
2427 | |
2428 | |
2429 | bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { |
2430 | |
2431 | |
2432 | FormTokenWithChars(Result, CurPtr, tok::comment); |
2433 | |
2434 | if (!ParsingPreprocessorDirective || LexingRawMode) |
2435 | return true; |
2436 | |
2437 | |
2438 | |
2439 | bool Invalid = false; |
2440 | std::string Spelling = PP->getSpelling(Result, &Invalid); |
2441 | if (Invalid) |
2442 | return true; |
2443 | |
2444 | assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); |
2445 | Spelling[1] = '*'; |
2446 | Spelling += "*/"; |
2447 | |
2448 | Result.setKind(tok::comment); |
2449 | PP->CreateString(Spelling, Result, |
2450 | Result.getLocation(), Result.getLocation()); |
2451 | return true; |
2452 | } |
2453 | |
2454 | |
2455 | |
2456 | |
2457 | static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, |
2458 | Lexer *L) { |
2459 | assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); |
2460 | |
2461 | |
2462 | const char *TrigraphPos = 0; |
2463 | |
2464 | const char *SpacePos = 0; |
2465 | |
2466 | while (true) { |
2467 | |
2468 | --CurPtr; |
2469 | |
2470 | |
2471 | if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { |
2472 | |
2473 | if (CurPtr[0] == CurPtr[1]) |
2474 | return false; |
2475 | |
2476 | --CurPtr; |
2477 | } |
2478 | |
2479 | |
2480 | |
2481 | while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { |
2482 | SpacePos = CurPtr; |
2483 | --CurPtr; |
2484 | } |
2485 | |
2486 | |
2487 | if (*CurPtr == '\\') { |
2488 | --CurPtr; |
2489 | } else if (CurPtr[0] == '/' && CurPtr[-1] == '?' && CurPtr[-2] == '?') { |
2490 | |
2491 | TrigraphPos = CurPtr - 2; |
2492 | CurPtr -= 3; |
2493 | } else { |
2494 | return false; |
2495 | } |
2496 | |
2497 | |
2498 | |
2499 | if (*CurPtr == '*') |
2500 | break; |
2501 | |
2502 | if (*CurPtr != '\n' && *CurPtr != '\r') |
2503 | return false; |
2504 | } |
2505 | |
2506 | if (TrigraphPos) { |
2507 | |
2508 | |
2509 | if (!L->getLangOpts().Trigraphs) { |
2510 | if (!L->isLexingRawMode()) |
2511 | L->Diag(TrigraphPos, diag::trigraph_ignored_block_comment); |
2512 | return false; |
2513 | } |
2514 | if (!L->isLexingRawMode()) |
2515 | L->Diag(TrigraphPos, diag::trigraph_ends_block_comment); |
2516 | } |
2517 | |
2518 | |
2519 | if (!L->isLexingRawMode()) |
2520 | L->Diag(CurPtr + 1, diag::escaped_newline_block_comment_end); |
2521 | |
2522 | |
2523 | if (SpacePos && !L->isLexingRawMode()) |
2524 | L->Diag(SpacePos, diag::backslash_newline_space); |
2525 | |
2526 | return true; |
2527 | } |
2528 | |
2529 | #ifdef __SSE2__ |
2530 | #include <emmintrin.h> |
2531 | #elif __ALTIVEC__ |
2532 | #include <altivec.h> |
2533 | #undef bool |
2534 | #endif |
2535 | |
2536 | |
2537 | |
2538 | |
2539 | |
2540 | |
2541 | |
2542 | |
2543 | |
2544 | |
2545 | bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr, |
2546 | bool &TokAtPhysicalStartOfLine) { |
2547 | |
2548 | |
2549 | |
2550 | |
2551 | |
2552 | |
2553 | |
2554 | |
2555 | unsigned CharSize; |
2556 | unsigned char C = getCharAndSize(CurPtr, CharSize); |
| 12 | | Calling 'Lexer::getCharAndSize' | |
|
| 15 | | Returning from 'Lexer::getCharAndSize' | |
|
2557 | CurPtr += CharSize; |
2558 | if (C == 0 && CurPtr == BufferEnd+1) { |
| 16 | | Assuming 'C' is not equal to 0 | |
|
2559 | if (!isLexingRawMode()) |
2560 | Diag(BufferPtr, diag::err_unterminated_block_comment); |
2561 | --CurPtr; |
2562 | |
2563 | |
2564 | |
2565 | if (isKeepWhitespaceMode()) { |
2566 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
2567 | return true; |
2568 | } |
2569 | |
2570 | BufferPtr = CurPtr; |
2571 | return false; |
2572 | } |
2573 | |
2574 | |
2575 | |
2576 | if (C == '/') |
| 17 | | Assuming the condition is false | |
|
| |
2577 | C = *CurPtr++; |
2578 | |
2579 | while (true) { |
| 19 | | Loop condition is true. Entering loop body | |
|
2580 | |
2581 | |
2582 | if (CurPtr + 24 < BufferEnd && |
| 20 | | Assuming the condition is true | |
|
| |
2583 | |
2584 | |
2585 | !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { |
| 21 | | Assuming field 'PP' is null | |
|
2586 | |
2587 | while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) |
| 23 | | Loop condition is false. Execution continues on line 2590 | |
|
2588 | C = *CurPtr++; |
2589 | |
2590 | if (C == '/') goto FoundSlash; |
| |
2591 | |
2592 | #ifdef __SSE2__ |
2593 | __m128i Slashes = _mm_set1_epi8('/'); |
2594 | while (CurPtr+16 <= BufferEnd) { |
| 25 | | Loop condition is true. Entering loop body | |
|
2595 | int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, |
2596 | Slashes)); |
2597 | if (cmp != 0) { |
| 26 | | Assuming 'cmp' is not equal to 0 | |
|
| |
2598 | |
2599 | |
2600 | |
2601 | CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1; |
2602 | goto FoundSlash; |
| 28 | | Control jumps to line 2635 | |
|
2603 | } |
2604 | CurPtr += 16; |
2605 | } |
2606 | #elif __ALTIVEC__ |
2607 | __vector unsigned char Slashes = { |
2608 | '/', '/', '/', '/', '/', '/', '/', '/', |
2609 | '/', '/', '/', '/', '/', '/', '/', '/' |
2610 | }; |
2611 | while (CurPtr + 16 <= BufferEnd && |
2612 | !vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes)) |
2613 | CurPtr += 16; |
2614 | #else |
2615 | |
2616 | while (CurPtr[0] != '/' && |
2617 | CurPtr[1] != '/' && |
2618 | CurPtr[2] != '/' && |
2619 | CurPtr[3] != '/' && |
2620 | CurPtr+4 < BufferEnd) { |
2621 | CurPtr += 4; |
2622 | } |
2623 | #endif |
2624 | |
2625 | |
2626 | C = *CurPtr++; |
2627 | } |
2628 | |
2629 | |
2630 | while (C != '/' && C != '\0') |
2631 | C = *CurPtr++; |
2632 | |
2633 | if (C == '/') { |
2634 | FoundSlash: |
2635 | if (CurPtr[-2] == '*') re done! |
| 29 | | Assuming the condition is false | |
|
| |
2636 | break; |
2637 | |
2638 | if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { |
| 31 | | Assuming the condition is false | |
|
| 32 | | Assuming the condition is false | |
|
| |
2639 | if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { |
2640 | |
2641 | |
2642 | break; |
2643 | } |
2644 | } |
2645 | if (CurPtr[0] == '*' && CurPtr[1] != '/') { |
| 34 | | Assuming the condition is true | |
|
| 35 | | Assuming the condition is true | |
|
| |
2646 | |
2647 | |
2648 | |
2649 | if (!isLexingRawMode()) |
| 37 | | Assuming the condition is true | |
|
| |
2650 | Diag(CurPtr-1, diag::warn_nested_block_comment); |
| |
2651 | } |
2652 | } else if (C == 0 && CurPtr == BufferEnd+1) { |
2653 | if (!isLexingRawMode()) |
2654 | Diag(BufferPtr, diag::err_unterminated_block_comment); |
2655 | |
2656 | |
2657 | |
2658 | --CurPtr; |
2659 | |
2660 | |
2661 | |
2662 | if (isKeepWhitespaceMode()) { |
2663 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
2664 | return true; |
2665 | } |
2666 | |
2667 | BufferPtr = CurPtr; |
2668 | return false; |
2669 | } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { |
2670 | PP->CodeCompleteNaturalLanguage(); |
2671 | cutOffLexing(); |
2672 | return false; |
2673 | } |
2674 | |
2675 | C = *CurPtr++; |
2676 | } |
2677 | |
2678 | |
2679 | if (PP && !isLexingRawMode() && |
2680 | PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), |
2681 | getSourceLocation(CurPtr)))) { |
2682 | BufferPtr = CurPtr; |
2683 | return true; |
2684 | } |
2685 | |
2686 | |
2687 | if (inKeepCommentMode()) { |
2688 | FormTokenWithChars(Result, CurPtr, tok::comment); |
2689 | return true; |
2690 | } |
2691 | |
2692 | |
2693 | |
2694 | |
2695 | |
2696 | if (isHorizontalWhitespace(*CurPtr)) { |
2697 | SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine); |
2698 | return false; |
2699 | } |
2700 | |
2701 | |
2702 | BufferPtr = CurPtr; |
2703 | Result.setFlag(Token::LeadingSpace); |
2704 | return false; |
2705 | } |
2706 | |
2707 | |
2708 | |
2709 | |
2710 | |
2711 | |
2712 | |
2713 | void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { |
2714 | assert(ParsingPreprocessorDirective && ParsingFilename == false && |
2715 | "Must be in a preprocessing directive!"); |
2716 | Token Tmp; |
2717 | Tmp.startToken(); |
2718 | |
2719 | |
2720 | const char *CurPtr = BufferPtr; |
2721 | while (true) { |
2722 | char Char = getAndAdvanceChar(CurPtr, Tmp); |
2723 | switch (Char) { |
2724 | default: |
2725 | if (Result) |
2726 | Result->push_back(Char); |
2727 | break; |
2728 | case 0: |
2729 | |
2730 | if (CurPtr-1 != BufferEnd) { |
2731 | if (isCodeCompletionPoint(CurPtr-1)) { |
2732 | PP->CodeCompleteNaturalLanguage(); |
2733 | cutOffLexing(); |
2734 | return; |
2735 | } |
2736 | |
2737 | |
2738 | if (Result) |
2739 | Result->push_back(Char); |
2740 | break; |
2741 | } |
2742 | |
2743 | LLVM_FALLTHROUGH; |
2744 | case '\r': |
2745 | case '\n': |
2746 | |
2747 | assert(CurPtr[-1] == Char && "Trigraphs for newline?"); |
2748 | BufferPtr = CurPtr-1; |
2749 | |
2750 | |
2751 | Lex(Tmp); |
2752 | if (Tmp.is(tok::code_completion)) { |
2753 | if (PP) |
2754 | PP->CodeCompleteNaturalLanguage(); |
2755 | Lex(Tmp); |
2756 | } |
2757 | assert(Tmp.is(tok::eod) && "Unexpected token!"); |
2758 | |
2759 | |
2760 | return; |
2761 | } |
2762 | } |
2763 | } |
2764 | |
2765 | |
2766 | |
2767 | |
2768 | |
2769 | bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { |
2770 | |
2771 | |
2772 | |
2773 | if (ParsingPreprocessorDirective) { |
2774 | |
2775 | ParsingPreprocessorDirective = false; |
2776 | |
2777 | FormTokenWithChars(Result, CurPtr, tok::eod); |
2778 | |
2779 | |
2780 | if (PP) |
2781 | resetExtendedTokenMode(); |
2782 | return true; |
2783 | } |
2784 | |
2785 | |
2786 | |
2787 | if (isLexingRawMode()) { |
2788 | Result.startToken(); |
2789 | BufferPtr = BufferEnd; |
2790 | FormTokenWithChars(Result, BufferEnd, tok::eof); |
2791 | return true; |
2792 | } |
2793 | |
2794 | if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) { |
2795 | PP->setRecordedPreambleConditionalStack(ConditionalStack); |
2796 | |
2797 | |
2798 | |
2799 | if (!ConditionalStack.empty()) |
2800 | MIOpt.ExitTopLevelConditional(); |
2801 | ConditionalStack.clear(); |
2802 | } |
2803 | |
2804 | |
2805 | |
2806 | |
2807 | while (!ConditionalStack.empty()) { |
2808 | if (PP->getCodeCompletionFileLoc() != FileLoc) |
2809 | PP->Diag(ConditionalStack.back().IfLoc, |
2810 | diag::err_pp_unterminated_conditional); |
2811 | ConditionalStack.pop_back(); |
2812 | } |
2813 | |
2814 | SourceLocation EndLoc = getSourceLocation(BufferEnd); |
2815 | |
2816 | |
2817 | if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) { |
2818 | DiagnosticsEngine &Diags = PP->getDiagnostics(); |
2819 | unsigned DiagID; |
2820 | |
2821 | if (LangOpts.CPlusPlus11) { |
2822 | |
2823 | |
2824 | |
2825 | if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) { |
2826 | DiagID = diag::warn_cxx98_compat_no_newline_eof; |
2827 | } else { |
2828 | DiagID = diag::warn_no_newline_eof; |
2829 | } |
2830 | } else { |
2831 | DiagID = diag::ext_no_newline_eof; |
2832 | } |
2833 | |
2834 | Diag(BufferEnd, DiagID) |
2835 | << FixItHint::CreateInsertion(EndLoc, "\n"); |
2836 | } |
2837 | |
2838 | BufferPtr = CurPtr; |
2839 | |
2840 | |
2841 | return PP->HandleEndOfFile(Result, EndLoc, isPragmaLexer()); |
2842 | } |
2843 | |
2844 | |
2845 | |
2846 | |
2847 | |
2848 | unsigned Lexer::isNextPPTokenLParen() { |
2849 | assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); |
2850 | |
2851 | |
2852 | |
2853 | |
2854 | LexingRawMode = true; |
2855 | |
2856 | |
2857 | const char *TmpBufferPtr = BufferPtr; |
2858 | bool inPPDirectiveMode = ParsingPreprocessorDirective; |
2859 | bool atStartOfLine = IsAtStartOfLine; |
2860 | bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; |
2861 | bool leadingSpace = HasLeadingSpace; |
2862 | |
2863 | Token Tok; |
2864 | Lex(Tok); |
2865 | |
2866 | |
2867 | BufferPtr = TmpBufferPtr; |
2868 | ParsingPreprocessorDirective = inPPDirectiveMode; |
2869 | HasLeadingSpace = leadingSpace; |
2870 | IsAtStartOfLine = atStartOfLine; |
2871 | IsAtPhysicalStartOfLine = atPhysicalStartOfLine; |
2872 | |
2873 | |
2874 | LexingRawMode = false; |
2875 | |
2876 | if (Tok.is(tok::eof)) |
2877 | return 2; |
2878 | return Tok.is(tok::l_paren); |
2879 | } |
2880 | |
2881 | |
2882 | static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, |
2883 | ConflictMarkerKind CMK) { |
2884 | const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; |
2885 | size_t TermLen = CMK == CMK_Perforce ? 5 : 7; |
2886 | auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(TermLen); |
2887 | size_t Pos = RestOfBuffer.find(Terminator); |
2888 | while (Pos != StringRef::npos) { |
2889 | |
2890 | if (Pos == 0 || |
2891 | (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) { |
2892 | RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); |
2893 | Pos = RestOfBuffer.find(Terminator); |
2894 | continue; |
2895 | } |
2896 | return RestOfBuffer.data()+Pos; |
2897 | } |
2898 | return nullptr; |
2899 | } |
2900 | |
2901 | |
2902 | |
2903 | |
2904 | |
2905 | bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { |
2906 | |
2907 | if (CurPtr != BufferStart && |
2908 | CurPtr[-1] != '\n' && CurPtr[-1] != '\r') |
2909 | return false; |
2910 | |
2911 | |
2912 | if (!StringRef(CurPtr, BufferEnd - CurPtr).startswith("<<<<<<<") && |
2913 | !StringRef(CurPtr, BufferEnd - CurPtr).startswith(">>>> ")) |
2914 | return false; |
2915 | |
2916 | |
2917 | |
2918 | if (CurrentConflictMarkerState || isLexingRawMode()) |
2919 | return false; |
2920 | |
2921 | ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; |
2922 | |
2923 | |
2924 | |
2925 | if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { |
2926 | |
2927 | |
2928 | Diag(CurPtr, diag::err_conflict_marker); |
2929 | CurrentConflictMarkerState = Kind; |
2930 | |
2931 | |
2932 | |
2933 | while (*CurPtr != '\r' && *CurPtr != '\n') { |
2934 | assert(CurPtr != BufferEnd && "Didn't find end of line"); |
2935 | ++CurPtr; |
2936 | } |
2937 | BufferPtr = CurPtr; |
2938 | return true; |
2939 | } |
2940 | |
2941 | |
2942 | return false; |
2943 | } |
2944 | |
2945 | |
2946 | |
2947 | |
2948 | |
2949 | bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { |
2950 | |
2951 | if (CurPtr != BufferStart && |
2952 | CurPtr[-1] != '\n' && CurPtr[-1] != '\r') |
2953 | return false; |
2954 | |
2955 | |
2956 | |
2957 | if (!CurrentConflictMarkerState || isLexingRawMode()) |
2958 | return false; |
2959 | |
2960 | |
2961 | for (unsigned i = 1; i != 4; ++i) |
2962 | if (CurPtr[i] != CurPtr[0]) |
2963 | return false; |
2964 | |
2965 | |
2966 | |
2967 | |
2968 | if (const char *End = FindConflictEnd(CurPtr, BufferEnd, |
2969 | CurrentConflictMarkerState)) { |
2970 | CurPtr = End; |
2971 | |
2972 | |
2973 | while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') |
2974 | ++CurPtr; |
2975 | |
2976 | BufferPtr = CurPtr; |
2977 | |
2978 | |
2979 | CurrentConflictMarkerState = CMK_None; |
2980 | return true; |
2981 | } |
2982 | |
2983 | return false; |
2984 | } |
2985 | |
2986 | static const char *findPlaceholderEnd(const char *CurPtr, |
2987 | const char *BufferEnd) { |
2988 | if (CurPtr == BufferEnd) |
2989 | return nullptr; |
2990 | BufferEnd -= 1; |
2991 | for (; CurPtr != BufferEnd; ++CurPtr) { |
2992 | if (CurPtr[0] == '#' && CurPtr[1] == '>') |
2993 | return CurPtr + 2; |
2994 | } |
2995 | return nullptr; |
2996 | } |
2997 | |
2998 | bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) { |
2999 | assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!"); |
3000 | if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode) |
3001 | return false; |
3002 | const char *End = findPlaceholderEnd(CurPtr + 1, BufferEnd); |
3003 | if (!End) |
3004 | return false; |
3005 | const char *Start = CurPtr - 1; |
3006 | if (!LangOpts.AllowEditorPlaceholders) |
3007 | Diag(Start, diag::err_placeholder_in_source); |
3008 | Result.startToken(); |
3009 | FormTokenWithChars(Result, End, tok::raw_identifier); |
3010 | Result.setRawIdentifierData(Start); |
3011 | PP->LookUpIdentifierInfo(Result); |
3012 | Result.setFlag(Token::IsEditorPlaceholder); |
3013 | BufferPtr = End; |
3014 | return true; |
3015 | } |
3016 | |
3017 | bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { |
3018 | if (PP && PP->isCodeCompletionEnabled()) { |
3019 | SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); |
3020 | return Loc == PP->getCodeCompletionLoc(); |
3021 | } |
3022 | |
3023 | return false; |
3024 | } |
3025 | |
3026 | uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, |
3027 | Token *Result) { |
3028 | unsigned CharSize; |
3029 | char Kind = getCharAndSize(StartPtr, CharSize); |
3030 | |
3031 | unsigned NumHexDigits; |
3032 | if (Kind == 'u') |
3033 | NumHexDigits = 4; |
3034 | else if (Kind == 'U') |
3035 | NumHexDigits = 8; |
3036 | else |
3037 | return 0; |
3038 | |
3039 | if (!LangOpts.CPlusPlus && !LangOpts.C99) { |
3040 | if (Result && !isLexingRawMode()) |
3041 | Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); |
3042 | return 0; |
3043 | } |
3044 | |
3045 | const char *CurPtr = StartPtr + CharSize; |
3046 | const char *KindLoc = &CurPtr[-1]; |
3047 | |
3048 | uint32_t CodePoint = 0; |
3049 | for (unsigned i = 0; i < NumHexDigits; ++i) { |
3050 | char C = getCharAndSize(CurPtr, CharSize); |
3051 | |
3052 | unsigned Value = llvm::hexDigitValue(C); |
3053 | if (Value == -1U) { |
3054 | if (Result && !isLexingRawMode()) { |
3055 | if (i == 0) { |
3056 | Diag(BufferPtr, diag::warn_ucn_escape_no_digits) |
3057 | << StringRef(KindLoc, 1); |
3058 | } else { |
3059 | Diag(BufferPtr, diag::warn_ucn_escape_incomplete); |
3060 | |
3061 | |
3062 | if (i == 4 && NumHexDigits == 8) { |
3063 | CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); |
3064 | Diag(KindLoc, diag::note_ucn_four_not_eight) |
3065 | << FixItHint::CreateReplacement(URange, "u"); |
3066 | } |
3067 | } |
3068 | } |
3069 | |
3070 | return 0; |
3071 | } |
3072 | |
3073 | CodePoint <<= 4; |
3074 | CodePoint += Value; |
3075 | |
3076 | CurPtr += CharSize; |
3077 | } |
3078 | |
3079 | if (Result) { |
3080 | Result->setFlag(Token::HasUCN); |
3081 | if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) |
3082 | StartPtr = CurPtr; |
3083 | else |
3084 | while (StartPtr != CurPtr) |
3085 | (void)getAndAdvanceChar(StartPtr, *Result); |
3086 | } else { |
3087 | StartPtr = CurPtr; |
3088 | } |
3089 | |
3090 | |
3091 | if (LangOpts.AsmPreprocessor) |
3092 | return CodePoint; |
3093 | |
3094 | |
3095 | |
3096 | |
3097 | |
3098 | |
3099 | |
3100 | |
3101 | |
3102 | |
3103 | |
3104 | |
3105 | if (CodePoint < 0xA0) { |
3106 | if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) |
3107 | return CodePoint; |
3108 | |
3109 | |
3110 | |
3111 | if (Result && PP) { |
3112 | if (CodePoint < 0x20 || CodePoint >= 0x7F) |
3113 | Diag(BufferPtr, diag::err_ucn_control_character); |
3114 | else { |
3115 | char C = static_cast<char>(CodePoint); |
3116 | Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); |
3117 | } |
3118 | } |
3119 | |
3120 | return 0; |
3121 | } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { |
3122 | |
3123 | |
3124 | |
3125 | if (Result && PP) { |
3126 | if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) |
3127 | Diag(BufferPtr, diag::warn_ucn_escape_surrogate); |
3128 | else |
3129 | Diag(BufferPtr, diag::err_ucn_escape_invalid); |
3130 | } |
3131 | return 0; |
3132 | } |
3133 | |
3134 | return CodePoint; |
3135 | } |
3136 | |
3137 | bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C, |
3138 | const char *CurPtr) { |
3139 | static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars( |
3140 | UnicodeWhitespaceCharRanges); |
3141 | if (!isLexingRawMode() && !PP->isPreprocessedOutput() && |
3142 | UnicodeWhitespaceChars.contains(C)) { |
3143 | Diag(BufferPtr, diag::ext_unicode_whitespace) |
3144 | << makeCharRange(*this, BufferPtr, CurPtr); |
3145 | |
3146 | Result.setFlag(Token::LeadingSpace); |
3147 | return true; |
3148 | } |
3149 | return false; |
3150 | } |
3151 | |
3152 | bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { |
3153 | if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { |
3154 | if (!isLexingRawMode() && !ParsingPreprocessorDirective && |
3155 | !PP->isPreprocessedOutput()) { |
3156 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, |
3157 | makeCharRange(*this, BufferPtr, CurPtr), |
3158 | true); |
3159 | maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), C, |
3160 | makeCharRange(*this, BufferPtr, CurPtr)); |
3161 | } |
3162 | |
3163 | MIOpt.ReadToken(); |
3164 | return LexIdentifier(Result, CurPtr); |
3165 | } |
3166 | |
3167 | if (!isLexingRawMode() && !ParsingPreprocessorDirective && |
3168 | !PP->isPreprocessedOutput() && |
3169 | !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { |
3170 | |
3171 | |
3172 | |
3173 | |
3174 | |
3175 | |
3176 | |
3177 | |
3178 | |
3179 | Diag(BufferPtr, diag::err_non_ascii) |
3180 | << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); |
3181 | |
3182 | BufferPtr = CurPtr; |
3183 | return false; |
3184 | } |
3185 | |
3186 | |
3187 | |
3188 | MIOpt.ReadToken(); |
3189 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
3190 | return true; |
3191 | } |
3192 | |
3193 | void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { |
3194 | IsAtStartOfLine = Result.isAtStartOfLine(); |
3195 | HasLeadingSpace = Result.hasLeadingSpace(); |
3196 | HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro(); |
3197 | |
3198 | } |
3199 | |
3200 | bool Lexer::Lex(Token &Result) { |
3201 | |
3202 | Result.startToken(); |
3203 | |
3204 | |
3205 | if (IsAtStartOfLine) { |
3206 | Result.setFlag(Token::StartOfLine); |
3207 | IsAtStartOfLine = false; |
3208 | } |
3209 | |
3210 | if (HasLeadingSpace) { |
3211 | Result.setFlag(Token::LeadingSpace); |
3212 | HasLeadingSpace = false; |
3213 | } |
3214 | |
3215 | if (HasLeadingEmptyMacro) { |
3216 | Result.setFlag(Token::LeadingEmptyMacro); |
3217 | HasLeadingEmptyMacro = false; |
3218 | } |
3219 | |
3220 | bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; |
3221 | IsAtPhysicalStartOfLine = false; |
3222 | bool isRawLex = isLexingRawMode(); |
3223 | (void) isRawLex; |
3224 | bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine); |
3225 | |
3226 | assert((returnedToken || !isRawLex) && "Raw lex must succeed"); |
3227 | return returnedToken; |
3228 | } |
3229 | |
3230 | |
3231 | |
3232 | |
3233 | |
3234 | |
3235 | bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) { |
3236 | LexNextToken: |
3237 | |
3238 | Result.clearFlag(Token::NeedsCleaning); |
3239 | Result.setIdentifierInfo(nullptr); |
3240 | |
3241 | |
3242 | const char *CurPtr = BufferPtr; |
3243 | |
3244 | |
3245 | if (isHorizontalWhitespace(*CurPtr)) { |
| 1 | Assuming the condition is false | |
|
| |
3246 | do { |
3247 | ++CurPtr; |
3248 | } while (isHorizontalWhitespace(*CurPtr)); |
3249 | |
3250 | |
3251 | |
3252 | |
3253 | if (isKeepWhitespaceMode()) { |
3254 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
3255 | |
3256 | return true; |
3257 | } |
3258 | |
3259 | BufferPtr = CurPtr; |
3260 | Result.setFlag(Token::LeadingSpace); |
3261 | } |
3262 | |
3263 | unsigned SizeTmp, SizeTmp2; |
3264 | |
3265 | |
3266 | char Char = getAndAdvanceChar(CurPtr, Result); |
3267 | tok::TokenKind Kind; |
3268 | |
3269 | if (!isVerticalWhitespace(Char)) |
| 3 | | Assuming the condition is false | |
|
| |
3270 | NewLinePtr = nullptr; |
3271 | |
3272 | switch (Char) { |
| 5 | | Control jumps to 'case 32:' at line 3341 | |
|
3273 | case 0: |
3274 | |
3275 | if (CurPtr-1 == BufferEnd) |
3276 | return LexEndOfFile(Result, CurPtr-1); |
3277 | |
3278 | |
3279 | if (isCodeCompletionPoint(CurPtr-1)) { |
3280 | |
3281 | Result.startToken(); |
3282 | FormTokenWithChars(Result, CurPtr, tok::code_completion); |
3283 | return true; |
3284 | } |
3285 | |
3286 | if (!isLexingRawMode()) |
3287 | Diag(CurPtr-1, diag::null_in_file); |
3288 | Result.setFlag(Token::LeadingSpace); |
3289 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
3290 | return true; |
3291 | |
3292 | |
3293 | |
3294 | goto LexNextToken; |
3295 | |
3296 | case 26: |
3297 | |
3298 | if (LangOpts.MicrosoftExt) { |
3299 | if (!isLexingRawMode()) |
3300 | Diag(CurPtr-1, diag::ext_ctrl_z_eof_microsoft); |
3301 | return LexEndOfFile(Result, CurPtr-1); |
3302 | } |
3303 | |
3304 | |
3305 | Kind = tok::unknown; |
3306 | break; |
3307 | |
3308 | case '\r': |
3309 | if (CurPtr[0] == '\n') |
3310 | (void)getAndAdvanceChar(CurPtr, Result); |
3311 | LLVM_FALLTHROUGH; |
3312 | case '\n': |
3313 | |
3314 | |
3315 | if (ParsingPreprocessorDirective) { |
3316 | |
3317 | ParsingPreprocessorDirective = false; |
3318 | |
3319 | |
3320 | if (PP) |
3321 | resetExtendedTokenMode(); |
3322 | |
3323 | |
3324 | IsAtStartOfLine = true; |
3325 | IsAtPhysicalStartOfLine = true; |
3326 | NewLinePtr = CurPtr - 1; |
3327 | |
3328 | Kind = tok::eod; |
3329 | break; |
3330 | } |
3331 | |
3332 | |
3333 | Result.clearFlag(Token::LeadingSpace); |
3334 | |
3335 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
3336 | return true; |
3337 | |
3338 | |
3339 | |
3340 | goto LexNextToken; |
3341 | case ' ': |
3342 | case '\t': |
3343 | case '\f': |
3344 | case '\v': |
3345 | SkipHorizontalWhitespace: |
3346 | Result.setFlag(Token::LeadingSpace); |
3347 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| |
3348 | return true; |
3349 | |
3350 | SkipIgnoredUnits: |
3351 | CurPtr = BufferPtr; |
3352 | |
3353 | |
3354 | |
3355 | if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && |
| 7 | | Assuming the condition is false | |
|
3356 | LangOpts.LineComment && |
3357 | (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) { |
3358 | if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) |
3359 | return true; |
3360 | goto SkipIgnoredUnits; |
3361 | } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { |
| 8 | | Assuming the condition is true | |
|
| 9 | | Assuming the condition is true | |
|
| |
3362 | if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) |
| 11 | | Calling 'Lexer::SkipBlockComment' | |
|
3363 | return true; |
3364 | goto SkipIgnoredUnits; |
3365 | } else if (isHorizontalWhitespace(*CurPtr)) { |
3366 | goto SkipHorizontalWhitespace; |
3367 | } |
3368 | |
3369 | |
3370 | goto LexNextToken; |
3371 | |
3372 | |
3373 | |
3374 | case '0': case '1': case '2': case '3': case '4': |
3375 | case '5': case '6': case '7': case '8': case '9': |
3376 | |
3377 | MIOpt.ReadToken(); |
3378 | return LexNumericConstant(Result, CurPtr); |
3379 | |
3380 | case 'u': |
3381 | |
3382 | MIOpt.ReadToken(); |
3383 | |
3384 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
3385 | Char = getCharAndSize(CurPtr, SizeTmp); |
3386 | |
3387 | |
3388 | if (Char == '"') |
3389 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3390 | tok::utf16_string_literal); |
3391 | |
3392 | |
3393 | if (Char == '\'') |
3394 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3395 | tok::utf16_char_constant); |
3396 | |
3397 | |
3398 | if (Char == 'R' && LangOpts.CPlusPlus11 && |
3399 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
3400 | return LexRawStringLiteral(Result, |
3401 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3402 | SizeTmp2, Result), |
3403 | tok::utf16_string_literal); |
3404 | |
3405 | if (Char == '8') { |
3406 | char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); |
3407 | |
3408 | |
3409 | if (Char2 == '"') |
3410 | return LexStringLiteral(Result, |
3411 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3412 | SizeTmp2, Result), |
3413 | tok::utf8_string_literal); |
3414 | if (Char2 == '\'' && LangOpts.CPlusPlus17) |
3415 | return LexCharConstant( |
3416 | Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3417 | SizeTmp2, Result), |
3418 | tok::utf8_char_constant); |
3419 | |
3420 | if (Char2 == 'R' && LangOpts.CPlusPlus11) { |
3421 | unsigned SizeTmp3; |
3422 | char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); |
3423 | |
3424 | if (Char3 == '"') { |
3425 | return LexRawStringLiteral(Result, |
3426 | ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3427 | SizeTmp2, Result), |
3428 | SizeTmp3, Result), |
3429 | tok::utf8_string_literal); |
3430 | } |
3431 | } |
3432 | } |
3433 | } |
3434 | |
3435 | |
3436 | return LexIdentifier(Result, CurPtr); |
3437 | |
3438 | case 'U': |
3439 | |
3440 | MIOpt.ReadToken(); |
3441 | |
3442 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
3443 | Char = getCharAndSize(CurPtr, SizeTmp); |
3444 | |
3445 | |
3446 | if (Char == '"') |
3447 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3448 | tok::utf32_string_literal); |
3449 | |
3450 | |
3451 | if (Char == '\'') |
3452 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3453 | tok::utf32_char_constant); |
3454 | |
3455 | |
3456 | if (Char == 'R' && LangOpts.CPlusPlus11 && |
3457 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
3458 | return LexRawStringLiteral(Result, |
3459 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3460 | SizeTmp2, Result), |
3461 | tok::utf32_string_literal); |
3462 | } |
3463 | |
3464 | |
3465 | return LexIdentifier(Result, CurPtr); |
3466 | |
3467 | case 'R': |
3468 | |
3469 | MIOpt.ReadToken(); |
3470 | |
3471 | if (LangOpts.CPlusPlus11) { |
3472 | Char = getCharAndSize(CurPtr, SizeTmp); |
3473 | |
3474 | if (Char == '"') |
3475 | return LexRawStringLiteral(Result, |
3476 | ConsumeChar(CurPtr, SizeTmp, Result), |
3477 | tok::string_literal); |
3478 | } |
3479 | |
3480 | |
3481 | return LexIdentifier(Result, CurPtr); |
3482 | |
3483 | case 'L': |
3484 | |
3485 | MIOpt.ReadToken(); |
3486 | Char = getCharAndSize(CurPtr, SizeTmp); |
3487 | |
3488 | |
3489 | if (Char == '"') |
3490 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3491 | tok::wide_string_literal); |
3492 | |
3493 | |
3494 | if (LangOpts.CPlusPlus11 && Char == 'R' && |
3495 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
3496 | return LexRawStringLiteral(Result, |
3497 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3498 | SizeTmp2, Result), |
3499 | tok::wide_string_literal); |
3500 | |
3501 | |
3502 | if (Char == '\'') |
3503 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3504 | tok::wide_char_constant); |
3505 | |
3506 | LLVM_FALLTHROUGH; |
3507 | |
3508 | |
3509 | case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': |
3510 | case 'H': case 'I': case 'J': case 'K': case 'M': case 'N': |
3511 | case 'O': case 'P': case 'Q': case 'S': case 'T': |
3512 | case 'V': case 'W': case 'X': case 'Y': case 'Z': |
3513 | case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': |
3514 | case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': |
3515 | case 'o': case 'p': case 'q': case 'r': case 's': case 't': |
3516 | case 'v': case 'w': case 'x': case 'y': case 'z': |
3517 | case '_': |
3518 | |
3519 | MIOpt.ReadToken(); |
3520 | return LexIdentifier(Result, CurPtr); |
3521 | |
3522 | case '$': |
3523 | if (LangOpts.DollarIdents) { |
3524 | if (!isLexingRawMode()) |
3525 | Diag(CurPtr-1, diag::ext_dollar_in_identifier); |
3526 | |
3527 | MIOpt.ReadToken(); |
3528 | return LexIdentifier(Result, CurPtr); |
3529 | } |
3530 | |
3531 | Kind = tok::unknown; |
3532 | break; |
3533 | |
3534 | |
3535 | case '\'': |
3536 | |
3537 | MIOpt.ReadToken(); |
3538 | return LexCharConstant(Result, CurPtr, tok::char_constant); |
3539 | |
3540 | |
3541 | case '"': |
3542 | |
3543 | MIOpt.ReadToken(); |
3544 | return LexStringLiteral(Result, CurPtr, |
3545 | ParsingFilename ? tok::header_name |
3546 | : tok::string_literal); |
3547 | |
3548 | |
3549 | case '?': |
3550 | Kind = tok::question; |
3551 | break; |
3552 | case '[': |
3553 | Kind = tok::l_square; |
3554 | break; |
3555 | case ']': |
3556 | Kind = tok::r_square; |
3557 | break; |
3558 | case '(': |
3559 | Kind = tok::l_paren; |
3560 | break; |
3561 | case ')': |
3562 | Kind = tok::r_paren; |
3563 | break; |
3564 | case '{': |
3565 | Kind = tok::l_brace; |
3566 | break; |
3567 | case '}': |
3568 | Kind = tok::r_brace; |
3569 | break; |
3570 | case '.': |
3571 | Char = getCharAndSize(CurPtr, SizeTmp); |
3572 | if (Char >= '0' && Char <= '9') { |
3573 | |
3574 | MIOpt.ReadToken(); |
3575 | |
3576 | return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); |
3577 | } else if (LangOpts.CPlusPlus && Char == '*') { |
3578 | Kind = tok::periodstar; |
3579 | CurPtr += SizeTmp; |
3580 | } else if (Char == '.' && |
3581 | getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { |
3582 | Kind = tok::ellipsis; |
3583 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3584 | SizeTmp2, Result); |
3585 | } else { |
3586 | Kind = tok::period; |
3587 | } |
3588 | break; |
3589 | case '&': |
3590 | Char = getCharAndSize(CurPtr, SizeTmp); |
3591 | if (Char == '&') { |
3592 | Kind = tok::ampamp; |
3593 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3594 | } else if (Char == '=') { |
3595 | Kind = tok::ampequal; |
3596 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3597 | } else { |
3598 | Kind = tok::amp; |
3599 | } |
3600 | break; |
3601 | case '*': |
3602 | if (getCharAndSize(CurPtr, SizeTmp) == '=') { |
3603 | Kind = tok::starequal; |
3604 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3605 | } else { |
3606 | Kind = tok::star; |
3607 | } |
3608 | break; |
3609 | case '+': |
3610 | Char = getCharAndSize(CurPtr, SizeTmp); |
3611 | if (Char == '+') { |
3612 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3613 | Kind = tok::plusplus; |
3614 | } else if (Char == '=') { |
3615 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3616 | Kind = tok::plusequal; |
3617 | } else { |
3618 | Kind = tok::plus; |
3619 | } |
3620 | break; |
3621 | case '-': |
3622 | Char = getCharAndSize(CurPtr, SizeTmp); |
3623 | if (Char == '-') { |
3624 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3625 | Kind = tok::minusminus; |
3626 | } else if (Char == '>' && LangOpts.CPlusPlus && |
3627 | getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { |
3628 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3629 | SizeTmp2, Result); |
3630 | Kind = tok::arrowstar; |
3631 | } else if (Char == '>') { |
3632 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3633 | Kind = tok::arrow; |
3634 | } else if (Char == '=') { |
3635 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3636 | Kind = tok::minusequal; |
3637 | } else { |
3638 | Kind = tok::minus; |
3639 | } |
3640 | break; |
3641 | case '~': |
3642 | Kind = tok::tilde; |
3643 | break; |
3644 | case '!': |
3645 | if (getCharAndSize(CurPtr, SizeTmp) == '=') { |
3646 | Kind = tok::exclaimequal; |
3647 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3648 | } else { |
3649 | Kind = tok::exclaim; |
3650 | } |
3651 | break; |
3652 | case '/': |
3653 | |
3654 | Char = getCharAndSize(CurPtr, SizeTmp); |
3655 | if (Char == '/') { |
3656 | |
3657 | |
3658 | |
3659 | |
3660 | |
3661 | |
3662 | |
3663 | |
3664 | bool TreatAsComment = LangOpts.LineComment && |
3665 | (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP); |
3666 | if (!TreatAsComment) |
3667 | if (!(PP && PP->isPreprocessedOutput())) |
3668 | TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; |
3669 | |
3670 | if (TreatAsComment) { |
3671 | if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3672 | TokAtPhysicalStartOfLine)) |
3673 | return true; |
3674 | |
3675 | |
3676 | |
3677 | |
3678 | goto SkipIgnoredUnits; |
3679 | } |
3680 | } |
3681 | |
3682 | if (Char == '*') { |
3683 | if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
3684 | TokAtPhysicalStartOfLine)) |
3685 | return true; |
3686 | |
3687 | |
3688 | |
3689 | goto LexNextToken; |
3690 | } |
3691 | |
3692 | if (Char == '=') { |
3693 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3694 | Kind = tok::slashequal; |
3695 | } else { |
3696 | Kind = tok::slash; |
3697 | } |
3698 | break; |
3699 | case '%': |
3700 | Char = getCharAndSize(CurPtr, SizeTmp); |
3701 | if (Char == '=') { |
3702 | Kind = tok::percentequal; |
3703 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3704 | } else if (LangOpts.Digraphs && Char == '>') { |
3705 | Kind = tok::r_brace; |
3706 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3707 | } else if (LangOpts.Digraphs && Char == ':') { |
3708 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3709 | Char = getCharAndSize(CurPtr, SizeTmp); |
3710 | if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { |
3711 | Kind = tok::hashhash; |
3712 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3713 | SizeTmp2, Result); |
3714 | } else if (Char == '@' && LangOpts.MicrosoftExt) { |
3715 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3716 | if (!isLexingRawMode()) |
3717 | Diag(BufferPtr, diag::ext_charize_microsoft); |
3718 | Kind = tok::hashat; |
3719 | } else { |
3720 | |
3721 | |
3722 | |
3723 | |
3724 | if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) |
3725 | goto HandleDirective; |
3726 | |
3727 | Kind = tok::hash; |
3728 | } |
3729 | } else { |
3730 | Kind = tok::percent; |
3731 | } |
3732 | break; |
3733 | case '<': |
3734 | Char = getCharAndSize(CurPtr, SizeTmp); |
3735 | if (ParsingFilename) { |
3736 | return LexAngledStringLiteral(Result, CurPtr); |
3737 | } else if (Char == '<') { |
3738 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
3739 | if (After == '=') { |
3740 | Kind = tok::lesslessequal; |
3741 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3742 | SizeTmp2, Result); |
3743 | } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { |
3744 | |
3745 | |
3746 | goto LexNextToken; |
3747 | } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { |
3748 | |
3749 | |
3750 | goto LexNextToken; |
3751 | } else if (LangOpts.CUDA && After == '<') { |
3752 | Kind = tok::lesslessless; |
3753 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3754 | SizeTmp2, Result); |
3755 | } else { |
3756 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3757 | Kind = tok::lessless; |
3758 | } |
3759 | } else if (Char == '=') { |
3760 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
3761 | if (After == '>') { |
3762 | if (getLangOpts().CPlusPlus20) { |
3763 | if (!isLexingRawMode()) |
3764 | Diag(BufferPtr, diag::warn_cxx17_compat_spaceship); |
3765 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3766 | SizeTmp2, Result); |
3767 | Kind = tok::spaceship; |
3768 | break; |
3769 | } |
3770 | |
3771 | |
3772 | if (getLangOpts().CPlusPlus && !isLexingRawMode()) { |
3773 | Diag(BufferPtr, diag::warn_cxx20_compat_spaceship) |
3774 | << FixItHint::CreateInsertion( |
3775 | getSourceLocation(CurPtr + SizeTmp, SizeTmp2), " "); |
3776 | } |
3777 | } |
3778 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3779 | Kind = tok::lessequal; |
3780 | } else if (LangOpts.Digraphs && Char == ':') { |
3781 | if (LangOpts.CPlusPlus11 && |
3782 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { |
3783 | |
3784 | |
3785 | |
3786 | |
3787 | |
3788 | unsigned SizeTmp3; |
3789 | char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); |
3790 | if (After != ':' && After != '>') { |
3791 | Kind = tok::less; |
3792 | if (!isLexingRawMode()) |
3793 | Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); |
3794 | break; |
3795 | } |
3796 | } |
3797 | |
3798 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3799 | Kind = tok::l_square; |
3800 | } else if (LangOpts.Digraphs && Char == '%') { |
3801 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3802 | Kind = tok::l_brace; |
3803 | } else if (Char == '#' && SizeTmp == 1 && |
3804 | lexEditorPlaceholder(Result, CurPtr)) { |
3805 | return true; |
3806 | } else { |
3807 | Kind = tok::less; |
3808 | } |
3809 | break; |
3810 | case '>': |
3811 | Char = getCharAndSize(CurPtr, SizeTmp); |
3812 | if (Char == '=') { |
3813 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3814 | Kind = tok::greaterequal; |
3815 | } else if (Char == '>') { |
3816 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
3817 | if (After == '=') { |
3818 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3819 | SizeTmp2, Result); |
3820 | Kind = tok::greatergreaterequal; |
3821 | } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { |
3822 | |
3823 | |
3824 | goto LexNextToken; |
3825 | } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { |
3826 | |
3827 | goto LexNextToken; |
3828 | } else if (LangOpts.CUDA && After == '>') { |
3829 | Kind = tok::greatergreatergreater; |
3830 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
3831 | SizeTmp2, Result); |
3832 | } else { |
3833 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3834 | Kind = tok::greatergreater; |
3835 | } |
3836 | } else { |
3837 | Kind = tok::greater; |
3838 | } |
3839 | break; |
3840 | case '^': |
3841 | Char = getCharAndSize(CurPtr, SizeTmp); |
3842 | if (Char == '=') { |
3843 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3844 | Kind = tok::caretequal; |
3845 | } else if (LangOpts.OpenCL && Char == '^') { |
3846 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3847 | Kind = tok::caretcaret; |
3848 | } else { |
3849 | Kind = tok::caret; |
3850 | } |
3851 | break; |
3852 | case '|': |
3853 | Char = getCharAndSize(CurPtr, SizeTmp); |
3854 | if (Char == '=') { |
3855 | Kind = tok::pipeequal; |
3856 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3857 | } else if (Char == '|') { |
3858 | |
3859 | if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) |
3860 | goto LexNextToken; |
3861 | Kind = tok::pipepipe; |
3862 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3863 | } else { |
3864 | Kind = tok::pipe; |
3865 | } |
3866 | break; |
3867 | case ':': |
3868 | Char = getCharAndSize(CurPtr, SizeTmp); |
3869 | if (LangOpts.Digraphs && Char == '>') { |
3870 | Kind = tok::r_square; |
3871 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3872 | } else if ((LangOpts.CPlusPlus || |
3873 | LangOpts.DoubleSquareBracketAttributes) && |
3874 | Char == ':') { |
3875 | Kind = tok::coloncolon; |
3876 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3877 | } else { |
3878 | Kind = tok::colon; |
3879 | } |
3880 | break; |
3881 | case ';': |
3882 | Kind = tok::semi; |
3883 | break; |
3884 | case '=': |
3885 | Char = getCharAndSize(CurPtr, SizeTmp); |
3886 | if (Char == '=') { |
3887 | |
3888 | if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) |
3889 | goto LexNextToken; |
3890 | |
3891 | Kind = tok::equalequal; |
3892 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3893 | } else { |
3894 | Kind = tok::equal; |
3895 | } |
3896 | break; |
3897 | case ',': |
3898 | Kind = tok::comma; |
3899 | break; |
3900 | case '#': |
3901 | Char = getCharAndSize(CurPtr, SizeTmp); |
3902 | if (Char == '#') { |
3903 | Kind = tok::hashhash; |
3904 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3905 | } else if (Char == '@' && LangOpts.MicrosoftExt) { |
3906 | Kind = tok::hashat; |
3907 | if (!isLexingRawMode()) |
3908 | Diag(BufferPtr, diag::ext_charize_microsoft); |
3909 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
3910 | } else { |
3911 | |
3912 | |
3913 | |
3914 | |
3915 | if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) |
3916 | goto HandleDirective; |
3917 | |
3918 | Kind = tok::hash; |
3919 | } |
3920 | break; |
3921 | |
3922 | case '@': |
3923 | |
3924 | if (CurPtr[-1] == '@' && LangOpts.ObjC) |
3925 | Kind = tok::at; |
3926 | else |
3927 | Kind = tok::unknown; |
3928 | break; |
3929 | |
3930 | |
3931 | case '\\': |
3932 | if (!LangOpts.AsmPreprocessor) { |
3933 | if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { |
3934 | if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { |
3935 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
3936 | return true; |
3937 | |
3938 | |
3939 | |
3940 | goto LexNextToken; |
3941 | } |
3942 | |
3943 | return LexUnicode(Result, CodePoint, CurPtr); |
3944 | } |
3945 | } |
3946 | |
3947 | Kind = tok::unknown; |
3948 | break; |
3949 | |
3950 | default: { |
3951 | if (isASCII(Char)) { |
3952 | Kind = tok::unknown; |
3953 | break; |
3954 | } |
3955 | |
3956 | llvm::UTF32 CodePoint; |
3957 | |
3958 | |
3959 | |
3960 | --CurPtr; |
3961 | llvm::ConversionResult Status = |
3962 | llvm::convertUTF8Sequence((const llvm::UTF8 **)&CurPtr, |
3963 | (const llvm::UTF8 *)BufferEnd, |
3964 | &CodePoint, |
3965 | llvm::strictConversion); |
3966 | if (Status == llvm::conversionOK) { |
3967 | if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { |
3968 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
3969 | return true; |
3970 | |
3971 | |
3972 | |
3973 | goto LexNextToken; |
3974 | } |
3975 | return LexUnicode(Result, CodePoint, CurPtr); |
3976 | } |
3977 | |
3978 | if (isLexingRawMode() || ParsingPreprocessorDirective || |
3979 | PP->isPreprocessedOutput()) { |
3980 | ++CurPtr; |
3981 | Kind = tok::unknown; |
3982 | break; |
3983 | } |
3984 | |
3985 | |
3986 | |
3987 | |
3988 | Diag(CurPtr, diag::err_invalid_utf8); |
3989 | |
3990 | BufferPtr = CurPtr+1; |
3991 | |
3992 | |
3993 | |
3994 | goto LexNextToken; |
3995 | } |
3996 | } |
3997 | |
3998 | |
3999 | MIOpt.ReadToken(); |
4000 | |
4001 | |
4002 | FormTokenWithChars(Result, CurPtr, Kind); |
4003 | return true; |
4004 | |
4005 | HandleDirective: |
4006 | |
4007 | |
4008 | FormTokenWithChars(Result, CurPtr, tok::hash); |
4009 | PP->HandleDirective(Result); |
4010 | |
4011 | if (PP->hadModuleLoaderFatalFailure()) { |
4012 | |
4013 | assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof"); |
4014 | return true; |
4015 | } |
4016 | |
4017 | |
4018 | return false; |
4019 | } |