mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-17 00:38:37 +00:00
Improve lexer and parser diagnostic locations (#7990)
Lexer diagnostics used a single `const char*` as their source location, which only allowed pointing at a single character rather than a run of characters. This limited the underlining of unrecognized tokens, and also made escape sequence diagnostics particularly strange by stopping at whatever point the lexer discovered the error and pointing only at that location. For example: `"\w"` was reported at the `w`, and `"\x"` at the closing `"`. This change tries to always keep the primary location of a diagnostic the point at which typing (inserting or deleting) might fix the error. So for `"\x"`, `\x` is underlined, but the position is at the `"` as that's where you would type a hex code. The lexer's emitter now takes a special lexing source location that represents either a single position _between_ two characters, a single character, or a range of the source buffer. The lexer diagnostics are updated to construct these for the character range as appropriate. Parser diagnostics for a missing token were located at the _next_ token, which is particularly confusing for terminators like missing `;`s. Now they are located at a position after the previous token, with or without a space depending on the token, but typically on the same line. Diagnostics still sort by how far the parser has read. The parser diagnostics for unclosed groups also now attaches a label to the opening token when it is on a separate line. And the diagnostics for mismatched indent in a multi-line string attaches a label to the closing `'''` that establishes that indent. There are likely more location improvements we should do here, but this seemed like a reasonable incremental improvement and is what Claude found when doing an initial audit. One example where I think we should make further improvements is around suggesting raw identifiers when we find an unexpected `Cpp` or `Core` or `base`. We do this consistently now, but raw identifiers should be rare. A TODO is added for these cases. Assisted-by: Claude Code --------- Co-authored-by: Richard Smith <richard@metafoo.co.uk>
This commit is contained in:
1 parent
611e6a210e
commit
68569d234d
173 files changed
+1782
-689
No files matched your search
@@ -1188,9 +1188,14 @@ TEST_F(LexerTest, TypeLiterals) {
|
||||
TEST_F(LexerTest, TypeLiteralTooManyDigits) {
|
||||
// We increase the number of digits until the first one that is to large.
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
Diagnostics::Kind::TooManyTypeBitWidthDigits,
|
||||
Diagnostics::Level::Error, 1, 2, _)));
|
||||
// The bit width's digits are marked as the range that is too long.
|
||||
EXPECT_CALL(
|
||||
consumer,
|
||||
HandleDiagnostic(Testing::IsDiagnosticWithRange(
|
||||
Diagnostics::Kind::TooManyTypeBitWidthDigits,
|
||||
Diagnostics::Level::Error,
|
||||
{.line_number = 1, .column_number = 2, .length = 1}, _,
|
||||
{.line_number = 1, .column_number = 2, .length = testing::Gt(1)})));
|
||||
std::string code = "i";
|
||||
// A 128-bit APInt should be plenty large, but if needed in the future it can
|
||||
// be widened without issue.
|
||||
@@ -1221,10 +1226,13 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) {
|
||||
// Make sure we can also gracefully reject very large number of digits without
|
||||
// crashing or hanging, and show the correct number.
|
||||
constexpr int Count = 10000;
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
Diagnostics::Kind::TooManyTypeBitWidthDigits,
|
||||
Diagnostics::Level::Error, 1, 2,
|
||||
HasSubstr(llvm::formatv(" {0} ", Count)))));
|
||||
EXPECT_CALL(consumer,
|
||||
HandleDiagnostic(Testing::IsDiagnosticWithRange(
|
||||
Diagnostics::Kind::TooManyTypeBitWidthDigits,
|
||||
Diagnostics::Level::Error,
|
||||
{.line_number = 1, .column_number = 2, .length = 1},
|
||||
HasSubstr(llvm::formatv(" {0} ", Count)),
|
||||
{.line_number = 1, .column_number = 2, .length = Count})));
|
||||
code = "i";
|
||||
code.append(Count, '9');
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(code, &consumer);
|
||||
@@ -1328,8 +1336,9 @@ TEST_F(LexerTest, InvalidCommentRunsLumpTogether) {
|
||||
// no matter which invalid byte follows each `//`; a valid comment ends the
|
||||
// run and starts its own record.
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
// The whitespace goes just past the `//`.
|
||||
EXPECT_CALL(consumer,
|
||||
HandleDiagnostic(IsSingleDiagnostic(
|
||||
HandleDiagnostic(Testing::IsPointDiagnostic(
|
||||
Diagnostics::Kind::NoWhitespaceAfterCommentIntroducer,
|
||||
Diagnostics::Level::Error, 1, 3, _)));
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(
|
||||
@@ -1369,8 +1378,9 @@ TEST_F(LexerTest, InvalidCommentRunAtEof) {
|
||||
|
||||
TEST_F(LexerTest, DiagnosticWhitespace) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
// The whitespace goes just past the `//`.
|
||||
EXPECT_CALL(consumer,
|
||||
HandleDiagnostic(IsSingleDiagnostic(
|
||||
HandleDiagnostic(Testing::IsPointDiagnostic(
|
||||
Diagnostics::Kind::NoWhitespaceAfterCommentIntroducer,
|
||||
Diagnostics::Level::Error, 1, 3, _)));
|
||||
compile_helper_.GetTokenizedBuffer("//no space after comment", &consumer);
|
||||
@@ -1378,23 +1388,34 @@ TEST_F(LexerTest, DiagnosticWhitespace) {
|
||||
|
||||
TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
EXPECT_CALL(consumer,
|
||||
HandleDiagnostic(IsSingleDiagnostic(
|
||||
Diagnostics::Kind::UnknownEscapeSequence,
|
||||
Diagnostics::Level::Error, 1, 8, HasSubstr("`b`"))));
|
||||
// The message is located at the `\` that starts the escape, and the escape
|
||||
// is marked.
|
||||
EXPECT_CALL(
|
||||
consumer,
|
||||
HandleDiagnostic(Testing::IsDiagnosticWithRange(
|
||||
Diagnostics::Kind::UnknownEscapeSequence, Diagnostics::Level::Error,
|
||||
{.line_number = 1, .column_number = 7, .length = 1}, HasSubstr("`b`"),
|
||||
{.line_number = 1, .column_number = 7, .length = 2})));
|
||||
compile_helper_.GetTokenizedBuffer(R"("hello\bworld")", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticBadHex) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
Diagnostics::Kind::HexadecimalEscapeMissingDigits,
|
||||
Diagnostics::Level::Error, 1, 9, _)));
|
||||
// The message is located at the first digit that isn't an uppercase
|
||||
// hexadecimal digit, and the escape before it is marked.
|
||||
EXPECT_CALL(consumer,
|
||||
HandleDiagnostic(Testing::IsDiagnosticWithRange(
|
||||
Diagnostics::Kind::HexadecimalEscapeMissingDigits,
|
||||
Diagnostics::Level::Error,
|
||||
{.line_number = 1, .column_number = 9, .length = 0}, _,
|
||||
{.line_number = 1, .column_number = 7, .length = 2})));
|
||||
compile_helper_.GetTokenizedBuffer(R"("hello\xabworld")", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticInvalidDigit) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
// Only the invalid digit is marked, since the message says which literal it
|
||||
// is in.
|
||||
EXPECT_CALL(consumer,
|
||||
HandleDiagnostic(IsSingleDiagnostic(
|
||||
Diagnostics::Kind::InvalidDigit, Diagnostics::Level::Error, 1,
|
||||
@@ -1420,17 +1441,82 @@ TEST_F(LexerTest, DiagnosticLfCr) {
|
||||
|
||||
TEST_F(LexerTest, DiagnosticMissingTerminator) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
Diagnostics::Kind::UnterminatedString,
|
||||
Diagnostics::Level::Error, 1, 1, _)));
|
||||
// The terminator goes where the literal stops, and the whole literal is
|
||||
// marked.
|
||||
EXPECT_CALL(
|
||||
consumer,
|
||||
HandleDiagnostic(Testing::IsDiagnosticWithRange(
|
||||
Diagnostics::Kind::UnterminatedString, Diagnostics::Level::Error,
|
||||
{.line_number = 1, .column_number = 5, .length = 0}, _,
|
||||
{.line_number = 1, .column_number = 1, .length = 4})));
|
||||
compile_helper_.GetTokenizedBuffer(R"(#" ")", &consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticMissingMultiLineTerminator) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
// The literal runs to the end of the file, which is where the terminator
|
||||
// goes, and the literal is marked from its introducer.
|
||||
EXPECT_CALL(
|
||||
consumer,
|
||||
HandleDiagnostic(Testing::IsDiagnosticWithRange(
|
||||
Diagnostics::Kind::UnterminatedString, Diagnostics::Level::Error,
|
||||
{.line_number = 3, .column_number = 1, .length = 0}, _,
|
||||
{.line_number = 1, .column_number = 1, .length = _})));
|
||||
compile_helper_.GetTokenizedBuffer(
|
||||
"'''\n"
|
||||
" text\n",
|
||||
&consumer);
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticPointAtEndOfFile) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
// A file that doesn't end in a newline ends at the end of its last line.
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(Testing::IsPointDiagnostic(
|
||||
Diagnostics::Kind::DollarIntLiteralMissingNumber,
|
||||
Diagnostics::Level::Error, 1, 2, _)));
|
||||
compile_helper_.GetTokenizedBuffer("$", &consumer);
|
||||
}
|
||||
|
||||
// Returns the line and column of the point just past `token`, checking that it
|
||||
// is a point.
|
||||
static auto PointAfter(const TokenizedBuffer& buffer, int token)
|
||||
-> std::pair<int, int> {
|
||||
auto loc = buffer.PointAfterTokenToDiagnosticLoc(TokenIndex(token)).loc;
|
||||
EXPECT_THAT(loc.length, Eq(0));
|
||||
return {loc.line_number, loc.column_number};
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, PointAfterToken) {
|
||||
// Tokens are `FileStart`, `a`, `b`, the string literal, and `FileEnd`.
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer(
|
||||
"a b\n"
|
||||
"'''\n"
|
||||
" c\n"
|
||||
"'''");
|
||||
// The point is before the spaces after the token.
|
||||
EXPECT_THAT(PointAfter(buffer, 1), Pair(1, 2));
|
||||
// The point just past the last token on a line is at the end of the line.
|
||||
EXPECT_THAT(PointAfter(buffer, 2), Pair(1, 5));
|
||||
// A token spanning lines ends on its last line, which here ends the file.
|
||||
EXPECT_THAT(PointAfter(buffer, 3), Pair(4, 4));
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, PointAfterRecoveryToken) {
|
||||
// The lexer inserts a `)` after `a`. It isn't in the source, so the point
|
||||
// just past it is the point just past `a`.
|
||||
auto& buffer = compile_helper_.GetTokenizedBuffer("(a");
|
||||
ASSERT_TRUE(buffer.IsRecoveryToken(TokenIndex(3)));
|
||||
EXPECT_THAT(PointAfter(buffer, 3), Pair(1, 3));
|
||||
}
|
||||
|
||||
TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
|
||||
Testing::MockDiagnosticConsumer consumer;
|
||||
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
|
||||
Diagnostics::Kind::UnrecognizedCharacters,
|
||||
Diagnostics::Level::Error, 1, 1, _)));
|
||||
EXPECT_CALL(
|
||||
consumer,
|
||||
HandleDiagnostic(Testing::IsDiagnosticWithRange(
|
||||
Diagnostics::Kind::UnrecognizedCharacters, Diagnostics::Level::Error,
|
||||
{.line_number = 1, .column_number = 1, .length = 1}, _,
|
||||
{.line_number = 1, .column_number = 1, .length = 1})));
|
||||
compile_helper_.GetTokenizedBuffer("\b", &consumer);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user