carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-09-24 22:02:23 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity
Files
b6396e97f84b258c0a43c3d6612100a75f351aba
carbon-lang/toolchain/lex/tokenized_buffer_test.cpp
T
Chandler CarruthandRichard Smith 565fc5cebb Initial support for CR+LF (DOS / Windows) line endings. (#4056) 
It turns out we can make these work with very minimal complexity because
the LF is still in the right place either way. This also lets us easily
support mixtures of LF and CR+LF line endings gracefully. We create the
line structures around the LF bytes and then have the byte-dispatch loop
notice a CR followed by an LF and skip to the LF behavior.

Rather than add the remaining complexity around supporting bare CR and
LF+CR sequences (both of which are quite rare now), this just adds
diagnostics when we encounter a CR byte that won't fall out of our CR+LF
handling. This is a better experience for users than the alternative. We
still have a TODO to handle the full complexity of vertical whitespace,
but I've updated it to reflect that the common case should be handled
already.

This isn't complete though: we need to add support in string literal
lexing, and we need to teach the diagnostic rendering to handle the
error messages above better. But those will be future PRs, this is
enough to unblock folks who happen to edit a Carbon source file with
notepad on Windows which seems important.

---------

Co-authored-by: Richard Smith <richard@metafoo.co.uk>
2024-06-18 02:23:25 +00:00

1159 lines
44 KiB
C++
Raw Blame History

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "toolchain/lex/tokenized_buffer.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <forward_list>
#include <iterator>
#include "llvm/ADT/ArrayRef.h"
#include "testing/base/test_raw_ostream.h"
#include "toolchain/base/value_store.h"
#include "toolchain/diagnostics/diagnostic_emitter.h"
#include "toolchain/diagnostics/mocks.h"
#include "toolchain/lex/lex.h"
#include "toolchain/lex/tokenized_buffer_test_helpers.h"
#include "toolchain/testing/yaml_test_helpers.h"
namespace Carbon::Lex {
namespace {
using ::Carbon::Testing::ExpectedToken;
using ::Carbon::Testing::IsSingleDiagnostic;
using ::Carbon::Testing::TestRawOstream;
using ::testing::_;
using ::testing::ElementsAre;
using ::testing::Eq;
using ::testing::HasSubstr;
using ::testing::Pair;
namespace Yaml = ::Carbon::Testing::Yaml;
class LexerTest : public ::testing::Test {
protected:
auto GetSourceBuffer(llvm::StringRef text) -> SourceBuffer& {
std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
llvm::MemoryBuffer::getMemBuffer(text)));
source_storage_.push_front(std::move(*SourceBuffer::MakeFromFile(
fs_, filename, ConsoleDiagnosticConsumer())));
return source_storage_.front();
}
auto Lex(llvm::StringRef text,
DiagnosticConsumer& consumer = ConsoleDiagnosticConsumer())
-> TokenizedBuffer {
return Lex::Lex(value_stores_, GetSourceBuffer(text), consumer);
}
SharedValueStores value_stores_;
llvm::vfs::InMemoryFileSystem fs_;
int file_index_ = 0;
std::forward_list<SourceBuffer> source_storage_;
};
TEST_F(LexerTest, HandlesEmptyBuffer) {
auto buffer = Lex("");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
}
TEST_F(LexerTest, TracksLinesAndColumns) {
auto buffer = Lex("\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 2, .column = 4, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 3, .column = 4, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 5, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 6, .indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 4},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 11,
.indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 6,
.column = 6,
.indent_column = 11,
.text = "y"},
{.kind = TokenKind::FileEnd, .line = 6, .column = 7},
}));
}
TEST_F(LexerTest, TracksLinesAndColumnsCRLF) {
auto buffer =
Lex("\r\n ;;\r\n ;;;\r\n x\"foo\" '''baz\r\n a\r\n ''' y");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 2, .column = 4, .indent_column = 3},
{.kind = TokenKind::Semi, .line = 3, .column = 4, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 5, .indent_column = 4},
{.kind = TokenKind::Semi, .line = 3, .column = 6, .indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 4},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 11,
.indent_column = 4},
{.kind = TokenKind::Identifier,
.line = 6,
.column = 6,
.indent_column = 11,
.text = "y"},
{.kind = TokenKind::FileEnd, .line = 6, .column = 7},
}));
}
TEST_F(LexerTest, InvalidCR) {
auto buffer = Lex("\n ;;\r ;\n x");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 2, .column = 6, .indent_column = 2},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::FileEnd, .line = 3, .column = 5},
}));
}
TEST_F(LexerTest, InvalidLFCR) {
auto buffer = Lex("\n ;;\n\r ;\n x");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart,
.line = 1,
.column = 1,
.indent_column = 1},
{.kind = TokenKind::Semi, .line = 2, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 1},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 4,
.indent_column = 4,
.text = "x"},
{.kind = TokenKind::FileEnd, .line = 4, .column = 5},
}));
}
TEST_F(LexerTest, HandlesNumericLiteral) {
auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::IntLiteral,
.line = 1,
.column = 1,
.indent_column = 1,
.text = "12"},
{.kind = TokenKind::Minus,
.line = 1,
.column = 3,
.indent_column = 1},
{.kind = TokenKind::IntLiteral,
.line = 1,
.column = 4,
.indent_column = 1,
.text = "578"},
{.kind = TokenKind::IntLiteral,
.line = 2,
.column = 3,
.indent_column = 3,
.text = "1"},
{.kind = TokenKind::IntLiteral,
.line = 2,
.column = 6,
.indent_column = 3,
.text = "2"},
{.kind = TokenKind::IntLiteral,
.line = 3,
.column = 1,
.indent_column = 1,
.text = "0x12_3ABC"},
{.kind = TokenKind::IntLiteral,
.line = 4,
.column = 1,
.indent_column = 1,
.text = "0b10_10_11"},
{.kind = TokenKind::IntLiteral,
.line = 5,
.column = 1,
.indent_column = 1,
.text = "1_234_567"},
{.kind = TokenKind::RealLiteral,
.line = 6,
.column = 1,
.indent_column = 1,
.text = "1.5e9"},
{.kind = TokenKind::FileEnd, .line = 6, .column = 6},
}));
auto token_start = buffer.tokens().begin();
auto token_12 = token_start + 1;
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_12)), 12);
auto token_578 = token_12 + 2;
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_578)), 578);
auto token_1 = token_578 + 1;
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_1)), 1);
auto token_2 = token_1 + 1;
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_2)), 2);
auto token_0x12_3abc = token_2 + 1;
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_0x12_3abc)),
0x12'3abc);
auto token_0b10_10_11 = token_0x12_3abc + 1;
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_0b10_10_11)),
0b10'10'11);
auto token_1_234_567 = token_0b10_10_11 + 1;
EXPECT_EQ(value_stores_.ints().Get(buffer.GetIntLiteral(*token_1_234_567)),
1'234'567);
auto token_1_5e9 = token_1_234_567 + 1;
auto value_1_5e9 =
value_stores_.reals().Get(buffer.GetRealLiteral(*token_1_5e9));
EXPECT_EQ(value_1_5e9.mantissa.getZExtValue(), 15);
EXPECT_EQ(value_1_5e9.exponent.getSExtValue(), 8);
EXPECT_EQ(value_1_5e9.is_decimal, true);
}
TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
auto buffer = Lex("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = "14x"},
{.kind = TokenKind::IntLiteral,
.line = 1,
.column = 5,
.indent_column = 1,
.text = "15_49"},
{.kind = TokenKind::Error,
.line = 1,
.column = 11,
.indent_column = 1,
.text = "0x3.5q"},
{.kind = TokenKind::RealLiteral,
.line = 1,
.column = 18,
.indent_column = 1,
.text = "0x3_4.5_6"},
{.kind = TokenKind::Error,
.line = 1,
.column = 28,
.indent_column = 1,
.text = "0ops"},
{.kind = TokenKind::FileEnd, .line = 1, .column = 32},
}));
}
TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
llvm::StringLiteral source_text = R"(
1.
.2
3.+foo
4.0-bar
5.0e+123+456
6.0e+1e+2
1e7
8..10
9.0.9.5
10.foo
11.0.foo
12e+1
13._
)";
auto buffer = Lex(source_text);
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::IntLiteral, .text = "1"},
{.kind = TokenKind::Period},
// newline
{.kind = TokenKind::Period},
{.kind = TokenKind::IntLiteral, .text = "2"},
// newline
{.kind = TokenKind::IntLiteral, .text = "3"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Plus},
{.kind = TokenKind::Identifier, .text = "foo"},
// newline
{.kind = TokenKind::RealLiteral, .text = "4.0"},
{.kind = TokenKind::Minus},
{.kind = TokenKind::Identifier, .text = "bar"},
// newline
{.kind = TokenKind::RealLiteral, .text = "5.0e+123"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntLiteral, .text = "456"},
// newline
{.kind = TokenKind::Error, .text = "6.0e+1e"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntLiteral, .text = "2"},
// newline
{.kind = TokenKind::Error, .text = "1e7"},
// newline
{.kind = TokenKind::IntLiteral, .text = "8"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Period},
{.kind = TokenKind::IntLiteral, .text = "10"},
// newline
{.kind = TokenKind::RealLiteral, .text = "9.0"},
{.kind = TokenKind::Period},
{.kind = TokenKind::RealLiteral, .text = "9.5"},
// newline
{.kind = TokenKind::Error, .text = "10.foo"},
// newline
{.kind = TokenKind::RealLiteral, .text = "11.0"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Identifier, .text = "foo"},
// newline
{.kind = TokenKind::Error, .text = "12e"},
{.kind = TokenKind::Plus},
{.kind = TokenKind::IntLiteral, .text = "1"},
// newline
{.kind = TokenKind::IntLiteral, .text = "13"},
{.kind = TokenKind::Period},
{.kind = TokenKind::Underscore},
// newline
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, HandlesGarbageCharacters) {
constexpr char GarbageText[] = "$$💩-$\n$\0$12$\n\\\"\\\n\"x";
auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
// 💩 takes 4 bytes, and we count column as bytes offset.
.text = llvm::StringRef("$$💩", 6)},
{.kind = TokenKind::Minus, .line = 1, .column = 7},
{.kind = TokenKind::Error, .line = 1, .column = 8, .text = "$"},
// newline
{.kind = TokenKind::Error,
.line = 2,
.column = 1,
.text = llvm::StringRef("$\0$", 3)},
{.kind = TokenKind::IntLiteral, .line = 2, .column = 4, .text = "12"},
{.kind = TokenKind::Error, .line = 2, .column = 6, .text = "$"},
// newline
{.kind = TokenKind::Backslash, .line = 3, .column = 1, .text = "\\"},
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
// newline
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
{.kind = TokenKind::FileEnd, .line = 4, .column = 3},
}));
}
TEST_F(LexerTest, Symbols) {
// We don't need to exhaustively test symbols here as they're handled with
// common code, but we want to check specific patterns to verify things like
// max-munch rule and handling of interesting symbols.
auto buffer = Lex("<<<");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::LessLess},
{.kind = TokenKind::Less},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("<<=>>");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::LessLessEqual},
{.kind = TokenKind::GreaterGreater},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("< <=> >");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Less},
{.kind = TokenKind::LessEqualGreater},
{.kind = TokenKind::Greater},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("\\/?@&^!");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Backslash},
{.kind = TokenKind::Slash},
{.kind = TokenKind::Question},
{.kind = TokenKind::At},
{.kind = TokenKind::Amp},
{.kind = TokenKind::Caret},
{.kind = TokenKind::Exclaim},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, Parens) {
auto buffer = Lex("()");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("((()()))");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::OpenParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::CloseParen},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, CurlyBraces) {
auto buffer = Lex("{}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("{{{}{}}}");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::OpenCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::CloseCurlyBrace},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, MatchingGroups) {
{
TokenizedBuffer buffer = Lex("(){}");
ASSERT_FALSE(buffer.has_errors());
auto it = ++buffer.tokens().begin();
auto open_paren_token = *it++;
auto close_paren_token = *it++;
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
auto open_curly_token = *it++;
auto close_curly_token = *it++;
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
{
TokenizedBuffer buffer = Lex("({x}){(y)} {{((z))}}");
ASSERT_FALSE(buffer.has_errors());
auto it = ++buffer.tokens().begin();
auto open_paren_token = *it++;
auto open_curly_token = *it++;
ASSERT_EQ("x",
value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
auto close_curly_token = *it++;
auto close_paren_token = *it++;
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
open_curly_token = *it++;
open_paren_token = *it++;
ASSERT_EQ("y",
value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
close_paren_token = *it++;
close_curly_token = *it++;
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
open_curly_token = *it++;
auto inner_open_curly_token = *it++;
open_paren_token = *it++;
auto inner_open_paren_token = *it++;
ASSERT_EQ("z",
value_stores_.identifiers().Get(buffer.GetIdentifier(*it++)));
auto inner_close_paren_token = *it++;
close_paren_token = *it++;
auto inner_close_curly_token = *it++;
close_curly_token = *it++;
EXPECT_EQ(close_curly_token,
buffer.GetMatchedClosingToken(open_curly_token));
EXPECT_EQ(open_curly_token,
buffer.GetMatchedOpeningToken(close_curly_token));
EXPECT_EQ(inner_close_curly_token,
buffer.GetMatchedClosingToken(inner_open_curly_token));
EXPECT_EQ(inner_open_curly_token,
buffer.GetMatchedOpeningToken(inner_close_curly_token));
EXPECT_EQ(close_paren_token,
buffer.GetMatchedClosingToken(open_paren_token));
EXPECT_EQ(open_paren_token,
buffer.GetMatchedOpeningToken(close_paren_token));
EXPECT_EQ(inner_close_paren_token,
buffer.GetMatchedClosingToken(inner_open_paren_token));
EXPECT_EQ(inner_open_paren_token,
buffer.GetMatchedOpeningToken(inner_close_paren_token));
auto eof_token = *it++;
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::FileEnd);
EXPECT_EQ(buffer.tokens().end(), it);
}
}
TEST_F(LexerTest, MismatchedGroups) {
auto buffer = Lex("{");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Error, .text = "{"},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("}");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Error, .text = "}"},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("{(}");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex(")({)");
EXPECT_TRUE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Error, .column = 1, .text = ")"},
{.kind = TokenKind::OpenParen, .column = 2},
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
{.kind = TokenKind::CloseParen, .column = 4},
{.kind = TokenKind::FileEnd},
}));
}
TEST_F(LexerTest, Whitespace) {
auto buffer = Lex("{( } {(");
// Whether there should be whitespace before/after each token.
bool space[] = {true,
// start-of-file
true,
// {
false,
// (
true,
// inserted )
true,
// }
true,
// error {
false,
// error (
true,
// EOF
false};
int pos = 0;
for (TokenIndex token : buffer.tokens()) {
SCOPED_TRACE(
llvm::formatv("Token #{0}: '{1}'", token, buffer.GetTokenText(token)));
ASSERT_LT(pos, std::size(space));
EXPECT_THAT(buffer.HasLeadingWhitespace(token), Eq(space[pos]));
++pos;
ASSERT_LT(pos, std::size(space));
EXPECT_THAT(buffer.HasTrailingWhitespace(token), Eq(space[pos]));
}
ASSERT_EQ(pos + 1, std::size(space));
}
TEST_F(LexerTest, Keywords) {
TokenKind keywords[] = {
#define CARBON_TOKEN(TokenName)
#define CARBON_KEYWORD_TOKEN(TokenName, ...) TokenKind::TokenName,
#include "toolchain/lex/token_kind.def"
};
for (const auto& keyword : keywords) {
auto buffer = Lex(keyword.fixed_spelling());
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = keyword, .column = 1, .indent_column = 1},
{.kind = TokenKind::FileEnd},
}));
}
}
TEST_F(LexerTest, Comments) {
auto buffer = Lex(" ;\n // foo\n ;\n");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(
buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
{.kind = TokenKind::FileEnd, .line = 3, .column = 4},
}));
buffer = Lex("// foo\n//\n// bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
// Make sure weird characters aren't a problem.
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
// Make sure we can lex a comment at the end of the input.
buffer = Lex("//");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::FileEnd}}));
}
TEST_F(LexerTest, InvalidComments) {
llvm::StringLiteral testcases[] = {
" /// foo\n",
"foo // bar\n",
"//! hello",
" //world",
};
for (llvm::StringLiteral testcase : testcases) {
auto buffer = Lex(testcase);
EXPECT_TRUE(buffer.has_errors());
}
}
TEST_F(LexerTest, Identifiers) {
auto buffer = Lex(" foobar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier,
.column = 4,
.indent_column = 4,
.text = "foobar"},
{.kind = TokenKind::FileEnd},
}));
// Check different kinds of identifier character sequences.
buffer = Lex("_foo_bar");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
{.kind = TokenKind::FileEnd},
}));
buffer = Lex("foo2bar00");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
{.kind = TokenKind::FileEnd},
}));
// Check that we can parse identifiers that start with a keyword.
buffer = Lex("fnord");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart},
{.kind = TokenKind::Identifier, .text = "fnord"},
{.kind = TokenKind::FileEnd},
}));
// Check multiple identifiers with indent and interning.
buffer = Lex(" foo;bar\nbar \n foo\tfoo");
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 1,
.column = 4,
.indent_column = 4,
.text = "foo"},
{.kind = TokenKind::Semi},
{.kind = TokenKind::Identifier,
.line = 1,
.column = 8,
.indent_column = 4,
.text = "bar"},
{.kind = TokenKind::Identifier,
.line = 2,
.column = 1,
.indent_column = 1,
.text = "bar"},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 3,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 7,
.indent_column = 3,
.text = "foo"},
{.kind = TokenKind::FileEnd, .line = 3, .column = 10},
}));
}
TEST_F(LexerTest, StringLiterals) {
llvm::StringLiteral testcase = R"(
"hello world\n"
'''foo
test \
\xAB
''' trailing
#"""#
"\0"
#"\0"foo"\1"#
"""x"""
)";
auto buffer = Lex(testcase);
EXPECT_FALSE(buffer.has_errors());
EXPECT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::StringLiteral,
.line = 2,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {"hello world\n"}},
{.kind = TokenKind::StringLiteral,
.line = 4,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {" test \xAB\n"}},
{.kind = TokenKind::Identifier,
.line = 7,
.column = 10,
.indent_column = 5,
.text = "trailing"},
{.kind = TokenKind::StringLiteral,
.line = 9,
.column = 7,
.indent_column = 7,
.value_stores = &value_stores_,
.string_contents = {"\""}},
{.kind = TokenKind::StringLiteral,
.line = 11,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = llvm::StringLiteral::withInnerNUL("\0")},
{.kind = TokenKind::StringLiteral,
.line = 13,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {"\\0\"foo\"\\1"}},
// """x""" is three string literals, not one invalid
// attempt at a block string literal.
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 5,
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {""}},
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 7,
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {"x"}},
{.kind = TokenKind::StringLiteral,
.line = 15,
.column = 10,
.indent_column = 5,
.value_stores = &value_stores_,
.string_contents = {""}},
{.kind = TokenKind::FileEnd, .line = 16, .column = 3},
}));
}
TEST_F(LexerTest, InvalidStringLiterals) {
llvm::StringLiteral invalid[] = {
// clang-format off
R"(")",
R"('''
'')",
R"("\)",
R"("\")",
R"("\\)",
R"("\\\")",
R"(''')",
R"('''
)",
R"('''\)",
R"(#'''
''')",
// clang-format on
};
for (llvm::StringLiteral test : invalid) {
SCOPED_TRACE(test);
auto buffer = Lex(test);
EXPECT_TRUE(buffer.has_errors());
// We should have formed at least one error token.
bool found_error = false;
for (TokenIndex token : buffer.tokens()) {
if (buffer.GetKind(token) == TokenKind::Error) {
found_error = true;
break;
}
}
EXPECT_TRUE(found_error);
}
}
TEST_F(LexerTest, TypeLiterals) {
llvm::StringLiteral testcase = R"(
i0 i1 i20 i999999999999 i0x1
u0 u1 u64 u64b
f32 f80 f1 fi
s1
)";
auto buffer = Lex(testcase);
EXPECT_FALSE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Identifier,
.line = 2,
.column = 5,
.indent_column = 5,
.text = {"i0"}},
{.kind = TokenKind::IntTypeLiteral,
.line = 2,
.column = 8,
.indent_column = 5,
.text = {"i1"}},
{.kind = TokenKind::IntTypeLiteral,
.line = 2,
.column = 11,
.indent_column = 5,
.text = {"i20"}},
{.kind = TokenKind::IntTypeLiteral,
.line = 2,
.column = 15,
.indent_column = 5,
.text = {"i999999999999"}},
{.kind = TokenKind::Identifier,
.line = 2,
.column = 29,
.indent_column = 5,
.text = {"i0x1"}},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 5,
.indent_column = 5,
.text = {"u0"}},
{.kind = TokenKind::UnsignedIntTypeLiteral,
.line = 3,
.column = 8,
.indent_column = 5,
.text = {"u1"}},
{.kind = TokenKind::UnsignedIntTypeLiteral,
.line = 3,
.column = 11,
.indent_column = 5,
.text = {"u64"}},
{.kind = TokenKind::Identifier,
.line = 3,
.column = 15,
.indent_column = 5,
.text = {"u64b"}},
{.kind = TokenKind::FloatTypeLiteral,
.line = 4,
.column = 5,
.indent_column = 5,
.text = {"f32"}},
{.kind = TokenKind::FloatTypeLiteral,
.line = 4,
.column = 9,
.indent_column = 5,
.text = {"f80"}},
{.kind = TokenKind::FloatTypeLiteral,
.line = 4,
.column = 13,
.indent_column = 5,
.text = {"f1"}},
{.kind = TokenKind::Identifier,
.line = 4,
.column = 16,
.indent_column = 5,
.text = {"fi"}},
{.kind = TokenKind::Identifier,
.line = 5,
.column = 5,
.indent_column = 5,
.text = {"s1"}},
{.kind = TokenKind::FileEnd, .line = 6, .column = 3},
}));
auto type_size = [&](int token_index) {
auto token = buffer.tokens().begin()[token_index];
return value_stores_.ints().Get(buffer.GetTypeLiteralSize(token));
};
EXPECT_EQ(type_size(2), 1);
EXPECT_EQ(type_size(3), 20);
EXPECT_EQ(type_size(4), 999999999999ULL);
EXPECT_EQ(type_size(7), 1);
EXPECT_EQ(type_size(8), 64);
EXPECT_EQ(type_size(10), 32);
EXPECT_EQ(type_size(11), 80);
EXPECT_EQ(type_size(12), 1);
}
TEST_F(LexerTest, TypeLiteralTooManyDigits) {
std::string code = "i";
constexpr int Count = 10000;
code.append(Count, '9');
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer,
HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::TooManyDigits, DiagnosticLevel::Error, 1, 2,
HasSubstr(llvm::formatv(" {0} ", Count)))));
auto buffer = Lex(code, consumer);
EXPECT_TRUE(buffer.has_errors());
ASSERT_THAT(buffer,
HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::FileStart, .line = 1, .column = 1},
{.kind = TokenKind::Error,
.line = 1,
.column = 1,
.indent_column = 1,
.text = code},
{.kind = TokenKind::FileEnd, .line = 1, .column = Count + 2},
}));
}
TEST_F(LexerTest, DiagnosticTrailingComment) {
llvm::StringLiteral testcase = R"(
// Hello!
var String x; // trailing comment
)";
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::TrailingComment,
DiagnosticLevel::Error, 3, 19, _)));
Lex(testcase, consumer);
}
TEST_F(LexerTest, DiagnosticWhitespace) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::NoWhitespaceAfterCommentIntroducer,
DiagnosticLevel::Error, 1, 3, _)));
Lex("//no space after comment", consumer);
}
TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::UnknownEscapeSequence,
DiagnosticLevel::Error, 1, 8, HasSubstr("`b`"))));
Lex(R"("hello\bworld")", consumer);
}
TEST_F(LexerTest, DiagnosticBadHex) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::HexadecimalEscapeMissingDigits,
DiagnosticLevel::Error, 1, 9, _)));
Lex(R"("hello\xabworld")", consumer);
}
TEST_F(LexerTest, DiagnosticInvalidDigit) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::InvalidDigit,
DiagnosticLevel::Error, 1, 6, HasSubstr("'a'"))));
Lex("0x123abc", consumer);
}
TEST_F(LexerTest, DiagnosticMissingTerminator) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::UnterminatedString,
DiagnosticLevel::Error, 1, 1, _)));
Lex(R"(#" ")", consumer);
}
TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
Testing::MockDiagnosticConsumer consumer;
EXPECT_CALL(consumer, HandleDiagnostic(IsSingleDiagnostic(
DiagnosticKind::UnrecognizedCharacters,
DiagnosticLevel::Error, 1, 1, _)));
Lex("\b", consumer);
}
TEST_F(LexerTest, PrintingOutputYaml) {
// Test that we can parse this into YAML and verify line and indent data.
auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
ASSERT_FALSE(buffer.has_errors());
TestRawOstream print_stream;
buffer.Print(print_stream);
EXPECT_THAT(
Yaml::Value::FromText(print_stream.TakeStr()),
IsYaml(ElementsAre(Yaml::Sequence(ElementsAre(Yaml::Mapping(ElementsAre(
Pair("filename", source_storage_.front().filename().str()),
Pair("tokens",
Yaml::Sequence(ElementsAre(
Yaml::Mapping(ElementsAre(
Pair("index", "0"), Pair("kind", "FileStart"),
Pair("line", "1"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", ""),
Pair("has_trailing_space", "true"))),
Yaml::Mapping(
ElementsAre(Pair("index", "1"), Pair("kind", "Semi"),
Pair("line", "2"), Pair("column", "2"),
Pair("indent", "2"), Pair("spelling", ";"),
Pair("has_trailing_space", "true"))),
Yaml::Mapping(
ElementsAre(Pair("index", "2"), Pair("kind", "Semi"),
Pair("line", "5"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", ";"),
Pair("has_trailing_space", "true"))),
Yaml::Mapping(
ElementsAre(Pair("index", "3"), Pair("kind", "Semi"),
Pair("line", "5"), Pair("column", "3"),
Pair("indent", "1"), Pair("spelling", ";"),
Pair("has_trailing_space", "true"))),
Yaml::Mapping(ElementsAre(
Pair("index", "4"), Pair("kind", "FileEnd"),
Pair("line", "15"), Pair("column", "1"),
Pair("indent", "1"), Pair("spelling", "")))))))))))));
}
} // namespace
} // namespace Carbon::Lex
Reference in New Issue View Git Blame Copy Permalink