mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 22:02:23 +01:00
Jon Ross-Perkins
2ecab78297
Lex now prints its yaml as: ``` - filename: name tokens: [ ... ] ``` New support in file_test allows the `filename` marker at the top to define the default file number for later lines, meaning multi-file output from lexing is now associated with the appropriate file. Similar support will probably also apply to lowering, semir, and other places that print a filename once for the full dump. This hammers a bit at how line number replacements work in file_test, allowing stacking them so that lex errors and stdout can both be line-associated properly. I've tried to make the autoupdate more frequently work in one pass, now also taking into account the file index when doing line replacements. There are still some issues with EndOfFile that it may be good to discuss: because CHECK lines are appended to the end of the file now, and the EndOfFile token points at the last line including comments, new lex tests now take two runs to autoupdate (because without CHECK lines, the EndOfFile points at a content line, which content is then inserted after). Note that removing CHECK lines from the test is not a solution: autoupdate also started inserting blank lines, which breaks this for a similar reason. One solution here might be to not have EndOfFile associate with a line or column, which has been a bit of an issue regardless. Also fixes a small issue with toolchain's autoupdate script.
1040 lines
38 KiB
C++
1040 lines
38 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "toolchain/lex/tokenized_buffer.h"
|
|
|
|
#include <gmock/gmock.h>
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <forward_list>
|
|
#include <iterator>
|
|
|
|
#include "llvm/ADT/ArrayRef.h"
|
|
#include "testing/base/test_raw_ostream.h"
|
|
#include "toolchain/base/yaml_test_helpers.h"
|
|
#include "toolchain/diagnostics/diagnostic_emitter.h"
|
|
#include "toolchain/diagnostics/mocks.h"
|
|
#include "toolchain/lex/tokenized_buffer_test_helpers.h"
|
|
|
|
namespace Carbon::Testing {
|
|
namespace {
|
|
|
|
using Lex::Token;
|
|
using Lex::TokenizedBuffer;
|
|
using Lex::TokenKind;
|
|
using ::testing::_;
|
|
using ::testing::ElementsAre;
|
|
using ::testing::Eq;
|
|
using ::testing::HasSubstr;
|
|
|
|
class LexerTest : public ::testing::Test {
|
|
protected:
|
|
auto GetSourceBuffer(llvm::StringRef text) -> SourceBuffer& {
|
|
std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
|
|
CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
|
|
llvm::MemoryBuffer::getMemBuffer(text)));
|
|
source_storage_.push_front(std::move(*SourceBuffer::CreateFromFile(
|
|
fs_, filename, ConsoleDiagnosticConsumer())));
|
|
return source_storage_.front();
|
|
}
|
|
|
|
auto Lex(llvm::StringRef text,
|
|
DiagnosticConsumer& consumer = ConsoleDiagnosticConsumer())
|
|
-> TokenizedBuffer {
|
|
return TokenizedBuffer::Lex(GetSourceBuffer(text), consumer);
|
|
}
|
|
|
|
llvm::vfs::InMemoryFileSystem fs_;
|
|
int file_index_ = 0;
|
|
std::forward_list<SourceBuffer> source_storage_;
|
|
};
|
|
|
|
TEST_F(LexerTest, HandlesEmptyBuffer) {
|
|
auto buffer = Lex("");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
|
|
}
|
|
|
|
TEST_F(LexerTest, TracksLinesAndColumns) {
|
|
auto buffer = Lex("\n ;;\n ;;;\n x\"foo\" '''baz\n a\n ''' y");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(
|
|
buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Semi, .line = 2, .column = 3, .indent_column = 3},
|
|
{.kind = TokenKind::Semi, .line = 2, .column = 4, .indent_column = 3},
|
|
{.kind = TokenKind::Semi, .line = 3, .column = 4, .indent_column = 4},
|
|
{.kind = TokenKind::Semi, .line = 3, .column = 5, .indent_column = 4},
|
|
{.kind = TokenKind::Semi, .line = 3, .column = 6, .indent_column = 4},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 4,
|
|
.column = 4,
|
|
.indent_column = 4,
|
|
.text = "x"},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 4,
|
|
.column = 5,
|
|
.indent_column = 4},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 4,
|
|
.column = 11,
|
|
.indent_column = 4},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 6,
|
|
.column = 6,
|
|
.indent_column = 11,
|
|
.text = "y"},
|
|
{.kind = TokenKind::EndOfFile, .line = 6, .column = 7},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, HandlesNumericLiteral) {
|
|
auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
ASSERT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 1,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = "12"},
|
|
{.kind = TokenKind::Minus,
|
|
.line = 1,
|
|
.column = 3,
|
|
.indent_column = 1},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 1,
|
|
.column = 4,
|
|
.indent_column = 1,
|
|
.text = "578"},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 2,
|
|
.column = 3,
|
|
.indent_column = 3,
|
|
.text = "1"},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 2,
|
|
.column = 6,
|
|
.indent_column = 3,
|
|
.text = "2"},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 3,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = "0x12_3ABC"},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 4,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = "0b10_10_11"},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 5,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = "1_234_567"},
|
|
{.kind = TokenKind::RealLiteral,
|
|
.line = 6,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = "1.5e9"},
|
|
{.kind = TokenKind::EndOfFile, .line = 6, .column = 6},
|
|
}));
|
|
auto token_12 = buffer.tokens().begin();
|
|
EXPECT_EQ(buffer.GetIntegerLiteral(*token_12), 12);
|
|
auto token_578 = buffer.tokens().begin() + 2;
|
|
EXPECT_EQ(buffer.GetIntegerLiteral(*token_578), 578);
|
|
auto token_1 = buffer.tokens().begin() + 3;
|
|
EXPECT_EQ(buffer.GetIntegerLiteral(*token_1), 1);
|
|
auto token_2 = buffer.tokens().begin() + 4;
|
|
EXPECT_EQ(buffer.GetIntegerLiteral(*token_2), 2);
|
|
auto token_0x12_3abc = buffer.tokens().begin() + 5;
|
|
EXPECT_EQ(buffer.GetIntegerLiteral(*token_0x12_3abc), 0x12'3abc);
|
|
auto token_0b10_10_11 = buffer.tokens().begin() + 6;
|
|
EXPECT_EQ(buffer.GetIntegerLiteral(*token_0b10_10_11), 0b10'10'11);
|
|
auto token_1_234_567 = buffer.tokens().begin() + 7;
|
|
EXPECT_EQ(buffer.GetIntegerLiteral(*token_1_234_567), 1'234'567);
|
|
auto token_1_5e9 = buffer.tokens().begin() + 8;
|
|
auto value_1_5e9 = buffer.GetRealLiteral(*token_1_5e9);
|
|
EXPECT_EQ(value_1_5e9.mantissa.getZExtValue(), 15);
|
|
EXPECT_EQ(value_1_5e9.exponent.getSExtValue(), 8);
|
|
EXPECT_EQ(value_1_5e9.is_decimal, true);
|
|
}
|
|
|
|
TEST_F(LexerTest, HandlesInvalidNumericLiterals) {
|
|
auto buffer = Lex("14x 15_49 0x3.5q 0x3_4.5_6 0ops");
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
ASSERT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Error,
|
|
.line = 1,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = "14x"},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 1,
|
|
.column = 5,
|
|
.indent_column = 1,
|
|
.text = "15_49"},
|
|
{.kind = TokenKind::Error,
|
|
.line = 1,
|
|
.column = 11,
|
|
.indent_column = 1,
|
|
.text = "0x3.5q"},
|
|
{.kind = TokenKind::RealLiteral,
|
|
.line = 1,
|
|
.column = 18,
|
|
.indent_column = 1,
|
|
.text = "0x3_4.5_6"},
|
|
{.kind = TokenKind::Error,
|
|
.line = 1,
|
|
.column = 28,
|
|
.indent_column = 1,
|
|
.text = "0ops"},
|
|
{.kind = TokenKind::EndOfFile, .line = 1, .column = 32},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, SplitsNumericLiteralsProperly) {
|
|
llvm::StringLiteral source_text = R"(
|
|
1.
|
|
.2
|
|
3.+foo
|
|
4.0-bar
|
|
5.0e+123+456
|
|
6.0e+1e+2
|
|
1e7
|
|
8..10
|
|
9.0.9.5
|
|
10.foo
|
|
11.0.foo
|
|
12e+1
|
|
13._
|
|
)";
|
|
auto buffer = Lex(source_text);
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::IntegerLiteral, .text = "1"},
|
|
{.kind = TokenKind::Period},
|
|
// newline
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::IntegerLiteral, .text = "2"},
|
|
// newline
|
|
{.kind = TokenKind::IntegerLiteral, .text = "3"},
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::Plus},
|
|
{.kind = TokenKind::Identifier, .text = "foo"},
|
|
// newline
|
|
{.kind = TokenKind::RealLiteral, .text = "4.0"},
|
|
{.kind = TokenKind::Minus},
|
|
{.kind = TokenKind::Identifier, .text = "bar"},
|
|
// newline
|
|
{.kind = TokenKind::RealLiteral, .text = "5.0e+123"},
|
|
{.kind = TokenKind::Plus},
|
|
{.kind = TokenKind::IntegerLiteral, .text = "456"},
|
|
// newline
|
|
{.kind = TokenKind::Error, .text = "6.0e+1e"},
|
|
{.kind = TokenKind::Plus},
|
|
{.kind = TokenKind::IntegerLiteral, .text = "2"},
|
|
// newline
|
|
{.kind = TokenKind::Error, .text = "1e7"},
|
|
// newline
|
|
{.kind = TokenKind::IntegerLiteral, .text = "8"},
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::IntegerLiteral, .text = "10"},
|
|
// newline
|
|
{.kind = TokenKind::RealLiteral, .text = "9.0"},
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::RealLiteral, .text = "9.5"},
|
|
// newline
|
|
{.kind = TokenKind::Error, .text = "10.foo"},
|
|
// newline
|
|
{.kind = TokenKind::RealLiteral, .text = "11.0"},
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::Identifier, .text = "foo"},
|
|
// newline
|
|
{.kind = TokenKind::Error, .text = "12e"},
|
|
{.kind = TokenKind::Plus},
|
|
{.kind = TokenKind::IntegerLiteral, .text = "1"},
|
|
// newline
|
|
{.kind = TokenKind::IntegerLiteral, .text = "13"},
|
|
{.kind = TokenKind::Period},
|
|
{.kind = TokenKind::Underscore},
|
|
// newline
|
|
{.kind = TokenKind::EndOfFile},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, HandlesGarbageCharacters) {
|
|
constexpr char GarbageText[] = "$$💩-$\n$\0$12$\n\\\"\\\n\"x";
|
|
auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1));
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_THAT(
|
|
buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Error,
|
|
.line = 1,
|
|
.column = 1,
|
|
// 💩 takes 4 bytes, and we count column as bytes offset.
|
|
.text = llvm::StringRef("$$💩", 6)},
|
|
{.kind = TokenKind::Minus, .line = 1, .column = 7},
|
|
{.kind = TokenKind::Error, .line = 1, .column = 8, .text = "$"},
|
|
// newline
|
|
{.kind = TokenKind::Error,
|
|
.line = 2,
|
|
.column = 1,
|
|
.text = llvm::StringRef("$\0$", 3)},
|
|
{.kind = TokenKind::IntegerLiteral,
|
|
.line = 2,
|
|
.column = 4,
|
|
.text = "12"},
|
|
{.kind = TokenKind::Error, .line = 2, .column = 6, .text = "$"},
|
|
// newline
|
|
{.kind = TokenKind::Backslash, .line = 3, .column = 1, .text = "\\"},
|
|
{.kind = TokenKind::Error, .line = 3, .column = 2, .text = "\"\\"},
|
|
// newline
|
|
{.kind = TokenKind::Error, .line = 4, .column = 1, .text = "\"x"},
|
|
{.kind = TokenKind::EndOfFile, .line = 4, .column = 3},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, Symbols) {
|
|
// We don't need to exhaustively test symbols here as they're handled with
|
|
// common code, but we want to check specific patterns to verify things like
|
|
// max-munch rule and handling of interesting symbols.
|
|
auto buffer = Lex("<<<");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::LessLess},
|
|
{TokenKind::Less},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("<<=>>");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::LessLessEqual},
|
|
{TokenKind::GreaterGreater},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("< <=> >");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::Less},
|
|
{TokenKind::LessEqualGreater},
|
|
{TokenKind::Greater},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("\\/?@&^!");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::Backslash},
|
|
{TokenKind::Slash},
|
|
{TokenKind::Question},
|
|
{TokenKind::At},
|
|
{TokenKind::Amp},
|
|
{TokenKind::Caret},
|
|
{TokenKind::Exclaim},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, Parens) {
|
|
auto buffer = Lex("()");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("((()()))");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::OpenParen},
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::CloseParen},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, CurlyBraces) {
|
|
auto buffer = Lex("{}");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("{{{}{}}}");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::OpenCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::CloseCurlyBrace},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, MatchingGroups) {
|
|
{
|
|
TokenizedBuffer buffer = Lex("(){}");
|
|
ASSERT_FALSE(buffer.has_errors());
|
|
auto it = buffer.tokens().begin();
|
|
auto open_paren_token = *it++;
|
|
auto close_paren_token = *it++;
|
|
EXPECT_EQ(close_paren_token,
|
|
buffer.GetMatchedClosingToken(open_paren_token));
|
|
EXPECT_EQ(open_paren_token,
|
|
buffer.GetMatchedOpeningToken(close_paren_token));
|
|
auto open_curly_token = *it++;
|
|
auto close_curly_token = *it++;
|
|
EXPECT_EQ(close_curly_token,
|
|
buffer.GetMatchedClosingToken(open_curly_token));
|
|
EXPECT_EQ(open_curly_token,
|
|
buffer.GetMatchedOpeningToken(close_curly_token));
|
|
auto eof_token = *it++;
|
|
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
|
|
EXPECT_EQ(buffer.tokens().end(), it);
|
|
}
|
|
|
|
{
|
|
TokenizedBuffer buffer = Lex("({x}){(y)} {{((z))}}");
|
|
ASSERT_FALSE(buffer.has_errors());
|
|
auto it = buffer.tokens().begin();
|
|
auto open_paren_token = *it++;
|
|
auto open_curly_token = *it++;
|
|
ASSERT_EQ("x", buffer.GetIdentifierText(buffer.GetIdentifier(*it++)));
|
|
auto close_curly_token = *it++;
|
|
auto close_paren_token = *it++;
|
|
EXPECT_EQ(close_paren_token,
|
|
buffer.GetMatchedClosingToken(open_paren_token));
|
|
EXPECT_EQ(open_paren_token,
|
|
buffer.GetMatchedOpeningToken(close_paren_token));
|
|
EXPECT_EQ(close_curly_token,
|
|
buffer.GetMatchedClosingToken(open_curly_token));
|
|
EXPECT_EQ(open_curly_token,
|
|
buffer.GetMatchedOpeningToken(close_curly_token));
|
|
|
|
open_curly_token = *it++;
|
|
open_paren_token = *it++;
|
|
ASSERT_EQ("y", buffer.GetIdentifierText(buffer.GetIdentifier(*it++)));
|
|
close_paren_token = *it++;
|
|
close_curly_token = *it++;
|
|
EXPECT_EQ(close_curly_token,
|
|
buffer.GetMatchedClosingToken(open_curly_token));
|
|
EXPECT_EQ(open_curly_token,
|
|
buffer.GetMatchedOpeningToken(close_curly_token));
|
|
EXPECT_EQ(close_paren_token,
|
|
buffer.GetMatchedClosingToken(open_paren_token));
|
|
EXPECT_EQ(open_paren_token,
|
|
buffer.GetMatchedOpeningToken(close_paren_token));
|
|
|
|
open_curly_token = *it++;
|
|
auto inner_open_curly_token = *it++;
|
|
open_paren_token = *it++;
|
|
auto inner_open_paren_token = *it++;
|
|
ASSERT_EQ("z", buffer.GetIdentifierText(buffer.GetIdentifier(*it++)));
|
|
auto inner_close_paren_token = *it++;
|
|
close_paren_token = *it++;
|
|
auto inner_close_curly_token = *it++;
|
|
close_curly_token = *it++;
|
|
EXPECT_EQ(close_curly_token,
|
|
buffer.GetMatchedClosingToken(open_curly_token));
|
|
EXPECT_EQ(open_curly_token,
|
|
buffer.GetMatchedOpeningToken(close_curly_token));
|
|
EXPECT_EQ(inner_close_curly_token,
|
|
buffer.GetMatchedClosingToken(inner_open_curly_token));
|
|
EXPECT_EQ(inner_open_curly_token,
|
|
buffer.GetMatchedOpeningToken(inner_close_curly_token));
|
|
EXPECT_EQ(close_paren_token,
|
|
buffer.GetMatchedClosingToken(open_paren_token));
|
|
EXPECT_EQ(open_paren_token,
|
|
buffer.GetMatchedOpeningToken(close_paren_token));
|
|
EXPECT_EQ(inner_close_paren_token,
|
|
buffer.GetMatchedClosingToken(inner_open_paren_token));
|
|
EXPECT_EQ(inner_open_paren_token,
|
|
buffer.GetMatchedOpeningToken(inner_close_paren_token));
|
|
|
|
auto eof_token = *it++;
|
|
EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile);
|
|
EXPECT_EQ(buffer.tokens().end(), it);
|
|
}
|
|
}
|
|
|
|
TEST_F(LexerTest, MismatchedGroups) {
|
|
auto buffer = Lex("{");
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::OpenCurlyBrace},
|
|
{.kind = TokenKind::CloseCurlyBrace, .recovery = true},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("}");
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Error, .text = "}"},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("{(}");
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_THAT(
|
|
buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::OpenCurlyBrace, .column = 1},
|
|
{.kind = TokenKind::OpenParen, .column = 2},
|
|
{.kind = TokenKind::CloseParen, .column = 3, .recovery = true},
|
|
{.kind = TokenKind::CloseCurlyBrace, .column = 3},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex(")({)");
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
EXPECT_THAT(
|
|
buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Error, .column = 1, .text = ")"},
|
|
{.kind = TokenKind::OpenParen, .column = 2},
|
|
{.kind = TokenKind::OpenCurlyBrace, .column = 3},
|
|
{.kind = TokenKind::CloseCurlyBrace, .column = 4, .recovery = true},
|
|
{.kind = TokenKind::CloseParen, .column = 4},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, Whitespace) {
|
|
auto buffer = Lex("{( } {(");
|
|
|
|
// Whether there should be whitespace before/after each token.
|
|
bool space[] = {true,
|
|
// {
|
|
false,
|
|
// (
|
|
true,
|
|
// inserted )
|
|
true,
|
|
// }
|
|
true,
|
|
// {
|
|
false,
|
|
// (
|
|
true,
|
|
// inserted )
|
|
true,
|
|
// inserted }
|
|
true,
|
|
// EOF
|
|
false};
|
|
int pos = 0;
|
|
for (Token token : buffer.tokens()) {
|
|
ASSERT_LT(pos, std::size(space));
|
|
EXPECT_THAT(buffer.HasLeadingWhitespace(token), Eq(space[pos]));
|
|
++pos;
|
|
ASSERT_LT(pos, std::size(space));
|
|
EXPECT_THAT(buffer.HasTrailingWhitespace(token), Eq(space[pos]));
|
|
}
|
|
ASSERT_EQ(pos + 1, std::size(space));
|
|
}
|
|
|
|
TEST_F(LexerTest, Keywords) {
|
|
auto buffer = Lex(" fn");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Fn, .column = 4, .indent_column = 4},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("and or not if else for return var break continue _");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{TokenKind::And},
|
|
{TokenKind::Or},
|
|
{TokenKind::Not},
|
|
{TokenKind::If},
|
|
{TokenKind::Else},
|
|
{TokenKind::For},
|
|
{TokenKind::Return},
|
|
{TokenKind::Var},
|
|
{TokenKind::Break},
|
|
{TokenKind::Continue},
|
|
{TokenKind::Underscore},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, Comments) {
|
|
auto buffer = Lex(" ;\n // foo\n ;\n");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(
|
|
buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Semi, .line = 1, .column = 2, .indent_column = 2},
|
|
{.kind = TokenKind::Semi, .line = 3, .column = 3, .indent_column = 3},
|
|
{.kind = TokenKind::EndOfFile, .line = 3, .column = 4},
|
|
}));
|
|
|
|
buffer = Lex("// foo\n//\n// bar");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
|
|
|
|
// Make sure weird characters aren't a problem.
|
|
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
|
|
|
|
// Make sure we can lex a comment at the end of the input.
|
|
buffer = Lex("//");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{{TokenKind::EndOfFile}}));
|
|
}
|
|
|
|
TEST_F(LexerTest, InvalidComments) {
|
|
llvm::StringLiteral testcases[] = {
|
|
" /// foo\n",
|
|
"foo // bar\n",
|
|
"//! hello",
|
|
" //world",
|
|
};
|
|
for (llvm::StringLiteral testcase : testcases) {
|
|
auto buffer = Lex(testcase);
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
}
|
|
}
|
|
|
|
TEST_F(LexerTest, Identifiers) {
|
|
auto buffer = Lex(" foobar");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Identifier,
|
|
.column = 4,
|
|
.indent_column = 4,
|
|
.text = "foobar"},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
// Check different kinds of identifier character sequences.
|
|
buffer = Lex("_foo_bar");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Identifier, .text = "_foo_bar"},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
buffer = Lex("foo2bar00");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Identifier, .text = "foo2bar00"},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
// Check that we can parse identifiers that start with a keyword.
|
|
buffer = Lex("fnord");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Identifier, .text = "fnord"},
|
|
{TokenKind::EndOfFile},
|
|
}));
|
|
|
|
// Check multiple identifiers with indent and interning.
|
|
buffer = Lex(" foo;bar\nbar \n foo\tfoo");
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 1,
|
|
.column = 4,
|
|
.indent_column = 4,
|
|
.text = "foo"},
|
|
{.kind = TokenKind::Semi},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 1,
|
|
.column = 8,
|
|
.indent_column = 4,
|
|
.text = "bar"},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 2,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = "bar"},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 3,
|
|
.column = 3,
|
|
.indent_column = 3,
|
|
.text = "foo"},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 3,
|
|
.column = 7,
|
|
.indent_column = 3,
|
|
.text = "foo"},
|
|
{.kind = TokenKind::EndOfFile, .line = 3, .column = 10},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, StringLiterals) {
|
|
llvm::StringLiteral testcase = R"(
|
|
"hello world\n"
|
|
|
|
'''foo
|
|
test \
|
|
\xAB
|
|
''' trailing
|
|
|
|
#"""#
|
|
|
|
"\0"
|
|
|
|
#"\0"foo"\1"#
|
|
|
|
"""x"""
|
|
)";
|
|
|
|
auto buffer = Lex(testcase);
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
EXPECT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 2,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.string_contents = {"hello world\n"}},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 4,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.string_contents = {" test \xAB\n"}},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 7,
|
|
.column = 10,
|
|
.indent_column = 5,
|
|
.text = "trailing"},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 9,
|
|
.column = 7,
|
|
.indent_column = 7,
|
|
.string_contents = {"\""}},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 11,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.string_contents = llvm::StringLiteral::withInnerNUL("\0")},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 13,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.string_contents = {"\\0\"foo\"\\1"}},
|
|
|
|
// """x""" is three string literals, not one invalid
|
|
// attempt at a block string literal.
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 15,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.string_contents = {""}},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 15,
|
|
.column = 7,
|
|
.indent_column = 5,
|
|
.string_contents = {"x"}},
|
|
{.kind = TokenKind::StringLiteral,
|
|
.line = 15,
|
|
.column = 10,
|
|
.indent_column = 5,
|
|
.string_contents = {""}},
|
|
{.kind = TokenKind::EndOfFile, .line = 16, .column = 3},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, InvalidStringLiterals) {
|
|
llvm::StringLiteral invalid[] = {
|
|
// clang-format off
|
|
R"(")",
|
|
R"('''
|
|
'')",
|
|
R"("\)",
|
|
R"("\")",
|
|
R"("\\)",
|
|
R"("\\\")",
|
|
R"(''')",
|
|
R"('''
|
|
)",
|
|
R"('''\)",
|
|
R"(#'''
|
|
''')",
|
|
// clang-format on
|
|
};
|
|
|
|
for (llvm::StringLiteral test : invalid) {
|
|
SCOPED_TRACE(test);
|
|
auto buffer = Lex(test);
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
|
|
// We should have formed at least one error token.
|
|
bool found_error = false;
|
|
for (Token token : buffer.tokens()) {
|
|
if (buffer.GetKind(token) == TokenKind::Error) {
|
|
found_error = true;
|
|
break;
|
|
}
|
|
}
|
|
EXPECT_TRUE(found_error);
|
|
}
|
|
}
|
|
|
|
TEST_F(LexerTest, TypeLiterals) {
|
|
llvm::StringLiteral testcase = R"(
|
|
i0 i1 i20 i999999999999 i0x1
|
|
u0 u1 u64 u64b
|
|
f32 f80 f1 fi
|
|
s1
|
|
)";
|
|
|
|
auto buffer = Lex(testcase);
|
|
EXPECT_FALSE(buffer.has_errors());
|
|
ASSERT_THAT(buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 2,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.text = {"i0"}},
|
|
{.kind = TokenKind::IntegerTypeLiteral,
|
|
.line = 2,
|
|
.column = 8,
|
|
.indent_column = 5,
|
|
.text = {"i1"}},
|
|
{.kind = TokenKind::IntegerTypeLiteral,
|
|
.line = 2,
|
|
.column = 11,
|
|
.indent_column = 5,
|
|
.text = {"i20"}},
|
|
{.kind = TokenKind::IntegerTypeLiteral,
|
|
.line = 2,
|
|
.column = 15,
|
|
.indent_column = 5,
|
|
.text = {"i999999999999"}},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 2,
|
|
.column = 29,
|
|
.indent_column = 5,
|
|
.text = {"i0x1"}},
|
|
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 3,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.text = {"u0"}},
|
|
{.kind = TokenKind::UnsignedIntegerTypeLiteral,
|
|
.line = 3,
|
|
.column = 8,
|
|
.indent_column = 5,
|
|
.text = {"u1"}},
|
|
{.kind = TokenKind::UnsignedIntegerTypeLiteral,
|
|
.line = 3,
|
|
.column = 11,
|
|
.indent_column = 5,
|
|
.text = {"u64"}},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 3,
|
|
.column = 15,
|
|
.indent_column = 5,
|
|
.text = {"u64b"}},
|
|
|
|
{.kind = TokenKind::FloatingPointTypeLiteral,
|
|
.line = 4,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.text = {"f32"}},
|
|
{.kind = TokenKind::FloatingPointTypeLiteral,
|
|
.line = 4,
|
|
.column = 9,
|
|
.indent_column = 5,
|
|
.text = {"f80"}},
|
|
{.kind = TokenKind::FloatingPointTypeLiteral,
|
|
.line = 4,
|
|
.column = 13,
|
|
.indent_column = 5,
|
|
.text = {"f1"}},
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 4,
|
|
.column = 16,
|
|
.indent_column = 5,
|
|
.text = {"fi"}},
|
|
|
|
{.kind = TokenKind::Identifier,
|
|
.line = 5,
|
|
.column = 5,
|
|
.indent_column = 5,
|
|
.text = {"s1"}},
|
|
|
|
{.kind = TokenKind::EndOfFile, .line = 6, .column = 3},
|
|
}));
|
|
|
|
auto token_i1 = buffer.tokens().begin() + 1;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_i1), 1);
|
|
auto token_i20 = buffer.tokens().begin() + 2;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_i20), 20);
|
|
auto token_i999999999999 = buffer.tokens().begin() + 3;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_i999999999999), 999999999999ULL);
|
|
auto token_u1 = buffer.tokens().begin() + 6;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_u1), 1);
|
|
auto token_u64 = buffer.tokens().begin() + 7;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_u64), 64);
|
|
auto token_f32 = buffer.tokens().begin() + 9;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_f32), 32);
|
|
auto token_f80 = buffer.tokens().begin() + 10;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_f80), 80);
|
|
auto token_f1 = buffer.tokens().begin() + 11;
|
|
EXPECT_EQ(buffer.GetTypeLiteralSize(*token_f1), 1);
|
|
}
|
|
|
|
TEST_F(LexerTest, TypeLiteralTooManyDigits) {
|
|
std::string code = "i";
|
|
constexpr int Count = 10000;
|
|
code.append(Count, '9');
|
|
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer,
|
|
HandleDiagnostic(IsDiagnostic(
|
|
DiagnosticKind::TooManyDigits, DiagnosticLevel::Error, 1, 2,
|
|
HasSubstr(llvm::formatv(" {0} ", Count)))));
|
|
auto buffer = Lex(code, consumer);
|
|
EXPECT_TRUE(buffer.has_errors());
|
|
ASSERT_THAT(
|
|
buffer,
|
|
HasTokens(llvm::ArrayRef<ExpectedToken>{
|
|
{.kind = TokenKind::Error,
|
|
.line = 1,
|
|
.column = 1,
|
|
.indent_column = 1,
|
|
.text = {code}},
|
|
{.kind = TokenKind::EndOfFile, .line = 1, .column = Count + 2},
|
|
}));
|
|
}
|
|
|
|
TEST_F(LexerTest, DiagnosticTrailingComment) {
|
|
llvm::StringLiteral testcase = R"(
|
|
// Hello!
|
|
var String x; // trailing comment
|
|
)";
|
|
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer,
|
|
HandleDiagnostic(IsDiagnostic(DiagnosticKind::TrailingComment,
|
|
DiagnosticLevel::Error, 3, 19, _)));
|
|
Lex(testcase, consumer);
|
|
}
|
|
|
|
TEST_F(LexerTest, DiagnosticWhitespace) {
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsDiagnostic(
|
|
DiagnosticKind::NoWhitespaceAfterCommentIntroducer,
|
|
DiagnosticLevel::Error, 1, 3, _)));
|
|
Lex("//no space after comment", consumer);
|
|
}
|
|
|
|
TEST_F(LexerTest, DiagnosticUnrecognizedEscape) {
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsDiagnostic(
|
|
DiagnosticKind::UnknownEscapeSequence,
|
|
DiagnosticLevel::Error, 1, 8, HasSubstr("`b`"))));
|
|
Lex(R"("hello\bworld")", consumer);
|
|
}
|
|
|
|
TEST_F(LexerTest, DiagnosticBadHex) {
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsDiagnostic(
|
|
DiagnosticKind::HexadecimalEscapeMissingDigits,
|
|
DiagnosticLevel::Error, 1, 9, _)));
|
|
Lex(R"("hello\xabworld")", consumer);
|
|
}
|
|
|
|
TEST_F(LexerTest, DiagnosticInvalidDigit) {
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer, HandleDiagnostic(IsDiagnostic(
|
|
DiagnosticKind::InvalidDigit,
|
|
DiagnosticLevel::Error, 1, 6, HasSubstr("'a'"))));
|
|
Lex("0x123abc", consumer);
|
|
}
|
|
|
|
TEST_F(LexerTest, DiagnosticMissingTerminator) {
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer,
|
|
HandleDiagnostic(IsDiagnostic(DiagnosticKind::UnterminatedString,
|
|
DiagnosticLevel::Error, 1, 1, _)));
|
|
Lex(R"(#" ")", consumer);
|
|
}
|
|
|
|
TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
EXPECT_CALL(consumer, HandleDiagnostic(
|
|
IsDiagnostic(DiagnosticKind::UnrecognizedCharacters,
|
|
DiagnosticLevel::Error, 1, 1, _)));
|
|
Lex("\b", consumer);
|
|
}
|
|
|
|
TEST_F(LexerTest, PrintingAsYaml) {
|
|
// Test that we can parse this into YAML and verify line and indent data.
|
|
auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
|
|
ASSERT_FALSE(buffer.has_errors());
|
|
TestRawOstream print_stream;
|
|
buffer.Print(print_stream);
|
|
|
|
EXPECT_THAT(
|
|
Yaml::Value::FromText(print_stream.TakeStr()),
|
|
ElementsAre(Yaml::SequenceValue{Yaml::MappingValue{
|
|
{"filename", Yaml::ScalarValue{source_storage_.front().filename()}},
|
|
{"tokens", Yaml::SequenceValue{
|
|
Yaml::MappingValue{{"index", "0"},
|
|
{"kind", "Semi"},
|
|
{"line", "2"},
|
|
{"column", "2"},
|
|
{"indent", "2"},
|
|
{"spelling", ";"},
|
|
{"has_trailing_space", "true"}},
|
|
Yaml::MappingValue{{"index", "1"},
|
|
{"kind", "Semi"},
|
|
{"line", "5"},
|
|
{"column", "1"},
|
|
{"indent", "1"},
|
|
{"spelling", ";"},
|
|
{"has_trailing_space", "true"}},
|
|
Yaml::MappingValue{{"index", "2"},
|
|
{"kind", "Semi"},
|
|
{"line", "5"},
|
|
{"column", "3"},
|
|
{"indent", "1"},
|
|
{"spelling", ";"},
|
|
{"has_trailing_space", "true"}},
|
|
Yaml::MappingValue{{"index", "3"},
|
|
{"kind", "EndOfFile"},
|
|
{"line", "15"},
|
|
{"column", "1"},
|
|
{"indent", "1"},
|
|
{"spelling", ""}}}}}}));
|
|
}
|
|
|
|
} // namespace
|
|
} // namespace Carbon::Testing
|