carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Migrate some lexer tests to file tests. (#2892)

Browse Source 
These tests are doing string comparisons on output that don't seem to be meaningfully different from a file_test.

I'm tempted to migrate lexer tests in general, but I'm not doing that here since others may find more value in the current approach.
This commit is contained in:
Jon Ross-Perkins
2023-06-14 09:50:48 -07:00
committed by GitHub
parent d18c1347d7
commit 8b1e820848
7 changed files with 69 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lexer/tokenized_buffer_test.cpp
+9 -128
View File
@@ -32,12 +32,12 @@ using ::testing::StrEq;
class LexerTest : public ::testing::Test {
protected:
auto GetSourceBuffer(llvm::StringRef text) -> SourceBuffer& {
std::string filename = llvm::formatv("test{0}.carbon", ++file_index);
CARBON_CHECK(fs.addFile(filename, /*ModificationTime=*/0,
llvm::MemoryBuffer::getMemBuffer(text)));
source_storage.push_front(
std::move(*SourceBuffer::CreateFromFile(fs, filename)));
return source_storage.front();
std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
llvm::MemoryBuffer::getMemBuffer(text)));
source_storage_.push_front(
std::move(*SourceBuffer::CreateFromFile(fs_, filename)));
return source_storage_.front();
}
auto Lex(llvm::StringRef text,
@@ -46,9 +46,9 @@ class LexerTest : public ::testing::Test {
return TokenizedBuffer::Lex(GetSourceBuffer(text), consumer);
}
llvm::vfs::InMemoryFileSystem fs;
int file_index = 0;
std::forward_list<SourceBuffer> source_storage;
llvm::vfs::InMemoryFileSystem fs_;
int file_index_ = 0;
std::forward_list<SourceBuffer> source_storage_;
};
TEST_F(LexerTest, HandlesEmptyBuffer) {
@@ -995,111 +995,6 @@ TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
Lex("\b", consumer);
}
auto GetAndDropLine(llvm::StringRef& text) -> std::string {
auto newline_offset = text.find_first_of('\n');
llvm::StringRef line = text.slice(0, newline_offset);
if (newline_offset != llvm::StringRef::npos) {
text = text.substr(newline_offset + 1);
} else {
text = "";
}
return line.str();
}
TEST_F(LexerTest, PrintingInteger) {
auto buffer = Lex("123");
ASSERT_FALSE(buffer.has_errors());
std::string print_storage;
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
"column: 1, indent: 1, spelling: '123', value: `123`, "
"has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
TEST_F(LexerTest, PrintingReal) {
auto buffer = Lex("2.5");
ASSERT_FALSE(buffer.has_errors());
std::string print_storage;
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(
GetAndDropLine(print),
StrEq("{ index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: "
"1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true "
"},"));
EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
TEST_F(LexerTest, PrintingPadding) {
// Test kind padding.
auto buffer = Lex("(;foo;)");
ASSERT_FALSE(buffer.has_errors());
std::string print_storage;
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 0, kind: 'OpenParen', line: 1, column: "
"1, indent: 1, spelling: '(', closing_token: 4 },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 1, kind: 'Semi', line: 1, column: "
"2, indent: 1, spelling: ';' },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 2, kind: 'Identifier', line: 1, column: "
"3, indent: 1, spelling: 'foo', identifier: 0 },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 3, kind: 'Semi', line: 1, column: "
"6, indent: 1, spelling: ';' },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 4, kind: 'CloseParen', line: 1, column: "
"7, indent: 1, spelling: ')', opening_token: 0, "
"has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 5, kind: 'EndOfFile', line: 1, column: "
"8, indent: 1, spelling: '' },"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
TEST_F(LexerTest, PrintingPaddingDigits) {
// Test digit padding with max values of 9, 10, and 11.
auto buffer = Lex(";\n\n\n\n\n\n\n\n\n\n ;;");
ASSERT_FALSE(buffer.has_errors());
std::string print_storage;
llvm::raw_string_ostream print_stream(print_storage);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 0, kind: 'Semi', line: 1, column: 1, "
"indent: 1, spelling: ';', has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 1, kind: 'Semi', line: 11, column: 9, "
"indent: 9, spelling: ';' },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 2, kind: 'Semi', line: 11, column: 10, "
"indent: 9, spelling: ';', has_trailing_space: true },"));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 3, kind: 'EndOfFile', line: 11, column: 11, "
"indent: 9, spelling: '' },"));
EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
EXPECT_TRUE(print.empty()) << print;
}
TEST_F(LexerTest, PrintingAsYaml) {
// Test that we can parse this into YAML and verify line and indent data.
auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
@@ -1138,19 +1033,5 @@ TEST_F(LexerTest, PrintingAsYaml) {
{"spelling", ""}}}));
}
TEST_F(LexerTest, PrintToken) {
auto buffer = Lex("0x9");
ASSERT_FALSE(buffer.has_errors());
std::string print_output;
llvm::raw_string_ostream print_stream(print_output);
buffer.Print(print_stream);
llvm::StringRef print = print_stream.str();
EXPECT_THAT(GetAndDropLine(print), StrEq("["));
EXPECT_THAT(GetAndDropLine(print),
StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
"column: 1, indent: 1, spelling: '0x9', value: `9`, "
"has_trailing_space: true },"));
}
} // namespace
} // namespace Carbon::Testing