carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-04 22:02:52 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Update comment lexing to match #198. (#292)

Browse Source 
Require a space after a `//`, and recognize but reject a `//` that follows non-whitespace text in the same line. Remove DocComment token kind that ended up not being part of the design.
This commit is contained in:
Richard Smith
2021-02-25 16:16:08 -08:00
committed by GitHub
parent 3ea84c9276
commit bbaa0e788c
3 changed files with 59 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lexer/tokenized_buffer_test.cpp
+17 -57
View File
@@ -520,67 +520,27 @@ TEST_F(LexerTest, Comments) {
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{}));
// Make sure weird characters aren't a problem.
buffer = Lex(" //foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
EXPECT_FALSE(buffer.HasErrors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{}));
// Make sure we can lex a comment at the end of the input.
buffer = Lex("//");
EXPECT_FALSE(buffer.HasErrors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{}));
}
TEST_F(LexerTest, DocComments) {
auto buffer = Lex(" /// foo");
EXPECT_FALSE(buffer.HasErrors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::DocComment(),
.line = 1,
.column = 3,
.indent_column = 3,
.text = "/// foo"},
}));
buffer = Lex("/// foo\n//\n/// bar");
EXPECT_FALSE(buffer.HasErrors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::DocComment(),
.line = 1,
.column = 1,
.indent_column = 1,
.text = "/// foo"},
{.kind = TokenKind::DocComment(),
.line = 3,
.column = 1,
.indent_column = 1,
.text = "/// bar"},
}));
buffer = Lex("/// foo\n///\n/// bar");
EXPECT_FALSE(buffer.HasErrors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::DocComment(),
.line = 1,
.column = 1,
.indent_column = 1,
.text = "/// foo"},
{.kind = TokenKind::DocComment(),
.line = 2,
.column = 1,
.indent_column = 1,
.text = "///"},
{.kind = TokenKind::DocComment(),
.line = 3,
.column = 1,
.indent_column = 1,
.text = "/// bar"},
}));
// Make sure weird characters aren't a problem.
buffer = Lex(" ///foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]");
EXPECT_FALSE(buffer.HasErrors());
EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef<ExpectedToken>{
{.kind = TokenKind::DocComment(),
.line = 1,
.column = 3,
.indent_column = 3,
.text = "///foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]"},
}));
TEST_F(LexerTest, InvalidComments) {
llvm::StringLiteral testcases[] = {
" /// foo\n",
"foo // bar\n",
"//! hello",
" //world",
};
for (llvm::StringLiteral testcase : testcases) {
auto buffer = Lex(testcase);
EXPECT_TRUE(buffer.HasErrors());
}
}
TEST_F(LexerTest, Identifiers) {