carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-04 22:02:52 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Issue #2121: Support new block string literal syntax in explorer (#2399)

Browse Source
This commit is contained in:
Priyananda Shenoy
2023-01-04 15:53:47 -08:00
committed by GitHub
parent 4daaa4866f
commit c87d271c53
14 changed files with 111 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/string_helpers.cpp
+1 -1
View File
@@ -14,7 +14,7 @@
namespace Carbon {
static constexpr llvm::StringRef TripleQuotes = R"(""")";
static constexpr llvm::StringRef TripleQuotes = "'''";
static constexpr llvm::StringRef HorizontalWhitespaceChars = " \t";
// Carbon only takes uppercase hex input.
common/string_helpers_test.cpp
+28 -28
View File
@@ -95,57 +95,57 @@ TEST(ParseBlockStringLiteral, FailNoLeadingTripleQuotes) {
}
TEST(ParseBlockStringLiteral, FailInvalideFiletypeIndicator) {
EXPECT_THAT(ParseBlockStringLiteral("\"\"\"carbon file\n").error().message(),
EXPECT_THAT(ParseBlockStringLiteral("'''carbon file\n").error().message(),
Eq("Invalid characters in file type indicator: carbon file"));
}
TEST(ParseBlockStringLiteral, FailEndingTripleQuotes) {
EXPECT_THAT(ParseBlockStringLiteral("\"\"\"\n").error().message(),
EXPECT_THAT(ParseBlockStringLiteral("'''\n").error().message(),
Eq("Should end with triple quotes: "));
}
TEST(ParseBlockStringLiteral, FailWrongIndent) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal
with wrong indent
""")";
''')";
EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
Eq("Wrong indent for line: with wrong indent, expected 5"));
}
TEST(ParseBlockStringLiteral, FailInvalidEscaping) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
\q
""")";
''')";
EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
Eq("Invalid escaping in \\q"));
constexpr char InputRaw[] = R"("""
constexpr char InputRaw[] = R"('''
\#q
""")";
''')";
EXPECT_THAT(ParseBlockStringLiteral(InputRaw, 1).error().message(),
Eq("Invalid escaping in \\#q"));
}
TEST(ParseBlockStringLiteral, OkEmptyString) {
constexpr char Input[] = R"("""
""")";
constexpr char Input[] = R"('''
''')";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(""));
}
TEST(ParseBlockStringLiteral, OkOneLineString) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal
""")";
''')";
constexpr char Expected[] = R"(A block string literal
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkTwoLineString) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal
with indent.
""")";
''')";
constexpr char Expected[] = R"(A block string literal
with indent.
)";
@@ -153,10 +153,10 @@ TEST(ParseBlockStringLiteral, OkTwoLineString) {
}
TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) {
constexpr char Input[] = R"("""carbon
constexpr char Input[] = R"('''carbon
A block string literal
with file type indicator.
""")";
''')";
constexpr char Expected[] = R"(A block string literal
with file type indicator.
)";
@@ -164,16 +164,16 @@ TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) {
}
TEST(ParseBlockStringLiteral, OkWhitespaceAfterOpeningQuotes) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal
""")";
''')";
constexpr char Expected[] = R"(A block string literal
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal
with
@@ -181,7 +181,7 @@ TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
empty
lines.
""")";
''')";
constexpr char Expected[] = R"(A block string literal
with
@@ -194,37 +194,37 @@ TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
}
TEST(ParseBlockStringLiteral, OkWithSlashNewlineEscape) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal\
""")";
''')";
constexpr char Expected[] = "A block string literal";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithDoubleSlashNewline) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal\\
""")";
''')";
constexpr char Expected[] = R"(A block string literal\
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithTripleSlashNewline) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal\\\
""")";
''')";
constexpr char Expected[] = R"(A block string literal\)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkMultipleSlashes) {
constexpr char Input[] = R"("""
constexpr char Input[] = R"('''
A block string literal\
\
\
\
""")";
''')";
constexpr char Expected[] = "A block string literal";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
explorer/syntax/lexer.lpp
+53 -43
View File
@@ -308,29 +308,15 @@ operand_start [(A-Za-z0-9_\"]
return CARBON_ARG_TOKEN(integer_literal, val);
}
#*(\"\"\"|\") {
// Raw string literal.
// yytext (the token that matches the above regex) and chars scanned by
// str_lex_helper hold the source text, not the string the source represents.
#*\" {
// Found a double quote character.
Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context);
const std::string& s = str_lex_helper.str();
const int hashtag_num = s.find_first_of('"');
const int leading_quotes = s.size() - hashtag_num;
if (leading_quotes == 3 && hashtag_num > 0) {
// Check if it's a single-line string, like #"""#.
// TODO: Extend with other single-line string cases, like #""""#, based on
// the definition of block string in the design doc.
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
return Carbon::ProcessSingleLineString(str_lex_helper.str(), context,
hashtag_num);
} else if (str_lex_helper.is_eof()) {
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
} else if (!str_lex_helper.Advance()) {
if (!str_lex_helper.Advance()) {
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
// 3 quotes indicates multi-line, otherwise it'll be one.
const bool multi_line = leading_quotes == 3;
while (!str_lex_helper.is_eof()) {
switch (str_lex_helper.last_char()) {
@@ -338,36 +324,60 @@ operand_start [(A-Za-z0-9_\"]
case '\v': // Fall through.
case '\f': // Fall through.
case '\r':
if (!multi_line) {
return context.RecordSyntaxError(
llvm::formatv("missing closing quote in single-line string: {0}",
str_lex_helper.str()));
}
str_lex_helper.Advance();
return context.RecordSyntaxError(
llvm::formatv("missing closing quote in single-line string: {0}",
str_lex_helper.str()));
break;
case '"':
if (multi_line) {
// Check for 2 more '"'s on block string.
if (!(str_lex_helper.Advance() &&
str_lex_helper.last_char() == '"')) {
continue;
}
if (!(str_lex_helper.Advance() &&
str_lex_helper.last_char() == '"')) {
continue;
}
// Now we are at the last " of """.
}
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
// Reach closing quotes, break out of the loop.
if (leading_quotes == 3) {
return Carbon::ProcessMultiLineString(str_lex_helper.str(), context,
hashtag_num);
} else {
return Carbon::ProcessSingleLineString(str_lex_helper.str(),
context, hashtag_num);
return Carbon::ProcessSingleLineString(str_lex_helper.str(), context,
hashtag_num);
}
break;
case '\\':
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
// Read the escaped char.
if (!str_lex_helper.Advance()) {
continue;
}
// Read the next char.
str_lex_helper.Advance();
}
break;
default:
str_lex_helper.Advance();
}
}
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
#*\'\'\' {
// Multi-line string literal.
Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context);
const std::string& s = str_lex_helper.str();
const int hashtag_num = s.find_first_of('\'');
if (!str_lex_helper.Advance()) {
return CARBON_SIMPLE_TOKEN(END_OF_FILE);
}
while (!str_lex_helper.is_eof()) {
switch (str_lex_helper.last_char()) {
case '\'':
// Check for 2 more '\''s on block string.
if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) {
continue;
}
if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) {
continue;
}
// Now we are at the last ' of '''.
if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) {
// Reach closing quotes, break out of the loop.
return Carbon::ProcessMultiLineString(str_lex_helper.str(), context,
hashtag_num);
}
break;
case '\\':
explorer/testdata/string/block.carbon
Vendored
+2 -2
View File
@@ -10,10 +10,10 @@
package ExplorerTest api;
fn Main() -> i32 {
var s: String = """
var s: String = '''
A "block" ""string"" literal
with indent.
""";
''';
if (s == "A \"block\" \"\"string\"\" literal\n with indent.\n") {
return 0;
} else {
explorer/testdata/string/block_escaped_triple_quotes.carbon
Vendored
+4 -4
View File
@@ -10,11 +10,11 @@
package ExplorerTest api;
fn Main() -> i32 {
var s: String = """
var s: String = '''
A block string literal
\"""
""";
if (s == "A block string literal\n\"\"\"\n") {
\'''
''';
if (s == "A block string literal\n'''\n") {
return 0;
} else {
return 1;
explorer/testdata/string/block_file_type_indicator.carbon
Vendored
+2 -2
View File
@@ -10,10 +10,10 @@
package ExplorerTest api;
fn Main() -> i32 {
var s: String = """filetype
var s: String = '''filetype
A "block" ""string"" literal
with file type indicator.
""";
''';
if (s == "A \"block\" \"\"string\"\" literal\n with file type indicator.\n") {
return 0;
} else {
explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon
Vendored
+4 -4
View File
@@ -9,10 +9,10 @@
package ExplorerTest api;
fn Main() -> i32 {
// CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """
var s: String = """
error: closing """ is not on its own line.
""";
// CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing '''
var s: String = '''
error: closing ''' is not on its own line.
''';
return 0;
}
explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon
Vendored
+2 -2
View File
@@ -10,9 +10,9 @@ package ExplorerTest api;
fn Main() -> i32 {
// CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon:[[@LINE+1]]: Unexpected end of file
var s: String = ##"""
var s: String = ##'''
error: there are more #s on the left than the right.
"""#;
'''#;
return 0;
}
explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon
Vendored
+2 -2
View File
@@ -10,9 +10,9 @@ package ExplorerTest api;
fn Main() -> i32 {
// CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file.
var s: String = #"""
var s: String = #'''
error: there are more #s on the right than the left.
"""##;
'''##;
return 0;
}
explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon
Vendored
+4 -4
View File
@@ -9,10 +9,10 @@
package ExplorerTest api;
fn Main() -> i32 {
// CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """
var s: String = #"""
error: closing """# is not on its own line.
"""#;
// CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing '''
var s: String = #'''
error: closing '''# is not on its own line.
'''#;
return 0;
}
explorer/testdata/string/fail_raw_block_single_line.carbon
Vendored
+1 -1
View File
@@ -10,7 +10,7 @@ package ExplorerTest api;
fn CompareStr(s: String) -> i32 {
// CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_single_line.carbon:[[@LINE+1]]: Invalid block string: Too few lines
if (s == #"""raw string literal starting with """#) {
if (s == #'''raw string literal starting with '''#) {
return 0;
}
return 1;
explorer/testdata/string/raw_block.carbon
Vendored
+2 -2
View File
@@ -10,10 +10,10 @@
package ExplorerTest api;
fn Main() -> i32 {
var s: String = #"""
var s: String = #'''
A "block" ""string"" literal
with indent.
"""#;
'''#;
if (s == #"A \#"block\#" \#"\#"string\#"\#" literal\#n with indent.\#n"#) {
return 0;
} else {
explorer/testdata/string/raw_block_escaped_back_slash.carbon
Vendored
+2 -2
View File
@@ -10,10 +10,10 @@
package ExplorerTest api;
fn Main() -> i32 {
var s: String = #"""
var s: String = #'''
A block string literal
\
"""#;
'''#;
if (s == "A block string literal\n\\\n") {
return 0;
} else {
explorer/testdata/string/raw_block_escaped_triple_quotes.carbon
Vendored
+4 -4
View File
@@ -10,11 +10,11 @@
package ExplorerTest api;
fn Main() -> i32 {
var s: String = #"""
var s: String = #'''
A block string literal
\#"""#
"""#;
if (s == "A block string literal\n\"\"\"#\n") {
\#'''#
'''#;
if (s == "A block string literal\n'''#\n") {
return 0;
} else {
return 1;