From c87d271c531ebad5e4c0ea2da9cca0ea1ee172fa Mon Sep 17 00:00:00 2001 From: Priyananda Shenoy <112149799+priyans-google@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:53:47 -0800 Subject: [PATCH] Issue #2121: Support new block string literal syntax in explorer (#2399) --- common/string_helpers.cpp | 2 +- common/string_helpers_test.cpp | 56 +++++------ explorer/syntax/lexer.lpp | 96 ++++++++++--------- explorer/testdata/string/block.carbon | 4 +- .../string/block_escaped_triple_quotes.carbon | 8 +- .../string/block_file_type_indicator.carbon | 4 +- .../fail_block_quotes_not_on_own_line.carbon | 8 +- ...il_raw_block_more_hash_tags_on_left.carbon | 4 +- ...l_raw_block_more_hash_tags_on_right.carbon | 4 +- ...il_raw_block_quotes_not_on_own_line.carbon | 8 +- .../string/fail_raw_block_single_line.carbon | 2 +- explorer/testdata/string/raw_block.carbon | 4 +- .../raw_block_escaped_back_slash.carbon | 4 +- .../raw_block_escaped_triple_quotes.carbon | 8 +- 14 files changed, 111 insertions(+), 101 deletions(-) diff --git a/common/string_helpers.cpp b/common/string_helpers.cpp index 583c3dc7b19a..2bfbcb8fb7e3 100644 --- a/common/string_helpers.cpp +++ b/common/string_helpers.cpp @@ -14,7 +14,7 @@ namespace Carbon { -static constexpr llvm::StringRef TripleQuotes = R"(""")"; +static constexpr llvm::StringRef TripleQuotes = "'''"; static constexpr llvm::StringRef HorizontalWhitespaceChars = " \t"; // Carbon only takes uppercase hex input. diff --git a/common/string_helpers_test.cpp b/common/string_helpers_test.cpp index bcb715e3de23..50f210b8aa0c 100644 --- a/common/string_helpers_test.cpp +++ b/common/string_helpers_test.cpp @@ -95,57 +95,57 @@ TEST(ParseBlockStringLiteral, FailNoLeadingTripleQuotes) { } TEST(ParseBlockStringLiteral, FailInvalideFiletypeIndicator) { - EXPECT_THAT(ParseBlockStringLiteral("\"\"\"carbon file\n").error().message(), + EXPECT_THAT(ParseBlockStringLiteral("'''carbon file\n").error().message(), Eq("Invalid characters in file type indicator: carbon file")); } TEST(ParseBlockStringLiteral, FailEndingTripleQuotes) { - EXPECT_THAT(ParseBlockStringLiteral("\"\"\"\n").error().message(), + EXPECT_THAT(ParseBlockStringLiteral("'''\n").error().message(), Eq("Should end with triple quotes: ")); } TEST(ParseBlockStringLiteral, FailWrongIndent) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal with wrong indent - """)"; + ''')"; EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(), Eq("Wrong indent for line: with wrong indent, expected 5")); } TEST(ParseBlockStringLiteral, FailInvalidEscaping) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' \q - """)"; + ''')"; EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(), Eq("Invalid escaping in \\q")); - constexpr char InputRaw[] = R"(""" + constexpr char InputRaw[] = R"(''' \#q - """)"; + ''')"; EXPECT_THAT(ParseBlockStringLiteral(InputRaw, 1).error().message(), Eq("Invalid escaping in \\#q")); } TEST(ParseBlockStringLiteral, OkEmptyString) { - constexpr char Input[] = R"(""" -""")"; + constexpr char Input[] = R"(''' +''')"; EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq("")); } TEST(ParseBlockStringLiteral, OkOneLineString) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal - """)"; + ''')"; constexpr char Expected[] = R"(A block string literal )"; EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected)); } TEST(ParseBlockStringLiteral, OkTwoLineString) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal with indent. - """)"; + ''')"; constexpr char Expected[] = R"(A block string literal with indent. )"; @@ -153,10 +153,10 @@ TEST(ParseBlockStringLiteral, OkTwoLineString) { } TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) { - constexpr char Input[] = R"("""carbon + constexpr char Input[] = R"('''carbon A block string literal with file type indicator. - """)"; + ''')"; constexpr char Expected[] = R"(A block string literal with file type indicator. )"; @@ -164,16 +164,16 @@ TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) { } TEST(ParseBlockStringLiteral, OkWhitespaceAfterOpeningQuotes) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal - """)"; + ''')"; constexpr char Expected[] = R"(A block string literal )"; EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected)); } TEST(ParseBlockStringLiteral, OkWithEmptyLines) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal with @@ -181,7 +181,7 @@ TEST(ParseBlockStringLiteral, OkWithEmptyLines) { empty lines. - """)"; + ''')"; constexpr char Expected[] = R"(A block string literal with @@ -194,37 +194,37 @@ TEST(ParseBlockStringLiteral, OkWithEmptyLines) { } TEST(ParseBlockStringLiteral, OkWithSlashNewlineEscape) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal\ - """)"; + ''')"; constexpr char Expected[] = "A block string literal"; EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected)); } TEST(ParseBlockStringLiteral, OkWithDoubleSlashNewline) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal\\ - """)"; + ''')"; constexpr char Expected[] = R"(A block string literal\ )"; EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected)); } TEST(ParseBlockStringLiteral, OkWithTripleSlashNewline) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal\\\ - """)"; + ''')"; constexpr char Expected[] = R"(A block string literal\)"; EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected)); } TEST(ParseBlockStringLiteral, OkMultipleSlashes) { - constexpr char Input[] = R"(""" + constexpr char Input[] = R"(''' A block string literal\ \ \ \ - """)"; + ''')"; constexpr char Expected[] = "A block string literal"; EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected)); } diff --git a/explorer/syntax/lexer.lpp b/explorer/syntax/lexer.lpp index 25f7fd2e8477..96dd7c08d2e7 100644 --- a/explorer/syntax/lexer.lpp +++ b/explorer/syntax/lexer.lpp @@ -308,29 +308,15 @@ operand_start [(A-Za-z0-9_\"] return CARBON_ARG_TOKEN(integer_literal, val); } -#*(\"\"\"|\") { - // Raw string literal. - // yytext (the token that matches the above regex) and chars scanned by - // str_lex_helper hold the source text, not the string the source represents. +#*\" { + // Found a double quote character. Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context); const std::string& s = str_lex_helper.str(); const int hashtag_num = s.find_first_of('"'); - const int leading_quotes = s.size() - hashtag_num; - if (leading_quotes == 3 && hashtag_num > 0) { - // Check if it's a single-line string, like #"""#. - // TODO: Extend with other single-line string cases, like #""""#, based on - // the definition of block string in the design doc. - if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) { - return Carbon::ProcessSingleLineString(str_lex_helper.str(), context, - hashtag_num); - } else if (str_lex_helper.is_eof()) { - return CARBON_SIMPLE_TOKEN(END_OF_FILE); - } - } else if (!str_lex_helper.Advance()) { + + if (!str_lex_helper.Advance()) { return CARBON_SIMPLE_TOKEN(END_OF_FILE); } - // 3 quotes indicates multi-line, otherwise it'll be one. - const bool multi_line = leading_quotes == 3; while (!str_lex_helper.is_eof()) { switch (str_lex_helper.last_char()) { @@ -338,36 +324,60 @@ operand_start [(A-Za-z0-9_\"] case '\v': // Fall through. case '\f': // Fall through. case '\r': - if (!multi_line) { - return context.RecordSyntaxError( - llvm::formatv("missing closing quote in single-line string: {0}", - str_lex_helper.str())); - } - str_lex_helper.Advance(); + return context.RecordSyntaxError( + llvm::formatv("missing closing quote in single-line string: {0}", + str_lex_helper.str())); break; case '"': - if (multi_line) { - // Check for 2 more '"'s on block string. - if (!(str_lex_helper.Advance() && - str_lex_helper.last_char() == '"')) { - continue; - } - if (!(str_lex_helper.Advance() && - str_lex_helper.last_char() == '"')) { - continue; - } - // Now we are at the last " of """. - } - if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) { // Reach closing quotes, break out of the loop. - if (leading_quotes == 3) { - return Carbon::ProcessMultiLineString(str_lex_helper.str(), context, - hashtag_num); - } else { - return Carbon::ProcessSingleLineString(str_lex_helper.str(), - context, hashtag_num); + return Carbon::ProcessSingleLineString(str_lex_helper.str(), context, + hashtag_num); + } + break; + case '\\': + if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) { + // Read the escaped char. + if (!str_lex_helper.Advance()) { + continue; } + // Read the next char. + str_lex_helper.Advance(); + } + break; + default: + str_lex_helper.Advance(); + } + } + + return CARBON_SIMPLE_TOKEN(END_OF_FILE); +} + +#*\'\'\' { + // Multi-line string literal. + Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context); + const std::string& s = str_lex_helper.str(); + const int hashtag_num = s.find_first_of('\''); + + if (!str_lex_helper.Advance()) { + return CARBON_SIMPLE_TOKEN(END_OF_FILE); + } + + while (!str_lex_helper.is_eof()) { + switch (str_lex_helper.last_char()) { + case '\'': + // Check for 2 more '\''s on block string. + if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) { + continue; + } + if (!(str_lex_helper.Advance() && str_lex_helper.last_char() == '\'')) { + continue; + } + // Now we are at the last ' of '''. + if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) { + // Reach closing quotes, break out of the loop. + return Carbon::ProcessMultiLineString(str_lex_helper.str(), context, + hashtag_num); } break; case '\\': diff --git a/explorer/testdata/string/block.carbon b/explorer/testdata/string/block.carbon index 9c9211053cb6..79ec971dcfef 100644 --- a/explorer/testdata/string/block.carbon +++ b/explorer/testdata/string/block.carbon @@ -10,10 +10,10 @@ package ExplorerTest api; fn Main() -> i32 { - var s: String = """ + var s: String = ''' A "block" ""string"" literal with indent. - """; + '''; if (s == "A \"block\" \"\"string\"\" literal\n with indent.\n") { return 0; } else { diff --git a/explorer/testdata/string/block_escaped_triple_quotes.carbon b/explorer/testdata/string/block_escaped_triple_quotes.carbon index f3caeb0fcd31..327948aa47f9 100644 --- a/explorer/testdata/string/block_escaped_triple_quotes.carbon +++ b/explorer/testdata/string/block_escaped_triple_quotes.carbon @@ -10,11 +10,11 @@ package ExplorerTest api; fn Main() -> i32 { - var s: String = """ + var s: String = ''' A block string literal - \""" - """; - if (s == "A block string literal\n\"\"\"\n") { + \''' + '''; + if (s == "A block string literal\n'''\n") { return 0; } else { return 1; diff --git a/explorer/testdata/string/block_file_type_indicator.carbon b/explorer/testdata/string/block_file_type_indicator.carbon index 784679d47536..07ac92d339be 100644 --- a/explorer/testdata/string/block_file_type_indicator.carbon +++ b/explorer/testdata/string/block_file_type_indicator.carbon @@ -10,10 +10,10 @@ package ExplorerTest api; fn Main() -> i32 { - var s: String = """filetype + var s: String = '''filetype A "block" ""string"" literal with file type indicator. - """; + '''; if (s == "A \"block\" \"\"string\"\" literal\n with file type indicator.\n") { return 0; } else { diff --git a/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon b/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon index 392a441401e1..7835639c513a 100644 --- a/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon +++ b/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon @@ -9,10 +9,10 @@ package ExplorerTest api; fn Main() -> i32 { - // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """ - var s: String = """ - error: closing """ is not on its own line. - """; + // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing ''' + var s: String = ''' + error: closing ''' is not on its own line. + '''; return 0; } diff --git a/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon index 423562045047..ed6e3a524955 100644 --- a/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon +++ b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon @@ -10,9 +10,9 @@ package ExplorerTest api; fn Main() -> i32 { // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon:[[@LINE+1]]: Unexpected end of file - var s: String = ##""" + var s: String = ##''' error: there are more #s on the left than the right. - """#; + '''#; return 0; } diff --git a/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon index e038eb82b7a8..d1f608522a08 100644 --- a/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon +++ b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon @@ -10,9 +10,9 @@ package ExplorerTest api; fn Main() -> i32 { // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file. - var s: String = #""" + var s: String = #''' error: there are more #s on the right than the left. - """##; + '''##; return 0; } diff --git a/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon b/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon index 68befe9cce7e..572b0d3a4620 100644 --- a/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon +++ b/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon @@ -9,10 +9,10 @@ package ExplorerTest api; fn Main() -> i32 { - // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """ - var s: String = #""" - error: closing """# is not on its own line. - """#; + // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing ''' + var s: String = #''' + error: closing '''# is not on its own line. + '''#; return 0; } diff --git a/explorer/testdata/string/fail_raw_block_single_line.carbon b/explorer/testdata/string/fail_raw_block_single_line.carbon index 5a86763f024c..d61ead0294ff 100644 --- a/explorer/testdata/string/fail_raw_block_single_line.carbon +++ b/explorer/testdata/string/fail_raw_block_single_line.carbon @@ -10,7 +10,7 @@ package ExplorerTest api; fn CompareStr(s: String) -> i32 { // CHECK:STDERR: SYNTAX ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_single_line.carbon:[[@LINE+1]]: Invalid block string: Too few lines - if (s == #"""raw string literal starting with """#) { + if (s == #'''raw string literal starting with '''#) { return 0; } return 1; diff --git a/explorer/testdata/string/raw_block.carbon b/explorer/testdata/string/raw_block.carbon index cc9c1c8c41ba..c24a85dea9c2 100644 --- a/explorer/testdata/string/raw_block.carbon +++ b/explorer/testdata/string/raw_block.carbon @@ -10,10 +10,10 @@ package ExplorerTest api; fn Main() -> i32 { - var s: String = #""" + var s: String = #''' A "block" ""string"" literal with indent. - """#; + '''#; if (s == #"A \#"block\#" \#"\#"string\#"\#" literal\#n with indent.\#n"#) { return 0; } else { diff --git a/explorer/testdata/string/raw_block_escaped_back_slash.carbon b/explorer/testdata/string/raw_block_escaped_back_slash.carbon index b9025deef47e..da058481c254 100644 --- a/explorer/testdata/string/raw_block_escaped_back_slash.carbon +++ b/explorer/testdata/string/raw_block_escaped_back_slash.carbon @@ -10,10 +10,10 @@ package ExplorerTest api; fn Main() -> i32 { - var s: String = #""" + var s: String = #''' A block string literal \ - """#; + '''#; if (s == "A block string literal\n\\\n") { return 0; } else { diff --git a/explorer/testdata/string/raw_block_escaped_triple_quotes.carbon b/explorer/testdata/string/raw_block_escaped_triple_quotes.carbon index ca29608555bd..0079abfef306 100644 --- a/explorer/testdata/string/raw_block_escaped_triple_quotes.carbon +++ b/explorer/testdata/string/raw_block_escaped_triple_quotes.carbon @@ -10,11 +10,11 @@ package ExplorerTest api; fn Main() -> i32 { - var s: String = #""" + var s: String = #''' A block string literal - \#"""# - """#; - if (s == "A block string literal\n\"\"\"#\n") { + \#'''# + '''#; + if (s == "A block string literal\n'''#\n") { return 0; } else { return 1;