From 8d0f3364d8f91510192b4a2bddc5535bae8fd447 Mon Sep 17 00:00:00 2001 From: SlaterLatiao <41390071+SlaterLatiao@users.noreply.github.com> Date: Wed, 22 Jun 2022 15:38:47 -0700 Subject: [PATCH] Initial implementation of raw string literals (#1304) * test cases for raw string literals * raw string literal implementation * match as block string if starting with triple ", and better error message for simple string except for *#"""#* * fix broken test case block string literal cannot be one line * test cases for raw string literals * raw string literal implementation * match as block string if starting with triple ", and better error message for simple string except for *#"""#* * fix broken test case block string literal cannot be one line * removed unused initial value * rename flag to indicate multi-line string and remove comment * use * to get value from std::optional * clean-ups * removed skip_scan flag and directly return in case of a single line string starting with #+\'\'\' * Updated error message: simple string -> single-line string. Co-authored-by: josh11b * Updated test cases according to changes in error message * Removed counting_hashtag flag. * Implemented ScanHelper class to handle scanning * Fixed explanation of ReadHashTags. * Addressed PR comment. * Clarify that scan_helper holds the source text. * Addressed PR comments. * Updated error messages in test cases. * Added const keyword to return type of GetCurrentStr(). * addressed PR comments. 1. Moved ScanHelper class to lex_scan_helper.h and lex_scan_helper.cpp. 2. Moved ReadHashTags and Process* functions to lex_scan_helper.cpp. Moved YY_USER_ACTION, SIMPLE_TOKEN and ARG_TOKEN to lex_helper.h. Added a wrapper function YyinputWrapper to call static function yyinput in lexer.lpp. 3. Renamed ScanHelper with StringLexHelper. 4. Modified BUILD accordingly. 5. Renamed data members and functions. * Addressed PR comments. 1. Adjusted order to keep ret usage close. 2. Used resize to construct the string to avoid creation of temp string. Co-authored-by: Jon Ross-Perkins * Removed the multi_line flag and skip_read field to improve readability. * Copied default parameter value to definition of UnescapeStringLiteral. Co-authored-by: Jon Ross-Perkins * Copied default parameter value to definition of ParseBlockStringLiteral. Co-authored-by: Jon Ross-Perkins * Prefix CARBON_ to SIMPLE_TOKEN and ARG_TOKEN macros. * Rollback redefinition of arguments. * Updated comment on the flex macro. Co-authored-by: Jon Ross-Perkins * Updated wording. Co-authored-by: Jon Ross-Perkins * Moved the EOF error out of the loop. * Removed duplicated declaration. * Changed type of `hashtag_num` and `leading_quotes` to int. * Minor fix: string copy. Co-authored-by: Jon Ross-Perkins * Added comment on YyinputWrapper. Co-authored-by: Jon Ross-Perkins * Garmmar in comment. Co-authored-by: Jon Ross-Perkins * Added check of eof before readling next char. * Minor updates based on PR comments. * Minor changes to address PR comments. * Used a clearer way to calculate `hashtag_num` and `leading_quotes`. Switched back to indicate muti-line string with a flag. * Directly copy StringRef for compilation error message. * Make str_with_quote const as we don't change it. Co-authored-by: josh11b * Added TODO for unsupported cases. * Fixed a typo. Co-authored-by: Jon Ross-Perkins Co-authored-by: josh11b Co-authored-by: Jon Ross-Perkins --- common/string_helpers.cpp | 142 ++++---- common/string_helpers.h | 6 +- common/string_helpers_test.cpp | 8 + explorer/syntax/BUILD | 3 + explorer/syntax/lex_helper.h | 25 ++ explorer/syntax/lex_scan_helper.cpp | 68 ++++ explorer/syntax/lex_scan_helper.h | 58 ++++ explorer/syntax/lexer.lpp | 311 +++++++++--------- .../string/block_file_type_indicator.carbon | 24 ++ explorer/testdata/string/fail_newline.carbon | 3 +- ...il_raw_block_more_hash_tags_on_left.carbon | 20 ++ ...l_raw_block_more_hash_tags_on_right.carbon | 20 ++ ...il_raw_block_quotes_not_on_own_line.carbon | 20 ++ .../string/fail_raw_block_single_line.carbon | 23 ++ .../fail_raw_more_hash_tags_on_left.carbon | 24 ++ .../fail_raw_more_hash_tags_on_right.carbon | 23 ++ explorer/testdata/string/raw_basic.carbon | 23 ++ .../string/raw_basic_multi_hashtag.carbon | 23 ++ explorer/testdata/string/raw_block.carbon | 24 ++ .../raw_block_escaped_back_slash.carbon | 24 ++ .../raw_block_escaped_triple_quotes.carbon | 24 ++ explorer/testdata/string/raw_nesting.carbon | 23 ++ 22 files changed, 683 insertions(+), 236 deletions(-) create mode 100644 explorer/syntax/lex_helper.h create mode 100644 explorer/syntax/lex_scan_helper.cpp create mode 100644 explorer/syntax/lex_scan_helper.h create mode 100644 explorer/testdata/string/block_file_type_indicator.carbon create mode 100644 explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon create mode 100644 explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon create mode 100644 explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon create mode 100644 explorer/testdata/string/fail_raw_block_single_line.carbon create mode 100644 explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon create mode 100644 explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon create mode 100644 explorer/testdata/string/raw_basic.carbon create mode 100644 explorer/testdata/string/raw_basic_multi_hashtag.carbon create mode 100644 explorer/testdata/string/raw_block.carbon create mode 100644 explorer/testdata/string/raw_block_escaped_back_slash.carbon create mode 100644 explorer/testdata/string/raw_block_escaped_triple_quotes.carbon create mode 100644 explorer/testdata/string/raw_nesting.carbon diff --git a/common/string_helpers.cpp b/common/string_helpers.cpp index cb010dcee2b9..15f1b3a2357b 100644 --- a/common/string_helpers.cpp +++ b/common/string_helpers.cpp @@ -27,87 +27,86 @@ static auto FromHex(char c) -> std::optional { return std::nullopt; } -auto UnescapeStringLiteral(llvm::StringRef source, bool is_block_string) - -> std::optional { +auto UnescapeStringLiteral(llvm::StringRef source, const int hashtag_num, + bool is_block_string) -> std::optional { std::string ret; ret.reserve(source.size()); + std::string escape = "\\"; + escape.resize(hashtag_num + 1, '#'); size_t i = 0; while (i < source.size()) { char c = source[i]; - switch (c) { - case '\\': - ++i; - if (i == source.size()) { - return std::nullopt; - } - switch (source[i]) { - case 'n': - ret.push_back('\n'); - break; - case 'r': - ret.push_back('\r'); - break; - case 't': - ret.push_back('\t'); - break; - case '0': - if (i + 1 < source.size() && llvm::isDigit(source[i + 1])) { - // \0[0-9] is reserved. - return std::nullopt; - } - ret.push_back('\0'); - break; - case '"': - ret.push_back('"'); - break; - case '\'': - ret.push_back('\''); - break; - case '\\': - ret.push_back('\\'); - break; - case 'x': { - i += 2; - if (i >= source.size()) { - return std::nullopt; - } - std::optional c1 = FromHex(source[i - 1]); - std::optional c2 = FromHex(source[i]); - if (c1 == std::nullopt || c2 == std::nullopt) { - return std::nullopt; - } - ret.push_back(16 * *c1 + *c2); - break; - } - case 'u': - CARBON_FATAL() << "\\u is not yet supported in string literals"; - case '\n': - if (!is_block_string) { - return std::nullopt; - } - break; - default: - // Unsupported. - return std::nullopt; - } - break; - - case '\t': - // Disallow non-` ` horizontal whitespace: - // https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/whitespace.md - // TODO: This doesn't handle unicode whitespace. + if (i + hashtag_num < source.size() && + source.slice(i, i + hashtag_num + 1).equals(escape)) { + i += hashtag_num + 1; + if (i == source.size()) { return std::nullopt; - - default: - ret.push_back(c); - break; + } + switch (source[i]) { + case 'n': + ret.push_back('\n'); + break; + case 'r': + ret.push_back('\r'); + break; + case 't': + ret.push_back('\t'); + break; + case '0': + if (i + 1 < source.size() && llvm::isDigit(source[i + 1])) { + // \0[0-9] is reserved. + return std::nullopt; + } + ret.push_back('\0'); + break; + case '"': + ret.push_back('"'); + break; + case '\'': + ret.push_back('\''); + break; + case '\\': + ret.push_back('\\'); + break; + case 'x': { + i += 2; + if (i >= source.size()) { + return std::nullopt; + } + std::optional c1 = FromHex(source[i - 1]); + std::optional c2 = FromHex(source[i]); + if (c1 == std::nullopt || c2 == std::nullopt) { + return std::nullopt; + } + ret.push_back(16 * *c1 + *c2); + break; + } + case 'u': + CARBON_FATAL() << "\\u is not yet supported in string literals"; + case '\n': + if (!is_block_string) { + return std::nullopt; + } + break; + default: + // Unsupported. + return std::nullopt; + } + } else if (c == '\t') { + // Disallow non-` ` horizontal whitespace: + // https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/whitespace.md + // TODO: This doesn't handle unicode whitespace. + return std::nullopt; + } else { + ret.push_back(c); } ++i; } return ret; } -auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr { +auto ParseBlockStringLiteral(llvm::StringRef source, const int hashtag_num) + -> ErrorOr { llvm::SmallVector lines; source.split(lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/true); if (lines.size() < 2) { @@ -150,8 +149,9 @@ auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr { } // Unescaping with \n appended to handle things like \\. llvm::SmallVector buffer; - std::optional unescaped = UnescapeStringLiteral( - (line + "\n").toStringRef(buffer), /*is_block_string=*/true); + std::optional unescaped = + UnescapeStringLiteral((line + "\n").toStringRef(buffer), hashtag_num, + /*is_block_string=*/true); if (!unescaped.has_value()) { return Error("Invalid escaping in " + line); } diff --git a/common/string_helpers.h b/common/string_helpers.h index c3b658d7a546..cbe9e2f3b724 100644 --- a/common/string_helpers.h +++ b/common/string_helpers.h @@ -19,11 +19,13 @@ namespace Carbon { // Unescapes Carbon escape sequences in the source string. Returns std::nullopt // on bad input. `is_block_string` enables escaping unique to block string // literals, such as \. -auto UnescapeStringLiteral(llvm::StringRef source, bool is_block_string = false) +auto UnescapeStringLiteral(llvm::StringRef source, int hashtag_num = 0, + bool is_block_string = false) -> std::optional; // Parses a block string literal in `source`. -auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr; +auto ParseBlockStringLiteral(llvm::StringRef source, int hashtag_num = 0) + -> ErrorOr; // Returns true if the pointer is in the string ref (including equality with // `ref.end()`). This should be used instead of `<=` comparisons for diff --git a/common/string_helpers_test.cpp b/common/string_helpers_test.cpp index 73a1fc36d78c..c7251528cf57 100644 --- a/common/string_helpers_test.cpp +++ b/common/string_helpers_test.cpp @@ -28,6 +28,8 @@ TEST(UnescapeStringLiteral, Valid) { EXPECT_THAT(UnescapeStringLiteral("test\\\\n"), Optional(Eq("test\\n"))); EXPECT_THAT(UnescapeStringLiteral("\\xAA"), Optional(Eq("\xAA"))); EXPECT_THAT(UnescapeStringLiteral("\\x12"), Optional(Eq("\x12"))); + EXPECT_THAT(UnescapeStringLiteral("test", 1), Optional(Eq("test"))); + EXPECT_THAT(UnescapeStringLiteral("test\\#n", 1), Optional(Eq("test\n"))); } TEST(UnescapeStringLiteral, Invalid) { @@ -43,6 +45,7 @@ TEST(UnescapeStringLiteral, Invalid) { EXPECT_THAT(UnescapeStringLiteral("\\xaa"), Eq(std::nullopt)); // Reserved. EXPECT_THAT(UnescapeStringLiteral("\\00"), Eq(std::nullopt)); + EXPECT_THAT(UnescapeStringLiteral("\\#00", 1), Eq(std::nullopt)); } TEST(UnescapeStringLiteral, Nul) { @@ -90,6 +93,11 @@ TEST(ParseBlockStringLiteral, FailInvalidEscaping) { """)"; EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(), Eq("Invalid escaping in \\q")); + constexpr char InputRaw[] = R"(""" + \#q + """)"; + EXPECT_THAT(ParseBlockStringLiteral(InputRaw, 1).error().message(), + Eq("Invalid escaping in \\#q")); } TEST(ParseBlockStringLiteral, OkEmptyString) { diff --git a/explorer/syntax/BUILD b/explorer/syntax/BUILD index 4c868ff0fefe..327b6470bc9b 100644 --- a/explorer/syntax/BUILD +++ b/explorer/syntax/BUILD @@ -53,6 +53,9 @@ cc_library( cc_library( name = "syntax", srcs = [ + "lex_helper.h", + "lex_scan_helper.cpp", + "lex_scan_helper.h", "lexer.cpp", "lexer.h", "parse.cpp", diff --git a/explorer/syntax/lex_helper.h b/explorer/syntax/lex_helper.h new file mode 100644 index 000000000000..c81f59149e52 --- /dev/null +++ b/explorer/syntax/lex_helper.h @@ -0,0 +1,25 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_ +#define CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_ + +// Flex expands this macro immediately before each action. +// +// Advances the current token position by yyleng columns without changing +// the line number, and takes us out of the after-whitespace / after-operand +// state. +#define YY_USER_ACTION \ + context.current_token_position.columns(yyleng); \ + if (YY_START == AFTER_WHITESPACE || YY_START == AFTER_OPERAND) { \ + BEGIN(INITIAL); \ + } + +#define CARBON_SIMPLE_TOKEN(name) \ + Carbon::Parser::make_##name(context.current_token_position); + +#define CARBON_ARG_TOKEN(name, arg) \ + Carbon::Parser::make_##name(arg, context.current_token_position); + +#endif // CARBON_EXPLORER_SYNTAX_LEX_HELPER_H_ diff --git a/explorer/syntax/lex_scan_helper.cpp b/explorer/syntax/lex_scan_helper.cpp new file mode 100644 index 000000000000..a33440e52055 --- /dev/null +++ b/explorer/syntax/lex_scan_helper.cpp @@ -0,0 +1,68 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "explorer/syntax/lex_scan_helper.h" + +#include "common/string_helpers.h" +#include "explorer/syntax/lex_helper.h" +#include "llvm/Support/FormatVariadic.h" + +namespace Carbon { + +auto StringLexHelper::Advance() -> bool { + CARBON_CHECK(is_eof_ == false); + const char c = YyinputWrapper(yyscanner_); + if (c <= 0) { + context_.RecordSyntaxError("Unexpected end of file"); + is_eof_ = true; + return false; + } + str_.push_back(c); + return true; +} + +auto ReadHashTags(Carbon::StringLexHelper& scan_helper, + const size_t hashtag_num) -> bool { + for (size_t i = 0; i < hashtag_num; ++i) { + if (!scan_helper.Advance() || scan_helper.last_char() != '#') { + return false; + } + } + return true; +} + +auto ProcessSingleLineString(llvm::StringRef str, + Carbon::ParseAndLexContext& context, + const size_t hashtag_num) + -> Carbon::Parser::symbol_type { + std::string hashtags(hashtag_num, '#'); + const auto str_with_quote = str; + CARBON_CHECK(str.consume_front(hashtags + "\"") && + str.consume_back("\"" + hashtags)); + + std::optional unescaped = + Carbon::UnescapeStringLiteral(str, hashtag_num); + if (unescaped == std::nullopt) { + return context.RecordSyntaxError( + llvm::formatv("Invalid escaping in string: {0}", str_with_quote)); + } + return CARBON_ARG_TOKEN(string_literal, *unescaped); +} + +auto ProcessMultiLineString(llvm::StringRef str, + Carbon::ParseAndLexContext& context, + const size_t hashtag_num) + -> Carbon::Parser::symbol_type { + std::string hashtags(hashtag_num, '#'); + CARBON_CHECK(str.consume_front(hashtags) && str.consume_back(hashtags)); + Carbon::ErrorOr block_string = + Carbon::ParseBlockStringLiteral(str, hashtag_num); + if (!block_string.ok()) { + return context.RecordSyntaxError(llvm::formatv( + "Invalid block string: {0}", block_string.error().message())); + } + return CARBON_ARG_TOKEN(string_literal, *block_string); +} + +} // namespace Carbon diff --git a/explorer/syntax/lex_scan_helper.h b/explorer/syntax/lex_scan_helper.h new file mode 100644 index 000000000000..8c7b4f597089 --- /dev/null +++ b/explorer/syntax/lex_scan_helper.h @@ -0,0 +1,58 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_ +#define CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_ + +#include + +#include "explorer/syntax/parse_and_lex_context.h" +#include "explorer/syntax/parser.h" + +// Exposes yyinput; defined in lexer.lpp. +extern auto YyinputWrapper(yyscan_t yyscanner) -> int; + +namespace Carbon { + +class StringLexHelper { + public: + StringLexHelper(const char* text, yyscan_t yyscanner, + Carbon::ParseAndLexContext& context) + : str_(text), yyscanner_(yyscanner), context_(context), is_eof_(false) {} + // Advances yyscanner by one char. Sets is_eof to true and returns false on + // EOF. + auto Advance() -> bool; + // Returns the last scanned char. + auto last_char() -> char { return str_.back(); }; + // Returns the scanned string. + auto str() -> const std::string& { return str_; }; + + auto is_eof() -> bool { return is_eof_; }; + + private: + std::string str_; + yyscan_t yyscanner_; + Carbon::ParseAndLexContext& context_; + // Skips reading next char. + bool is_eof_; +}; + +// Tries to Read `hashtag_num` hashtags. Returns true on success. +// Reads `hashtag_num` characters on success, and number of consecutive hashtags +// (< `hashtag_num`) + 1 characters on failure. +auto ReadHashTags(Carbon::StringLexHelper& scan_helper, size_t hashtag_num) + -> bool; + +// Removes quotes and escapes a single line string. Reports an error on +// invalid escaping. +auto ProcessSingleLineString(llvm::StringRef str, + Carbon::ParseAndLexContext& context, + size_t hashtag_num) -> Carbon::Parser::symbol_type; +auto ProcessMultiLineString(llvm::StringRef str, + Carbon::ParseAndLexContext& context, + size_t hashtag_num) -> Carbon::Parser::symbol_type; + +} // namespace Carbon + +#endif // CARBON_EXPLORER_SYNTAX_LEX_SCAN_HELPER_H_ diff --git a/explorer/syntax/lexer.lpp b/explorer/syntax/lexer.lpp index f4e93ae024f9..5451f496f927 100644 --- a/explorer/syntax/lexer.lpp +++ b/explorer/syntax/lexer.lpp @@ -9,14 +9,12 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "common/check.h" #include "common/error.h" - #include "common/string_helpers.h" + #include "explorer/syntax/lex_helper.h" + #include "explorer/syntax/lex_scan_helper.h" #include "explorer/syntax/parse_and_lex_context.h" #include "explorer/syntax/parser.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/FormatVariadic.h" - - // Reads and returns a single character. Reports an error on EOF. - auto ReadChar(yyscan_t yyscanner, Carbon::ParseAndLexContext& context) -> int; %} /* Turn off legacy bits we don't need. */ @@ -109,28 +107,6 @@ whitespace [ \t\r\n] one_line_comment \/\/[^\n]*\n operand_start [(A-Za-z0-9_\"] -/* Single-line string literals should reject vertical whitespace. */ -string_literal \"([^\\\"\n\v\f\r]|\\.)*\" - -%{ - // This macro is expanded immediately before each action specified below. - // - // Advances the current token position by yyleng columns without changing - // the line number, and takes us out of the after-whitespace / after-operand - // state. - #define YY_USER_ACTION \ - context.current_token_position.columns(yyleng); \ - if (YY_START == AFTER_WHITESPACE || YY_START == AFTER_OPERAND) { \ - BEGIN(INITIAL); \ - } - - #define SIMPLE_TOKEN(name) \ - Carbon::Parser::make_##name(context.current_token_position); - - #define ARG_TOKEN(name, arg) \ - Carbon::Parser::make_##name(arg, context.current_token_position); -%} - %% %{ @@ -141,83 +117,83 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\" %} /* table-begin */ -{ADDR} { return SIMPLE_TOKEN(ADDR); } -{ALIAS} { return SIMPLE_TOKEN(ALIAS); } -{AMPERSAND} { return SIMPLE_TOKEN(AMPERSAND); } -{AND} { return SIMPLE_TOKEN(AND); } -{API} { return SIMPLE_TOKEN(API); } -{ARROW} { return SIMPLE_TOKEN(ARROW); } -{AS} { return SIMPLE_TOKEN(AS); } -{AUTO} { return SIMPLE_TOKEN(AUTO); } -{AWAIT} { return SIMPLE_TOKEN(AWAIT); } -{BOOL} { return SIMPLE_TOKEN(BOOL); } -{BREAK} { return SIMPLE_TOKEN(BREAK); } -{CASE} { return SIMPLE_TOKEN(CASE); } -{CHOICE} { return SIMPLE_TOKEN(CHOICE); } -{CLASS} { return SIMPLE_TOKEN(CLASS); } -{COLON_BANG} { return SIMPLE_TOKEN(COLON_BANG); } -{COLON} { return SIMPLE_TOKEN(COLON); } -{COMMA} { return SIMPLE_TOKEN(COMMA); } -{CONTINUATION_TYPE} { return SIMPLE_TOKEN(CONTINUATION_TYPE); } -{CONTINUATION} { return SIMPLE_TOKEN(CONTINUATION); } -{CONTINUE} { return SIMPLE_TOKEN(CONTINUE); } -{DEFAULT} { return SIMPLE_TOKEN(DEFAULT); } -{DOUBLE_ARROW} { return SIMPLE_TOKEN(DOUBLE_ARROW); } -{ELSE} { return SIMPLE_TOKEN(ELSE); } -{EQUAL_EQUAL} { return SIMPLE_TOKEN(EQUAL_EQUAL); } -{EQUAL} { return SIMPLE_TOKEN(EQUAL); } -{EXTERNAL} { return SIMPLE_TOKEN(EXTERNAL); } -{FALSE} { return SIMPLE_TOKEN(FALSE); } -{FN_TYPE} { return SIMPLE_TOKEN(FN_TYPE); } -{FN} { return SIMPLE_TOKEN(FN); } -{FORALL} { return SIMPLE_TOKEN(FORALL); } -{IF} { return SIMPLE_TOKEN(IF); } -{IMPL} { return SIMPLE_TOKEN(IMPL); } -{IMPORT} { return SIMPLE_TOKEN(IMPORT); } -{INTERFACE} { return SIMPLE_TOKEN(INTERFACE); } -{IS} { return SIMPLE_TOKEN(IS); } -{LEFT_CURLY_BRACE} { return SIMPLE_TOKEN(LEFT_CURLY_BRACE); } -{LEFT_PARENTHESIS} { return SIMPLE_TOKEN(LEFT_PARENTHESIS); } -{LEFT_SQUARE_BRACKET} { return SIMPLE_TOKEN(LEFT_SQUARE_BRACKET); } -{LET} { return SIMPLE_TOKEN(LET); } -{LIBRARY} { return SIMPLE_TOKEN(LIBRARY); } -{MATCH} { return SIMPLE_TOKEN(MATCH); } -{MINUS} { return SIMPLE_TOKEN(MINUS); } -{NOT} { return SIMPLE_TOKEN(NOT); } -{OR} { return SIMPLE_TOKEN(OR); } -{PACKAGE} { return SIMPLE_TOKEN(PACKAGE); } -{PERIOD} { return SIMPLE_TOKEN(PERIOD); } -{PLUS} { return SIMPLE_TOKEN(PLUS); } -{RETURN} { return SIMPLE_TOKEN(RETURN); } -{RUN} { return SIMPLE_TOKEN(RUN); } -{SELF} { return SIMPLE_TOKEN(SELF); } -{SEMICOLON} { return SIMPLE_TOKEN(SEMICOLON); } -{SLASH} { return SIMPLE_TOKEN(SLASH); } -{STRING} { return SIMPLE_TOKEN(STRING); } -{THEN} { return SIMPLE_TOKEN(THEN); } -{TRUE} { return SIMPLE_TOKEN(TRUE); } -{TYPE} { return SIMPLE_TOKEN(TYPE); } -{UNDERSCORE} { return SIMPLE_TOKEN(UNDERSCORE); } -{UNIMPL_EXAMPLE} { return SIMPLE_TOKEN(UNIMPL_EXAMPLE); } -{VAR} { return SIMPLE_TOKEN(VAR); } -{WHERE} { return SIMPLE_TOKEN(WHERE); } -{WHILE} { return SIMPLE_TOKEN(WHILE); } +{ADDR} { return CARBON_SIMPLE_TOKEN(ADDR); } +{ALIAS} { return CARBON_SIMPLE_TOKEN(ALIAS); } +{AMPERSAND} { return CARBON_SIMPLE_TOKEN(AMPERSAND); } +{AND} { return CARBON_SIMPLE_TOKEN(AND); } +{API} { return CARBON_SIMPLE_TOKEN(API); } +{ARROW} { return CARBON_SIMPLE_TOKEN(ARROW); } +{AS} { return CARBON_SIMPLE_TOKEN(AS); } +{AUTO} { return CARBON_SIMPLE_TOKEN(AUTO); } +{AWAIT} { return CARBON_SIMPLE_TOKEN(AWAIT); } +{BOOL} { return CARBON_SIMPLE_TOKEN(BOOL); } +{BREAK} { return CARBON_SIMPLE_TOKEN(BREAK); } +{CASE} { return CARBON_SIMPLE_TOKEN(CASE); } +{CHOICE} { return CARBON_SIMPLE_TOKEN(CHOICE); } +{CLASS} { return CARBON_SIMPLE_TOKEN(CLASS); } +{COLON_BANG} { return CARBON_SIMPLE_TOKEN(COLON_BANG); } +{COLON} { return CARBON_SIMPLE_TOKEN(COLON); } +{COMMA} { return CARBON_SIMPLE_TOKEN(COMMA); } +{CONTINUATION_TYPE} { return CARBON_SIMPLE_TOKEN(CONTINUATION_TYPE); } +{CONTINUATION} { return CARBON_SIMPLE_TOKEN(CONTINUATION); } +{CONTINUE} { return CARBON_SIMPLE_TOKEN(CONTINUE); } +{DEFAULT} { return CARBON_SIMPLE_TOKEN(DEFAULT); } +{DOUBLE_ARROW} { return CARBON_SIMPLE_TOKEN(DOUBLE_ARROW); } +{ELSE} { return CARBON_SIMPLE_TOKEN(ELSE); } +{EQUAL_EQUAL} { return CARBON_SIMPLE_TOKEN(EQUAL_EQUAL); } +{EQUAL} { return CARBON_SIMPLE_TOKEN(EQUAL); } +{EXTERNAL} { return CARBON_SIMPLE_TOKEN(EXTERNAL); } +{FALSE} { return CARBON_SIMPLE_TOKEN(FALSE); } +{FN_TYPE} { return CARBON_SIMPLE_TOKEN(FN_TYPE); } +{FN} { return CARBON_SIMPLE_TOKEN(FN); } +{FORALL} { return CARBON_SIMPLE_TOKEN(FORALL); } +{IF} { return CARBON_SIMPLE_TOKEN(IF); } +{IMPL} { return CARBON_SIMPLE_TOKEN(IMPL); } +{IMPORT} { return CARBON_SIMPLE_TOKEN(IMPORT); } +{INTERFACE} { return CARBON_SIMPLE_TOKEN(INTERFACE); } +{IS} { return CARBON_SIMPLE_TOKEN(IS); } +{LEFT_CURLY_BRACE} { return CARBON_SIMPLE_TOKEN(LEFT_CURLY_BRACE); } +{LEFT_PARENTHESIS} { return CARBON_SIMPLE_TOKEN(LEFT_PARENTHESIS); } +{LEFT_SQUARE_BRACKET} { return CARBON_SIMPLE_TOKEN(LEFT_SQUARE_BRACKET); } +{LET} { return CARBON_SIMPLE_TOKEN(LET); } +{LIBRARY} { return CARBON_SIMPLE_TOKEN(LIBRARY); } +{MATCH} { return CARBON_SIMPLE_TOKEN(MATCH); } +{MINUS} { return CARBON_SIMPLE_TOKEN(MINUS); } +{NOT} { return CARBON_SIMPLE_TOKEN(NOT); } +{OR} { return CARBON_SIMPLE_TOKEN(OR); } +{PACKAGE} { return CARBON_SIMPLE_TOKEN(PACKAGE); } +{PERIOD} { return CARBON_SIMPLE_TOKEN(PERIOD); } +{PLUS} { return CARBON_SIMPLE_TOKEN(PLUS); } +{RETURN} { return CARBON_SIMPLE_TOKEN(RETURN); } +{RUN} { return CARBON_SIMPLE_TOKEN(RUN); } +{SELF} { return CARBON_SIMPLE_TOKEN(SELF); } +{SEMICOLON} { return CARBON_SIMPLE_TOKEN(SEMICOLON); } +{SLASH} { return CARBON_SIMPLE_TOKEN(SLASH); } +{STRING} { return CARBON_SIMPLE_TOKEN(STRING); } +{THEN} { return CARBON_SIMPLE_TOKEN(THEN); } +{TRUE} { return CARBON_SIMPLE_TOKEN(TRUE); } +{TYPE} { return CARBON_SIMPLE_TOKEN(TYPE); } +{UNDERSCORE} { return CARBON_SIMPLE_TOKEN(UNDERSCORE); } +{UNIMPL_EXAMPLE} { return CARBON_SIMPLE_TOKEN(UNIMPL_EXAMPLE); } +{VAR} { return CARBON_SIMPLE_TOKEN(VAR); } +{WHERE} { return CARBON_SIMPLE_TOKEN(WHERE); } +{WHILE} { return CARBON_SIMPLE_TOKEN(WHILE); } /* table-end */ /* More modern Bisons provide make_EOF. */ -<> { return SIMPLE_TOKEN(END_OF_FILE); } +<> { return CARBON_SIMPLE_TOKEN(END_OF_FILE); } {RIGHT_PARENTHESIS} { BEGIN(AFTER_OPERAND); - return SIMPLE_TOKEN(RIGHT_PARENTHESIS); + return CARBON_SIMPLE_TOKEN(RIGHT_PARENTHESIS); } {RIGHT_CURLY_BRACE} { BEGIN(AFTER_OPERAND); - return SIMPLE_TOKEN(RIGHT_CURLY_BRACE); + return CARBON_SIMPLE_TOKEN(RIGHT_CURLY_BRACE); } {RIGHT_SQUARE_BRACKET} { BEGIN(AFTER_OPERAND); - return SIMPLE_TOKEN(RIGHT_SQUARE_BRACKET); + return CARBON_SIMPLE_TOKEN(RIGHT_SQUARE_BRACKET); } /* @@ -244,23 +220,23 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\" /* `*` operator case 1: */ "*"{whitespace}+ { BEGIN(AFTER_WHITESPACE); - return SIMPLE_TOKEN(BINARY_STAR); + return CARBON_SIMPLE_TOKEN(BINARY_STAR); } /* `*` operator case 2: */ -"*"/{operand_start} { return SIMPLE_TOKEN(BINARY_STAR); } +"*"/{operand_start} { return CARBON_SIMPLE_TOKEN(BINARY_STAR); } /* `*` operator case 3: */ -"*" { return SIMPLE_TOKEN(PREFIX_STAR); } +"*" { return CARBON_SIMPLE_TOKEN(PREFIX_STAR); } /* `*` operator case 4: */ "*"{whitespace}+ { BEGIN(AFTER_WHITESPACE); - return SIMPLE_TOKEN(POSTFIX_STAR); + return CARBON_SIMPLE_TOKEN(POSTFIX_STAR); } /* `*` operator case 5: */ -"*" { return SIMPLE_TOKEN(UNARY_STAR); } +"*" { return CARBON_SIMPLE_TOKEN(UNARY_STAR); } {sized_type_literal} { BEGIN(AFTER_OPERAND); - return ARG_TOKEN(sized_type_literal, yytext); + return CARBON_ARG_TOKEN(sized_type_literal, yytext); } {intrinsic_identifier} { @@ -268,7 +244,7 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\" Carbon::ErrorOr intrinsic = Carbon::IntrinsicExpression::FindIntrinsic(yytext, context.source_loc()); if (intrinsic.ok()) { - return ARG_TOKEN(intrinsic_identifier, *intrinsic); + return CARBON_ARG_TOKEN(intrinsic_identifier, *intrinsic); } else { return context.RecordSyntaxError(intrinsic.error().message()); } @@ -276,7 +252,7 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\" {identifier} { BEGIN(AFTER_OPERAND); - return ARG_TOKEN(identifier, yytext); + return CARBON_ARG_TOKEN(identifier, yytext); } {integer_literal} { @@ -286,69 +262,86 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\" return context.RecordSyntaxError( llvm::formatv("Invalid integer literal: {0}", yytext)); } - return ARG_TOKEN(integer_literal, val); + return CARBON_ARG_TOKEN(integer_literal, val); } -{string_literal} { - llvm::StringRef str(yytext); - CARBON_CHECK(str.consume_front("\"") && str.consume_back("\"")); - std::optional unescaped = Carbon::UnescapeStringLiteral(str); - if (unescaped == std::nullopt) { - return context.RecordSyntaxError( - llvm::formatv("Invalid escaping in string: {0}", yytext)); +#*(\"\"\"|\") { + // Raw string literal. + // yytext (the token that matches the above regex) and chars scanned by + // str_lex_helper hold the source text, not the string the source represents. + Carbon::StringLexHelper str_lex_helper(yytext, yyscanner, context); + const std::string& s = str_lex_helper.str(); + const int hashtag_num = s.find_first_of('"'); + const int leading_quotes = s.size() - hashtag_num; + if (leading_quotes == 3 && hashtag_num > 0) { + // Check if it's a single-line string, like #"""#. + // TODO: Extend with other single-line string cases, like #""""#, based on + // the definition of block string in the design doc. + if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) { + return Carbon::ProcessSingleLineString(str_lex_helper.str(), context, + hashtag_num); + } else if (str_lex_helper.is_eof()) { + return CARBON_SIMPLE_TOKEN(END_OF_FILE); + } + } else if (!str_lex_helper.Advance()) { + return CARBON_SIMPLE_TOKEN(END_OF_FILE); } - return ARG_TOKEN(string_literal, *unescaped); -} + // 3 quotes indicates multi-line, otherwise it'll be one. + const bool multi_line = leading_quotes == 3; -\"\"\" { - // Block string literal. - std::string s(yytext); - // Scans for the closing """, checking for possible escape sequences - // like \""". - for (;;) { - int c = ReadChar(yyscanner, context); - if (c <= 0) { - return SIMPLE_TOKEN(END_OF_FILE); - } - s.push_back(c); - if (c != '"' && c != '\\') { - continue; - } - if (c == '\\') { - // \" in \""" is not a terminator. - c = ReadChar(yyscanner, context); - if (c <= 0) { - return SIMPLE_TOKEN(END_OF_FILE); - } - s.push_back(c); - continue; - } + while (!str_lex_helper.is_eof()) { + switch (str_lex_helper.last_char()) { + case '\n': // Fall through. + case '\v': // Fall through. + case '\f': // Fall through. + case '\r': + if (!multi_line) { + return context.RecordSyntaxError( + llvm::formatv("missing closing quote in single-line string: {0}", + str_lex_helper.str())); + } + str_lex_helper.Advance(); + break; + case '"': + if (multi_line) { + // Check for 2 more '"'s on block string. + if (!(str_lex_helper.Advance() && + str_lex_helper.last_char() == '"')) { + continue; + } + if (!(str_lex_helper.Advance() && + str_lex_helper.last_char() == '"')) { + continue; + } + // Now we are at the last " of """. + } - c = ReadChar(yyscanner, context); - if (c <= 0) { - return SIMPLE_TOKEN(END_OF_FILE); - } - s.push_back(c); - if (c != '"') { - continue; - } - - c = ReadChar(yyscanner, context); - if (c <= 0) { - return SIMPLE_TOKEN(END_OF_FILE); - } - s.push_back(c); - if (c == '"') { - break; + if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) { + // Reach closing quotes, break out of the loop. + if (leading_quotes == 3) { + return Carbon::ProcessMultiLineString(str_lex_helper.str(), context, + hashtag_num); + } else { + return Carbon::ProcessSingleLineString(str_lex_helper.str(), + context, hashtag_num); + } + } + break; + case '\\': + if (Carbon::ReadHashTags(str_lex_helper, hashtag_num)) { + // Read the escaped char. + if (!str_lex_helper.Advance()) { + continue; + } + // Read the next char. + str_lex_helper.Advance(); + } + break; + default: + str_lex_helper.Advance(); } } - Carbon::ErrorOr block_string = - Carbon::ParseBlockStringLiteral(s); - if (!block_string.ok()) { - return context.RecordSyntaxError(llvm::formatv( - "Invalid block string: {0}", block_string.error().message())); - } - return ARG_TOKEN(string_literal, *block_string); + return CARBON_SIMPLE_TOKEN(END_OF_FILE); } {one_line_comment} { @@ -380,10 +373,4 @@ string_literal \"([^\\\"\n\v\f\r]|\\.)*\" %% -auto ReadChar(yyscan_t yyscanner, Carbon::ParseAndLexContext& context) -> int { - const int c = yyinput(yyscanner); - if (c <= 0) { - context.RecordSyntaxError("Unexpected end of file"); - } - return c; -} +auto YyinputWrapper(yyscan_t yyscanner) -> int { return yyinput(yyscanner); } diff --git a/explorer/testdata/string/block_file_type_indicator.carbon b/explorer/testdata/string/block_file_type_indicator.carbon new file mode 100644 index 000000000000..341addae6b3a --- /dev/null +++ b/explorer/testdata/string/block_file_type_indicator.carbon @@ -0,0 +1,24 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s +// CHECK: result: 0 + +package ExplorerTest api; + +fn Main() -> i32 { + var s: String = """filetype + A "block" ""string"" literal + with file type indicator. + """; + if (s == "A \"block\" \"\"string\"\" literal\n with file type indicator.\n") { + return 0; + } else { + return 1; + } +} diff --git a/explorer/testdata/string/fail_newline.carbon b/explorer/testdata/string/fail_newline.carbon index 5e3ea57ddad8..a279f25ce561 100644 --- a/explorer/testdata/string/fail_newline.carbon +++ b/explorer/testdata/string/fail_newline.carbon @@ -11,7 +11,8 @@ package ExplorerTest api; fn Main() -> i32 { - // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_newline.carbon:[[@LINE+1]]: invalid character '\x22' in source file. + // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_newline.carbon:[[@LINE+2]]: missing closing quote in single-line string: "new + // CHECK-EMPTY: Print("new line"); return 0; diff --git a/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon new file mode 100644 index 000000000000..ef38648787c6 --- /dev/null +++ b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon @@ -0,0 +1,20 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{not} %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s + +package ExplorerTest api; + +fn Main() -> i32 { + // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon:[[@LINE+1]]: Unexpected end of file + var s: String = ##""" + error: there are more #s on the left than the right. + """#; + + return 0; +} diff --git a/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon new file mode 100644 index 000000000000..d92ac406a514 --- /dev/null +++ b/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon @@ -0,0 +1,20 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{not} %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s + +package ExplorerTest api; + +fn Main() -> i32 { + // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file. + var s: String = #""" + error: there are more #s on the right than the left. + """##; + + return 0; +} diff --git a/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon b/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon new file mode 100644 index 000000000000..dfbaec9f174f --- /dev/null +++ b/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon @@ -0,0 +1,20 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{not} %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s + +package ExplorerTest api; + +fn Main() -> i32 { + // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon:[[@LINE+1]]: Invalid block string: Should end with triple quotes: error: closing """ + var s: String = #""" + error: closing """# is not on its own line. + """#; + + return 0; +} diff --git a/explorer/testdata/string/fail_raw_block_single_line.carbon b/explorer/testdata/string/fail_raw_block_single_line.carbon new file mode 100644 index 000000000000..cae3b5e9f514 --- /dev/null +++ b/explorer/testdata/string/fail_raw_block_single_line.carbon @@ -0,0 +1,23 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{not} %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s + +package ExplorerTest api; + +fn CompareStr(s: String) -> i32 { + // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_block_single_line.carbon:[[@LINE+1]]: Invalid block string: Too few lines + if (s == #"""raw string literal starting with """#) { + return 0; + } + return 1; +} + +fn Main() -> i32 { + return CompareStr("\"\"raw string literal starting with \"\""); +} diff --git a/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon b/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon new file mode 100644 index 000000000000..75dc531d3de4 --- /dev/null +++ b/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon @@ -0,0 +1,24 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{not} %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s + +package ExplorerTest api; + +fn CompareStr(s: String) -> i32 { + // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_more_hash_tags_on_left.carbon:[[@LINE+2]]: missing closing quote in single-line string: ##"str"#) { + // CHECK-EMPTY: + if (s == ##"str"#) { + return 0; + } + return 1; +} + +fn Main() -> i32 { + return CompareStr("str"); +} diff --git a/explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon b/explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon new file mode 100644 index 000000000000..26a93c578a84 --- /dev/null +++ b/explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon @@ -0,0 +1,23 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{not} %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{not} %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s + +package ExplorerTest api; + +fn CompareStr(s: String) -> i32 { + // CHECK: COMPILATION ERROR: {{.*}}/explorer/testdata/string/fail_raw_more_hash_tags_on_right.carbon:[[@LINE+1]]: invalid character '\x23' in source file. + if (s == "str"#) { + return 0; + } + return 1; +} + +fn Main() -> i32 { + return CompareStr("str"); +} diff --git a/explorer/testdata/string/raw_basic.carbon b/explorer/testdata/string/raw_basic.carbon new file mode 100644 index 000000000000..2749ada7719f --- /dev/null +++ b/explorer/testdata/string/raw_basic.carbon @@ -0,0 +1,23 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s +// CHECK: result: 0 + +package ExplorerTest api; + +fn CompareStr(s: String) -> i32 { + if (s == #"str"#) { + return 0; + } + return 1; +} + +fn Main() -> i32 { + return CompareStr("str"); +} diff --git a/explorer/testdata/string/raw_basic_multi_hashtag.carbon b/explorer/testdata/string/raw_basic_multi_hashtag.carbon new file mode 100644 index 000000000000..9cbe6c608d94 --- /dev/null +++ b/explorer/testdata/string/raw_basic_multi_hashtag.carbon @@ -0,0 +1,23 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s +// CHECK: result: 0 + +package ExplorerTest api; + +fn CompareStr(s: String) -> i32 { + if (s == ##"str"##) { + return 0; + } + return 1; +} + +fn Main() -> i32 { + return CompareStr(#####"str"#####); +} diff --git a/explorer/testdata/string/raw_block.carbon b/explorer/testdata/string/raw_block.carbon new file mode 100644 index 000000000000..fd19c490c437 --- /dev/null +++ b/explorer/testdata/string/raw_block.carbon @@ -0,0 +1,24 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s +// CHECK: result: 0 + +package ExplorerTest api; + +fn Main() -> i32 { + var s: String = #""" + A "block" ""string"" literal + with indent. + """#; + if (s == #"A \#"block\#" \#"\#"string\#"\#" literal\#n with indent.\#n"#) { + return 0; + } else { + return 1; + } +} diff --git a/explorer/testdata/string/raw_block_escaped_back_slash.carbon b/explorer/testdata/string/raw_block_escaped_back_slash.carbon new file mode 100644 index 000000000000..f22a71636f89 --- /dev/null +++ b/explorer/testdata/string/raw_block_escaped_back_slash.carbon @@ -0,0 +1,24 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s +// CHECK: result: 0 + +package ExplorerTest api; + +fn Main() -> i32 { + var s: String = #""" + A block string literal + \ + """#; + if (s == "A block string literal\n\\\n") { + return 0; + } else { + return 1; + } +} diff --git a/explorer/testdata/string/raw_block_escaped_triple_quotes.carbon b/explorer/testdata/string/raw_block_escaped_triple_quotes.carbon new file mode 100644 index 000000000000..53d2306a3db1 --- /dev/null +++ b/explorer/testdata/string/raw_block_escaped_triple_quotes.carbon @@ -0,0 +1,24 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s +// CHECK: result: 0 + +package ExplorerTest api; + +fn Main() -> i32 { + var s: String = #""" + A block string literal + \#"""# + """#; + if (s == "A block string literal\n\"\"\"#\n") { + return 0; + } else { + return 1; + } +} diff --git a/explorer/testdata/string/raw_nesting.carbon b/explorer/testdata/string/raw_nesting.carbon new file mode 100644 index 000000000000..054e4a622c15 --- /dev/null +++ b/explorer/testdata/string/raw_nesting.carbon @@ -0,0 +1,23 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{explorer} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{explorer} %s +// CHECK: result: 0 + +package ExplorerTest api; + +fn CompareStr(s: String) -> i32 { + if (s == ##"#"str"#"##) { + return 0; + } + return 1; +} + +fn Main() -> i32 { + return CompareStr("#\"str\"#"); +}