mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 21:20:11 +01:00
Initial implementation of raw string literals (#1304)
* test cases for raw string literals * raw string literal implementation * match as block string if starting with triple ", and better error message for simple string except for *#"""#* * fix broken test case block string literal cannot be one line * test cases for raw string literals * raw string literal implementation * match as block string if starting with triple ", and better error message for simple string except for *#"""#* * fix broken test case block string literal cannot be one line * removed unused initial value * rename flag to indicate multi-line string and remove comment * use * to get value from std::optional * clean-ups * removed skip_scan flag and directly return in case of a single line string starting with #+\'\'\' * Updated error message: simple string -> single-line string. Co-authored-by: josh11b <josh11b@users.noreply.github.com> * Updated test cases according to changes in error message * Removed counting_hashtag flag. * Implemented ScanHelper class to handle scanning * Fixed explanation of ReadHashTags. * Addressed PR comment. * Clarify that scan_helper holds the source text. * Addressed PR comments. * Updated error messages in test cases. * Added const keyword to return type of GetCurrentStr(). * addressed PR comments. 1. Moved ScanHelper class to lex_scan_helper.h and lex_scan_helper.cpp. 2. Moved ReadHashTags and Process* functions to lex_scan_helper.cpp. Moved YY_USER_ACTION, SIMPLE_TOKEN and ARG_TOKEN to lex_helper.h. Added a wrapper function YyinputWrapper to call static function yyinput in lexer.lpp. 3. Renamed ScanHelper with StringLexHelper. 4. Modified BUILD accordingly. 5. Renamed data members and functions. * Addressed PR comments. 1. Adjusted order to keep ret usage close. 2. Used resize to construct the string to avoid creation of temp string. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Removed the multi_line flag and skip_read field to improve readability. * Copied default parameter value to definition of UnescapeStringLiteral. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Copied default parameter value to definition of ParseBlockStringLiteral. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Prefix CARBON_ to SIMPLE_TOKEN and ARG_TOKEN macros. * Rollback redefinition of arguments. * Updated comment on the flex macro. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Updated wording. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Moved the EOF error out of the loop. * Removed duplicated declaration. * Changed type of `hashtag_num` and `leading_quotes` to int. * Minor fix: string copy. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Added comment on YyinputWrapper. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Garmmar in comment. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Added check of eof before readling next char. * Minor updates based on PR comments. * Minor changes to address PR comments. * Used a clearer way to calculate `hashtag_num` and `leading_quotes`. Switched back to indicate muti-line string with a flag. * Directly copy StringRef for compilation error message. * Make str_with_quote const as we don't change it. Co-authored-by: josh11b <josh11b@users.noreply.github.com> * Added TODO for unsupported cases. * Fixed a typo. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> Co-authored-by: josh11b <josh11b@users.noreply.github.com> Co-authored-by: Jon Ross-Perkins <jperkins@google.com>
This commit is contained in:
SlaterLatiao
committed by
GitHub
GitHub
co-authored by
Jon Ross-Perkins
josh11b
Jon Ross-Perkins
josh11b
parent
a1be2a8a38
commit
8d0f3364d8
+71
-71
@@ -27,87 +27,86 @@ static auto FromHex(char c) -> std::optional<char> {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto UnescapeStringLiteral(llvm::StringRef source, bool is_block_string)
|
||||
-> std::optional<std::string> {
|
||||
auto UnescapeStringLiteral(llvm::StringRef source, const int hashtag_num,
|
||||
bool is_block_string) -> std::optional<std::string> {
|
||||
std::string ret;
|
||||
ret.reserve(source.size());
|
||||
std::string escape = "\\";
|
||||
escape.resize(hashtag_num + 1, '#');
|
||||
size_t i = 0;
|
||||
while (i < source.size()) {
|
||||
char c = source[i];
|
||||
switch (c) {
|
||||
case '\\':
|
||||
++i;
|
||||
if (i == source.size()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
switch (source[i]) {
|
||||
case 'n':
|
||||
ret.push_back('\n');
|
||||
break;
|
||||
case 'r':
|
||||
ret.push_back('\r');
|
||||
break;
|
||||
case 't':
|
||||
ret.push_back('\t');
|
||||
break;
|
||||
case '0':
|
||||
if (i + 1 < source.size() && llvm::isDigit(source[i + 1])) {
|
||||
// \0[0-9] is reserved.
|
||||
return std::nullopt;
|
||||
}
|
||||
ret.push_back('\0');
|
||||
break;
|
||||
case '"':
|
||||
ret.push_back('"');
|
||||
break;
|
||||
case '\'':
|
||||
ret.push_back('\'');
|
||||
break;
|
||||
case '\\':
|
||||
ret.push_back('\\');
|
||||
break;
|
||||
case 'x': {
|
||||
i += 2;
|
||||
if (i >= source.size()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
std::optional<char> c1 = FromHex(source[i - 1]);
|
||||
std::optional<char> c2 = FromHex(source[i]);
|
||||
if (c1 == std::nullopt || c2 == std::nullopt) {
|
||||
return std::nullopt;
|
||||
}
|
||||
ret.push_back(16 * *c1 + *c2);
|
||||
break;
|
||||
}
|
||||
case 'u':
|
||||
CARBON_FATAL() << "\\u is not yet supported in string literals";
|
||||
case '\n':
|
||||
if (!is_block_string) {
|
||||
return std::nullopt;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
// Unsupported.
|
||||
return std::nullopt;
|
||||
}
|
||||
break;
|
||||
|
||||
case '\t':
|
||||
// Disallow non-` ` horizontal whitespace:
|
||||
// https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/whitespace.md
|
||||
// TODO: This doesn't handle unicode whitespace.
|
||||
if (i + hashtag_num < source.size() &&
|
||||
source.slice(i, i + hashtag_num + 1).equals(escape)) {
|
||||
i += hashtag_num + 1;
|
||||
if (i == source.size()) {
|
||||
return std::nullopt;
|
||||
|
||||
default:
|
||||
ret.push_back(c);
|
||||
break;
|
||||
}
|
||||
switch (source[i]) {
|
||||
case 'n':
|
||||
ret.push_back('\n');
|
||||
break;
|
||||
case 'r':
|
||||
ret.push_back('\r');
|
||||
break;
|
||||
case 't':
|
||||
ret.push_back('\t');
|
||||
break;
|
||||
case '0':
|
||||
if (i + 1 < source.size() && llvm::isDigit(source[i + 1])) {
|
||||
// \0[0-9] is reserved.
|
||||
return std::nullopt;
|
||||
}
|
||||
ret.push_back('\0');
|
||||
break;
|
||||
case '"':
|
||||
ret.push_back('"');
|
||||
break;
|
||||
case '\'':
|
||||
ret.push_back('\'');
|
||||
break;
|
||||
case '\\':
|
||||
ret.push_back('\\');
|
||||
break;
|
||||
case 'x': {
|
||||
i += 2;
|
||||
if (i >= source.size()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
std::optional<char> c1 = FromHex(source[i - 1]);
|
||||
std::optional<char> c2 = FromHex(source[i]);
|
||||
if (c1 == std::nullopt || c2 == std::nullopt) {
|
||||
return std::nullopt;
|
||||
}
|
||||
ret.push_back(16 * *c1 + *c2);
|
||||
break;
|
||||
}
|
||||
case 'u':
|
||||
CARBON_FATAL() << "\\u is not yet supported in string literals";
|
||||
case '\n':
|
||||
if (!is_block_string) {
|
||||
return std::nullopt;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
// Unsupported.
|
||||
return std::nullopt;
|
||||
}
|
||||
} else if (c == '\t') {
|
||||
// Disallow non-` ` horizontal whitespace:
|
||||
// https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/whitespace.md
|
||||
// TODO: This doesn't handle unicode whitespace.
|
||||
return std::nullopt;
|
||||
} else {
|
||||
ret.push_back(c);
|
||||
}
|
||||
++i;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr<std::string> {
|
||||
auto ParseBlockStringLiteral(llvm::StringRef source, const int hashtag_num)
|
||||
-> ErrorOr<std::string> {
|
||||
llvm::SmallVector<llvm::StringRef> lines;
|
||||
source.split(lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/true);
|
||||
if (lines.size() < 2) {
|
||||
@@ -150,8 +149,9 @@ auto ParseBlockStringLiteral(llvm::StringRef source) -> ErrorOr<std::string> {
|
||||
}
|
||||
// Unescaping with \n appended to handle things like \\<newline>.
|
||||
llvm::SmallVector<char> buffer;
|
||||
std::optional<std::string> unescaped = UnescapeStringLiteral(
|
||||
(line + "\n").toStringRef(buffer), /*is_block_string=*/true);
|
||||
std::optional<std::string> unescaped =
|
||||
UnescapeStringLiteral((line + "\n").toStringRef(buffer), hashtag_num,
|
||||
/*is_block_string=*/true);
|
||||
if (!unescaped.has_value()) {
|
||||
return Error("Invalid escaping in " + line);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user