mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 20:30:14 +01:00
* test cases for raw string literals * raw string literal implementation * match as block string if starting with triple ", and better error message for simple string except for *#"""#* * fix broken test case block string literal cannot be one line * test cases for raw string literals * raw string literal implementation * match as block string if starting with triple ", and better error message for simple string except for *#"""#* * fix broken test case block string literal cannot be one line * removed unused initial value * rename flag to indicate multi-line string and remove comment * use * to get value from std::optional * clean-ups * removed skip_scan flag and directly return in case of a single line string starting with #+\'\'\' * Updated error message: simple string -> single-line string. Co-authored-by: josh11b <josh11b@users.noreply.github.com> * Updated test cases according to changes in error message * Removed counting_hashtag flag. * Implemented ScanHelper class to handle scanning * Fixed explanation of ReadHashTags. * Addressed PR comment. * Clarify that scan_helper holds the source text. * Addressed PR comments. * Updated error messages in test cases. * Added const keyword to return type of GetCurrentStr(). * addressed PR comments. 1. Moved ScanHelper class to lex_scan_helper.h and lex_scan_helper.cpp. 2. Moved ReadHashTags and Process* functions to lex_scan_helper.cpp. Moved YY_USER_ACTION, SIMPLE_TOKEN and ARG_TOKEN to lex_helper.h. Added a wrapper function YyinputWrapper to call static function yyinput in lexer.lpp. 3. Renamed ScanHelper with StringLexHelper. 4. Modified BUILD accordingly. 5. Renamed data members and functions. * Addressed PR comments. 1. Adjusted order to keep ret usage close. 2. Used resize to construct the string to avoid creation of temp string. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Removed the multi_line flag and skip_read field to improve readability. * Copied default parameter value to definition of UnescapeStringLiteral. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Copied default parameter value to definition of ParseBlockStringLiteral. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Prefix CARBON_ to SIMPLE_TOKEN and ARG_TOKEN macros. * Rollback redefinition of arguments. * Updated comment on the flex macro. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Updated wording. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Moved the EOF error out of the loop. * Removed duplicated declaration. * Changed type of `hashtag_num` and `leading_quotes` to int. * Minor fix: string copy. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Added comment on YyinputWrapper. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Garmmar in comment. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> * Added check of eof before readling next char. * Minor updates based on PR comments. * Minor changes to address PR comments. * Used a clearer way to calculate `hashtag_num` and `leading_quotes`. Switched back to indicate muti-line string with a flag. * Directly copy StringRef for compilation error message. * Make str_with_quote const as we don't change it. Co-authored-by: josh11b <josh11b@users.noreply.github.com> * Added TODO for unsupported cases. * Fixed a typo. Co-authored-by: Jon Ross-Perkins <jperkins@google.com> Co-authored-by: josh11b <josh11b@users.noreply.github.com> Co-authored-by: Jon Ross-Perkins <jperkins@google.com>
208 lines
6.2 KiB
C++
208 lines
6.2 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "common/string_helpers.h"
|
|
|
|
#include <gmock/gmock.h>
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <string>
|
|
|
|
#include "llvm/Support/Error.h"
|
|
|
|
using ::llvm::toString;
|
|
using ::testing::Eq;
|
|
using ::testing::Optional;
|
|
|
|
namespace Carbon::Testing {
|
|
namespace {
|
|
|
|
TEST(UnescapeStringLiteral, Valid) {
|
|
EXPECT_THAT(UnescapeStringLiteral("test"), Optional(Eq("test")));
|
|
EXPECT_THAT(UnescapeStringLiteral("okay whitespace"),
|
|
Optional(Eq("okay whitespace")));
|
|
EXPECT_THAT(UnescapeStringLiteral("test\n"), Optional(Eq("test\n")));
|
|
EXPECT_THAT(UnescapeStringLiteral("test\\n"), Optional(Eq("test\n")));
|
|
EXPECT_THAT(UnescapeStringLiteral("abc\\ndef"), Optional(Eq("abc\ndef")));
|
|
EXPECT_THAT(UnescapeStringLiteral("test\\\\n"), Optional(Eq("test\\n")));
|
|
EXPECT_THAT(UnescapeStringLiteral("\\xAA"), Optional(Eq("\xAA")));
|
|
EXPECT_THAT(UnescapeStringLiteral("\\x12"), Optional(Eq("\x12")));
|
|
EXPECT_THAT(UnescapeStringLiteral("test", 1), Optional(Eq("test")));
|
|
EXPECT_THAT(UnescapeStringLiteral("test\\#n", 1), Optional(Eq("test\n")));
|
|
}
|
|
|
|
TEST(UnescapeStringLiteral, Invalid) {
|
|
// Missing char after `\`.
|
|
EXPECT_THAT(UnescapeStringLiteral("a\\"), Eq(std::nullopt));
|
|
// Not a supported escape.
|
|
EXPECT_THAT(UnescapeStringLiteral("\\e"), Eq(std::nullopt));
|
|
// Needs 2 hex chars.
|
|
EXPECT_THAT(UnescapeStringLiteral("\\x"), Eq(std::nullopt));
|
|
// Needs 2 hex chars.
|
|
EXPECT_THAT(UnescapeStringLiteral("\\xA"), Eq(std::nullopt));
|
|
// Needs uppercase hex.
|
|
EXPECT_THAT(UnescapeStringLiteral("\\xaa"), Eq(std::nullopt));
|
|
// Reserved.
|
|
EXPECT_THAT(UnescapeStringLiteral("\\00"), Eq(std::nullopt));
|
|
EXPECT_THAT(UnescapeStringLiteral("\\#00", 1), Eq(std::nullopt));
|
|
}
|
|
|
|
TEST(UnescapeStringLiteral, Nul) {
|
|
std::optional<std::string> str = UnescapeStringLiteral("a\\0b");
|
|
ASSERT_NE(str, std::nullopt);
|
|
EXPECT_THAT(str->size(), Eq(3));
|
|
EXPECT_THAT(strlen(str->c_str()), Eq(1));
|
|
EXPECT_THAT((*str)[0], Eq('a'));
|
|
EXPECT_THAT((*str)[1], Eq('\0'));
|
|
EXPECT_THAT((*str)[2], Eq('b'));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, FailTooFewLines) {
|
|
EXPECT_THAT(ParseBlockStringLiteral("").error().message(),
|
|
Eq("Too few lines"));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, FailNoLeadingTripleQuotes) {
|
|
EXPECT_THAT(ParseBlockStringLiteral("'a'\n").error().message(),
|
|
Eq("Should start with triple quotes: 'a'"));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, FailInvalideFiletypeIndicator) {
|
|
EXPECT_THAT(ParseBlockStringLiteral("\"\"\"carbon file\n").error().message(),
|
|
Eq("Invalid characters in file type indicator: carbon file"));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, FailEndingTripleQuotes) {
|
|
EXPECT_THAT(ParseBlockStringLiteral("\"\"\"\n").error().message(),
|
|
Eq("Should end with triple quotes: "));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, FailWrongIndent) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal
|
|
with wrong indent
|
|
""")";
|
|
EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
|
|
Eq("Wrong indent for line: with wrong indent, expected 5"));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, FailInvalidEscaping) {
|
|
constexpr char Input[] = R"("""
|
|
\q
|
|
""")";
|
|
EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
|
|
Eq("Invalid escaping in \\q"));
|
|
constexpr char InputRaw[] = R"("""
|
|
\#q
|
|
""")";
|
|
EXPECT_THAT(ParseBlockStringLiteral(InputRaw, 1).error().message(),
|
|
Eq("Invalid escaping in \\#q"));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkEmptyString) {
|
|
constexpr char Input[] = R"("""
|
|
""")";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(""));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkOneLineString) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal
|
|
""")";
|
|
constexpr char Expected[] = R"(A block string literal
|
|
)";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkTwoLineString) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal
|
|
with indent.
|
|
""")";
|
|
constexpr char Expected[] = R"(A block string literal
|
|
with indent.
|
|
)";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) {
|
|
constexpr char Input[] = R"("""carbon
|
|
A block string literal
|
|
with file type indicator.
|
|
""")";
|
|
constexpr char Expected[] = R"(A block string literal
|
|
with file type indicator.
|
|
)";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkWhitespaceAfterOpeningQuotes) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal
|
|
""")";
|
|
constexpr char Expected[] = R"(A block string literal
|
|
)";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal
|
|
|
|
with
|
|
|
|
empty
|
|
|
|
lines.
|
|
""")";
|
|
constexpr char Expected[] = R"(A block string literal
|
|
|
|
with
|
|
|
|
empty
|
|
|
|
lines.
|
|
)";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkWithSlashNewlineEscape) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal\
|
|
""")";
|
|
constexpr char Expected[] = "A block string literal";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkWithDoubleSlashNewline) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal\\
|
|
""")";
|
|
constexpr char Expected[] = R"(A block string literal\
|
|
)";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkWithTripleSlashNewline) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal\\\
|
|
""")";
|
|
constexpr char Expected[] = R"(A block string literal\)";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
TEST(ParseBlockStringLiteral, OkMultipleSlashes) {
|
|
constexpr char Input[] = R"("""
|
|
A block string literal\
|
|
\
|
|
\
|
|
\
|
|
""")";
|
|
constexpr char Expected[] = "A block string literal";
|
|
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
|
|
}
|
|
|
|
} // namespace
|
|
} // namespace Carbon::Testing
|