Files
carbon-lang/common/string_helpers_test.cpp
T
8d0f3364d8 Initial implementation of raw string literals (#1304)
* test cases for raw string literals

* raw string literal implementation

* match as block string if starting with triple ", and better error message for simple string

except for *#"""#*

* fix broken test case

block string  literal cannot be one line

* test cases for raw string literals

* raw string literal implementation

* match as block string if starting with triple ", and better error message for simple string

except for *#"""#*

* fix broken test case

block string  literal cannot be one line

* removed unused initial value

* rename flag to indicate multi-line string and remove comment

* use * to get value from std::optional

* clean-ups

* removed skip_scan flag and directly return in case of a single line string starting with #+\'\'\'

* Updated error message: simple string -> single-line string.

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

* Updated test cases according to changes in error message

* Removed counting_hashtag flag.

* Implemented ScanHelper class to handle scanning

* Fixed explanation of ReadHashTags.

* Addressed PR comment.

* Clarify that scan_helper holds the source text.

* Addressed PR comments.

* Updated error messages in test cases.

* Added const keyword to return type of GetCurrentStr().

* addressed PR comments.

1. Moved ScanHelper class to lex_scan_helper.h and lex_scan_helper.cpp.
2. Moved ReadHashTags and Process* functions to lex_scan_helper.cpp. Moved YY_USER_ACTION, SIMPLE_TOKEN and ARG_TOKEN to lex_helper.h. Added a wrapper function YyinputWrapper to call static function yyinput in lexer.lpp.
3. Renamed ScanHelper with StringLexHelper.
4. Modified BUILD accordingly.
5. Renamed data members and functions.

* Addressed PR comments.

1. Adjusted order to keep ret usage close.
2. Used resize to construct the string to avoid creation of temp string.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Removed the multi_line flag and skip_read field to improve readability.

* Copied default parameter value to definition of UnescapeStringLiteral.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Copied default parameter value to definition of ParseBlockStringLiteral.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Prefix CARBON_ to SIMPLE_TOKEN and ARG_TOKEN macros.

* Rollback redefinition of arguments.

* Updated comment on the flex macro.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Updated wording.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Moved the EOF error out of the loop.

* Removed duplicated declaration.

* Changed type of `hashtag_num` and `leading_quotes` to int.

* Minor fix: string copy.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Added comment on YyinputWrapper.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Garmmar in comment.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

* Added check of eof before readling next char.

* Minor updates based on PR comments.

* Minor changes to address PR comments.

* Used a clearer way to calculate `hashtag_num` and `leading_quotes`. Switched back to indicate muti-line string with a flag.

* Directly copy StringRef for compilation error message.

* Make str_with_quote const as we don't change it.

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

* Added TODO for unsupported cases.

* Fixed a typo.

Co-authored-by: Jon Ross-Perkins <jperkins@google.com>

Co-authored-by: josh11b <josh11b@users.noreply.github.com>
Co-authored-by: Jon Ross-Perkins <jperkins@google.com>
2022-06-22 15:38:47 -07:00

208 lines
6.2 KiB
C++

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "common/string_helpers.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <string>
#include "llvm/Support/Error.h"
using ::llvm::toString;
using ::testing::Eq;
using ::testing::Optional;
namespace Carbon::Testing {
namespace {
TEST(UnescapeStringLiteral, Valid) {
EXPECT_THAT(UnescapeStringLiteral("test"), Optional(Eq("test")));
EXPECT_THAT(UnescapeStringLiteral("okay whitespace"),
Optional(Eq("okay whitespace")));
EXPECT_THAT(UnescapeStringLiteral("test\n"), Optional(Eq("test\n")));
EXPECT_THAT(UnescapeStringLiteral("test\\n"), Optional(Eq("test\n")));
EXPECT_THAT(UnescapeStringLiteral("abc\\ndef"), Optional(Eq("abc\ndef")));
EXPECT_THAT(UnescapeStringLiteral("test\\\\n"), Optional(Eq("test\\n")));
EXPECT_THAT(UnescapeStringLiteral("\\xAA"), Optional(Eq("\xAA")));
EXPECT_THAT(UnescapeStringLiteral("\\x12"), Optional(Eq("\x12")));
EXPECT_THAT(UnescapeStringLiteral("test", 1), Optional(Eq("test")));
EXPECT_THAT(UnescapeStringLiteral("test\\#n", 1), Optional(Eq("test\n")));
}
TEST(UnescapeStringLiteral, Invalid) {
// Missing char after `\`.
EXPECT_THAT(UnescapeStringLiteral("a\\"), Eq(std::nullopt));
// Not a supported escape.
EXPECT_THAT(UnescapeStringLiteral("\\e"), Eq(std::nullopt));
// Needs 2 hex chars.
EXPECT_THAT(UnescapeStringLiteral("\\x"), Eq(std::nullopt));
// Needs 2 hex chars.
EXPECT_THAT(UnescapeStringLiteral("\\xA"), Eq(std::nullopt));
// Needs uppercase hex.
EXPECT_THAT(UnescapeStringLiteral("\\xaa"), Eq(std::nullopt));
// Reserved.
EXPECT_THAT(UnescapeStringLiteral("\\00"), Eq(std::nullopt));
EXPECT_THAT(UnescapeStringLiteral("\\#00", 1), Eq(std::nullopt));
}
TEST(UnescapeStringLiteral, Nul) {
std::optional<std::string> str = UnescapeStringLiteral("a\\0b");
ASSERT_NE(str, std::nullopt);
EXPECT_THAT(str->size(), Eq(3));
EXPECT_THAT(strlen(str->c_str()), Eq(1));
EXPECT_THAT((*str)[0], Eq('a'));
EXPECT_THAT((*str)[1], Eq('\0'));
EXPECT_THAT((*str)[2], Eq('b'));
}
TEST(ParseBlockStringLiteral, FailTooFewLines) {
EXPECT_THAT(ParseBlockStringLiteral("").error().message(),
Eq("Too few lines"));
}
TEST(ParseBlockStringLiteral, FailNoLeadingTripleQuotes) {
EXPECT_THAT(ParseBlockStringLiteral("'a'\n").error().message(),
Eq("Should start with triple quotes: 'a'"));
}
TEST(ParseBlockStringLiteral, FailInvalideFiletypeIndicator) {
EXPECT_THAT(ParseBlockStringLiteral("\"\"\"carbon file\n").error().message(),
Eq("Invalid characters in file type indicator: carbon file"));
}
TEST(ParseBlockStringLiteral, FailEndingTripleQuotes) {
EXPECT_THAT(ParseBlockStringLiteral("\"\"\"\n").error().message(),
Eq("Should end with triple quotes: "));
}
TEST(ParseBlockStringLiteral, FailWrongIndent) {
constexpr char Input[] = R"("""
A block string literal
with wrong indent
""")";
EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
Eq("Wrong indent for line: with wrong indent, expected 5"));
}
TEST(ParseBlockStringLiteral, FailInvalidEscaping) {
constexpr char Input[] = R"("""
\q
""")";
EXPECT_THAT(ParseBlockStringLiteral(Input).error().message(),
Eq("Invalid escaping in \\q"));
constexpr char InputRaw[] = R"("""
\#q
""")";
EXPECT_THAT(ParseBlockStringLiteral(InputRaw, 1).error().message(),
Eq("Invalid escaping in \\#q"));
}
TEST(ParseBlockStringLiteral, OkEmptyString) {
constexpr char Input[] = R"("""
""")";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(""));
}
TEST(ParseBlockStringLiteral, OkOneLineString) {
constexpr char Input[] = R"("""
A block string literal
""")";
constexpr char Expected[] = R"(A block string literal
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkTwoLineString) {
constexpr char Input[] = R"("""
A block string literal
with indent.
""")";
constexpr char Expected[] = R"(A block string literal
with indent.
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithFileTypeIndicator) {
constexpr char Input[] = R"("""carbon
A block string literal
with file type indicator.
""")";
constexpr char Expected[] = R"(A block string literal
with file type indicator.
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWhitespaceAfterOpeningQuotes) {
constexpr char Input[] = R"("""
A block string literal
""")";
constexpr char Expected[] = R"(A block string literal
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithEmptyLines) {
constexpr char Input[] = R"("""
A block string literal
with
empty
lines.
""")";
constexpr char Expected[] = R"(A block string literal
with
empty
lines.
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithSlashNewlineEscape) {
constexpr char Input[] = R"("""
A block string literal\
""")";
constexpr char Expected[] = "A block string literal";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithDoubleSlashNewline) {
constexpr char Input[] = R"("""
A block string literal\\
""")";
constexpr char Expected[] = R"(A block string literal\
)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkWithTripleSlashNewline) {
constexpr char Input[] = R"("""
A block string literal\\\
""")";
constexpr char Expected[] = R"(A block string literal\)";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
TEST(ParseBlockStringLiteral, OkMultipleSlashes) {
constexpr char Input[] = R"("""
A block string literal\
\
\
\
""")";
constexpr char Expected[] = "A block string literal";
EXPECT_THAT(*ParseBlockStringLiteral(Input), Eq(Expected));
}
} // namespace
} // namespace Carbon::Testing