carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-06 08:54:47 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Adjust string parsing to consume everything until the terminator. (#1111)

Browse Source 
Note I've added a few TODOs, particularly that multi-line strings should only consume until the dedent.

Co-authored-by: Richard Smith <richard@metafoo.co.uk>
This commit is contained in:
Jon Meow
2022-03-03 10:21:16 -08:00
committed by GitHub
co-authored by Richard Smith
parent f4f9b23291
commit 0a8c0dc271
7 changed files with 164 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/lexer/tokenized_buffer.cpp
+27 -13
View File
@@ -57,6 +57,12 @@ struct UnrecognizedCharacters : DiagnosticBase<UnrecognizedCharacters> {
"Encountered unrecognized characters while parsing.";
};
struct UnterminatedString : DiagnosticBase<UnterminatedString> {
static constexpr llvm::StringLiteral ShortName = "syntax-string-terminator";
static constexpr llvm::StringLiteral Message =
"String is missing a terminator.";
};
// TODO: Move Overload and VariantMatch somewhere more central.
// Form an overload set from a list of functions. For example:
@@ -267,7 +273,7 @@ class TokenizedBuffer::Lexer {
Line string_line = current_line_;
int string_column = current_column_;
int literal_size = literal->Text().size();
int literal_size = literal->text().size();
source_text = source_text.drop_front(literal_size);
if (!set_indent_) {
@@ -276,10 +282,10 @@ class TokenizedBuffer::Lexer {
}
// Update line and column information.
if (!literal->IsMultiLine()) {
if (!literal->is_multi_line()) {
current_column_ += literal_size;
} else {
for (char c : literal->Text()) {
for (char c : literal->text()) {
if (c == '\n') {
HandleNewline();
// The indentation of all lines in a multi-line string literal is
@@ -292,13 +298,23 @@ class TokenizedBuffer::Lexer {
}
}
auto token = buffer_.AddToken({.kind = TokenKind::StringLiteral(),
.token_line = string_line,
.column = string_column});
buffer_.GetTokenInfo(token).literal_index =
buffer_.literal_string_storage_.size();
buffer_.literal_string_storage_.push_back(literal->ComputeValue(emitter_));
return token;
if (literal->is_terminated()) {
auto token =
buffer_.AddToken({.kind = TokenKind::StringLiteral(),
.token_line = string_line,
.column = string_column,
.literal_index = static_cast<int32_t>(
buffer_.literal_string_storage_.size())});
buffer_.literal_string_storage_.push_back(
literal->ComputeValue(emitter_));
return token;
} else {
emitter_.EmitError<UnterminatedString>(literal->text().begin());
return buffer_.AddToken({.kind = TokenKind::Error(),
.token_line = string_line,
.column = string_column,
.error_length = literal_size});
}
}
auto LexSymbolToken(llvm::StringRef& source_text) -> LexResult {
@@ -506,8 +522,6 @@ class TokenizedBuffer::Lexer {
error_text = source_text.take_front(1);
}
// Longer errors get to be two tokens.
error_text = error_text.substr(0, std::numeric_limits<int32_t>::max());
auto token = buffer_.AddToken(
{.kind = TokenKind::Error(),
.token_line = current_line_,
@@ -632,7 +646,7 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef {
llvm::Optional<LexedStringLiteral> relexed_token =
LexedStringLiteral::Lex(source_->Text().substr(token_start));
CHECK(relexed_token) << "Could not reform string literal token.";
return relexed_token->Text();
return relexed_token->text();
}
// Refer back to the source text to avoid needing to reconstruct the