carbon-language/carbon-lang
1
0 Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-11-15 22:38:10 +00:00
Code Issues Packages Projects Releases 10 Wiki Activity

Factor out character set handling into a separate file. (#352)

Browse Source
This commit is contained in:
Richard Smith authored and GitHub committed 2021-03-09 12:38:46 -08:00
1 parent 67ee1fcaa7
commit 97bb6f4e80
5 files changed
+108 -38
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lexer/tokenized_buffer.cpp
+9 -12
View File
@@ -9,6 +9,7 @@
#include <iterator>
#include <string>
#include "lexer/character_set.h"
#include "lexer/numeric_literal.h"
#include "lexer/string_literal.h"
#include "llvm/ADT/StringRef.h"
@@ -53,10 +54,6 @@ struct UnrecognizedCharacters : SimpleDiagnostic<UnrecognizedCharacters> {
"Encountered unrecognized characters while parsing.";
};
// TODO(zygoloid): Update this to match whatever we decide qualifies as
// acceptable whitespace.
static bool isSpace(char c) { return c == ' ' || c == '\n' || c == '\t'; }
// Implementation of the lexer logic itself.
//
// The design is that lexing can loop over the source buffer, consuming it into
@@ -110,8 +107,8 @@ class TokenizedBuffer::Lexer {
auto HandleNewline() -> void {
current_line_info->length = current_column;
current_line = buffer.AddLine(
{current_line_info->start + current_column + 1, 0, 0});
current_line =
buffer.AddLine({current_line_info->start + current_column + 1, 0, 0});
current_line_info = &buffer.GetLineInfo(current_line);
current_column = 0;
set_indent = false;
@@ -128,7 +125,7 @@ class TokenizedBuffer::Lexer {
buffer.has_errors = true;
}
// The introducer '//' must be followed by whitespace or EOF.
if (source_text.size() > 2 && !isSpace(source_text[2])) {
if (source_text.size() > 2 && !IsSpace(source_text[2])) {
emitter.EmitError<NoWhitespaceAfterCommentIntroducer>();
buffer.has_errors = true;
}
@@ -145,7 +142,7 @@ class TokenizedBuffer::Lexer {
default:
// If we find a non-whitespace character without exhausting the
// buffer, return true to continue lexing.
assert(!isSpace(source_text.front()));
assert(!IsSpace(source_text.front()));
return true;
case '\n':
@@ -383,7 +380,7 @@ class TokenizedBuffer::Lexer {
}
auto LexKeywordOrIdentifier(llvm::StringRef& source_text) -> LexResult {
if (!llvm::isAlpha(source_text.front()) && source_text.front() != '_') {
if (!IsAlpha(source_text.front()) && source_text.front() != '_') {
return LexResult::NoMatch();
}
@@ -393,8 +390,8 @@ class TokenizedBuffer::Lexer {
}
// Take the valid characters off the front of the source buffer.
llvm::StringRef identifier_text = source_text.take_while(
[](char c) { return llvm::isAlnum(c) || c == '_'; });
llvm::StringRef identifier_text =
source_text.take_while([](char c) { return IsAlnum(c) || c == '_'; });
assert(!identifier_text.empty() && "Must have at least one character!");
int identifier_column = current_column;
current_column += identifier_text.size();
@@ -420,7 +417,7 @@ class TokenizedBuffer::Lexer {
auto LexError(llvm::StringRef& source_text) -> LexResult {
llvm::StringRef error_text = source_text.take_while([](char c) {
if (llvm::isAlnum(c)) {
if (IsAlnum(c)) {
return false;
}
switch (c) {