diff --git a/language_server/language_server.cpp b/language_server/language_server.cpp index eb06bdecf428..f11ed7816650 100644 --- a/language_server/language_server.cpp +++ b/language_server/language_server.cpp @@ -95,7 +95,7 @@ void LanguageServer::OnDocumentSymbol( llvm::MemoryBuffer::getMemBufferCopy(files_.at(file))); auto buf = SourceBuffer::CreateFromFile(vfs, file); - auto lexed = TokenizedBuffer::Lex(*buf, NullDiagnosticConsumer()); + auto lexed = Lex::TokenizedBuffer::Lex(*buf, NullDiagnosticConsumer()); auto parsed = Parse::Tree::Parse(lexed, NullDiagnosticConsumer(), nullptr); std::vector result; for (const auto& node : parsed.postorder()) { diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp index 1f2e046555d3..0ecc98c82fa2 100644 --- a/toolchain/driver/driver.cpp +++ b/toolchain/driver/driver.cpp @@ -406,10 +406,10 @@ auto Driver::Compile(const CompileOptions& options) -> bool { } CARBON_VLOG() << "*** file:\n```\n" << source->text() << "\n```\n"; - CARBON_VLOG() << "*** TokenizedBuffer::Lex ***\n"; - auto tokenized_source = TokenizedBuffer::Lex(*source, *consumer); + CARBON_VLOG() << "*** Lex::TokenizedBuffer::Lex ***\n"; + auto tokenized_source = Lex::TokenizedBuffer::Lex(*source, *consumer); bool has_errors = tokenized_source.has_errors(); - CARBON_VLOG() << "*** TokenizedBuffer::Lex done ***\n"; + CARBON_VLOG() << "*** Lex::TokenizedBuffer::Lex done ***\n"; if (options.dump_tokens) { CARBON_VLOG() << "Finishing output."; consumer->Flush(); diff --git a/toolchain/lexer/character_set.h b/toolchain/lexer/character_set.h index 3dc9e916c2b5..d2e9ced996ad 100644 --- a/toolchain/lexer/character_set.h +++ b/toolchain/lexer/character_set.h @@ -7,7 +7,7 @@ #include "llvm/ADT/StringExtras.h" -namespace Carbon { +namespace Carbon::Lex { // TODO: These definitions need to be updated to match whatever Unicode lexical // rules we pick. The function interfaces will need to change to accommodate @@ -71,6 +71,6 @@ inline auto IsSpace(char c) -> bool { return IsHorizontalWhitespace(c) || IsVerticalWhitespace(c); } -} // namespace Carbon +} // namespace Carbon::Lex #endif // CARBON_TOOLCHAIN_LEXER_CHARACTER_SET_H_ diff --git a/toolchain/lexer/lex_helpers.cpp b/toolchain/lexer/lex_helpers.cpp index 0ce1de17aa0f..da1209062eb1 100644 --- a/toolchain/lexer/lex_helpers.cpp +++ b/toolchain/lexer/lex_helpers.cpp @@ -4,7 +4,7 @@ #include "toolchain/lexer/lex_helpers.h" -namespace Carbon { +namespace Carbon::Lex { auto CanLexInteger(DiagnosticEmitter& emitter, llvm::StringRef text) -> bool { @@ -29,4 +29,4 @@ auto CanLexInteger(DiagnosticEmitter& emitter, return true; } -} // namespace Carbon +} // namespace Carbon::Lex diff --git a/toolchain/lexer/lex_helpers.h b/toolchain/lexer/lex_helpers.h index 5728610f083d..d70685e2361f 100644 --- a/toolchain/lexer/lex_helpers.h +++ b/toolchain/lexer/lex_helpers.h @@ -7,13 +7,13 @@ #include "toolchain/diagnostics/diagnostic_emitter.h" -namespace Carbon { +namespace Carbon::Lex { // Should guard calls to getAsInteger due to performance issues with large // integers. Emits an error if the text cannot be lexed. auto CanLexInteger(DiagnosticEmitter& emitter, llvm::StringRef text) -> bool; -} // namespace Carbon +} // namespace Carbon::Lex #endif // CARBON_TOOLCHAIN_LEXER_LEX_HELPERS_H_ diff --git a/toolchain/lexer/numeric_literal.cpp b/toolchain/lexer/numeric_literal.cpp index f5663ad56a44..a593a0771053 100644 --- a/toolchain/lexer/numeric_literal.cpp +++ b/toolchain/lexer/numeric_literal.cpp @@ -11,7 +11,7 @@ #include "toolchain/lexer/character_set.h" #include "toolchain/lexer/lex_helpers.h" -namespace Carbon { +namespace Carbon::Lex { // Adapts Radix for use with formatv. // NOTE: clangd may see this as unused, but it will be invoked by diagnostics. @@ -461,4 +461,4 @@ auto LexedNumericLiteral::ComputeValue( .exponent = parser.GetExponent()}; } -} // namespace Carbon +} // namespace Carbon::Lex diff --git a/toolchain/lexer/numeric_literal.h b/toolchain/lexer/numeric_literal.h index 672f9e7f0eb1..783d7abad0e0 100644 --- a/toolchain/lexer/numeric_literal.h +++ b/toolchain/lexer/numeric_literal.h @@ -12,7 +12,7 @@ #include "llvm/ADT/StringRef.h" #include "toolchain/diagnostics/diagnostic_emitter.h" -namespace Carbon { +namespace Carbon::Lex { // A numeric literal token that has been extracted from a source buffer. class LexedNumericLiteral { @@ -71,6 +71,6 @@ class LexedNumericLiteral { int exponent_; }; -} // namespace Carbon +} // namespace Carbon::Lex #endif // CARBON_TOOLCHAIN_LEXER_NUMERIC_LITERAL_H_ diff --git a/toolchain/lexer/numeric_literal_benchmark.cpp b/toolchain/lexer/numeric_literal_benchmark.cpp index 04bd3e974523..e56d5f70a0e0 100644 --- a/toolchain/lexer/numeric_literal_benchmark.cpp +++ b/toolchain/lexer/numeric_literal_benchmark.cpp @@ -11,6 +11,8 @@ namespace Carbon::Testing { namespace { +using Lex::LexedNumericLiteral; + static void BM_Lex_Float(benchmark::State& state) { for (auto _ : state) { CARBON_CHECK(LexedNumericLiteral::Lex("0.000001")); diff --git a/toolchain/lexer/numeric_literal_fuzzer.cpp b/toolchain/lexer/numeric_literal_fuzzer.cpp index c6db06f47609..7ef0ac16a503 100644 --- a/toolchain/lexer/numeric_literal_fuzzer.cpp +++ b/toolchain/lexer/numeric_literal_fuzzer.cpp @@ -13,7 +13,7 @@ namespace Carbon::Testing { // NOLINTNEXTLINE: Match the documented fuzzer entry point declaration style. extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, std::size_t size) { - auto token = LexedNumericLiteral::Lex( + auto token = Lex::LexedNumericLiteral::Lex( llvm::StringRef(reinterpret_cast(data), size)); if (!token) { // Lexically not a numeric literal. diff --git a/toolchain/lexer/numeric_literal_test.cpp b/toolchain/lexer/numeric_literal_test.cpp index 776684776099..c674200b718e 100644 --- a/toolchain/lexer/numeric_literal_test.cpp +++ b/toolchain/lexer/numeric_literal_test.cpp @@ -14,6 +14,7 @@ namespace Carbon::Testing { namespace { +using Lex::LexedNumericLiteral; using ::testing::_; using ::testing::Field; using ::testing::Matcher; diff --git a/toolchain/lexer/string_literal.cpp b/toolchain/lexer/string_literal.cpp index 83c2b880cb89..1fed48f97d97 100644 --- a/toolchain/lexer/string_literal.cpp +++ b/toolchain/lexer/string_literal.cpp @@ -12,7 +12,7 @@ #include "toolchain/lexer/character_set.h" #include "toolchain/lexer/lex_helpers.h" -namespace Carbon { +namespace Carbon::Lex { using LexerDiagnosticEmitter = DiagnosticEmitter; @@ -455,4 +455,4 @@ auto LexedStringLiteral::ComputeValue(LexerDiagnosticEmitter& emitter) const indent); } -} // namespace Carbon +} // namespace Carbon::Lex diff --git a/toolchain/lexer/string_literal.h b/toolchain/lexer/string_literal.h index 72cdbad3a709..1aff5661b08f 100644 --- a/toolchain/lexer/string_literal.h +++ b/toolchain/lexer/string_literal.h @@ -11,7 +11,7 @@ #include "llvm/ADT/StringRef.h" #include "toolchain/diagnostics/diagnostic_emitter.h" -namespace Carbon { +namespace Carbon::Lex { class LexedStringLiteral { public: @@ -70,6 +70,6 @@ class LexedStringLiteral { bool is_terminated_; }; -} // namespace Carbon +} // namespace Carbon::Lex #endif // CARBON_TOOLCHAIN_LEXER_STRING_LITERAL_H_ diff --git a/toolchain/lexer/string_literal_benchmark.cpp b/toolchain/lexer/string_literal_benchmark.cpp index cce31fc566b9..c9058353ed76 100644 --- a/toolchain/lexer/string_literal_benchmark.cpp +++ b/toolchain/lexer/string_literal_benchmark.cpp @@ -10,6 +10,8 @@ namespace Carbon::Testing { namespace { +using Lex::LexedStringLiteral; + static void BM_ValidString(benchmark::State& state, std::string_view introducer, std::string_view terminator) { std::string x(introducer); diff --git a/toolchain/lexer/string_literal_fuzzer.cpp b/toolchain/lexer/string_literal_fuzzer.cpp index 46ef59d25870..deceb3546958 100644 --- a/toolchain/lexer/string_literal_fuzzer.cpp +++ b/toolchain/lexer/string_literal_fuzzer.cpp @@ -14,7 +14,7 @@ namespace Carbon::Testing { // NOLINTNEXTLINE: Match the documented fuzzer entry point declaration style. extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, std::size_t size) { - auto token = LexedStringLiteral::Lex( + auto token = Lex::LexedStringLiteral::Lex( llvm::StringRef(reinterpret_cast(data), size)); if (!token) { // Lexically not a string literal. diff --git a/toolchain/lexer/string_literal_test.cpp b/toolchain/lexer/string_literal_test.cpp index acfbcae88e1b..b1c594c5b3cb 100644 --- a/toolchain/lexer/string_literal_test.cpp +++ b/toolchain/lexer/string_literal_test.cpp @@ -14,6 +14,8 @@ namespace Carbon::Testing { namespace { +using Lex::LexedStringLiteral; + class StringLiteralTest : public ::testing::Test { protected: StringLiteralTest() : error_tracker(ConsoleDiagnosticConsumer()) {} diff --git a/toolchain/lexer/token_kind.cpp b/toolchain/lexer/token_kind.cpp index f22fb5e47e8e..1eb67b8d6cd7 100644 --- a/toolchain/lexer/token_kind.cpp +++ b/toolchain/lexer/token_kind.cpp @@ -4,7 +4,7 @@ #include "toolchain/lexer/token_kind.h" -namespace Carbon { +namespace Carbon::Lex { CARBON_DEFINE_ENUM_CLASS_NAMES(TokenKind) = { #define CARBON_TOKEN(TokenName) CARBON_ENUM_CLASS_NAME_STRING(TokenName) @@ -79,4 +79,4 @@ constexpr int8_t TokenKind::ExpectedParseTreeSize[] = { #include "toolchain/lexer/token_kind.def" }; -} // namespace Carbon +} // namespace Carbon::Lex diff --git a/toolchain/lexer/token_kind.h b/toolchain/lexer/token_kind.h index 04615f0ac19e..2962ea1fd2bc 100644 --- a/toolchain/lexer/token_kind.h +++ b/toolchain/lexer/token_kind.h @@ -13,7 +13,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Support/FormatVariadicDetails.h" -namespace Carbon { +namespace Carbon::Lex { CARBON_DEFINE_RAW_ENUM_CLASS(TokenKind, uint8_t) { #define CARBON_TOKEN(TokenName) CARBON_RAW_ENUM_ENUMERATOR(TokenName) @@ -138,15 +138,15 @@ constexpr TokenKind TokenKind::KeywordTokensStorage[] = { constexpr llvm::ArrayRef TokenKind::KeywordTokens = KeywordTokensStorage; -} // namespace Carbon +} // namespace Carbon::Lex namespace llvm { // We use formatv primarily for diagnostics. In these cases, it's expected that // the spelling in source code should be used. template <> -struct format_provider { - static void format(const Carbon::TokenKind& kind, raw_ostream& out, +struct format_provider { + static void format(const Carbon::Lex::TokenKind& kind, raw_ostream& out, StringRef /*style*/) { auto spelling = kind.fixed_spelling(); if (!spelling.empty()) { diff --git a/toolchain/lexer/token_kind_test.cpp b/toolchain/lexer/token_kind_test.cpp index ad07657370cc..8bd6d966927a 100644 --- a/toolchain/lexer/token_kind_test.cpp +++ b/toolchain/lexer/token_kind_test.cpp @@ -12,6 +12,7 @@ namespace Carbon::Testing { namespace { +using Lex::TokenKind; using ::testing::MatchesRegex; // We restrict symbols to punctuation characters that are expected to be widely diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index d835c27ae975..2540cd4c9b2b 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -25,7 +25,7 @@ #include #endif -namespace Carbon { +namespace Carbon::Lex { // TODO: Move Overload and VariantMatch somewhere more central. @@ -1213,4 +1213,4 @@ auto TokenizedBuffer::TokenLocationTranslator::GetLocation(Token token) return SourceBufferLocationTranslator(buffer_).GetLocation(token_start); } -} // namespace Carbon +} // namespace Carbon::Lex diff --git a/toolchain/lexer/tokenized_buffer.h b/toolchain/lexer/tokenized_buffer.h index 7ece0b6cfddc..e2be132a299d 100644 --- a/toolchain/lexer/tokenized_buffer.h +++ b/toolchain/lexer/tokenized_buffer.h @@ -21,9 +21,7 @@ #include "toolchain/lexer/token_kind.h" #include "toolchain/source/source_buffer.h" -namespace Carbon { - -class TokenizedBuffer; +namespace Carbon::Lex { // A buffer of tokenized Carbon source code. // @@ -422,6 +420,6 @@ using LexerDiagnosticEmitter = DiagnosticEmitter; // A diagnostic emitter that uses tokens as its source of location information. using TokenDiagnosticEmitter = DiagnosticEmitter; -} // namespace Carbon +} // namespace Carbon::Lex #endif // CARBON_TOOLCHAIN_LEXER_TOKENIZED_BUFFER_H_ diff --git a/toolchain/lexer/tokenized_buffer_benchmark.cpp b/toolchain/lexer/tokenized_buffer_benchmark.cpp index 1affe01a6e8a..5080474b95e0 100644 --- a/toolchain/lexer/tokenized_buffer_benchmark.cpp +++ b/toolchain/lexer/tokenized_buffer_benchmark.cpp @@ -17,6 +17,9 @@ namespace Carbon::Testing { namespace { +using Lex::TokenizedBuffer; +using Lex::TokenKind; + // A large value for measurement stability without making benchmarking too slow. // Needs to be a multiple of 100 so we can easily divide it up into percentages, // and 1% itself needs to not be too tiny. This makes 100,000 a great balance. diff --git a/toolchain/lexer/tokenized_buffer_fuzzer.cpp b/toolchain/lexer/tokenized_buffer_fuzzer.cpp index 9e2161553443..55103e4c3e13 100644 --- a/toolchain/lexer/tokenized_buffer_fuzzer.cpp +++ b/toolchain/lexer/tokenized_buffer_fuzzer.cpp @@ -32,7 +32,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, /*RequiresNullTerminator=*/false))); auto source = SourceBuffer::CreateFromFile(fs, TestFileName); - auto buffer = TokenizedBuffer::Lex(*source, NullDiagnosticConsumer()); + auto buffer = Lex::TokenizedBuffer::Lex(*source, NullDiagnosticConsumer()); if (buffer.has_errors()) { return 0; } @@ -41,7 +41,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, // // TODO: We should enhance this to do more sanity checks on the resulting // token stream. - for (TokenizedBuffer::Token token : buffer.tokens()) { + for (Lex::TokenizedBuffer::Token token : buffer.tokens()) { int line_number = buffer.GetLineNumber(token); CARBON_CHECK(line_number > 0) << "Invalid line number!"; CARBON_CHECK(line_number < INT_MAX) << "Invalid line number!"; diff --git a/toolchain/lexer/tokenized_buffer_test.cpp b/toolchain/lexer/tokenized_buffer_test.cpp index 78b6b101a4d2..722e1a661fd4 100644 --- a/toolchain/lexer/tokenized_buffer_test.cpp +++ b/toolchain/lexer/tokenized_buffer_test.cpp @@ -20,6 +20,8 @@ namespace Carbon::Testing { namespace { +using Lex::TokenizedBuffer; +using Lex::TokenKind; using ::testing::_; using ::testing::ElementsAre; using ::testing::Eq; diff --git a/toolchain/lexer/tokenized_buffer_test_helpers.h b/toolchain/lexer/tokenized_buffer_test_helpers.h index dd5a9a545b92..37b2d93a49e6 100644 --- a/toolchain/lexer/tokenized_buffer_test_helpers.h +++ b/toolchain/lexer/tokenized_buffer_test_helpers.h @@ -40,7 +40,7 @@ struct ExpectedToken { return output; } - TokenKind kind; + Lex::TokenKind kind; int line = -1; int column = -1; int indent_column = -1; @@ -54,7 +54,7 @@ struct ExpectedToken { // mismatches first. // NOLINTNEXTLINE: Expands from GoogleTest. MATCHER_P(HasTokens, raw_all_expected, "") { - const TokenizedBuffer& buffer = arg; + const Lex::TokenizedBuffer& buffer = arg; llvm::ArrayRef all_expected = raw_all_expected; bool matches = true; @@ -68,7 +68,7 @@ MATCHER_P(HasTokens, raw_all_expected, "") { int index = buffer_it - buffer.tokens().begin(); auto token = *buffer_it++; - TokenKind actual_kind = buffer.GetKind(token); + Lex::TokenKind actual_kind = buffer.GetKind(token); if (actual_kind != expected.kind) { *result_listener << "\nToken " << index << " is a " << actual_kind << ", expected a " << expected.kind << "."; @@ -119,8 +119,9 @@ MATCHER_P(HasTokens, raw_all_expected, "") { } CARBON_CHECK(!expected.string_contents || - expected.kind == TokenKind::StringLiteral); - if (expected.string_contents && actual_kind == TokenKind::StringLiteral) { + expected.kind == Lex::TokenKind::StringLiteral); + if (expected.string_contents && + actual_kind == Lex::TokenKind::StringLiteral) { llvm::StringRef actual_contents = buffer.GetStringLiteral(token); if (actual_contents != *expected.string_contents) { *result_listener << "\nToken " << index << " has contents `" diff --git a/toolchain/parser/parse_tree.cpp b/toolchain/parser/parse_tree.cpp index 2ea31183e447..f7342a9d3ce8 100644 --- a/toolchain/parser/parse_tree.cpp +++ b/toolchain/parser/parse_tree.cpp @@ -15,10 +15,10 @@ namespace Carbon::Parse { -auto Tree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer, +auto Tree::Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer, llvm::raw_ostream* vlog_stream) -> Tree { - TokenizedBuffer::TokenLocationTranslator translator(&tokens); - TokenDiagnosticEmitter emitter(translator, consumer); + Lex::TokenizedBuffer::TokenLocationTranslator translator(&tokens); + Lex::TokenDiagnosticEmitter emitter(translator, consumer); // Delegate to the parser. Tree tree(tokens); @@ -30,7 +30,7 @@ auto Tree::Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer, // The package should always be the first token, if it's present. Any other // use is invalid. - if (context.PositionIs(TokenKind::Package)) { + if (context.PositionIs(Lex::TokenKind::Package)) { context.PushState(State::Package); } @@ -95,7 +95,7 @@ auto Tree::node_kind(Node n) const -> NodeKind { return node_impls_[n.index].kind; } -auto Tree::node_token(Node n) const -> TokenizedBuffer::Token { +auto Tree::node_token(Node n) const -> Lex::TokenizedBuffer::Token { CARBON_CHECK(n.is_valid()); return node_impls_[n.index].token; } @@ -281,7 +281,7 @@ auto Tree::Verify() const -> ErrorOr { tokens_->expected_parse_tree_size()) { return Error( llvm::formatv("Tree has {0} nodes and no errors, but " - "TokenizedBuffer expected {1} nodes for {2} tokens.", + "Lex::TokenizedBuffer expected {1} nodes for {2} tokens.", node_impls_.size(), tokens_->expected_parse_tree_size(), tokens_->size())); } diff --git a/toolchain/parser/parse_tree.h b/toolchain/parser/parse_tree.h index 85dc97502279..68ecec1c8f00 100644 --- a/toolchain/parser/parse_tree.h +++ b/toolchain/parser/parse_tree.h @@ -65,7 +65,7 @@ class Tree : public Printable { // Parses the token buffer into a `Tree`. // // This is the factory function which is used to build parse trees. - static auto Parse(TokenizedBuffer& tokens, DiagnosticConsumer& consumer, + static auto Parse(Lex::TokenizedBuffer& tokens, DiagnosticConsumer& consumer, llvm::raw_ostream* vlog_stream) -> Tree; // Tests whether there are any errors in the parse tree. @@ -103,7 +103,7 @@ class Tree : public Printable { [[nodiscard]] auto node_kind(Node n) const -> NodeKind; // Returns the token the given parse tree node models. - [[nodiscard]] auto node_token(Node n) const -> TokenizedBuffer::Token; + [[nodiscard]] auto node_token(Node n) const -> Lex::TokenizedBuffer::Token; [[nodiscard]] auto node_subtree_size(Node n) const -> int32_t; @@ -171,7 +171,7 @@ class Tree : public Printable { // tree. struct NodeImpl { explicit NodeImpl(NodeKind kind, bool has_error, - TokenizedBuffer::Token token, int subtree_size) + Lex::TokenizedBuffer::Token token, int subtree_size) : kind(kind), has_error(has_error), token(token), @@ -199,7 +199,7 @@ class Tree : public Printable { bool has_error = false; // The token root of this node. - TokenizedBuffer::Token token; + Lex::TokenizedBuffer::Token token; // The size of this node's subtree of the parse tree. This is the number of // nodes (and thus tokens) that are covered by this node (and its @@ -223,7 +223,7 @@ class Tree : public Printable { // Wires up the reference to the tokenized buffer. The `Parse` function should // be used to actually parse the tokens into a tree. - explicit Tree(TokenizedBuffer& tokens_arg) : tokens_(&tokens_arg) { + explicit Tree(Lex::TokenizedBuffer& tokens_arg) : tokens_(&tokens_arg) { // If the tree is valid, there will be one node per token, so reserve once. node_impls_.reserve(tokens_->expected_parse_tree_size()); } @@ -236,7 +236,7 @@ class Tree : public Printable { // Depth-first postorder sequence of node implementation data. llvm::SmallVector node_impls_; - TokenizedBuffer* tokens_; + Lex::TokenizedBuffer* tokens_; // Indicates if any errors were encountered while parsing. // diff --git a/toolchain/parser/parse_tree_fuzzer.cpp b/toolchain/parser/parse_tree_fuzzer.cpp index 957187a4bf50..5123d05572e8 100644 --- a/toolchain/parser/parse_tree_fuzzer.cpp +++ b/toolchain/parser/parse_tree_fuzzer.cpp @@ -30,7 +30,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, auto source = SourceBuffer::CreateFromFile(fs, TestFileName); // Lex the input. - auto tokens = TokenizedBuffer::Lex(*source, NullDiagnosticConsumer()); + auto tokens = Lex::TokenizedBuffer::Lex(*source, NullDiagnosticConsumer()); if (tokens.has_errors()) { return 0; } diff --git a/toolchain/parser/parse_tree_node_location_translator.h b/toolchain/parser/parse_tree_node_location_translator.h index 91bc044bd954..b4d40a6c1bde 100644 --- a/toolchain/parser/parse_tree_node_location_translator.h +++ b/toolchain/parser/parse_tree_node_location_translator.h @@ -11,7 +11,7 @@ namespace Carbon::Parse { class NodeLocationTranslator : public DiagnosticLocationTranslator { public: - explicit NodeLocationTranslator(const TokenizedBuffer* tokens, + explicit NodeLocationTranslator(const Lex::TokenizedBuffer* tokens, const Tree* parse_tree) : token_translator_(tokens), parse_tree_(parse_tree) {} @@ -21,7 +21,7 @@ class NodeLocationTranslator : public DiagnosticLocationTranslator { } private: - TokenizedBuffer::TokenLocationTranslator token_translator_; + Lex::TokenizedBuffer::TokenLocationTranslator token_translator_; const Tree* parse_tree_; }; diff --git a/toolchain/parser/parse_tree_test.cpp b/toolchain/parser/parse_tree_test.cpp index 10aa387bc679..5b44119bd492 100644 --- a/toolchain/parser/parse_tree_test.cpp +++ b/toolchain/parser/parse_tree_test.cpp @@ -31,26 +31,26 @@ class TreeTest : public ::testing::Test { return source_storage.front(); } - auto GetTokenizedBuffer(llvm::StringRef t) -> TokenizedBuffer& { + auto GetTokenizedBuffer(llvm::StringRef t) -> Lex::TokenizedBuffer& { token_storage.push_front( - TokenizedBuffer::Lex(GetSourceBuffer(t), consumer)); + Lex::TokenizedBuffer::Lex(GetSourceBuffer(t), consumer)); return token_storage.front(); } llvm::vfs::InMemoryFileSystem fs; std::forward_list source_storage; - std::forward_list token_storage; + std::forward_list token_storage; DiagnosticConsumer& consumer = ConsoleDiagnosticConsumer(); }; TEST_F(TreeTest, IsValid) { - TokenizedBuffer tokens = GetTokenizedBuffer(""); + Lex::TokenizedBuffer tokens = GetTokenizedBuffer(""); Tree tree = Tree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_TRUE((*tree.postorder().begin()).is_valid()); } TEST_F(TreeTest, PrintPostorderAsYAML) { - TokenizedBuffer tokens = GetTokenizedBuffer("fn F();"); + Lex::TokenizedBuffer tokens = GetTokenizedBuffer("fn F();"); Tree tree = Tree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); TestRawOstream print_stream; @@ -72,7 +72,7 @@ TEST_F(TreeTest, PrintPostorderAsYAML) { } TEST_F(TreeTest, PrintPreorderAsYAML) { - TokenizedBuffer tokens = GetTokenizedBuffer("fn F();"); + Lex::TokenizedBuffer tokens = GetTokenizedBuffer("fn F();"); Tree tree = Tree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); EXPECT_FALSE(tree.has_errors()); TestRawOstream print_stream; @@ -112,7 +112,7 @@ TEST_F(TreeTest, HighRecursion) { code.append(10000, '('); code.append(10000, ')'); code += "; }"; - TokenizedBuffer tokens = GetTokenizedBuffer(code); + Lex::TokenizedBuffer tokens = GetTokenizedBuffer(code); ASSERT_FALSE(tokens.has_errors()); Testing::MockDiagnosticConsumer consumer; Tree tree = Tree::Parse(tokens, consumer, /*vlog_stream=*/nullptr); diff --git a/toolchain/parser/parser_context.cpp b/toolchain/parser/parser_context.cpp index 15700e53dd52..4f28ca57523f 100644 --- a/toolchain/parser/parser_context.cpp +++ b/toolchain/parser/parser_context.cpp @@ -40,8 +40,8 @@ static auto operator<<(llvm::raw_ostream& out, RelativeLocation loc) return out; } -Context::Context(Tree& tree, TokenizedBuffer& tokens, - TokenDiagnosticEmitter& emitter, +Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens, + Lex::TokenDiagnosticEmitter& emitter, llvm::raw_ostream* vlog_stream) : tree_(&tree), tokens_(&tokens), @@ -51,12 +51,12 @@ Context::Context(Tree& tree, TokenizedBuffer& tokens, end_(tokens_->tokens().end()) { CARBON_CHECK(position_ != end_) << "Empty TokenizedBuffer"; --end_; - CARBON_CHECK(tokens_->GetKind(*end_) == TokenKind::EndOfFile) + CARBON_CHECK(tokens_->GetKind(*end_) == Lex::TokenKind::EndOfFile) << "TokenizedBuffer should end with EndOfFile, ended with " << tokens_->GetKind(*end_); } -auto Context::AddLeafNode(NodeKind kind, TokenizedBuffer::Token token, +auto Context::AddLeafNode(NodeKind kind, Lex::TokenizedBuffer::Token token, bool has_error) -> void { tree_->node_impls_.push_back( Tree::NodeImpl(kind, has_error, token, /*subtree_size=*/1)); @@ -65,7 +65,7 @@ auto Context::AddLeafNode(NodeKind kind, TokenizedBuffer::Token token, } } -auto Context::AddNode(NodeKind kind, TokenizedBuffer::Token token, +auto Context::AddNode(NodeKind kind, Lex::TokenizedBuffer::Token token, int subtree_start, bool has_error) -> void { int subtree_size = tree_->size() - subtree_start + 1; tree_->node_impls_.push_back( @@ -75,15 +75,15 @@ auto Context::AddNode(NodeKind kind, TokenizedBuffer::Token token, } } -auto Context::ConsumeAndAddOpenParen(TokenizedBuffer::Token default_token, +auto Context::ConsumeAndAddOpenParen(Lex::TokenizedBuffer::Token default_token, NodeKind start_kind) - -> std::optional { - if (auto open_paren = ConsumeIf(TokenKind::OpenParen)) { + -> std::optional { + if (auto open_paren = ConsumeIf(Lex::TokenKind::OpenParen)) { AddLeafNode(start_kind, *open_paren, /*has_error=*/false); return open_paren; } else { CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, "Expected `(` after `{0}`.", - TokenKind); + Lex::TokenKind); emitter_->Emit(*position_, ExpectedParenAfter, tokens().GetKind(default_token)); AddLeafNode(start_kind, default_token, /*has_error=*/true); @@ -91,10 +91,10 @@ auto Context::ConsumeAndAddOpenParen(TokenizedBuffer::Token default_token, } } -auto Context::ConsumeAndAddCloseSymbol(TokenizedBuffer::Token expected_open, - StateStackEntry state, - NodeKind close_kind) -> void { - TokenKind open_token_kind = tokens().GetKind(expected_open); +auto Context::ConsumeAndAddCloseSymbol( + Lex::TokenizedBuffer::Token expected_open, StateStackEntry state, + NodeKind close_kind) -> void { + Lex::TokenKind open_token_kind = tokens().GetKind(expected_open); if (!open_token_kind.is_opening_symbol()) { AddNode(close_kind, state.token, state.subtree_start, /*has_error=*/true); @@ -113,8 +113,8 @@ auto Context::ConsumeAndAddCloseSymbol(TokenizedBuffer::Token expected_open, } } -auto Context::ConsumeAndAddLeafNodeIf(TokenKind token_kind, NodeKind node_kind) - -> bool { +auto Context::ConsumeAndAddLeafNodeIf(Lex::TokenKind token_kind, + NodeKind node_kind) -> bool { auto token = ConsumeIf(token_kind); if (!token) { return false; @@ -124,21 +124,22 @@ auto Context::ConsumeAndAddLeafNodeIf(TokenKind token_kind, NodeKind node_kind) return true; } -auto Context::ConsumeChecked(TokenKind kind) -> TokenizedBuffer::Token { +auto Context::ConsumeChecked(Lex::TokenKind kind) + -> Lex::TokenizedBuffer::Token { CARBON_CHECK(PositionIs(kind)) << "Required " << kind << ", found " << PositionKind(); return Consume(); } -auto Context::ConsumeIf(TokenKind kind) - -> std::optional { +auto Context::ConsumeIf(Lex::TokenKind kind) + -> std::optional { if (!PositionIs(kind)) { return std::nullopt; } return Consume(); } -auto Context::ConsumeIfPatternKeyword(TokenKind keyword_token, +auto Context::ConsumeIfPatternKeyword(Lex::TokenKind keyword_token, State keyword_state, int subtree_start) -> void { if (auto token = ConsumeIf(keyword_token)) { @@ -148,22 +149,22 @@ auto Context::ConsumeIfPatternKeyword(TokenKind keyword_token, } } -auto Context::FindNextOf(std::initializer_list desired_kinds) - -> std::optional { +auto Context::FindNextOf(std::initializer_list desired_kinds) + -> std::optional { auto new_position = position_; while (true) { - TokenizedBuffer::Token token = *new_position; - TokenKind kind = tokens().GetKind(token); + Lex::TokenizedBuffer::Token token = *new_position; + Lex::TokenKind kind = tokens().GetKind(token); if (kind.IsOneOf(desired_kinds)) { return token; } // Step to the next token at the current bracketing level. - if (kind.is_closing_symbol() || kind == TokenKind::EndOfFile) { + if (kind.is_closing_symbol() || kind == Lex::TokenKind::EndOfFile) { // There are no more tokens at this level. return std::nullopt; } else if (kind.is_opening_symbol()) { - new_position = TokenizedBuffer::TokenIterator( + new_position = Lex::TokenizedBuffer::TokenIterator( tokens().GetMatchedClosingToken(token)); // Advance past the closing token. ++new_position; @@ -183,20 +184,20 @@ auto Context::SkipMatchingGroup() -> bool { return true; } -auto Context::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) - -> std::optional { +auto Context::SkipPastLikelyEnd(Lex::TokenizedBuffer::Token skip_root) + -> std::optional { if (position_ == end_) { return std::nullopt; } - TokenizedBuffer::Line root_line = tokens().GetLine(skip_root); + Lex::TokenizedBuffer::Line root_line = tokens().GetLine(skip_root); int root_line_indent = tokens().GetIndentColumnNumber(root_line); // We will keep scanning through tokens on the same line as the root or // lines with greater indentation than root's line. auto is_same_line_or_indent_greater_than_root = - [&](TokenizedBuffer::Token t) { - TokenizedBuffer::Line l = tokens().GetLine(t); + [&](Lex::TokenizedBuffer::Token t) { + Lex::TokenizedBuffer::Line l = tokens().GetLine(t); if (l == root_line) { return true; } @@ -205,7 +206,7 @@ auto Context::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) }; do { - if (PositionIs(TokenKind::CloseCurlyBrace)) { + if (PositionIs(Lex::TokenKind::CloseCurlyBrace)) { // Immediately bail out if we hit an unmatched close curly, this will // pop us up a level of the syntax grouping. return std::nullopt; @@ -213,7 +214,7 @@ auto Context::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) // We assume that a semicolon is always intended to be the end of the // current construct. - if (auto semi = ConsumeIf(TokenKind::Semi)) { + if (auto semi = ConsumeIf(Lex::TokenKind::Semi)) { return semi; } @@ -230,37 +231,41 @@ auto Context::SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) return std::nullopt; } -auto Context::SkipTo(TokenizedBuffer::Token t) -> void { - CARBON_CHECK(t >= *position_) << "Tried to skip backwards from " << position_ - << " to " << TokenizedBuffer::TokenIterator(t); - position_ = TokenizedBuffer::TokenIterator(t); +auto Context::SkipTo(Lex::TokenizedBuffer::Token t) -> void { + CARBON_CHECK(t >= *position_) + << "Tried to skip backwards from " << position_ << " to " + << Lex::TokenizedBuffer::TokenIterator(t); + position_ = Lex::TokenizedBuffer::TokenIterator(t); CARBON_CHECK(position_ != end_) << "Skipped past EOF."; } // Determines whether the given token is considered to be the start of an // operand according to the rules for infix operator parsing. -static auto IsAssumedStartOfOperand(TokenKind kind) -> bool { - return kind.IsOneOf({TokenKind::OpenParen, TokenKind::Identifier, - TokenKind::IntegerLiteral, TokenKind::RealLiteral, - TokenKind::StringLiteral}); +static auto IsAssumedStartOfOperand(Lex::TokenKind kind) -> bool { + return kind.IsOneOf({Lex::TokenKind::OpenParen, Lex::TokenKind::Identifier, + Lex::TokenKind::IntegerLiteral, + Lex::TokenKind::RealLiteral, + Lex::TokenKind::StringLiteral}); } // Determines whether the given token is considered to be the end of an // operand according to the rules for infix operator parsing. -static auto IsAssumedEndOfOperand(TokenKind kind) -> bool { - return kind.IsOneOf({TokenKind::CloseParen, TokenKind::CloseCurlyBrace, - TokenKind::CloseSquareBracket, TokenKind::Identifier, - TokenKind::IntegerLiteral, TokenKind::RealLiteral, - TokenKind::StringLiteral}); +static auto IsAssumedEndOfOperand(Lex::TokenKind kind) -> bool { + return kind.IsOneOf( + {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseCurlyBrace, + Lex::TokenKind::CloseSquareBracket, Lex::TokenKind::Identifier, + Lex::TokenKind::IntegerLiteral, Lex::TokenKind::RealLiteral, + Lex::TokenKind::StringLiteral}); } // Determines whether the given token could possibly be the start of an // operand. This is conservatively correct, and will never incorrectly return // `false`, but can incorrectly return `true`. -static auto IsPossibleStartOfOperand(TokenKind kind) -> bool { - return !kind.IsOneOf({TokenKind::CloseParen, TokenKind::CloseCurlyBrace, - TokenKind::CloseSquareBracket, TokenKind::Comma, - TokenKind::Semi, TokenKind::Colon}); +static auto IsPossibleStartOfOperand(Lex::TokenKind kind) -> bool { + return !kind.IsOneOf( + {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseCurlyBrace, + Lex::TokenKind::CloseSquareBracket, Lex::TokenKind::Comma, + Lex::TokenKind::Semi, Lex::TokenKind::Colon}); } auto Context::IsLexicallyValidInfixOperator() -> bool { @@ -359,19 +364,19 @@ auto Context::DiagnoseOperatorFixity(OperatorFixity fixity) -> void { } } -auto Context::ConsumeListToken(NodeKind comma_kind, TokenKind close_kind, +auto Context::ConsumeListToken(NodeKind comma_kind, Lex::TokenKind close_kind, bool already_has_error) -> ListTokenKind { - if (!PositionIs(TokenKind::Comma) && !PositionIs(close_kind)) { + if (!PositionIs(Lex::TokenKind::Comma) && !PositionIs(close_kind)) { // Don't error a second time on the same element. if (!already_has_error) { CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error, - "Expected `,` or `{0}`.", TokenKind); + "Expected `,` or `{0}`.", Lex::TokenKind); emitter_->Emit(*position_, UnexpectedTokenAfterListElement, close_kind); ReturnErrorOnState(); } // Recover from the invalid token. - auto end_of_element = FindNextOf({TokenKind::Comma, close_kind}); + auto end_of_element = FindNextOf({Lex::TokenKind::Comma, close_kind}); // The lexer guarantees that parentheses are balanced. CARBON_CHECK(end_of_element) << "missing matching `" << close_kind.opening_symbol() << "` for `" @@ -426,18 +431,19 @@ auto Context::RecoverFromDeclarationError(StateStackEntry state, /*has_error=*/true); } -auto Context::EmitExpectedDeclarationSemi(TokenKind expected_kind) -> void { +auto Context::EmitExpectedDeclarationSemi(Lex::TokenKind expected_kind) + -> void { CARBON_DIAGNOSTIC(ExpectedDeclarationSemi, Error, - "`{0}` declarations must end with a `;`.", TokenKind); + "`{0}` declarations must end with a `;`.", Lex::TokenKind); emitter().Emit(*position(), ExpectedDeclarationSemi, expected_kind); } -auto Context::EmitExpectedDeclarationSemiOrDefinition(TokenKind expected_kind) - -> void { +auto Context::EmitExpectedDeclarationSemiOrDefinition( + Lex::TokenKind expected_kind) -> void { CARBON_DIAGNOSTIC(ExpectedDeclarationSemiOrDefinition, Error, "`{0}` declarations must either end with a `;` or " "have a `{{ ... }` block for a definition.", - TokenKind); + Lex::TokenKind); emitter().Emit(*position(), ExpectedDeclarationSemiOrDefinition, expected_kind); } @@ -453,7 +459,7 @@ auto Context::PrintForStackDump(llvm::raw_ostream& output) const -> void { } auto Context::PrintTokenForStackDump(llvm::raw_ostream& output, - TokenizedBuffer::Token token) const + Lex::TokenizedBuffer::Token token) const -> void { output << " @ " << tokens_->GetLineNumber(tokens_->GetLine(token)) << ":" << tokens_->GetColumnNumber(token) << ": token " << token << " : " diff --git a/toolchain/parser/parser_context.h b/toolchain/parser/parser_context.h index 7136edaa2948..a0416df7f0cd 100644 --- a/toolchain/parser/parser_context.h +++ b/toolchain/parser/parser_context.h @@ -43,7 +43,7 @@ class Context { struct StateStackEntry : public Printable { explicit StateStackEntry(State state, PrecedenceGroup ambient_precedence, PrecedenceGroup lhs_precedence, - TokenizedBuffer::Token token, + Lex::TokenizedBuffer::Token token, int32_t subtree_start) : state(state), ambient_precedence(ambient_precedence), @@ -73,7 +73,7 @@ class Context { // A token providing context based on the subtree. This will typically be // the first token in the subtree, but may sometimes be a token within. It // will typically be used for the subtree's root node. - TokenizedBuffer::Token token; + Lex::TokenizedBuffer::Token token; // The offset within the Tree of the subtree start. int32_t subtree_start; }; @@ -90,54 +90,55 @@ class Context { static_assert(sizeof(StateStackEntry) == 12, "StateStackEntry has unexpected size!"); - explicit Context(Tree& tree, TokenizedBuffer& tokens, - TokenDiagnosticEmitter& emitter, + explicit Context(Tree& tree, Lex::TokenizedBuffer& tokens, + Lex::TokenDiagnosticEmitter& emitter, llvm::raw_ostream* vlog_stream); // Adds a node to the parse tree that has no children (a leaf). - auto AddLeafNode(NodeKind kind, TokenizedBuffer::Token token, + auto AddLeafNode(NodeKind kind, Lex::TokenizedBuffer::Token token, bool has_error = false) -> void; // Adds a node to the parse tree that has children. - auto AddNode(NodeKind kind, TokenizedBuffer::Token token, int subtree_start, - bool has_error) -> void; + auto AddNode(NodeKind kind, Lex::TokenizedBuffer::Token token, + int subtree_start, bool has_error) -> void; // Returns the current position and moves past it. - auto Consume() -> TokenizedBuffer::Token { return *(position_++); } + auto Consume() -> Lex::TokenizedBuffer::Token { return *(position_++); } // Parses an open paren token, possibly diagnosing if necessary. Creates a // leaf parse node of the specified start kind. The default_token is used when // there's no open paren. Returns the open paren token if it was found. - auto ConsumeAndAddOpenParen(TokenizedBuffer::Token default_token, + auto ConsumeAndAddOpenParen(Lex::TokenizedBuffer::Token default_token, NodeKind start_kind) - -> std::optional; + -> std::optional; // Parses a closing symbol corresponding to the opening symbol // `expected_open`, possibly skipping forward and diagnosing if necessary. // Creates a parse node of the specified close kind. If `expected_open` is not // an opening symbol, the parse node will be associated with `state.token`, // no input will be consumed, and no diagnostic will be emitted. - auto ConsumeAndAddCloseSymbol(TokenizedBuffer::Token expected_open, + auto ConsumeAndAddCloseSymbol(Lex::TokenizedBuffer::Token expected_open, StateStackEntry state, NodeKind close_kind) -> void; // Composes `ConsumeIf` and `AddLeafNode`, returning false when ConsumeIf // fails. - auto ConsumeAndAddLeafNodeIf(TokenKind token_kind, NodeKind node_kind) + auto ConsumeAndAddLeafNodeIf(Lex::TokenKind token_kind, NodeKind node_kind) -> bool; // Returns the current position and moves past it. Requires the token is the // expected kind. - auto ConsumeChecked(TokenKind kind) -> TokenizedBuffer::Token; + auto ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenizedBuffer::Token; // If the current position's token matches this `Kind`, returns it and // advances to the next position. Otherwise returns an empty optional. - auto ConsumeIf(TokenKind kind) -> std::optional; + auto ConsumeIf(Lex::TokenKind kind) + -> std::optional; // Find the next token of any of the given kinds at the current bracketing // level. - auto FindNextOf(std::initializer_list desired_kinds) - -> std::optional; + auto FindNextOf(std::initializer_list desired_kinds) + -> std::optional; // If the token is an opening symbol for a matched group, skips to the matched // closing symbol and returns true. Otherwise, returns false. @@ -159,11 +160,11 @@ class Context { // declarations or statements across multiple lines should be indented. // // Returns a semicolon token if one is the likely end. - auto SkipPastLikelyEnd(TokenizedBuffer::Token skip_root) - -> std::optional; + auto SkipPastLikelyEnd(Lex::TokenizedBuffer::Token skip_root) + -> std::optional; // Skip forward to the given token. Verifies that it is actually forward. - auto SkipTo(TokenizedBuffer::Token t) -> void; + auto SkipTo(Lex::TokenizedBuffer::Token t) -> void; // Returns true if the current token satisfies the lexical validity rules // for an infix operator. @@ -184,16 +185,16 @@ class Context { // `,)`). Handles cases where invalid tokens are present by advancing the // position, and may emit errors. Pass already_has_error in order to suppress // duplicate errors. - auto ConsumeListToken(NodeKind comma_kind, TokenKind close_kind, + auto ConsumeListToken(NodeKind comma_kind, Lex::TokenKind close_kind, bool already_has_error) -> ListTokenKind; // Gets the kind of the next token to be consumed. - auto PositionKind() const -> TokenKind { + auto PositionKind() const -> Lex::TokenKind { return tokens_->GetKind(*position_); } // Tests whether the next token to be consumed is of the specified kind. - auto PositionIs(TokenKind kind) const -> bool { + auto PositionIs(Lex::TokenKind kind) const -> bool { return PositionKind() == kind; } @@ -220,7 +221,7 @@ class Context { // Pushes a new state with a specific token for context. Used when forming a // new subtree with a token that isn't the start of the subtree. - auto PushState(State state, TokenizedBuffer::Token token) -> void { + auto PushState(State state, Lex::TokenizedBuffer::Token token) -> void { PushState(StateStackEntry(state, PrecedenceGroup::ForTopLevelExpression(), PrecedenceGroup::ForTopLevelExpression(), token, tree_->size())); @@ -262,14 +263,15 @@ class Context { auto ReturnErrorOnState() -> void { state_stack_.back().has_error = true; } // For HandlePattern, tries to consume a wrapping keyword. - auto ConsumeIfPatternKeyword(TokenKind keyword_token, State keyword_state, - int subtree_start) -> void; + auto ConsumeIfPatternKeyword(Lex::TokenKind keyword_token, + State keyword_state, int subtree_start) -> void; // Emits a diagnostic for a declaration missing a semi. - auto EmitExpectedDeclarationSemi(TokenKind expected_kind) -> void; + auto EmitExpectedDeclarationSemi(Lex::TokenKind expected_kind) -> void; // Emits a diagnostic for a declaration missing a semi or definition. - auto EmitExpectedDeclarationSemiOrDefinition(TokenKind expected_kind) -> void; + auto EmitExpectedDeclarationSemiOrDefinition(Lex::TokenKind expected_kind) + -> void; // Handles error recovery in a declaration, particularly before any possible // definition has started (although one could be present). Recover to a @@ -284,12 +286,14 @@ class Context { auto tree() const -> const Tree& { return *tree_; } - auto tokens() const -> const TokenizedBuffer& { return *tokens_; } + auto tokens() const -> const Lex::TokenizedBuffer& { return *tokens_; } - auto emitter() -> TokenDiagnosticEmitter& { return *emitter_; } + auto emitter() -> Lex::TokenDiagnosticEmitter& { return *emitter_; } - auto position() -> TokenizedBuffer::TokenIterator& { return position_; } - auto position() const -> TokenizedBuffer::TokenIterator { return position_; } + auto position() -> Lex::TokenizedBuffer::TokenIterator& { return position_; } + auto position() const -> Lex::TokenizedBuffer::TokenIterator { + return position_; + } auto state_stack() -> llvm::SmallVector& { return state_stack_; @@ -302,19 +306,19 @@ class Context { private: // Prints a single token for a stack dump. Used by PrintForStackDump. auto PrintTokenForStackDump(llvm::raw_ostream& output, - TokenizedBuffer::Token token) const -> void; + Lex::TokenizedBuffer::Token token) const -> void; Tree* tree_; - TokenizedBuffer* tokens_; - TokenDiagnosticEmitter* emitter_; + Lex::TokenizedBuffer* tokens_; + Lex::TokenDiagnosticEmitter* emitter_; // Whether to print verbose output. llvm::raw_ostream* vlog_stream_; // The current position within the token buffer. - TokenizedBuffer::TokenIterator position_; + Lex::TokenizedBuffer::TokenIterator position_; // The EndOfFile token. - TokenizedBuffer::TokenIterator end_; + Lex::TokenizedBuffer::TokenIterator end_; llvm::SmallVector state_stack_; }; diff --git a/toolchain/parser/parser_handle_array_expression.cpp b/toolchain/parser/parser_handle_array_expression.cpp index 02c261adba5e..8cfc4583684a 100644 --- a/toolchain/parser/parser_handle_array_expression.cpp +++ b/toolchain/parser/parser_handle_array_expression.cpp @@ -14,7 +14,7 @@ auto HandleArrayExpression(Context& context) -> void { auto state = context.PopState(); state.state = State::ArrayExpressionSemi; context.AddLeafNode(NodeKind::ArrayExpressionStart, - context.ConsumeChecked(TokenKind::OpenSquareBracket), + context.ConsumeChecked(Lex::TokenKind::OpenSquareBracket), state.has_error); context.PushState(state); context.PushState(State::Expression); @@ -22,7 +22,7 @@ auto HandleArrayExpression(Context& context) -> void { auto HandleArrayExpressionSemi(Context& context) -> void { auto state = context.PopState(); - auto semi = context.ConsumeIf(TokenKind::Semi); + auto semi = context.ConsumeIf(Lex::TokenKind::Semi); if (!semi) { context.AddNode(NodeKind::ArrayExpressionSemi, *context.position(), state.subtree_start, true); @@ -35,7 +35,7 @@ auto HandleArrayExpressionSemi(Context& context) -> void { } state.state = State::ArrayExpressionFinish; context.PushState(state); - if (!context.PositionIs(TokenKind::CloseSquareBracket)) { + if (!context.PositionIs(Lex::TokenKind::CloseSquareBracket)) { context.PushState(State::Expression); } } @@ -43,7 +43,7 @@ auto HandleArrayExpressionSemi(Context& context) -> void { auto HandleArrayExpressionFinish(Context& context) -> void { auto state = context.PopState(); context.ConsumeAndAddCloseSymbol( - *(TokenizedBuffer::TokenIterator(state.token)), state, + *(Lex::TokenizedBuffer::TokenIterator(state.token)), state, NodeKind::ArrayExpression); } diff --git a/toolchain/parser/parser_handle_brace_expression.cpp b/toolchain/parser/parser_handle_brace_expression.cpp index 699c00aa554a..4210cd7dae4c 100644 --- a/toolchain/parser/parser_handle_brace_expression.cpp +++ b/toolchain/parser/parser_handle_brace_expression.cpp @@ -13,9 +13,9 @@ auto HandleBraceExpression(Context& context) -> void { context.PushState(state); CARBON_CHECK(context.ConsumeAndAddLeafNodeIf( - TokenKind::OpenCurlyBrace, + Lex::TokenKind::OpenCurlyBrace, NodeKind::StructLiteralOrStructTypeLiteralStart)); - if (!context.PositionIs(TokenKind::CloseCurlyBrace)) { + if (!context.PositionIs(Lex::TokenKind::CloseCurlyBrace)) { context.PushState(State::BraceExpressionParameterAsUnknown); } } @@ -50,7 +50,7 @@ static auto HandleBraceExpressionParameter(Context& context, State param_finish_state) -> void { auto state = context.PopState(); - if (!context.PositionIs(TokenKind::Period)) { + if (!context.PositionIs(Lex::TokenKind::Period)) { HandleBraceExpressionParameterError(context, state, param_finish_state); return; } @@ -85,9 +85,9 @@ static auto HandleBraceExpressionParameterAfterDesignator( if (state.has_error) { auto recovery_pos = context.FindNextOf( - {TokenKind::Equal, TokenKind::Colon, TokenKind::Comma}); + {Lex::TokenKind::Equal, Lex::TokenKind::Colon, Lex::TokenKind::Comma}); if (!recovery_pos || - context.tokens().GetKind(*recovery_pos) == TokenKind::Comma) { + context.tokens().GetKind(*recovery_pos) == Lex::TokenKind::Comma) { state.state = param_finish_state; context.PushState(state); return; @@ -97,9 +97,9 @@ static auto HandleBraceExpressionParameterAfterDesignator( // Work out the kind of this element. bool is_type; - if (context.PositionIs(TokenKind::Colon)) { + if (context.PositionIs(Lex::TokenKind::Colon)) { is_type = true; - } else if (context.PositionIs(TokenKind::Equal)) { + } else if (context.PositionIs(Lex::TokenKind::Equal)) { is_type = false; } else { HandleBraceExpressionParameterError(context, state, param_finish_state); @@ -168,9 +168,9 @@ static auto HandleBraceExpressionParameterFinish(Context& context, /*has_error=*/false); } - if (context.ConsumeListToken(NodeKind::StructComma, - TokenKind::CloseCurlyBrace, state.has_error) == - Context::ListTokenKind::Comma) { + if (context.ConsumeListToken( + NodeKind::StructComma, Lex::TokenKind::CloseCurlyBrace, + state.has_error) == Context::ListTokenKind::Comma) { context.PushState(param_state); } } diff --git a/toolchain/parser/parser_handle_call_expression.cpp b/toolchain/parser/parser_handle_call_expression.cpp index fa317e8ac806..4ed63a2b7f03 100644 --- a/toolchain/parser/parser_handle_call_expression.cpp +++ b/toolchain/parser/parser_handle_call_expression.cpp @@ -14,7 +14,7 @@ auto HandleCallExpression(Context& context) -> void { context.AddNode(NodeKind::CallExpressionStart, context.Consume(), state.subtree_start, state.has_error); - if (!context.PositionIs(TokenKind::CloseParen)) { + if (!context.PositionIs(Lex::TokenKind::CloseParen)) { context.PushState(State::CallExpressionParameterFinish); context.PushState(State::Expression); } @@ -28,7 +28,7 @@ auto HandleCallExpressionParameterFinish(Context& context) -> void { } if (context.ConsumeListToken(NodeKind::CallExpressionComma, - TokenKind::CloseParen, state.has_error) == + Lex::TokenKind::CloseParen, state.has_error) == Context::ListTokenKind::Comma) { context.PushState(State::CallExpressionParameterFinish); context.PushState(State::Expression); diff --git a/toolchain/parser/parser_handle_code_block.cpp b/toolchain/parser/parser_handle_code_block.cpp index 6b31c2d44e13..6c4e6b3984c4 100644 --- a/toolchain/parser/parser_handle_code_block.cpp +++ b/toolchain/parser/parser_handle_code_block.cpp @@ -10,7 +10,7 @@ auto HandleCodeBlock(Context& context) -> void { context.PopAndDiscardState(); context.PushState(State::CodeBlockFinish); - if (context.ConsumeAndAddLeafNodeIf(TokenKind::OpenCurlyBrace, + if (context.ConsumeAndAddLeafNodeIf(Lex::TokenKind::OpenCurlyBrace, NodeKind::CodeBlockStart)) { context.PushState(State::StatementScopeLoop); } else { @@ -29,7 +29,7 @@ auto HandleCodeBlockFinish(Context& context) -> void { auto state = context.PopState(); // If the block started with an open curly, this is a close curly. - if (context.tokens().GetKind(state.token) == TokenKind::OpenCurlyBrace) { + if (context.tokens().GetKind(state.token) == Lex::TokenKind::OpenCurlyBrace) { context.AddNode(NodeKind::CodeBlock, context.Consume(), state.subtree_start, state.has_error); } else { diff --git a/toolchain/parser/parser_handle_declaration_name_and_params.cpp b/toolchain/parser/parser_handle_declaration_name_and_params.cpp index cbd352523800..ad5859072915 100644 --- a/toolchain/parser/parser_handle_declaration_name_and_params.cpp +++ b/toolchain/parser/parser_handle_declaration_name_and_params.cpp @@ -12,11 +12,11 @@ static auto HandleDeclarationNameAndParams(Context& context, State after_name) auto state = context.PopState(); // TODO: Should handle designated names. - if (auto identifier = context.ConsumeIf(TokenKind::Identifier)) { + if (auto identifier = context.ConsumeIf(Lex::TokenKind::Identifier)) { state.state = after_name; context.PushState(state); - if (context.PositionIs(TokenKind::Period)) { + if (context.PositionIs(Lex::TokenKind::Period)) { // Because there's a qualifier, we process the first segment as an // expression for simplicity. This just means semantics has one less thing // to handle here. @@ -29,7 +29,7 @@ static auto HandleDeclarationNameAndParams(Context& context, State after_name) } else { CARBON_DIAGNOSTIC(ExpectedDeclarationName, Error, "`{0}` introducer should be followed by a name.", - TokenKind); + Lex::TokenKind); context.emitter().Emit(*context.position(), ExpectedDeclarationName, context.tokens().GetKind(state.token)); context.ReturnErrorOnState(); @@ -62,7 +62,7 @@ static auto HandleDeclarationNameAndParamsAfterName(Context& context, Params params) -> void { auto state = context.PopState(); - if (context.PositionIs(TokenKind::Period)) { + if (context.PositionIs(Lex::TokenKind::Period)) { // Continue designator processing. context.PushState(state); state.state = State::PeriodAsDeclaration; @@ -74,14 +74,14 @@ static auto HandleDeclarationNameAndParamsAfterName(Context& context, return; } - if (context.PositionIs(TokenKind::OpenSquareBracket)) { + if (context.PositionIs(Lex::TokenKind::OpenSquareBracket)) { context.PushState(State::DeclarationNameAndParamsAfterDeduced); context.PushState(State::ParameterListAsDeduced); - } else if (context.PositionIs(TokenKind::OpenParen)) { + } else if (context.PositionIs(Lex::TokenKind::OpenParen)) { context.PushState(State::ParameterListAsRegular); } else if (params == Params::Required) { CARBON_DIAGNOSTIC(ParametersRequiredByIntroducer, Error, - "`{0}` requires a `(` for parameters.", TokenKind); + "`{0}` requires a `(` for parameters.", Lex::TokenKind); context.emitter().Emit(*context.position(), ParametersRequiredByIntroducer, context.tokens().GetKind(state.token)); context.ReturnErrorOnState(); @@ -105,7 +105,7 @@ auto HandleDeclarationNameAndParamsAfterNameAsRequired(Context& context) auto HandleDeclarationNameAndParamsAfterDeduced(Context& context) -> void { context.PopAndDiscardState(); - if (context.PositionIs(TokenKind::OpenParen)) { + if (context.PositionIs(Lex::TokenKind::OpenParen)) { context.PushState(State::ParameterListAsRegular); } else { CARBON_DIAGNOSTIC( diff --git a/toolchain/parser/parser_handle_declaration_scope_loop.cpp b/toolchain/parser/parser_handle_declaration_scope_loop.cpp index 3488674b55d6..f0ea0ab01d0f 100644 --- a/toolchain/parser/parser_handle_declaration_scope_loop.cpp +++ b/toolchain/parser/parser_handle_declaration_scope_loop.cpp @@ -23,37 +23,37 @@ auto HandleDeclarationScopeLoop(Context& context) -> void { // This maintains the current state unless we're at the end of the scope. switch (context.PositionKind()) { - case TokenKind::CloseCurlyBrace: - case TokenKind::EndOfFile: { + case Lex::TokenKind::CloseCurlyBrace: + case Lex::TokenKind::EndOfFile: { // This is the end of the scope, so the loop state ends. context.PopAndDiscardState(); break; } - case TokenKind::Class: { + case Lex::TokenKind::Class: { context.PushState(State::TypeIntroducerAsClass); break; } - case TokenKind::Constraint: { + case Lex::TokenKind::Constraint: { context.PushState(State::TypeIntroducerAsNamedConstraint); break; } - case TokenKind::Fn: { + case Lex::TokenKind::Fn: { context.PushState(State::FunctionIntroducer); break; } - case TokenKind::Interface: { + case Lex::TokenKind::Interface: { context.PushState(State::TypeIntroducerAsInterface); break; } - case TokenKind::Namespace: { + case Lex::TokenKind::Namespace: { context.PushState(State::Namespace); break; } - case TokenKind::Semi: { + case Lex::TokenKind::Semi: { context.AddLeafNode(NodeKind::EmptyDeclaration, context.Consume()); break; } - case TokenKind::Var: { + case Lex::TokenKind::Var: { context.PushState(State::VarAsSemicolon); break; } diff --git a/toolchain/parser/parser_handle_expression.cpp b/toolchain/parser/parser_handle_expression.cpp index f65f3f82732b..da74bdbaf4a2 100644 --- a/toolchain/parser/parser_handle_expression.cpp +++ b/toolchain/parser/parser_handle_expression.cpp @@ -9,7 +9,7 @@ namespace Carbon::Parse { static auto DiagnoseStatementOperatorAsSubexpression(Context& context) -> void { CARBON_DIAGNOSTIC(StatementOperatorAsSubexpression, Error, "Operator `{0}` can only be used as a complete statement.", - TokenKind); + Lex::TokenKind); context.emitter().Emit(*context.position(), StatementOperatorAsSubexpression, context.PositionKind()); } @@ -31,7 +31,7 @@ auto HandleExpression(Context& context) -> void { CARBON_DIAGNOSTIC( UnaryOperatorRequiresParentheses, Error, "Parentheses are required around this unary `{0}` operator.", - TokenKind); + Lex::TokenKind); context.emitter().Emit(*context.position(), UnaryOperatorRequiresParentheses, context.PositionKind()); @@ -44,7 +44,7 @@ auto HandleExpression(Context& context) -> void { context.DiagnoseOperatorFixity(Context::OperatorFixity::Prefix); } - if (context.PositionIs(TokenKind::If)) { + if (context.PositionIs(Lex::TokenKind::If)) { context.PushState(State::IfExpressionFinish); context.PushState(State::IfExpressionFinishCondition); } else { @@ -73,47 +73,47 @@ auto HandleExpressionInPostfix(Context& context) -> void { // expression tree, such as an identifier or literal, or a parenthesized // expression. switch (context.PositionKind()) { - case TokenKind::Identifier: { + case Lex::TokenKind::Identifier: { context.AddLeafNode(NodeKind::NameExpression, context.Consume()); context.PushState(state); break; } - case TokenKind::False: - case TokenKind::True: - case TokenKind::IntegerLiteral: - case TokenKind::RealLiteral: - case TokenKind::StringLiteral: - case TokenKind::Bool: - case TokenKind::IntegerTypeLiteral: - case TokenKind::UnsignedIntegerTypeLiteral: - case TokenKind::FloatingPointTypeLiteral: - case TokenKind::StringTypeLiteral: - case TokenKind::Type: { + case Lex::TokenKind::False: + case Lex::TokenKind::True: + case Lex::TokenKind::IntegerLiteral: + case Lex::TokenKind::RealLiteral: + case Lex::TokenKind::StringLiteral: + case Lex::TokenKind::Bool: + case Lex::TokenKind::IntegerTypeLiteral: + case Lex::TokenKind::UnsignedIntegerTypeLiteral: + case Lex::TokenKind::FloatingPointTypeLiteral: + case Lex::TokenKind::StringTypeLiteral: + case Lex::TokenKind::Type: { context.AddLeafNode(NodeKind::Literal, context.Consume()); context.PushState(state); break; } - case TokenKind::OpenCurlyBrace: { + case Lex::TokenKind::OpenCurlyBrace: { context.PushState(state); context.PushState(State::BraceExpression); break; } - case TokenKind::OpenParen: { + case Lex::TokenKind::OpenParen: { context.PushState(state); context.PushState(State::ParenExpression); break; } - case TokenKind::OpenSquareBracket: { + case Lex::TokenKind::OpenSquareBracket: { context.PushState(state); context.PushState(State::ArrayExpression); break; } - case TokenKind::SelfValueIdentifier: { + case Lex::TokenKind::SelfValueIdentifier: { context.AddLeafNode(NodeKind::SelfValueName, context.Consume()); context.PushState(state); break; } - case TokenKind::SelfTypeIdentifier: { + case Lex::TokenKind::SelfTypeIdentifier: { context.AddLeafNode(NodeKind::SelfTypeNameExpression, context.Consume()); context.PushState(state); break; @@ -136,25 +136,25 @@ auto HandleExpressionInPostfixLoop(Context& context) -> void { auto state = context.PopState(); state.token = *context.position(); switch (context.PositionKind()) { - case TokenKind::Period: { + case Lex::TokenKind::Period: { context.PushState(state); state.state = State::PeriodAsExpression; context.PushState(state); break; } - case TokenKind::MinusGreater: { + case Lex::TokenKind::MinusGreater: { context.PushState(state); state.state = State::ArrowExpression; context.PushState(state); break; } - case TokenKind::OpenParen: { + case Lex::TokenKind::OpenParen: { context.PushState(state); state.state = State::CallExpression; context.PushState(state); break; } - case TokenKind::OpenSquareBracket: { + case Lex::TokenKind::OpenSquareBracket: { context.PushState(state); state.state = State::IndexExpression; context.PushState(state); @@ -224,7 +224,8 @@ auto HandleExpressionLoop(Context& context) -> void { state.lhs_precedence = operator_precedence; if (is_binary) { - if (operator_kind == TokenKind::And || operator_kind == TokenKind::Or) { + if (operator_kind == Lex::TokenKind::And || + operator_kind == Lex::TokenKind::Or) { // For `and` and `or`, wrap the first operand in a virtual parse tree // node so that semantics can insert control flow here. context.AddNode(NodeKind::ShortCircuitOperand, state.token, @@ -268,10 +269,11 @@ auto HandleIfExpressionFinishCondition(Context& context) -> void { context.AddNode(NodeKind::IfExpressionIf, state.token, state.subtree_start, state.has_error); - if (context.PositionIs(TokenKind::Then)) { + if (context.PositionIs(Lex::TokenKind::Then)) { context.PushState(State::IfExpressionFinishThen); - context.ConsumeChecked(TokenKind::Then); - context.PushStateForExpression(*PrecedenceGroup::ForLeading(TokenKind::If)); + context.ConsumeChecked(Lex::TokenKind::Then); + context.PushStateForExpression( + *PrecedenceGroup::ForLeading(Lex::TokenKind::If)); } else { // TODO: Include the location of the `if` token. CARBON_DIAGNOSTIC(ExpectedThenAfterIf, Error, @@ -294,10 +296,11 @@ auto HandleIfExpressionFinishThen(Context& context) -> void { context.AddNode(NodeKind::IfExpressionThen, state.token, state.subtree_start, state.has_error); - if (context.PositionIs(TokenKind::Else)) { + if (context.PositionIs(Lex::TokenKind::Else)) { context.PushState(State::IfExpressionFinishElse); - context.ConsumeChecked(TokenKind::Else); - context.PushStateForExpression(*PrecedenceGroup::ForLeading(TokenKind::If)); + context.ConsumeChecked(Lex::TokenKind::Else); + context.PushStateForExpression( + *PrecedenceGroup::ForLeading(Lex::TokenKind::If)); } else { // TODO: Include the location of the `if` token. CARBON_DIAGNOSTIC(ExpectedElseAfterIf, Error, @@ -332,7 +335,7 @@ auto HandleIfExpressionFinish(Context& context) -> void { auto HandleExpressionStatementFinish(Context& context) -> void { auto state = context.PopState(); - if (auto semi = context.ConsumeIf(TokenKind::Semi)) { + if (auto semi = context.ConsumeIf(Lex::TokenKind::Semi)) { context.AddNode(NodeKind::ExpressionStatement, *semi, state.subtree_start, state.has_error); return; diff --git a/toolchain/parser/parser_handle_function.cpp b/toolchain/parser/parser_handle_function.cpp index 189efb634bd0..f5d9a0849a12 100644 --- a/toolchain/parser/parser_handle_function.cpp +++ b/toolchain/parser/parser_handle_function.cpp @@ -25,7 +25,7 @@ auto HandleFunctionAfterParameters(Context& context) -> void { // If there is a return type, parse the expression before adding the return // type nod.e - if (context.PositionIs(TokenKind::MinusGreater)) { + if (context.PositionIs(Lex::TokenKind::MinusGreater)) { context.PushState(State::FunctionReturnTypeFinish); ++context.position(); context.PushStateForExpression(PrecedenceGroup::ForType()); @@ -43,12 +43,12 @@ auto HandleFunctionSignatureFinish(Context& context) -> void { auto state = context.PopState(); switch (context.PositionKind()) { - case TokenKind::Semi: { + case Lex::TokenKind::Semi: { context.AddNode(NodeKind::FunctionDeclaration, context.Consume(), state.subtree_start, state.has_error); break; } - case TokenKind::OpenCurlyBrace: { + case Lex::TokenKind::OpenCurlyBrace: { if (auto decl_context = context.GetDeclarationContext(); decl_context == Context::DeclarationContext::Interface || decl_context == Context::DeclarationContext::NamedConstraint) { @@ -73,7 +73,7 @@ auto HandleFunctionSignatureFinish(Context& context) -> void { } default: { if (!state.has_error) { - context.EmitExpectedDeclarationSemiOrDefinition(TokenKind::Fn); + context.EmitExpectedDeclarationSemiOrDefinition(Lex::TokenKind::Fn); } // Only need to skip if we've not already found a new line. bool skip_past_likely_end = diff --git a/toolchain/parser/parser_handle_index_expression.cpp b/toolchain/parser/parser_handle_index_expression.cpp index eddf70ed15c5..adf6b1d74b05 100644 --- a/toolchain/parser/parser_handle_index_expression.cpp +++ b/toolchain/parser/parser_handle_index_expression.cpp @@ -12,7 +12,7 @@ auto HandleIndexExpression(Context& context) -> void { state.state = State::IndexExpressionFinish; context.PushState(state); context.AddNode(NodeKind::IndexExpressionStart, - context.ConsumeChecked(TokenKind::OpenSquareBracket), + context.ConsumeChecked(Lex::TokenKind::OpenSquareBracket), state.subtree_start, state.has_error); context.PushState(State::Expression); } diff --git a/toolchain/parser/parser_handle_namespace.cpp b/toolchain/parser/parser_handle_namespace.cpp index e9108ceb951b..38aa8c68965a 100644 --- a/toolchain/parser/parser_handle_namespace.cpp +++ b/toolchain/parser/parser_handle_namespace.cpp @@ -26,11 +26,11 @@ auto HandleNamespaceFinish(Context& context) -> void { return; } - if (auto semi = context.ConsumeIf(TokenKind::Semi)) { + if (auto semi = context.ConsumeIf(Lex::TokenKind::Semi)) { context.AddNode(NodeKind::Namespace, *semi, state.subtree_start, state.has_error); } else { - context.EmitExpectedDeclarationSemi(TokenKind::Namespace); + context.EmitExpectedDeclarationSemi(Lex::TokenKind::Namespace); context.RecoverFromDeclarationError(state, NodeKind::Namespace, /*skip_past_likely_end=*/true); } diff --git a/toolchain/parser/parser_handle_package.cpp b/toolchain/parser/parser_handle_package.cpp index 2c337f268799..3b2273520ca6 100644 --- a/toolchain/parser/parser_handle_package.cpp +++ b/toolchain/parser/parser_handle_package.cpp @@ -19,7 +19,8 @@ auto HandlePackage(Context& context) -> void { /*has_error=*/true); }; - if (!context.ConsumeAndAddLeafNodeIf(TokenKind::Identifier, NodeKind::Name)) { + if (!context.ConsumeAndAddLeafNodeIf(Lex::TokenKind::Identifier, + NodeKind::Name)) { CARBON_DIAGNOSTIC(ExpectedIdentifierAfterPackage, Error, "Expected identifier after `package`."); context.emitter().Emit(*context.position(), ExpectedIdentifierAfterPackage); @@ -28,10 +29,10 @@ auto HandlePackage(Context& context) -> void { } bool library_parsed = false; - if (auto library_token = context.ConsumeIf(TokenKind::Library)) { + if (auto library_token = context.ConsumeIf(Lex::TokenKind::Library)) { auto library_start = context.tree().size(); - if (!context.ConsumeAndAddLeafNodeIf(TokenKind::StringLiteral, + if (!context.ConsumeAndAddLeafNodeIf(Lex::TokenKind::StringLiteral, NodeKind::Literal)) { CARBON_DIAGNOSTIC( ExpectedLibraryName, Error, @@ -48,16 +49,17 @@ auto HandlePackage(Context& context) -> void { switch (auto api_or_impl_token = context.tokens().GetKind(*(context.position()))) { - case TokenKind::Api: { + case Lex::TokenKind::Api: { context.AddLeafNode(NodeKind::PackageApi, context.Consume()); break; } - case TokenKind::Impl: { + case Lex::TokenKind::Impl: { context.AddLeafNode(NodeKind::PackageImpl, context.Consume()); break; } default: { - if (!library_parsed && api_or_impl_token == TokenKind::StringLiteral) { + if (!library_parsed && + api_or_impl_token == Lex::TokenKind::StringLiteral) { // If we come acroess a string literal and we didn't parse `library // "..."` yet, then most probably the user forgot to add `library` // before the library name. @@ -74,8 +76,8 @@ auto HandlePackage(Context& context) -> void { } } - if (!context.PositionIs(TokenKind::Semi)) { - context.EmitExpectedDeclarationSemi(TokenKind::Package); + if (!context.PositionIs(Lex::TokenKind::Semi)) { + context.EmitExpectedDeclarationSemi(Lex::TokenKind::Package); exit_on_parse_error(); return; } diff --git a/toolchain/parser/parser_handle_parameter.cpp b/toolchain/parser/parser_handle_parameter.cpp index e6eece27a530..48c4d53cc2d7 100644 --- a/toolchain/parser/parser_handle_parameter.cpp +++ b/toolchain/parser/parser_handle_parameter.cpp @@ -26,7 +26,7 @@ auto HandleParameterAsRegular(Context& context) -> void { } // Handles ParameterFinishAs(Deduced|Regular). -static auto HandleParameterFinish(Context& context, TokenKind close_token, +static auto HandleParameterFinish(Context& context, Lex::TokenKind close_token, State param_state) -> void { auto state = context.PopState(); @@ -42,20 +42,20 @@ static auto HandleParameterFinish(Context& context, TokenKind close_token, } auto HandleParameterFinishAsDeduced(Context& context) -> void { - HandleParameterFinish(context, TokenKind::CloseSquareBracket, + HandleParameterFinish(context, Lex::TokenKind::CloseSquareBracket, State::ParameterAsDeduced); } auto HandleParameterFinishAsRegular(Context& context) -> void { - HandleParameterFinish(context, TokenKind::CloseParen, + HandleParameterFinish(context, Lex::TokenKind::CloseParen, State::ParameterAsRegular); } // Handles ParameterListAs(Deduced|Regular). static auto HandleParameterList(Context& context, NodeKind parse_node_kind, - TokenKind open_token_kind, - TokenKind close_token_kind, State param_state, - State finish_state) -> void { + Lex::TokenKind open_token_kind, + Lex::TokenKind close_token_kind, + State param_state, State finish_state) -> void { context.PopAndDiscardState(); context.PushState(finish_state); @@ -67,15 +67,15 @@ static auto HandleParameterList(Context& context, NodeKind parse_node_kind, } auto HandleParameterListAsDeduced(Context& context) -> void { - HandleParameterList(context, NodeKind::DeducedParameterListStart, - TokenKind::OpenSquareBracket, - TokenKind::CloseSquareBracket, State::ParameterAsDeduced, - State::ParameterListFinishAsDeduced); + HandleParameterList( + context, NodeKind::DeducedParameterListStart, + Lex::TokenKind::OpenSquareBracket, Lex::TokenKind::CloseSquareBracket, + State::ParameterAsDeduced, State::ParameterListFinishAsDeduced); } auto HandleParameterListAsRegular(Context& context) -> void { HandleParameterList(context, NodeKind::ParameterListStart, - TokenKind::OpenParen, TokenKind::CloseParen, + Lex::TokenKind::OpenParen, Lex::TokenKind::CloseParen, State::ParameterAsRegular, State::ParameterListFinishAsRegular); } @@ -83,7 +83,7 @@ auto HandleParameterListAsRegular(Context& context) -> void { // Handles ParameterListFinishAs(Deduced|Regular). static auto HandleParameterListFinish(Context& context, NodeKind parse_node_kind, - TokenKind token_kind) -> void { + Lex::TokenKind token_kind) -> void { auto state = context.PopState(); context.AddNode(parse_node_kind, context.ConsumeChecked(token_kind), @@ -92,12 +92,12 @@ static auto HandleParameterListFinish(Context& context, auto HandleParameterListFinishAsDeduced(Context& context) -> void { HandleParameterListFinish(context, NodeKind::DeducedParameterList, - TokenKind::CloseSquareBracket); + Lex::TokenKind::CloseSquareBracket); } auto HandleParameterListFinishAsRegular(Context& context) -> void { HandleParameterListFinish(context, NodeKind::ParameterList, - TokenKind::CloseParen); + Lex::TokenKind::CloseParen); } } // namespace Carbon::Parse diff --git a/toolchain/parser/parser_handle_paren_condition.cpp b/toolchain/parser/parser_handle_paren_condition.cpp index ece601ddb314..d5917f24198a 100644 --- a/toolchain/parser/parser_handle_paren_condition.cpp +++ b/toolchain/parser/parser_handle_paren_condition.cpp @@ -11,7 +11,7 @@ static auto HandleParenCondition(Context& context, NodeKind start_kind, State finish_state) -> void { auto state = context.PopState(); - std::optional open_paren = + std::optional open_paren = context.ConsumeAndAddOpenParen(state.token, start_kind); if (open_paren) { state.token = *open_paren; @@ -19,7 +19,7 @@ static auto HandleParenCondition(Context& context, NodeKind start_kind, state.state = finish_state; context.PushState(state); - if (!open_paren && context.PositionIs(TokenKind::OpenCurlyBrace)) { + if (!open_paren && context.PositionIs(Lex::TokenKind::OpenCurlyBrace)) { // For an open curly, assume the condition was completely omitted. // Expression parsing would treat the { as a struct, but instead assume it's // a code block and just emit an invalid parse. diff --git a/toolchain/parser/parser_handle_paren_expression.cpp b/toolchain/parser/parser_handle_paren_expression.cpp index 5190a07ed60f..37346951bb99 100644 --- a/toolchain/parser/parser_handle_paren_expression.cpp +++ b/toolchain/parser/parser_handle_paren_expression.cpp @@ -11,9 +11,9 @@ auto HandleParenExpression(Context& context) -> void { // Advance past the open paren. context.AddLeafNode(NodeKind::ParenExpressionOrTupleLiteralStart, - context.ConsumeChecked(TokenKind::OpenParen)); + context.ConsumeChecked(Lex::TokenKind::OpenParen)); - if (context.PositionIs(TokenKind::CloseParen)) { + if (context.PositionIs(Lex::TokenKind::CloseParen)) { state.state = State::ParenExpressionFinishAsTuple; context.PushState(state); } else { @@ -30,7 +30,7 @@ static auto HandleParenExpressionParameterFinish(Context& context, auto state = context.PopState(); auto list_token_kind = context.ConsumeListToken( - NodeKind::TupleLiteralComma, TokenKind::CloseParen, state.has_error); + NodeKind::TupleLiteralComma, Lex::TokenKind::CloseParen, state.has_error); if (list_token_kind == Context::ListTokenKind::Close) { return; } diff --git a/toolchain/parser/parser_handle_pattern.cpp b/toolchain/parser/parser_handle_pattern.cpp index 05bfca33aaa8..7594c893397e 100644 --- a/toolchain/parser/parser_handle_pattern.cpp +++ b/toolchain/parser/parser_handle_pattern.cpp @@ -14,9 +14,9 @@ static auto HandlePattern(Context& context, Context::PatternKind pattern_kind) // Parameters may have keywords prefixing the pattern. They become the parent // for the full PatternBinding. if (pattern_kind != Context::PatternKind::Variable) { - context.ConsumeIfPatternKeyword(TokenKind::Template, State::PatternTemplate, - state.subtree_start); - context.ConsumeIfPatternKeyword(TokenKind::Addr, State::PatternAddress, + context.ConsumeIfPatternKeyword( + Lex::TokenKind::Template, State::PatternTemplate, state.subtree_start); + context.ConsumeIfPatternKeyword(Lex::TokenKind::Addr, State::PatternAddress, state.subtree_start); } @@ -47,11 +47,11 @@ static auto HandlePattern(Context& context, Context::PatternKind pattern_kind) // The first item should be an identifier or, for deduced parameters, `self`. bool has_name = false; - if (auto identifier = context.ConsumeIf(TokenKind::Identifier)) { + if (auto identifier = context.ConsumeIf(Lex::TokenKind::Identifier)) { context.AddLeafNode(NodeKind::Name, *identifier); has_name = true; } else if (pattern_kind == Context::PatternKind::DeducedParameter) { - if (auto self = context.ConsumeIf(TokenKind::SelfValueIdentifier)) { + if (auto self = context.ConsumeIf(Lex::TokenKind::SelfValueIdentifier)) { context.AddLeafNode(NodeKind::SelfValueName, *self); has_name = true; } @@ -65,9 +65,9 @@ static auto HandlePattern(Context& context, Context::PatternKind pattern_kind) } if (auto kind = context.PositionKind(); - kind == TokenKind::Colon || kind == TokenKind::ColonExclaim) { - state.state = kind == TokenKind::Colon ? State::PatternFinishAsRegular - : State::PatternFinishAsGeneric; + kind == Lex::TokenKind::Colon || kind == Lex::TokenKind::ColonExclaim) { + state.state = kind == Lex::TokenKind::Colon ? State::PatternFinishAsRegular + : State::PatternFinishAsGeneric; // Use the `:` or `:!` for the root node. state.token = context.Consume(); context.PushState(state); diff --git a/toolchain/parser/parser_handle_period.cpp b/toolchain/parser/parser_handle_period.cpp index 4188a3aa399a..1af5d4ff6980 100644 --- a/toolchain/parser/parser_handle_period.cpp +++ b/toolchain/parser/parser_handle_period.cpp @@ -14,10 +14,11 @@ static auto HandlePeriodOrArrow(Context& context, NodeKind node_kind, auto state = context.PopState(); // `.` identifier - auto dot = context.ConsumeChecked(is_arrow ? TokenKind::MinusGreater - : TokenKind::Period); + auto dot = context.ConsumeChecked(is_arrow ? Lex::TokenKind::MinusGreater + : Lex::TokenKind::Period); - if (!context.ConsumeAndAddLeafNodeIf(TokenKind::Identifier, NodeKind::Name)) { + if (!context.ConsumeAndAddLeafNodeIf(Lex::TokenKind::Identifier, + NodeKind::Name)) { CARBON_DIAGNOSTIC(ExpectedIdentifierAfterDotOrArrow, Error, "Expected identifier after `{0}`.", llvm::StringRef); context.emitter().Emit(*context.position(), diff --git a/toolchain/parser/parser_handle_statement.cpp b/toolchain/parser/parser_handle_statement.cpp index 91fe7cd5312b..9cec75a20bf9 100644 --- a/toolchain/parser/parser_handle_statement.cpp +++ b/toolchain/parser/parser_handle_statement.cpp @@ -10,35 +10,35 @@ auto HandleStatement(Context& context) -> void { context.PopAndDiscardState(); switch (context.PositionKind()) { - case TokenKind::Break: { + case Lex::TokenKind::Break: { context.PushState(State::StatementBreakFinish); context.AddLeafNode(NodeKind::BreakStatementStart, context.Consume()); break; } - case TokenKind::Continue: { + case Lex::TokenKind::Continue: { context.PushState(State::StatementContinueFinish); context.AddLeafNode(NodeKind::ContinueStatementStart, context.Consume()); break; } - case TokenKind::For: { + case Lex::TokenKind::For: { context.PushState(State::StatementForFinish); context.PushState(State::StatementForHeader); ++context.position(); break; } - case TokenKind::If: { + case Lex::TokenKind::If: { context.PushState(State::StatementIf); break; } - case TokenKind::Return: { + case Lex::TokenKind::Return: { context.PushState(State::StatementReturn); break; } - case TokenKind::Var: { + case Lex::TokenKind::Var: { context.PushState(State::VarAsSemicolon); break; } - case TokenKind::While: { + case Lex::TokenKind::While: { context.PushState(State::StatementWhile); break; } @@ -55,10 +55,10 @@ static auto HandleStatementKeywordFinish(Context& context, NodeKind node_kind) -> void { auto state = context.PopState(); - auto semi = context.ConsumeIf(TokenKind::Semi); + auto semi = context.ConsumeIf(Lex::TokenKind::Semi); if (!semi) { CARBON_DIAGNOSTIC(ExpectedStatementSemi, Error, - "`{0}` statements must end with a `;`.", TokenKind); + "`{0}` statements must end with a `;`.", Lex::TokenKind); context.emitter().Emit(*context.position(), ExpectedStatementSemi, context.tokens().GetKind(state.token)); state.has_error = true; @@ -83,14 +83,14 @@ auto HandleStatementContinueFinish(Context& context) -> void { auto HandleStatementForHeader(Context& context) -> void { auto state = context.PopState(); - std::optional open_paren = + std::optional open_paren = context.ConsumeAndAddOpenParen(state.token, NodeKind::ForHeaderStart); if (open_paren) { state.token = *open_paren; } state.state = State::StatementForHeaderIn; - if (context.PositionIs(TokenKind::Var)) { + if (context.PositionIs(Lex::TokenKind::Var)) { context.PushState(state); context.PushState(State::VarAsFor); } else { @@ -98,7 +98,7 @@ auto HandleStatementForHeader(Context& context) -> void { "Expected `var` declaration."); context.emitter().Emit(*context.position(), ExpectedVariableDeclaration); - if (auto next_in = context.FindNextOf({TokenKind::In})) { + if (auto next_in = context.FindNextOf({Lex::TokenKind::In})) { context.SkipTo(*next_in); ++context.position(); } @@ -149,13 +149,14 @@ auto HandleStatementIfConditionFinish(Context& context) -> void { auto HandleStatementIfThenBlockFinish(Context& context) -> void { auto state = context.PopState(); - if (context.ConsumeAndAddLeafNodeIf(TokenKind::Else, + if (context.ConsumeAndAddLeafNodeIf(Lex::TokenKind::Else, NodeKind::IfStatementElse)) { state.state = State::StatementIfElseBlockFinish; context.PushState(state); // `else if` is permitted as a special case. - context.PushState(context.PositionIs(TokenKind::If) ? State::StatementIf - : State::CodeBlock); + context.PushState(context.PositionIs(Lex::TokenKind::If) + ? State::StatementIf + : State::CodeBlock); } else { context.AddNode(NodeKind::IfStatement, state.token, state.subtree_start, state.has_error); @@ -174,7 +175,7 @@ auto HandleStatementReturn(Context& context) -> void { context.PushState(state); context.AddLeafNode(NodeKind::ReturnStatementStart, context.Consume()); - if (!context.PositionIs(TokenKind::Semi)) { + if (!context.PositionIs(Lex::TokenKind::Semi)) { context.PushState(State::Expression); } } @@ -187,7 +188,7 @@ auto HandleStatementScopeLoop(Context& context) -> void { // This maintains the current state until we're at the end of the scope. auto token_kind = context.PositionKind(); - if (token_kind == TokenKind::CloseCurlyBrace) { + if (token_kind == Lex::TokenKind::CloseCurlyBrace) { auto state = context.PopState(); if (state.has_error) { context.ReturnErrorOnState(); diff --git a/toolchain/parser/parser_handle_type.cpp b/toolchain/parser/parser_handle_type.cpp index d688cbbe231e..58d78d167b5a 100644 --- a/toolchain/parser/parser_handle_type.cpp +++ b/toolchain/parser/parser_handle_type.cpp @@ -46,13 +46,13 @@ static auto HandleTypeAfterParams(Context& context, NodeKind declaration_kind, return; } - if (auto semi = context.ConsumeIf(TokenKind::Semi)) { + if (auto semi = context.ConsumeIf(Lex::TokenKind::Semi)) { context.AddNode(declaration_kind, *semi, state.subtree_start, state.has_error); return; } - if (!context.PositionIs(TokenKind::OpenCurlyBrace)) { + if (!context.PositionIs(Lex::TokenKind::OpenCurlyBrace)) { context.EmitExpectedDeclarationSemiOrDefinition( context.tokens().GetKind(state.token)); context.RecoverFromDeclarationError(state, declaration_kind, diff --git a/toolchain/parser/parser_handle_var.cpp b/toolchain/parser/parser_handle_var.cpp index 67005855ddf0..4f07676db9d8 100644 --- a/toolchain/parser/parser_handle_var.cpp +++ b/toolchain/parser/parser_handle_var.cpp @@ -33,12 +33,12 @@ auto HandleVarAfterPattern(Context& context) -> void { if (state.has_error) { if (auto after_pattern = - context.FindNextOf({TokenKind::Equal, TokenKind::Semi})) { + context.FindNextOf({Lex::TokenKind::Equal, Lex::TokenKind::Semi})) { context.SkipTo(*after_pattern); } } - if (auto equals = context.ConsumeIf(TokenKind::Equal)) { + if (auto equals = context.ConsumeIf(Lex::TokenKind::Equal)) { context.AddLeafNode(NodeKind::VariableInitializer, *equals); context.PushState(State::Expression); } @@ -48,11 +48,11 @@ auto HandleVarFinishAsSemicolon(Context& context) -> void { auto state = context.PopState(); auto end_token = state.token; - if (context.PositionIs(TokenKind::Semi)) { + if (context.PositionIs(Lex::TokenKind::Semi)) { end_token = context.Consume(); } else { // TODO: Disambiguate between statement and member declaration. - context.EmitExpectedDeclarationSemi(TokenKind::Var); + context.EmitExpectedDeclarationSemi(Lex::TokenKind::Var); state.has_error = true; if (auto semi_token = context.SkipPastLikelyEnd(state.token)) { end_token = *semi_token; @@ -66,9 +66,9 @@ auto HandleVarFinishAsFor(Context& context) -> void { auto state = context.PopState(); auto end_token = state.token; - if (context.PositionIs(TokenKind::In)) { + if (context.PositionIs(Lex::TokenKind::In)) { end_token = context.Consume(); - } else if (context.PositionIs(TokenKind::Colon)) { + } else if (context.PositionIs(Lex::TokenKind::Colon)) { CARBON_DIAGNOSTIC(ExpectedInNotColon, Error, "`:` should be replaced by `in`."); context.emitter().Emit(*context.position(), ExpectedInNotColon); diff --git a/toolchain/parser/parser_state.def b/toolchain/parser/parser_state.def index 6016a5f6ea6b..2cd759f2329e 100644 --- a/toolchain/parser/parser_state.def +++ b/toolchain/parser/parser_state.def @@ -21,7 +21,7 @@ // the stack. // // Where state output is conditional on a lexed token, the name of -// the TokenKind should be used rather than the string name in order to make it +// the Lex::TokenKind should be used rather than the string name in order to make it // easier to compare with code. #ifndef CARBON_PARSE_STATE diff --git a/toolchain/parser/precedence.cpp b/toolchain/parser/precedence.cpp index 57cda50d5edb..1f05d2566390 100644 --- a/toolchain/parser/precedence.cpp +++ b/toolchain/parser/precedence.cpp @@ -193,30 +193,30 @@ auto PrecedenceGroup::ForType() -> PrecedenceGroup { return ForTopLevelExpression(); } -auto PrecedenceGroup::ForLeading(TokenKind kind) +auto PrecedenceGroup::ForLeading(Lex::TokenKind kind) -> std::optional { switch (kind) { - case TokenKind::Star: - case TokenKind::Amp: + case Lex::TokenKind::Star: + case Lex::TokenKind::Amp: return PrecedenceGroup(TermPrefix); - case TokenKind::Not: + case Lex::TokenKind::Not: return PrecedenceGroup(LogicalPrefix); - case TokenKind::Minus: + case Lex::TokenKind::Minus: return PrecedenceGroup(NumericPrefix); - case TokenKind::MinusMinus: - case TokenKind::PlusPlus: + case Lex::TokenKind::MinusMinus: + case Lex::TokenKind::PlusPlus: return PrecedenceGroup(IncrementDecrement); - case TokenKind::Caret: + case Lex::TokenKind::Caret: return PrecedenceGroup(BitwisePrefix); - case TokenKind::If: + case Lex::TokenKind::If: return PrecedenceGroup(If); - case TokenKind::Const: + case Lex::TokenKind::Const: return PrecedenceGroup(TypePrefix); default: @@ -224,96 +224,96 @@ auto PrecedenceGroup::ForLeading(TokenKind kind) } } -auto PrecedenceGroup::ForTrailing(TokenKind kind, bool infix) +auto PrecedenceGroup::ForTrailing(Lex::TokenKind kind, bool infix) -> std::optional { switch (kind) { // Assignment operators. - case TokenKind::Equal: - case TokenKind::PlusEqual: - case TokenKind::MinusEqual: - case TokenKind::StarEqual: - case TokenKind::SlashEqual: - case TokenKind::PercentEqual: - case TokenKind::AmpEqual: - case TokenKind::PipeEqual: - case TokenKind::CaretEqual: - case TokenKind::GreaterGreaterEqual: - case TokenKind::LessLessEqual: + case Lex::TokenKind::Equal: + case Lex::TokenKind::PlusEqual: + case Lex::TokenKind::MinusEqual: + case Lex::TokenKind::StarEqual: + case Lex::TokenKind::SlashEqual: + case Lex::TokenKind::PercentEqual: + case Lex::TokenKind::AmpEqual: + case Lex::TokenKind::PipeEqual: + case Lex::TokenKind::CaretEqual: + case Lex::TokenKind::GreaterGreaterEqual: + case Lex::TokenKind::LessLessEqual: return Trailing{.level = Assignment, .is_binary = true}; // Logical operators. - case TokenKind::And: + case Lex::TokenKind::And: return Trailing{.level = LogicalAnd, .is_binary = true}; - case TokenKind::Or: + case Lex::TokenKind::Or: return Trailing{.level = LogicalOr, .is_binary = true}; // Bitwise operators. - case TokenKind::Amp: + case Lex::TokenKind::Amp: return Trailing{.level = BitwiseAnd, .is_binary = true}; - case TokenKind::Pipe: + case Lex::TokenKind::Pipe: return Trailing{.level = BitwiseOr, .is_binary = true}; - case TokenKind::Caret: + case Lex::TokenKind::Caret: return Trailing{.level = BitwiseXor, .is_binary = true}; - case TokenKind::GreaterGreater: - case TokenKind::LessLess: + case Lex::TokenKind::GreaterGreater: + case Lex::TokenKind::LessLess: return Trailing{.level = BitShift, .is_binary = true}; // Relational operators. - case TokenKind::EqualEqual: - case TokenKind::ExclaimEqual: - case TokenKind::Less: - case TokenKind::LessEqual: - case TokenKind::Greater: - case TokenKind::GreaterEqual: - case TokenKind::LessEqualGreater: + case Lex::TokenKind::EqualEqual: + case Lex::TokenKind::ExclaimEqual: + case Lex::TokenKind::Less: + case Lex::TokenKind::LessEqual: + case Lex::TokenKind::Greater: + case Lex::TokenKind::GreaterEqual: + case Lex::TokenKind::LessEqualGreater: return Trailing{.level = Relational, .is_binary = true}; // Additive operators. - case TokenKind::Plus: - case TokenKind::Minus: + case Lex::TokenKind::Plus: + case Lex::TokenKind::Minus: return Trailing{.level = Additive, .is_binary = true}; // Multiplicative operators. - case TokenKind::Slash: + case Lex::TokenKind::Slash: return Trailing{.level = Multiplicative, .is_binary = true}; - case TokenKind::Percent: + case Lex::TokenKind::Percent: return Trailing{.level = Modulo, .is_binary = true}; // `*` could be multiplication or pointer type formation. - case TokenKind::Star: + case Lex::TokenKind::Star: return infix ? Trailing{.level = Multiplicative, .is_binary = true} : Trailing{.level = TypePostfix, .is_binary = false}; // Cast operator. - case TokenKind::As: + case Lex::TokenKind::As: return Trailing{.level = As, .is_binary = true}; // Prefix-only operators. - case TokenKind::Const: - case TokenKind::MinusMinus: - case TokenKind::Not: - case TokenKind::PlusPlus: + case Lex::TokenKind::Const: + case Lex::TokenKind::MinusMinus: + case Lex::TokenKind::Not: + case Lex::TokenKind::PlusPlus: break; // Symbolic tokens that might be operators eventually. - case TokenKind::Tilde: - case TokenKind::Backslash: - case TokenKind::Comma: - case TokenKind::TildeEqual: - case TokenKind::Exclaim: - case TokenKind::LessGreater: - case TokenKind::Question: - case TokenKind::Colon: + case Lex::TokenKind::Tilde: + case Lex::TokenKind::Backslash: + case Lex::TokenKind::Comma: + case Lex::TokenKind::TildeEqual: + case Lex::TokenKind::Exclaim: + case Lex::TokenKind::LessGreater: + case Lex::TokenKind::Question: + case Lex::TokenKind::Colon: break; // Symbolic tokens that are intentionally not operators. - case TokenKind::At: - case TokenKind::LessMinus: - case TokenKind::MinusGreater: - case TokenKind::EqualGreater: - case TokenKind::ColonEqual: - case TokenKind::Period: - case TokenKind::Semi: + case Lex::TokenKind::At: + case Lex::TokenKind::LessMinus: + case Lex::TokenKind::MinusGreater: + case Lex::TokenKind::EqualGreater: + case Lex::TokenKind::ColonEqual: + case Lex::TokenKind::Period: + case Lex::TokenKind::Semi: break; default: diff --git a/toolchain/parser/precedence.h b/toolchain/parser/precedence.h index e72f7d807577..2fcc7a4f3ea6 100644 --- a/toolchain/parser/precedence.h +++ b/toolchain/parser/precedence.h @@ -56,14 +56,14 @@ class PrecedenceGroup { // Look up the operator information of the given prefix operator token, or // return std::nullopt if the given token is not a prefix operator. - static auto ForLeading(TokenKind kind) -> std::optional; + static auto ForLeading(Lex::TokenKind kind) -> std::optional; // Look up the operator information of the given infix or postfix operator // token, or return std::nullopt if the given token is not an infix or postfix // operator. `infix` indicates whether this is a valid infix operator, but is // only considered if the same operator symbol is available as both infix and // postfix. - static auto ForTrailing(TokenKind kind, bool infix) + static auto ForTrailing(Lex::TokenKind kind, bool infix) -> std::optional; friend auto operator==(PrecedenceGroup lhs, PrecedenceGroup rhs) -> bool { diff --git a/toolchain/parser/precedence_test.cpp b/toolchain/parser/precedence_test.cpp index 7997281d99bc..bd44a62c1faf 100644 --- a/toolchain/parser/precedence_test.cpp +++ b/toolchain/parser/precedence_test.cpp @@ -18,123 +18,143 @@ using Parse::PrecedenceGroup; using ::testing::Eq; TEST(PrecedenceTest, OperatorsAreRecognized) { - EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Minus).has_value()); - EXPECT_TRUE(PrecedenceGroup::ForLeading(TokenKind::Caret).has_value()); - EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Slash).has_value()); - EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Identifier).has_value()); - EXPECT_FALSE(PrecedenceGroup::ForLeading(TokenKind::Tilde).has_value()); + EXPECT_TRUE(PrecedenceGroup::ForLeading(Lex::TokenKind::Minus).has_value()); + EXPECT_TRUE(PrecedenceGroup::ForLeading(Lex::TokenKind::Caret).has_value()); + EXPECT_FALSE(PrecedenceGroup::ForLeading(Lex::TokenKind::Slash).has_value()); + EXPECT_FALSE( + PrecedenceGroup::ForLeading(Lex::TokenKind::Identifier).has_value()); + EXPECT_FALSE(PrecedenceGroup::ForLeading(Lex::TokenKind::Tilde).has_value()); EXPECT_TRUE( - PrecedenceGroup::ForTrailing(TokenKind::Minus, false).has_value()); + PrecedenceGroup::ForTrailing(Lex::TokenKind::Minus, false).has_value()); EXPECT_TRUE( - PrecedenceGroup::ForTrailing(TokenKind::Caret, false).has_value()); + PrecedenceGroup::ForTrailing(Lex::TokenKind::Caret, false).has_value()); EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::Tilde, false).has_value()); - EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Slash, true).has_value()); - EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::Identifier, false).has_value()); + PrecedenceGroup::ForTrailing(Lex::TokenKind::Tilde, false).has_value()); + EXPECT_TRUE( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Slash, true).has_value()); + EXPECT_FALSE(PrecedenceGroup::ForTrailing(Lex::TokenKind::Identifier, false) + .has_value()); - EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Minus, true)->is_binary); - EXPECT_FALSE( - PrecedenceGroup::ForTrailing(TokenKind::MinusMinus, false).has_value()); + EXPECT_TRUE( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Minus, true)->is_binary); + EXPECT_FALSE(PrecedenceGroup::ForTrailing(Lex::TokenKind::MinusMinus, false) + .has_value()); } TEST(PrecedenceTest, InfixVsPostfix) { // A trailing `-` is always binary, even when written with whitespace that // suggests it's postfix. - EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Minus, /*infix*/ false) - ->is_binary); + EXPECT_TRUE( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Minus, /*infix*/ false) + ->is_binary); // A trailing `*` is interpreted based on context. - EXPECT_TRUE(PrecedenceGroup::ForTrailing(TokenKind::Star, true)->is_binary); - EXPECT_FALSE(PrecedenceGroup::ForTrailing(TokenKind::Star, false)->is_binary); + EXPECT_TRUE( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, true)->is_binary); + EXPECT_FALSE( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, false)->is_binary); // Infix `*` can appear in `+` contexts; postfix `*` cannot. - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level), - Eq(OperatorPriority::LeftFirst)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star, false)->level, - PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level), - Eq(OperatorPriority::Ambiguous)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Plus, true)->level), + Eq(OperatorPriority::LeftFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, false)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Plus, true)->level), + Eq(OperatorPriority::Ambiguous)); } TEST(PrecedenceTest, Associativity) { - EXPECT_THAT(PrecedenceGroup::ForLeading(TokenKind::Minus)->GetAssociativity(), - Eq(Associativity::None)); - EXPECT_THAT(PrecedenceGroup::ForLeading(TokenKind::Star)->GetAssociativity(), - Eq(Associativity::RightToLeft)); - EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Plus, true) + EXPECT_THAT( + PrecedenceGroup::ForLeading(Lex::TokenKind::Minus)->GetAssociativity(), + Eq(Associativity::None)); + EXPECT_THAT( + PrecedenceGroup::ForLeading(Lex::TokenKind::Star)->GetAssociativity(), + Eq(Associativity::RightToLeft)); + EXPECT_THAT(PrecedenceGroup::ForTrailing(Lex::TokenKind::Plus, true) ->level.GetAssociativity(), Eq(Associativity::LeftToRight)); - EXPECT_THAT(PrecedenceGroup::ForTrailing(TokenKind::Equal, true) + EXPECT_THAT(PrecedenceGroup::ForTrailing(Lex::TokenKind::Equal, true) ->level.GetAssociativity(), Eq(Associativity::None)); } TEST(PrecedenceTest, DirectRelations) { - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level), - Eq(OperatorPriority::LeftFirst)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level), - Eq(OperatorPriority::RightFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Plus, true)->level), + Eq(OperatorPriority::LeftFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Plus, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, true)->level), + Eq(OperatorPriority::RightFirst)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Less, true)->level), - Eq(OperatorPriority::LeftFirst)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Less, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level), - Eq(OperatorPriority::RightFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Amp, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Less, true)->level), + Eq(OperatorPriority::LeftFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Less, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Amp, true)->level), + Eq(OperatorPriority::RightFirst)); } TEST(PrecedenceTest, IndirectRelations) { - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Or, true)->level), - Eq(OperatorPriority::LeftFirst)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Or, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Star, true)->level), - Eq(OperatorPriority::RightFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Or, true)->level), + Eq(OperatorPriority::LeftFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Or, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Star, true)->level), + Eq(OperatorPriority::RightFirst)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Caret), - PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level), - Eq(OperatorPriority::LeftFirst)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level, - *PrecedenceGroup::ForLeading(TokenKind::Caret)), - Eq(OperatorPriority::RightFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + *PrecedenceGroup::ForLeading(Lex::TokenKind::Caret), + PrecedenceGroup::ForTrailing(Lex::TokenKind::Equal, true)->level), + Eq(OperatorPriority::LeftFirst)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Equal, true)->level, + *PrecedenceGroup::ForLeading(Lex::TokenKind::Caret)), + Eq(OperatorPriority::RightFirst)); } TEST(PrecedenceTest, IncomparableOperators) { EXPECT_THAT(PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Caret), - *PrecedenceGroup::ForLeading(TokenKind::Not)), + *PrecedenceGroup::ForLeading(Lex::TokenKind::Caret), + *PrecedenceGroup::ForLeading(Lex::TokenKind::Not)), Eq(OperatorPriority::Ambiguous)); EXPECT_THAT(PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Caret), - *PrecedenceGroup::ForLeading(TokenKind::Minus)), - Eq(OperatorPriority::Ambiguous)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - *PrecedenceGroup::ForLeading(TokenKind::Not), - PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level), + *PrecedenceGroup::ForLeading(Lex::TokenKind::Caret), + *PrecedenceGroup::ForLeading(Lex::TokenKind::Minus)), Eq(OperatorPriority::Ambiguous)); EXPECT_THAT( PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Equal, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::PipeEqual, true)->level), + *PrecedenceGroup::ForLeading(Lex::TokenKind::Not), + PrecedenceGroup::ForTrailing(Lex::TokenKind::Amp, true)->level), + Eq(OperatorPriority::Ambiguous)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Equal, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::PipeEqual, true)->level), + Eq(OperatorPriority::Ambiguous)); + EXPECT_THAT( + PrecedenceGroup::GetPriority( + PrecedenceGroup::ForTrailing(Lex::TokenKind::Plus, true)->level, + PrecedenceGroup::ForTrailing(Lex::TokenKind::Amp, true)->level), Eq(OperatorPriority::Ambiguous)); - EXPECT_THAT(PrecedenceGroup::GetPriority( - PrecedenceGroup::ForTrailing(TokenKind::Plus, true)->level, - PrecedenceGroup::ForTrailing(TokenKind::Amp, true)->level), - Eq(OperatorPriority::Ambiguous)); } } // namespace diff --git a/toolchain/semantics/check.cpp b/toolchain/semantics/check.cpp index a8666a9718e1..1b9a26b905be 100644 --- a/toolchain/semantics/check.cpp +++ b/toolchain/semantics/check.cpp @@ -12,7 +12,7 @@ namespace Carbon::Check { auto CheckParseTree(const SemIR::File& builtin_ir, - const TokenizedBuffer& tokens, + const Lex::TokenizedBuffer& tokens, const Parse::Tree& parse_tree, DiagnosticConsumer& consumer, llvm::raw_ostream* vlog_stream) -> SemIR::File { auto semantics_ir = SemIR::File(&builtin_ir); diff --git a/toolchain/semantics/check.h b/toolchain/semantics/check.h index b0e44ca51188..5fad50a55217 100644 --- a/toolchain/semantics/check.h +++ b/toolchain/semantics/check.h @@ -19,7 +19,7 @@ inline auto MakeBuiltins() -> SemIR::File { return SemIR::File(); } // Produces and checks the IR for the provided Parse::Tree. extern auto CheckParseTree(const SemIR::File& builtin_ir, - const TokenizedBuffer& tokens, + const Lex::TokenizedBuffer& tokens, const Parse::Tree& parse_tree, DiagnosticConsumer& consumer, llvm::raw_ostream* vlog_stream) -> SemIR::File; diff --git a/toolchain/semantics/semantics_context.cpp b/toolchain/semantics/semantics_context.cpp index 3be86a102b2f..e2d548113d72 100644 --- a/toolchain/semantics/semantics_context.cpp +++ b/toolchain/semantics/semantics_context.cpp @@ -20,7 +20,7 @@ namespace Carbon::Check { -Context::Context(const TokenizedBuffer& tokens, +Context::Context(const Lex::TokenizedBuffer& tokens, DiagnosticEmitter& emitter, const Parse::Tree& parse_tree, SemIR::File& semantics_ir, llvm::raw_ostream* vlog_stream) diff --git a/toolchain/semantics/semantics_context.h b/toolchain/semantics/semantics_context.h index e14e2358cf28..49b7c1ec7ca4 100644 --- a/toolchain/semantics/semantics_context.h +++ b/toolchain/semantics/semantics_context.h @@ -22,7 +22,7 @@ namespace Carbon::Check { class Context { public: // Stores references for work. - explicit Context(const TokenizedBuffer& tokens, + explicit Context(const Lex::TokenizedBuffer& tokens, DiagnosticEmitter& emitter, const Parse::Tree& parse_tree, SemIR::File& semantics, llvm::raw_ostream* vlog_stream); @@ -229,7 +229,7 @@ class Context { // Prints information for a stack dump. auto PrintForStackDump(llvm::raw_ostream& output) const -> void; - auto tokens() -> const TokenizedBuffer& { return *tokens_; } + auto tokens() -> const Lex::TokenizedBuffer& { return *tokens_; } auto emitter() -> DiagnosticEmitter& { return *emitter_; } @@ -333,7 +333,7 @@ class Context { auto current_scope() -> ScopeStackEntry& { return scope_stack_.back(); } // Tokens for getting data on literals. - const TokenizedBuffer* tokens_; + const Lex::TokenizedBuffer* tokens_; // Handles diagnostics. DiagnosticEmitter* emitter_; diff --git a/toolchain/semantics/semantics_handle_literal.cpp b/toolchain/semantics/semantics_handle_literal.cpp index d4c9f445e727..d62705739ffb 100644 --- a/toolchain/semantics/semantics_handle_literal.cpp +++ b/toolchain/semantics/semantics_handle_literal.cpp @@ -9,18 +9,18 @@ namespace Carbon::Check { auto HandleLiteral(Context& context, Parse::Node parse_node) -> bool { auto token = context.parse_tree().node_token(parse_node); switch (auto token_kind = context.tokens().GetKind(token)) { - case TokenKind::False: - case TokenKind::True: { + case Lex::TokenKind::False: + case Lex::TokenKind::True: { context.AddNodeAndPush( parse_node, SemIR::Node::BoolLiteral::Make( parse_node, context.CanonicalizeType(SemIR::NodeId::BuiltinBoolType), - token_kind == TokenKind::True ? SemIR::BoolValue::True - : SemIR::BoolValue::False)); + token_kind == Lex::TokenKind::True ? SemIR::BoolValue::True + : SemIR::BoolValue::False)); break; } - case TokenKind::IntegerLiteral: { + case Lex::TokenKind::IntegerLiteral: { auto id = context.semantics_ir().AddIntegerLiteral( context.tokens().GetIntegerLiteral(token)); context.AddNodeAndPush( @@ -30,7 +30,7 @@ auto HandleLiteral(Context& context, Parse::Node parse_node) -> bool { context.CanonicalizeType(SemIR::NodeId::BuiltinIntegerType), id)); break; } - case TokenKind::RealLiteral: { + case Lex::TokenKind::RealLiteral: { auto token_value = context.tokens().GetRealLiteral(token); auto id = context.semantics_ir().AddRealLiteral( {.mantissa = token_value.Mantissa(), @@ -44,7 +44,7 @@ auto HandleLiteral(Context& context, Parse::Node parse_node) -> bool { id)); break; } - case TokenKind::StringLiteral: { + case Lex::TokenKind::StringLiteral: { auto id = context.semantics_ir().AddString( context.tokens().GetStringLiteral(token)); context.AddNodeAndPush( @@ -54,15 +54,15 @@ auto HandleLiteral(Context& context, Parse::Node parse_node) -> bool { context.CanonicalizeType(SemIR::NodeId::BuiltinStringType), id)); break; } - case TokenKind::Type: { + case Lex::TokenKind::Type: { context.node_stack().Push(parse_node, SemIR::NodeId::BuiltinTypeType); break; } - case TokenKind::Bool: { + case Lex::TokenKind::Bool: { context.node_stack().Push(parse_node, SemIR::NodeId::BuiltinBoolType); break; } - case TokenKind::IntegerTypeLiteral: { + case Lex::TokenKind::IntegerTypeLiteral: { auto text = context.tokens().GetTokenText(token); if (text != "i32") { return context.TODO(parse_node, "Currently only i32 is allowed"); @@ -70,7 +70,7 @@ auto HandleLiteral(Context& context, Parse::Node parse_node) -> bool { context.node_stack().Push(parse_node, SemIR::NodeId::BuiltinIntegerType); break; } - case TokenKind::FloatingPointTypeLiteral: { + case Lex::TokenKind::FloatingPointTypeLiteral: { auto text = context.tokens().GetTokenText(token); if (text != "f64") { return context.TODO(parse_node, "Currently only f64 is allowed"); @@ -79,7 +79,7 @@ auto HandleLiteral(Context& context, Parse::Node parse_node) -> bool { SemIR::NodeId::BuiltinFloatingPointType); break; } - case TokenKind::StringTypeLiteral: { + case Lex::TokenKind::StringTypeLiteral: { context.node_stack().Push(parse_node, SemIR::NodeId::BuiltinStringType); break; } diff --git a/toolchain/semantics/semantics_handle_operator.cpp b/toolchain/semantics/semantics_handle_operator.cpp index 9d9832096380..a36588cc627a 100644 --- a/toolchain/semantics/semantics_handle_operator.cpp +++ b/toolchain/semantics/semantics_handle_operator.cpp @@ -13,7 +13,7 @@ auto HandleInfixOperator(Context& context, Parse::Node parse_node) -> bool { // Figure out the operator for the token. auto token = context.parse_tree().node_token(parse_node); switch (auto token_kind = context.tokens().GetKind(token)) { - case TokenKind::Plus: + case Lex::TokenKind::Plus: // TODO: This should search for a compatible interface. For now, it's a // very trivial check of validity on the operation. lhs_id = context.ConvertToValueOfType( @@ -27,8 +27,8 @@ auto HandleInfixOperator(Context& context, Parse::Node parse_node) -> bool { lhs_id, rhs_id)); return true; - case TokenKind::And: - case TokenKind::Or: { + case Lex::TokenKind::And: + case Lex::TokenKind::Or: { // The first operand is wrapped in a ShortCircuitOperand, which we // already handled by creating a RHS block and a resumption block, which // are the current block and its enclosing block. @@ -51,7 +51,7 @@ auto HandleInfixOperator(Context& context, Parse::Node parse_node) -> bool { resume_block_id)); return true; } - case TokenKind::Equal: { + case Lex::TokenKind::Equal: { // TODO: handle complex assignment expression such as `a += 1`. if (SemIR::GetExpressionCategory(context.semantics_ir(), lhs_id) != SemIR::ExpressionCategory::DurableReference) { @@ -81,7 +81,7 @@ auto HandlePostfixOperator(Context& context, Parse::Node parse_node) -> bool { // Figure out the operator for the token. auto token = context.parse_tree().node_token(parse_node); switch (auto token_kind = context.tokens().GetKind(token)) { - case TokenKind::Star: { + case Lex::TokenKind::Star: { auto inner_type_id = context.ExpressionAsType(parse_node, value_id); context.AddNodeAndPush( parse_node, SemIR::Node::PointerType::Make( @@ -100,7 +100,7 @@ auto HandlePrefixOperator(Context& context, Parse::Node parse_node) -> bool { // Figure out the operator for the token. auto token = context.parse_tree().node_token(parse_node); switch (auto token_kind = context.tokens().GetKind(token)) { - case TokenKind::Amp: { + case Lex::TokenKind::Amp: { // Only durable reference expressions can have their address taken. switch (SemIR::GetExpressionCategory(context.semantics_ir(), value_id)) { case SemIR::ExpressionCategory::DurableReference: @@ -128,7 +128,7 @@ auto HandlePrefixOperator(Context& context, Parse::Node parse_node) -> bool { return true; } - case TokenKind::Const: { + case Lex::TokenKind::Const: { // `const (const T)` is probably not what the developer intended. // TODO: Detect `const (const T)*` and suggest moving the `*` inside the // parentheses. @@ -146,7 +146,7 @@ auto HandlePrefixOperator(Context& context, Parse::Node parse_node) -> bool { return true; } - case TokenKind::Not: + case Lex::TokenKind::Not: value_id = context.ConvertToBoolValue(parse_node, value_id); context.AddNodeAndPush( parse_node, @@ -155,7 +155,7 @@ auto HandlePrefixOperator(Context& context, Parse::Node parse_node) -> bool { value_id)); return true; - case TokenKind::Star: { + case Lex::TokenKind::Star: { auto type_id = context.GetUnqualifiedType( context.semantics_ir().GetNode(value_id).type_id()); auto type_node = context.semantics_ir().GetNode( @@ -204,13 +204,13 @@ auto HandleShortCircuitOperand(Context& context, Parse::Node parse_node) SemIR::NodeId branch_value_id = SemIR::NodeId::Invalid; auto short_circuit_result_id = SemIR::NodeId::Invalid; switch (auto token_kind = context.tokens().GetKind(token)) { - case TokenKind::And: + case Lex::TokenKind::And: branch_value_id = cond_value_id; short_circuit_result_id = context.AddNode(SemIR::Node::BoolLiteral::Make( parse_node, bool_type_id, SemIR::BoolValue::False)); break; - case TokenKind::Or: + case Lex::TokenKind::Or: branch_value_id = context.AddNode(SemIR::Node::UnaryOperatorNot::Make( parse_node, bool_type_id, cond_value_id)); short_circuit_result_id = context.AddNode(SemIR::Node::BoolLiteral::Make( diff --git a/toolchain/semantics/semantics_ir_formatter.cpp b/toolchain/semantics/semantics_ir_formatter.cpp index 0bbc318eff18..86674ebc30f3 100644 --- a/toolchain/semantics/semantics_ir_formatter.cpp +++ b/toolchain/semantics/semantics_ir_formatter.cpp @@ -30,7 +30,7 @@ class NodeNamer { }; static_assert(sizeof(ScopeIndex) == sizeof(FunctionId)); - NodeNamer(const TokenizedBuffer& tokenized_buffer, + NodeNamer(const Lex::TokenizedBuffer& tokenized_buffer, const Parse::Tree& parse_tree, const File& semantics_ir) : tokenized_buffer_(tokenized_buffer), parse_tree_(parse_tree), @@ -309,7 +309,7 @@ class NodeNamer { case Parse::NodeKind::ShortCircuitOperand: { bool is_rhs = node.kind() == NodeKind::BranchIf; bool is_and = tokenized_buffer_.GetKind(parse_tree_.node_token( - node.parse_node())) == TokenKind::And; + node.parse_node())) == Lex::TokenKind::And; name = is_and ? (is_rhs ? "and.rhs" : "and.result") : (is_rhs ? "or.rhs" : "or.result"); break; @@ -382,7 +382,7 @@ class NodeNamer { } } - const TokenizedBuffer& tokenized_buffer_; + const Lex::TokenizedBuffer& tokenized_buffer_; const Parse::Tree& parse_tree_; const File& semantics_ir_; @@ -396,7 +396,7 @@ class NodeNamer { // Formatter for printing textual Semantics IR. class Formatter { public: - explicit Formatter(const TokenizedBuffer& tokenized_buffer, + explicit Formatter(const Lex::TokenizedBuffer& tokenized_buffer, const Parse::Tree& parse_tree, const File& semantics_ir, llvm::raw_ostream& out) : semantics_ir_(semantics_ir), @@ -770,7 +770,7 @@ class Formatter { bool in_terminator_sequence = false; }; -auto FormatFile(const TokenizedBuffer& tokenized_buffer, +auto FormatFile(const Lex::TokenizedBuffer& tokenized_buffer, const Parse::Tree& parse_tree, const File& semantics_ir, llvm::raw_ostream& out) -> void { Formatter(tokenized_buffer, parse_tree, semantics_ir, out).Format(); diff --git a/toolchain/semantics/semantics_ir_formatter.h b/toolchain/semantics/semantics_ir_formatter.h index 3562af1d099c..947b84242367 100644 --- a/toolchain/semantics/semantics_ir_formatter.h +++ b/toolchain/semantics/semantics_ir_formatter.h @@ -10,7 +10,7 @@ namespace Carbon::SemIR { -auto FormatFile(const TokenizedBuffer& tokenized_buffer, +auto FormatFile(const Lex::TokenizedBuffer& tokenized_buffer, const Parse::Tree& parse_tree, const File& semantics_ir, llvm::raw_ostream& out) -> void;