diff --git a/toolchain/lex/tokenized_buffer.cpp b/toolchain/lex/tokenized_buffer.cpp index d6eccc6fdcae..3ab3188d759a 100644 --- a/toolchain/lex/tokenized_buffer.cpp +++ b/toolchain/lex/tokenized_buffer.cpp @@ -20,10 +20,6 @@ namespace Carbon::Lex { -auto TokenizedBuffer::GetKind(TokenIndex token) const -> TokenKind { - return GetTokenInfo(token).kind(); -} - auto TokenizedBuffer::GetLine(TokenIndex token) const -> LineIndex { return FindLineIndex(GetTokenInfo(token).byte_offset()); } @@ -159,16 +155,6 @@ auto TokenizedBuffer::GetMatchedOpeningToken(TokenIndex closing_token) const return closing_token_info.opening_token_index(); } -auto TokenizedBuffer::HasLeadingWhitespace(TokenIndex token) const -> bool { - return GetTokenInfo(token).has_leading_space(); -} - -auto TokenizedBuffer::HasTrailingWhitespace(TokenIndex token) const -> bool { - TokenIterator it(token); - ++it; - return it != tokens().end() && GetTokenInfo(*it).has_leading_space(); -} - auto TokenizedBuffer::IsRecoveryToken(TokenIndex token) const -> bool { if (recovery_tokens_.empty()) { return false; @@ -359,20 +345,6 @@ auto TokenizedBuffer::AddLine(LineInfo info) -> LineIndex { return LineIndex(static_cast(line_infos_.size()) - 1); } -auto TokenizedBuffer::GetTokenInfo(TokenIndex token) -> TokenInfo& { - return token_infos_[token.index]; -} - -auto TokenizedBuffer::GetTokenInfo(TokenIndex token) const -> const TokenInfo& { - return token_infos_[token.index]; -} - -auto TokenizedBuffer::AddToken(TokenInfo info) -> TokenIndex { - token_infos_.push_back(info); - expected_max_parse_tree_size_ += info.kind().expected_max_parse_tree_size(); - return TokenIndex(static_cast(token_infos_.size()) - 1); -} - auto TokenizedBuffer::CollectMemUsage(MemUsage& mem_usage, llvm::StringRef label) const -> void { mem_usage.Add(MemUsage::ConcatLabel(label, "allocator_"), allocator_); diff --git a/toolchain/lex/tokenized_buffer.h b/toolchain/lex/tokenized_buffer.h index da2ca1d1f18a..52641fbe60f1 100644 --- a/toolchain/lex/tokenized_buffer.h +++ b/toolchain/lex/tokenized_buffer.h @@ -477,6 +477,38 @@ using LexerDiagnosticEmitter = DiagnosticEmitter; // A diagnostic emitter that uses tokens as its source of location information. using TokenDiagnosticEmitter = DiagnosticEmitter; +inline auto TokenizedBuffer::GetKind(TokenIndex token) const -> TokenKind { + return GetTokenInfo(token).kind(); +} + +inline auto TokenizedBuffer::HasLeadingWhitespace(TokenIndex token) const + -> bool { + return GetTokenInfo(token).has_leading_space(); +} + +inline auto TokenizedBuffer::HasTrailingWhitespace(TokenIndex token) const + -> bool { + TokenIterator it(token); + ++it; + return it != tokens().end() && GetTokenInfo(*it).has_leading_space(); +} + +inline auto TokenizedBuffer::GetTokenInfo(TokenIndex token) -> TokenInfo& { + return token_infos_[token.index]; +} + +inline auto TokenizedBuffer::GetTokenInfo(TokenIndex token) const + -> const TokenInfo& { + return token_infos_[token.index]; +} + +inline auto TokenizedBuffer::AddToken(TokenInfo info) -> TokenIndex { + TokenIndex index(token_infos_.size()); + token_infos_.push_back(info); + expected_max_parse_tree_size_ += info.kind().expected_max_parse_tree_size(); + return index; +} + } // namespace Carbon::Lex #endif // CARBON_TOOLCHAIN_LEX_TOKENIZED_BUFFER_H_ diff --git a/toolchain/parse/context.cpp b/toolchain/parse/context.cpp index 3d9a557353a8..31312fd42f48 100644 --- a/toolchain/parse/context.cpp +++ b/toolchain/parse/context.cpp @@ -66,16 +66,6 @@ Context::Context(Tree& tree, Lex::TokenizedBuffer& tokens, tokens_->GetKind(*end_)); } -auto Context::AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error) - -> void { - tree_->node_impls_.push_back(Tree::NodeImpl(kind, has_error, token)); -} - -auto Context::AddNode(NodeKind kind, Lex::TokenIndex token, bool has_error) - -> void { - tree_->node_impls_.push_back(Tree::NodeImpl(kind, has_error, token)); -} - auto Context::ReplacePlaceholderNode(int32_t position, NodeKind kind, Lex::TokenIndex token, bool has_error) -> void { @@ -143,13 +133,6 @@ auto Context::ConsumeChecked(Lex::TokenKind kind) -> Lex::TokenIndex { return Consume(); } -auto Context::ConsumeIf(Lex::TokenKind kind) -> std::optional { - if (!PositionIs(kind)) { - return std::nullopt; - } - return Consume(); -} - auto Context::FindNextOf(std::initializer_list desired_kinds) -> std::optional { auto new_position = position_; diff --git a/toolchain/parse/context.h b/toolchain/parse/context.h index a8ae5b09b329..ec594adfcd50 100644 --- a/toolchain/parse/context.h +++ b/toolchain/parse/context.h @@ -97,10 +97,14 @@ class Context { // Adds a node to the parse tree that has no children (a leaf). auto AddLeafNode(NodeKind kind, Lex::TokenIndex token, bool has_error = false) - -> void; + -> void { + tree_->node_impls_.push_back(Tree::NodeImpl(kind, has_error, token)); + } // Adds a node to the parse tree that has children. - auto AddNode(NodeKind kind, Lex::TokenIndex token, bool has_error) -> void; + auto AddNode(NodeKind kind, Lex::TokenIndex token, bool has_error) -> void { + tree_->node_impls_.push_back(Tree::NodeImpl(kind, has_error, token)); + } // Replaces the placeholder node at the indicated position with a leaf node. // @@ -154,7 +158,12 @@ class Context { // If the current position's token matches this `Kind`, returns it and // advances to the next position. Otherwise returns an empty optional. - auto ConsumeIf(Lex::TokenKind kind) -> std::optional; + auto ConsumeIf(Lex::TokenKind kind) -> std::optional { + if (!PositionIs(kind)) { + return std::nullopt; + } + return Consume(); + } // Find the next token of any of the given kinds at the current bracketing // level.