carbon-lang/toolchain/parse/context.cpp

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include "toolchain/parse/context.h"

#include <initializer_list>
#include <optional>

#include "common/check.h"
#include "common/ostream.h"
#include "llvm/ADT/STLExtras.h"
#include "toolchain/diagnostics/emitter.h"
#include "toolchain/diagnostics/format_providers.h"
#include "toolchain/lex/token_kind.h"
#include "toolchain/lex/tokenized_buffer.h"
#include "toolchain/parse/node_ids.h"
#include "toolchain/parse/node_kind.h"
#include "toolchain/parse/state.h"
#include "toolchain/parse/tree.h"
#include "toolchain/parse/typed_nodes.h"

namespace Carbon::Parse {

Context::Context(Tree* tree, Lex::TokenizedBuffer* tokens,
                 Diagnostics::Consumer* consumer,
                 llvm::raw_ostream* vlog_stream)
    : tree_(tree),
      tokens_(tokens),
      err_tracker_(*consumer),
      emitter_(&err_tracker_, this),
      vlog_stream_(vlog_stream),
      position_(tokens_->tokens().begin()),
      end_(tokens_->tokens().end()) {
  CARBON_CHECK(position_ != end_, "Empty TokenizedBuffer");
  --end_;
  CARBON_CHECK(tokens_->GetKind(*end_) == Lex::TokenKind::FileEnd,
               "TokenizedBuffer should end with FileEnd, ended with {0}",
               tokens_->GetKind(*end_));
}

auto Context::VlogAddNode() const -> void {
  CARBON_VLOG("Add #node{0}: {1}",
              llvm::format_hex_no_prefix(tree_->node_impls_.size() - 1, 0,
                                         /*Upper=*/true),
              tree_->node_impls_.back());
}

auto Context::VlogPushState() const -> void {
  CARBON_VLOG("Push {0}: {1}\n", state_stack_.size() - 1, state_stack_.back());
}

auto Context::VlogPopState(llvm::StringLiteral label) const -> void {
  CARBON_VLOG("{0} {1}: {2}\n", label, state_stack_.size() - 1,
              state_stack_.back());
}

auto Context::OpeningBracketInsertionPoint() const -> DiagnosticLoc {
  auto previous_token = *(position_ - 1);
  if (tokens().GetEndLoc(previous_token).first ==
      tokens().GetLine(*position_)) {
    return DiagnosticLoc::Before(*position_);
  }
  return DiagnosticLoc::After(previous_token);
}

auto Context::DiagnoseExpectedOpenCurlyBrace(Lex::TokenIndex default_token,
                                             NodeKind start_kind) -> void {
  CARBON_DIAGNOSTIC(ExpectedCurlyBraceAfter, Error, "expected `{` after `{0}`",
                    Lex::TokenKind);
  emitter_.Emit(OpeningBracketInsertionPoint(), ExpectedCurlyBraceAfter,
                tokens().GetKind(default_token));
  AddLeafNode(start_kind, default_token, /*has_error=*/true);
}

auto Context::DiagnoseExpectedOpenParen(Lex::TokenIndex default_token,
                                        NodeKind start_kind) -> void {
  CARBON_DIAGNOSTIC(ExpectedParenAfter, Error, "expected `(` after `{0}`",
                    Lex::TokenKind);
  emitter_.Emit(OpeningBracketInsertionPoint(), ExpectedParenAfter,
                tokens().GetKind(default_token));
  AddLeafNode(start_kind, default_token, /*has_error=*/true);
}

auto Context::DiagnoseAndSkipToCloseSymbol(State state, NodeKind close_kind)
    -> void {
  Lex::TokenKind open_token_kind = tokens().GetKind(state.token);
  if (!open_token_kind.is_opening_symbol()) {
    AddNode(close_kind, state.token, /*has_error=*/true);
    return;
  }

  CARBON_DIAGNOSTIC(ExpectedCloseSymbol, Error,
                    "unexpected tokens before `{0}`", Lex::TokenKind);
  CARBON_DIAGNOSTIC_LABEL(GroupOpenedHere, Info, "matching this `{0}`",
                          Lex::TokenKind);
  auto matched_close = tokens().GetMatchedClosingToken(state.token);
  emitter_
      .Build(DiagnosticLoc::Tokens(
                 {.begin = *position_,
                  .end = Lex::TokenIndex(matched_close.index - 1)}),
             ExpectedCloseSymbol, open_token_kind.closing_symbol())
      .Attach(state.token, GroupOpenedHere, open_token_kind)
      .Emit();

  SkipTo(matched_close);
  AddNode(close_kind, Consume(), /*has_error=*/true);
}

auto Context::FindNextOf(std::initializer_list<Lex::TokenKind> desired_kinds)
    -> std::optional<Lex::TokenIndex> {
  auto new_position = position_;
  while (true) {
    Lex::TokenIndex token = *new_position;
    Lex::TokenKind kind = tokens().GetKind(token);
    if (kind.IsOneOf(desired_kinds)) {
      return token;
    }

    // Step to the next token at the current bracketing level.
    if (kind.is_closing_symbol() || kind == Lex::TokenKind::FileEnd) {
      // There are no more tokens at this level.
      return std::nullopt;
    } else if (kind.is_opening_symbol()) {
      new_position = Lex::TokenIterator(tokens().GetMatchedClosingToken(token));
      // Advance past the closing token.
      ++new_position;
    } else {
      ++new_position;
    }
  }
}

auto Context::SkipMatchingGroup() -> bool {
  if (!PositionKind().is_opening_symbol()) {
    return false;
  }

  SkipTo(tokens().GetMatchedClosingToken(*position_));
  ++position_;
  return true;
}

auto Context::SkipPastLikelyEnd(Lex::TokenIndex skip_root) -> Lex::TokenIndex {
  if (position_ == end_) {
    return *(position_ - 1);
  }

  int root_line_indent =
      tokens().GetIndentColumnNumber(tokens().GetLine(skip_root));

  // We keep scanning through tokens that don't start their own line and
  // through lines indented more than the root's line. Tokens that don't start
  // their line include the rest of the root's line and, because comments run
  // to the end of the line, tokens after a multi-line string literal's
  // closing delimiter (as in `''' + "more"`), both of which continue the
  // construct regardless of indentation.
  auto keep_scanning = [&](Lex::TokenIndex t) {
    int indent = tokens().GetIndentColumnNumber(tokens().GetLine(t));
    return tokens().GetColumnNumber(t) > indent || indent > root_line_indent;
  };

  do {
    if (PositionIs(Lex::TokenKind::CloseCurlyBrace)) {
      // Immediately bail out if we hit an unmatched close curly, this will
      // pop us up a level of the syntax grouping.
      return *(position_ - 1);
    }

    // We assume that a semicolon is always intended to be the end of the
    // current construct.
    if (auto semi = ConsumeIf(Lex::TokenKind::Semi)) {
      return *semi;
    }

    // Skip over any matching group of tokens().
    if (SkipMatchingGroup()) {
      continue;
    }

    // Otherwise just step forward one token.
    ++position_;
  } while (position_ != end_ && keep_scanning(*position_));

  return *(position_ - 1);
}

auto Context::SkipTo(Lex::TokenIndex t) -> void {
  CARBON_CHECK(t >= *position_, "Tried to skip backwards from {0} to {1}",
               position_, Lex::TokenIterator(t));
  position_ = Lex::TokenIterator(t);
  CARBON_CHECK(position_ != end_, "Skipped past EOF.");
}

auto Context::DiagnoseInvalidOperatorFixity(OperatorFixity fixity) -> void {
  if (fixity == OperatorFixity::Infix) {
    // Infix operators must satisfy the infix operator rules.
    CARBON_DIAGNOSTIC(BinaryOperatorRequiresWhitespace, Error,
                      "whitespace missing {0:=-1:before|=0:around|=1:after} "
                      "binary operator",
                      Diagnostics::IntAsSelect);
    // Whitespace missing on one side goes at that side. Whitespace missing
    // around the operator is about the operator.
    Diagnostics::IntAsSelect pos(0);
    DiagnosticLoc loc = *position_;
    if (tokens().HasLeadingWhitespace(*position_)) {
      pos.value = 1;
      loc = DiagnosticLoc::After(*position_);
    } else if (tokens().HasTrailingWhitespace(*position_)) {
      pos.value = -1;
      loc = DiagnosticLoc::Before(*position_);
    }
    emitter_.Emit(loc, BinaryOperatorRequiresWhitespace, pos);
  } else {
    bool prefix = fixity == OperatorFixity::Prefix;

    // Whitespace is not permitted between a symbolic pre/postfix operator and
    // its operand.
    if ((prefix ? tokens().HasTrailingWhitespace(*position_)
                : tokens().HasLeadingWhitespace(*position_))) {
      CARBON_DIAGNOSTIC(
          UnaryOperatorHasWhitespace, Error,
          "whitespace is not allowed {0:after|before} this unary operator",
          Diagnostics::BoolAsSelect);
      emitter_.Emit(*position_, UnaryOperatorHasWhitespace, prefix);
    } else {
      // Pre/postfix operators must not satisfy the infix operator rules.
      CARBON_DIAGNOSTIC(
          UnaryOperatorRequiresWhitespace, Error,
          "whitespace is required {0:before|after} this unary operator",
          Diagnostics::BoolAsSelect);
      emitter_.Emit(prefix ? DiagnosticLoc::Before(*position_)
                           : DiagnosticLoc::After(*position_),
                    UnaryOperatorRequiresWhitespace, prefix);
    }
  }
}

auto Context::RecoverFromInvalidListToken(Lex::TokenKind close_kind,
                                          bool already_has_error) -> void {
  // Don't error a second time on the same element.
  if (!already_has_error) {
    // TODO: Also label the bracket that opened the list, to show which list
    // this is. That needs the opening token, which callers hold in
    // `state.token` but don't pass down.
    CARBON_DIAGNOSTIC(UnexpectedTokenAfterListElement, Error,
                      "expected `,` or `{0}`", Lex::TokenKind);
    emitter_.Emit(PointAfterPreviousToken(), UnexpectedTokenAfterListElement,
                  close_kind);
    ReturnErrorOnState();
  }

  // Recover from the invalid token.
  auto end_of_element = FindNextOf({Lex::TokenKind::Comma, close_kind});
  // The lexer guarantees that parentheses are balanced.
  CARBON_CHECK(end_of_element, "missing matching `{0}` for `{1}`",
               close_kind.opening_symbol(), close_kind);

  SkipTo(*end_of_element);
}

auto Context::RecoverFromMissingDeclSemi(State state, NodeKind node_kind,
                                         Lex::TokenKind decl_kind,
                                         bool is_def_allowed) -> void {
  // TODO: This could better handle things like:
  //   base: { }
  //   var n: i32;
  //       ^ Ends up at `n`, instead of `var`.
  if (!state.has_error) {
    if (is_def_allowed) {
      DiagnoseExpectedDeclSemiOrDefinition(decl_kind);
    } else {
      DiagnoseExpectedDeclSemi(decl_kind);
    }
  }
  RecoverFromDeclError(state, node_kind,
                       /*skip_past_likely_end=*/true);
}

auto Context::RecoverFromDeclError(State state, NodeKind node_kind,
                                   bool skip_past_likely_end) -> void {
  auto token = state.token;
  if (skip_past_likely_end) {
    token = SkipPastLikelyEnd(token);
  }
  AddNode(node_kind, token, /*has_error=*/true);
}

auto Context::ParseLibraryName(bool accept_default)
    -> std::optional<StringLiteralValueId> {
  if (auto library_name_token = ConsumeIf(Lex::TokenKind::StringLiteral)) {
    AddLeafNode(NodeKind::LibraryName, *library_name_token);
    return tokens().GetStringLiteralValue(*library_name_token);
  }

  if (accept_default) {
    if (auto default_token = ConsumeIf(Lex::TokenKind::Default)) {
      AddLeafNode(NodeKind::DefaultLibrary, *default_token);
      return StringLiteralValueId::None;
    }
  }

  CARBON_DIAGNOSTIC(
      ExpectedLibraryNameOrDefault, Error,
      "expected `default` or a string literal to specify the library name");
  CARBON_DIAGNOSTIC(ExpectedLibraryName, Error,
                    "expected a string literal to specify the library name");
  emitter().Emit(PointAfterPreviousToken(), accept_default
                                                ? ExpectedLibraryNameOrDefault
                                                : ExpectedLibraryName);
  return std::nullopt;
}

auto Context::ParseLibrarySpecifier(bool accept_default)
    -> std::optional<StringLiteralValueId> {
  auto library_token = ConsumeChecked(Lex::TokenKind::Library);
  auto library_id = ParseLibraryName(accept_default);
  if (!library_id) {
    AddLeafNode(NodeKind::LibraryName, *position_, /*has_error=*/true);
  }
  AddNode(NodeKind::LibrarySpecifier, library_token, /*has_error=*/false);
  return library_id;
}

auto Context::DiagnoseExpectedDeclSemi(Lex::TokenKind expected_kind) -> void {
  CARBON_DIAGNOSTIC(ExpectedDeclSemi, Error,
                    "`{0}` declarations must end with a `;`", Lex::TokenKind);
  emitter().Emit(PointAfterPreviousToken(), ExpectedDeclSemi, expected_kind);
}

auto Context::DiagnoseExpectedDeclSemiOrDefinition(Lex::TokenKind expected_kind)
    -> void {
  CARBON_DIAGNOSTIC(ExpectedDeclSemiOrDefinition, Error,
                    "`{0}` declarations must either end with a `;` or "
                    "have a `{{ ... }` block for a definition",
                    Lex::TokenKind);
  emitter().Emit(PointAfterPreviousToken(), ExpectedDeclSemiOrDefinition,
                 expected_kind);
}

// Returns whether we are currently parsing in a scope in which function
// definitions are deferred, such as a class or interface.
static auto ParsingInDeferredDefinitionScope(Context& context) -> bool {
  auto& stack = context.state_stack();
  if (stack.size() < 2 ||
      (stack.back().kind != StateKind::DeclScopeLoopAsClass &&
       stack.back().kind != StateKind::DeclScopeLoopAsInterface &&
       stack.back().kind != StateKind::DeclScopeLoopAsRegular)) {
    return false;
  }
  auto kind = stack[stack.size() - 2].kind;
  return kind == StateKind::DeclDefinitionFinishAsClass ||
         kind == StateKind::DeclDefinitionFinishAsImpl ||
         kind == StateKind::DeclDefinitionFinishAsInterface ||
         kind == StateKind::DeclDefinitionFinishAsNamedConstraint;
}

auto Context::AddFunctionDefinitionStart(Lex::TokenIndex token, bool has_error)
    -> void {
  auto start_id = AddNode<NodeKind::FunctionDefinitionStart>(token, has_error);
  if (ParsingInDeferredDefinitionScope(*this)) {
    deferred_definition_stack_.push_back(
        tree_->deferred_definitions_.Add({.start_id = start_id}));
  }
}

auto Context::AddFunctionTerseDefinitionStart(Lex::TokenIndex token,
                                              bool has_error) -> void {
  auto start_id =
      AddNode<NodeKind::FunctionTerseDefinitionStart>(token, has_error);
  if (ParsingInDeferredDefinitionScope(*this)) {
    deferred_definition_stack_.push_back(
        tree_->deferred_definitions_.Add({.start_id = start_id}));
  }
}

auto Context::AddFunctionDefinition(Lex::TokenIndex token, bool has_error)
    -> void {
  auto definition_id = AddNode<NodeKind::FunctionDefinition>(token, has_error);
  if (ParsingInDeferredDefinitionScope(*this)) {
    auto definition_index = deferred_definition_stack_.pop_back_val();
    auto& definition = tree_->deferred_definitions_.Get(definition_index);
    definition.definition_id = definition_id;
    definition.next_definition_index =
        DeferredDefinitionIndex(tree_->deferred_definitions().size());
  }
}

auto Context::AddFunctionTerseDefinition(Lex::TokenIndex token, bool has_error)
    -> void {
  auto definition_id =
      AddNode<NodeKind::FunctionTerseDefinition>(token, has_error);
  if (ParsingInDeferredDefinitionScope(*this)) {
    auto definition_index = deferred_definition_stack_.pop_back_val();
    auto& definition = tree_->deferred_definitions_.Get(definition_index);
    definition.definition_id = definition_id;
    definition.next_definition_index =
        DeferredDefinitionIndex(tree_->deferred_definitions().size());
  }
}

auto Context::PrintForStackDump(llvm::raw_ostream& output) const -> void {
  output << "Parser stack:\n";
  for (auto [i, entry] : llvm::enumerate(state_stack_)) {
    output << "\t" << i << ".\t" << entry.kind;
    PrintTokenForStackDump(output, entry.token);
  }
  output << "\tcursor\tposition_";
  PrintTokenForStackDump(output, *position_);
}

auto Context::PrintTokenForStackDump(llvm::raw_ostream& output,
                                     Lex::TokenIndex token) const -> void {
  output << " @ " << tokens_->GetLineNumber(token) << ":"
         << tokens_->GetColumnNumber(token) << ": token " << token << " : "
         << tokens_->GetKind(token) << "\n";
}

}  // namespace Carbon::Parse