Parsing for generic and template parameters. (#2685)

Also cleans up some comments about related parse nodes. Currently basic and not heavily validated.

Co-authored-by: Richard Smith <richard@metafoo.co.uk>

toolchain/lexer/token_kind.def

@@ -45,6 +45,7 @@ CARBON_SYMBOL_TOKEN(LessLessEqual,       "<<=")
 CARBON_SYMBOL_TOKEN(AmpEqual,            "&=")
 CARBON_SYMBOL_TOKEN(CaretEqual,          "^=")
 CARBON_SYMBOL_TOKEN(ColonEqual,          ":=")
+CARBON_SYMBOL_TOKEN(ColonExclaim,        ":!")
 CARBON_SYMBOL_TOKEN(EqualEqual,          "==")
 CARBON_SYMBOL_TOKEN(EqualGreater,        "=>")
 CARBON_SYMBOL_TOKEN(ExclaimEqual,        "!=")
@@ -161,6 +162,7 @@ CARBON_KEYWORD_TOKEN(SelfParameter, "self")
 CARBON_KEYWORD_TOKEN(SelfType,      "Self")
 // TODO: Although we provide a String type literal, it's not standardized.
 CARBON_KEYWORD_TOKEN(StringTypeLiteral, "String")
+CARBON_KEYWORD_TOKEN(Template,      "template")
 CARBON_KEYWORD_TOKEN(Then,          "then")
 // Underscore is tokenized as a keyword because it's part of identifiers.
 CARBON_KEYWORD_TOKEN(Underscore,    "_")

toolchain/parser/parse_node_kind.def

@@ -88,23 +88,30 @@ CARBON_PARSE_NODE_KIND_BRACKET(FunctionDefinitionStart, FunctionIntroducer)
 CARBON_PARSE_NODE_KIND_BRACKET(FunctionDefinition, FunctionDefinitionStart)
 CARBON_PARSE_NODE_KIND_BRACKET(FunctionDeclaration, FunctionIntroducer)
 
-// A parameter list:
-//   ParamertListStart
-//   _external_: expression
+// A parameter list, possibly deduced:
+//   [Deduced]ParamertListStart
+//   _external_: [Generic]PatternBinding
 //   ParameterListComma
-// ParameterList
+// [Deduced]ParameterList
 //
 // Expressions and ParameterListComma may repeat with ParameterListComma as a
 // separator.
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(ParameterListStart, 0)
+CARBON_PARSE_NODE_KIND_CHILD_COUNT(DeducedParameterListStart, 0)
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(ParameterListComma, 0)
 CARBON_PARSE_NODE_KIND_BRACKET(ParameterList, ParameterListStart)
+CARBON_PARSE_NODE_KIND_BRACKET(DeducedParameterList, DeducedParameterListStart)
 
 // A pattern binding, such as `name: Type`:
 //   DeclaredName
 //   _external_: type expression
-// PatternBinding
+// [Generic]PatternBinding
+//
+// Address and Template may be parents to [Generic]PatternBinding, in that order.
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(PatternBinding, 2)
+CARBON_PARSE_NODE_KIND_CHILD_COUNT(GenericPatternBinding, 2)
+CARBON_PARSE_NODE_KIND_CHILD_COUNT(Address, 1)
+CARBON_PARSE_NODE_KIND_CHILD_COUNT(Template, 1)
 
 // `var`:
 //   VariableIntroducer
@@ -330,27 +337,13 @@ CARBON_PARSE_NODE_KIND_BRACKET(NamedConstraintDefinition,
 CARBON_PARSE_NODE_KIND_BRACKET(NamedConstraintDeclaration,
                                NamedConstraintIntroducer)
 
-// A pattern binding for `self` deduced parameter:
-//   `self`
-//   _external_: type expression
-// SelfDeducedParameter
+// The `self` name and `Self` type keywords. Typically of the form `self: Self`:
+//   SelfDeducedParameter
+//   SelfType
+// PatternBinding
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(SelfDeducedParameter, 0)
-
-// A pattern binding for `addr self` deduced parameter:
-//   `addr self`
-//   _external_: type expression
-// SelfDeducedParameter
-// CARBON_PARSE_NODE_KIND_CHILD_COUNT(SelfDeducedParameterAddress, 1)
-CARBON_PARSE_NODE_KIND_CHILD_COUNT(Address, 1)
 CARBON_PARSE_NODE_KIND_CHILD_COUNT(SelfType, 0)
 
-// Deduced parameters, such as `[self: Self]`:
-//   DeducedParameterListStart
-//   _external_: Address or PatternBinding
-// DeducedParameterList
-CARBON_PARSE_NODE_KIND_CHILD_COUNT(DeducedParameterListStart, 0)
-CARBON_PARSE_NODE_KIND_BRACKET(DeducedParameterList, DeducedParameterListStart)
-
 #undef CARBON_PARSE_NODE_KIND
 #undef CARBON_PARSE_NODE_KIND_BRACKET
 #undef CARBON_PARSE_NODE_KIND_CHILD_COUNT

toolchain/parser/parser.cpp

@@ -1451,23 +1451,26 @@ auto Parser::HandleParenExpressionFinishAsTupleState() -> void {
           state.has_error);
 }
 
+auto Parser::ConsumeIfPatternKeyword(TokenKind keyword_token,
+                                     ParserState keyword_state,
+                                     int subtree_start) -> void {
+  if (auto token = ConsumeIf(keyword_token)) {
+    PushState(StateStackEntry(
+        keyword_state, PrecedenceGroup::ForTopLevelExpression(),
+        PrecedenceGroup::ForTopLevelExpression(), *token, subtree_start));
+  }
+}
+
 auto Parser::HandlePattern(PatternKind pattern_kind) -> void {
   auto state = PopState();
 
-  // Ensure the finish state always follows, including for errors.
-  state.state = ParserState::PatternFinish;
-
-  // Parameters may have an `addr` keyword prefixing the pattern. This becomes
-  // the parent for the full PatternBinding.
+  // Parameters may have keywords prefixing the pattern. They become the parent
+  // for the full PatternBinding.
   if (pattern_kind != PatternKind::Variable) {
-    if (auto addr = ConsumeIf(TokenKind::Addr)) {
-      // Mirror information including subtree_size, but we need to change a copy
-      // to avoid affecting the token.
-      auto addr_state = state;
-      addr_state.state = ParserState::PatternAddress;
-      addr_state.token = *addr;
-      PushState(addr_state);
-    }
+    ConsumeIfPatternKeyword(TokenKind::Template, ParserState::PatternTemplate,
+                            state.subtree_start);
+    ConsumeIfPatternKeyword(TokenKind::Addr, ParserState::PatternAddress,
+                            state.subtree_start);
   }
 
   // Handle an invalid pattern introducer for parameters and variables.
@@ -1487,6 +1490,8 @@ auto Parser::HandlePattern(PatternKind pattern_kind) -> void {
         break;
       }
     }
+    // Still use the finish state for errors.
+    state.state = ParserState::PatternFinishAsRegular;
     state.has_error = true;
     PushState(state);
   };
@@ -1506,9 +1511,13 @@ auto Parser::HandlePattern(PatternKind pattern_kind) -> void {
     return;
   }
 
-  if (auto colon = ConsumeIf(TokenKind::Colon)) {
-    // Use the colon for the root node.
-    state.token = *colon;
+  if (auto kind = PositionKind();
+      kind == TokenKind::Colon || kind == TokenKind::ColonExclaim) {
+    state.state = kind == TokenKind::Colon
+                      ? ParserState::PatternFinishAsRegular
+                      : ParserState::PatternFinishAsGeneric;
+    // Use the `:` or `:!` for the root node.
+    state.token = Consume();
     PushState(state);
     PushStateForExpression(PrecedenceGroup::ForType());
   } else {
@@ -1529,7 +1538,7 @@ auto Parser::HandlePatternAsVariableState() -> void {
   HandlePattern(PatternKind::Variable);
 }
 
-auto Parser::HandlePatternFinishState() -> void {
+auto Parser::HandlePatternFinish(ParseNodeKind node_kind) -> void {
   auto state = PopState();
 
   // If an error was encountered, propagate it without adding a node.
@@ -1539,8 +1548,15 @@ auto Parser::HandlePatternFinishState() -> void {
   }
 
   // TODO: may need to mark has_error if !type.
-  AddNode(ParseNodeKind::PatternBinding, state.token, state.subtree_start,
-          /*has_error=*/false);
+  AddNode(node_kind, state.token, state.subtree_start, /*has_error=*/false);
+}
+
+auto Parser::HandlePatternFinishAsGenericState() -> void {
+  HandlePatternFinish(ParseNodeKind::GenericPatternBinding);
+}
+
+auto Parser::HandlePatternFinishAsRegularState() -> void {
+  HandlePatternFinish(ParseNodeKind::PatternBinding);
 }
 
 auto Parser::HandlePatternAddressState() -> void {
@@ -1555,6 +1571,18 @@ auto Parser::HandlePatternAddressState() -> void {
   }
 }
 
+auto Parser::HandlePatternTemplateState() -> void {
+  auto state = PopState();
+
+  AddNode(ParseNodeKind::Template, state.token, state.subtree_start,
+          state.has_error);
+
+  // If an error was encountered, propagate it while adding a node.
+  if (state.has_error) {
+    ReturnErrorOnState();
+  }
+}
+
 auto Parser::HandleStatementState() -> void {
   PopAndDiscardState();
 

toolchain/parser/parser.h

@@ -339,6 +339,14 @@ class Parser {
   // Handles PatternAs(DeducedParameter|FunctionParameter|Variable).
   auto HandlePattern(PatternKind pattern_kind) -> void;
 
+  // Handles PatternFinishAs(Generic|Regular).
+  auto HandlePatternFinish(ParseNodeKind node_kind) -> void;
+
+  // For HandlePattern, tries to consume a wrapping keyword.
+  auto ConsumeIfPatternKeyword(TokenKind keyword_token,
+                               ParserState keyword_state, int subtree_start)
+      -> void;
+
   // Handles the `;` after a keyword statement.
   auto HandleStatementKeywordFinish(ParseNodeKind node_kind) -> void;
 

toolchain/parser/parser_state.def

@@ -380,7 +380,7 @@ CARBON_PARSER_STATE_VARIANTS2(ParenExpressionFinish, Normal, Tuple)
 //
 // If valid:
 //   1. Expression
-//   2. PatternFinish
+//   2. PatternFinishAs(Generic|Regular)
 // Else:
 //   1. PatternFinish
 CARBON_PARSER_STATE_VARIANTS3(Pattern, DeducedParameter, Parameter, Variable)
@@ -391,11 +391,17 @@ CARBON_PARSER_STATE_VARIANTS3(Pattern, DeducedParameter, Parameter, Variable)
 //   (state done)
 CARBON_PARSER_STATE(PatternAddress)
 
+// Handles `template` in a pattern.
+//
+// Always:
+//   (state done)
+CARBON_PARSER_STATE(PatternTemplate)
+
 // Finishes pattern processing.
 //
 // Always:
 //   (state done)
-CARBON_PARSER_STATE(PatternFinish)
+CARBON_PARSER_STATE_VARIANTS2(PatternFinish, Generic, Regular)
 
 // Handles a single statement. While typically within a statement block, this
 // can also be used for error recovery where we expect a statement block and

toolchain/parser/testdata/generics/generic_params/basic.carbon

@@ -0,0 +1,19 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// RUN: %{carbon-run-parser}
+// CHECK:STDOUT: [
+// CHECK:STDOUT:   {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:   {kind: 'DeclaredName', text: 'foo'},
+// CHECK:STDOUT:     {kind: 'ParameterListStart', text: '('},
+// CHECK:STDOUT:       {kind: 'DeclaredName', text: 'a'},
+// CHECK:STDOUT:       {kind: 'Literal', text: 'i32'},
+// CHECK:STDOUT:     {kind: 'GenericPatternBinding', text: ':!', subtree_size: 3},
+// CHECK:STDOUT:   {kind: 'ParameterList', text: ')', subtree_size: 5},
+// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', subtree_size: 8},
+// CHECK:STDOUT: {kind: 'FileEnd', text: ''},
+// CHECK:STDOUT: ]
+
+fn foo(a:! i32);

toolchain/parser/testdata/generics/generic_params/template.carbon

@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// RUN: %{carbon-run-parser}
+// CHECK:STDOUT: [
+// CHECK:STDOUT:   {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:   {kind: 'DeclaredName', text: 'foo'},
+// CHECK:STDOUT:     {kind: 'ParameterListStart', text: '('},
+// CHECK:STDOUT:         {kind: 'DeclaredName', text: 'a'},
+// CHECK:STDOUT:         {kind: 'Literal', text: 'i32'},
+// CHECK:STDOUT:       {kind: 'GenericPatternBinding', text: ':!', subtree_size: 3},
+// CHECK:STDOUT:     {kind: 'Template', text: 'template', subtree_size: 4},
+// CHECK:STDOUT:   {kind: 'ParameterList', text: ')', subtree_size: 6},
+// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', subtree_size: 9},
+// CHECK:STDOUT: {kind: 'FileEnd', text: ''},
+// CHECK:STDOUT: ]
+
+fn foo(template a:! i32);

toolchain/parser/testdata/generics/generic_params/template_addr.carbon

@@ -0,0 +1,21 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// RUN: %{carbon-run-parser}
+// CHECK:STDOUT: [
+// CHECK:STDOUT:   {kind: 'FunctionIntroducer', text: 'fn'},
+// CHECK:STDOUT:   {kind: 'DeclaredName', text: 'foo'},
+// CHECK:STDOUT:     {kind: 'ParameterListStart', text: '('},
+// CHECK:STDOUT:           {kind: 'DeclaredName', text: 'a'},
+// CHECK:STDOUT:           {kind: 'Literal', text: 'i32'},
+// CHECK:STDOUT:         {kind: 'GenericPatternBinding', text: ':!', subtree_size: 3},
+// CHECK:STDOUT:       {kind: 'Address', text: 'addr', subtree_size: 4},
+// CHECK:STDOUT:     {kind: 'Template', text: 'template', subtree_size: 5},
+// CHECK:STDOUT:   {kind: 'ParameterList', text: ')', subtree_size: 7},
+// CHECK:STDOUT: {kind: 'FunctionDeclaration', text: ';', subtree_size: 10},
+// CHECK:STDOUT: {kind: 'FileEnd', text: ''},
+// CHECK:STDOUT: ]
+
+fn foo(template addr a:! i32);

toolchain/semantics/semantics_parse_tree_handler.cpp

@@ -553,6 +553,12 @@ auto SemanticsParseTreeHandler::HandleFunctionIntroducer(
   return true;
 }
 
+auto SemanticsParseTreeHandler::HandleGenericPatternBinding(
+    ParseTree::Node parse_node) -> bool {
+  emitter_->Emit(parse_node, SemanticsTodo, "GenericPatternBinding");
+  return false;
+}
+
 auto SemanticsParseTreeHandler::HandleIfCondition(ParseTree::Node parse_node)
     -> bool {
   emitter_->Emit(parse_node, SemanticsTodo, "HandleIfCondition");
@@ -977,6 +983,12 @@ auto SemanticsParseTreeHandler::HandleStructTypeLiteral(
   return false;
 }
 
+auto SemanticsParseTreeHandler::HandleTemplate(ParseTree::Node parse_node)
+    -> bool {
+  emitter_->Emit(parse_node, SemanticsTodo, "HandleTemplate");
+  return false;
+}
+
 auto SemanticsParseTreeHandler::HandleTupleLiteral(ParseTree::Node parse_node)
     -> bool {
   emitter_->Emit(parse_node, SemanticsTodo, "HandleTupleLiteral");