carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

[toolchain] Parse variables and parameters as 'name: Type'. (#574)

Browse Source 
Factor out code for pattern parsing, that only recognizes this form for
now, and uniformly form a 'PatternBinding' parse node for this, for
both variables and patterns.
This commit is contained in:
Richard Smith
2021-06-14 14:28:45 -07:00
committed by GitHub
parent 47ee66feef
commit 61b30b1243
6 changed files with 103 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/parser/parse_node_kind.def
+1 -1
View File
@@ -21,7 +21,7 @@ CARBON_PARSE_NODE_KIND(FunctionDeclaration)
CARBON_PARSE_NODE_KIND(ParameterList)
CARBON_PARSE_NODE_KIND(ParameterListComma)
CARBON_PARSE_NODE_KIND(ParameterListEnd)
CARBON_PARSE_NODE_KIND(ParameterDeclaration)
CARBON_PARSE_NODE_KIND(PatternBinding)
CARBON_PARSE_NODE_KIND(ReturnType)
CARBON_PARSE_NODE_KIND(VariableDeclaration)
CARBON_PARSE_NODE_KIND(VariableInitializer)
toolchain/parser/parse_tree_test.cpp
+34 -29
View File
@@ -151,26 +151,27 @@ TEST_F(ParseTreeTest,
}
TEST_F(ParseTreeTest, FunctionDeclarationWithParameterList) {
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo(Int bar, Int baz);");
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo(bar: Int, baz: Int);");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_FALSE(tree.HasErrors());
EXPECT_THAT(
tree,
MatchParseTreeNodes(
{MatchFunctionDeclaration(
MatchDeclaredName("foo"),
MatchParameterList(
MatchParameterDeclaration(MatchNameReference("Int"), "bar"),
MatchParameterListComma(),
MatchParameterDeclaration(MatchNameReference("Int"), "baz"),
MatchParameterListEnd()),
MatchDeclarationEnd()),
MatchFileEnd()}));
EXPECT_THAT(tree,
MatchParseTreeNodes(
{MatchFunctionDeclaration(
MatchDeclaredName("foo"),
MatchParameterList(
MatchPatternBinding(MatchDeclaredName("bar"), ":",
MatchNameReference("Int")),
MatchParameterListComma(),
MatchPatternBinding(MatchDeclaredName("baz"), ":",
MatchNameReference("Int")),
MatchParameterListEnd()),
MatchDeclarationEnd()),
MatchFileEnd()}));
}
TEST_F(ParseTreeTest, FunctionDefinitionWithParameterList) {
TokenizedBuffer tokens = GetTokenizedBuffer(
"fn foo(Int bar, Int baz) {\n"
"fn foo(bar: Int, baz: Int) {\n"
" foo(baz, bar + baz);\n"
"}");
ParseTree tree = ParseTree::Parse(tokens, consumer);
@@ -181,9 +182,11 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithParameterList) {
{MatchFunctionDeclaration(
MatchDeclaredName("foo"),
MatchParameterList(
MatchParameterDeclaration(MatchNameReference("Int"), "bar"),
MatchPatternBinding(MatchDeclaredName("bar"), ":",
MatchNameReference("Int")),
MatchParameterListComma(),
MatchParameterDeclaration(MatchNameReference("Int"), "baz"),
MatchPatternBinding(MatchDeclaredName("baz"), ":",
MatchNameReference("Int")),
MatchParameterListEnd()),
MatchCodeBlock(
MatchExpressionStatement(MatchCallExpression(
@@ -237,8 +240,7 @@ TEST_F(ParseTreeTest, FunctionDeclarationWithSingleIdentifierParameterList) {
MatchParseTreeNodes(
{MatchFunctionDeclaration(
MatchDeclaredName("foo"),
MatchParameterList(MatchNameReference("bar"), HasError,
MatchParameterListEnd()),
MatchParameterList(HasError, MatchParameterListEnd()),
MatchDeclarationEnd()),
MatchFileEnd()}));
}
@@ -530,10 +532,10 @@ TEST_F(ParseTreeTest, Operators) {
TEST_F(ParseTreeTest, VariableDeclarations) {
TokenizedBuffer tokens = GetTokenizedBuffer(
"var Int v = 0;\n"
"var Int w;\n"
"var v: Int = 0;\n"
"var w: Int;\n"
"fn F() {\n"
" var String s = \"hello\";\n"
" var s: String = \"hello\";\n"
"}");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_FALSE(tree.HasErrors());
@@ -541,14 +543,17 @@ TEST_F(ParseTreeTest, VariableDeclarations) {
EXPECT_THAT(tree,
MatchParseTreeNodes(
{MatchVariableDeclaration(
MatchNameReference("Int"), MatchDeclaredName("v"),
MatchPatternBinding(MatchDeclaredName("v"), ":",
MatchNameReference("Int")),
MatchVariableInitializer(MatchLiteral("0")),
MatchDeclarationEnd()),
MatchVariableDeclaration(MatchNameReference("Int"),
MatchDeclaredName("w"),
MatchDeclarationEnd()),
MatchVariableDeclaration(
MatchPatternBinding(MatchDeclaredName("w"), ":",
MatchNameReference("Int")),
MatchDeclarationEnd()),
MatchFunctionWithBody(MatchVariableDeclaration(
MatchNameReference("String"), MatchDeclaredName("s"),
MatchPatternBinding(MatchDeclaredName("s"), ":",
MatchNameReference("String")),
MatchVariableInitializer(MatchLiteral("\"hello\"")),
MatchDeclarationEnd())),
MatchFileEnd()}));
@@ -699,7 +704,7 @@ TEST_F(ParseTreeTest, Return) {
" if (c)\n"
" return;\n"
"}\n"
"fn G(Int x) -> Int {\n"
"fn G(x: Int) -> Int {\n"
" return x;\n"
"}");
ParseTree tree = ParseTree::Parse(tokens, consumer);
@@ -713,8 +718,8 @@ TEST_F(ParseTreeTest, Return) {
MatchReturnStatement(MatchStatementEnd()))),
MatchFunctionDeclaration(
MatchDeclaredName(),
MatchParameters(
MatchParameterDeclaration(MatchNameReference("Int"), "x")),
MatchParameters(MatchPatternBinding(MatchDeclaredName("x"), ":",
MatchNameReference("Int"))),
MatchReturnType(MatchNameReference("Int")),
MatchCodeBlock(MatchReturnStatement(MatchNameReference("x"),
MatchStatementEnd()),
toolchain/parser/parser_impl.cpp
+39 -27
View File
@@ -45,13 +45,13 @@ struct ExpectedFunctionBodyOrSemi
struct ExpectedVariableName : SimpleDiagnostic<ExpectedVariableName> {
static constexpr llvm::StringLiteral ShortName = "syntax-error";
static constexpr llvm::StringLiteral Message =
"Expected variable name after type in `var` declaration.";
"Expected pattern in `var` declaration.";
};
struct ExpectedParameterName : SimpleDiagnostic<ExpectedParameterName> {
static constexpr llvm::StringLiteral ShortName = "syntax-error";
static constexpr llvm::StringLiteral Message =
"Expected parameter name after type in parameter declaration.";
"Expected parameter declaration.";
};
struct UnrecognizedDeclaration : SimpleDiagnostic<UnrecognizedDeclaration> {
@@ -376,23 +376,34 @@ auto ParseTree::Parser::ParseParenList(ListElementParser list_element_parser,
return list_handler(open_paren, Consume(TokenKind::CloseParen()), has_errors);
}
auto ParseTree::Parser::ParseFunctionParameter() -> llvm::Optional<Node> {
// A parameter is of the form
// type identifier
auto start = GetSubtreeStartPosition();
auto type = ParseType();
// FIXME: We can't use DeclaredName here because we need to use the
// identifier token as the root token in the parameter node.
auto name = ConsumeIf(TokenKind::Identifier());
if (!name) {
emitter.EmitError<ExpectedParameterName>(*position);
return llvm::None;
auto ParseTree::Parser::ParsePattern(PatternKind kind) -> llvm::Optional<Node> {
if (NextTokenIs(TokenKind::Identifier()) &&
tokens.GetKind(*(position + 1)) == TokenKind::Colon()) {
// identifier `:` type
auto start = GetSubtreeStartPosition();
AddLeafNode(ParseNodeKind::DeclaredName(),
Consume(TokenKind::Identifier()));
auto colon = Consume(TokenKind::Colon());
auto type = ParseType();
return AddNode(ParseNodeKind::PatternBinding(), colon, start,
/*has_error=*/!type);
}
return AddNode(ParseNodeKind::ParameterDeclaration(), *name, start,
/*has_error=*/!type);
switch (kind) {
case PatternKind::Parameter:
emitter.EmitError<ExpectedParameterName>(*position);
break;
case PatternKind::Variable:
emitter.EmitError<ExpectedVariableName>(*position);
break;
}
return llvm::None;
}
auto ParseTree::Parser::ParseFunctionParameter() -> llvm::Optional<Node> {
return ParsePattern(PatternKind::Parameter);
}
auto ParseTree::Parser::ParseFunctionSignature() -> bool {
@@ -507,18 +518,15 @@ auto ParseTree::Parser::ParseFunctionDeclaration() -> Node {
}
auto ParseTree::Parser::ParseVariableDeclaration() -> Node {
// `var` expression identifier [= expression] `;`
// `var` pattern [= expression] `;`
TokenizedBuffer::Token var_token = Consume(TokenKind::VarKeyword());
auto start = GetSubtreeStartPosition();
auto type = ParseType();
auto name = ConsumeAndAddLeafNodeIf(TokenKind::Identifier(),
ParseNodeKind::DeclaredName());
if (!name) {
emitter.EmitError<ExpectedVariableName>(*position);
if (auto after_name = FindNextOf({TokenKind::Equal(), TokenKind::Semi()})) {
SkipTo(*after_name);
auto pattern = ParsePattern(PatternKind::Variable);
if (!pattern) {
if (auto after_pattern =
FindNextOf({TokenKind::Equal(), TokenKind::Semi()})) {
SkipTo(*after_pattern);
}
}
@@ -538,7 +546,7 @@ auto ParseTree::Parser::ParseVariableDeclaration() -> Node {
}
return AddNode(ParseNodeKind::VariableDeclaration(), var_token, start,
/*has_error=*/!type || !name || !semi);
/*has_error=*/!pattern || !semi);
}
auto ParseTree::Parser::ParseEmptyDeclaration() -> Node {
@@ -744,6 +752,10 @@ auto ParseTree::Parser::ParseExpression() -> llvm::Optional<Node> {
return ParseOperatorExpression(PrecedenceGroup::ForTopLevelExpression());
}
auto ParseTree::Parser::ParseType() -> llvm::Optional<Node> {
return ParseOperatorExpression(PrecedenceGroup::ForType());
}
auto ParseTree::Parser::ParseExpressionStatement() -> llvm::Optional<Node> {
TokenizedBuffer::Token start_token = *position;
auto start = GetSubtreeStartPosition();
toolchain/parser/parser_impl.h
+9 -1
View File
@@ -200,7 +200,7 @@ class ParseTree::Parser {
auto ParseExpression() -> llvm::Optional<Node>;
// Parses a type expression.
auto ParseType() -> llvm::Optional<Node> { return ParseExpression(); }
auto ParseType() -> llvm::Optional<Node>;
// Parses an expression statement: an expression followed by a semicolon.
auto ParseExpressionStatement() -> llvm::Optional<Node>;
@@ -228,6 +228,14 @@ class ParseTree::Parser {
// Parses a statement.
auto ParseStatement() -> llvm::Optional<Node>;
enum class PatternKind {
Parameter,
Variable,
};
// Parses a pattern.
auto ParsePattern(PatternKind kind) -> llvm::Optional<Node>;
ParseTree& tree;
TokenizedBuffer& tokens;
TokenDiagnosticEmitter& emitter;
toolchain/parser/precedence.cpp
+16
View File
@@ -32,6 +32,8 @@ enum PrecedenceLevel : int8_t {
// Assignment.
SimpleAssignment,
CompoundAssignment,
// Sentinel representing a type context.
Type,
// Sentinel representing a context in which any operator can appear.
Lowest,
};
@@ -58,6 +60,16 @@ struct OperatorPriorityTable {
{SimpleAssignment, CompoundAssignment, LogicalAnd, LogicalOr},
{Lowest});
// FIXME: Decide upon a precedence level to use for types. It's important
// that this is no higher than simple assignment, otherwise
// var x: T = y;
// would be parsed as
// var x: (T = y);
// For now, we have no type operators and no operator overloading, so we
// only parse primary expressions in types.
MarkHigherThan({Highest}, {Type});
MarkHigherThan({Type}, {Lowest});
// Compute the transitive closure of the above relationships: if we parse
// `a $ b @ c` as `(a $ b) @ c` and parse `b @ c % d` as `(b @ c) % d`,
// then we will parse `a $ b @ c % d` as `((a $ b) @ c) % d` and should
@@ -181,6 +193,10 @@ auto PrecedenceGroup::ForTopLevelExpression() -> PrecedenceGroup {
return PrecedenceGroup(Lowest);
}
auto PrecedenceGroup::ForType() -> PrecedenceGroup {
return PrecedenceGroup(Type);
}
auto PrecedenceGroup::ForLeading(TokenKind kind)
-> llvm::Optional<PrecedenceGroup> {
switch (kind) {
toolchain/parser/precedence.h
+4
View File
@@ -45,6 +45,10 @@ class PrecedenceGroup {
// operators should have higher precedence than this.
static auto ForTopLevelExpression() -> PrecedenceGroup;
// Get the precedence level at which to parse a type expression. All type
// operators should have higher precedence than this.
static auto ForType() -> PrecedenceGroup;
// Look up the operator information of the given prefix operator token, or
// return llvm::None if the given token is not a prefix operator.
static auto ForLeading(TokenKind kind) -> llvm::Optional<PrecedenceGroup>;