carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 11:21:05 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

[toolchain] Implement lexing and parsing support for #543. (#693)

Browse Source 
Lex [iuf][1-9][0-9]* as a new kind of "sized type literal" token. When
parsing that token, form a literal expression.

Co-authored-by: Chandler Carruth <chandlerc@gmail.com>
This commit is contained in:
Richard Smith
2021-08-02 15:37:43 -07:00
committed by GitHub
co-authored by Chandler Carruth
parent 9829f188b7
commit a83c22288f
8 changed files with 257 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
toolchain/parser/parse_tree_test.cpp
+63 -64
View File
@@ -151,27 +151,27 @@ TEST_F(ParseTreeTest,
}
TEST_F(ParseTreeTest, FunctionDeclarationWithParameterList) {
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo(bar: Int, baz: Int);");
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo(bar: i32, baz: i32);");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_FALSE(tree.HasErrors());
EXPECT_THAT(tree,
MatchParseTreeNodes(
{MatchFunctionDeclaration(
MatchDeclaredName("foo"),
MatchParameterList(
MatchPatternBinding(MatchDeclaredName("bar"), ":",
MatchNameReference("Int")),
MatchParameterListComma(),
MatchPatternBinding(MatchDeclaredName("baz"), ":",
MatchNameReference("Int")),
MatchParameterListEnd()),
MatchDeclarationEnd()),
MatchFileEnd()}));
EXPECT_THAT(
tree,
MatchParseTreeNodes(
{MatchFunctionDeclaration(
MatchDeclaredName("foo"),
MatchParameterList(MatchPatternBinding(MatchDeclaredName("bar"),
":", MatchLiteral("i32")),
MatchParameterListComma(),
MatchPatternBinding(MatchDeclaredName("baz"),
":", MatchLiteral("i32")),
MatchParameterListEnd()),
MatchDeclarationEnd()),
MatchFileEnd()}));
}
TEST_F(ParseTreeTest, FunctionDefinitionWithParameterList) {
TokenizedBuffer tokens = GetTokenizedBuffer(
"fn foo(bar: Int, baz: Int) {\n"
"fn foo(bar: i64, baz: i64) {\n"
" foo(baz, bar + baz);\n"
"}");
ParseTree tree = ParseTree::Parse(tokens, consumer);
@@ -181,13 +181,12 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithParameterList) {
MatchParseTreeNodes(
{MatchFunctionDeclaration(
MatchDeclaredName("foo"),
MatchParameterList(
MatchPatternBinding(MatchDeclaredName("bar"), ":",
MatchNameReference("Int")),
MatchParameterListComma(),
MatchPatternBinding(MatchDeclaredName("baz"), ":",
MatchNameReference("Int")),
MatchParameterListEnd()),
MatchParameterList(MatchPatternBinding(MatchDeclaredName("bar"),
":", MatchLiteral("i64")),
MatchParameterListComma(),
MatchPatternBinding(MatchDeclaredName("baz"),
":", MatchLiteral("i64")),
MatchParameterListEnd()),
MatchCodeBlock(
MatchExpressionStatement(MatchCallExpression(
MatchNameReference("foo"), MatchNameReference("baz"),
@@ -200,21 +199,21 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithParameterList) {
}
TEST_F(ParseTreeTest, FunctionDeclarationWithReturnType) {
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo() -> Int;");
TokenizedBuffer tokens = GetTokenizedBuffer("fn foo() -> u32;");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_FALSE(tree.HasErrors());
EXPECT_THAT(
tree,
MatchParseTreeNodes(
{MatchFunctionDeclaration(MatchDeclaredName("foo"), MatchParameters(),
MatchReturnType(MatchNameReference("Int")),
MatchReturnType(MatchLiteral("u32")),
MatchDeclarationEnd()),
MatchFileEnd()}));
}
TEST_F(ParseTreeTest, FunctionDefinitionWithReturnType) {
TokenizedBuffer tokens = GetTokenizedBuffer(
"fn foo() -> Int {\n"
"fn foo() -> f64 {\n"
" return 42;\n"
"}");
ParseTree tree = ParseTree::Parse(tokens, consumer);
@@ -223,7 +222,7 @@ TEST_F(ParseTreeTest, FunctionDefinitionWithReturnType) {
MatchParseTreeNodes(
{MatchFunctionDeclaration(
MatchDeclaredName("foo"), MatchParameters(),
MatchReturnType(MatchNameReference("Int")),
MatchReturnType(MatchLiteral("f64")),
MatchCodeBlock(MatchReturnStatement(MatchLiteral("42"),
MatchStatementEnd()),
MatchCodeBlockEnd())),
@@ -490,14 +489,14 @@ TEST_F(ParseTreeTest, Operators) {
TEST_F(ParseTreeTest, OperatorFixity) {
TokenizedBuffer tokens = GetTokenizedBuffer(
"fn F(p: Int*, n: Int) {\n"
" var q: Int* = p;\n"
" var t: Type = Int*;\n"
"fn F(p: i32*, n: i32) {\n"
" var q: i32* = p;\n"
" var t: Type = i32*;\n"
" t = t**;\n"
" n = n * n;\n"
" n = n * *p;\n"
" n = n*n;\n"
" G(Int*, n * n);\n"
" G(i32*, n * n);\n"
"}");
ParseTree tree = ParseTree::Parse(tokens, consumer);
EXPECT_FALSE(tree.HasErrors());
@@ -510,22 +509,22 @@ TEST_F(ParseTreeTest, OperatorFixity) {
MatchParameters(
MatchPatternBinding(
MatchDeclaredName("p"),
MatchPostfixOperator(MatchNameReference("Int"), "*")),
MatchPostfixOperator(MatchLiteral("i32"), "*")),
MatchParameterListComma(),
MatchPatternBinding(MatchDeclaredName("n"),
MatchNameReference("Int"))),
MatchLiteral("i32"))),
MatchCodeBlock(
MatchVariableDeclaration(
MatchPatternBinding(MatchDeclaredName("q"),
MatchPostfixOperator(
MatchNameReference("Int"), "*")),
MatchPatternBinding(
MatchDeclaredName("q"),
MatchPostfixOperator(MatchLiteral("i32"), "*")),
MatchVariableInitializer(MatchNameReference("p")),
MatchDeclarationEnd()),
MatchVariableDeclaration(
MatchPatternBinding(MatchDeclaredName("t"),
MatchNameReference("Type")),
MatchVariableInitializer(MatchPostfixOperator(
MatchNameReference("Int"), "*")),
MatchVariableInitializer(
MatchPostfixOperator(MatchLiteral("i32"), "*")),
MatchDeclarationEnd()),
MatchExpressionStatement(MatchInfixOperator(
MatchNameReference("t"), "=",
@@ -547,7 +546,7 @@ TEST_F(ParseTreeTest, OperatorFixity) {
MatchNameReference("n")))),
MatchExpressionStatement(MatchCallExpression(
MatchNameReference("G"),
MatchPostfixOperator(MatchNameReference("Int"), "*"),
MatchPostfixOperator(MatchLiteral("i32"), "*"),
MatchCallExpressionComma(),
MatchInfixOperator(MatchNameReference("n"), "*",
MatchNameReference("n")),
@@ -565,31 +564,31 @@ TEST_F(ParseTreeTest, OperatorWhitespaceErrors) {
const char* input;
Kind kind;
} testcases[] = {
{"var v: Type = Int*;", Valid},
{"var v: Type = Int *;", Recovered},
{"var v: Type = Int* ;", Valid},
{"var v: Type = Int * ;", Recovered},
{"var n: Int = n * n;", Valid},
{"var n: Int = n*n;", Valid},
{"var n: Int = (n)*3;", Valid},
{"var n: Int = 3*(n);", Valid},
{"var n: Int = n *n;", Recovered},
{"var v: Type = i8*;", Valid},
{"var v: Type = i8 *;", Recovered},
{"var v: Type = i8* ;", Valid},
{"var v: Type = i8 * ;", Recovered},
{"var n: i8 = n * n;", Valid},
{"var n: i8 = n*n;", Valid},
{"var n: i8 = (n)*3;", Valid},
{"var n: i8 = 3*(n);", Valid},
{"var n: i8 = n *n;", Recovered},
// FIXME: We could figure out that this first Failed example is infix
// with one-token lookahead.
{"var n: Int = n* n;", Failed},
{"var n: Int = n* -n;", Failed},
{"var n: Int = n* *p;", Failed},
{"var n: i8 = n* n;", Failed},
{"var n: i8 = n* -n;", Failed},
{"var n: i8 = n* *p;", Failed},
// FIXME: We try to form (n*)*p and reject due to missing parentheses
// before we notice the missing whitespace around the second `*`.
// It'd be better to (somehow) form n*(*p) and reject due to the missing
// whitespace around the first `*`.
{"var n: Int = n**p;", Failed},
{"var n: Int = -n;", Valid},
{"var n: Int = - n;", Recovered},
{"var n: Int =-n;", Valid},
{"var n: Int =- n;", Recovered},
{"var n: Int = F(Int *);", Recovered},
{"var n: Int = F(Int *, 0);", Recovered},
{"var n: i8 = n**p;", Failed},
{"var n: i8 = -n;", Valid},
{"var n: i8 = - n;", Recovered},
{"var n: i8 =-n;", Valid},
{"var n: i8 =- n;", Recovered},
{"var n: i8 = F(i8 *);", Recovered},
{"var n: i8 = F(i8 *, 0);", Recovered},
};
for (auto [input, kind] : testcases) {
@@ -603,8 +602,8 @@ TEST_F(ParseTreeTest, OperatorWhitespaceErrors) {
TEST_F(ParseTreeTest, VariableDeclarations) {
TokenizedBuffer tokens = GetTokenizedBuffer(
"var v: Int = 0;\n"
"var w: Int;\n"
"var v: i32 = 0;\n"
"var w: i32;\n"
"fn F() {\n"
" var s: String = \"hello\";\n"
"}");
@@ -615,12 +614,12 @@ TEST_F(ParseTreeTest, VariableDeclarations) {
MatchParseTreeNodes(
{MatchVariableDeclaration(
MatchPatternBinding(MatchDeclaredName("v"), ":",
MatchNameReference("Int")),
MatchLiteral("i32")),
MatchVariableInitializer(MatchLiteral("0")),
MatchDeclarationEnd()),
MatchVariableDeclaration(
MatchPatternBinding(MatchDeclaredName("w"), ":",
MatchNameReference("Int")),
MatchLiteral("i32")),
MatchDeclarationEnd()),
MatchFunctionWithBody(MatchVariableDeclaration(
MatchPatternBinding(MatchDeclaredName("s"), ":",
@@ -914,7 +913,7 @@ TEST_F(ParseTreeTest, Return) {
" return;\n"
" }\n"
"}\n"
"fn G(x: Int) -> Int {\n"
"fn G(x: Foo) -> Foo {\n"
" return x;\n"
"}");
ParseTree tree = ParseTree::Parse(tokens, consumer);
@@ -930,8 +929,8 @@ TEST_F(ParseTreeTest, Return) {
MatchFunctionDeclaration(
MatchDeclaredName(),
MatchParameters(MatchPatternBinding(MatchDeclaredName("x"), ":",
MatchNameReference("Int"))),
MatchReturnType(MatchNameReference("Int")),
MatchNameReference("Foo"))),
MatchReturnType(MatchNameReference("Foo")),
MatchCodeBlock(MatchReturnStatement(MatchNameReference("x"),
MatchStatementEnd()),
MatchCodeBlockEnd())),