From 5ae6d2241543fe07f81d90a9126db0f3750b7556 Mon Sep 17 00:00:00 2001 From: pk19604014 <95385881+pk19604014@users.noreply.github.com> Date: Fri, 25 Mar 2022 15:29:31 -0400 Subject: [PATCH] Carbon fuzzing 1/3: added a proto for representing Carbon AST + ast_to_proto library (#1128) * initial fuzzer proto * visibility change * use newer protocol buffer version which has the defs.bzl bug fixed * Adjusted fix_cc_deps to work with protobuf external repo * explicitly load rules_cc to avoid a frozenset bug in the version loaded by protobuf * use explit deps, use llvm's zlib * restored cxx settings * deps change * Cleaned up WORKSPACE and changed the test to read carbon sources from testdata * updated comment * adapted to new ErrorOr return value * proto buffer 3.19.2 -> 3.19.4 * changed comment * Apply suggestions from code review Co-authored-by: Jon Meow * Apply suggestions from code review Co-authored-by: Jon Meow * Update executable_semantics/fuzzing/BUILD Co-authored-by: Jon Meow * addressed review comments * updated Unimplemented error message * Addressed review comments * more review comments * switched to loading protobuf via rules_proto() * Ignore protobuf headers in fix_cc_deps.py until the script supports alias rules * renamed repeated proto fields to be plural * added @zlib to check_non_test_cc_deps * Update common/fuzzing/BUILD Co-authored-by: Jon Meow * review comments * set is_omitted_expression for return value Co-authored-by: Jon Meow --- .pre-commit-config.yaml | 2 +- WORKSPACE | 33 +- bazel/check_deps/check_non_test_cc_deps.py | 2 +- common/fuzzing/BUILD | 18 + common/fuzzing/carbon.proto | 341 +++++++++++ executable_semantics/fuzzing/BUILD | 38 ++ executable_semantics/fuzzing/ast_to_proto.cpp | 541 ++++++++++++++++++ executable_semantics/fuzzing/ast_to_proto.h | 18 + .../fuzzing/ast_to_proto_test.cpp | 130 +++++ .../interpreter/resolve_names.cpp | 3 +- executable_semantics/syntax/BUILD | 5 +- executable_semantics/testdata/BUILD | 6 + .../fail_unimplemented_example.carbon | 16 + scripts/fix_cc_deps.py | 28 +- 14 files changed, 1171 insertions(+), 10 deletions(-) create mode 100644 common/fuzzing/BUILD create mode 100644 common/fuzzing/carbon.proto create mode 100644 executable_semantics/fuzzing/BUILD create mode 100644 executable_semantics/fuzzing/ast_to_proto.cpp create mode 100644 executable_semantics/fuzzing/ast_to_proto.h create mode 100644 executable_semantics/fuzzing/ast_to_proto_test.cpp create mode 100644 executable_semantics/testdata/basic_syntax/fail_unimplemented_example.carbon diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c6f679d1cacc..9366b47fb13b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -97,7 +97,7 @@ repos: Exceptions. See /LICENSE for license information. SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - --custom_format - - '\.(carbon|ypp)$' + - '\.(carbon|proto|ypp)$' - '' - '// ' - '' diff --git a/WORKSPACE b/WORKSPACE index db81f8a4d013..c781ede63b82 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -143,6 +143,7 @@ load("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") llvm_configure( name = "llvm-project", + repo_mapping = {"@llvm_zlib": "@zlib"}, targets = [ "AArch64", "X86", @@ -157,7 +158,7 @@ llvm_terminfo_system(name = "llvm_terminfo") load("@llvm-raw//utils/bazel:zlib.bzl", "llvm_zlib_system") # We require successful detection and use of a system zlib library. -llvm_zlib_system(name = "llvm_zlib") +llvm_zlib_system(name = "zlib") ############################################################################### # Flex/Bison rules @@ -217,6 +218,36 @@ load("@rules_bison//bison:bison.bzl", "bison_register_toolchains") # fix them anyways. bison_register_toolchains(extra_copts = ["-w"]) +############################################################################### +# Protocol buffers - for structured fuzzer testing. +############################################################################### + +# TODO: `rules_proto` pulls in a version of `rules_cc` with a frozenset bug. +rules_cc_version = "0.0.1" + +http_archive( + name = "rules_cc", + sha256 = "4dccbfd22c0def164c8f47458bd50e0c7148f3d92002cdb459c2a96a68498241", + urls = ["https://github.com/bazelbuild/rules_cc/releases/download/%s/rules_cc-%s.tar.gz" % (rules_cc_version, rules_cc_version)], +) + +rules_proto_version = "4.0.0-3.19.2" + +http_archive( + name = "rules_proto", + sha256 = "c22cfcb3f22a0ae2e684801ea8dfed070ba5bed25e73f73580564f250475e72d", + strip_prefix = "rules_proto-%s" % rules_proto_version, + urls = [ + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/%s.tar.gz" % rules_proto_version, + ], +) + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + +rules_proto_dependencies() + +rules_proto_toolchains() + ############################################################################### # Example conversion repositories ############################################################################### diff --git a/bazel/check_deps/check_non_test_cc_deps.py b/bazel/check_deps/check_non_test_cc_deps.py index 7a0f26d6cc5a..29a1b51e72ed 100644 --- a/bazel/check_deps/check_non_test_cc_deps.py +++ b/bazel/check_deps/check_non_test_cc_deps.py @@ -56,7 +56,7 @@ for dep in deps: # The rest of LLVM, LLD, and Clang themselves are safe to depend on. continue - if repo in ("@llvm_terminfo", "@llvm_zlib"): + if repo in ("@llvm_terminfo", "@llvm_zlib", "@zlib"): # These are stubs wrapping system libraries for LLVM. They aren't # distributed and so should be fine. continue diff --git a/common/fuzzing/BUILD b/common/fuzzing/BUILD new file mode 100644 index 000000000000..492dca495c7e --- /dev/null +++ b/common/fuzzing/BUILD @@ -0,0 +1,18 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "carbon_proto", + srcs = ["carbon.proto"], +) + +cc_proto_library( + name = "carbon_cc_proto", + deps = [":carbon_proto"], +) diff --git a/common/fuzzing/carbon.proto b/common/fuzzing/carbon.proto new file mode 100644 index 000000000000..76437d8062d6 --- /dev/null +++ b/common/fuzzing/carbon.proto @@ -0,0 +1,341 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +syntax = "proto2"; + +package Carbon.Fuzzing; + +message LibraryName { + optional string package_name = 1; + optional string path = 2; +} + +// Expressions. + +message CallExpression { + optional Expression function = 1; + optional Expression argument = 2; +} + +message FunctionTypeLiteral { + optional Expression parameter = 1; + optional Expression return_type = 2; +} + +message FieldAccessExpression { + optional string field = 1; + optional Expression aggregate = 2; +} + +message IndexExpression { + optional Expression aggregate = 1; + optional Expression offset = 2; +} + +message PrimitiveOperatorExpression { + enum Operator { + UnknownOperator = 0; + Add = 1; + AddressOf = 2; + And = 3; + Deref = 4; + Eq = 5; + Mul = 6; + Neg = 7; + Not = 8; + Or = 9; + Sub = 10; + Ptr = 11; + } + optional Operator op = 1; + repeated Expression arguments = 2; +} + +message TupleLiteralExpression { + repeated Expression fields = 1; +} + +message FieldInitializer { + optional string name = 1; + optional Expression expression = 2; +} + +message StructLiteralExpression { + repeated FieldInitializer fields = 1; +} + +message StructTypeLiteralExpression { + repeated FieldInitializer fields = 1; +} + +message IdentifierExpression { + optional string name = 1; +} + +message IntrinsicExpression { + enum Intrinsic { + UnknownIntrinsic = 0; + Print = 1; + } + optional Intrinsic intrinsic = 1; + optional TupleLiteralExpression argument = 2; +} + +message IfExpression { + optional Expression condition = 1; + optional Expression then_expression = 2; + optional Expression else_expression = 3; +} + +message BoolTypeLiteral {} + +message BoolLiteral { + optional bool value = 1; +} + +message IntTypeLiteral {} +message ContinuationTypeLiteral {} + +message IntLiteral { + optional int64 value = 1; +} + +message StringLiteral { + optional string value = 1; +} + +message StringTypeLiteral {} +message TypeTypeLiteral {} + +message UnimplementedExpression {} + +message Expression { + oneof kind { + CallExpression call = 1; + FunctionTypeLiteral function_type = 2; + FieldAccessExpression field_access = 3; + IndexExpression index = 4; + PrimitiveOperatorExpression primitive_operator = 5; + TupleLiteralExpression tuple_literal = 6; + StructLiteralExpression struct_literal = 7; + StructTypeLiteralExpression struct_type_literal = 8; + IdentifierExpression identifier = 9; + IntrinsicExpression intrinsic = 10; + IfExpression if_expression = 11; + BoolTypeLiteral bool_type_literal = 12; + BoolLiteral bool_literal = 13; + IntTypeLiteral int_type_literal = 14; + ContinuationTypeLiteral continuation_type_literal = 15; + IntLiteral int_literal = 16; + StringLiteral string_literal = 17; + StringTypeLiteral string_type_literal = 18; + TypeTypeLiteral type_type_literal = 19; + UnimplementedExpression unimplemented_expression = 20; + } +} + +// Patterns. + +message BindingPattern { + optional string name = 1; + optional Pattern type = 2; +} + +message TuplePattern { + repeated Pattern fields = 1; +} + +message AlternativePattern { + optional Expression choice_type = 1; + optional string alternative_name = 2; + optional TuplePattern arguments = 3; +} + +message ExpressionPattern { + optional Expression expression = 1; +} + +message AutoPattern {} + +message VarPattern { + optional Pattern pattern = 1; +} + +message Pattern { + oneof kind { + BindingPattern binding_pattern = 1; + TuplePattern tuple_pattern = 2; + AlternativePattern alternative_pattern = 3; + ExpressionPattern expression_pattern = 4; + AutoPattern auto_pattern = 5; + VarPattern var_pattern = 6; + } +} + +// Statements. + +message ExpressionStatement { + optional Expression expression = 1; +} + +message AssignStatement { + optional Expression lhs = 1; + optional Expression rhs = 2; +} + +message VariableDefinitionStatement { + optional Pattern pattern = 1; + optional Expression init = 2; +} + +message IfStatement { + optional Expression condition = 1; + optional BlockStatement then_block = 2; + optional BlockStatement else_block = 3; +} + +message ReturnStatement { + optional Expression expression = 1; // Can be omitted. + optional bool is_omitted_expression = 2; +} + +message BlockStatement { + repeated Statement statements = 1; +} + +message WhileStatement { + optional Expression condition = 1; + optional BlockStatement body = 2; +} + +message MatchClause { + optional Pattern pattern = 1; + optional Statement statement = 2; +} + +message MatchStatement { + optional Expression expression = 1; + repeated MatchClause clauses = 2; +} + +message ContinuationStatement { + optional string name = 1; + optional BlockStatement body = 2; +} + +message RunStatement { + optional Expression argument = 1; +} + +message AwaitStatement {} + +message BreakStatement {} +message ContinueStatement {} + +message Statement { + oneof kind { + ExpressionStatement expression_statement = 1; + AssignStatement assign = 2; + VariableDefinitionStatement variable_definition = 3; + IfStatement if_statement = 4; + ReturnStatement return_statement = 5; + BlockStatement block = 6; + WhileStatement while_statement = 7; + MatchStatement match = 8; + ContinuationStatement continuation = 9; + RunStatement run = 10; + AwaitStatement await = 11; + BreakStatement break_statement = 12; + ContinueStatement continue_statement = 13; + } +} + +// Declarations. + +message ReturnTerm { + enum ReturnKind { + UnknownReturnKind = 0; + Omitted = 1; + Auto = 2; + Expression = 3; + } + optional ReturnKind kind = 1; + optional Expression type = 2; +} + +message GenericBinding { + optional string name = 1; + optional Expression type = 2; +} + +message FunctionDeclaration { + optional string name = 1; + repeated GenericBinding deduced_parameters = 2; + optional BindingPattern me_pattern = 3; + optional TuplePattern param_pattern = 4; + optional ReturnTerm return_term = 5; + optional BlockStatement body = 6; +} + +message ClassDeclaration { + optional string name = 1; + repeated Declaration members = 2; +} + +message AlternativeSignature { + optional string name = 1; + optional Expression signature = 2; +} + +message ChoiceDeclaration { + optional string name = 1; + repeated AlternativeSignature alternatives = 2; +} + +message VariableDeclaration { + optional BindingPattern binding = 1; + optional Expression initializer = 2; +} + +message InterfaceDeclaration { + optional string name = 1; + repeated Declaration members = 2; + optional GenericBinding self = 3; +} + +message ImplDeclaration { + enum ImplKind { + UnknownImplKind = 0; + InternalImpl = 1; + ExternalImpl = 2; + } + + optional ImplKind kind = 1; + optional Expression impl_type = 2; + optional Expression interface = 3; + repeated Declaration members = 4; +} + +message Declaration { + oneof kind { + FunctionDeclaration function = 1; + ClassDeclaration class_declaration = 2; + ChoiceDeclaration choice = 3; + VariableDeclaration variable = 4; + InterfaceDeclaration interface = 5; + ImplDeclaration impl = 6; + } +} + +message CompilationUnit { + optional LibraryName package_statement = 1; + optional bool is_api = 2; + repeated Declaration declarations = 3; + // TODO Add support for imports if they are useful in fuzzing. +} + +// Top-level fuzzer input. +message Carbon { + optional CompilationUnit compilation_unit = 1; +} diff --git a/executable_semantics/fuzzing/BUILD b/executable_semantics/fuzzing/BUILD new file mode 100644 index 000000000000..58b482154805 --- /dev/null +++ b/executable_semantics/fuzzing/BUILD @@ -0,0 +1,38 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cc_library( + name = "ast_to_proto_lib", + srcs = ["ast_to_proto.cpp"], + hdrs = ["ast_to_proto.h"], + deps = [ + "//common/fuzzing:carbon_cc_proto", + "//executable_semantics/ast", + "//executable_semantics/ast:declaration", + "//executable_semantics/ast:expression", + "//executable_semantics/ast:generic_binding", + "@llvm-project//llvm:Support", + ], +) + +cc_test( + name = "ast_to_proto_test", + srcs = ["ast_to_proto_test.cpp"], + args = [ + "$(location //executable_semantics:data/prelude.carbon)", + "$(locations //executable_semantics/testdata:carbon_files)", + ], + data = [ + "//executable_semantics:data/prelude.carbon", + "//executable_semantics/testdata:carbon_files", + ], + deps = [ + ":ast_to_proto_lib", + "//common/fuzzing:carbon_cc_proto", + "//executable_semantics/syntax", + "@com_google_googletest//:gtest", + "@com_google_protobuf//:protobuf_headers", + "@llvm-project//llvm:Support", + ], +) diff --git a/executable_semantics/fuzzing/ast_to_proto.cpp b/executable_semantics/fuzzing/ast_to_proto.cpp new file mode 100644 index 000000000000..2eda74df0d99 --- /dev/null +++ b/executable_semantics/fuzzing/ast_to_proto.cpp @@ -0,0 +1,541 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "executable_semantics/fuzzing/ast_to_proto.h" + +#include + +#include "executable_semantics/ast/declaration.h" +#include "executable_semantics/ast/expression.h" +#include "executable_semantics/ast/generic_binding.h" +#include "llvm/Support/Casting.h" + +namespace Carbon { + +using ::llvm::cast; + +static auto ExpressionToProto(const Expression& expression) + -> Fuzzing::Expression; +static auto PatternToProto(const Pattern& pattern) -> Fuzzing::Pattern; +static auto StatementToProto(const Statement& statement) -> Fuzzing::Statement; +static auto DeclarationToProto(const Declaration& declaration) + -> Fuzzing::Declaration; + +static auto LibraryNameToProto(const LibraryName& library_name) + -> Fuzzing::LibraryName { + Fuzzing::LibraryName library_name_proto; + library_name_proto.set_package_name(library_name.package); + if (!library_name.path.empty()) { + library_name_proto.set_path(library_name.path); + } + return library_name_proto; +} + +static auto OperatorToProtoEnum(const Operator op) + -> Fuzzing::PrimitiveOperatorExpression::Operator { + switch (op) { + case Operator::AddressOf: + return Fuzzing::PrimitiveOperatorExpression::AddressOf; + case Operator::Deref: + return Fuzzing::PrimitiveOperatorExpression::Deref; + case Operator::Neg: + return Fuzzing::PrimitiveOperatorExpression::Neg; + case Operator::Not: + return Fuzzing::PrimitiveOperatorExpression::Not; + case Operator::Ptr: + return Fuzzing::PrimitiveOperatorExpression::Ptr; + case Operator::Add: + return Fuzzing::PrimitiveOperatorExpression::Add; + case Operator::And: + return Fuzzing::PrimitiveOperatorExpression::And; + case Operator::Eq: + return Fuzzing::PrimitiveOperatorExpression::Eq; + case Operator::Mul: + return Fuzzing::PrimitiveOperatorExpression::Mul; + case Operator::Or: + return Fuzzing::PrimitiveOperatorExpression::Or; + case Operator::Sub: + return Fuzzing::PrimitiveOperatorExpression::Sub; + } +} + +static auto FieldInitializerToProto(const FieldInitializer& field) + -> Fuzzing::FieldInitializer { + Fuzzing::FieldInitializer field_proto; + field_proto.set_name(field.name()); + *field_proto.mutable_expression() = ExpressionToProto(field.expression()); + return field_proto; +} + +static auto TupleLiteralExpressionToProto(const TupleLiteral& tuple_literal) + -> Fuzzing::TupleLiteralExpression { + Fuzzing::TupleLiteralExpression tuple_literal_proto; + for (Nonnull field : tuple_literal.fields()) { + *tuple_literal_proto.add_fields() = ExpressionToProto(*field); + } + return tuple_literal_proto; +} + +static auto ExpressionToProto(const Expression& expression) + -> Fuzzing::Expression { + Fuzzing::Expression expression_proto; + switch (expression.kind()) { + case ExpressionKind::CallExpression: { + const auto& call = cast(expression); + auto* call_proto = expression_proto.mutable_call(); + *call_proto->mutable_function() = ExpressionToProto(call.function()); + *call_proto->mutable_argument() = ExpressionToProto(call.argument()); + break; + } + + case ExpressionKind::FunctionTypeLiteral: { + const auto& fun_type = cast(expression); + auto* fun_type_proto = expression_proto.mutable_function_type(); + *fun_type_proto->mutable_parameter() = + ExpressionToProto(fun_type.parameter()); + *fun_type_proto->mutable_return_type() = + ExpressionToProto(fun_type.return_type()); + break; + } + + case ExpressionKind::FieldAccessExpression: { + const auto& field_access = cast(expression); + auto* field_access_proto = expression_proto.mutable_field_access(); + field_access_proto->set_field(field_access.field()); + *field_access_proto->mutable_aggregate() = + ExpressionToProto(field_access.aggregate()); + break; + } + + case ExpressionKind::IndexExpression: { + const auto& index = cast(expression); + auto* index_proto = expression_proto.mutable_index(); + *index_proto->mutable_aggregate() = ExpressionToProto(index.aggregate()); + *index_proto->mutable_offset() = ExpressionToProto(index.offset()); + break; + } + + case ExpressionKind::PrimitiveOperatorExpression: { + const auto& primitive_operator = + cast(expression); + auto* operator_proto = expression_proto.mutable_primitive_operator(); + operator_proto->set_op(OperatorToProtoEnum(primitive_operator.op())); + for (Nonnull arg : primitive_operator.arguments()) { + *operator_proto->add_arguments() = ExpressionToProto(*arg); + } + break; + } + + case ExpressionKind::TupleLiteral: + *expression_proto.mutable_tuple_literal() = + TupleLiteralExpressionToProto(cast(expression)); + break; + + case ExpressionKind::StructLiteral: { + const auto& struct_literal = cast(expression); + auto* struct_literal_proto = expression_proto.mutable_struct_literal(); + for (const FieldInitializer& field : struct_literal.fields()) { + *struct_literal_proto->add_fields() = FieldInitializerToProto(field); + } + break; + } + + case ExpressionKind::StructTypeLiteral: { + const auto& struct_type_literal = cast(expression); + auto* struct_type_literal_proto = + expression_proto.mutable_struct_type_literal(); + for (const FieldInitializer& field : struct_type_literal.fields()) { + *struct_type_literal_proto->add_fields() = + FieldInitializerToProto(field); + } + break; + } + + case ExpressionKind::IdentifierExpression: { + const auto& identifier = cast(expression); + auto* identifier_proto = expression_proto.mutable_identifier(); + identifier_proto->set_name(identifier.name()); + break; + } + + case ExpressionKind::IntrinsicExpression: { + const auto& intrinsic = cast(expression); + auto* intrinsic_proto = expression_proto.mutable_intrinsic(); + switch (intrinsic.intrinsic()) { + case IntrinsicExpression::Intrinsic::Print: + intrinsic_proto->set_intrinsic(Fuzzing::IntrinsicExpression::Print); + break; + } + *intrinsic_proto->mutable_argument() = + TupleLiteralExpressionToProto(intrinsic.args()); + break; + } + + case ExpressionKind::IfExpression: { + const auto& if_expression = cast(expression); + auto* if_proto = expression_proto.mutable_if_expression(); + if (if_expression.condition()) { + *if_proto->mutable_condition() = + ExpressionToProto(*if_expression.condition()); + } + if (if_expression.then_expression()) { + *if_proto->mutable_then_expression() = + ExpressionToProto(*if_expression.then_expression()); + } + if (if_expression.else_expression()) { + *if_proto->mutable_else_expression() = + ExpressionToProto(*if_expression.else_expression()); + } + break; + } + + case ExpressionKind::BoolTypeLiteral: + expression_proto.mutable_bool_type_literal(); + break; + + case ExpressionKind::BoolLiteral: + expression_proto.mutable_bool_literal()->set_value( + cast(expression).value()); + break; + + case ExpressionKind::IntTypeLiteral: + expression_proto.mutable_int_type_literal(); + break; + + case ExpressionKind::IntLiteral: + expression_proto.mutable_int_literal()->set_value( + cast(expression).value()); + break; + + case ExpressionKind::StringLiteral: + expression_proto.mutable_string_literal()->set_value( + cast(expression).value()); + break; + + case ExpressionKind::StringTypeLiteral: + expression_proto.mutable_string_type_literal(); + break; + + case ExpressionKind::ContinuationTypeLiteral: + expression_proto.mutable_continuation_type_literal(); + break; + + case ExpressionKind::TypeTypeLiteral: + expression_proto.mutable_type_type_literal(); + break; + + case ExpressionKind::UnimplementedExpression: + expression_proto.mutable_unimplemented_expression(); + break; + } + return expression_proto; +} + +static auto BindingPatternToProto(const BindingPattern& pattern) + -> Fuzzing::BindingPattern { + Fuzzing::BindingPattern pattern_proto; + pattern_proto.set_name(pattern.name()); + *pattern_proto.mutable_type() = PatternToProto(pattern.type()); + return pattern_proto; +} + +static auto TuplePatternToProto(const TuplePattern& tuple_pattern) + -> Fuzzing::TuplePattern { + Fuzzing::TuplePattern tuple_pattern_proto; + for (Nonnull field : tuple_pattern.fields()) { + *tuple_pattern_proto.add_fields() = PatternToProto(*field); + } + return tuple_pattern_proto; +} + +static auto PatternToProto(const Pattern& pattern) -> Fuzzing::Pattern { + Fuzzing::Pattern pattern_proto; + switch (pattern.kind()) { + case PatternKind::BindingPattern: { + const auto& binding = cast(pattern); + *pattern_proto.mutable_binding_pattern() = BindingPatternToProto(binding); + break; + } + case PatternKind::TuplePattern: + *pattern_proto.mutable_tuple_pattern() = + TuplePatternToProto(cast(pattern)); + break; + + case PatternKind::AlternativePattern: { + const auto& alternative = cast(pattern); + auto* alternative_proto = pattern_proto.mutable_alternative_pattern(); + alternative_proto->set_alternative_name(alternative.alternative_name()); + *alternative_proto->mutable_choice_type() = + ExpressionToProto(alternative.choice_type()); + *alternative_proto->mutable_arguments() = + TuplePatternToProto(alternative.arguments()); + break; + } + + case PatternKind::ExpressionPattern: + *pattern_proto.mutable_expression_pattern()->mutable_expression() = + ExpressionToProto(cast(pattern).expression()); + break; + + case PatternKind::AutoPattern: + pattern_proto.mutable_auto_pattern(); + break; + + case PatternKind::VarPattern: + *pattern_proto.mutable_var_pattern()->mutable_pattern() = + PatternToProto(cast(pattern).pattern()); + break; + } + return pattern_proto; +} + +static auto BlockStatementToProto(const Block& block) + -> Fuzzing::BlockStatement { + Fuzzing::BlockStatement block_proto; + for (Nonnull statement : block.statements()) { + *block_proto.add_statements() = StatementToProto(*statement); + } + return block_proto; +} + +static auto StatementToProto(const Statement& statement) -> Fuzzing::Statement { + Fuzzing::Statement statement_proto; + switch (statement.kind()) { + case StatementKind::ExpressionStatement: + *statement_proto.mutable_expression_statement()->mutable_expression() = + ExpressionToProto(cast(statement).expression()); + break; + + case StatementKind::Assign: { + const auto& assign = cast(statement); + auto* assign_proto = statement_proto.mutable_assign(); + *assign_proto->mutable_lhs() = ExpressionToProto(assign.lhs()); + *assign_proto->mutable_rhs() = ExpressionToProto(assign.rhs()); + break; + } + + case StatementKind::VariableDefinition: { + const auto& def = cast(statement); + auto* def_proto = statement_proto.mutable_variable_definition(); + *def_proto->mutable_pattern() = PatternToProto(def.pattern()); + *def_proto->mutable_init() = ExpressionToProto(def.init()); + break; + } + + case StatementKind::If: { + const auto& if_stmt = cast(statement); + auto* if_proto = statement_proto.mutable_if_statement(); + *if_proto->mutable_condition() = ExpressionToProto(if_stmt.condition()); + *if_proto->mutable_then_block() = + BlockStatementToProto(if_stmt.then_block()); + if (if_stmt.else_block().has_value()) { + *if_proto->mutable_else_block() = + BlockStatementToProto(**if_stmt.else_block()); + } + break; + } + + case StatementKind::Return: { + const auto& ret = cast(statement); + auto* ret_proto = statement_proto.mutable_return_statement(); + if (!ret.is_omitted_expression()) { + *ret_proto->mutable_expression() = ExpressionToProto(ret.expression()); + } else { + ret_proto->set_is_omitted_expression(true); + } + break; + } + + case StatementKind::Block: + *statement_proto.mutable_block() = + BlockStatementToProto(cast(statement)); + break; + + case StatementKind::While: { + const auto& while_stmt = cast(statement); + auto* while_proto = statement_proto.mutable_while_statement(); + *while_proto->mutable_condition() = + ExpressionToProto(while_stmt.condition()); + *while_proto->mutable_body() = BlockStatementToProto(while_stmt.body()); + break; + } + + case StatementKind::Match: { + const auto& match = cast(statement); + auto* match_proto = statement_proto.mutable_match(); + *match_proto->mutable_expression() = + ExpressionToProto(match.expression()); + for (const Match::Clause& clause : match.clauses()) { + auto* clause_proto = match_proto->add_clauses(); + *clause_proto->mutable_pattern() = PatternToProto(clause.pattern()); + *clause_proto->mutable_statement() = + StatementToProto(clause.statement()); + } + break; + } + + case StatementKind::Continuation: { + const auto& continuation = cast(statement); + auto* continuation_proto = statement_proto.mutable_continuation(); + continuation_proto->set_name(continuation.name()); + *continuation_proto->mutable_body() = + BlockStatementToProto(continuation.body()); + break; + } + + case StatementKind::Run: + *statement_proto.mutable_run()->mutable_argument() = + ExpressionToProto(cast(statement).argument()); + break; + + case StatementKind::Await: + // Initializes with the default value; there's nothing to set. + statement_proto.mutable_await(); + break; + + case StatementKind::Break: + // Initializes with the default value; there's nothing to set. + statement_proto.mutable_break_statement(); + break; + + case StatementKind::Continue: + // Initializes with the default value; there's nothing to set. + statement_proto.mutable_continue_statement(); + break; + } + return statement_proto; +} + +static auto ReturnTermToProto(const ReturnTerm& return_term) + -> Fuzzing::ReturnTerm { + Fuzzing::ReturnTerm return_term_proto; + if (return_term.is_omitted()) { + return_term_proto.set_kind(Fuzzing::ReturnTerm::Omitted); + } else if (return_term.is_auto()) { + return_term_proto.set_kind(Fuzzing::ReturnTerm::Auto); + } else { + return_term_proto.set_kind(Fuzzing::ReturnTerm::Expression); + *return_term_proto.mutable_type() = + ExpressionToProto(**return_term.type_expression()); + } + return return_term_proto; +} + +static auto GenericBindingToProto(const GenericBinding& binding) + -> Fuzzing::GenericBinding { + Fuzzing::GenericBinding binding_proto; + binding_proto.set_name(binding.name()); + *binding_proto.mutable_type() = ExpressionToProto(binding.type()); + return binding_proto; +} + +static auto DeclarationToProto(const Declaration& declaration) + -> Fuzzing::Declaration { + Fuzzing::Declaration declaration_proto; + switch (declaration.kind()) { + case DeclarationKind::FunctionDeclaration: { + const auto& function = cast(declaration); + auto* function_proto = declaration_proto.mutable_function(); + function_proto->set_name(function.name()); + for (Nonnull binding : + function.deduced_parameters()) { + *function_proto->add_deduced_parameters() = + GenericBindingToProto(*binding); + } + if (function.is_method()) { + *function_proto->mutable_me_pattern() = + BindingPatternToProto(function.me_pattern()); + } + *function_proto->mutable_param_pattern() = + TuplePatternToProto(function.param_pattern()); + *function_proto->mutable_return_term() = + ReturnTermToProto(function.return_term()); + if (function.body().has_value()) { + *function_proto->mutable_body() = + BlockStatementToProto(**function.body()); + } + break; + } + + case DeclarationKind::ClassDeclaration: { + const auto& class_decl = cast(declaration); + auto* class_proto = declaration_proto.mutable_class_declaration(); + class_proto->set_name(class_decl.name()); + for (Nonnull member : class_decl.members()) { + *class_proto->add_members() = DeclarationToProto(*member); + } + break; + } + + case DeclarationKind::ChoiceDeclaration: { + const auto& choice = cast(declaration); + auto* choice_proto = declaration_proto.mutable_choice(); + choice_proto->set_name(choice.name()); + for (Nonnull alternative : + choice.alternatives()) { + auto* alternative_proto = choice_proto->add_alternatives(); + alternative_proto->set_name(alternative->name()); + *alternative_proto->mutable_signature() = + ExpressionToProto(alternative->signature()); + } + break; + } + + case DeclarationKind::VariableDeclaration: { + const auto& var = cast(declaration); + auto* var_proto = declaration_proto.mutable_variable(); + *var_proto->mutable_binding() = BindingPatternToProto(var.binding()); + if (var.has_initializer()) { + *var_proto->mutable_initializer() = + ExpressionToProto(var.initializer()); + } + break; + } + + case DeclarationKind::InterfaceDeclaration: { + const auto& interface = cast(declaration); + auto* interface_proto = declaration_proto.mutable_interface(); + interface_proto->set_name(interface.name()); + for (const auto& member : interface.members()) { + *interface_proto->add_members() = DeclarationToProto(*member); + } + *interface_proto->mutable_self() = + GenericBindingToProto(*interface.self()); + break; + } + + case DeclarationKind::ImplDeclaration: { + const auto& impl = cast(declaration); + auto* impl_proto = declaration_proto.mutable_impl(); + switch (impl.kind()) { + case ImplKind::InternalImpl: + impl_proto->set_kind(Fuzzing::ImplDeclaration::InternalImpl); + break; + case ImplKind::ExternalImpl: + impl_proto->set_kind(Fuzzing::ImplDeclaration::ExternalImpl); + break; + } + *impl_proto->mutable_impl_type() = ExpressionToProto(*impl.impl_type()); + *impl_proto->mutable_interface() = ExpressionToProto(impl.interface()); + for (const auto& member : impl.members()) { + *impl_proto->add_members() = DeclarationToProto(*member); + } + break; + } + } + return declaration_proto; +} + +Fuzzing::CompilationUnit AstToProto(const AST& ast) { + Fuzzing::CompilationUnit compilation_unit; + *compilation_unit.mutable_package_statement() = + LibraryNameToProto(ast.package); + compilation_unit.set_is_api(ast.is_api); + for (const Declaration* declaration : ast.declarations) { + *compilation_unit.add_declarations() = DeclarationToProto(*declaration); + } + return compilation_unit; +} + +} // namespace Carbon diff --git a/executable_semantics/fuzzing/ast_to_proto.h b/executable_semantics/fuzzing/ast_to_proto.h new file mode 100644 index 000000000000..f07ada43444e --- /dev/null +++ b/executable_semantics/fuzzing/ast_to_proto.h @@ -0,0 +1,18 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef THIRD_PARTY_CARBON_LANG_EXECUTABLE_SEMANTICS_FUZZING_AST_TO_PROTO_H_ +#define THIRD_PARTY_CARBON_LANG_EXECUTABLE_SEMANTICS_FUZZING_AST_TO_PROTO_H_ + +#include "common/fuzzing/carbon.pb.h" +#include "executable_semantics/ast/ast.h" + +namespace Carbon { + +// Builds a protobuf representation of `ast`. +auto AstToProto(const AST& ast) -> Fuzzing::CompilationUnit; + +} // namespace Carbon + +#endif // THIRD_PARTY_CARBON_LANG_EXECUTABLE_SEMANTICS_FUZZING_AST_TO_PROTO_H_ diff --git a/executable_semantics/fuzzing/ast_to_proto_test.cpp b/executable_semantics/fuzzing/ast_to_proto_test.cpp new file mode 100644 index 000000000000..1471376b81ef --- /dev/null +++ b/executable_semantics/fuzzing/ast_to_proto_test.cpp @@ -0,0 +1,130 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "executable_semantics/fuzzing/ast_to_proto.h" + +#include +#include + +#include +#include +#include +#include + +#include "executable_semantics/syntax/parse.h" +#include "google/protobuf/descriptor.h" +#include "llvm/Support/Error.h" + +namespace Carbon::Testing { +namespace { + +using ::google::protobuf::Descriptor; +using ::google::protobuf::FieldDescriptor; +using ::google::protobuf::Message; +using ::google::protobuf::Reflection; + +static std::vector* carbon_files = nullptr; + +// Concatenates message and field names. +auto FieldName(const Descriptor& descriptor, const FieldDescriptor& field) + -> std::string { + return descriptor.full_name() + "." + field.name(); +} + +// Traverses the proto to find all unique messages and fields. +auto CollectAllFields(const Descriptor& descriptor, + std::set& all_messages, + std::set& all_fields) -> void { + all_messages.insert(descriptor.full_name()); + for (int i = 0; i < descriptor.field_count(); ++i) { + const FieldDescriptor* field = descriptor.field(i); + all_fields.insert(FieldName(descriptor, *field)); + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE && + all_messages.find(field->message_type()->full_name()) == + all_messages.end()) { + CollectAllFields(*field->message_type(), all_messages, all_fields); + } + } +} + +// Traverses an instance of the proto to find all used fields. +auto CollectUsedFields(const Message& message, + std::set& used_fields) -> void { + const Descriptor* descriptor = message.GetDescriptor(); + const Reflection* reflection = message.GetReflection(); + for (int i = 0; i < descriptor->field_count(); ++i) { + const FieldDescriptor* field = descriptor->field(i); + if (!field->is_repeated()) { + if (reflection->HasField(message, field)) { + used_fields.insert(FieldName(*descriptor, *field)); + } + } else { + if (reflection->FieldSize(message, field) > 0) { + used_fields.insert(FieldName(*descriptor, *field)); + } + } + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + if (!field->is_repeated()) { + if (reflection->HasField(message, field)) { + CollectUsedFields(reflection->GetMessage(message, field), + used_fields); + } + } else { + for (int i = 0; i < reflection->FieldSize(message, field); ++i) { + CollectUsedFields(reflection->GetRepeatedMessage(message, field, i), + used_fields); + } + } + } + } +} + +// Determines which fields in the proto have not been used at all. +auto GetUnusedFields(const Message& message) -> std::set { + std::set all_messages; + std::set all_fields; + CollectAllFields(*message.GetDescriptor(), all_messages, all_fields); + + std::set used_fields; + CollectUsedFields(message, used_fields); + + std::set unused_fields; + std::set_difference(all_fields.begin(), all_fields.end(), used_fields.begin(), + used_fields.end(), + std::inserter(unused_fields, unused_fields.begin())); + return unused_fields; +} + +// A 'smoke' test to check that each field present in `carbon.proto` is set at +// least once after converting all Carbon test sources to proto represention. +TEST(AstToProtoTest, SetsAllProtoFields) { + Carbon::Fuzzing::CompilationUnit merged_proto; + for (const llvm::StringRef f : *carbon_files) { + Carbon::Arena arena; + const ErrorOr ast = Carbon::Parse(&arena, f, /*trace=*/false); + if (ast.ok()) { + merged_proto.MergeFrom(AstToProto(*ast)); + } + } + + std::set unused_fields = GetUnusedFields(merged_proto); + EXPECT_EQ(unused_fields.size(), 0) + << "Unused fields" + << std::accumulate(unused_fields.begin(), unused_fields.end(), + std::string(), + [](const std::string& a, const std::string& b) { + return a + '\n' + b; + }); +} + +} // namespace +} // namespace Carbon::Testing + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + // gtest should remove flags, leaving just input files. + Carbon::Testing::carbon_files = + new std::vector(&argv[1], &argv[argc]); + return RUN_ALL_TESTS(); +} diff --git a/executable_semantics/interpreter/resolve_names.cpp b/executable_semantics/interpreter/resolve_names.cpp index cf9e8649fd9a..9687848a2f79 100644 --- a/executable_semantics/interpreter/resolve_names.cpp +++ b/executable_semantics/interpreter/resolve_names.cpp @@ -161,7 +161,8 @@ static auto ResolveNames(Expression& expression, case ExpressionKind::TypeTypeLiteral: break; case ExpressionKind::UnimplementedExpression: - FATAL() << "Unimplemented"; + return FATAL_COMPILATION_ERROR(expression.source_loc()) + << "Unimplemented"; } return Success(); } diff --git a/executable_semantics/syntax/BUILD b/executable_semantics/syntax/BUILD index 38c6770447b3..36e12fda8e52 100644 --- a/executable_semantics/syntax/BUILD +++ b/executable_semantics/syntax/BUILD @@ -4,7 +4,10 @@ load("@mypy_integration//:mypy.bzl", "mypy_test") -package(default_visibility = ["//executable_semantics:__pkg__"]) +package(default_visibility = [ + "//executable_semantics:__pkg__", + "//executable_semantics/fuzzing:__pkg__", +]) cc_library( name = "bison_wrap", diff --git a/executable_semantics/testdata/BUILD b/executable_semantics/testdata/BUILD index 1e6ea3028b3f..e35943a9ffce 100644 --- a/executable_semantics/testdata/BUILD +++ b/executable_semantics/testdata/BUILD @@ -13,3 +13,9 @@ glob_lit_tests( driver = "lit.cfg.py", test_file_exts = ["carbon"], ) + +filegroup( + name = "carbon_files", + srcs = glob(["**/*.carbon"]), + visibility = ["//visibility:public"], +) diff --git a/executable_semantics/testdata/basic_syntax/fail_unimplemented_example.carbon b/executable_semantics/testdata/basic_syntax/fail_unimplemented_example.carbon new file mode 100644 index 000000000000..691bedd06d2e --- /dev/null +++ b/executable_semantics/testdata/basic_syntax/fail_unimplemented_example.carbon @@ -0,0 +1,16 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RUN: %{not} %{executable_semantics} %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s +// RUN: %{not} %{executable_semantics} --trace %s 2>&1 | \ +// RUN: %{FileCheck} --match-full-lines --allow-unused-prefixes %s +// AUTOUPDATE: %{executable_semantics} %s +// CHECK: COMPILATION ERROR: {{.*}}/executable_semantics/testdata/basic_syntax/fail_unimplemented_example.carbon:15: Unimplemented + +package ExecutableSemanticsTest api; + +fn Main() -> i32 { + return 1 __unimplemented_example_infix 2; +} diff --git a/scripts/fix_cc_deps.py b/scripts/fix_cc_deps.py index e1c61a8e5f88..55dd641adfc4 100755 --- a/scripts/fix_cc_deps.py +++ b/scripts/fix_cc_deps.py @@ -25,9 +25,20 @@ import scripts_utils # type: ignore # Maps external repository names to a method translating bazel labels to file # paths for that repository. EXTERNAL_REPOS: Dict[str, Callable[[str], str]] = { - "@llvm-project": lambda x: re.sub("^(.*:(lib|include))/", "", x) + # @llvm-project//llvm:include/llvm/Support/Error.h -> + # llvm/Support/Error.h + "@llvm-project": lambda x: re.sub("^(.*:(lib|include))/", "", x), + # @com_google_protobuf//:src/google/protobuf/descriptor.h -> + # google/protobuf/descriptor.h + "@com_google_protobuf": lambda x: re.sub("^(.*:src)/", "", x), } +# TODO: proto rules are aspect-based and their generated files don't show up in +# `bazel query` output. +# Try using `bazel cquery --output=starlark` to print `target.files`. +# For protobuf, need to add support for `alias` rule kind. +IGNORE_HEADER_REGEX = re.compile("^(.*\\.pb\\.h)|(.*google/protobuf/.*)$") + class Rule(NamedTuple): # For cc_* rules: @@ -157,10 +168,17 @@ def get_missing_deps( if header in rule_files: continue if header not in header_to_rule_map: - exit( - f"Missing rule for #include '{header}' in " - f"'{source_file}'" - ) + if IGNORE_HEADER_REGEX.match(header): + print( + f"Ignored missing #include '{header}' in " + f"'{source_file}'" + ) + continue + else: + exit( + f"Missing rule for #include '{header}' in " + f"'{source_file}'" + ) dep_choices = header_to_rule_map[header] if not dep_choices.intersection(rule.deps): if len(dep_choices) > 1: