Carbon fuzzing 1/3: added a proto for representing Carbon AST + ast_to_proto library (#1128)

* initial fuzzer proto * visibility change * use newer protocol buffer version which has the defs.bzl bug fixed * Adjusted fix_cc_deps to work with protobuf external repo * explicitly load rules_cc to avoid a frozenset bug in the version loaded by protobuf * use explit deps, use llvm's zlib * restored cxx settings * deps change * Cleaned up WORKSPACE and changed the test to read carbon sources from testdata * updated comment * adapted to new ErrorOr return value * proto buffer 3.19.2 -> 3.19.4 * changed comment * Apply suggestions from code review Co-authored-by: Jon Meow <jperkins@google.com> * Apply suggestions from code review Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/BUILD Co-authored-by: Jon Meow <jperkins@google.com> * addressed review comments * updated Unimplemented error message * Addressed review comments * more review comments * switched to loading protobuf via rules_proto() * Ignore protobuf headers in fix_cc_deps.py until the script supports alias rules * renamed repeated proto fields to be plural * added @zlib to check_non_test_cc_deps * Update common/fuzzing/BUILD Co-authored-by: Jon Meow <jperkins@google.com> * review comments * set is_omitted_expression for return value Co-authored-by: Jon Meow <jperkins@google.com>
2026-10-05 19:51:09 +01:00 · 2022-03-25 15:29:31 -04:00
parent 770279e02d
commit 5ae6d22415
14 changed files with 1171 additions and 10 deletions
@@ -0,0 +1,130 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "executable_semantics/fuzzing/ast_to_proto.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <filesystem>
+#include <numeric>
+#include <set>
+#include <variant>
+
+#include "executable_semantics/syntax/parse.h"
+#include "google/protobuf/descriptor.h"
+#include "llvm/Support/Error.h"
+
+namespace Carbon::Testing {
+namespace {
+
+using ::google::protobuf::Descriptor;
+using ::google::protobuf::FieldDescriptor;
+using ::google::protobuf::Message;
+using ::google::protobuf::Reflection;
+
+static std::vector<llvm::StringRef>* carbon_files = nullptr;
+
+// Concatenates message and field names.
+auto FieldName(const Descriptor& descriptor, const FieldDescriptor& field)
+    -> std::string {
+  return descriptor.full_name() + "." + field.name();
+}
+
+// Traverses the proto to find all unique messages and fields.
+auto CollectAllFields(const Descriptor& descriptor,
+                      std::set<std::string>& all_messages,
+                      std::set<std::string>& all_fields) -> void {
+  all_messages.insert(descriptor.full_name());
+  for (int i = 0; i < descriptor.field_count(); ++i) {
+    const FieldDescriptor* field = descriptor.field(i);
+    all_fields.insert(FieldName(descriptor, *field));
+    if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE &&
+        all_messages.find(field->message_type()->full_name()) ==
+            all_messages.end()) {
+      CollectAllFields(*field->message_type(), all_messages, all_fields);
+    }
+  }
+}
+
+// Traverses an instance of the proto to find all used fields.
+auto CollectUsedFields(const Message& message,
+                       std::set<std::string>& used_fields) -> void {
+  const Descriptor* descriptor = message.GetDescriptor();
+  const Reflection* reflection = message.GetReflection();
+  for (int i = 0; i < descriptor->field_count(); ++i) {
+    const FieldDescriptor* field = descriptor->field(i);
+    if (!field->is_repeated()) {
+      if (reflection->HasField(message, field)) {
+        used_fields.insert(FieldName(*descriptor, *field));
+      }
+    } else {
+      if (reflection->FieldSize(message, field) > 0) {
+        used_fields.insert(FieldName(*descriptor, *field));
+      }
+    }
+    if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+      if (!field->is_repeated()) {
+        if (reflection->HasField(message, field)) {
+          CollectUsedFields(reflection->GetMessage(message, field),
+                            used_fields);
+        }
+      } else {
+        for (int i = 0; i < reflection->FieldSize(message, field); ++i) {
+          CollectUsedFields(reflection->GetRepeatedMessage(message, field, i),
+                            used_fields);
+        }
+      }
+    }
+  }
+}
+
+// Determines which fields in the proto have not been used at all.
+auto GetUnusedFields(const Message& message) -> std::set<std::string> {
+  std::set<std::string> all_messages;
+  std::set<std::string> all_fields;
+  CollectAllFields(*message.GetDescriptor(), all_messages, all_fields);
+
+  std::set<std::string> used_fields;
+  CollectUsedFields(message, used_fields);
+
+  std::set<std::string> unused_fields;
+  std::set_difference(all_fields.begin(), all_fields.end(), used_fields.begin(),
+                      used_fields.end(),
+                      std::inserter(unused_fields, unused_fields.begin()));
+  return unused_fields;
+}
+
+// A 'smoke' test to check that each field present in `carbon.proto` is set at
+// least once after converting all Carbon test sources to proto represention.
+TEST(AstToProtoTest, SetsAllProtoFields) {
+  Carbon::Fuzzing::CompilationUnit merged_proto;
+  for (const llvm::StringRef f : *carbon_files) {
+    Carbon::Arena arena;
+    const ErrorOr<AST> ast = Carbon::Parse(&arena, f, /*trace=*/false);
+    if (ast.ok()) {
+      merged_proto.MergeFrom(AstToProto(*ast));
+    }
+  }
+
+  std::set<std::string> unused_fields = GetUnusedFields(merged_proto);
+  EXPECT_EQ(unused_fields.size(), 0)
+      << "Unused fields"
+      << std::accumulate(unused_fields.begin(), unused_fields.end(),
+                         std::string(),
+                         [](const std::string& a, const std::string& b) {
+                           return a + '\n' + b;
+                         });
+}
+
+}  // namespace
+}  // namespace Carbon::Testing
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  // gtest should remove flags, leaving just input files.
+  Carbon::Testing::carbon_files =
+      new std::vector<llvm::StringRef>(&argv[1], &argv[argc]);
+  return RUN_ALL_TESTS();
+}