Carbon fuzzing 3/3: added actual fuzzer implementation and a fuzzverter utility for investigating crashing protos (#1156)

* finished fuzzer and added fuzzverter util * fixed typo * renamed cmd line params * fixed libproto_mutator download path * small fixes * small fixes * small fixes * renamed sample corpus proto * small fixes * try building on github with LIBCPP_DEBUG enabled * temporarily marked proto fuzzer as a manual test * code review * use a dedicated proto-fuzzer feature to work around LIBCPP_DEBUG=1 crash in proto code * code review comments, added README.md * minor fixes to the text * Update bazel/cc_toolchains/clang_cc_toolchain_config.bzl Co-authored-by: Jon Meow <jperkins@google.com> * use Carbon source representation for "empty Main()" instead of text format proto representation * fixed typo * made FuzzerUtil produce the full carbon source (proto converted + Main if needed) to decrease code duplication a bit * typo * switched to text proto format per code review * Update executable_semantics/prelude.h Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <jperkins@google.com> * review comments * removed unnecessary file mode variables * Update executable_semantics/fuzzing/fuzzverter.cpp Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <jperkins@google.com> * Update executable_semantics/fuzzing/README.md Co-authored-by: Jon Meow <jperkins@google.com> * code review comments * Update executable_semantics/syntax/BUILD Co-authored-by: Jon Meow <jperkins@google.com> * buildifier Co-authored-by: Jon Meow <jperkins@google.com>
2026-10-05 22:02:55 +01:00 · 2022-04-11 16:47:03 -04:00
parent db0de1004f
commit 22462a0d7f
18 changed files with 497 additions and 20 deletions
@@ -17,13 +17,13 @@ filegroup(
 cc_binary(
    name = "executable_semantics",
    srcs = ["main.cpp"],
-    data = [":standard_libraries"],
    deps = [
        "//common:error",
        "//executable_semantics/common:arena",
        "//executable_semantics/common:nonnull",
        "//executable_semantics/interpreter:exec_program",
        "//executable_semantics/syntax",
+        "//executable_semantics/syntax:prelude",
        "@llvm-project//llvm:Support",
    ],
 )
@@ -2,6 +2,8 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

+load("//bazel/fuzzing:rules.bzl", "cc_fuzz_test")
+
 cc_library(
    name = "ast_to_proto_lib",
    srcs = ["ast_to_proto.cpp"],
@@ -15,6 +17,18 @@ cc_library(
    ],
 )

+cc_library(
+    name = "fuzzer_util",
+    srcs = ["fuzzer_util.cpp"],
+    hdrs = ["fuzzer_util.h"],
+    deps = [
+        "//common:check",
+        "//common/fuzzing:carbon_cc_proto",
+        "//common/fuzzing:proto_to_carbon_lib",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_test(
    name = "ast_to_proto_test",
    srcs = ["ast_to_proto_test.cpp"],
@@ -36,6 +50,22 @@ cc_test(
    ],
 )

+cc_binary(
+    name = "fuzzverter",
+    srcs = ["fuzzverter.cpp"],
+    deps = [
+        ":ast_to_proto_lib",
+        ":fuzzer_util",
+        "//common:error",
+        "//common/fuzzing:carbon_cc_proto",
+        "//executable_semantics/common:error",
+        "//executable_semantics/common:nonnull",
+        "//executable_semantics/syntax",
+        "@com_google_protobuf//:protobuf_headers",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
 cc_test(
    name = "proto_to_carbon_test",
    srcs = ["proto_to_carbon_test.cpp"],
@@ -57,3 +87,22 @@ cc_test(
        "@llvm-project//llvm:Support",
    ],
 )
+
+# Needs `--config=proto-fuzzer` for `bazel build` / `bazel test`.
+cc_fuzz_test(
+    name = "executable_semantics_fuzzer",
+    size = "small",
+    srcs = ["executable_semantics_fuzzer.cpp"],
+    corpus = glob(["fuzzer_corpus/**"]),
+    tags = ["manual"],
+    deps = [
+        ":fuzzer_util",
+        "//common/fuzzing:carbon_cc_proto",
+        "//executable_semantics/interpreter:exec_program",
+        "//executable_semantics/syntax",
+        "//executable_semantics/syntax:prelude",
+        "@com_google_libprotobuf_mutator//:libprotobuf_mutator",
+        "@com_google_protobuf//:protobuf_headers",
+        "@llvm-project//llvm:Support",
+    ],
+)
@@ -0,0 +1,89 @@
+# Executable semantics structured fuzzer
+
+<!--
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+-->
+
+## Overview
+
+Fuzz testing is based on generating a large amount of random inputs for a
+software component in order to trigger bugs and unexpected behavior. Basic
+fuzzing uses randomly generated arrays of bytes as inputs, which works great for
+some applications but is problematic for testing the logic that operates on
+highly structured data, as most random inputs are immediately rejected as
+invalid before any interesting parts of the code get a chance to run.
+
+Structured fuzzing addresses this issue by ensuring the randomly generated data
+is itself structured, and as such has a high chance of presenting a valid input.
+
+`executable_semantics_fuzzer` is a structured fuzzer based on
+[libprotobuf-mutator](https://github.com/google/libprotobuf-mutator), which is a
+library to randomly mutate
+[protobuffers](https://github.com/protocolbuffers/protobuf).
+
+The input to the fuzzer is an instance of `Carbon::Fuzzing::Carbon` proto
+randomly generated by the `libprotobuf-mutator` framework.
+`executable_semantics_fuzzer` converts the proto to a Carbon source code string,
+and tries to parse and execute the code using `executable_semantics`
+implementation.
+
+## Fuzzer data format
+
+`libprotobuf-mutator` supports fuzzer inputs in either text or binary protocol
+buffer format. `executable_semantics_fuzzer` uses text proto format with
+`Carbon` proto message definition in `common/fuzzing/carbon.proto`.
+
+## Running the fuzzer
+
+The fuzzer can be run in 'unit test' mode, where the fuzzer executes on each
+input file from the `fuzzer_corpus/` folder, or in 'fuzzing' mode, where the
+fuzzer will keep generating random inputs and executing the logic on them until
+a crash is triggered, or forever in a bug-free program ;).
+
+To run in 'unit test' mode:
+
+```bash
+bazel test --config=proto-fuzzer --test_output=all //executable_semantics/fuzzing:executable_semantics_fuzzer
+```
+
+To run in 'fuzzing' mode:
+
+```bash
+bazel build --config=proto-fuzzer //executable_semantics/fuzzing:executable_semantics_fuzzer
+
+bazel-bin/executable_semantics/fuzzing/executable_semantics_fuzzer
+```
+
+It's also possible to run the fuzzer on a single input:
+
+```bash
+bazel-bin/executable_semantics/fuzzing/executable_semantics_fuzzer /tmp/crash.textproto
+```
+
+## Investigating a crash
+
+To reproduce a crash, run the fuzzer on the crashing input as described above.
+
+A separate tool called `fuzzverter` can be used for things like converting a
+crashing input to Carbon source code for running `executable_semantics` on the
+code directly.
+
+To convert a `Fuzzing::Carbon` text proto to Carbon source:
+
+```bash
+bazel-bin/executable_semantics/fuzzing/fuzzverter --mode proto_to_carbon --input /tmp/crash.textproto
+```
+
+## Generating new fuzzer corpus entries
+
+The ability of the fuzzing framework to generate 'interesting' inputs can be
+improved by providing 'seed' inputs known as the fuzzer corpus. The inputs need
+to be a `Fuzzing::Carbon` text proto.
+
+To generate a text proto from Carbon source:
+
+```bash
+bazel-bin/executable_semantics/fuzzing/fuzzverter --mode carbon_to_proto --input /tmp/crash.carbon --output /tmp/crash.textproto
+```
@@ -0,0 +1,42 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <google/protobuf/text_format.h>
+#include <libprotobuf_mutator/src/libfuzzer/libfuzzer_macro.h>
+
+#include "common/fuzzing/carbon.pb.h"
+#include "executable_semantics/fuzzing/fuzzer_util.h"
+#include "executable_semantics/interpreter/exec_program.h"
+#include "executable_semantics/syntax/parse.h"
+#include "executable_semantics/syntax/prelude.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace Carbon {
+
+// Parses and executes a fuzzer-generated program.
+void ParseAndExecute(const Fuzzing::CompilationUnit& compilation_unit) {
+  const std::string source = ProtoToCarbonWithMain(compilation_unit);
+
+  Arena arena;
+  ErrorOr<AST> ast = ParseFromString(&arena, "Fuzzer.carbon", source,
+                                     /*trace=*/false);
+  if (!ast.ok()) {
+    llvm::errs() << "Parsing failed: " << ast.error().message() << "\n";
+    return;
+  }
+  AddPrelude("executable_semantics/data/prelude.carbon", &arena,
+             &ast->declarations);
+  const ErrorOr<int> result = ExecProgram(&arena, *ast, /*trace=*/false);
+  if (!result.ok()) {
+    llvm::errs() << "Execution failed: " << result.error().message() << "\n";
+    return;
+  }
+  llvm::outs() << "Executed OK: " << *result << "\n";
+}
+
+}  // namespace Carbon
+
+DEFINE_TEXT_PROTO_FUZZER(const Carbon::Fuzzing::Carbon& input) {
+  Carbon::ParseAndExecute(input.compilation_unit());
+}
@@ -0,0 +1,32 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "executable_semantics/fuzzing/fuzzer_util.h"
+
+#include "common/check.h"
+#include "common/fuzzing/proto_to_carbon.h"
+
+namespace Carbon {
+
+// Appended to fuzzer-generated Carbon source when the source is missing
+// `Main()` definition, to prevent early error return in semantic analysis.
+static constexpr char EmptyMain[] = R"(
+fn Main() -> i32 {
+  return 0;
+}
+)";
+
+auto ProtoToCarbonWithMain(const Fuzzing::CompilationUnit& compilation_unit)
+    -> std::string {
+  const bool has_main = std::any_of(
+      compilation_unit.declarations().begin(),
+      compilation_unit.declarations().end(),
+      [](const Fuzzing::Declaration& decl) {
+        return decl.kind_case() == Fuzzing::Declaration::kFunction &&
+               decl.function().name() == "Main";
+      });
+  return Carbon::ProtoToCarbon(compilation_unit) + (has_main ? "" : EmptyMain);
+}
+
+}  // namespace Carbon
@@ -0,0 +1,19 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef EXECUTABLE_SEMANTICS_FUZZING_FUZZER_UTIL_H_
+#define EXECUTABLE_SEMANTICS_FUZZING_FUZZER_UTIL_H_
+
+#include "common/fuzzing/carbon.pb.h"
+
+namespace Carbon {
+
+// Converts `compilation_unit` to Carbon. Adds an default `Main()`
+// definition if one is not present in the proto.
+auto ProtoToCarbonWithMain(const Fuzzing::CompilationUnit& compilation_unit)
+    -> std::string;
+
+}  // namespace Carbon
+
+#endif  // EXECUTABLE_SEMANTICS_FUZZING_FUZZER_UTIL_H_
@@ -0,0 +1,136 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// An utility for converting between fuzzer protos and Carbon sources.
+//
+// For example, to convert a crashing input in text proto to carbon source:
+// `fuzzverter --mode=proto_to_carbon --input file.textproto`
+//
+// To generate a new text proto from carbon source for seeding the corpus:
+// `fuzzverter --mode=carbon_to_proto --input file.carbon`
+
+#include <google/protobuf/text_format.h>
+
+#include <cstdlib>
+#include <fstream>
+#include <ios>
+#include <sstream>
+
+#include "common/error.h"
+#include "common/fuzzing/carbon.pb.h"
+#include "executable_semantics/common/error.h"
+#include "executable_semantics/fuzzing/ast_to_proto.h"
+#include "executable_semantics/fuzzing/fuzzer_util.h"
+#include "executable_semantics/syntax/parse.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+
+namespace Carbon {
+
+// Reads a file and returns its contents as a string.
+static auto ReadFile(std::string_view file_name) -> ErrorOr<std::string> {
+  std::ifstream file(file_name, std::ios::in);
+  if (!file.is_open()) {
+    return ErrorBuilder() << "Could not open " << file_name << " for reading";
+  }
+  std::stringstream ss;
+  ss << file.rdbuf();
+  return ss.str();
+}
+
+// Writes string `s` to `file_name`.
+static auto WriteFile(std::string_view s, std::string_view file_name)
+    -> ErrorOr<Success> {
+  std::ofstream file(file_name, std::ios::out);
+  if (!file.is_open()) {
+    return ErrorBuilder() << "Could not open " << file_name << " for writing";
+  }
+  file << s;
+  return Success();
+}
+
+// Converts text proto to Carbon source.
+static auto TextProtoToCarbon(std::string_view input_file_name,
+                              std::string_view output_file_name)
+    -> ErrorOr<Success> {
+  ASSIGN_OR_RETURN(const std::string input_contents, ReadFile(input_file_name));
+  Fuzzing::Carbon carbon_proto;
+  if (!google::protobuf::TextFormat::ParseFromString(input_contents,
+                                                     &carbon_proto)) {
+    return Error("Could not parse text proto");
+  }
+  const std::string carbon_source =
+      ProtoToCarbonWithMain(carbon_proto.compilation_unit());
+  return WriteFile(carbon_source, output_file_name);
+}
+
+// Converts Carbon source to text proto.
+static auto CarbonToTextProto(std::string_view input_file_name,
+                              std::string_view output_file_name)
+    -> ErrorOr<Success> {
+  Carbon::Arena arena;
+  const ErrorOr<AST> ast = Carbon::Parse(&arena, input_file_name,
+                                         /*trace=*/false);
+  if (!ast.ok()) {
+    return ErrorBuilder() << "Parsing failed: " << ast.error().message();
+  }
+  Fuzzing::Carbon carbon_proto;
+  *carbon_proto.mutable_compilation_unit() = AstToProto(*ast);
+
+  std::string proto_string;
+  google::protobuf::TextFormat::Printer p;
+  if (!p.PrintToString(carbon_proto, &proto_string)) {
+    return Error("Failed to convert to text proto");
+  }
+  return WriteFile(proto_string, output_file_name);
+}
+
+// Command line options for defining input/output format.
+enum class ConversionMode { TextProtoToCarbon, CarbonToTextProto };
+
+// Returns string representation of an enum option.
+static auto GetEnumString(llvm::cl::opt<ConversionMode>& o) -> llvm::StringRef {
+  // TODO: is there a better way?
+  return o.getParser().getOption(static_cast<int>(ConversionMode(o)));
+}
+
+// Performs the conversion specified by `mode`.
+auto Convert(const ConversionMode mode, std::string_view input_file_name,
+             std::string_view output_file_name) -> ErrorOr<Success> {
+  switch (mode) {
+    case ConversionMode::TextProtoToCarbon:
+      return TextProtoToCarbon(input_file_name, output_file_name);
+    case ConversionMode::CarbonToTextProto:
+      return CarbonToTextProto(input_file_name, output_file_name);
+  }
+}
+
+}  // namespace Carbon
+
+auto main(int argc, char* argv[]) -> int {
+  llvm::InitLLVM init_llvm(argc, argv);
+
+  llvm::cl::opt<Carbon::ConversionMode> mode(
+      "mode", llvm::cl::desc("Conversion mode"),
+      llvm::cl::values(
+          clEnumValN(Carbon::ConversionMode::TextProtoToCarbon,
+                     "proto_to_carbon", "Convert text proto to Carbon source"),
+          clEnumValN(Carbon::ConversionMode::CarbonToTextProto,
+                     "carbon_to_proto",
+                     "Convert Carbon source to text proto")));
+  llvm::cl::opt<std::string> input_file_name(
+      "input", llvm::cl::desc("<input file>"), llvm::cl::init("/dev/stdin"));
+  llvm::cl::opt<std::string> output_file_name(
+      "output", llvm::cl::desc("<output file>"), llvm::cl::init("/dev/stdout"));
+  llvm::cl::ParseCommandLineOptions(argc, argv);
+
+  if (const auto result =
+          Carbon::Convert(mode, input_file_name, output_file_name);
+      !result.ok()) {
+    llvm::errs() << GetEnumString(mode)
+                 << " conversion failed: " << result.error().message() << "\n";
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}
@@ -68,6 +68,10 @@ cc_library(
    name = "exec_program",
    srcs = ["exec_program.cpp"],
    hdrs = ["exec_program.h"],
+    visibility = [
+        "//executable_semantics:__pkg__",
+        "//executable_semantics/fuzzing:__pkg__",
+    ],
    deps = [
        ":interpreter",
        ":resolve_control_flow",
@@ -15,27 +15,10 @@
 #include "executable_semantics/common/nonnull.h"
 #include "executable_semantics/interpreter/exec_program.h"
 #include "executable_semantics/syntax/parse.h"
+#include "executable_semantics/syntax/prelude.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/InitLLVM.h"

-// Adds the Carbon prelude to `declarations`.
-static void AddPrelude(
-    std::string_view prelude_file_name, Carbon::Nonnull<Carbon::Arena*> arena,
-    std::vector<Carbon::Nonnull<Carbon::Declaration*>>* declarations) {
-  Carbon::ErrorOr<Carbon::AST> parse_result =
-      Carbon::Parse(arena, prelude_file_name, false);
-  if (!parse_result.ok()) {
-    // Try again with tracing, to help diagnose the problem.
-    Carbon::ErrorOr<Carbon::AST> trace_parse_result =
-        Carbon::Parse(arena, prelude_file_name, true);
-    FATAL() << "Failed to parse prelude: "
-            << trace_parse_result.error().message();
-  }
-  const auto& prelude = *parse_result;
-  declarations->insert(declarations->begin(), prelude.declarations.begin(),
-                       prelude.declarations.end());
-}
-
 // Prints an error message and returns error code value.
 auto PrintError(const Carbon::Error& error) -> int {
  llvm::errs() << error.message() << "\n";
@@ -37,6 +37,20 @@ cc_library(
    ],
 )

+cc_library(
+    name = "prelude",
+    srcs = ["prelude.cpp"],
+    hdrs = ["prelude.h"],
+    data = ["//executable_semantics:standard_libraries"],
+    deps = [
+        ":syntax",
+        "//common:error",
+        "//executable_semantics/ast:declaration",
+        "//executable_semantics/common:arena",
+        "//executable_semantics/common:nonnull",
+    ],
+)
+
 cc_library(
    name = "syntax",
    srcs = [
@@ -0,0 +1,26 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "executable_semantics/syntax/prelude.h"
+
+#include "executable_semantics/syntax/parse.h"
+
+namespace Carbon {
+
+// Adds the Carbon prelude to `declarations`.
+void AddPrelude(std::string_view prelude_file_name, Nonnull<Arena*> arena,
+                std::vector<Nonnull<Declaration*>>* declarations) {
+  ErrorOr<AST> parse_result = Parse(arena, prelude_file_name, false);
+  if (!parse_result.ok()) {
+    // Try again with tracing, to help diagnose the problem.
+    ErrorOr<AST> trace_parse_result = Parse(arena, prelude_file_name, true);
+    FATAL() << "Failed to parse prelude: "
+            << trace_parse_result.error().message();
+  }
+  const auto& prelude = *parse_result;
+  declarations->insert(declarations->begin(), prelude.declarations.begin(),
+                       prelude.declarations.end());
+}
+
+}  // namespace Carbon
@@ -0,0 +1,22 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef EXECUTABLE_SEMANTICS_SYNTAX_PRELUDE_H_
+#define EXECUTABLE_SEMANTICS_SYNTAX_PRELUDE_H_
+
+#include <string_view>
+
+#include "executable_semantics/ast/declaration.h"
+#include "executable_semantics/common/arena.h"
+#include "executable_semantics/common/nonnull.h"
+
+namespace Carbon {
+
+// Adds the Carbon prelude to `declarations`.
+void AddPrelude(std::string_view prelude_file_name, Nonnull<Arena*> arena,
+                std::vector<Nonnull<Declaration*>>* declarations);
+
+}  // namespace Carbon
+
+#endif  // EXECUTABLE_SEMANTICS_SYNTAX_PRELUDE_H_