Extract ParseAndExecute as its own library. (#2813)

Currently ParseAndExecute-style logic is done in main.cpp and a test. #2799 is adding another test that needs it, as is #2811.

Also more clearly marks fuzzing as testing.

This is being extracted out of #2799 in order to try unblocking progress while review continues.
This commit is contained in:
Jon Ross-Perkins
2023-05-12 08:00:43 -07:00
committed by GitHub
parent 681dcfdcef
commit b1ee45c1e4
20 changed files with 226 additions and 168 deletions
+2 -6
View File
@@ -21,12 +21,8 @@ cc_library(
hdrs = ["main.h"],
deps = [
"//common:error",
"//explorer/common:arena",
"//explorer/common:nonnull",
"//explorer/interpreter:exec_program",
"//explorer/interpreter:trace_stream",
"//explorer/syntax",
"//explorer/syntax:prelude",
"//explorer/common:trace_stream",
"//explorer/parse_and_execute",
"@llvm-project//llvm:Support",
],
)
+10
View File
@@ -48,3 +48,13 @@ cc_library(
"//common:ostream",
],
)
cc_library(
name = "trace_stream",
hdrs = ["trace_stream.h"],
deps = [
"//common:check",
"//common:ostream",
"//explorer/common:nonnull",
],
)
@@ -2,8 +2,8 @@
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#ifndef CARBON_EXPLORER_INTERPRETER_TRACE_STREAM_H_
#define CARBON_EXPLORER_INTERPRETER_TRACE_STREAM_H_
#ifndef CARBON_EXPLORER_COMMON_TRACE_STREAM_H_
#define CARBON_EXPLORER_COMMON_TRACE_STREAM_H_
#include <optional>
#include <string>
@@ -59,4 +59,4 @@ class TraceStream {
} // namespace Carbon
#endif // CARBON_EXPLORER_INTERPRETER_TRACE_STREAM_H_
#endif // CARBON_EXPLORER_COMMON_TRACE_STREAM_H_
+1 -5
View File
@@ -29,7 +29,6 @@ cc_binary(
"//explorer/common:arena",
"//explorer/syntax",
"@com_google_protobuf//:protobuf_headers",
"@llvm-project//llvm:Support",
],
)
@@ -65,10 +64,7 @@ cc_library(
"//common/fuzzing:carbon_cc_proto",
"//common/fuzzing:proto_to_carbon_lib",
"//explorer/ast",
"//explorer/interpreter:exec_program",
"//explorer/interpreter:trace_stream",
"//explorer/syntax",
"//explorer/syntax:prelude",
"//explorer/parse_and_execute",
"@bazel_tools//tools/cpp/runfiles",
"@com_google_protobuf//:protobuf_headers",
"@llvm-project//llvm:Support",
+2 -2
View File
@@ -10,7 +10,7 @@
#include "explorer/ast/expression.h"
#include "llvm/Support/Casting.h"
namespace Carbon {
namespace Carbon::Testing {
using ::llvm::cast;
using ::llvm::dyn_cast;
@@ -854,4 +854,4 @@ auto AstToProto(const AST& ast) -> Fuzzing::Carbon {
return carbon;
}
} // namespace Carbon
} // namespace Carbon::Testing
+2 -2
View File
@@ -8,11 +8,11 @@
#include "common/fuzzing/carbon.pb.h"
#include "explorer/ast/ast.h"
namespace Carbon {
namespace Carbon::Testing {
// Builds a protobuf representation of `ast`.
auto AstToProto(const AST& ast) -> Fuzzing::Carbon;
} // namespace Carbon
} // namespace Carbon::Testing
#endif // CARBON_EXPLORER_FUZZING_AST_TO_PROTO_H_
+4 -4
View File
@@ -19,10 +19,10 @@
#include "explorer/fuzzing/ast_to_proto.h"
#include "explorer/syntax/parse.h"
namespace Carbon {
namespace Carbon::Testing {
auto Main(int argc, char** argv) -> ErrorOr<Success> {
Carbon::SetWorkingDirForBazel();
SetWorkingDirForBazel();
if (argc != 2) {
return Error("Syntax: ast_to_proto <file.carbon>");
@@ -54,10 +54,10 @@ auto Main(int argc, char** argv) -> ErrorOr<Success> {
return Success();
}
} // namespace Carbon
} // namespace Carbon::Testing
auto main(int argc, char** argv) -> int {
auto err = Carbon::Main(argc, argv);
auto err = Carbon::Testing::Main(argc, argv);
if (!err.ok()) {
std::cerr << err.error().message() << "\n";
return EXIT_FAILURE;
+1 -1
View File
@@ -10,5 +10,5 @@
DEFINE_TEXT_PROTO_FUZZER(const Carbon::Fuzzing::Carbon& input) {
// Only verifying it doesn't crash.
(void)Carbon::ParseAndExecute(input);
(void)Carbon::Testing::ParseAndExecuteProto(input);
}
+6 -21
View File
@@ -9,15 +9,12 @@
#include "common/check.h"
#include "common/error.h"
#include "common/fuzzing/proto_to_carbon.h"
#include "explorer/interpreter/exec_program.h"
#include "explorer/interpreter/trace_stream.h"
#include "explorer/syntax/parse.h"
#include "explorer/syntax/prelude.h"
#include "explorer/parse_and_execute/parse_and_execute.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "tools/cpp/runfiles/runfiles.h"
namespace Carbon {
namespace Carbon::Testing {
auto GetRunfilesFile(const std::string& file) -> ErrorOr<std::string> {
using bazel::tools::cpp::runfiles::Runfiles;
@@ -35,26 +32,14 @@ auto GetRunfilesFile(const std::string& file) -> ErrorOr<std::string> {
return full_path;
}
auto ParseAndExecute(const Fuzzing::Carbon& carbon) -> ErrorOr<int> {
const std::string source = ProtoToCarbon(carbon, /*maybe_add_main=*/true);
Arena arena;
CARBON_ASSIGN_OR_RETURN(AST ast,
ParseFromString(&arena, "Fuzzer.carbon", source,
/*parser_debug=*/false));
auto ParseAndExecuteProto(const Fuzzing::Carbon& carbon) -> ErrorOr<int> {
const ErrorOr<std::string> prelude_path =
GetRunfilesFile("carbon/explorer/data/prelude.carbon");
// Can't do anything without a prelude, so it's a fatal error.
CARBON_CHECK(prelude_path.ok()) << prelude_path.error();
AddPrelude(*prelude_path, &arena, &ast.declarations,
&ast.num_prelude_declarations);
TraceStream trace_stream;
// Use llvm::nulls() to suppress output from the Print intrinsic.
CARBON_ASSIGN_OR_RETURN(
ast, AnalyzeProgram(&arena, ast, &trace_stream, &llvm::nulls()));
return ExecProgram(&arena, ast, &trace_stream, &llvm::nulls());
const std::string source = ProtoToCarbon(carbon, /*maybe_add_main=*/true);
return ParseAndExecute(*prelude_path, source);
}
} // namespace Carbon
} // namespace Carbon::Testing
+3 -3
View File
@@ -9,16 +9,16 @@
#include "common/fuzzing/carbon.pb.h"
#include "explorer/ast/ast.h"
namespace Carbon {
namespace Carbon::Testing {
// Parses and executes a fuzzer-generated program.
// Returns program result if execution was successful.
auto ParseAndExecute(const Fuzzing::Carbon& carbon) -> ErrorOr<int>;
auto ParseAndExecuteProto(const Fuzzing::Carbon& carbon) -> ErrorOr<int>;
// Returns a full path for a file under bazel runfiles.
// Exposed for testing.
auto GetRunfilesFile(const std::string& file) -> ErrorOr<std::string>;
} // namespace Carbon
} // namespace Carbon::Testing
#endif // CARBON_EXPLORER_FUZZING_FUZZER_UTIL_H_
+2 -2
View File
@@ -16,7 +16,7 @@
namespace Carbon::Testing {
namespace {
TEST(FuzzerUtilTest, ParseAndExecute) {
TEST(FuzzerUtilTest, ParseAndExecuteProto) {
const ErrorOr<Fuzzing::Carbon> carbon_proto = ParseCarbonTextProto(R"(
compilation_unit {
package_statement { package_name: "P" }
@@ -42,7 +42,7 @@ TEST(FuzzerUtilTest, ParseAndExecute) {
}
})");
ASSERT_TRUE(carbon_proto.ok());
const ErrorOr<int> result = ParseAndExecute(*carbon_proto);
const ErrorOr<int> result = ParseAndExecuteProto(*carbon_proto);
ASSERT_TRUE(result.ok()) << "Execution failed: " << result.error();
EXPECT_EQ(*result, 0);
}
+20 -38
View File
@@ -2,7 +2,7 @@
# Exceptions. See /LICENSE for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
package(default_visibility = ["//explorer:__pkg__"])
package(default_visibility = ["//explorer/parse_and_execute:__pkg__"])
cc_library(
name = "action",
@@ -56,21 +56,17 @@ cc_library(
name = "exec_program",
srcs = ["exec_program.cpp"],
hdrs = ["exec_program.h"],
visibility = [
"//explorer:__pkg__",
"//explorer/fuzzing:__pkg__",
],
deps = [
":interpreter",
":resolve_control_flow",
":resolve_names",
":resolve_unformed",
":trace_stream",
":type_checker",
"//common:check",
"//common:ostream",
"//explorer/ast",
"//explorer/common:arena",
"//explorer/common:trace_stream",
"@llvm-project//llvm:Support",
],
)
@@ -114,7 +110,6 @@ cc_library(
":action_stack",
":heap",
":stack",
":trace_stream",
"//common:check",
"//common:error",
"//common:ostream",
@@ -122,6 +117,7 @@ cc_library(
"//explorer/common:arena",
"//explorer/common:error_builders",
"//explorer/common:source_location",
"//explorer/common:trace_stream",
"@llvm-project//llvm:Support",
],
)
@@ -175,20 +171,6 @@ cc_library(
],
)
cc_library(
name = "trace_stream",
hdrs = ["trace_stream.h"],
visibility = [
"//explorer:__pkg__",
"//explorer/fuzzing:__pkg__",
],
deps = [
"//common:check",
"//common:ostream",
"//explorer/common:nonnull",
],
)
cc_library(
name = "type_checker",
srcs = [
@@ -211,7 +193,6 @@ cc_library(
":dictionary",
":interpreter",
":pattern_analysis",
":trace_stream",
":type_structure",
"//common:check",
"//common:enum_base",
@@ -222,22 +203,7 @@ cc_library(
"//explorer/common:error_builders",
"//explorer/common:nonnull",
"//explorer/common:source_location",
"@llvm-project//llvm:Support",
],
)
cc_library(
name = "type_structure",
srcs = [
"type_structure.cpp",
],
hdrs = [
"type_structure.h",
],
deps = [
"//common:ostream",
"//explorer/ast",
"//explorer/common:nonnull",
"//explorer/common:trace_stream",
"@llvm-project//llvm:Support",
],
)
@@ -259,3 +225,19 @@ cc_library(
"@llvm-project//llvm:Support",
],
)
cc_library(
name = "type_structure",
srcs = [
"type_structure.cpp",
],
hdrs = [
"type_structure.h",
],
deps = [
"//common:ostream",
"//explorer/ast",
"//explorer/common:nonnull",
"@llvm-project//llvm:Support",
],
)
+1 -1
View File
@@ -10,7 +10,7 @@
#define CARBON_EXPLORER_INTERPRETER_EXEC_PROGRAM_H_
#include "explorer/ast/ast.h"
#include "explorer/interpreter/trace_stream.h"
#include "explorer/common/trace_stream.h"
#include "llvm/Support/raw_ostream.h"
namespace Carbon {
+1 -1
View File
@@ -15,9 +15,9 @@
#include "explorer/ast/expression.h"
#include "explorer/ast/pattern.h"
#include "explorer/ast/value.h"
#include "explorer/common/trace_stream.h"
#include "explorer/interpreter/action.h"
#include "explorer/interpreter/heap.h"
#include "explorer/interpreter/trace_stream.h"
#include "llvm/ADT/ArrayRef.h"
namespace Carbon {
+1 -1
View File
@@ -19,12 +19,12 @@
#include "explorer/ast/statement.h"
#include "explorer/ast/value.h"
#include "explorer/common/nonnull.h"
#include "explorer/common/trace_stream.h"
#include "explorer/interpreter/builtins.h"
#include "explorer/interpreter/dictionary.h"
#include "explorer/interpreter/impl_scope.h"
#include "explorer/interpreter/interpreter.h"
#include "explorer/interpreter/matching_impl_set.h"
#include "explorer/interpreter/trace_stream.h"
#include "llvm/ADT/identity.h"
namespace Carbon {
+11 -74
View File
@@ -15,12 +15,8 @@
#include <vector>
#include "common/error.h"
#include "explorer/common/arena.h"
#include "explorer/common/nonnull.h"
#include "explorer/interpreter/exec_program.h"
#include "explorer/interpreter/trace_stream.h"
#include "explorer/syntax/parse.h"
#include "explorer/syntax/prelude.h"
#include "explorer/common/trace_stream.h"
#include "explorer/parse_and_execute/parse_and_execute.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
@@ -85,81 +81,22 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr,
}
}
auto time_start = std::chrono::system_clock::now();
Arena arena;
AST ast;
if (ErrorOr<AST> parse_result = Parse(&arena, input_file_name, parser_debug);
parse_result.ok()) {
ast = *std::move(parse_result);
} else {
llvm::errs() << "SYNTAX ERROR: " << parse_result.error() << "\n";
return EXIT_FAILURE;
}
auto time_after_parse = std::chrono::system_clock::now();
AddPrelude(prelude_file_name, &arena, &ast.declarations,
&ast.num_prelude_declarations);
auto time_after_prelude = std::chrono::system_clock::now();
// Semantically analyze the parsed program.
if (ErrorOr<AST> analyze_result =
AnalyzeProgram(&arena, ast, &trace_stream, &llvm::outs());
analyze_result.ok()) {
ast = *std::move(analyze_result);
} else {
llvm::errs() << "COMPILATION ERROR: " << analyze_result.error() << "\n";
return EXIT_FAILURE;
}
auto time_after_analyze = std::chrono::system_clock::now();
// Run the program.
auto ret = EXIT_SUCCESS;
if (ErrorOr<int> exec_result =
ExecProgram(&arena, ast, &trace_stream, &llvm::outs());
exec_result.ok()) {
ErrorOr<int> result =
ParseAndExecuteFile(prelude_file_name, input_file_name, parser_debug,
&trace_stream, &llvm::outs());
if (result.ok()) {
// Print the return code to stdout.
llvm::outs() << "result: " << *exec_result << "\n";
llvm::outs() << "result: " << *result << "\n";
// When there's a dedicated trace file, print the return code to it too.
if (scoped_trace_stream) {
trace_stream << "result: " << *exec_result << "\n";
trace_stream << "result: " << *result << "\n";
}
return EXIT_SUCCESS;
} else {
llvm::errs() << "RUNTIME ERROR: " << exec_result.error() << "\n";
ret = EXIT_FAILURE;
llvm::errs() << result.error() << "\n";
return EXIT_FAILURE;
}
auto time_after_exec = std::chrono::system_clock::now();
if (trace_stream.is_enabled()) {
trace_stream << "Timings:\n"
<< "- Parse: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
time_after_parse - time_start)
.count()
<< "ms\n"
<< "- AddPrelude: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
time_after_prelude - time_after_parse)
.count()
<< "ms\n"
<< "- AnalyzeProgram: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
time_after_analyze - time_after_prelude)
.count()
<< "ms\n"
<< "- ExecProgram: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
time_after_exec - time_after_analyze)
.count()
<< "ms\n";
}
return ret;
}
} // namespace Carbon
+25
View File
@@ -0,0 +1,25 @@
# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
# Exceptions. See /LICENSE for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
load("//bazel/cc_toolchains:defs.bzl", "cc_env")
package(default_visibility = [
"//explorer:__pkg__",
"//explorer/fuzzing:__pkg__",
])
cc_library(
name = "parse_and_execute",
srcs = ["parse_and_execute.cpp"],
hdrs = ["parse_and_execute.h"],
deps = [
"//common:check",
"//common:error",
"//explorer/common:trace_stream",
"//explorer/interpreter:exec_program",
"//explorer/syntax",
"//explorer/syntax:prelude",
"@llvm-project//llvm:Support",
],
)
@@ -0,0 +1,98 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "explorer/parse_and_execute/parse_and_execute.h"
#include <locale>
#include "common/check.h"
#include "common/error.h"
#include "explorer/interpreter/exec_program.h"
#include "explorer/syntax/parse.h"
#include "explorer/syntax/prelude.h"
#include "llvm/ADT/ScopeExit.h"
namespace Carbon {
// Returns a scope exit function for printing the timing of a step on scope
// exit. Note the use prints step timings in reverse order.
static auto PrintTimingOnExit(TraceStream* trace_stream, const char* label,
std::chrono::steady_clock::time_point* cursor) {
auto end = std::chrono::steady_clock::now();
auto duration = end - *cursor;
*cursor = end;
return llvm::make_scope_exit([=]() {
if (trace_stream->is_enabled()) {
*trace_stream << "Time elapsed in " << label << ": "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
duration)
.count()
<< "ms\n";
}
});
}
static auto ParseAndExecuteHelper(std::function<ErrorOr<AST>(Arena*)> parse,
const std::string& prelude_path,
Nonnull<TraceStream*> trace_stream,
Nonnull<llvm::raw_ostream*> print_stream)
-> ErrorOr<int> {
Arena arena;
auto cursor = std::chrono::steady_clock::now();
ErrorOr<AST> parse_result = parse(&arena);
auto print_parse_time = PrintTimingOnExit(trace_stream, "Parse", &cursor);
if (!parse_result.ok()) {
return ErrorBuilder() << "SYNTAX ERROR: " << parse_result.error();
}
AddPrelude(prelude_path, &arena, &parse_result->declarations,
&parse_result->num_prelude_declarations);
auto print_prelude_time =
PrintTimingOnExit(trace_stream, "AddPrelude", &cursor);
// Semantically analyze the parsed program.
ErrorOr<AST> analyze_result =
AnalyzeProgram(&arena, *parse_result, trace_stream, print_stream);
auto print_analyze_time =
PrintTimingOnExit(trace_stream, "AnalyzeProgram", &cursor);
if (!analyze_result.ok()) {
return ErrorBuilder() << "COMPILATION ERROR: " << analyze_result.error();
}
// Run the program.
ErrorOr<int> exec_result =
ExecProgram(&arena, *analyze_result, trace_stream, print_stream);
auto print_exec_time =
PrintTimingOnExit(trace_stream, "ExecProgram", &cursor);
if (!exec_result.ok()) {
return ErrorBuilder() << "RUNTIME ERROR: " << exec_result.error();
}
return exec_result;
}
auto ParseAndExecuteFile(const std::string& prelude_path,
const std::string& input_file_name, bool parser_debug,
Nonnull<TraceStream*> trace_stream,
Nonnull<llvm::raw_ostream*> print_stream)
-> ErrorOr<int> {
auto parse = [&](Arena* arena) {
return Parse(arena, input_file_name, parser_debug);
};
return ParseAndExecuteHelper(parse, prelude_path, trace_stream, print_stream);
}
auto ParseAndExecute(const std::string& prelude_path, const std::string& source)
-> ErrorOr<int> {
auto parse = [&](Arena* arena) {
return ParseFromString(arena, "test.carbon", source,
/*parser_debug=*/false);
};
TraceStream trace_stream;
return ParseAndExecuteHelper(parse, prelude_path, &trace_stream,
&llvm::nulls());
}
} // namespace Carbon
@@ -0,0 +1,28 @@
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#ifndef CARBON_EXPLORER_PARSE_AND_EXECUTE_PARSE_AND_EXECUTE_H_
#define CARBON_EXPLORER_PARSE_AND_EXECUTE_PARSE_AND_EXECUTE_H_
#include "common/error.h"
#include "explorer/common/trace_stream.h"
namespace Carbon {
// Parses and executes the input file, returning the program result on success.
// This API is intended for use by main execution.
auto ParseAndExecuteFile(const std::string& prelude_path,
const std::string& input_file_name, bool parser_debug,
Nonnull<TraceStream*> trace_stream,
Nonnull<llvm::raw_ostream*> print_stream)
-> ErrorOr<int>;
// Parses and executes the source, returning the program result on success.
// Discards output.
auto ParseAndExecute(const std::string& prelude_path, const std::string& source)
-> ErrorOr<int>;
} // namespace Carbon
#endif // CARBON_EXPLORER_PARSE_AND_EXECUTE_PARSE_AND_EXECUTE_H_
+5 -4
View File
@@ -2,10 +2,7 @@
# Exceptions. See /LICENSE for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
package(default_visibility = [
"//explorer:__pkg__",
"//explorer/fuzzing:__pkg__",
])
package(default_visibility = ["//explorer/parse_and_execute:__pkg__"])
cc_library(
name = "bison_wrap",
@@ -75,6 +72,10 @@ cc_library(
"-Wno-unused-function",
"-Wno-writable-strings",
],
visibility = [
"//explorer/fuzzing:__pkg__",
"//explorer/parse_and_execute:__pkg__",
],
deps = [
":bison_wrap",
"//common:check",