From 9e03d5efe295a18090c2f3caf78e7dc9ae49b9ea Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Fri, 28 Jul 2023 08:29:51 -0700 Subject: [PATCH] Change explorer's file_test to support argument passing to main. (#3032) This shifts explorer's file_test to use ExplorerMain in order to be able to pass arguments similar to how the command line would. It also means that various output wrapping done by the command line is shared, no longer copied by file_test. In order to help make this work, I plugged vfs::FileSystem into explorer, similar to how we're doing this on the toolchain side (might try to share more code later). I was having trouble getting an overlay working, I think due to how paths are specified -- but due to the issues in fixing this, I'm just loading the prelude into the InMemoryFilesystem for now. Under this approach, I'm unifying more of the file versus string handling. I think we should shift towards an approach where, like toolchain code, anyone trying to use Explorer should use a vfs implementation to supply code. This should reduce the number of distinct code paths which we're maintaining/testing. Short-term, this allows the trace testdata to be merged into file_test with less special casing, using ARGS:. Long-term, it means that the file_test knows the ARGS to pass to generate checks to match against, which is the direction I'm planning for autoupdate. --- explorer/BUILD | 4 +- explorer/common/source_location.h | 2 +- explorer/file_test.cpp | 83 ++++++++++--------- explorer/fuzzing/ast_to_proto_main.cpp | 11 +-- explorer/fuzzing/ast_to_proto_test.cpp | 12 +-- explorer/fuzzing/fuzzer_util.cpp | 12 ++- explorer/main.cpp | 35 +++++--- explorer/main.h | 15 ++++ .../parse_and_execute/parse_and_execute.cpp | 38 ++------- .../parse_and_execute/parse_and_execute.h | 15 +--- .../parse_and_execute_test.cpp | 13 ++- explorer/syntax/BUILD | 1 + explorer/syntax/parse.cpp | 46 +++++----- explorer/syntax/parse.h | 6 +- explorer/syntax/prelude.cpp | 7 +- explorer/syntax/prelude.h | 4 +- explorer/trace_testdata/full_trace.carbon | 1 + toolchain/driver/driver_file_test_base.h | 1 - 18 files changed, 167 insertions(+), 139 deletions(-) diff --git a/explorer/BUILD b/explorer/BUILD index dd205e89d9ee..076a1f034740 100644 --- a/explorer/BUILD +++ b/explorer/BUILD @@ -23,6 +23,7 @@ cc_library( hdrs = ["main.h"], deps = [ "//common:error", + "//common:ostream", "//explorer/common:trace_stream", "//explorer/parse_and_execute", "@llvm-project//llvm:Support", @@ -45,7 +46,8 @@ cc_library( testonly = 1, srcs = ["file_test.cpp"], deps = [ - "//explorer/parse_and_execute", + ":main", + "//common:check", "//testing/file_test:file_test_base", "//testing/util:test_raw_ostream", "@com_google_absl//absl/flags:flag", diff --git a/explorer/common/source_location.h b/explorer/common/source_location.h index aa3232472c76..a8f3bb305389 100644 --- a/explorer/common/source_location.h +++ b/explorer/common/source_location.h @@ -26,7 +26,7 @@ class SourceLocation { } // The filename should be eternal or arena-allocated to eliminate copies. - explicit constexpr SourceLocation(const char* filename, int line_num, + explicit constexpr SourceLocation(std::string_view filename, int line_num, FileKind file_kind) : filename_(filename), line_num_(line_num), file_kind_(file_kind) {} explicit SourceLocation(Nonnull filename, int line_num, diff --git a/explorer/file_test.cpp b/explorer/file_test.cpp index 69baa3141fab..6bd836c2e605 100644 --- a/explorer/file_test.cpp +++ b/explorer/file_test.cpp @@ -3,7 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "absl/flags/flag.h" -#include "explorer/parse_and_execute/parse_and_execute.h" +#include "common/check.h" +#include "explorer/main.h" #include "testing/file_test/file_test_base.h" #include "testing/util/test_raw_ostream.h" @@ -24,61 +25,65 @@ class ParseAndExecuteTestFile : public FileTestBase { const llvm::SmallVector& test_files, llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr) -> bool override { - CARBON_CHECK(test_args.empty()) - << "ARGS are not currently used in explorer's file_test."; + // Create the files in-memory. + llvm::vfs::InMemoryFileSystem fs(new llvm::vfs::InMemoryFileSystem()); + for (const auto& test_file : test_files) { + if (!fs.addFile(test_file.filename, /*ModificationTime=*/0, + llvm::MemoryBuffer::getMemBuffer(test_file.content))) { + ADD_FAILURE() << "File is repeated: " << test_file.filename; + return false; + } + } - if (test_files.size() != 1) { - ADD_FAILURE() << "Only 1 file is supported: " << test_files.size() - << " provided"; + // Add the prelude. + llvm::ErrorOr> prelude = + llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon"); + if (prelude.getError()) { + ADD_FAILURE() << prelude.getError().message(); + return false; + } + // TODO: This path is long with a prefix / because of the path expectations + // in tests. Change those to allow a shorter path (e.g., `prelude.carbon`) + // here. + static constexpr llvm::StringLiteral PreludePath = + "/explorer/data/prelude.carbon"; + if (!fs.addFile(PreludePath, /*ModificationTime=*/0, std::move(*prelude))) { + ADD_FAILURE() << "Duplicate prelude.carbon"; return false; } + llvm::SmallVector args = {"explorer"}; + for (auto arg : test_args) { + args.push_back(arg.data()); + } + TestRawOstream trace_stream; + // Trace output is only checked for a few tests. bool check_trace_output = - path().string().find("/trace_testdata/") != std::string::npos; + path().string().find("trace_testdata/") != std::string::npos; - // Capture trace streaming, but only when in debug mode. - TraceStream trace_stream; - TestRawOstream trace_stream_ostream; - if (check_trace_output || absl::GetFlag(FLAGS_trace)) { - trace_stream.set_stream(check_trace_output ? &stdout - : &trace_stream_ostream); - trace_stream.set_allowed_phases({ProgramPhase::All}); - trace_stream.set_allowed_file_kinds({FileKind::Main}); - } - - // Set the location of the prelude. - char* test_srcdir = getenv("TEST_SRCDIR"); - CARBON_CHECK(test_srcdir != nullptr); - std::string prelude_path(test_srcdir); - prelude_path += "/carbon/explorer/data/prelude.carbon"; - - // Run the parse. Parser debug output is always off because it's difficult - // to redirect. - auto result = ParseAndExecute( - prelude_path, test_files[0].filename, test_files[0].content, - /*parser_debug=*/false, &trace_stream, &stdout); - // This mirrors printing currently done by main.cpp. - if (result.ok()) { - stdout << "result: " << *result << "\n"; - } else { - stderr << result.error() << "\n"; - } + int exit_code = ExplorerMain( + args.size(), args.data(), /*install_path=*/"", PreludePath, stdout, + stderr, check_trace_output ? stdout : trace_stream, fs); // Skip trace test check as they use stdout stream instead of // trace_stream_ostream if (absl::GetFlag(FLAGS_trace)) { - CARBON_CHECK(!check_trace_output) - << "trace tests should only be run in the default mode."; - EXPECT_FALSE(trace_stream_ostream.TakeStr().empty()) + EXPECT_FALSE(trace_stream.TakeStr().empty()) << "Tracing should always do something"; } - return result.ok(); + return exit_code == EXIT_SUCCESS; } auto GetDefaultArgs() -> llvm::SmallVector override { - return {}; + llvm::SmallVector args; + if (absl::GetFlag(FLAGS_trace)) { + args.push_back("--trace_file=-"); + args.push_back("--trace_phase=all"); + } + args.push_back("%s"); + return args; } }; diff --git a/explorer/fuzzing/ast_to_proto_main.cpp b/explorer/fuzzing/ast_to_proto_main.cpp index f7962400a3d4..590ca9ed4460 100644 --- a/explorer/fuzzing/ast_to_proto_main.cpp +++ b/explorer/fuzzing/ast_to_proto_main.cpp @@ -31,15 +31,10 @@ auto Main(int argc, char** argv) -> ErrorOr { return Error("Argument must be a file."); } - // Read the input file. - std::ifstream proto_file(argv[1]); - std::stringstream buffer; - buffer << proto_file.rdbuf(); - proto_file.close(); - Arena arena; - const ErrorOr ast = Parse(&arena, argv[1], FileKind::Main, - /*parser_debug=*/false); + const ErrorOr ast = + Parse(*llvm::vfs::getRealFileSystem(), &arena, argv[1], FileKind::Main, + /*parser_debug=*/false); if (!ast.ok()) { return ErrorBuilder() << "Parsing failed: " << ast.error().message(); } diff --git a/explorer/fuzzing/ast_to_proto_test.cpp b/explorer/fuzzing/ast_to_proto_test.cpp index 744307d8dac1..ac149cd9b576 100644 --- a/explorer/fuzzing/ast_to_proto_test.cpp +++ b/explorer/fuzzing/ast_to_proto_test.cpp @@ -99,8 +99,8 @@ TEST(AstToProtoTest, SetsAllProtoFields) { Fuzzing::Carbon merged_proto; for (const llvm::StringRef f : *carbon_files) { Arena arena; - const ErrorOr ast = - Parse(&arena, f, FileKind::Main, /*parser_debug=*/false); + const ErrorOr ast = Parse(*llvm::vfs::getRealFileSystem(), &arena, f, + FileKind::Main, /*parser_debug=*/false); if (ast.ok()) { merged_proto.MergeFrom(AstToProto(*ast)); } @@ -137,8 +137,8 @@ TEST(AstToProtoTest, Roundtrip) { int parsed_ok_count = 0; for (const llvm::StringRef f : *carbon_files) { Arena arena; - const ErrorOr ast = - Parse(&arena, f, FileKind::Main, /*parser_debug=*/false); + const ErrorOr ast = Parse(*llvm::vfs::getRealFileSystem(), &arena, f, + FileKind::Main, /*parser_debug=*/false); if (ast.ok()) { ++parsed_ok_count; const std::string source_from_proto = @@ -177,8 +177,8 @@ TEST(AstToProtoTest, SameProtoAfterClone) { int parsed_ok_count = 0; for (const llvm::StringRef f : *carbon_files) { Arena arena; - const ErrorOr ast = - Parse(&arena, f, FileKind::Main, /*parser_debug=*/false); + const ErrorOr ast = Parse(*llvm::vfs::getRealFileSystem(), &arena, f, + FileKind::Main, /*parser_debug=*/false); if (ast.ok()) { ++parsed_ok_count; const AST clone = CloneAST(arena, *ast); diff --git a/explorer/fuzzing/fuzzer_util.cpp b/explorer/fuzzing/fuzzer_util.cpp index a40488d658ef..6f6fe713e360 100644 --- a/explorer/fuzzing/fuzzer_util.cpp +++ b/explorer/fuzzing/fuzzer_util.cpp @@ -33,14 +33,24 @@ auto GetRunfilesFile(const std::string& file) -> ErrorOr { } auto ParseAndExecuteProto(const Fuzzing::Carbon& carbon) -> ErrorOr { + llvm::vfs::InMemoryFileSystem fs; + const ErrorOr prelude_path = GetRunfilesFile("carbon/explorer/data/prelude.carbon"); // Can't do anything without a prelude, so it's a fatal error. CARBON_CHECK(prelude_path.ok()) << prelude_path.error(); + llvm::ErrorOr> prelude = + llvm::MemoryBuffer::getFile(*prelude_path); + CARBON_CHECK(!prelude.getError()) << prelude.getError().message(); + CARBON_CHECK(fs.addFile("prelude.carbon", /*ModificationTime=*/0, + std::move(*prelude))); const std::string source = ProtoToCarbon(carbon, /*maybe_add_main=*/true); + CARBON_CHECK(fs.addFile("fuzzer.carbon", /*ModificationTime=*/0, + llvm::MemoryBuffer::getMemBuffer(source))); + TraceStream trace_stream; - return ParseAndExecute(*prelude_path, "fuzzer.carbon", source, + return ParseAndExecute(fs, "prelude.carbon", "fuzzer.carbon", /*parser_debug=*/false, &trace_stream, &llvm::nulls()); } diff --git a/explorer/main.cpp b/explorer/main.cpp index 476ee5fc8cbf..ead23a2166c1 100644 --- a/explorer/main.cpp +++ b/explorer/main.cpp @@ -17,6 +17,7 @@ #include "common/error.h" #include "explorer/common/trace_stream.h" #include "explorer/parse_and_execute/parse_and_execute.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" @@ -32,7 +33,6 @@ namespace path = llvm::sys::path; auto ExplorerMain(int argc, char** argv, void* static_for_main_addr, llvm::StringRef relative_prelude_path) -> int { llvm::setBugReportMsg( - "Please report issues to " "https://github.com/carbon-language/carbon-lang/issues and include the " "crash backtrace.\n"); @@ -42,6 +42,20 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr, // is piped to stdout. llvm::errs().tie(&llvm::outs()); + std::string exe = + llvm::sys::fs::getMainExecutable(argv[0], static_for_main_addr); + llvm::StringRef install_path = path::parent_path(exe); + + return ExplorerMain(argc, const_cast(argv), install_path, + relative_prelude_path, llvm::outs(), llvm::errs(), + llvm::outs(), *llvm::vfs::getRealFileSystem()); +} + +auto ExplorerMain(int argc, const char** argv, llvm::StringRef install_path, + llvm::StringRef relative_prelude_path, + llvm::raw_ostream& out_stream, llvm::raw_ostream& err_stream, + llvm::raw_ostream& out_stream_for_trace, + llvm::vfs::FileSystem& fs) -> int { cl::opt input_file_name(cl::Positional, cl::desc(""), cl::Required); cl::opt parser_debug("parser_debug", @@ -98,10 +112,9 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr, "Include trace output for all files")), cl::CommaSeparated); + CARBON_CHECK(argc > 0); + // Use the executable path as a base for the relative prelude path. - std::string exe = - llvm::sys::fs::getMainExecutable(argv[0], static_for_main_addr); - llvm::StringRef install_path = path::parent_path(exe); llvm::SmallString<256> default_prelude_file(install_path); path::append(default_prelude_file, path::begin(relative_prelude_path, path::Style::posix), @@ -111,6 +124,8 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr, cl::init(default_prelude_file_str)); cl::ParseCommandLineOptions(argc, argv); + auto reset_parser = + llvm::make_scope_exit([] { cl::ResetCommandLineParser(); }); // Set up a stream for trace output. std::unique_ptr scoped_trace_stream; @@ -147,13 +162,13 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr, trace_stream.set_allowed_file_kinds(trace_file_kinds); if (trace_file_name == "-") { - trace_stream.set_stream(&llvm::outs()); + trace_stream.set_stream(&out_stream_for_trace); } else { std::error_code err; scoped_trace_stream = std::make_unique(trace_file_name, err); if (err) { - llvm::errs() << err.message() << "\n"; + err_stream << err.message() << "\n"; return EXIT_FAILURE; } trace_stream.set_stream(scoped_trace_stream.get()); @@ -161,14 +176,14 @@ auto ExplorerMain(int argc, char** argv, void* static_for_main_addr, } ErrorOr result = - ParseAndExecuteFile(prelude_file_name, input_file_name, parser_debug, - &trace_stream, &llvm::outs()); + ParseAndExecute(fs, prelude_file_name, input_file_name, parser_debug, + &trace_stream, &out_stream); if (result.ok()) { // Print the return code to stdout. - llvm::outs() << "result: " << *result << "\n"; + out_stream << "result: " << *result << "\n"; return EXIT_SUCCESS; } else { - llvm::errs() << result.error() << "\n"; + err_stream << result.error() << "\n"; return EXIT_FAILURE; } } diff --git a/explorer/main.h b/explorer/main.h index 73916609416f..bb1813f01c15 100644 --- a/explorer/main.h +++ b/explorer/main.h @@ -5,7 +5,9 @@ #ifndef CARBON_EXPLORER_MAIN_H_ #define CARBON_EXPLORER_MAIN_H_ +#include "common/ostream.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/VirtualFileSystem.h" namespace Carbon { @@ -14,6 +16,19 @@ namespace Carbon { auto ExplorerMain(int argc, char** argv, void* static_for_main_addr, llvm::StringRef relative_prelude_path) -> int; +// Wrapped by the above, but without some process init. This is used directly by +// tests, whereas the above is used directly by main binaries. The +// out_stream_for_trace is only used when `--trace_file=-` is specified. +// +// TODO: We use argc/argv for parameters because command line parsing requires +// it. When that's replaced, we should switch to +// llvm::SmallVector args because it'll work better with tests. +auto ExplorerMain(int argc, const char** argv, llvm::StringRef install_path, + llvm::StringRef relative_prelude_path, + llvm::raw_ostream& out_stream, llvm::raw_ostream& err_stream, + llvm::raw_ostream& out_stream_for_trace, + llvm::vfs::FileSystem& fs) -> int; + } // namespace Carbon #endif // CARBON_EXPLORER_MAIN_H_ diff --git a/explorer/parse_and_execute/parse_and_execute.cpp b/explorer/parse_and_execute/parse_and_execute.cpp index a5a9eeb026ed..733962c9af97 100644 --- a/explorer/parse_and_execute/parse_and_execute.cpp +++ b/explorer/parse_and_execute/parse_and_execute.cpp @@ -6,7 +6,6 @@ #include -#include "common/check.h" #include "common/error.h" #include "explorer/common/trace_stream.h" #include "explorer/interpreter/exec_program.h" @@ -37,22 +36,22 @@ static auto PrintTimingOnExit(TraceStream* trace_stream, const char* label, return exit_scope_function; } -static auto ParseAndExecuteHelper(std::function(Arena*)> parse, - std::string_view prelude_path, - Nonnull trace_stream, - Nonnull print_stream) - -> ErrorOr { +auto ParseAndExecute(llvm::vfs::FileSystem& fs, std::string_view prelude_path, + std::string_view input_file_name, bool parser_debug, + Nonnull trace_stream, + Nonnull print_stream) -> ErrorOr { return RunWithExtraStack([&]() -> ErrorOr { Arena arena; auto cursor = std::chrono::steady_clock::now(); - ErrorOr parse_result = parse(&arena); + ErrorOr parse_result = + Parse(fs, &arena, input_file_name, FileKind::Main, parser_debug); auto print_parse_time = PrintTimingOnExit(trace_stream, "Parse", &cursor); if (!parse_result.ok()) { return ErrorBuilder() << "SYNTAX ERROR: " << parse_result.error(); } - AddPrelude(prelude_path, &arena, &parse_result->declarations, + AddPrelude(fs, prelude_path, &arena, &parse_result->declarations, &parse_result->num_prelude_declarations); auto print_prelude_time = PrintTimingOnExit(trace_stream, "AddPrelude", &cursor); @@ -87,27 +86,4 @@ static auto ParseAndExecuteHelper(std::function(Arena*)> parse, }); } -auto ParseAndExecuteFile(std::string_view prelude_path, - std::string_view input_file_name, bool parser_debug, - Nonnull trace_stream, - Nonnull print_stream) - -> ErrorOr { - auto parse = [&](Arena* arena) { - return Parse(arena, input_file_name, FileKind::Main, parser_debug); - }; - return ParseAndExecuteHelper(parse, prelude_path, trace_stream, print_stream); -} - -auto ParseAndExecute(std::string_view prelude_path, - std::string_view input_file_name, - std::string_view file_contents, bool parser_debug, - Nonnull trace_stream, - Nonnull print_stream) -> ErrorOr { - auto parse = [&](Arena* arena) { - return ParseFromString(arena, input_file_name, FileKind::Main, - file_contents, parser_debug); - }; - return ParseAndExecuteHelper(parse, prelude_path, trace_stream, print_stream); -} - } // namespace Carbon diff --git a/explorer/parse_and_execute/parse_and_execute.h b/explorer/parse_and_execute/parse_and_execute.h index fa355423c92a..d31eae6edbd3 100644 --- a/explorer/parse_and_execute/parse_and_execute.h +++ b/explorer/parse_and_execute/parse_and_execute.h @@ -7,22 +7,13 @@ #include "common/error.h" #include "explorer/common/trace_stream.h" +#include "llvm/Support/VirtualFileSystem.h" namespace Carbon { // Parses and executes the input file, returning the program result on success. -// This API is intended for use by main execution. -auto ParseAndExecuteFile(std::string_view prelude_path, - std::string_view input_file_name, bool parser_debug, - Nonnull trace_stream, - Nonnull print_stream) - -> ErrorOr; - -// Parses and executes the source, returning the program result on success. -// Discards output. -auto ParseAndExecute(std::string_view prelude_path, - std::string_view input_file_name, - std::string_view file_contents, bool parser_debug, +auto ParseAndExecute(llvm::vfs::FileSystem& fs, std::string_view prelude_path, + std::string_view input_file_name, bool parser_debug, Nonnull trace_stream, Nonnull print_stream) -> ErrorOr; diff --git a/explorer/parse_and_execute/parse_and_execute_test.cpp b/explorer/parse_and_execute/parse_and_execute_test.cpp index 923dbf93c474..8eb08e5604da 100644 --- a/explorer/parse_and_execute/parse_and_execute_test.cpp +++ b/explorer/parse_and_execute/parse_and_execute_test.cpp @@ -10,10 +10,16 @@ namespace Carbon::Testing { namespace { -using ::testing::Eq; using ::testing::MatchesRegex; TEST(ParseAndExecuteTest, Recursion) { + llvm::vfs::InMemoryFileSystem fs; + llvm::ErrorOr> prelude = + llvm::MemoryBuffer::getFile("explorer/data/prelude.carbon"); + ASSERT_FALSE(prelude.getError()) << prelude.getError().message(); + ASSERT_TRUE(fs.addFile("prelude.carbon", /*ModificationTime=*/0, + std::move(*prelude))); + std::string source = R"( package Test api; fn Main() -> i32 { @@ -32,9 +38,12 @@ TEST(ParseAndExecuteTest, Recursion) { ; } )"; + ASSERT_TRUE(fs.addFile("test.carbon", /*ModificationTime=*/0, + llvm::MemoryBuffer::getMemBuffer(source))); + TraceStream trace_stream; auto err = - ParseAndExecute("explorer/data/prelude.carbon", "test.carbon", source, + ParseAndExecute(fs, "prelude.carbon", "test.carbon", /*parser_debug=*/false, &trace_stream, &llvm::nulls()); ASSERT_FALSE(err.ok()); // Don't expect any particular source location for the error. diff --git a/explorer/syntax/BUILD b/explorer/syntax/BUILD index d5d5ade15a08..9efc2771c36e 100644 --- a/explorer/syntax/BUILD +++ b/explorer/syntax/BUILD @@ -44,6 +44,7 @@ cc_library( "//explorer/ast", "//explorer/common:arena", "//explorer/common:nonnull", + "@llvm-project//llvm:Support", ], ) diff --git a/explorer/syntax/parse.cpp b/explorer/syntax/parse.cpp index 69ef8825e913..c7d13948e389 100644 --- a/explorer/syntax/parse.cpp +++ b/explorer/syntax/parse.cpp @@ -10,7 +10,6 @@ #include "explorer/syntax/lexer.h" #include "explorer/syntax/parse_and_lex_context.h" #include "explorer/syntax/parser.h" -#include "llvm/Support/Error.h" namespace Carbon { @@ -42,30 +41,35 @@ static auto ParseImpl(yyscan_t scanner, Nonnull arena, return *ast; } -auto Parse(Nonnull arena, std::string_view input_file_name, - FileKind file_kind, bool parser_debug) -> ErrorOr { - std::string name_str(input_file_name); - FILE* input_file = fopen(name_str.c_str(), "r"); - if (input_file == nullptr) { - return ProgramError(SourceLocation(name_str.c_str(), 0, file_kind)) - << "Error opening file: " << std::strerror(errno); +auto Parse(llvm::vfs::FileSystem& fs, Nonnull arena, + std::string_view input_file_name, FileKind file_kind, + bool parser_debug) -> ErrorOr { + llvm::ErrorOr> input_file = + fs.openFileForRead(input_file_name); + if (input_file.getError()) { + return ProgramError(SourceLocation(input_file_name, 0, file_kind)) + << "Error opening file: " << input_file.getError().message(); } - // Prepare the lexer. - yyscan_t scanner; - yylex_init(&scanner); - auto* buffer = yy_create_buffer(input_file, YY_BUF_SIZE, scanner); - yy_switch_to_buffer(buffer, scanner); + llvm::ErrorOr status = (*input_file)->status(); + if (status.getError()) { + return Error(status.getError().message()); + } + auto size = status->getSize(); + if (size >= std::numeric_limits::max()) { + return ProgramError(SourceLocation(input_file_name, 0, file_kind)) + << "File is over the 2GiB input limit."; + } - ErrorOr result = - ParseImpl(scanner, arena, input_file_name, file_kind, parser_debug); + llvm::ErrorOr> buffer = + (*input_file) + ->getBuffer(input_file_name, size, /*RequiresNullTerminator=*/false); + if (buffer.getError()) { + return Error(buffer.getError().message()); + } - // Clean up the lexer. - yy_delete_buffer(buffer, scanner); - yylex_destroy(scanner); - fclose(input_file); - - return result; + return ParseFromString(arena, input_file_name, file_kind, + (*buffer)->getBuffer(), parser_debug); } auto ParseFromString(Nonnull arena, std::string_view input_file_name, diff --git a/explorer/syntax/parse.h b/explorer/syntax/parse.h index bce0fffc6a11..b0ff5816366c 100644 --- a/explorer/syntax/parse.h +++ b/explorer/syntax/parse.h @@ -11,13 +11,15 @@ #include "explorer/ast/ast.h" #include "explorer/common/arena.h" #include "explorer/common/source_location.h" +#include "llvm/Support/VirtualFileSystem.h" namespace Carbon { // Returns the AST representing the contents of the named file, or an error code // if parsing fails. Allocations go into the provided arena. -auto Parse(Nonnull arena, std::string_view input_file_name, - FileKind file_kind, bool parser_debug) -> ErrorOr; +auto Parse(llvm::vfs::FileSystem& fs, Nonnull arena, + std::string_view input_file_name, FileKind file_kind, + bool parser_debug) -> ErrorOr; // Equivalent to `Parse`, but parses the contents of `file_contents`. // `input_file_name` is used only for reporting source locations, and does diff --git a/explorer/syntax/prelude.cpp b/explorer/syntax/prelude.cpp index 90116026997f..1ab9cd08194b 100644 --- a/explorer/syntax/prelude.cpp +++ b/explorer/syntax/prelude.cpp @@ -9,15 +9,16 @@ namespace Carbon { // Adds the Carbon prelude to `declarations`. -void AddPrelude(std::string_view prelude_file_name, Nonnull arena, +void AddPrelude(llvm::vfs::FileSystem& fs, std::string_view prelude_file_name, + Nonnull arena, std::vector>* declarations, int* num_prelude_declarations) { ErrorOr parse_result = - Parse(arena, prelude_file_name, FileKind::Prelude, false); + Parse(fs, arena, prelude_file_name, FileKind::Prelude, false); if (!parse_result.ok()) { // Try again with tracing, to help diagnose the problem. ErrorOr trace_parse_result = - Parse(arena, prelude_file_name, FileKind::Prelude, true); + Parse(fs, arena, prelude_file_name, FileKind::Prelude, true); CARBON_FATAL() << "Failed to parse prelude:\n" << trace_parse_result.error(); } diff --git a/explorer/syntax/prelude.h b/explorer/syntax/prelude.h index c0c2bfb531d4..d17ecb5a4a35 100644 --- a/explorer/syntax/prelude.h +++ b/explorer/syntax/prelude.h @@ -10,11 +10,13 @@ #include "explorer/ast/declaration.h" #include "explorer/common/arena.h" #include "explorer/common/nonnull.h" +#include "llvm/Support/VirtualFileSystem.h" namespace Carbon { // Adds the Carbon prelude to `declarations`. -void AddPrelude(std::string_view prelude_file_name, Nonnull arena, +void AddPrelude(llvm::vfs::FileSystem& fs, std::string_view prelude_file_name, + Nonnull arena, std::vector>* declarations, int* num_prelude_declarations); diff --git a/explorer/trace_testdata/full_trace.carbon b/explorer/trace_testdata/full_trace.carbon index 7baeecad0adc..619184430ee1 100644 --- a/explorer/trace_testdata/full_trace.carbon +++ b/explorer/trace_testdata/full_trace.carbon @@ -18,6 +18,7 @@ fn Main() -> i32 { } // Place checks after code so that line numbers are stable, reducing merge conflicts. +// ARGS: --trace_file=- --trace_phase=all %s // AUTOUPDATE // CHECK:STDOUT: ********** source program ********** // CHECK:STDOUT: interface TestInterface { diff --git a/toolchain/driver/driver_file_test_base.h b/toolchain/driver/driver_file_test_base.h index cc80d9dec17a..7247c4a337bd 100644 --- a/toolchain/driver/driver_file_test_base.h +++ b/toolchain/driver/driver_file_test_base.h @@ -8,7 +8,6 @@ #include #include -#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/MemoryBuffer.h"