From ed2d171703eaf053c355c972fc9fa06e39d50ba1 Mon Sep 17 00:00:00 2001 From: Jon Meow <46229924+jonmeow@users.noreply.github.com> Date: Mon, 30 Aug 2021 14:23:16 -0700 Subject: [PATCH] Configure reentrant bison/flex (#789) Make the parser reentrant, working towards #769 and the ability to load other packages cleanly. --- executable_semantics/main.cpp | 2 +- executable_semantics/syntax/BUILD | 7 +++++- executable_semantics/syntax/lexer.lpp | 13 +++++----- executable_semantics/syntax/parse.cpp | 25 ++++++++++++++----- executable_semantics/syntax/parse.h | 2 +- .../syntax/parse_and_lex_context.h | 5 ++-- executable_semantics/syntax/parser.ypp | 12 ++++++--- .../testdata/invalid_char.golden | 2 +- .../testdata/string_fail1.golden | 2 +- .../testdata/string_fail2.golden | 2 +- .../testdata/string_fail3.golden | 2 +- .../testdata/string_fail4.golden | 2 +- .../testdata/string_fail5.golden | 2 +- .../testdata/string_fail6.golden | 2 +- 14 files changed, 51 insertions(+), 29 deletions(-) diff --git a/executable_semantics/main.cpp b/executable_semantics/main.cpp index 30ca2d0529bd..05ed40557735 100644 --- a/executable_semantics/main.cpp +++ b/executable_semantics/main.cpp @@ -31,7 +31,7 @@ int main(int argc, char* argv[]) { } std::variant ast_or_error = - Carbon::parse(input_file_name); + Carbon::Parse(input_file_name); if (auto* error = std::get_if(&ast_or_error)) { // Diagnostic already reported to std::cerr; this is just a return code. diff --git a/executable_semantics/syntax/BUILD b/executable_semantics/syntax/BUILD index 7eefbf921768..713a7ac3649b 100644 --- a/executable_semantics/syntax/BUILD +++ b/executable_semantics/syntax/BUILD @@ -14,6 +14,7 @@ cc_library( name = "syntax", srcs = [ "lexer.cpp", + "lexer.h", "parse.cpp", "parse_and_lex_context.cpp", "parse_and_lex_context.h", @@ -68,9 +69,13 @@ genrule( genrule( name = "syntax_flex_srcs", srcs = ["lexer.lpp"], - outs = ["lexer.cpp"], + outs = [ + "lexer.cpp", + "lexer.h", + ], cmd = "M4=$(M4) $(FLEX) " + "--outfile=$(location lexer.cpp) " + + "--header-file=$(location lexer.h) " + "$(location lexer.lpp)", toolchains = [ "@rules_flex//flex:current_flex_toolchain", diff --git a/executable_semantics/syntax/lexer.lpp b/executable_semantics/syntax/lexer.lpp index 4507900f902f..c9f89408cebd 100644 --- a/executable_semantics/syntax/lexer.lpp +++ b/executable_semantics/syntax/lexer.lpp @@ -5,22 +5,21 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ %{ + #include #include "common/check.h" #include "common/string_helpers.h" #include "executable_semantics/common/tracing_flag.h" #include "executable_semantics/syntax/parse_and_lex_context.h" +#include "executable_semantics/syntax/parser.h" #include "llvm/ADT/StringExtras.h" %} -/* Turn off legacy bits we don't need */ +/* Turn off legacy bits we don't need. */ %option noyywrap nounput nodefault noinput -/* maintains the number of the current line read from input in the - global variable yylineno. -*/ -%option yylineno +%option reentrant /* Lexing a token immediately after consuming some whitespace. */ %s AFTER_WHITESPACE @@ -203,7 +202,7 @@ operator and its operand, leading to three more cases: // "Reading a token: ". llvm::errs() << "\n"; } - FATAL_COMPILATION_ERROR(yylineno) + FATAL_COMPILATION_ERROR(context.SourceLoc()) << "Invalid escaping in string: " << yytext; } return Carbon::Parser::make_string_literal( @@ -237,7 +236,7 @@ operator and its operand, leading to three more cases: // "Reading a token: ". llvm::errs() << "\n"; } - FATAL_COMPILATION_ERROR(yylineno) << "invalid character '\\x" + FATAL_COMPILATION_ERROR(context.SourceLoc()) << "invalid character '\\x" << llvm::toHex(llvm::StringRef(yytext, 1)) << "' in source file."; } diff --git a/executable_semantics/syntax/parse.cpp b/executable_semantics/syntax/parse.cpp index 283442d252d0..2e21c2788335 100644 --- a/executable_semantics/syntax/parse.cpp +++ b/executable_semantics/syntax/parse.cpp @@ -7,36 +7,49 @@ #include "common/check.h" #include "executable_semantics/common/error.h" #include "executable_semantics/common/tracing_flag.h" +#include "executable_semantics/syntax/lexer.h" #include "executable_semantics/syntax/parse_and_lex_context.h" #include "executable_semantics/syntax/parser.h" -extern FILE* yyin; - namespace Carbon { // Returns an abstract representation of the program contained in the // well-formed input file, or if the file was malformed, a description of the // problem. -auto parse(const std::string& input_file_name) +auto Parse(const std::string& input_file_name) -> std::variant { - yyin = fopen(input_file_name.c_str(), "r"); - if (yyin == nullptr) { + FILE* input_file = fopen(input_file_name.c_str(), "r"); + if (input_file == nullptr) { FATAL_PROGRAM_ERROR_NO_LINE() << "Error opening '" << input_file_name << "': " << std::strerror(errno); } + // Prepare the lexer. + yyscan_t scanner; + yylex_init(&scanner); + yyset_in(input_file, scanner); + + // Prepare other parser arguments. std::optional parsed_input = std::nullopt; ParseAndLexContext context(input_file_name); - auto parser = Parser(parsed_input, context); + // Do the parse. + auto parser = Parser(parsed_input, scanner, context); if (tracing_output) { parser.set_debug_level(1); } auto syntax_error_code = parser(); + + // Clean up the lexer. + fclose(input_file); + yylex_destroy(scanner); + + // Return an error if appropriate. if (syntax_error_code != 0) { return syntax_error_code; } + // Return parse results. CHECK(parsed_input != std::nullopt) << "parser validated syntax yet didn't produce an AST."; return *parsed_input; diff --git a/executable_semantics/syntax/parse.h b/executable_semantics/syntax/parse.h index 1b4936fb54dc..19e9edde257b 100644 --- a/executable_semantics/syntax/parse.h +++ b/executable_semantics/syntax/parse.h @@ -17,7 +17,7 @@ using SyntaxErrorCode = int; // Returns the AST representing the contents of the named file, or an error code // if parsing fails. -auto parse(const std::string& input_file_name) +auto Parse(const std::string& input_file_name) -> std::variant; } // namespace Carbon diff --git a/executable_semantics/syntax/parse_and_lex_context.h b/executable_semantics/syntax/parse_and_lex_context.h index 658fdeea7008..46616b04bc50 100644 --- a/executable_semantics/syntax/parse_and_lex_context.h +++ b/executable_semantics/syntax/parse_and_lex_context.h @@ -40,8 +40,9 @@ class ParseAndLexContext { } // namespace Carbon // Gives flex the yylex prototype we want. -#define YY_DECL \ - Carbon::Parser::symbol_type yylex(Carbon::ParseAndLexContext& context) +#define YY_DECL \ + Carbon::Parser::symbol_type yylex(yyscan_t yyscanner, \ + Carbon::ParseAndLexContext& context) // Declares yylex for the parser's sake. YY_DECL; diff --git a/executable_semantics/syntax/parser.ypp b/executable_semantics/syntax/parser.ypp index 4739e7fb8e9a..a80d09c4ef40 100644 --- a/executable_semantics/syntax/parser.ypp +++ b/executable_semantics/syntax/parser.ypp @@ -29,8 +29,10 @@ // Enable support for parser debugging %define parse.trace true -// -// Parameters to the parser and lexer +// Generate location structs. +%locations + +// Parameters to the parser and lexer. // // Parameters to the parser are stored therein as protected data members, and // thus available to its methods. @@ -38,7 +40,8 @@ // "out" parameter passed to the parser, where the AST is written. %parse-param {std::optional& parsed_program} -// "inout" parameter passed to both the parser and the lexer. +// "inout" parameters passed to both the parser and the lexer. +%param {yyscan_t yyscanner} %param {ParseAndLexContext& context} // No shift-reduce conflicts are expected. @@ -77,6 +80,8 @@ namespace Carbon { class ParseAndLexContext; } // namespace Carbon +typedef void* yyscan_t; + } // %code requires %code { @@ -206,7 +211,6 @@ void Carbon::Parser::error(const location_type&, const std::string& message) { %precedence "(" ")" "[" "]" %start input -%locations %% input: declaration_list { parsed_program = $1; } diff --git a/executable_semantics/testdata/invalid_char.golden b/executable_semantics/testdata/invalid_char.golden index ffaf1a0d2b1e..46c5219e3de6 100644 --- a/executable_semantics/testdata/invalid_char.golden +++ b/executable_semantics/testdata/invalid_char.golden @@ -1,2 +1,2 @@ -COMPILATION ERROR: 1: invalid character '\xFE' in source file. +COMPILATION ERROR: executable_semantics/testdata/invalid_char.carbon:1: invalid character '\xFE' in source file. EXIT CODE: 255 diff --git a/executable_semantics/testdata/string_fail1.golden b/executable_semantics/testdata/string_fail1.golden index e5cccb823fb2..1f6df390396b 100644 --- a/executable_semantics/testdata/string_fail1.golden +++ b/executable_semantics/testdata/string_fail1.golden @@ -1,2 +1,2 @@ -COMPILATION ERROR: 6: Invalid escaping in string: "str\e" +COMPILATION ERROR: executable_semantics/testdata/string_fail1.carbon:6: Invalid escaping in string: "str\e" EXIT CODE: 255 diff --git a/executable_semantics/testdata/string_fail2.golden b/executable_semantics/testdata/string_fail2.golden index cdb87bd14a6a..2b97b5f0cfc6 100644 --- a/executable_semantics/testdata/string_fail2.golden +++ b/executable_semantics/testdata/string_fail2.golden @@ -1,2 +1,2 @@ -COMPILATION ERROR: 6: Invalid escaping in string: "str\x" +COMPILATION ERROR: executable_semantics/testdata/string_fail2.carbon:6: Invalid escaping in string: "str\x" EXIT CODE: 255 diff --git a/executable_semantics/testdata/string_fail3.golden b/executable_semantics/testdata/string_fail3.golden index f3329845368c..9b594ffd8f79 100644 --- a/executable_semantics/testdata/string_fail3.golden +++ b/executable_semantics/testdata/string_fail3.golden @@ -1,2 +1,2 @@ -COMPILATION ERROR: 6: Invalid escaping in string: "str\xaa" +COMPILATION ERROR: executable_semantics/testdata/string_fail3.carbon:6: Invalid escaping in string: "str\xaa" EXIT CODE: 255 diff --git a/executable_semantics/testdata/string_fail4.golden b/executable_semantics/testdata/string_fail4.golden index 2da23bb3047d..8d1f9466af7f 100644 --- a/executable_semantics/testdata/string_fail4.golden +++ b/executable_semantics/testdata/string_fail4.golden @@ -1,2 +1,2 @@ -COMPILATION ERROR: 6: Invalid escaping in string: "str\01" +COMPILATION ERROR: executable_semantics/testdata/string_fail4.carbon:6: Invalid escaping in string: "str\01" EXIT CODE: 255 diff --git a/executable_semantics/testdata/string_fail5.golden b/executable_semantics/testdata/string_fail5.golden index c2abd374a6be..cb2e7da40069 100644 --- a/executable_semantics/testdata/string_fail5.golden +++ b/executable_semantics/testdata/string_fail5.golden @@ -1,2 +1,2 @@ -COMPILATION ERROR: 6: invalid character '\x22' in source file. +COMPILATION ERROR: executable_semantics/testdata/string_fail5.carbon:6: invalid character '\x22' in source file. EXIT CODE: 255 diff --git a/executable_semantics/testdata/string_fail6.golden b/executable_semantics/testdata/string_fail6.golden index c2abd374a6be..ded07e79810e 100644 --- a/executable_semantics/testdata/string_fail6.golden +++ b/executable_semantics/testdata/string_fail6.golden @@ -1,2 +1,2 @@ -COMPILATION ERROR: 6: invalid character '\x22' in source file. +COMPILATION ERROR: executable_semantics/testdata/string_fail6.carbon:6: invalid character '\x22' in source file. EXIT CODE: 255