[executable semantics] Add Syntax driver (#362)

Slowly bringing this into line with Bison's C++ example parser
so we can use strong semantic values for symbols rather than
leaking pointers.  First step is to thread a `ParseAndLexContext` 
object through the whole syntactic analysis state, like the 
example has.  In the example, it's called `driver`.

Co-authored-by: Geoff Romer <gromer@google.com>

executable_semantics/ast/BUILD

@@ -9,7 +9,10 @@ package(default_visibility = ["//executable_semantics:__subpackages__"])
 cc_library(
     name = "declaration",
     srcs = ["declaration.cpp"],
-    hdrs = ["declaration.h"],
+    hdrs = [
+        "abstract_syntax_tree.h",
+        "declaration.h",
+    ],
     deps = [
         ":function_definition",
         ":member",

executable_semantics/ast/abstract_syntax_tree.h

@@ -0,0 +1,16 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef EXECUTABLE_SEMANTICS_AST_ABSTRACT_SYNTAX_TREE_H_
+#define EXECUTABLE_SEMANTICS_AST_ABSTRACT_SYNTAX_TREE_H_
+
+#include <variant>
+
+#include "executable_semantics/ast/declaration.h"
+
+namespace Carbon {
+using AST = std::list<Carbon::Declaration>*;
+}
+
+#endif  // EXECUTABLE_SEMANTICS_AST_ABSTRACT_SYNTAX_TREE_H_

executable_semantics/main.cpp

@@ -6,50 +6,33 @@
 #include <cstring>
 #include <iostream>
 
-#include "executable_semantics/syntax/parser.h"
+#include "executable_semantics/syntax/parse.h"
 #include "executable_semantics/syntax/syntax_helpers.h"
 #include "executable_semantics/tracing_flag.h"
 #include "llvm/Support/CommandLine.h"
 
-extern FILE* yyin;
-
 int main(int argc, char* argv[]) {
   // yydebug = 1;
 
   using llvm::cl::desc;
   using llvm::cl::opt;
-  opt<bool> quiet_option("quiet", desc("Disable tracing"));
-  opt<std::string> input_filename(llvm::cl::Positional, desc("<input file>"));
-  llvm::cl::ParseCommandLineOptions(argc, argv);
+  opt<bool> quietOption("quiet", desc("Disable tracing"));
+  opt<std::string> inputFileName(llvm::cl::Positional, desc("<input file>"),
+                                 llvm::cl::Required);
 
-  if (input_filename.getNumOccurrences() > 0) {
-    Carbon::input_filename = input_filename.c_str();
-    yyin = fopen(input_filename.c_str(), "r");
-    if (yyin == nullptr) {
-      std::cerr << "Error opening '" << input_filename
-                << "': " << strerror(errno) << std::endl;
-      return 1;
-    }
-  }
-  if (quiet_option) {
+  llvm::cl::ParseCommandLineOptions(argc, argv);
+  if (quietOption) {
     Carbon::tracing_output = false;
   }
 
-  // No AST yet
-  std::optional<Carbon::AST> parsedInput = std::nullopt;
+  std::variant<Carbon::AST, Carbon::SyntaxErrorCode> astOrError =
+      Carbon::parse(inputFileName);
 
-  // Parse and handle syntax errors
-  auto syntaxErrorCode = yyparse(parsedInput);
-  if (syntaxErrorCode != 0) {
-    return syntaxErrorCode;
-  }
-
-  if (parsedInput == std::nullopt) {
-    std::cerr << "Internal error: parser validated syntax yet didn't produce "
-                 "an AST.\n";
-    return 1;
+  if (auto* error = std::get_if<Carbon::SyntaxErrorCode>(&astOrError)) {
+    // Diagnostic already reported to std::cerr; this is just a return code.
+    return *error;
   }
 
   // Typecheck and run the parsed program.
-  Carbon::ExecProgram(*parsedInput);
+  Carbon::ExecProgram(std::get<Carbon::AST>(astOrError));
 }

executable_semantics/syntax/BUILD

@@ -13,11 +13,17 @@ cc_library(
     name = "syntax",
     srcs = [
         "lexer.cpp",
+        "parse.cpp",
+        "parse_and_lex_context.cpp",
+        "parse_and_lex_context.h",
         "parser.cpp",
+        "parser.h",
         "syntax_helpers.cpp",
         "syntax_helpers.h",
     ],
-    hdrs = ["parser.h"],
+    hdrs = [
+        "parse.h",
+    ],
     # Disable warnings for generated code.
     copts = [
         "-Wno-unneeded-internal-declaration",

executable_semantics/syntax/lexer.lpp

@@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 %{
 #include <cstdlib>
-#include "executable_semantics/syntax/parser.h"
+#include "executable_semantics/syntax/parse_and_lex_context.h"
 %}
 
 /* Turn off legacy bits we don't need */

executable_semantics/syntax/parse.cpp

@@ -0,0 +1,44 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "executable_semantics/syntax/parse.h"
+
+#include <iostream>
+
+#include "executable_semantics/syntax/parse_and_lex_context.h"
+#include "executable_semantics/tracing_flag.h"
+
+extern FILE* yyin;
+
+namespace Carbon {
+
+// Returns an abstract representation of the program contained in the
+// well-formed input file, or if the file was malformed, a description of the
+// problem.
+auto parse(const std::string& inputFileName)
+    -> std::variant<AST, SyntaxErrorCode> {
+  yyin = fopen(inputFileName.c_str(), "r");
+  if (yyin == nullptr) {
+    std::cerr << "Error opening '" << inputFileName
+              << "': " << std::strerror(errno) << std::endl;
+    exit(1);
+  }
+
+  std::optional<AST> parsedInput = std::nullopt;
+  ParseAndLexContext context(inputFileName);
+
+  auto syntaxErrorCode = yyparse(parsedInput, context);
+  if (syntaxErrorCode != 0) {
+    return syntaxErrorCode;
+  }
+
+  if (parsedInput == std::nullopt) {
+    std::cerr << "Internal error: parser validated syntax yet didn't produce "
+                 "an AST.\n";
+    exit(1);
+  }
+  return *parsedInput;
+}
+
+}  // namespace Carbon

executable_semantics/syntax/parse.h

@@ -0,0 +1,25 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef EXECUTABLE_SEMANTICS_SYNTAX_PARSE_H_
+#define EXECUTABLE_SEMANTICS_SYNTAX_PARSE_H_
+
+#include <string>
+#include <variant>
+
+#include "executable_semantics/ast/abstract_syntax_tree.h"
+
+namespace Carbon {
+
+// This is the code given us by Bison, for now.
+using SyntaxErrorCode = int;
+
+// Returns the AST representing the contents of the named file, or an error code
+// if parsing fails.
+auto parse(const std::string& inputFileName)
+    -> std::variant<Carbon::AST, SyntaxErrorCode>;
+
+}  // namespace Carbon
+
+#endif  // EXECUTABLE_SEMANTICS_SYNTAX_PARSE_H_

executable_semantics/syntax/parse_and_lex_context.cpp

@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "executable_semantics/syntax/parse_and_lex_context.h"
+
+#include <cstring>
+#include <iostream>
+
+#include "executable_semantics/tracing_flag.h"
+
+// Writes a syntax error diagnostic, containing message, for the input file at
+// the given line, to standard error.
+auto Carbon::ParseAndLexContext::PrintDiagnostic(const char* message,
+                                                 int line_num) -> void {
+  std::cerr << inputFileName << ":" << line_num << ": " << message << std::endl;
+  exit(-1);  // TODO: do we really want this here?  It makes the comment and the
+             // name a lie, and renders some of the other yyparse() result
+             // propagation code moot.
+}

executable_semantics/syntax/parse_and_lex_context.h

@@ -0,0 +1,40 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef EXECUTABLE_SEMANTICS_SYNTAX_DRIVER_H_
+#define EXECUTABLE_SEMANTICS_SYNTAX_DRIVER_H_
+
+#include <variant>
+
+#include "executable_semantics/ast/abstract_syntax_tree.h"
+#include "executable_semantics/syntax/parser.h"  // from parser.ypp
+
+namespace Carbon {
+
+// The state and functionality that is threaded "globally" through the
+// lexing/parsing process.
+class ParseAndLexContext {
+ public:
+  // Creates an instance analyzing the given input file.
+  ParseAndLexContext(const std::string& inputFile) : inputFileName(inputFile) {}
+
+  // Writes a syntax error diagnostic, containing message, for the input file at
+  // the given line, to standard error.
+  auto PrintDiagnostic(const char* message, int lineNumber) -> void;
+
+ private:
+  // A path to the file processed, relative to the current working directory
+  // when *this is called.
+  const std::string inputFileName;
+};
+
+}  // namespace Carbon
+
+// Gives flex the yylex prototype we want.
+#define YY_DECL int yylex(Carbon::ParseAndLexContext& context)
+
+// Declares yylex for the parser's sake.
+YY_DECL;
+
+#endif  // EXECUTABLE_SYNTAX_DRIVER_H_

executable_semantics/syntax/parser.ypp

@@ -11,33 +11,46 @@
 #include <list>
 
 #include "executable_semantics/syntax/syntax_helpers.h"
+#include "executable_semantics/syntax/parse_and_lex_context.h"
 }
 
 %code requires {
 #include <optional>
 
+#include "executable_semantics/ast/abstract_syntax_tree.h"
 #include "executable_semantics/ast/declaration.h"
 #include "executable_semantics/ast/field_list.h"
 #include "executable_semantics/ast/function_definition.h"
 
 namespace Carbon {
-// A representation of the parsed program.
-using AST = std::list<Carbon::Declaration>*;
+class ParseAndLexContext;
 }
 
 }
 
 %code {
+
 extern int yylineno;
-extern int yylex();
 
-void yyerror(std::optional<Carbon::AST> const& parsedProgram, char* error) {
-  Carbon::PrintSyntaxError(error, yylineno);
+void yyerror(
+    std::optional<Carbon::AST> const& /* parsedProgram */,
+    Carbon::ParseAndLexContext& context,
+    char* message
+) {
+  context.PrintDiagnostic(message, yylineno);
 }
 }
 
+//
+// Parameters to yyparse and yylex.
+//
+
+// "out" parameter passed to the parser, where the AST is written.
 %parse-param {std::optional<Carbon::AST>& parsedProgram}
 
+// "inout" parameter passed to both the parser and the lexer.
+%param {Carbon::ParseAndLexContext& context}
+
 %union {
    char* str;
    int num;

executable_semantics/syntax/syntax_helpers.cpp

@@ -12,13 +12,6 @@
 
 namespace Carbon {
 
-const char* input_filename = nullptr;
-
-void PrintSyntaxError(char* error, int line_num) {
-  std::cerr << input_filename << ":" << line_num << ": " << error << std::endl;
-  exit(-1);
-}
-
 void ExecProgram(std::list<Declaration>* fs) {
   if (tracing_output) {
     std::cout << "********** source program **********" << std::endl;

executable_semantics/syntax/syntax_helpers.h

@@ -15,12 +15,6 @@
 
 namespace Carbon {
 
-// Initialized by main(), and used when printing errors.
-extern const char* input_filename;
-
-// Prints errors.
-void PrintSyntaxError(char* error, int line_num);
-
 // Runs the top-level declaration list.
 void ExecProgram(std::list<Declaration>* fs);