Configure reentrant bison/flex (#789)

Make the parser reentrant, working towards #769 and the ability to load other packages cleanly.

executable_semantics/main.cpp

@@ -31,7 +31,7 @@ int main(int argc, char* argv[]) {
   }
 
   std::variant<Carbon::AST, Carbon::SyntaxErrorCode> ast_or_error =
-      Carbon::parse(input_file_name);
+      Carbon::Parse(input_file_name);
 
   if (auto* error = std::get_if<Carbon::SyntaxErrorCode>(&ast_or_error)) {
     // Diagnostic already reported to std::cerr; this is just a return code.

executable_semantics/syntax/BUILD

@@ -14,6 +14,7 @@ cc_library(
     name = "syntax",
     srcs = [
         "lexer.cpp",
+        "lexer.h",
         "parse.cpp",
         "parse_and_lex_context.cpp",
         "parse_and_lex_context.h",
@@ -68,9 +69,13 @@ genrule(
 genrule(
     name = "syntax_flex_srcs",
     srcs = ["lexer.lpp"],
-    outs = ["lexer.cpp"],
+    outs = [
+        "lexer.cpp",
+        "lexer.h",
+    ],
     cmd = "M4=$(M4) $(FLEX) " +
           "--outfile=$(location lexer.cpp) " +
+          "--header-file=$(location lexer.h) " +
           "$(location lexer.lpp)",
     toolchains = [
         "@rules_flex//flex:current_flex_toolchain",

executable_semantics/syntax/lexer.lpp

@@ -5,22 +5,21 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 */
 
 %{
+
 #include <cstdlib>
 
 #include "common/check.h"
 #include "common/string_helpers.h"
 #include "executable_semantics/common/tracing_flag.h"
 #include "executable_semantics/syntax/parse_and_lex_context.h"
+#include "executable_semantics/syntax/parser.h"
 #include "llvm/ADT/StringExtras.h"
 %}
 
-/* Turn off legacy bits we don't need */
+/* Turn off legacy bits we don't need. */
 %option noyywrap nounput nodefault noinput
 
-/* maintains the number of the current line read from input in the
-   global variable yylineno.
-*/
-%option yylineno
+%option reentrant
 
 /* Lexing a token immediately after consuming some whitespace. */
 %s AFTER_WHITESPACE
@@ -203,7 +202,7 @@ operator and its operand, leading to three more cases:
       // "Reading a token: ".
       llvm::errs() << "\n";
     }
-    FATAL_COMPILATION_ERROR(yylineno)
+    FATAL_COMPILATION_ERROR(context.SourceLoc())
         << "Invalid escaping in string: " << yytext;
   }
   return Carbon::Parser::make_string_literal(
@@ -237,7 +236,7 @@ operator and its operand, leading to three more cases:
     // "Reading a token: ".
     llvm::errs() << "\n";
   }
-  FATAL_COMPILATION_ERROR(yylineno) << "invalid character '\\x"
+  FATAL_COMPILATION_ERROR(context.SourceLoc()) << "invalid character '\\x"
             << llvm::toHex(llvm::StringRef(yytext, 1)) << "' in source file.";
 }
 

executable_semantics/syntax/parse.cpp

@@ -7,36 +7,49 @@
 #include "common/check.h"
 #include "executable_semantics/common/error.h"
 #include "executable_semantics/common/tracing_flag.h"
+#include "executable_semantics/syntax/lexer.h"
 #include "executable_semantics/syntax/parse_and_lex_context.h"
 #include "executable_semantics/syntax/parser.h"
 
-extern FILE* yyin;
-
 namespace Carbon {
 
 // Returns an abstract representation of the program contained in the
 // well-formed input file, or if the file was malformed, a description of the
 // problem.
-auto parse(const std::string& input_file_name)
+auto Parse(const std::string& input_file_name)
     -> std::variant<AST, SyntaxErrorCode> {
-  yyin = fopen(input_file_name.c_str(), "r");
-  if (yyin == nullptr) {
+  FILE* input_file = fopen(input_file_name.c_str(), "r");
+  if (input_file == nullptr) {
     FATAL_PROGRAM_ERROR_NO_LINE() << "Error opening '" << input_file_name
                                   << "': " << std::strerror(errno);
   }
 
+  // Prepare the lexer.
+  yyscan_t scanner;
+  yylex_init(&scanner);
+  yyset_in(input_file, scanner);
+
+  // Prepare other parser arguments.
   std::optional<AST> parsed_input = std::nullopt;
   ParseAndLexContext context(input_file_name);
 
-  auto parser = Parser(parsed_input, context);
+  // Do the parse.
+  auto parser = Parser(parsed_input, scanner, context);
   if (tracing_output) {
     parser.set_debug_level(1);
   }
   auto syntax_error_code = parser();
+
+  // Clean up the lexer.
+  fclose(input_file);
+  yylex_destroy(scanner);
+
+  // Return an error if appropriate.
   if (syntax_error_code != 0) {
     return syntax_error_code;
   }
 
+  // Return parse results.
   CHECK(parsed_input != std::nullopt)
       << "parser validated syntax yet didn't produce an AST.";
   return *parsed_input;

executable_semantics/syntax/parse.h

@@ -17,7 +17,7 @@ using SyntaxErrorCode = int;
 
 // Returns the AST representing the contents of the named file, or an error code
 // if parsing fails.
-auto parse(const std::string& input_file_name)
+auto Parse(const std::string& input_file_name)
     -> std::variant<Carbon::AST, SyntaxErrorCode>;
 
 }  // namespace Carbon

executable_semantics/syntax/parse_and_lex_context.h

@@ -40,8 +40,9 @@ class ParseAndLexContext {
 }  // namespace Carbon
 
 // Gives flex the yylex prototype we want.
-#define YY_DECL \
-  Carbon::Parser::symbol_type yylex(Carbon::ParseAndLexContext& context)
+#define YY_DECL                                         \
+  Carbon::Parser::symbol_type yylex(yyscan_t yyscanner, \
+                                    Carbon::ParseAndLexContext& context)
 
 // Declares yylex for the parser's sake.
 YY_DECL;

executable_semantics/syntax/parser.ypp

@@ -29,8 +29,10 @@
 // Enable support for parser debugging
 %define parse.trace true
 
-//
-// Parameters to the parser and lexer
+// Generate location structs.
+%locations
+
+// Parameters to the parser and lexer.
 //
 // Parameters to the parser are stored therein as protected data members, and
 // thus available to its methods.
@@ -38,7 +40,8 @@
 // "out" parameter passed to the parser, where the AST is written.
 %parse-param {std::optional<AST>& parsed_program}
 
-// "inout" parameter passed to both the parser and the lexer.
+// "inout" parameters passed to both the parser and the lexer.
+%param {yyscan_t yyscanner}
 %param {ParseAndLexContext& context}
 
 // No shift-reduce conflicts are expected.
@@ -77,6 +80,8 @@ namespace Carbon {
 class ParseAndLexContext;
 }  // namespace Carbon
 
+typedef void* yyscan_t;
+
 }  // %code requires
 
 %code {
@@ -206,7 +211,6 @@ void Carbon::Parser::error(const location_type&, const std::string& message) {
 %precedence "(" ")" "[" "]"
 
 %start input
-%locations
 %%
 input: declaration_list
     { parsed_program = $1; }

executable_semantics/testdata/invalid_char.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: 1: invalid character '\xFE' in source file.
+COMPILATION ERROR: executable_semantics/testdata/invalid_char.carbon:1: invalid character '\xFE' in source file.
 EXIT CODE: 255

executable_semantics/testdata/string_fail1.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: 6: Invalid escaping in string: "str\e"
+COMPILATION ERROR: executable_semantics/testdata/string_fail1.carbon:6: Invalid escaping in string: "str\e"
 EXIT CODE: 255

executable_semantics/testdata/string_fail2.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: 6: Invalid escaping in string: "str\x"
+COMPILATION ERROR: executable_semantics/testdata/string_fail2.carbon:6: Invalid escaping in string: "str\x"
 EXIT CODE: 255

executable_semantics/testdata/string_fail3.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: 6: Invalid escaping in string: "str\xaa"
+COMPILATION ERROR: executable_semantics/testdata/string_fail3.carbon:6: Invalid escaping in string: "str\xaa"
 EXIT CODE: 255

executable_semantics/testdata/string_fail4.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: 6: Invalid escaping in string: "str\01"
+COMPILATION ERROR: executable_semantics/testdata/string_fail4.carbon:6: Invalid escaping in string: "str\01"
 EXIT CODE: 255

executable_semantics/testdata/string_fail5.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: 6: invalid character '\x22' in source file.
+COMPILATION ERROR: executable_semantics/testdata/string_fail5.carbon:6: invalid character '\x22' in source file.
 EXIT CODE: 255

executable_semantics/testdata/string_fail6.golden

@@ -1,2 +1,2 @@
-COMPILATION ERROR: 6: invalid character '\x22' in source file.
+COMPILATION ERROR: executable_semantics/testdata/string_fail6.carbon:6: invalid character '\x22' in source file.
 EXIT CODE: 255