Migrate some lexer tests to file tests. (#2892)

These tests are doing string comparisons on output that don't seem to be meaningfully different from a file_test.

I'm tempted to migrate lexer tests in general, but I'm not doing that here since others may find more value in the current approach.

toolchain/driver/driver_test.cpp

@@ -21,15 +21,15 @@ using ::testing::ElementsAre;
 using ::testing::HasSubstr;
 using ::testing::StrEq;
 
-static constexpr llvm::StringLiteral TestFileName = "test_file.carbon";
-
 class DriverTest : public testing::Test {
  protected:
   DriverTest() : driver_(fs_, test_output_stream_, test_error_stream_) {}
 
-  auto CreateTestFile(llvm::StringRef text) -> void {
+  auto CreateTestFile(llvm::StringRef text) -> llvm::StringRef {
+    static constexpr llvm::StringLiteral TestFileName = "test_file.carbon";
     fs_.addFile(TestFileName, /*ModificationTime=*/0,
                 llvm::MemoryBuffer::getMemBuffer(text));
+    return TestFileName;
   }
 
   llvm::vfs::InMemoryFileSystem fs_;
@@ -82,9 +82,9 @@ TEST_F(DriverTest, HelpErrors) {
 }
 
 TEST_F(DriverTest, DumpTokens) {
-  CreateTestFile("Hello World");
-  EXPECT_TRUE(driver_.RunDumpSubcommand(ConsoleDiagnosticConsumer(),
-                                        {"tokens", TestFileName}));
+  auto file = CreateTestFile("Hello World");
+  EXPECT_TRUE(
+      driver_.RunDumpSubcommand(ConsoleDiagnosticConsumer(), {"tokens", file}));
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   auto tokenized_text = test_output_stream_.TakeStr();
 
@@ -114,7 +114,7 @@ TEST_F(DriverTest, DumpTokens) {
                                      {"spelling", ""}}}));
 
   // Check that the subcommand dispatch works.
-  EXPECT_TRUE(driver_.RunFullCommand({"dump", "tokens", TestFileName}));
+  EXPECT_TRUE(driver_.RunFullCommand({"dump", "tokens", file}));
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   EXPECT_THAT(test_output_stream_.TakeStr(), StrEq(tokenized_text));
 }
@@ -141,15 +141,15 @@ TEST_F(DriverTest, DumpErrors) {
 }
 
 TEST_F(DriverTest, DumpParseTree) {
-  CreateTestFile("var v: Int = 42;");
+  auto file = CreateTestFile("var v: Int = 42;");
   EXPECT_TRUE(driver_.RunDumpSubcommand(ConsoleDiagnosticConsumer(),
-                                        {"parse-tree", TestFileName}));
+                                        {"parse-tree", file}));
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   // Verify there is output without examining it.
   EXPECT_FALSE(test_output_stream_.TakeStr().empty());
 
   // Check that the subcommand dispatch works.
-  EXPECT_TRUE(driver_.RunFullCommand({"dump", "parse-tree", TestFileName}));
+  EXPECT_TRUE(driver_.RunFullCommand({"dump", "parse-tree", file}));
   EXPECT_THAT(test_error_stream_.TakeStr(), StrEq(""));
   // Verify there is output without examining it.
   EXPECT_FALSE(test_output_stream_.TakeStr().empty());

toolchain/lexer/testdata/printing_digit_padding.carbon

@@ -0,0 +1,17 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// CHECK:STDOUT: [
+
+// CHECK:STDOUT: { index: 0, kind:      'Semi', line: {{ *}}[[@LINE+1]], column:  1, indent: 1, spelling: ';', has_trailing_space: true },
+;
+
+// Test digit padding with values of 9, 10, and 11.
+        // CHECK:STDOUT: { index: 1, kind:      'Semi', line: {{ *}}[[@LINE+5]], column:  9, indent: 9, spelling: ';' },
+        // CHECK:STDOUT: { index: 2, kind:      'Semi', line: {{ *}}[[@LINE+4]], column: 10, indent: 9, spelling: ';' },
+        // CHECK:STDOUT: { index: 3, kind:      'Semi', line: {{ *}}[[@LINE+3]], column: 11, indent: 9, spelling: ';', has_trailing_space: true },
+        // CHECK:STDOUT: { index: 4, kind: 'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 9, spelling: '' },
+        // CHECK:STDOUT: ]
+        ;;;

toolchain/lexer/testdata/printing_integer_literal.carbon

@@ -0,0 +1,11 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// CHECK:STDOUT: [
+
+// CHECK:STDOUT: { index: 0, kind: 'IntegerLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '123', value: `123`, has_trailing_space: true },
+// CHECK:STDOUT: { index: 1, kind:      'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' },
+// CHECK:STDOUT: ]
+123

toolchain/lexer/testdata/printing_real_literal.carbon

@@ -0,0 +1,11 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// CHECK:STDOUT: [
+
+// CHECK:STDOUT: { index: 0, kind: 'RealLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true },
+// CHECK:STDOUT: { index: 1, kind:   'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' },
+// CHECK:STDOUT: ]
+2.5

toolchain/lexer/testdata/printing_token.carbon

@@ -0,0 +1,11 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// CHECK:STDOUT: [
+
+// CHECK:STDOUT: { index: 0, kind: 'IntegerLiteral', line: {{ *}}[[@LINE+3]], column: 1, indent: 1, spelling: '0x9', value: `9`, has_trailing_space: true },
+// CHECK:STDOUT: { index: 1, kind:      'EndOfFile', line: {{ *}}[[@LINE+2]], column: {{[0-9]+}}, indent: 1, spelling: '' },
+// CHECK:STDOUT: ]
+0x9

toolchain/lexer/tokenized_buffer_test.cpp

@@ -32,12 +32,12 @@ using ::testing::StrEq;
 class LexerTest : public ::testing::Test {
  protected:
   auto GetSourceBuffer(llvm::StringRef text) -> SourceBuffer& {
-    std::string filename = llvm::formatv("test{0}.carbon", ++file_index);
-    CARBON_CHECK(fs.addFile(filename, /*ModificationTime=*/0,
-                            llvm::MemoryBuffer::getMemBuffer(text)));
-    source_storage.push_front(
-        std::move(*SourceBuffer::CreateFromFile(fs, filename)));
-    return source_storage.front();
+    std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
+    CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
+                             llvm::MemoryBuffer::getMemBuffer(text)));
+    source_storage_.push_front(
+        std::move(*SourceBuffer::CreateFromFile(fs_, filename)));
+    return source_storage_.front();
   }
 
   auto Lex(llvm::StringRef text,
@@ -46,9 +46,9 @@ class LexerTest : public ::testing::Test {
     return TokenizedBuffer::Lex(GetSourceBuffer(text), consumer);
   }
 
-  llvm::vfs::InMemoryFileSystem fs;
-  int file_index = 0;
-  std::forward_list<SourceBuffer> source_storage;
+  llvm::vfs::InMemoryFileSystem fs_;
+  int file_index_ = 0;
+  std::forward_list<SourceBuffer> source_storage_;
 };
 
 TEST_F(LexerTest, HandlesEmptyBuffer) {
@@ -995,111 +995,6 @@ TEST_F(LexerTest, DiagnosticUnrecognizedChar) {
   Lex("\b", consumer);
 }
 
-auto GetAndDropLine(llvm::StringRef& text) -> std::string {
-  auto newline_offset = text.find_first_of('\n');
-  llvm::StringRef line = text.slice(0, newline_offset);
-
-  if (newline_offset != llvm::StringRef::npos) {
-    text = text.substr(newline_offset + 1);
-  } else {
-    text = "";
-  }
-
-  return line.str();
-}
-
-TEST_F(LexerTest, PrintingInteger) {
-  auto buffer = Lex("123");
-  ASSERT_FALSE(buffer.has_errors());
-  std::string print_storage;
-  llvm::raw_string_ostream print_stream(print_storage);
-  buffer.Print(print_stream);
-  llvm::StringRef print = print_stream.str();
-  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
-                    "column: 1, indent: 1, spelling: '123', value: `123`, "
-                    "has_trailing_space: true },"));
-  EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
-  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
-  EXPECT_TRUE(print.empty()) << print;
-}
-
-TEST_F(LexerTest, PrintingReal) {
-  auto buffer = Lex("2.5");
-  ASSERT_FALSE(buffer.has_errors());
-  std::string print_storage;
-  llvm::raw_string_ostream print_stream(print_storage);
-  buffer.Print(print_stream);
-  llvm::StringRef print = print_stream.str();
-  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
-  EXPECT_THAT(
-      GetAndDropLine(print),
-      StrEq("{ index: 0, kind: 'RealLiteral', line: 1, column: 1, indent: "
-            "1, spelling: '2.5', value: `25*10^-1`, has_trailing_space: true "
-            "},"));
-  EXPECT_THAT(GetAndDropLine(print), HasSubstr("'EndOfFile'"));
-  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
-  EXPECT_TRUE(print.empty()) << print;
-}
-
-TEST_F(LexerTest, PrintingPadding) {
-  // Test kind padding.
-  auto buffer = Lex("(;foo;)");
-  ASSERT_FALSE(buffer.has_errors());
-  std::string print_storage;
-  llvm::raw_string_ostream print_stream(print_storage);
-  buffer.Print(print_stream);
-  llvm::StringRef print = print_stream.str();
-  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 0, kind:  'OpenParen', line: 1, column: "
-                    "1, indent: 1, spelling: '(', closing_token: 4 },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 1, kind:       'Semi', line: 1, column: "
-                    "2, indent: 1, spelling: ';' },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 2, kind: 'Identifier', line: 1, column: "
-                    "3, indent: 1, spelling: 'foo', identifier: 0 },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 3, kind:       'Semi', line: 1, column: "
-                    "6, indent: 1, spelling: ';' },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 4, kind: 'CloseParen', line: 1, column: "
-                    "7, indent: 1, spelling: ')', opening_token: 0, "
-                    "has_trailing_space: true },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 5, kind:  'EndOfFile', line: 1, column: "
-                    "8, indent: 1, spelling: '' },"));
-  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
-  EXPECT_TRUE(print.empty()) << print;
-}
-
-TEST_F(LexerTest, PrintingPaddingDigits) {
-  // Test digit padding with max values of 9, 10, and 11.
-  auto buffer = Lex(";\n\n\n\n\n\n\n\n\n\n        ;;");
-  ASSERT_FALSE(buffer.has_errors());
-  std::string print_storage;
-  llvm::raw_string_ostream print_stream(print_storage);
-  buffer.Print(print_stream);
-  llvm::StringRef print = print_stream.str();
-  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 0, kind:      'Semi', line:  1, column:  1, "
-                    "indent: 1, spelling: ';', has_trailing_space: true },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 1, kind:      'Semi', line: 11, column:  9, "
-                    "indent: 9, spelling: ';' },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 2, kind:      'Semi', line: 11, column: 10, "
-                    "indent: 9, spelling: ';', has_trailing_space: true },"));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 3, kind: 'EndOfFile', line: 11, column: 11, "
-                    "indent: 9, spelling: '' },"));
-  EXPECT_THAT(GetAndDropLine(print), StrEq("]"));
-  EXPECT_TRUE(print.empty()) << print;
-}
-
 TEST_F(LexerTest, PrintingAsYaml) {
   // Test that we can parse this into YAML and verify line and indent data.
   auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n");
@@ -1138,19 +1033,5 @@ TEST_F(LexerTest, PrintingAsYaml) {
                                      {"spelling", ""}}}));
 }
 
-TEST_F(LexerTest, PrintToken) {
-  auto buffer = Lex("0x9");
-  ASSERT_FALSE(buffer.has_errors());
-  std::string print_output;
-  llvm::raw_string_ostream print_stream(print_output);
-  buffer.Print(print_stream);
-  llvm::StringRef print = print_stream.str();
-  EXPECT_THAT(GetAndDropLine(print), StrEq("["));
-  EXPECT_THAT(GetAndDropLine(print),
-              StrEq("{ index: 0, kind: 'IntegerLiteral', line: 1, "
-                    "column: 1, indent: 1, spelling: '0x9', value: `9`, "
-                    "has_trailing_space: true },"));
-}
-
 }  // namespace
 }  // namespace Carbon::Testing