Add a framework for LSP testing. (#4841)

Also adds tests for exit and initialize, just basic things.

testing/file_test/README.md

@@ -86,6 +86,12 @@ they have an associated error. An exception is that the main test file may omit
 
 Some keywords can be inserted for content:
 
+-   ```
+    [[@LSP:<method>:<extra content>]]
+    ```
+
+    Produces JSON for an LSP method call, complete with `Content-Length` header.
+
 -   ```
     [[@TEST_NAME]]
     ```

testing/file_test/file_test_base.cpp

@@ -344,13 +344,16 @@ auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
   llvm::PrettyStackTraceProgram stack_trace_entry(
       test_argv_for_stack_trace.size() - 1, test_argv_for_stack_trace.data());
 
+  // Execution must be serialized for either serial tests or console output.
+  std::unique_lock<std::mutex> output_lock;
+  if (output_mutex_ &&
+      (context.capture_console_output || !AllowParallelRun())) {
+    output_lock = std::unique_lock<std::mutex>(*output_mutex_);
+  }
+
   // Conditionally capture console output. We use a scope exit to ensure the
   // captures terminate even on run failures.
-  std::unique_lock<std::mutex> output_lock;
   if (context.capture_console_output) {
-    if (output_mutex_) {
-      output_lock = std::unique_lock<std::mutex>(*output_mutex_);
-    }
     CaptureStderr();
     CaptureStdout();
   }
@@ -514,6 +517,84 @@ struct SplitState {
   int file_index = 0;
 };
 
+// Replaces the keyword at the given position. Returns the position to start a
+// find for the next keyword.
+static auto ReplaceContentKeywordAt(std::string* content, size_t keyword_pos,
+                                    llvm::StringRef test_name, int* lsp_id)
+    -> ErrorOr<size_t> {
+  auto keyword = llvm::StringRef(*content).substr(keyword_pos);
+
+  // Line replacements aren't handled here.
+  static constexpr llvm::StringLiteral Line = "[[@LINE";
+  if (keyword.starts_with(Line)) {
+    // Just move past the prefix to find the next one.
+    return keyword_pos + Line.size();
+  }
+
+  // Replaced with the actual test name.
+  static constexpr llvm::StringLiteral TestName = "[[@TEST_NAME]]";
+  if (keyword.starts_with(TestName)) {
+    content->replace(keyword_pos, TestName.size(), test_name);
+    return keyword_pos + test_name.size();
+  }
+
+  // Reformatted as an LSP call with headers.
+  static constexpr llvm::StringLiteral Lsp = "[[@LSP:";
+  if (keyword.starts_with(Lsp)) {
+    auto method_start = keyword_pos + Lsp.size();
+
+    static constexpr llvm::StringLiteral LspEnd = "]]";
+    auto keyword_end = content->find("]]", method_start);
+    if (keyword_end == std::string::npos) {
+      return ErrorBuilder()
+             << "Missing `" << LspEnd << "` after `" << Lsp << "`";
+    }
+
+    auto method_end = content->find(":", method_start);
+    auto extra_content_start = method_end + 1;
+    if (method_end == std::string::npos || method_end > keyword_end) {
+      method_end = keyword_end;
+      extra_content_start = keyword_end;
+    }
+    auto method = content->substr(method_start, method_end - method_start);
+
+    auto extra_content =
+        content->substr(extra_content_start, keyword_end - extra_content_start);
+    std::string extra_content_sep;
+    if (!extra_content.empty()) {
+      extra_content_sep = ",";
+      if (!extra_content.starts_with("\n")) {
+        extra_content_sep += " ";
+      }
+    }
+
+    // Form the JSON.
+    std::string json;
+    if (method == "exit") {
+      if (!extra_content.empty()) {
+        return Error("`[[@LSP:exit:` cannot include extra content");
+      }
+      json = R"({"jsonrpc": "2.0", "method": "exit"})";
+    } else {
+      json = llvm::formatv(
+                 R"({{"jsonrpc": "2.0", "id": "{0}", "method": "{1}"{2}{3}})",
+                 ++(*lsp_id), method, extra_content_sep, extra_content)
+                 .str();
+    }
+    // Add the Content-Length header. The `2` accounts for extra newlines.
+    auto json_with_header =
+        llvm::formatv("Content-Length: {0}\n\n{1}\n", json.size() + 2, json)
+            .str();
+    // Insert the content.
+    content->replace(keyword_pos, keyword_end + 2 - keyword_pos,
+                     json_with_header);
+    return keyword_pos + json_with_header.size();
+  }
+
+  return ErrorBuilder() << "Unexpected use of `[[@` at `"
+                        << keyword.substr(0, 5) << "`";
+}
+
 // Replaces the content keywords.
 //
 // TEST_NAME is the only content keyword at present, but we do validate that
@@ -543,20 +624,13 @@ static auto ReplaceContentKeywords(llvm::StringRef filename,
   test_name.consume_front("fail_");
   test_name.consume_front("todo_");
 
+  // A counter for LSP calls.
+  int lsp_id = 0;
   while (keyword_pos != std::string::npos) {
-    static constexpr llvm::StringLiteral TestName = "[[@TEST_NAME]]";
-    auto keyword = llvm::StringRef(*content).substr(keyword_pos);
-    if (keyword.starts_with(TestName)) {
-      content->replace(keyword_pos, TestName.size(), test_name);
-      keyword_pos += test_name.size();
-    } else if (keyword.starts_with("[[@LINE")) {
-      // Just move past the prefix to find the next one.
-      keyword_pos += Prefix.size();
-    } else {
-      return ErrorBuilder()
-             << "Unexpected use of `[[@` at `" << keyword.substr(0, 5) << "`";
-    }
-    keyword_pos = content->find(Prefix, keyword_pos);
+    CARBON_ASSIGN_OR_RETURN(
+        auto keyword_end,
+        ReplaceContentKeywordAt(content, keyword_pos, test_name, &lsp_id));
+    keyword_pos = content->find(Prefix, keyword_end);
   }
   return Success();
 }

testing/file_test/file_test_base.h

@@ -116,6 +116,12 @@ class FileTestBase : public testing::Test {
   // Optionally allows children to provide extra replacements for autoupdate.
   virtual auto DoExtraCheckReplacements(std::string& /*check_line*/) -> void {}
 
+  // Whether to allow running the test in parallel, particularly for autoupdate.
+  // This can be overridden to force some tests to be run serially. At any given
+  // time, all parallel tests and a single non-parallel test will be allowed to
+  // run.
+  virtual auto AllowParallelRun() const -> bool { return true; }
+
   // Runs a test and compares output. This keeps output split by line so that
   // issues are a little easier to identify by the different line.
   auto TestBody() -> void final;

testing/file_test/file_test_base_test.cpp

@@ -198,21 +198,6 @@ static auto TestEscaping(TestParams& params)
   return {{.success = true}};
 }
 
-// Prints and returns expected results for stdin.carbon.
-static auto TestStdin(TestParams& params)
-    -> ErrorOr<FileTestBaseTest::RunResult> {
-  CARBON_CHECK(params.input_stream);
-  constexpr int ReadSize = 256;
-  char buf[ReadSize];
-  while (feof(params.input_stream) == 0) {
-    auto read = fread(&buf, sizeof(char), ReadSize, params.input_stream);
-    if (read > 0) {
-      params.error_stream.write(buf, read);
-    }
-  }
-  return {{.success = true}};
-}
-
 // Prints and returns expected results for unattached_multi_file.carbon.
 static auto TestUnattachedMultiFile(TestParams& params)
     -> ErrorOr<FileTestBaseTest::RunResult> {
@@ -258,6 +243,17 @@ static auto EchoFileContent(TestParams& params)
       buffer = remainder;
     }
   }
+  if (params.input_stream) {
+    params.error_stream << "--- STDIN:\n";
+    constexpr int ReadSize = 1024;
+    char buf[ReadSize];
+    while (feof(params.input_stream) == 0) {
+      auto read = fread(&buf, sizeof(char), ReadSize, params.input_stream);
+      if (read > 0) {
+        params.error_stream.write(buf, read);
+      }
+    }
+  }
   return {{.success = true}};
 }
 
@@ -287,8 +283,6 @@ auto FileTestBaseTest::Run(
           .Case("file_only_re_one_file.carbon", &TestFileOnlyREOneFile)
           .Case("file_only_re_multi_file.carbon", &TestFileOnlyREMultiFile)
           .Case("no_line_number.carbon", &TestNoLineNumber)
-          .Case("stdin.carbon", &TestStdin)
-          .Case("stdin_and_autoupdate_split.carbon", &TestStdin)
           .Case("unattached_multi_file.carbon", &TestUnattachedMultiFile)
           .Case("fail_multi_success_overall_fail.carbon",
                 [&](TestParams&) {

testing/file_test/testdata/lsp_calls.carbon

@@ -0,0 +1,48 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //testing/file_test:file_test_base_test --test_arg=--file_tests=testing/file_test/testdata/lsp_calls.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //testing/file_test:file_test_base_test -- --dump_output --file_tests=testing/file_test/testdata/lsp_calls.carbon
+
+// --- STDIN
+[[@LSP:foo:]]
+[[@LSP:foo]]
+[[@LSP:bar:content]]
+[[@LSP:baz:
+multi
+line
+]]
+[[@LSP:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: --- STDIN:
+// CHECK:STDERR: Content-Length: 48
+// CHECK:STDERR:
+// CHECK:STDERR: {"jsonrpc": "2.0", "id": "1", "method": "foo"}
+// CHECK:STDERR:
+// CHECK:STDERR: Content-Length: 48
+// CHECK:STDERR:
+// CHECK:STDERR: {"jsonrpc": "2.0", "id": "2", "method": "foo"}
+// CHECK:STDERR:
+// CHECK:STDERR: Content-Length: 57
+// CHECK:STDERR:
+// CHECK:STDERR: {"jsonrpc": "2.0", "id": "3", "method": "bar", content}
+// CHECK:STDERR:
+// CHECK:STDERR: Content-Length: 61
+// CHECK:STDERR:
+// CHECK:STDERR: {"jsonrpc": "2.0", "id": "4", "method": "baz",
+// CHECK:STDERR: multi
+// CHECK:STDERR: line
+// CHECK:STDERR: }
+// CHECK:STDERR:
+// CHECK:STDERR: Content-Length: 38
+// CHECK:STDERR:
+// CHECK:STDERR: {"jsonrpc": "2.0", "method": "exit"}
+// CHECK:STDERR:
+// CHECK:STDERR:
+// CHECK:STDOUT: 1 args: `default_args`

testing/file_test/testdata/stdin.carbon

@@ -20,6 +20,7 @@ n
 
 // --- AUTOUPDATE-SPLIT
 
+// CHECK:STDERR: --- STDIN:
 // CHECK:STDERR:
 // CHECK:STDERR: S
 // CHECK:STDERR: t

toolchain/language_server/BUILD

@@ -6,6 +6,11 @@ load("@rules_cc//cc:defs.bzl", "cc_library")
 
 package(default_visibility = ["//visibility:public"])
 
+filegroup(
+    name = "testdata",
+    data = glob(["testdata/**/*.carbon"]),
+)
+
 cc_library(
     name = "language_server",
     srcs = ["language_server.cpp"],

toolchain/language_server/language_server.cpp

@@ -6,6 +6,7 @@
 
 #include "clang-tools-extra/clangd/LSPBinder.h"
 #include "clang-tools-extra/clangd/Transport.h"
+#include "clang-tools-extra/clangd/support/Logger.h"
 #include "common/raw_string_ostream.h"
 #include "toolchain/language_server/context.h"
 #include "toolchain/language_server/incoming_messages.h"
@@ -14,7 +15,12 @@
 namespace Carbon::LanguageServer {
 
 auto Run(FILE* input_stream, llvm::raw_ostream& output_stream,
-         llvm::raw_ostream& /*error_stream*/) -> ErrorOr<Success> {
+         llvm::raw_ostream& error_stream) -> ErrorOr<Success> {
+  // TODO: Consider implementing a custom logger that splits vlog to
+  // vlog_stream when provided. For now, this disables verbose logging.
+  clang::clangd::StreamLogger logger(error_stream, clang::clangd::Logger::Info);
+  clang::clangd::LoggingSession logging_session(logger);
+
   // Set up the connection.
   std::unique_ptr<clang::clangd::Transport> transport(
       clang::clangd::newJSONTransport(input_stream, output_stream,

toolchain/language_server/testdata/exit.carbon

@@ -0,0 +1,16 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/exit.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/exit.carbon
+
+// --- STDIN
+[[@LSP:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDOUT:

toolchain/language_server/testdata/fail_empty_stdin.carbon

@@ -0,0 +1,16 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+
+// --- STDIN
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: error: Input/output error
+// CHECK:STDERR:
+// CHECK:STDOUT:

toolchain/language_server/testdata/initialize.carbon

@@ -0,0 +1,28 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/initialize.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/initialize.carbon
+
+// --- STDIN
+[[@LSP:initialize]]
+[[@LSP:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDOUT: Content-Length: 148{{\r}}
+// CHECK:STDOUT: {{\r}}
+// CHECK:STDOUT: {
+// CHECK:STDOUT:   "id": "1",
+// CHECK:STDOUT:   "jsonrpc": "2.0",
+// CHECK:STDOUT:   "result": {
+// CHECK:STDOUT:     "capabilities": {
+// CHECK:STDOUT:       "documentSymbolProvider": true,
+// CHECK:STDOUT:       "textDocumentSync": 1
+// CHECK:STDOUT:     }
+// CHECK:STDOUT:   }
+// CHECK:STDOUT: }

toolchain/testing/BUILD

@@ -16,6 +16,7 @@ filegroup(
         "//toolchain/diagnostics:testdata",
         "//toolchain/driver:testdata",
         "//toolchain/format:testdata",
+        "//toolchain/language_server:testdata",
         "//toolchain/lex:testdata",
         "//toolchain/lower:testdata",
         "//toolchain/parse:testdata",

toolchain/testing/file_test.cpp

@@ -20,7 +20,7 @@ namespace Carbon::Testing {
 namespace {
 
 // Provides common test support for the driver. This is used by file tests in
-// phase subdirectories.
+// component subdirectories.
 class ToolchainFileTest : public FileTestBase {
  public:
   explicit ToolchainFileTest(llvm::StringRef exe_path, std::mutex* output_mutex,
@@ -51,6 +51,12 @@ class ToolchainFileTest : public FileTestBase {
   // driver.
   auto DoExtraCheckReplacements(std::string& check_line) -> void override;
 
+  // Most tests can be run in parallel, but clangd has a global for its logging
+  // system so we need language-server tests to be run in serial.
+  auto AllowParallelRun() const -> bool override {
+    return component_ != "language_server";
+  }
+
  private:
   // Adds a file to the fs.
   auto AddFile(llvm::vfs::InMemoryFileSystem& fs, llvm::StringRef path)
@@ -98,6 +104,10 @@ auto ToolchainFileTest::Run(
     llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
     FILE* input_stream, llvm::raw_pwrite_stream& output_stream,
     llvm::raw_pwrite_stream& error_stream) -> ErrorOr<RunResult> {
+  if (component_ == "language_server" && !input_stream) {
+    return Error("language_server tests must provide STDIN");
+  }
+
   CARBON_ASSIGN_OR_RETURN(auto prelude, installation_.ReadPreludeManifest());
   if (!is_no_prelude()) {
     for (const auto& file : prelude) {
@@ -130,12 +140,20 @@ auto ToolchainFileTest::Run(
                           entry.first.starts_with("not_file") ||
                           llvm::is_contained(prelude, entry.first);
                  });
+
+  if (component_ == "language_server") {
+    // The language server doesn't always add a suffix newline, so add one for
+    // tests to be happy.
+    output_stream << "\n";
+  }
   return result;
 }
 
 auto ToolchainFileTest::GetDefaultArgs() -> llvm::SmallVector<std::string> {
   if (component_ == "format") {
     return {"format", "%s"};
+  } else if (component_ == "language_server") {
+    return {"language-server"};
   }
 
   llvm::SmallVector<std::string> args = {"compile", "--include-diagnostic-kind",