Cache calculated file state in LSP (#4897)

Add caching of parsed documents, and testing of the textDocument
handlers. This is based on #4896, which splits out some of the
boilerplate to calls.

Note, this caches the entire parse state because we'll want to try to
emit diagnostics when we see the update, without waiting. It may be
helpful to do that asynchronously, but we don't want to wait for another
call (such as documentSymbol). Really, we'll probably want to also add
check for diagnostics, at least.

toolchain/diagnostics/diagnostic_kind.def

@@ -28,11 +28,6 @@ CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError)
 CARBON_DIAGNOSTIC_KIND(CompileInputNotRegularFile)
 CARBON_DIAGNOSTIC_KIND(CompileOutputFileOpenError)
 CARBON_DIAGNOSTIC_KIND(FormatMultipleFilesToOneOutput)
-CARBON_DIAGNOSTIC_KIND(LanguageServerMissingInputStream)
-CARBON_DIAGNOSTIC_KIND(LanguageServerNotificationParseError)
-CARBON_DIAGNOSTIC_KIND(LanguageServerTransportError)
-CARBON_DIAGNOSTIC_KIND(LanguageServerUnexpectedReply)
-CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedNotification)
 
 // ============================================================================
 // SourceBuffer diagnostics
@@ -441,6 +436,23 @@ CARBON_DIAGNOSTIC_KIND(AssociatedConstantWithDifferentValues)
 CARBON_DIAGNOSTIC_KIND(ImplsOnNonFacetType)
 CARBON_DIAGNOSTIC_KIND(WhereOnNonFacetType)
 
+// ============================================================================
+// Language server diagnostics
+// ============================================================================
+
+CARBON_DIAGNOSTIC_KIND(LanguageServerFileUnknown)
+CARBON_DIAGNOSTIC_KIND(LanguageServerFileUnsupported)
+CARBON_DIAGNOSTIC_KIND(LanguageServerMissingInputStream)
+CARBON_DIAGNOSTIC_KIND(LanguageServerNotificationParseError)
+CARBON_DIAGNOSTIC_KIND(LanguageServerTransportError)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnexpectedReply)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedNotification)
+
+// Document handling.
+CARBON_DIAGNOSTIC_KIND(LanguageServerOpenDuplicateFile)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedChanges)
+CARBON_DIAGNOSTIC_KIND(LanguageServerCloseUnknownFile)
+
 // ============================================================================
 // Other diagnostics
 // ============================================================================

toolchain/language_server/BUILD

@@ -28,10 +28,20 @@ cc_library(
 
 cc_library(
     name = "context",
+    srcs = ["context.cpp"],
     hdrs = ["context.h"],
     deps = [
         "//common:map",
+        "//toolchain/base:shared_value_stores",
         "//toolchain/diagnostics:diagnostic_emitter",
+        "//toolchain/diagnostics:file_diagnostics",
+        "//toolchain/diagnostics:null_diagnostics",
+        "//toolchain/lex",
+        "//toolchain/lex:tokenized_buffer",
+        "//toolchain/parse",
+        "//toolchain/parse:tree",
+        "//toolchain/sem_ir:file",
+        "//toolchain/source:source_buffer",
     ],
 )
 

toolchain/language_server/context.cpp

@@ -0,0 +1,68 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/language_server/context.h"
+
+#include <memory>
+
+#include "toolchain/base/shared_value_stores.h"
+#include "toolchain/diagnostics/null_diagnostics.h"
+#include "toolchain/lex/lex.h"
+#include "toolchain/lex/tokenized_buffer.h"
+#include "toolchain/parse/parse.h"
+#include "toolchain/parse/tree_and_subtrees.h"
+
+namespace Carbon::LanguageServer {
+
+auto Context::File::SetText(Context& context, llvm::StringRef text) -> void {
+  // Clear state dependent on the source text.
+  tree_and_subtrees_.reset();
+  tree_.reset();
+  tokens_.reset();
+  value_stores_.reset();
+  source_.reset();
+
+  // TODO: Make the processing asynchronous, to better handle rapid text
+  // updates.
+  CARBON_CHECK(!source_ && !value_stores_ && !tokens_ && !tree_,
+               "We currently cache everything together");
+  // TODO: Diagnostics should be passed to the LSP instead of dropped.
+  auto& null_consumer = NullDiagnosticConsumer();
+  std::optional source =
+      SourceBuffer::MakeFromStringCopy(filename_, text, null_consumer);
+  if (!source) {
+    // Failing here should be rare, but provide stub data for recovery so that
+    // we can have a simple API.
+    source = SourceBuffer::MakeFromStringCopy(filename_, "", null_consumer);
+    CARBON_CHECK(source, "Making an empty buffer should always succeed");
+  }
+  source_ = std::make_unique<SourceBuffer>(std::move(*source));
+  value_stores_ = std::make_unique<SharedValueStores>();
+  tokens_ = std::make_unique<Lex::TokenizedBuffer>(
+      Lex::Lex(*value_stores_, *source_, null_consumer));
+  tree_ = std::make_unique<Parse::Tree>(
+      Parse::Parse(*tokens_, null_consumer, context.vlog_stream()));
+  tree_and_subtrees_ =
+      std::make_unique<Parse::TreeAndSubtrees>(*tokens_, *tree_);
+}
+
+auto Context::LookupFile(llvm::StringRef filename) -> File* {
+  if (!filename.ends_with(".carbon")) {
+    CARBON_DIAGNOSTIC(LanguageServerFileUnsupported, Warning,
+                      "non-Carbon file requested");
+    file_emitter_.Emit(filename, LanguageServerFileUnsupported);
+    return nullptr;
+  }
+
+  if (auto lookup_result = files().Lookup(filename)) {
+    return &lookup_result.value();
+  } else {
+    CARBON_DIAGNOSTIC(LanguageServerFileUnknown, Warning,
+                      "unknown file requested");
+    file_emitter_.Emit(filename, LanguageServerFileUnknown);
+    return nullptr;
+  }
+}
+
+}  // namespace Carbon::LanguageServer

toolchain/language_server/context.h

@@ -5,29 +5,72 @@
 #ifndef CARBON_TOOLCHAIN_LANGUAGE_SERVER_CONTEXT_H_
 #define CARBON_TOOLCHAIN_LANGUAGE_SERVER_CONTEXT_H_
 
+#include <memory>
 #include <string>
 
 #include "common/map.h"
+#include "toolchain/base/shared_value_stores.h"
 #include "toolchain/diagnostics/diagnostic_consumer.h"
 #include "toolchain/diagnostics/diagnostic_emitter.h"
+#include "toolchain/diagnostics/file_diagnostics.h"
+#include "toolchain/lex/tokenized_buffer.h"
+#include "toolchain/parse/tree_and_subtrees.h"
+#include "toolchain/sem_ir/file.h"
+#include "toolchain/source/source_buffer.h"
 
 namespace Carbon::LanguageServer {
 
 // Context for LSP call handling.
 class Context {
  public:
-  // `consumer` is required.
-  explicit Context(DiagnosticConsumer* consumer) : no_loc_emitter_(consumer) {}
+  // Cached information for an open file.
+  class File {
+   public:
+    explicit File(std::string filename) : filename_(std::move(filename)) {}
 
+    // Changes the file's text, updating dependent state.
+    auto SetText(Context& context, llvm::StringRef text) -> void;
+
+    auto tree_and_subtrees() const -> const Parse::TreeAndSubtrees& {
+      return *tree_and_subtrees_;
+    }
+
+   private:
+    // The filename, stable across instances.
+    std::string filename_;
+
+    // Current file content, and derived values.
+    std::unique_ptr<SourceBuffer> source_;
+    std::unique_ptr<SharedValueStores> value_stores_;
+    std::unique_ptr<Lex::TokenizedBuffer> tokens_;
+    std::unique_ptr<Parse::Tree> tree_;
+    std::unique_ptr<Parse::TreeAndSubtrees> tree_and_subtrees_;
+  };
+
+  // `consumer` and `emitter` are required. `vlog_stream` is optional.
+  explicit Context(llvm::raw_ostream* vlog_stream, DiagnosticConsumer* consumer)
+      : vlog_stream_(vlog_stream),
+        file_emitter_(consumer),
+        no_loc_emitter_(consumer) {}
+
+  // Returns a reference to the file if it's known, or diagnoses and returns
+  // null.
+  auto LookupFile(llvm::StringRef filename) -> File*;
+
+  auto file_emitter() -> FileDiagnosticEmitter& { return file_emitter_; }
   auto no_loc_emitter() -> NoLocDiagnosticEmitter& { return no_loc_emitter_; }
+  auto vlog_stream() -> llvm::raw_ostream* { return vlog_stream_; }
 
-  auto files() -> Map<std::string, std::string>& { return files_; }
+  auto files() -> Map<std::string, File>& { return files_; }
 
  private:
+  // Diagnostic and output streams.
+  llvm::raw_ostream* vlog_stream_;
+  FileDiagnosticEmitter file_emitter_;
   NoLocDiagnosticEmitter no_loc_emitter_;
 
   // Content of files managed by the language client.
-  Map<std::string, std::string> files_;
+  Map<std::string, File> files_;
 };
 
 }  // namespace Carbon::LanguageServer

toolchain/language_server/handle.h

@@ -15,6 +15,11 @@ auto HandleDidChangeTextDocument(
     Context& context, const clang::clangd::DidChangeTextDocumentParams& params)
     -> void;
 
+// Closes a document.
+auto HandleDidCloseTextDocument(
+    Context& context, const clang::clangd::DidCloseTextDocumentParams& params)
+    -> void;
+
 // Updates the content of already-open documents.
 auto HandleDidOpenTextDocument(
     Context& context, const clang::clangd::DidOpenTextDocumentParams& params)

toolchain/language_server/handle_document_symbol.cpp

@@ -15,18 +15,17 @@ namespace Carbon::LanguageServer {
 
 // Returns the text of first child of kind IdentifierNameBeforeParams or
 // IdentifierNameNotBeforeParams.
-static auto GetIdentifierName(const SharedValueStores& value_stores,
-                              const Lex::TokenizedBuffer& tokens,
-                              const Parse::TreeAndSubtrees& tree_and_subtrees,
+static auto GetIdentifierName(const Parse::TreeAndSubtrees& tree_and_subtrees,
                               Parse::NodeId node)
     -> std::optional<llvm::StringRef> {
+  const auto& tokens = tree_and_subtrees.tree().tokens();
   for (auto child : tree_and_subtrees.children(node)) {
     switch (tree_and_subtrees.tree().node_kind(child)) {
       case Parse::NodeKind::IdentifierNameBeforeParams:
       case Parse::NodeKind::IdentifierNameNotBeforeParams: {
         auto token = tree_and_subtrees.tree().node_token(child);
         if (tokens.GetKind(token) == Lex::TokenKind::Identifier) {
-          return value_stores.identifiers().Get(tokens.GetIdentifier(token));
+          return tokens.GetTokenText(token);
         }
         break;
       }
@@ -42,22 +41,19 @@ auto HandleDocumentSymbol(
     llvm::function_ref<
         void(llvm::Expected<std::vector<clang::clangd::DocumentSymbol>>)>
         on_done) -> void {
-  SharedValueStores value_stores;
-  llvm::vfs::InMemoryFileSystem vfs;
-  auto lookup = context.files().Lookup(params.textDocument.uri.file());
-  CARBON_CHECK(lookup);
-  vfs.addFile(lookup.key(), /*mtime=*/0,
-              llvm::MemoryBuffer::getMemBufferCopy(lookup.value()));
+  auto* file = context.LookupFile(params.textDocument.uri.file());
+  if (!file) {
+    return;
+  }
+
+  const auto& tree_and_subtrees = file->tree_and_subtrees();
+  const auto& tree = tree_and_subtrees.tree();
+  const auto& tokens = tree.tokens();
 
-  auto source =
-      SourceBuffer::MakeFromFile(vfs, lookup.key(), NullDiagnosticConsumer());
-  auto tokens = Lex::Lex(value_stores, *source, NullDiagnosticConsumer());
-  auto tree = Parse::Parse(tokens, NullDiagnosticConsumer(), nullptr);
-  Parse::TreeAndSubtrees tree_and_subtrees(tokens, tree);
   std::vector<clang::clangd::DocumentSymbol> result;
-  for (const auto& node : tree.postorder()) {
+  for (const auto& node_id : tree.postorder()) {
     clang::clangd::SymbolKind symbol_kind;
-    switch (tree.node_kind(node)) {
+    switch (tree.node_kind(node_id)) {
       case Parse::NodeKind::FunctionDecl:
       case Parse::NodeKind::FunctionDefinitionStart:
         symbol_kind = clang::clangd::SymbolKind::Function;
@@ -76,9 +72,8 @@ auto HandleDocumentSymbol(
         continue;
     }
 
-    if (auto name =
-            GetIdentifierName(value_stores, tokens, tree_and_subtrees, node)) {
-      auto token = tree.node_token(node);
+    if (auto name = GetIdentifierName(tree_and_subtrees, node_id)) {
+      auto token = tree.node_token(node_id);
       clang::clangd::Position pos{tokens.GetLineNumber(token) - 1,
                                   tokens.GetColumnNumber(token) - 1};
 

toolchain/language_server/handle_text_document.cpp

@@ -9,17 +9,58 @@ namespace Carbon::LanguageServer {
 auto HandleDidOpenTextDocument(
     Context& context, const clang::clangd::DidOpenTextDocumentParams& params)
     -> void {
-  context.files().Update(params.textDocument.uri.file(),
-                         params.textDocument.text);
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
+  auto insert_result = context.files().Insert(
+      filename, [&] { return Context::File(filename.str()); });
+  insert_result.value().SetText(context, params.textDocument.text);
+  if (!insert_result.is_inserted()) {
+    CARBON_DIAGNOSTIC(LanguageServerOpenDuplicateFile, Warning,
+                      "duplicate open file request; updating content");
+    context.file_emitter().Emit(filename, LanguageServerOpenDuplicateFile);
+  }
 }
 
 auto HandleDidChangeTextDocument(
     Context& context, const clang::clangd::DidChangeTextDocumentParams& params)
     -> void {
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
   // Full text is sent if full sync is specified in capabilities.
-  CARBON_CHECK(params.contentChanges.size() == 1);
-  context.files().Update(params.textDocument.uri.file(),
-                         params.contentChanges[0].text);
+  if (params.contentChanges.size() != 1) {
+    CARBON_DIAGNOSTIC(LanguageServerUnsupportedChanges, Warning,
+                      "received unsupported contentChanges count: {0}", int);
+    context.file_emitter().Emit(filename, LanguageServerUnsupportedChanges,
+                                params.contentChanges.size());
+    return;
+  }
+  if (auto* file = context.LookupFile(filename)) {
+    file->SetText(context, params.contentChanges[0].text);
+  }
+}
+
+auto HandleDidCloseTextDocument(
+    Context& context, const clang::clangd::DidCloseTextDocumentParams& params)
+    -> void {
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
+  if (!context.files().Erase(filename)) {
+    CARBON_DIAGNOSTIC(LanguageServerCloseUnknownFile, Warning,
+                      "tried closing unknown file; ignoring request");
+    context.file_emitter().Emit(filename, LanguageServerCloseUnknownFile);
+  }
 }
 
 }  // namespace Carbon::LanguageServer

toolchain/language_server/incoming_messages.cpp

@@ -76,6 +76,7 @@ IncomingMessages::IncomingMessages(clang::clangd::Transport* transport,
   AddCallHandler("initialize", &HandleInitialize);
   AddNotificationHandler("textDocument/didChange",
                          &HandleDidChangeTextDocument);
+  AddNotificationHandler("textDocument/didClose", &HandleDidCloseTextDocument);
   AddNotificationHandler("textDocument/didOpen", &HandleDidOpenTextDocument);
 }
 

toolchain/language_server/language_server.cpp

@@ -55,7 +55,7 @@ auto Run(FILE* input_stream, llvm::raw_ostream& output_stream,
       clang::clangd::newJSONTransport(input_stream, output_stream,
                                       /*InMirror=*/nullptr,
                                       /*Pretty=*/true));
-  Context context(&consumer);
+  Context context(vlog_stream, &consumer);
   // TODO: Use error_stream in IncomingMessages to report dropped errors.
   IncomingMessages incoming(transport.get(), &context);
   OutgoingMessages outgoing(transport.get());

toolchain/language_server/testdata/exit.carbon → toolchain/language_server/testdata/basics/exit.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/exit.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/exit.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/exit.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/exit.carbon
 
 // --- STDIN
 [[@LSP-NOTIFY:exit]]

toolchain/language_server/testdata/fail_empty_stdin.carbon → toolchain/language_server/testdata/basics/fail_empty_stdin.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/fail_empty_stdin.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/fail_empty_stdin.carbon
 
 // --- STDIN
 // --- AUTOUPDATE-SPLIT

toolchain/language_server/testdata/fail_no_stdin.carbon → toolchain/language_server/testdata/basics/fail_no_stdin.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_no_stdin.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/fail_no_stdin.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_no_stdin.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/fail_no_stdin.carbon
 
 // --- AUTOUPDATE-SPLIT
 

toolchain/language_server/testdata/initialize.carbon → toolchain/language_server/testdata/basics/initialize.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/initialize.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/initialize.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/initialize.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/initialize.carbon
 
 // --- STDIN
 [[@LSP-CALL:initialize]]

toolchain/language_server/testdata/notify_parse_error.carbon → toolchain/language_server/testdata/basics/notify_parse_error.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/notify_parse_error.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/notify_parse_error.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/notify_parse_error.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/notify_parse_error.carbon
 
 
 // --- STDIN

toolchain/language_server/testdata/unexpected_reply.carbon → toolchain/language_server/testdata/basics/unexpected_reply.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/unexpected_reply.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/unexpected_reply.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/unexpected_reply.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/unexpected_reply.carbon
 
 
 // --- STDIN

toolchain/language_server/testdata/unsupported_call.carbon → toolchain/language_server/testdata/basics/unsupported_call.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/unsupported_call.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/unsupported_call.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/unsupported_call.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/unsupported_call.carbon
 
 
 // --- STDIN

toolchain/language_server/testdata/unsupported_notification.carbon → toolchain/language_server/testdata/basics/unsupported_notification.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/unsupported_notification.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/unsupported_notification.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/unsupported_notification.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/unsupported_notification.carbon
 
 
 // --- STDIN

toolchain/language_server/testdata/document_symbol/basics.carbon

@@ -0,0 +1,80 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/document_symbol/basics.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/document_symbol/basics.carbon
+
+// --- STDIN
+[[@LSP-NOTIFY:textDocument/didOpen:
+  "textDocument": {"uri": "file:/empty.carbon", "languageId": "carbon",
+                   "text": ""}
+]]
+[[@LSP-CALL:textDocument/documentSymbol:
+  "textDocument": {"uri": "file:/empty.carbon"}
+]]
+[[@LSP-NOTIFY:textDocument/didOpen:
+  "textDocument": {"uri": "file:/invalid.carbon", "languageId": "carbon",
+                   "text": "text"}
+]]
+[[@LSP-CALL:textDocument/documentSymbol:
+  "textDocument": {"uri": "file:/invalid.carbon"}
+]]
+[[@LSP-NOTIFY:textDocument/didOpen:
+  "textDocument": {"uri": "file:/fn.carbon", "languageId": "carbon",
+                   "text": "fn F() {}"}
+]]
+[[@LSP-CALL:textDocument/documentSymbol:
+  "textDocument": {"uri": "file:/fn.carbon"}
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDOUT: Content-Length: 49{{\r}}
+// CHECK:STDOUT: {{\r}}
+// CHECK:STDOUT: {
+// CHECK:STDOUT:   "id": 1,
+// CHECK:STDOUT:   "jsonrpc": "2.0",
+// CHECK:STDOUT:   "result": []
+// CHECK:STDOUT: }Content-Length: 49{{\r}}
+// CHECK:STDOUT: {{\r}}
+// CHECK:STDOUT: {
+// CHECK:STDOUT:   "id": 2,
+// CHECK:STDOUT:   "jsonrpc": "2.0",
+// CHECK:STDOUT:   "result": []
+// CHECK:STDOUT: }Content-Length: 459{{\r}}
+// CHECK:STDOUT: {{\r}}
+// CHECK:STDOUT: {
+// CHECK:STDOUT:   "id": 3,
+// CHECK:STDOUT:   "jsonrpc": "2.0",
+// CHECK:STDOUT:   "result": [
+// CHECK:STDOUT:     {
+// CHECK:STDOUT:       "kind": 12,
+// CHECK:STDOUT:       "name": "F",
+// CHECK:STDOUT:       "range": {
+// CHECK:STDOUT:         "end": {
+// CHECK:STDOUT:           "character": 7,
+// CHECK:STDOUT:           "line": 0
+// CHECK:STDOUT:         },
+// CHECK:STDOUT:         "start": {
+// CHECK:STDOUT:           "character": 7,
+// CHECK:STDOUT:           "line": 0
+// CHECK:STDOUT:         }
+// CHECK:STDOUT:       },
+// CHECK:STDOUT:       "selectionRange": {
+// CHECK:STDOUT:         "end": {
+// CHECK:STDOUT:           "character": 7,
+// CHECK:STDOUT:           "line": 0
+// CHECK:STDOUT:         },
+// CHECK:STDOUT:         "start": {
+// CHECK:STDOUT:           "character": 7,
+// CHECK:STDOUT:           "line": 0
+// CHECK:STDOUT:         }
+// CHECK:STDOUT:       }
+// CHECK:STDOUT:     }
+// CHECK:STDOUT:   ]
+// CHECK:STDOUT: }

toolchain/language_server/testdata/document_symbol/language.carbon

@@ -0,0 +1,21 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/document_symbol/language.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/document_symbol/language.carbon
+
+// --- STDIN
+[[@LSP-CALL:textDocument/documentSymbol:
+  "textDocument": {"uri": "file:/test.cpp"}
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: /test.cpp: warning: non-Carbon file requested [LanguageServerFileUnsupported]
+// CHECK:STDERR:
+// CHECK:STDOUT:

toolchain/language_server/testdata/document_symbol/unknown.carbon

@@ -0,0 +1,21 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/document_symbol/unknown.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/document_symbol/unknown.carbon
+
+// --- STDIN
+[[@LSP-CALL:textDocument/documentSymbol:
+  "textDocument": {"uri": "file:/test.carbon"}
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: /test.carbon: warning: unknown file requested [LanguageServerFileUnknown]
+// CHECK:STDERR:
+// CHECK:STDOUT:

toolchain/language_server/testdata/text_document/change_count.carbon

@@ -0,0 +1,39 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/text_document/change_count.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/text_document/change_count.carbon
+
+// --- STDIN
+[[@LSP-NOTIFY:textDocument/didOpen:
+  "textDocument": {"uri": "file:/test.carbon", "languageId": "carbon",
+                   "text": "test"}
+]]
+[[@LSP-NOTIFY:textDocument/didChange:
+  "textDocument": {"uri": "file:/test.carbon"},
+  "contentChanges": []
+]]
+[[@LSP-NOTIFY:textDocument/didChange:
+  "textDocument": {"uri": "file:/test.carbon"},
+  "contentChanges": [
+    {"range": {"start": {"line": 5, "character": 23 },
+               "end": {"line": 6, "character": 0 }},
+     "text": "a"},
+    {"range": {"start": {"line": 5, "character": 23 },
+               "end": {"line": 6, "character": 0 }},
+     "text": "a"}
+  ]
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: /test.carbon: warning: received unsupported contentChanges count: 0 [LanguageServerUnsupportedChanges]
+// CHECK:STDERR:
+// CHECK:STDERR: /test.carbon: warning: received unsupported contentChanges count: 2 [LanguageServerUnsupportedChanges]
+// CHECK:STDERR:
+// CHECK:STDOUT:

toolchain/language_server/testdata/text_document/change_unknown.carbon

@@ -0,0 +1,22 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/text_document/change_unknown.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/text_document/change_unknown.carbon
+
+// --- STDIN
+[[@LSP-NOTIFY:textDocument/didChange:
+  "textDocument": {"uri": "file:/test.carbon"},
+  "contentChanges": [{"text": "new content"}]
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: /test.carbon: warning: unknown file requested [LanguageServerFileUnknown]
+// CHECK:STDERR:
+// CHECK:STDOUT:

toolchain/language_server/testdata/text_document/close_unknown.carbon

@@ -0,0 +1,21 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/text_document/close_unknown.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/text_document/close_unknown.carbon
+
+// --- STDIN
+[[@LSP-NOTIFY:textDocument/didClose:
+  "textDocument": {"uri": "file:/test.carbon"}
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: /test.carbon: warning: tried closing unknown file; ignoring request [LanguageServerCloseUnknownFile]
+// CHECK:STDERR:
+// CHECK:STDOUT:

toolchain/language_server/testdata/text_document/open_change_close.carbon

@@ -0,0 +1,27 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/text_document/open_change_close.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/text_document/open_change_close.carbon
+
+// --- STDIN
+[[@LSP-NOTIFY:textDocument/didOpen:
+  "textDocument": {"uri": "file:/test.carbon", "languageId": "carbon",
+                   "text": "test"}
+]]
+[[@LSP-NOTIFY:textDocument/didChange:
+  "textDocument": {"uri": "file:/test.carbon"},
+  "contentChanges": [{"text": "new content"}]
+]]
+[[@LSP-NOTIFY:textDocument/didClose:
+  "textDocument": {"uri": "file:/test.carbon"}
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDOUT:

toolchain/language_server/testdata/text_document/open_duplicate.carbon

@@ -0,0 +1,26 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// AUTOUPDATE
+// TIP: To test this file alone, run:
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/text_document/open_duplicate.carbon
+// TIP: To dump output, run:
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/text_document/open_duplicate.carbon
+
+// --- STDIN
+[[@LSP-NOTIFY:textDocument/didOpen:
+  "textDocument": {"uri": "file:/test.carbon", "languageId": "carbon",
+                   "text": "test"}
+]]
+[[@LSP-NOTIFY:textDocument/didOpen:
+  "textDocument": {"uri": "file:/test.carbon", "languageId": "carbon",
+                   "text": "test"}
+]]
+[[@LSP-NOTIFY:exit]]
+
+// --- AUTOUPDATE-SPLIT
+
+// CHECK:STDERR: /test.carbon: warning: duplicate open file request; updating content [LanguageServerOpenDuplicateFile]
+// CHECK:STDERR:
+// CHECK:STDOUT:

toolchain/source/source_buffer.cpp

@@ -51,6 +51,15 @@ auto SourceBuffer::MakeFromFile(llvm::vfs::FileSystem& fs,
       filename, is_regular_file, consumer);
 }
 
+auto SourceBuffer::MakeFromStringCopy(llvm::StringRef filename,
+                                      llvm::StringRef text,
+                                      DiagnosticConsumer& consumer)
+    -> std::optional<SourceBuffer> {
+  return MakeFromMemoryBuffer(
+      llvm::MemoryBuffer::getMemBufferCopy(text, filename), filename,
+      /*is_regular_file=*/true, consumer);
+}
+
 auto SourceBuffer::MakeFromMemoryBuffer(
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer,
     llvm::StringRef filename, bool is_regular_file,

toolchain/source/source_buffer.h

@@ -57,6 +57,12 @@ class SourceBuffer {
     }
   }
 
+  // Returns a source buffer with the provided text content. Copies `filename`
+  // and `text` to take ownership.
+  static auto MakeFromStringCopy(llvm::StringRef filename, llvm::StringRef text,
+                                 DiagnosticConsumer& consumer)
+      -> std::optional<SourceBuffer>;
+
   // Use one of the factory functions above to create a source buffer.
   SourceBuffer() = delete;