Split deferred node traversal out from check.cpp (#4559)

I'm looking at adding more significant logic to checking, particularly
for interop. But check.cpp is getting large, and I think just adding
more logic will make it harder to reason about, so I'm looking at
splitting it up. This moves out NodeIdTraversal and
DeferredDefinitionWorklist because they're already independent from the
other code, and are reasonably sized to have their own files.

toolchain/check/BUILD

@@ -96,7 +96,11 @@ cc_library(
     name = "check",
     srcs = [
         "check.cpp",
+        "deferred_definition_worklist.cpp",
+        "deferred_definition_worklist.h",
         "handle.h",
+        "node_id_traversal.cpp",
+        "node_id_traversal.h",
     ] +
     # Glob handler files to avoid missing any.
     glob([

toolchain/check/check.cpp

@@ -4,25 +4,19 @@
 
 #include "toolchain/check/check.h"
 
-#include <variant>
-
 #include "common/check.h"
-#include "common/error.h"
 #include "common/map.h"
-#include "common/variant_helpers.h"
-#include "common/vlog.h"
 #include "toolchain/base/kind_switch.h"
 #include "toolchain/base/pretty_stack_trace_function.h"
 #include "toolchain/check/context.h"
 #include "toolchain/check/diagnostic_helpers.h"
-#include "toolchain/check/function.h"
 #include "toolchain/check/generic.h"
 #include "toolchain/check/handle.h"
 #include "toolchain/check/import.h"
 #include "toolchain/check/import_ref.h"
+#include "toolchain/check/node_id_traversal.h"
 #include "toolchain/check/sem_ir_diagnostic_converter.h"
 #include "toolchain/diagnostics/diagnostic.h"
-#include "toolchain/diagnostics/diagnostic_emitter.h"
 #include "toolchain/diagnostics/format_providers.h"
 #include "toolchain/lex/token_kind.h"
 #include "toolchain/parse/node_ids.h"
@@ -374,417 +368,6 @@ static auto InitPackageScopeAndImports(Context& context, UnitInfo& unit_info,
   ImportOtherPackages(context, unit_info, total_ir_count, namespace_type_id);
 }
 
-namespace {
-// State used to track the next deferred function definition that we will
-// encounter and need to reorder.
-class NextDeferredDefinitionCache {
- public:
-  explicit NextDeferredDefinitionCache(const Parse::Tree* tree) : tree_(tree) {
-    SkipTo(Parse::DeferredDefinitionIndex(0));
-  }
-
-  // Set the specified deferred definition index as being the next one that will
-  // be encountered.
-  auto SkipTo(Parse::DeferredDefinitionIndex next_index) -> void {
-    index_ = next_index;
-    if (static_cast<size_t>(index_.index) ==
-        tree_->deferred_definitions().size()) {
-      start_id_ = Parse::NodeId::Invalid;
-    } else {
-      start_id_ = tree_->deferred_definitions().Get(index_).start_id;
-    }
-  }
-
-  // Returns the index of the next deferred definition to be encountered.
-  auto index() const -> Parse::DeferredDefinitionIndex { return index_; }
-
-  // Returns the ID of the start node of the next deferred definition.
-  auto start_id() const -> Parse::NodeId { return start_id_; }
-
- private:
-  const Parse::Tree* tree_;
-  Parse::DeferredDefinitionIndex index_ =
-      Parse::DeferredDefinitionIndex::Invalid;
-  Parse::NodeId start_id_ = Parse::NodeId::Invalid;
-};
-}  // namespace
-
-// Determines whether this node kind is the start of a deferred definition
-// scope.
-static auto IsStartOfDeferredDefinitionScope(Parse::NodeKind kind) -> bool {
-  switch (kind) {
-    case Parse::NodeKind::ClassDefinitionStart:
-    case Parse::NodeKind::ImplDefinitionStart:
-    case Parse::NodeKind::InterfaceDefinitionStart:
-    case Parse::NodeKind::NamedConstraintDefinitionStart:
-      // TODO: Mixins.
-      return true;
-    default:
-      return false;
-  }
-}
-
-// Determines whether this node kind is the end of a deferred definition scope.
-static auto IsEndOfDeferredDefinitionScope(Parse::NodeKind kind) -> bool {
-  switch (kind) {
-    case Parse::NodeKind::ClassDefinition:
-    case Parse::NodeKind::ImplDefinition:
-    case Parse::NodeKind::InterfaceDefinition:
-    case Parse::NodeKind::NamedConstraintDefinition:
-      // TODO: Mixins.
-      return true;
-    default:
-      return false;
-  }
-}
-
-namespace {
-// A worklist of pending tasks to perform to check deferred function definitions
-// in the right order.
-class DeferredDefinitionWorklist {
- public:
-  // A worklist task that indicates we should check a deferred function
-  // definition that we previously skipped.
-  struct CheckSkippedDefinition {
-    // The definition that we skipped.
-    Parse::DeferredDefinitionIndex definition_index;
-    // The suspended function.
-    SuspendedFunction suspended_fn;
-  };
-
-  // A worklist task that indicates we should enter a nested deferred definition
-  // scope.
-  struct EnterDeferredDefinitionScope {
-    // The suspended scope. This is only set once we reach the end of the scope.
-    std::optional<DeclNameStack::SuspendedName> suspended_name;
-    // Whether this scope is itself within an outer deferred definition scope.
-    // If so, we'll delay processing its contents until we reach the end of the
-    // parent scope. For example:
-    //
-    // ```
-    // class A {
-    //   class B {
-    //     fn F() -> A { return {}; }
-    //   }
-    // } // A.B.F is type-checked here, with A complete.
-    //
-    // fn F() {
-    //   class C {
-    //     fn G() {}
-    //   } // C.G is type-checked here.
-    // }
-    // ```
-    bool in_deferred_definition_scope;
-  };
-
-  // A worklist task that indicates we should leave a deferred definition scope.
-  struct LeaveDeferredDefinitionScope {
-    // Whether this scope is within another deferred definition scope.
-    bool in_deferred_definition_scope;
-  };
-
-  // A pending type-checking task.
-  using Task =
-      std::variant<CheckSkippedDefinition, EnterDeferredDefinitionScope,
-                   LeaveDeferredDefinitionScope>;
-
-  explicit DeferredDefinitionWorklist(llvm::raw_ostream* vlog_stream)
-      : vlog_stream_(vlog_stream) {
-    // See declaration of `worklist_`.
-    worklist_.reserve(64);
-  }
-
-  static constexpr llvm::StringLiteral VlogPrefix =
-      "DeferredDefinitionWorklist ";
-
-  // Suspend the current function definition and push a task onto the worklist
-  // to finish it later.
-  auto SuspendFunctionAndPush(Context& context,
-                              Parse::DeferredDefinitionIndex index,
-                              Parse::FunctionDefinitionStartId node_id)
-      -> void {
-    worklist_.push_back(CheckSkippedDefinition{
-        index, HandleFunctionDefinitionSuspend(context, node_id)});
-    CARBON_VLOG("{0}Push CheckSkippedDefinition {1}\n", VlogPrefix,
-                index.index);
-  }
-
-  // Push a task to re-enter a function scope, so that functions defined within
-  // it are type-checked in the right context.
-  auto PushEnterDeferredDefinitionScope(Context& context) -> void {
-    bool nested = !entered_scopes_.empty() &&
-                  entered_scopes_.back().scope_index ==
-                      context.decl_name_stack().PeekInitialScopeIndex();
-    entered_scopes_.push_back(
-        {.worklist_start_index = worklist_.size(),
-         .scope_index = context.scope_stack().PeekIndex()});
-    worklist_.push_back(
-        EnterDeferredDefinitionScope{.suspended_name = std::nullopt,
-                                     .in_deferred_definition_scope = nested});
-    CARBON_VLOG("{0}Push EnterDeferredDefinitionScope {1}\n", VlogPrefix,
-                nested ? "(nested)" : "(non-nested)");
-  }
-
-  // Suspend the current deferred definition scope, which is finished but still
-  // on the decl_name_stack, and push a task to leave the scope when we're
-  // type-checking deferred definitions. Returns `true` if the current list of
-  // deferred definitions should be type-checked immediately.
-  auto SuspendFinishedScopeAndPush(Context& context) -> bool;
-
-  // Pop the next task off the worklist.
-  auto Pop() -> Task {
-    if (vlog_stream_) {
-      VariantMatch(
-          worklist_.back(),
-          [&](CheckSkippedDefinition& definition) {
-            CARBON_VLOG("{0}Handle CheckSkippedDefinition {1}\n", VlogPrefix,
-                        definition.definition_index.index);
-          },
-          [&](EnterDeferredDefinitionScope& enter) {
-            CARBON_CHECK(enter.in_deferred_definition_scope);
-            CARBON_VLOG("{0}Handle EnterDeferredDefinitionScope (nested)\n",
-                        VlogPrefix);
-          },
-          [&](LeaveDeferredDefinitionScope& leave) {
-            bool nested = leave.in_deferred_definition_scope;
-            CARBON_VLOG("{0}Handle LeaveDeferredDefinitionScope {1}\n",
-                        VlogPrefix, nested ? "(nested)" : "(non-nested)");
-          });
-    }
-
-    return worklist_.pop_back_val();
-  }
-
-  // CHECK that the work list has no further work.
-  auto VerifyEmpty() {
-    CARBON_CHECK(worklist_.empty() && entered_scopes_.empty(),
-                 "Tasks left behind on worklist.");
-  }
-
- private:
-  llvm::raw_ostream* vlog_stream_;
-
-  // A worklist of type-checking tasks we'll need to do later.
-  //
-  // Don't allocate any inline storage here. A Task is fairly large, so we never
-  // want this to live on the stack. Instead, we reserve space in the
-  // constructor for a fairly large number of deferred definitions.
-  llvm::SmallVector<Task, 0> worklist_;
-
-  // A deferred definition scope that is currently still open.
-  struct EnteredScope {
-    // The index in worklist_ of the EnterDeferredDefinitionScope task.
-    size_t worklist_start_index;
-    // The corresponding lexical scope index.
-    ScopeIndex scope_index;
-  };
-
-  // The deferred definition scopes for the current checking actions.
-  llvm::SmallVector<EnteredScope> entered_scopes_;
-};
-}  // namespace
-
-auto DeferredDefinitionWorklist::SuspendFinishedScopeAndPush(Context& context)
-    -> bool {
-  auto start_index = entered_scopes_.pop_back_val().worklist_start_index;
-
-  // If we've not found any deferred definitions in this scope, clean up the
-  // stack.
-  if (start_index == worklist_.size() - 1) {
-    context.decl_name_stack().PopScope();
-    worklist_.pop_back();
-    CARBON_VLOG("{0}Pop EnterDeferredDefinitionScope (empty)\n", VlogPrefix);
-    return false;
-  }
-
-  // If we're finishing a nested deferred definition scope, keep track of that
-  // but don't type-check deferred definitions now.
-  auto& enter_scope = get<EnterDeferredDefinitionScope>(worklist_[start_index]);
-  if (enter_scope.in_deferred_definition_scope) {
-    // This is a nested deferred definition scope. Suspend the inner scope so we
-    // can restore it when we come to type-check the deferred definitions.
-    enter_scope.suspended_name = context.decl_name_stack().Suspend();
-
-    // Enqueue a task to leave the nested scope.
-    worklist_.push_back(
-        LeaveDeferredDefinitionScope{.in_deferred_definition_scope = true});
-    CARBON_VLOG("{0}Push LeaveDeferredDefinitionScope (nested)\n", VlogPrefix);
-    return false;
-  }
-
-  // We're at the end of a non-nested deferred definition scope. Prepare to
-  // start checking deferred definitions. Enqueue a task to leave this outer
-  // scope and end checking deferred definitions.
-  worklist_.push_back(
-      LeaveDeferredDefinitionScope{.in_deferred_definition_scope = false});
-  CARBON_VLOG("{0}Push LeaveDeferredDefinitionScope (non-nested)\n",
-              VlogPrefix);
-
-  // We'll process the worklist in reverse index order, so reverse the part of
-  // it we're about to execute so we run our tasks in the order in which they
-  // were pushed.
-  std::reverse(worklist_.begin() + start_index, worklist_.end());
-
-  // Pop the `EnterDeferredDefinitionScope` that's now on the end of the
-  // worklist. We stay in that scope rather than suspending then immediately
-  // resuming it.
-  CARBON_CHECK(
-      holds_alternative<EnterDeferredDefinitionScope>(worklist_.back()),
-      "Unexpected task in worklist.");
-  worklist_.pop_back();
-  CARBON_VLOG("{0}Handle EnterDeferredDefinitionScope (non-nested)\n",
-              VlogPrefix);
-  return true;
-}
-
-namespace {
-// A traversal of the node IDs in the parse tree, in the order in which we need
-// to check them.
-class NodeIdTraversal {
- public:
-  explicit NodeIdTraversal(Context& context, llvm::raw_ostream* vlog_stream)
-      : context_(context),
-        next_deferred_definition_(&context.parse_tree()),
-        worklist_(vlog_stream) {
-    auto range = context.parse_tree().postorder();
-    chunks_.push_back(
-        {.it = range.begin(),
-         .end = range.end(),
-         .next_definition = Parse::DeferredDefinitionIndex::Invalid});
-  }
-
-  // Finds the next `NodeId` to type-check. Returns nullopt if the traversal is
-  // complete.
-  auto Next() -> std::optional<Parse::NodeId>;
-
-  // Performs any processing necessary after we type-check a node.
-  auto Handle(Parse::NodeKind parse_kind) -> void {
-    // When we reach the start of a deferred definition scope, add a task to the
-    // worklist to check future skipped definitions in the new context.
-    if (IsStartOfDeferredDefinitionScope(parse_kind)) {
-      worklist_.PushEnterDeferredDefinitionScope(context_);
-    }
-
-    // When we reach the end of a deferred definition scope, add a task to the
-    // worklist to leave the scope. If this is not a nested scope, start
-    // checking the deferred definitions now.
-    if (IsEndOfDeferredDefinitionScope(parse_kind)) {
-      chunks_.back().checking_deferred_definitions =
-          worklist_.SuspendFinishedScopeAndPush(context_);
-    }
-  }
-
- private:
-  // A chunk of the parse tree that we need to type-check.
-  struct Chunk {
-    Parse::Tree::PostorderIterator it;
-    Parse::Tree::PostorderIterator end;
-    // The next definition that will be encountered after this chunk completes.
-    Parse::DeferredDefinitionIndex next_definition;
-    // Whether we are currently checking deferred definitions, rather than the
-    // tokens of this chunk. If so, we'll pull tasks off `worklist` and execute
-    // them until we're done with this batch of deferred definitions. Otherwise,
-    // we'll pull node IDs from `*it` until it reaches `end`.
-    bool checking_deferred_definitions = false;
-  };
-
-  // Re-enter a nested deferred definition scope.
-  auto PerformTask(
-      DeferredDefinitionWorklist::EnterDeferredDefinitionScope&& enter)
-      -> void {
-    CARBON_CHECK(enter.suspended_name,
-                 "Entering a scope with no suspension information.");
-    context_.decl_name_stack().Restore(std::move(*enter.suspended_name));
-  }
-
-  // Leave a nested or top-level deferred definition scope.
-  auto PerformTask(
-      DeferredDefinitionWorklist::LeaveDeferredDefinitionScope&& leave)
-      -> void {
-    if (!leave.in_deferred_definition_scope) {
-      // We're done with checking deferred definitions.
-      chunks_.back().checking_deferred_definitions = false;
-    }
-    context_.decl_name_stack().PopScope();
-  }
-
-  // Resume checking a deferred definition.
-  auto PerformTask(
-      DeferredDefinitionWorklist::CheckSkippedDefinition&& parse_definition)
-      -> void {
-    auto& [definition_index, suspended_fn] = parse_definition;
-    const auto& definition_info =
-        context_.parse_tree().deferred_definitions().Get(definition_index);
-    HandleFunctionDefinitionResume(context_, definition_info.start_id,
-                                   std::move(suspended_fn));
-    auto range = Parse::Tree::PostorderIterator::MakeRange(
-        definition_info.start_id, definition_info.definition_id);
-    chunks_.push_back({.it = range.begin() + 1,
-                       .end = range.end(),
-                       .next_definition = next_deferred_definition_.index()});
-    ++definition_index.index;
-    next_deferred_definition_.SkipTo(definition_index);
-  }
-
-  Context& context_;
-  NextDeferredDefinitionCache next_deferred_definition_;
-  DeferredDefinitionWorklist worklist_;
-  llvm::SmallVector<Chunk> chunks_;
-};
-}  // namespace
-
-auto NodeIdTraversal::Next() -> std::optional<Parse::NodeId> {
-  while (true) {
-    // If we're checking deferred definitions, find the next definition we
-    // should check, restore its suspended state, and add a corresponding
-    // `Chunk` to the top of the chunk list.
-    if (chunks_.back().checking_deferred_definitions) {
-      std::visit(
-          [&](auto&& task) { PerformTask(std::forward<decltype(task)>(task)); },
-          worklist_.Pop());
-      continue;
-    }
-
-    // If we're not checking deferred definitions, produce the next parse node
-    // for this chunk. If we've run out of parse nodes, we're done with this
-    // chunk of the parse tree.
-    if (chunks_.back().it == chunks_.back().end) {
-      auto old_chunk = chunks_.pop_back_val();
-
-      // If we're out of chunks, then we're done entirely.
-      if (chunks_.empty()) {
-        worklist_.VerifyEmpty();
-        return std::nullopt;
-      }
-
-      next_deferred_definition_.SkipTo(old_chunk.next_definition);
-      continue;
-    }
-
-    auto node_id = *chunks_.back().it;
-
-    // If we've reached the start of a deferred definition, skip to the end of
-    // it, and track that we need to check it later.
-    if (node_id == next_deferred_definition_.start_id()) {
-      const auto& definition_info =
-          context_.parse_tree().deferred_definitions().Get(
-              next_deferred_definition_.index());
-      worklist_.SuspendFunctionAndPush(context_,
-                                       next_deferred_definition_.index(),
-                                       definition_info.start_id);
-
-      // Continue type-checking the parse tree after the end of the definition.
-      chunks_.back().it =
-          Parse::Tree::PostorderIterator(definition_info.definition_id) + 1;
-      next_deferred_definition_.SkipTo(definition_info.next_definition_index);
-      continue;
-    }
-
-    ++chunks_.back().it;
-    return node_id;
-  }
-}
-
 // Checks that each required definition is available. If the definition can be
 // generated by resolving a specific, does so, otherwise emits a diagnostic for
 // each declaration in context.definitions_required() that doesn't have a
@@ -1217,7 +800,8 @@ auto CheckParseTrees(
     llvm::MutableArrayRef<Unit> units,
     llvm::MutableArrayRef<Parse::NodeLocConverter> node_converters,
     bool prelude_import, llvm::raw_ostream* vlog_stream) -> void {
-  // UnitInfo is big due to its SmallVectors, so we default to 0 on the stack.
+  // UnitInfo is big due to its SmallVectors, so we default to 0 on the
+  // stack.
   llvm::SmallVector<UnitInfo, 0> unit_infos;
   unit_infos.reserve(units.size());
   for (auto [i, unit] : llvm::enumerate(units)) {

toolchain/check/deferred_definition_worklist.cpp

@@ -0,0 +1,121 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/check/deferred_definition_worklist.h"
+
+#include "common/variant_helpers.h"
+#include "common/vlog.h"
+#include "toolchain/check/handle.h"
+
+namespace Carbon::Check {
+
+static constexpr llvm::StringLiteral VlogPrefix = "DeferredDefinitionWorklist ";
+
+DeferredDefinitionWorklist::DeferredDefinitionWorklist(
+    llvm::raw_ostream* vlog_stream)
+    : vlog_stream_(vlog_stream) {
+  // See declaration of `worklist_`.
+  worklist_.reserve(64);
+}
+
+auto DeferredDefinitionWorklist::SuspendFunctionAndPush(
+    Context& context, Parse::DeferredDefinitionIndex index,
+    Parse::FunctionDefinitionStartId node_id) -> void {
+  // TODO: Investigate factoring out `HandleFunctionDefinitionSuspend` to make
+  // `DeferredDefinitionWorklist` reusable.
+  worklist_.push_back(CheckSkippedDefinition{
+      index, HandleFunctionDefinitionSuspend(context, node_id)});
+  CARBON_VLOG("{0}Push CheckSkippedDefinition {1}\n", VlogPrefix, index.index);
+}
+
+auto DeferredDefinitionWorklist::PushEnterDeferredDefinitionScope(
+    Context& context) -> void {
+  bool nested = !entered_scopes_.empty() &&
+                entered_scopes_.back().scope_index ==
+                    context.decl_name_stack().PeekInitialScopeIndex();
+  entered_scopes_.push_back({.worklist_start_index = worklist_.size(),
+                             .scope_index = context.scope_stack().PeekIndex()});
+  worklist_.push_back(EnterDeferredDefinitionScope{
+      .suspended_name = std::nullopt, .in_deferred_definition_scope = nested});
+  CARBON_VLOG("{0}Push EnterDeferredDefinitionScope {1}\n", VlogPrefix,
+              nested ? "(nested)" : "(non-nested)");
+}
+
+auto DeferredDefinitionWorklist::SuspendFinishedScopeAndPush(Context& context)
+    -> bool {
+  auto start_index = entered_scopes_.pop_back_val().worklist_start_index;
+
+  // If we've not found any deferred definitions in this scope, clean up the
+  // stack.
+  if (start_index == worklist_.size() - 1) {
+    context.decl_name_stack().PopScope();
+    worklist_.pop_back();
+    CARBON_VLOG("{0}Pop EnterDeferredDefinitionScope (empty)\n", VlogPrefix);
+    return false;
+  }
+
+  // If we're finishing a nested deferred definition scope, keep track of that
+  // but don't type-check deferred definitions now.
+  auto& enter_scope = get<EnterDeferredDefinitionScope>(worklist_[start_index]);
+  if (enter_scope.in_deferred_definition_scope) {
+    // This is a nested deferred definition scope. Suspend the inner scope so we
+    // can restore it when we come to type-check the deferred definitions.
+    enter_scope.suspended_name = context.decl_name_stack().Suspend();
+
+    // Enqueue a task to leave the nested scope.
+    worklist_.push_back(
+        LeaveDeferredDefinitionScope{.in_deferred_definition_scope = true});
+    CARBON_VLOG("{0}Push LeaveDeferredDefinitionScope (nested)\n", VlogPrefix);
+    return false;
+  }
+
+  // We're at the end of a non-nested deferred definition scope. Prepare to
+  // start checking deferred definitions. Enqueue a task to leave this outer
+  // scope and end checking deferred definitions.
+  worklist_.push_back(
+      LeaveDeferredDefinitionScope{.in_deferred_definition_scope = false});
+  CARBON_VLOG("{0}Push LeaveDeferredDefinitionScope (non-nested)\n",
+              VlogPrefix);
+
+  // We'll process the worklist in reverse index order, so reverse the part of
+  // it we're about to execute so we run our tasks in the order in which they
+  // were pushed.
+  std::reverse(worklist_.begin() + start_index, worklist_.end());
+
+  // Pop the `EnterDeferredDefinitionScope` that's now on the end of the
+  // worklist. We stay in that scope rather than suspending then immediately
+  // resuming it.
+  CARBON_CHECK(
+      holds_alternative<EnterDeferredDefinitionScope>(worklist_.back()),
+      "Unexpected task in worklist.");
+  worklist_.pop_back();
+  CARBON_VLOG("{0}Handle EnterDeferredDefinitionScope (non-nested)\n",
+              VlogPrefix);
+  return true;
+}
+
+auto DeferredDefinitionWorklist::Pop() -> Task {
+  if (vlog_stream_) {
+    VariantMatch(
+        worklist_.back(),
+        [&](CheckSkippedDefinition& definition) {
+          CARBON_VLOG("{0}Handle CheckSkippedDefinition {1}\n", VlogPrefix,
+                      definition.definition_index.index);
+        },
+        [&](EnterDeferredDefinitionScope& enter) {
+          CARBON_CHECK(enter.in_deferred_definition_scope);
+          CARBON_VLOG("{0}Handle EnterDeferredDefinitionScope (nested)\n",
+                      VlogPrefix);
+        },
+        [&](LeaveDeferredDefinitionScope& leave) {
+          bool nested = leave.in_deferred_definition_scope;
+          CARBON_VLOG("{0}Handle LeaveDeferredDefinitionScope {1}\n",
+                      VlogPrefix, nested ? "(nested)" : "(non-nested)");
+        });
+  }
+
+  return worklist_.pop_back_val();
+}
+
+}  // namespace Carbon::Check

toolchain/check/deferred_definition_worklist.h

@@ -0,0 +1,119 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_CHECK_DEFERRED_DEFINITION_WORKLIST_H_
+#define CARBON_TOOLCHAIN_CHECK_DEFERRED_DEFINITION_WORKLIST_H_
+
+#include <optional>
+#include <variant>
+
+#include "common/ostream.h"
+#include "llvm/ADT/SmallVector.h"
+#include "toolchain/check/decl_name_stack.h"
+#include "toolchain/check/function.h"
+#include "toolchain/parse/tree.h"
+
+namespace Carbon::Check {
+
+// A worklist of pending tasks to perform to check deferred function definitions
+// in the right order.
+class DeferredDefinitionWorklist {
+ public:
+  // A worklist task that indicates we should check a deferred function
+  // definition that we previously skipped.
+  struct CheckSkippedDefinition {
+    // The definition that we skipped.
+    Parse::DeferredDefinitionIndex definition_index;
+    // The suspended function.
+    SuspendedFunction suspended_fn;
+  };
+
+  // A worklist task that indicates we should enter a nested deferred definition
+  // scope.
+  struct EnterDeferredDefinitionScope {
+    // The suspended scope. This is only set once we reach the end of the scope.
+    std::optional<DeclNameStack::SuspendedName> suspended_name;
+    // Whether this scope is itself within an outer deferred definition scope.
+    // If so, we'll delay processing its contents until we reach the end of the
+    // parent scope. For example:
+    //
+    // ```
+    // class A {
+    //   class B {
+    //     fn F() -> A { return {}; }
+    //   }
+    // } // A.B.F is type-checked here, with A complete.
+    //
+    // fn F() {
+    //   class C {
+    //     fn G() {}
+    //   } // C.G is type-checked here.
+    // }
+    // ```
+    bool in_deferred_definition_scope;
+  };
+
+  // A worklist task that indicates we should leave a deferred definition scope.
+  struct LeaveDeferredDefinitionScope {
+    // Whether this scope is within another deferred definition scope.
+    bool in_deferred_definition_scope;
+  };
+
+  // A pending type-checking task.
+  using Task =
+      std::variant<CheckSkippedDefinition, EnterDeferredDefinitionScope,
+                   LeaveDeferredDefinitionScope>;
+
+  explicit DeferredDefinitionWorklist(llvm::raw_ostream* vlog_stream);
+
+  // Suspend the current function definition and push a task onto the worklist
+  // to finish it later.
+  auto SuspendFunctionAndPush(Context& context,
+                              Parse::DeferredDefinitionIndex index,
+                              Parse::FunctionDefinitionStartId node_id) -> void;
+
+  // Push a task to re-enter a function scope, so that functions defined within
+  // it are type-checked in the right context.
+  auto PushEnterDeferredDefinitionScope(Context& context) -> void;
+
+  // Suspend the current deferred definition scope, which is finished but still
+  // on the decl_name_stack, and push a task to leave the scope when we're
+  // type-checking deferred definitions. Returns `true` if the current list of
+  // deferred definitions should be type-checked immediately.
+  auto SuspendFinishedScopeAndPush(Context& context) -> bool;
+
+  // Pop the next task off the worklist.
+  auto Pop() -> Task;
+
+  // CHECK that the work list has no further work.
+  auto VerifyEmpty() {
+    CARBON_CHECK(worklist_.empty() && entered_scopes_.empty(),
+                 "Tasks left behind on worklist.");
+  }
+
+ private:
+  // A deferred definition scope that is currently still open.
+  struct EnteredScope {
+    // The index in worklist_ of the EnterDeferredDefinitionScope task.
+    size_t worklist_start_index;
+    // The corresponding lexical scope index.
+    ScopeIndex scope_index;
+  };
+
+  llvm::raw_ostream* vlog_stream_;
+
+  // A worklist of type-checking tasks we'll need to do later.
+  //
+  // Don't allocate any inline storage here. A Task is fairly large, so we never
+  // want this to live on the stack. Instead, we reserve space in the
+  // constructor for a fairly large number of deferred definitions.
+  llvm::SmallVector<Task, 0> worklist_;
+
+  // The deferred definition scopes for the current checking actions.
+  llvm::SmallVector<EnteredScope> entered_scopes_;
+};
+
+}  // namespace Carbon::Check
+
+#endif  // CARBON_TOOLCHAIN_CHECK_DEFERRED_DEFINITION_WORKLIST_H_

toolchain/check/node_id_traversal.cpp

@@ -0,0 +1,175 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/check/node_id_traversal.h"
+
+#include "toolchain/check/handle.h"
+
+namespace Carbon::Check {
+
+NodeIdTraversal::NodeIdTraversal(Context& context,
+                                 llvm::raw_ostream* vlog_stream)
+    : context_(context),
+      next_deferred_definition_(&context.parse_tree()),
+      worklist_(vlog_stream) {
+  auto range = context.parse_tree().postorder();
+  chunks_.push_back(
+      {.it = range.begin(),
+       .end = range.end(),
+       .next_definition = Parse::DeferredDefinitionIndex::Invalid});
+}
+
+auto NodeIdTraversal::Next() -> std::optional<Parse::NodeId> {
+  while (true) {
+    // If we're checking deferred definitions, find the next definition we
+    // should check, restore its suspended state, and add a corresponding
+    // `Chunk` to the top of the chunk list.
+    if (chunks_.back().checking_deferred_definitions) {
+      std::visit(
+          [&](auto&& task) { PerformTask(std::forward<decltype(task)>(task)); },
+          worklist_.Pop());
+      continue;
+    }
+
+    // If we're not checking deferred definitions, produce the next parse node
+    // for this chunk. If we've run out of parse nodes, we're done with this
+    // chunk of the parse tree.
+    if (chunks_.back().it == chunks_.back().end) {
+      auto old_chunk = chunks_.pop_back_val();
+
+      // If we're out of chunks, then we're done entirely.
+      if (chunks_.empty()) {
+        worklist_.VerifyEmpty();
+        return std::nullopt;
+      }
+
+      next_deferred_definition_.SkipTo(old_chunk.next_definition);
+      continue;
+    }
+
+    auto node_id = *chunks_.back().it;
+
+    // If we've reached the start of a deferred definition, skip to the end of
+    // it, and track that we need to check it later.
+    if (node_id == next_deferred_definition_.start_id()) {
+      const auto& definition_info =
+          context_.parse_tree().deferred_definitions().Get(
+              next_deferred_definition_.index());
+      worklist_.SuspendFunctionAndPush(context_,
+                                       next_deferred_definition_.index(),
+                                       definition_info.start_id);
+
+      // Continue type-checking the parse tree after the end of the definition.
+      chunks_.back().it =
+          Parse::Tree::PostorderIterator(definition_info.definition_id) + 1;
+      next_deferred_definition_.SkipTo(definition_info.next_definition_index);
+      continue;
+    }
+
+    ++chunks_.back().it;
+    return node_id;
+  }
+}
+
+// Determines whether this node kind is the start of a deferred definition
+// scope.
+static auto IsStartOfDeferredDefinitionScope(Parse::NodeKind kind) -> bool {
+  switch (kind) {
+    case Parse::NodeKind::ClassDefinitionStart:
+    case Parse::NodeKind::ImplDefinitionStart:
+    case Parse::NodeKind::InterfaceDefinitionStart:
+    case Parse::NodeKind::NamedConstraintDefinitionStart:
+      // TODO: Mixins.
+      return true;
+    default:
+      return false;
+  }
+}
+
+// Determines whether this node kind is the end of a deferred definition scope.
+static auto IsEndOfDeferredDefinitionScope(Parse::NodeKind kind) -> bool {
+  switch (kind) {
+    case Parse::NodeKind::ClassDefinition:
+    case Parse::NodeKind::ImplDefinition:
+    case Parse::NodeKind::InterfaceDefinition:
+    case Parse::NodeKind::NamedConstraintDefinition:
+      // TODO: Mixins.
+      return true;
+    default:
+      return false;
+  }
+}
+
+// TODO: Investigate factoring out `IsStartOfDeferredDefinitionScope` and
+// `IsEndOfDeferredDefinitionScope` in order to make `NodeIdTraversal`
+// reusable.
+auto NodeIdTraversal::Handle(Parse::NodeKind parse_kind) -> void {
+  // When we reach the start of a deferred definition scope, add a task to the
+  // worklist to check future skipped definitions in the new context.
+  if (IsStartOfDeferredDefinitionScope(parse_kind)) {
+    worklist_.PushEnterDeferredDefinitionScope(context_);
+  }
+
+  // When we reach the end of a deferred definition scope, add a task to the
+  // worklist to leave the scope. If this is not a nested scope, start
+  // checking the deferred definitions now.
+  if (IsEndOfDeferredDefinitionScope(parse_kind)) {
+    chunks_.back().checking_deferred_definitions =
+        worklist_.SuspendFinishedScopeAndPush(context_);
+  }
+}
+
+auto NodeIdTraversal::PerformTask(
+    DeferredDefinitionWorklist::EnterDeferredDefinitionScope&& enter) -> void {
+  CARBON_CHECK(enter.suspended_name,
+               "Entering a scope with no suspension information.");
+  context_.decl_name_stack().Restore(std::move(*enter.suspended_name));
+}
+
+auto NodeIdTraversal::PerformTask(
+    DeferredDefinitionWorklist::LeaveDeferredDefinitionScope&& leave) -> void {
+  if (!leave.in_deferred_definition_scope) {
+    // We're done with checking deferred definitions.
+    chunks_.back().checking_deferred_definitions = false;
+  }
+  context_.decl_name_stack().PopScope();
+}
+
+auto NodeIdTraversal::PerformTask(
+    DeferredDefinitionWorklist::CheckSkippedDefinition&& parse_definition)
+    -> void {
+  auto& [definition_index, suspended_fn] = parse_definition;
+  const auto& definition_info =
+      context_.parse_tree().deferred_definitions().Get(definition_index);
+  HandleFunctionDefinitionResume(context_, definition_info.start_id,
+                                 std::move(suspended_fn));
+  auto range = Parse::Tree::PostorderIterator::MakeRange(
+      definition_info.start_id, definition_info.definition_id);
+  chunks_.push_back({.it = range.begin() + 1,
+                     .end = range.end(),
+                     .next_definition = next_deferred_definition_.index()});
+  ++definition_index.index;
+  next_deferred_definition_.SkipTo(definition_index);
+}
+
+NodeIdTraversal::NextDeferredDefinitionCache::NextDeferredDefinitionCache(
+    const Parse::Tree* tree)
+    : tree_(tree) {
+  SkipTo(Parse::DeferredDefinitionIndex(0));
+}
+
+// Set the specified deferred definition index as being the next one that
+// will be encountered.
+auto NodeIdTraversal::NextDeferredDefinitionCache::SkipTo(
+    Parse::DeferredDefinitionIndex next_index) -> void {
+  index_ = next_index;
+  if (static_cast<size_t>(index_.index) ==
+      tree_->deferred_definitions().size()) {
+    start_id_ = Parse::NodeId::Invalid;
+  } else {
+    start_id_ = tree_->deferred_definitions().Get(index_).start_id;
+  }
+}
+
+}  // namespace Carbon::Check

toolchain/check/node_id_traversal.h

@@ -0,0 +1,87 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_CHECK_NODE_ID_TRAVERSAL_H_
+#define CARBON_TOOLCHAIN_CHECK_NODE_ID_TRAVERSAL_H_
+
+#include "common/ostream.h"
+#include "llvm/ADT/SmallVector.h"
+#include "toolchain/check/context.h"
+#include "toolchain/check/deferred_definition_worklist.h"
+#include "toolchain/parse/tree.h"
+
+namespace Carbon::Check {
+
+// A traversal of the node IDs in the parse tree, in the order in which we need
+// to check them.
+class NodeIdTraversal {
+ public:
+  explicit NodeIdTraversal(Context& context, llvm::raw_ostream* vlog_stream);
+
+  // Finds the next `NodeId` to type-check. Returns nullopt if the traversal is
+  // complete.
+  auto Next() -> std::optional<Parse::NodeId>;
+
+  // Performs any processing necessary after we type-check a node.
+  auto Handle(Parse::NodeKind parse_kind) -> void;
+
+ private:
+  // State used to track the next deferred function definition that we will
+  // encounter and need to reorder.
+  class NextDeferredDefinitionCache {
+   public:
+    explicit NextDeferredDefinitionCache(const Parse::Tree* tree);
+
+    // Set the specified deferred definition index as being the next one that
+    // will be encountered.
+    auto SkipTo(Parse::DeferredDefinitionIndex next_index) -> void;
+
+    // Returns the index of the next deferred definition to be encountered.
+    auto index() const -> Parse::DeferredDefinitionIndex { return index_; }
+
+    // Returns the ID of the start node of the next deferred definition.
+    auto start_id() const -> Parse::NodeId { return start_id_; }
+
+   private:
+    const Parse::Tree* tree_;
+    Parse::DeferredDefinitionIndex index_ =
+        Parse::DeferredDefinitionIndex::Invalid;
+    Parse::NodeId start_id_ = Parse::NodeId::Invalid;
+  };
+
+  // A chunk of the parse tree that we need to type-check.
+  struct Chunk {
+    Parse::Tree::PostorderIterator it;
+    Parse::Tree::PostorderIterator end;
+    // The next definition that will be encountered after this chunk completes.
+    Parse::DeferredDefinitionIndex next_definition;
+    // Whether we are currently checking deferred definitions, rather than the
+    // tokens of this chunk. If so, we'll pull tasks off `worklist` and execute
+    // them until we're done with this batch of deferred definitions. Otherwise,
+    // we'll pull node IDs from `*it` until it reaches `end`.
+    bool checking_deferred_definitions = false;
+  };
+
+  // Re-enter a nested deferred definition scope.
+  auto PerformTask(
+      DeferredDefinitionWorklist::EnterDeferredDefinitionScope&& enter) -> void;
+
+  // Leave a nested or top-level deferred definition scope.
+  auto PerformTask(
+      DeferredDefinitionWorklist::LeaveDeferredDefinitionScope&& leave) -> void;
+
+  // Resume checking a deferred definition.
+  auto PerformTask(
+      DeferredDefinitionWorklist::CheckSkippedDefinition&& parse_definition)
+      -> void;
+
+  Context& context_;
+  NextDeferredDefinitionCache next_deferred_definition_;
+  DeferredDefinitionWorklist worklist_;
+  llvm::SmallVector<Chunk> chunks_;
+};
+
+}  // namespace Carbon::Check
+
+#endif  // CARBON_TOOLCHAIN_CHECK_NODE_ID_TRAVERSAL_H_