Add a clang subcommand. (#4322)

This makes something like `bazel run :toolchain -- clang -- -c test.cpp`
work, because that can be run in-process. Note that `bazel run
:toolchain -- clang -- test.cpp` still requires subprocessing, and does
not work.

Note, the vision here is that we are trying to align how clang and
carbon compile c++ code. This is work towards intertwining command
execution.

---------

Co-authored-by: Chandler Carruth <chandlerc@gmail.com>

toolchain/driver/BUILD

@@ -26,6 +26,7 @@ cc_library(
         "//common:vlog",
         "//toolchain/install:install_paths",
         "@llvm-project//clang:basic",
+        "@llvm-project//clang:clang-driver",
         "@llvm-project//clang:driver",
         "@llvm-project//clang:frontend",
         "@llvm-project//llvm:Core",
@@ -77,6 +78,8 @@ sh_test(
 cc_library(
     name = "driver",
     srcs = [
+        "clang_subcommand.cpp",
+        "clang_subcommand.h",
         "codegen_options.cpp",
         "codegen_options.h",
         "compile_subcommand.cpp",

toolchain/driver/clang_runner.cpp

@@ -23,11 +23,22 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/LLVMDriver.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/Host.h"
 
+// Defined in:
+// https://github.com/llvm/llvm-project/blob/main/clang/tools/driver/driver.cpp
+//
+// While not in a header, this is the API used by llvm-driver.cpp for
+// busyboxing.
+//
+// NOLINTNEXTLINE(readability-identifier-naming)
+auto clang_main(int Argc, char** Argv, const llvm::ToolContext& ToolContext)
+    -> int;
+
 namespace Carbon {
 
 ClangRunner::ClangRunner(const InstallPaths* install_paths,
@@ -51,7 +62,6 @@ auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args) -> bool {
     maybe_v_arg = v_arg_storage;
   }
 
-  CARBON_CHECK(!args.empty());
   CARBON_VLOG("Running Clang driver with arguments: \n");
 
   // Render the arguments into null-terminated C-strings for use by the Clang
@@ -113,9 +123,23 @@ auto ClangRunner::Run(llvm::ArrayRef<llvm::StringRef> args) -> bool {
   driver.Dir = installation_->llvm_install_bin();
   CARBON_VLOG("Setting bin directory to: {0}\n", driver.Dir);
 
-  // TODO: Directly run in-process rather than using a subprocess. This is both
-  // more efficient and makes debugging (much) easier. Needs code like:
-  // driver.CC1Main = [](llvm::SmallVectorImpl<const char*>& argv) {};
+  // When there's only one command being run, this will run it in-process.
+  // However, a `clang` invocation may cause multiple `cc1` invocations, which
+  // still subprocess. See `InProcess` comment at:
+  // https://github.com/llvm/llvm-project/blob/86ce8e4504c06ecc3cc42f002ad4eb05cac10925/clang/lib/Driver/Job.cpp#L411-L413
+  //
+  // TODO: It would be nice to find a way to set up the driver's understanding
+  // of the executable name in a way that causes the multiple `cc1` invocations
+  // to actually result in `carbon clang -- ...` invocations (even if as
+  // subprocesses). This may dovetail with having symlinks that redirect to a
+  // busybox of LLD as well, and having even the subprocesses consistently run
+  // the Carbon install toolchain and not a system toolchain whenever possible.
+  driver.CC1Main = [](llvm::SmallVectorImpl<const char*>& argv) -> int {
+    llvm::ToolContext tool_context;
+    return clang_main(argv.size(), const_cast<char**>(argv.data()),
+                      tool_context);
+  };
+
   std::unique_ptr<clang::driver::Compilation> compilation(
       driver.BuildCompilation(cstr_args));
   CARBON_CHECK(compilation, "Should always successfully allocate!");

toolchain/driver/clang_runner_test.cpp

@@ -152,5 +152,47 @@ TEST(ClangRunnerTest, LinkCommandEcho) {
   EXPECT_THAT(out, StrEq(""));
 }
 
+TEST(ClangRunnerTest, NoArgs) {
+  const auto install_paths =
+      InstallPaths::MakeForBazelRunfiles(Testing::GetExePath());
+  std::string verbose_out;
+  llvm::raw_string_ostream verbose_os(verbose_out);
+  std::string target = llvm::sys::getDefaultTargetTriple();
+  ClangRunner runner(&install_paths, target, &verbose_os);
+  std::string out;
+  std::string err;
+  EXPECT_FALSE(RunWithCapturedOutput(out, err, [&] { return runner.Run({}); }))
+      << "Verbose output from runner:\n"
+      << verbose_out << "\n";
+
+  EXPECT_THAT(out, StrEq(""));
+  EXPECT_THAT(err, HasSubstr("error: no input files"));
+}
+
+TEST(ClangRunnerTest, DashC) {
+  std::filesystem::path test_file =
+      WriteTestFile("test.cpp", "int test() { return 0; }");
+
+  const auto install_paths =
+      InstallPaths::MakeForBazelRunfiles(Testing::GetExePath());
+  std::string verbose_out;
+  llvm::raw_string_ostream verbose_os(verbose_out);
+  std::string target = llvm::sys::getDefaultTargetTriple();
+  ClangRunner runner(&install_paths, target, &verbose_os);
+  std::string out;
+  std::string err;
+  EXPECT_TRUE(
+      RunWithCapturedOutput(out, err,
+                            [&] {
+                              return runner.Run({"-c", test_file.string()});
+                            }))
+      << "Verbose output from runner:\n"
+      << verbose_out << "\n";
+
+  // No output should be produced.
+  EXPECT_THAT(out, StrEq(""));
+  EXPECT_THAT(err, StrEq(""));
+}
+
 }  // namespace
 }  // namespace Carbon

toolchain/driver/clang_subcommand.cpp

@@ -0,0 +1,50 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/driver/clang_subcommand.h"
+
+#include "llvm/TargetParser/Host.h"
+#include "toolchain/driver/clang_runner.h"
+
+namespace Carbon {
+
+constexpr CommandLine::CommandInfo ClangOptions::Info = {
+    .name = "clang",
+    .help = R"""(
+Runs Clang on arguments.
+
+This is equivalent to running the `clang` command line directly, and provides
+the full command line interface.
+
+Use `carbon clang -- ARGS` to pass flags to `clang`. Although there are
+currently no flags for `carbon clang`, the `--` reserves the ability to add
+flags in the future.
+
+This is provided to help guarantee consistent compilation of C++ files, both
+when Clang is invoked directly and when a Carbon file importing a C++ file
+results in an indirect Clang invocation.
+)""",
+};
+
+auto ClangOptions::Build(CommandLine::CommandBuilder& b) -> void {
+  b.AddStringPositionalArg(
+      {
+          .name = "ARG",
+          .help = R"""(
+Arguments passed to Clang.
+)""",
+      },
+      [&](auto& arg_b) { arg_b.Append(&args); });
+}
+
+// TODO: This lacks a lot of features from the main driver code. We may need to
+// add more.
+// https://github.com/llvm/llvm-project/blob/main/clang/tools/driver/driver.cpp
+auto ClangSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
+  std::string target = llvm::sys::getDefaultTargetTriple();
+  ClangRunner runner(driver_env.installation, target, driver_env.vlog_stream);
+  return {.success = runner.Run(options_.args)};
+}
+
+}  // namespace Carbon

toolchain/driver/clang_subcommand.h

@@ -0,0 +1,40 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_DRIVER_CLANG_SUBCOMMAND_H_
+#define CARBON_TOOLCHAIN_DRIVER_CLANG_SUBCOMMAND_H_
+
+#include "common/command_line.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "toolchain/driver/driver_env.h"
+#include "toolchain/driver/driver_subcommand.h"
+
+namespace Carbon {
+
+// Options for the clang subcommand, which is just a thin wrapper.
+//
+// See the implementation of `Build` for documentation on members.
+struct ClangOptions {
+  static const CommandLine::CommandInfo Info;
+
+  auto Build(CommandLine::CommandBuilder& b) -> void;
+
+  llvm::SmallVector<llvm::StringRef> args;
+};
+
+// Implements the clang subcommand of the driver.
+class ClangSubcommand : public DriverSubcommand {
+ public:
+  auto BuildOptions(CommandLine::CommandBuilder& b) { options_.Build(b); }
+
+  auto Run(DriverEnv& driver_env) -> DriverResult override;
+
+ private:
+  ClangOptions options_;
+};
+
+}  // namespace Carbon
+
+#endif  // CARBON_TOOLCHAIN_DRIVER_CLANG_SUBCOMMAND_H_

toolchain/driver/driver.cpp

@@ -10,12 +10,9 @@
 
 #include "common/command_line.h"
 #include "common/version.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/Support/Path.h"
-#include "toolchain/base/value_store.h"
+#include "toolchain/driver/clang_subcommand.h"
+#include "toolchain/driver/compile_subcommand.h"
+#include "toolchain/driver/link_subcommand.h"
 
 namespace Carbon {
 
@@ -27,6 +24,7 @@ struct Options {
 
   bool verbose;
 
+  ClangSubcommand clang;
   CompileSubcommand compile;
   LinkSubcommand link;
 
@@ -63,6 +61,11 @@ auto Options::Build(CommandLine::CommandBuilder& b) -> void {
       },
       [&](CommandLine::FlagBuilder& arg_b) { arg_b.Set(&verbose); });
 
+  b.AddSubcommand(ClangOptions::Info, [&](CommandLine::CommandBuilder& sub_b) {
+    clang.BuildOptions(sub_b);
+    sub_b.Do([&] { subcommand = &clang; });
+  });
+
   b.AddSubcommand(CompileOptions::Info,
                   [&](CommandLine::CommandBuilder& sub_b) {
                     compile.BuildOptions(sub_b);

toolchain/driver/driver.h

@@ -8,11 +8,8 @@
 #include "common/command_line.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
-#include "toolchain/driver/codegen_options.h"
-#include "toolchain/driver/compile_subcommand.h"
 #include "toolchain/driver/driver_env.h"
 #include "toolchain/driver/driver_subcommand.h"
-#include "toolchain/driver/link_subcommand.h"
 
 namespace Carbon {