Move the language server into toolchain's busybox. (#4469)

Removes the separate language server binary; I'm not sure we need to
provide it. Instead, `carbon language-server` is added as a subcommand.

Moves //language_server to //toolchain/language_server. Splits into a
trivial language_server.h, and a substantive server.h. I wasn't sure
about a better name, but wanted the split similar to check/check.h,
lex/lex.h, etc. At the same time, the class is probably going to be a
little big so not a good fit to through into just a cpp file.

This fixes some style issues with the language server class, but
generally I'm trying to not address things here in order to keep it
simpler.

.github/workflows/auto_label_prs.yaml

@@ -63,7 +63,6 @@ jobs:
             toolchain:
               - 'common/**'
               - 'core/**'
-              - 'language_server/**'
               - 'testing/**'
               - 'toolchain/**'
 

bazel/check_deps/BUILD

@@ -15,7 +15,6 @@ filegroup(
     data = [
         "//explorer",
         "//installers/local:carbon",
-        "//language_server",
         "//migrate_cpp:rewriter",
         "//migrate_cpp/cpp_refactoring",
         "//toolchain/install:carbon-busybox",

language_server/main.cpp

@@ -1,10 +0,0 @@
-// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
-// Exceptions. See /LICENSE for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#include "language_server/language_server.h"
-
-auto main(int /*argc*/, char** /*argv*/) -> int {
-  Carbon::LS::LanguageServer::Start();
-  return 0;
-}

toolchain/driver/BUILD

@@ -91,6 +91,8 @@ cc_library(
         "driver_env.h",
         "format_subcommand.cpp",
         "format_subcommand.h",
+        "language_server_subcommand.cpp",
+        "language_server_subcommand.h",
         "link_subcommand.cpp",
         "link_subcommand.h",
     ],
@@ -116,6 +118,7 @@ cc_library(
         "//toolchain/diagnostics:sorting_diagnostic_consumer",
         "//toolchain/format",
         "//toolchain/install:install_paths",
+        "//toolchain/language_server",
         "//toolchain/lex",
         "//toolchain/lower",
         "//toolchain/parse",

toolchain/driver/driver.cpp

@@ -13,6 +13,7 @@
 #include "toolchain/driver/clang_subcommand.h"
 #include "toolchain/driver/compile_subcommand.h"
 #include "toolchain/driver/format_subcommand.h"
+#include "toolchain/driver/language_server_subcommand.h"
 #include "toolchain/driver/link_subcommand.h"
 
 namespace Carbon {
@@ -29,6 +30,7 @@ struct Options {
   ClangSubcommand clang;
   CompileSubcommand compile;
   FormatSubcommand format;
+  LanguageServerSubcommand language_server;
   LinkSubcommand link;
 
   // On success, this is set to the subcommand to run.
@@ -87,6 +89,11 @@ auto Options::Build(CommandLine::CommandBuilder& b) -> void {
     sub_b.Do([&] { subcommand = &format; });
   });
 
+  b.AddSubcommand(LanguageServerSubcommand::Info,
+                  [&](CommandLine::CommandBuilder& sub_b) {
+                    sub_b.Do([&] { subcommand = &language_server; });
+                  });
+
   b.AddSubcommand(LinkOptions::Info, [&](CommandLine::CommandBuilder& sub_b) {
     link.BuildOptions(sub_b);
     sub_b.Do([&] { subcommand = &link; });

toolchain/driver/language_server_subcommand.cpp

@@ -0,0 +1,28 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/driver/language_server_subcommand.h"
+
+#include "toolchain/language_server/language_server.h"
+
+namespace Carbon {
+
+constexpr CommandLine::CommandInfo LanguageServerSubcommand::Info = {
+    .name = "language-server",
+    .help = R"""(
+Runs the language server.
+)""",
+};
+
+auto LanguageServerSubcommand::Run(DriverEnv& driver_env) -> DriverResult {
+  // TODO: Consider a way to override stdin, but it's a `FILE*` so less
+  // convenient to work with.
+  auto err = LanguageServer::Run(stdin, driver_env.output_stream);
+  if (!err.ok()) {
+    driver_env.error_stream << "error: " << err.error() << "\n";
+  }
+  return {.success = err.ok()};
+}
+
+}  // namespace Carbon

toolchain/driver/language_server_subcommand.h

@@ -0,0 +1,27 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_DRIVER_LANGUAGE_SERVER_SUBCOMMAND_H_
+#define CARBON_TOOLCHAIN_DRIVER_LANGUAGE_SERVER_SUBCOMMAND_H_
+
+#include "common/command_line.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "toolchain/driver/codegen_options.h"
+#include "toolchain/driver/driver_env.h"
+#include "toolchain/driver/driver_subcommand.h"
+
+namespace Carbon {
+
+// Implements the link subcommand of the driver.
+class LanguageServerSubcommand : public DriverSubcommand {
+ public:
+  static const CommandLine::CommandInfo Info;
+
+  auto Run(DriverEnv& driver_env) -> DriverResult override;
+};
+
+}  // namespace Carbon
+
+#endif  // CARBON_TOOLCHAIN_DRIVER_LANGUAGE_SERVER_SUBCOMMAND_H_

toolchain/language_server/BUILD

@@ -2,25 +2,23 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("@rules_cc//cc:defs.bzl", "cc_binary")
+load("@rules_cc//cc:defs.bzl", "cc_library")
 
-package(default_visibility = [
-    "//bazel/check_deps:__pkg__",
-    "//installers:__subpackages__",
-    "//language_server:__subpackages__",
-])
+package(default_visibility = ["//visibility:public"])
 
-cc_binary(
+cc_library(
     name = "language_server",
     srcs = [
         "language_server.cpp",
-        "language_server.h",
-        "main.cpp",
+        "server.cpp",
+        "server.h",
     ],
+    hdrs = ["language_server.h"],
     # Some parameters are unused in clangd headers.
     copts = ["-Wno-unused-parameter"],
     deps = [
         "//common:error",
+        "//common:ostream",
         "//toolchain/base:value_store",
         "//toolchain/diagnostics:null_diagnostics",
         "//toolchain/lex",

toolchain/language_server/language_server.cpp

@@ -0,0 +1,17 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/language_server/language_server.h"
+
+#include "toolchain/language_server/server.h"
+
+namespace Carbon::LanguageServer {
+
+auto Run(std::FILE* input_stream, llvm::raw_ostream& output_stream)
+    -> ErrorOr<Success> {
+  Server server(input_stream, output_stream);
+  return server.Run();
+}
+
+}  // namespace Carbon::LanguageServer

toolchain/language_server/language_server.h

@@ -0,0 +1,19 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_TOOLCHAIN_LANGUAGE_SERVER_LANGUAGE_SERVER_H_
+#define CARBON_TOOLCHAIN_LANGUAGE_SERVER_LANGUAGE_SERVER_H_
+
+#include "common/error.h"
+#include "common/ostream.h"
+
+namespace Carbon::LanguageServer {
+
+// Start the language server.
+auto Run(std::FILE* input_stream, llvm::raw_ostream& output_stream)
+    -> ErrorOr<Success>;
+
+}  // namespace Carbon::LanguageServer
+
+#endif  // CARBON_TOOLCHAIN_LANGUAGE_SERVER_LANGUAGE_SERVER_H_

toolchain/language_server/server.cpp

@@ -2,9 +2,8 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "language_server/language_server.h"
+#include "toolchain/language_server/server.h"
 
-#include "clang-tools-extra/clangd/Protocol.h"
 #include "toolchain/base/value_store.h"
 #include "toolchain/diagnostics/null_diagnostics.h"
 #include "toolchain/lex/lex.h"
@@ -13,22 +12,48 @@
 #include "toolchain/parse/tree_and_subtrees.h"
 #include "toolchain/source/source_buffer.h"
 
-namespace Carbon::LS {
+namespace Carbon::LanguageServer {
 
-void LanguageServer::OnDidOpenTextDocument(
+Server::Server(std::FILE* input_stream, llvm::raw_ostream& output_stream)
+    : transport_(clang::clangd::newJSONTransport(input_stream, output_stream,
+                                                 /*InMirror=*/nullptr,
+                                                 /*Pretty=*/true)),
+      binder_(handlers_, *this) {
+  binder_.notification("textDocument/didOpen", this,
+                       &Server::OnDidOpenTextDocument);
+  binder_.notification("textDocument/didChange", this,
+                       &Server::OnDidChangeTextDocument);
+  binder_.method("initialize", this, &Server::OnInitialize);
+  binder_.method("textDocument/documentSymbol", this,
+                 &Server::OnDocumentSymbol);
+}
+
+auto Server::Run() -> ErrorOr<Success> {
+  llvm::Error err = transport_->loop(*this);
+  if (err.success()) {
+    return Success();
+  } else {
+    std::string str;
+    llvm::raw_string_ostream out(str);
+    out << err;
+    return Error(str);
+  }
+}
+
+void Server::OnDidOpenTextDocument(
     clang::clangd::DidOpenTextDocumentParams const& params) {
   files_.emplace(params.textDocument.uri.file(), params.textDocument.text);
 }
 
-void LanguageServer::OnDidChangeTextDocument(
+void Server::OnDidChangeTextDocument(
     clang::clangd::DidChangeTextDocumentParams const& params) {
-  // full text is sent if full sync is specified in capabilities.
-  assert(params.contentChanges.size() == 1);
+  // Full text is sent if full sync is specified in capabilities.
+  CARBON_CHECK(params.contentChanges.size() == 1);
   std::string file = params.textDocument.uri.file().str();
   files_[file] = params.contentChanges[0].text;
 }
 
-void LanguageServer::OnInitialize(
+void Server::OnInitialize(
     clang::clangd::NoParams const& /*client_capabilities*/,
     clang::clangd::Callback<llvm::json::Object> cb) {
   llvm::json::Object capabilities{{"documentSymbolProvider", true},
@@ -38,8 +63,7 @@ void LanguageServer::OnInitialize(
   cb(reply);
 }
 
-auto LanguageServer::onNotify(llvm::StringRef method, llvm::json::Value value)
-    -> bool {
+auto Server::onNotify(llvm::StringRef method, llvm::json::Value value) -> bool {
   if (method == "exit") {
     return false;
   }
@@ -53,8 +77,8 @@ auto LanguageServer::onNotify(llvm::StringRef method, llvm::json::Value value)
   return true;
 }
 
-auto LanguageServer::onCall(llvm::StringRef method, llvm::json::Value params,
-                            llvm::json::Value id) -> bool {
+auto Server::onCall(llvm::StringRef method, llvm::json::Value params,
+                    llvm::json::Value id) -> bool {
   if (auto handler = handlers_.MethodHandlers.find(method);
       handler != handlers_.MethodHandlers.end()) {
     // TODO: improve this if add threads
@@ -71,9 +95,8 @@ auto LanguageServer::onCall(llvm::StringRef method, llvm::json::Value params,
   return true;
 }
 
-auto LanguageServer::onReply(llvm::json::Value /*id*/,
-                             llvm::Expected<llvm::json::Value> /*result*/)
-    -> bool {
+auto Server::onReply(llvm::json::Value /*id*/,
+                     llvm::Expected<llvm::json::Value> /*result*/) -> bool {
   return true;
 }
 
@@ -94,7 +117,7 @@ static auto GetIdentifierName(const SharedValueStores& value_stores,
   return std::nullopt;
 }
 
-void LanguageServer::OnDocumentSymbol(
+void Server::OnDocumentSymbol(
     clang::clangd::DocumentSymbolParams const& params,
     clang::clangd::Callback<std::vector<clang::clangd::DocumentSymbol>> cb) {
   SharedValueStores value_stores;
@@ -148,19 +171,4 @@ void LanguageServer::OnDocumentSymbol(
   cb(result);
 }
 
-void LanguageServer::Start() {
-  auto transport =
-      clang::clangd::newJSONTransport(stdin, llvm::outs(), nullptr, true);
-  LanguageServer ls(std::move(transport));
-  clang::clangd::LSPBinder binder(ls.handlers_, ls);
-  binder.notification("textDocument/didOpen", &ls,
-                      &LanguageServer::OnDidOpenTextDocument);
-  binder.notification("textDocument/didChange", &ls,
-                      &LanguageServer::OnDidChangeTextDocument);
-  binder.method("initialize", &ls, &LanguageServer::OnInitialize);
-  binder.method("textDocument/documentSymbol", &ls,
-                &LanguageServer::OnDocumentSymbol);
-  auto error = ls.transport_->loop(ls);
-  llvm::errs() << "Error: " << error << "\n";
-}
-}  // namespace Carbon::LS
+}  // namespace Carbon::LanguageServer

toolchain/language_server/server.h

@@ -2,25 +2,30 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#ifndef CARBON_LANGUAGE_SERVER_LANGUAGE_SERVER_H_
-#define CARBON_LANGUAGE_SERVER_LANGUAGE_SERVER_H_
+#ifndef CARBON_TOOLCHAIN_LANGUAGE_SERVER_SERVER_H_
+#define CARBON_TOOLCHAIN_LANGUAGE_SERVER_SERVER_H_
+
+#include <memory>
 #include <unordered_map>
 #include <vector>
 
 #include "clang-tools-extra/clangd/LSPBinder.h"
 #include "clang-tools-extra/clangd/Protocol.h"
 #include "clang-tools-extra/clangd/Transport.h"
-#include "clang-tools-extra/clangd/support/Function.h"
-#include "toolchain/lex/tokenized_buffer.h"
-#include "toolchain/parse/tree.h"
-#include "toolchain/source/source_buffer.h"
+#include "common/error.h"
 
-namespace Carbon::LS {
-class LanguageServer : public clang::clangd::Transport::MessageHandler,
-                       public clang::clangd::LSPBinder::RawOutgoing {
+namespace Carbon::LanguageServer {
+
+// Provides a LSP implementation for Carbon.
+class Server : public clang::clangd::Transport::MessageHandler,
+               public clang::clangd::LSPBinder::RawOutgoing {
  public:
-  // Start the language server.
-  static void Start();
+  // Prepares the server to run on the provided streams.
+  explicit Server(std::FILE* input_stream, llvm::raw_ostream& output_stream);
+
+  // Runs the server in a loop, returning the result. Currently this always
+  // returns an error when the input stream is closed.
+  auto Run() -> ErrorOr<Success>;
 
   // Transport::MessageHandler
   // Handlers returns true to keep processing messages, or false to shut down.
@@ -38,46 +43,48 @@ class LanguageServer : public clang::clangd::Transport::MessageHandler,
   // LSPBinder::RawOutgoing
 
   // Send method call to client
-  void callMethod(llvm::StringRef method, llvm::json::Value params,
-                  clang::clangd::Callback<llvm::json::Value> reply) override {
+  auto callMethod(llvm::StringRef method, llvm::json::Value params,
+                  clang::clangd::Callback<llvm::json::Value> reply)
+      -> void override {
     // TODO: implement when needed
   }
 
   // Send notification to client
-  void notify(llvm::StringRef method, llvm::json::Value params) override {
+  auto notify(llvm::StringRef method, llvm::json::Value params)
+      -> void override {
     transport_->notify(method, params);
   }
 
  private:
+  // Typed handlers for notifications and method calls by client.
+
+  // Client opened a document.
+  auto OnDidOpenTextDocument(
+      clang::clangd::DidOpenTextDocumentParams const& params) -> void;
+
+  // Client updated content of a document.
+  auto OnDidChangeTextDocument(
+      clang::clangd::DidChangeTextDocumentParams const& params) -> void;
+
+  // Capabilities negotiation
+  auto OnInitialize(clang::clangd::NoParams const& client_capabilities,
+                    clang::clangd::Callback<llvm::json::Object> cb) -> void;
+
+  // Code outline
+  auto OnDocumentSymbol(
+      clang::clangd::DocumentSymbolParams const& params,
+      clang::clangd::Callback<std::vector<clang::clangd::DocumentSymbol>> cb)
+      -> void;
+
   const std::unique_ptr<clang::clangd::Transport> transport_;
   // content of files managed by the language client.
   std::unordered_map<std::string, std::string> files_;
   // handlers for client methods and notifications
   clang::clangd::LSPBinder::RawHandlers handlers_;
-
-  explicit LanguageServer(std::unique_ptr<clang::clangd::Transport> transport)
-      : transport_(std::move(transport)) {}
-
-  // Typed handlers for notifications and method calls by client.
-
-  // Client opened a document.
-  void OnDidOpenTextDocument(
-      clang::clangd::DidOpenTextDocumentParams const& params);
-
-  // Client updated content of a document.
-  void OnDidChangeTextDocument(
-      clang::clangd::DidChangeTextDocumentParams const& params);
-
-  // Capabilities negotiation
-  void OnInitialize(clang::clangd::NoParams const& client_capabilities,
-                    clang::clangd::Callback<llvm::json::Object> cb);
-
-  // Code outline
-  void OnDocumentSymbol(
-      clang::clangd::DocumentSymbolParams const& params,
-      clang::clangd::Callback<std::vector<clang::clangd::DocumentSymbol>> cb);
+  // Binds client calls to member methods.
+  clang::clangd::LSPBinder binder_;
 };
 
-}  // namespace Carbon::LS
+}  // namespace Carbon::LanguageServer
 
-#endif  // CARBON_LANGUAGE_SERVER_LANGUAGE_SERVER_H_
+#endif  // CARBON_TOOLCHAIN_LANGUAGE_SERVER_SERVER_H_

utils/README.md

@@ -21,5 +21,5 @@ developers and developers writing Carbon code.
 
 Any editor that supports Language server protocol and/or tree-sitter is
 supported. The editor just needs to be configured manually.
-`bazel build language_server` produces the language server binary.
+`bazel build //toolchain` produces the language server binary.
 `utils/treesitter` contains the treesitter grammar.

utils/nvim/README.md

@@ -11,7 +11,7 @@ Treesitter based syntax highlighting and language server client for Neovim.
 This requires neovim >= 0.9 and
 [nvim-lspconfig](https://github.com/neovim/nvim-lspconfig) to be installed.
 
-1. Run `bazel build language_server` in project root.
+1. Run `bazel build //toolchain` in project root.
 2. Run `utils/nvim/setup.sh`.
 3. Start nvim in carbon-lang root folder and open a carbon file.
 4. View document symbols. If you have telescope.nvim installed, you can use

utils/nvim/carbon.lua

@@ -19,7 +19,7 @@ local util = require 'lspconfig.util'
 if not configs.carbon then
   configs.carbon = {
     default_config = {
-      cmd = { "./bazel-bin/language_server/language_server" },
+      cmd = { "./bazel-bin/toolchain/install/run_carbon language-server" },
       filetypes = { "carbon" },
       root_dir = util.find_git_ancestor,
     }

utils/vscode/README.md

@@ -25,7 +25,7 @@ code --install-extension out/carbon.vsix
 
 ## Development
 
-1. `bazel build language_server` in project root.
+1. `bazel build //toolchain` in project root.
 2. Open utils/vscode folder in VS Code.
 3. Launch the extension using Run command (F5).
 4. In the opened window, open the carbon-lang repository as folder.

utils/vscode/src/extension.js

@@ -7,10 +7,11 @@
 const { LanguageClient } = require('vscode-languageclient/node');
 
 function activate(context) {
-  const command = './bazel-bin/language_server/language_server';
+  const command = './bazel-bin/toolchain/install/run_carbon';
+  const args = ['language-server'];
   const serverOptions = {
-    run: { command },
-    debug: { command },
+    run: { command: command, args: args },
+    debug: { command: command, args: args },
   };
 
   const clientOptions = {