Treesitter parser (#2902)

A push towards better editor support.
See utils/treesitter/README.md

---------

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

utils/treesitter/src/scanner.c

@@ -0,0 +1,87 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "tree_sitter/parser.h"
+
+enum TokenType {
+  BINARY_STAR,
+  POSTFIX_STAR,
+};
+
+// our scanner is stateless
+void* tree_sitter_carbon_external_scanner_create() { return NULL; }
+
+unsigned tree_sitter_carbon_external_scanner_serialize(
+    __attribute__((unused)) void* payload,
+    __attribute__((unused)) char* buffer) {
+  return 0;  // zero bytes used to serialize
+}
+
+void tree_sitter_carbon_external_scanner_deserialize(
+    __attribute__((unused)) void* payload,
+    __attribute__((unused)) const char* buffer,
+    __attribute__((unused)) unsigned length) {}
+
+void tree_sitter_carbon_external_scanner_destroy(
+    __attribute__((unused)) void* payload) {}
+
+// https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md#overview
+// > the token after the operator must be an identifier, a literal, or any kind
+// of opening bracket (for example, (, [, or {).
+static bool token_allowed_after_binary_operator(char c) {
+  return
+      // identifier
+      c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+      // string literal
+      c == '\"' ||
+      // TODO: character literal
+      // number literal
+      (c >= '0' && c <= '9') ||
+      // opening bracket
+      c == '(' || c == '[' || c == '{';
+}
+
+static bool is_whitespace(char c) { return c == ' ' || c == '\n'; }
+
+// https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners
+//
+// > If a token in the externals array is valid at a given position in the
+// > parse, the external scanner will be called first before anything else is
+// > done.
+//
+// > But the external scanner may return false and in this case Tree-sitter
+// > fallbacks to the internal lexing mechanism.
+bool tree_sitter_carbon_external_scanner_scan(
+    __attribute__((unused)) void* payload,
+    __attribute__((unused)) TSLexer* lexer,
+    __attribute__((unused)) const bool* valid_symbols) {
+  // skip past whitespace if any
+  bool whitespace = false;
+  while (is_whitespace(lexer->lookahead)) {
+    whitespace = true;
+    lexer->advance(lexer, /* skip= */ true);
+  }
+
+  // if any other symbol than *, fallback to treesitter internal lexer
+  if (lexer->lookahead != '*') {
+    return false;
+  }
+
+  // move to past the *, add * to current token
+  lexer->advance(lexer, /* skip= */ false);
+
+  // https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md
+  if (is_whitespace(lexer->lookahead) && whitespace) {
+    // foo * bar
+    lexer->result_symbol = BINARY_STAR;
+  } else if (!whitespace &&
+             token_allowed_after_binary_operator(lexer->lookahead)) {
+    // foo*bar or foo*(bar)
+    lexer->result_symbol = BINARY_STAR;
+  } else {
+    // foo*
+    lexer->result_symbol = POSTFIX_STAR;
+  }
+  return true;
+}