Files
carbon-lang/utils/tree_sitter/src/scanner.c
T
Jon Ross-Perkins 6f6e46ef57 Migrate tree-sitter support to MODULE.bazel (#4783)
The WORKSPACE file is deprecated; support is already off by default, and
it'll be removed in the next major bazel release. Our main dependency is
tree-sitter, and I'm trying to address that here.

We're currently using https://github.com/elliottt/rules_tree_sitter, but
that hasn't been updated in a couple years, meaning it lacks
MODULE.bazel support. In the registry, there's
https://registry.bazel.build/modules/tree-sitter-bazel, but this is only
the *parser* libraries of tree-sitter, not the *generator*. I'm using it
for that much, at least.

For the *generator*, which transforms grammar.js to parser.c/h, I'm just
requiring a non-hermetic invocation (i.e., people who want to work on it
will need to install tree-sitter; see the README.md updates). I tried
running it manually, but parser.c is about 600 KB; pre-commit rejects
files that large and I don't think an exception makes sense to override
for this (it'd probably also grow substantially if the grammar were
updated to cover more syntax). In order to make the non-hermetic call
not break "bazel build //..." for most developers, I'm marking most
targets in the package as manual.

Note, I did look long and hard at using `aspect_rules_js`/`rules_nodejs`
to invoke npm. This took a lot of time, and I have a commit that's
mostly working, except I hit a point where it uses `declare_symlink`
which we disallow for compatibility reasons (commit "Lots of work for
figuring out rule_js uses declare_symlink" on the PR). As a consequence,
I think we can't use the primary supported ways to have hermetic npm
calls.

Also, `treesitter` -> `tree_sitter` because it's generally called
`tree-sitter`, two words. We even had a `treesitter/src/tree_sitter`
directory so it's a bit inconsistent.

As far as bugs here, the parser library breaks bazel queries, e.g. the
error:
```
ERROR: Evaluation of query "somepath(//..., @llvm-project//third-party/unittest:gtest)" failed: preloading transitive closure failed: no such package '@@[unknown repo 'platforms' requested from @@tree-sitter-bazel+]//': The repository '@@[unknown repo 'platforms' requested from @@tree-sitter-bazel+]' could not be resolved: No repository visible as '@platforms' from repository '@@tree-sitter-bazel+'
```

I'm just excluding tree_sitter from queries where I can to work around
the error.
2025-01-13 19:04:10 +00:00

182 lines
5.7 KiB
C

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "utils/tree_sitter/src/tree_sitter/parser.h"
enum TokenType {
BINARY_STAR,
POSTFIX_STAR,
STRING,
};
// This is part of a special rule that doesn't allow `copts` in Bazel, so we
// disable warnings using `#pragma`s here.
#pragma clang diagnostic push
// The tree_sitter build uses C functions without prototypes.
#pragma clang diagnostic ignored "-Wmissing-prototypes"
// our scanner is stateless
void* tree_sitter_carbon_external_scanner_create(void) { return NULL; }
unsigned tree_sitter_carbon_external_scanner_serialize(
__attribute__((unused)) void* payload,
__attribute__((unused)) char* buffer) {
return 0; // zero bytes used to serialize
}
void tree_sitter_carbon_external_scanner_deserialize(
__attribute__((unused)) void* payload,
__attribute__((unused)) const char* buffer,
__attribute__((unused)) unsigned length) {}
void tree_sitter_carbon_external_scanner_destroy(
__attribute__((unused)) void* payload) {}
// https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md#overview
// > the token after the operator must be an identifier, a literal, or any kind
// of opening bracket (for example, (, [, or {).
static bool token_allowed_after_binary_operator(char c) {
return
// identifier
c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
// string literal
c == '\"' ||
// TODO: character literal
// number literal
(c >= '0' && c <= '9') ||
// opening bracket
c == '(' || c == '[' || c == '{';
}
static bool is_whitespace(char c) { return c == ' ' || c == '\n'; }
static void advance(struct TSLexer* lexer) {
lexer->advance(lexer, /* skip= */ false);
}
static int eat_count(struct TSLexer* lexer, char ch, int count) {
int matched = 0;
while (matched != count) {
if (lexer->lookahead != ch) {
break;
}
advance(lexer);
matched++;
}
return matched;
}
// https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners
//
// > If a token in the externals array is valid at a given position in the
// > parse, the external scanner will be called first before anything else is
// > done.
//
// > But the external scanner may return false and in this case Tree-sitter
// > fallbacks to the internal lexing mechanism.
bool tree_sitter_carbon_external_scanner_scan(
__attribute__((unused)) void* payload,
__attribute__((unused)) TSLexer* lexer, const bool* valid_symbols) {
// skip past whitespace if any
bool whitespace = false;
while (is_whitespace(lexer->lookahead)) {
whitespace = true;
lexer->advance(lexer, /* skip= */ true);
}
if (lexer->lookahead == '*' &&
(valid_symbols[BINARY_STAR] || valid_symbols[POSTFIX_STAR])) {
// move to past the *, add * to current token
lexer->advance(lexer, /* skip= */ false);
// https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md
if (is_whitespace(lexer->lookahead) && whitespace) {
// foo * bar
lexer->result_symbol = BINARY_STAR;
} else if (!whitespace &&
token_allowed_after_binary_operator(lexer->lookahead)) {
// foo*bar or foo*(bar)
lexer->result_symbol = BINARY_STAR;
} else {
// foo*
lexer->result_symbol = POSTFIX_STAR;
}
return true;
} else if ((lexer->lookahead == '#' || lexer->lookahead == '\'' ||
lexer->lookahead == '\"') &&
valid_symbols[STRING]) {
lexer->result_symbol = STRING;
int hash_count = 0;
while (lexer->lookahead == '#') {
advance(lexer);
hash_count++;
}
if (lexer->lookahead == '\'') {
if (eat_count(lexer, '\'', 3) != 3) {
// lexer state is ignored on return false
return false;
}
while (!lexer->eof(lexer)) {
// ''' must be on its own line.
// so look for \n *'''#{hash_count}
// we can ignore escapes
if (lexer->lookahead == '\n') {
advance(lexer);
while (lexer->lookahead == ' ') {
advance(lexer);
}
if (eat_count(lexer, '\'', 3) != 3) {
// treat less than 3 `'`s as normal characters
continue;
}
if (eat_count(lexer, '#', hash_count) == hash_count) {
// end of string
return true;
} else {
// treat as normal characters
continue;
}
} else {
// skip not new lines
advance(lexer);
}
}
} else if (lexer->lookahead == '"') {
advance(lexer);
while (!lexer->eof(lexer)) {
if (lexer->lookahead == '\\') {
advance(lexer);
if (eat_count(lexer, '#', hash_count) == hash_count) {
// treat next character as not special
// but \n is not allowed in simple string
if (lexer->lookahead != '\n') {
advance(lexer);
}
} else {
// not an escape
continue;
}
} else if (lexer->lookahead == '"') {
advance(lexer);
if (eat_count(lexer, '#', hash_count) == hash_count) {
// end of string
return true;
} else {
continue;
}
} else if (lexer->lookahead == '\n') {
// new line is not allowed in simple string
return false;
} else {
advance(lexer);
}
}
} else {
// # not followed by ' or "
return false;
}
}
return false;
}