mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-11-21 05:36:39 +00:00
The WORKSPACE file is deprecated; support is already off by default, and it'll be removed in the next major bazel release. Our main dependency is tree-sitter, and I'm trying to address that here. We're currently using https://github.com/elliottt/rules_tree_sitter, but that hasn't been updated in a couple years, meaning it lacks MODULE.bazel support. In the registry, there's https://registry.bazel.build/modules/tree-sitter-bazel, but this is only the *parser* libraries of tree-sitter, not the *generator*. I'm using it for that much, at least. For the *generator*, which transforms grammar.js to parser.c/h, I'm just requiring a non-hermetic invocation (i.e., people who want to work on it will need to install tree-sitter; see the README.md updates). I tried running it manually, but parser.c is about 600 KB; pre-commit rejects files that large and I don't think an exception makes sense to override for this (it'd probably also grow substantially if the grammar were updated to cover more syntax). In order to make the non-hermetic call not break "bazel build //..." for most developers, I'm marking most targets in the package as manual. Note, I did look long and hard at using `aspect_rules_js`/`rules_nodejs` to invoke npm. This took a lot of time, and I have a commit that's mostly working, except I hit a point where it uses `declare_symlink` which we disallow for compatibility reasons (commit "Lots of work for figuring out rule_js uses declare_symlink" on the PR). As a consequence, I think we can't use the primary supported ways to have hermetic npm calls. Also, `treesitter` -> `tree_sitter` because it's generally called `tree-sitter`, two words. We even had a `treesitter/src/tree_sitter` directory so it's a bit inconsistent. As far as bugs here, the parser library breaks bazel queries, e.g. the error: ``` ERROR: Evaluation of query "somepath(//..., @llvm-project//third-party/unittest:gtest)" failed: preloading transitive closure failed: no such package '@@[unknown repo 'platforms' requested from @@tree-sitter-bazel+]//': The repository '@@[unknown repo 'platforms' requested from @@tree-sitter-bazel+]' could not be resolved: No repository visible as '@platforms' from repository '@@tree-sitter-bazel+' ``` I'm just excluding tree_sitter from queries where I can to work around the error.
182 lines
5.7 KiB
C
182 lines
5.7 KiB
C
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "utils/tree_sitter/src/tree_sitter/parser.h"
|
|
|
|
enum TokenType {
|
|
BINARY_STAR,
|
|
POSTFIX_STAR,
|
|
STRING,
|
|
};
|
|
|
|
// This is part of a special rule that doesn't allow `copts` in Bazel, so we
|
|
// disable warnings using `#pragma`s here.
|
|
#pragma clang diagnostic push
|
|
// The tree_sitter build uses C functions without prototypes.
|
|
#pragma clang diagnostic ignored "-Wmissing-prototypes"
|
|
|
|
// our scanner is stateless
|
|
void* tree_sitter_carbon_external_scanner_create(void) { return NULL; }
|
|
|
|
unsigned tree_sitter_carbon_external_scanner_serialize(
|
|
__attribute__((unused)) void* payload,
|
|
__attribute__((unused)) char* buffer) {
|
|
return 0; // zero bytes used to serialize
|
|
}
|
|
|
|
void tree_sitter_carbon_external_scanner_deserialize(
|
|
__attribute__((unused)) void* payload,
|
|
__attribute__((unused)) const char* buffer,
|
|
__attribute__((unused)) unsigned length) {}
|
|
|
|
void tree_sitter_carbon_external_scanner_destroy(
|
|
__attribute__((unused)) void* payload) {}
|
|
|
|
// https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md#overview
|
|
// > the token after the operator must be an identifier, a literal, or any kind
|
|
// of opening bracket (for example, (, [, or {).
|
|
static bool token_allowed_after_binary_operator(char c) {
|
|
return
|
|
// identifier
|
|
c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
|
// string literal
|
|
c == '\"' ||
|
|
// TODO: character literal
|
|
// number literal
|
|
(c >= '0' && c <= '9') ||
|
|
// opening bracket
|
|
c == '(' || c == '[' || c == '{';
|
|
}
|
|
|
|
static bool is_whitespace(char c) { return c == ' ' || c == '\n'; }
|
|
|
|
static void advance(struct TSLexer* lexer) {
|
|
lexer->advance(lexer, /* skip= */ false);
|
|
}
|
|
|
|
static int eat_count(struct TSLexer* lexer, char ch, int count) {
|
|
int matched = 0;
|
|
while (matched != count) {
|
|
if (lexer->lookahead != ch) {
|
|
break;
|
|
}
|
|
advance(lexer);
|
|
matched++;
|
|
}
|
|
return matched;
|
|
}
|
|
|
|
// https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners
|
|
//
|
|
// > If a token in the externals array is valid at a given position in the
|
|
// > parse, the external scanner will be called first before anything else is
|
|
// > done.
|
|
//
|
|
// > But the external scanner may return false and in this case Tree-sitter
|
|
// > fallbacks to the internal lexing mechanism.
|
|
bool tree_sitter_carbon_external_scanner_scan(
|
|
__attribute__((unused)) void* payload,
|
|
__attribute__((unused)) TSLexer* lexer, const bool* valid_symbols) {
|
|
// skip past whitespace if any
|
|
bool whitespace = false;
|
|
while (is_whitespace(lexer->lookahead)) {
|
|
whitespace = true;
|
|
lexer->advance(lexer, /* skip= */ true);
|
|
}
|
|
|
|
if (lexer->lookahead == '*' &&
|
|
(valid_symbols[BINARY_STAR] || valid_symbols[POSTFIX_STAR])) {
|
|
// move to past the *, add * to current token
|
|
lexer->advance(lexer, /* skip= */ false);
|
|
|
|
// https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md
|
|
if (is_whitespace(lexer->lookahead) && whitespace) {
|
|
// foo * bar
|
|
lexer->result_symbol = BINARY_STAR;
|
|
} else if (!whitespace &&
|
|
token_allowed_after_binary_operator(lexer->lookahead)) {
|
|
// foo*bar or foo*(bar)
|
|
lexer->result_symbol = BINARY_STAR;
|
|
} else {
|
|
// foo*
|
|
lexer->result_symbol = POSTFIX_STAR;
|
|
}
|
|
return true;
|
|
} else if ((lexer->lookahead == '#' || lexer->lookahead == '\'' ||
|
|
lexer->lookahead == '\"') &&
|
|
valid_symbols[STRING]) {
|
|
lexer->result_symbol = STRING;
|
|
int hash_count = 0;
|
|
while (lexer->lookahead == '#') {
|
|
advance(lexer);
|
|
hash_count++;
|
|
}
|
|
if (lexer->lookahead == '\'') {
|
|
if (eat_count(lexer, '\'', 3) != 3) {
|
|
// lexer state is ignored on return false
|
|
return false;
|
|
}
|
|
while (!lexer->eof(lexer)) {
|
|
// ''' must be on its own line.
|
|
// so look for \n *'''#{hash_count}
|
|
// we can ignore escapes
|
|
if (lexer->lookahead == '\n') {
|
|
advance(lexer);
|
|
while (lexer->lookahead == ' ') {
|
|
advance(lexer);
|
|
}
|
|
if (eat_count(lexer, '\'', 3) != 3) {
|
|
// treat less than 3 `'`s as normal characters
|
|
continue;
|
|
}
|
|
if (eat_count(lexer, '#', hash_count) == hash_count) {
|
|
// end of string
|
|
return true;
|
|
} else {
|
|
// treat as normal characters
|
|
continue;
|
|
}
|
|
} else {
|
|
// skip not new lines
|
|
advance(lexer);
|
|
}
|
|
}
|
|
} else if (lexer->lookahead == '"') {
|
|
advance(lexer);
|
|
while (!lexer->eof(lexer)) {
|
|
if (lexer->lookahead == '\\') {
|
|
advance(lexer);
|
|
if (eat_count(lexer, '#', hash_count) == hash_count) {
|
|
// treat next character as not special
|
|
// but \n is not allowed in simple string
|
|
if (lexer->lookahead != '\n') {
|
|
advance(lexer);
|
|
}
|
|
} else {
|
|
// not an escape
|
|
continue;
|
|
}
|
|
} else if (lexer->lookahead == '"') {
|
|
advance(lexer);
|
|
if (eat_count(lexer, '#', hash_count) == hash_count) {
|
|
// end of string
|
|
return true;
|
|
} else {
|
|
continue;
|
|
}
|
|
} else if (lexer->lookahead == '\n') {
|
|
// new line is not allowed in simple string
|
|
return false;
|
|
} else {
|
|
advance(lexer);
|
|
}
|
|
}
|
|
} else {
|
|
// # not followed by ' or "
|
|
return false;
|
|
}
|
|
}
|
|
return false;
|
|
}
|