diff --git a/explorer/BUILD b/explorer/BUILD index 949910a21bc4..ba3ae2f68b4c 100644 --- a/explorer/BUILD +++ b/explorer/BUILD @@ -87,9 +87,6 @@ filegroup( ["testdata/**/*.carbon"], exclude = [ "testdata/**/fail_*", - - # multiline strings - "testdata/string/*", ], ), visibility = ["//utils/treesitter:__pkg__"], diff --git a/utils/treesitter/BUILD b/utils/treesitter/BUILD index de2b190fda27..58e449a07657 100644 --- a/utils/treesitter/BUILD +++ b/utils/treesitter/BUILD @@ -27,3 +27,39 @@ cc_test( ":treesitter", ], ) + +filegroup( + name = "string_testdata", + srcs = glob( + ["testdata/string/*.carbon"], + exclude = ["testdata/string/fail_*.carbon"], + ), +) + +filegroup( + name = "string_fail_testdata", + srcs = glob(["testdata/string/fail_*.carbon"]), +) + +cc_test( + name = "string_tests", + srcs = ["test_runner.cpp"], + args = ["$(locations :string_testdata)"], + data = [":string_testdata"], + deps = [ + ":treesitter", + ], +) + +cc_test( + name = "string_fail_tests", + srcs = ["test_runner.cpp"], + args = ["$(locations :string_fail_testdata)"], + data = [":string_fail_testdata"], + env = { + "FAIL_TESTS": "1", + }, + deps = [ + ":treesitter", + ], +) diff --git a/utils/treesitter/grammar.js b/utils/treesitter/grammar.js index 0a64ed400b14..090a0617153b 100644 --- a/utils/treesitter/grammar.js +++ b/utils/treesitter/grammar.js @@ -51,7 +51,7 @@ module.exports = grammar({ extras: ($) => [/\s/, $.comment], // NOTE: This must match the order in src/scanner.c, names are not used for matching. - externals: ($) => [$.binary_star, $.postfix_star], + externals: ($) => [$.binary_star, $.postfix_star, $.string], rules: { source_file: ($) => @@ -63,7 +63,7 @@ module.exports = grammar({ api_or_impl: ($) => choice('api', 'impl'), - library_path: ($) => seq('library', $.string_literal), + library_path: ($) => seq('library', $.string), package_directive: ($) => seq('package', $.ident, optional($.library_path), $.api_or_impl, ';'), @@ -124,34 +124,6 @@ module.exports = grammar({ ); }, - _string_content: ($) => token.immediate(/[^\\"]+/), - - escape_sequence: ($) => - token.immediate( - seq( - '\\', - choice( - 'n', - 't', - 'r', - "'", - '"', - '\\', - '0', - /x[0-9A-F]{2}/, - /u\{[0-9A-F]+\}/ - ) - ) - ), - - // TODO: multiline string - string_literal: ($) => - seq( - '"', - repeat(choice($._string_content, $.escape_sequence)), - token.immediate('"') - ), - array_literal: ($) => seq( '[', @@ -174,7 +146,7 @@ module.exports = grammar({ $.bool_literal, $.numeric_literal, $.numeric_type_literal, - $.string_literal, + $.string, $.struct_literal, $.struct_type_literal ), diff --git a/utils/treesitter/queries/highlights.scm b/utils/treesitter/queries/highlights.scm index e6b906d93fb9..da90e3917f96 100644 --- a/utils/treesitter/queries/highlights.scm +++ b/utils/treesitter/queries/highlights.scm @@ -8,8 +8,7 @@ (comment) @comment (builtin_type) @type.builtin (bool_literal) @constant.builtin -(escape_sequence) @constant.character.escape -(string_literal) @string +(string) @string (numeric_literal) @constant.builtin (numeric_type_literal) @type.builtin diff --git a/utils/treesitter/src/scanner.c b/utils/treesitter/src/scanner.c index b2c6665251ac..36d885029f80 100644 --- a/utils/treesitter/src/scanner.c +++ b/utils/treesitter/src/scanner.c @@ -7,6 +7,7 @@ enum TokenType { BINARY_STAR, POSTFIX_STAR, + STRING, }; // our scanner is stateless @@ -44,6 +45,22 @@ static bool token_allowed_after_binary_operator(char c) { static bool is_whitespace(char c) { return c == ' ' || c == '\n'; } +static void advance(struct TSLexer* lexer) { + lexer->advance(lexer, /* skip= */ false); +} + +static int eat_count(struct TSLexer* lexer, char ch, int count) { + int matched = 0; + while (matched != count) { + if (lexer->lookahead != ch) { + break; + } + advance(lexer); + matched++; + } + return matched; +} + // https://tree-sitter.github.io/tree-sitter/creating-parsers#external-scanners // // > If a token in the externals array is valid at a given position in the @@ -54,8 +71,7 @@ static bool is_whitespace(char c) { return c == ' ' || c == '\n'; } // > fallbacks to the internal lexing mechanism. bool tree_sitter_carbon_external_scanner_scan( __attribute__((unused)) void* payload, - __attribute__((unused)) TSLexer* lexer, - __attribute__((unused)) const bool* valid_symbols) { + __attribute__((unused)) TSLexer* lexer, const bool* valid_symbols) { // skip past whitespace if any bool whitespace = false; while (is_whitespace(lexer->lookahead)) { @@ -63,25 +79,97 @@ bool tree_sitter_carbon_external_scanner_scan( lexer->advance(lexer, /* skip= */ true); } - // if any other symbol than *, fallback to treesitter internal lexer - if (lexer->lookahead != '*') { - return false; - } + if (lexer->lookahead == '*' && + (valid_symbols[BINARY_STAR] || valid_symbols[POSTFIX_STAR])) { + // move to past the *, add * to current token + lexer->advance(lexer, /* skip= */ false); - // move to past the *, add * to current token - lexer->advance(lexer, /* skip= */ false); - - // https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md - if (is_whitespace(lexer->lookahead) && whitespace) { - // foo * bar - lexer->result_symbol = BINARY_STAR; - } else if (!whitespace && - token_allowed_after_binary_operator(lexer->lookahead)) { - // foo*bar or foo*(bar) - lexer->result_symbol = BINARY_STAR; - } else { - // foo* - lexer->result_symbol = POSTFIX_STAR; + // https://github.com/carbon-language/carbon-lang/blob/trunk/docs/design/lexical_conventions/symbolic_tokens.md + if (is_whitespace(lexer->lookahead) && whitespace) { + // foo * bar + lexer->result_symbol = BINARY_STAR; + } else if (!whitespace && + token_allowed_after_binary_operator(lexer->lookahead)) { + // foo*bar or foo*(bar) + lexer->result_symbol = BINARY_STAR; + } else { + // foo* + lexer->result_symbol = POSTFIX_STAR; + } + return true; + } else if ((lexer->lookahead == '#' || lexer->lookahead == '\'' || + lexer->lookahead == '\"') && + valid_symbols[STRING]) { + lexer->result_symbol = STRING; + int hash_count = 0; + while (lexer->lookahead == '#') { + advance(lexer); + hash_count++; + } + if (lexer->lookahead == '\'') { + if (eat_count(lexer, '\'', 3) != 3) { + // lexer state is ignored on return false + return false; + } + while (!lexer->eof(lexer)) { + // ''' must be on its own line. + // so look for \n *'''#{hash_count} + // we can ignore escapes + if (lexer->lookahead == '\n') { + advance(lexer); + while (lexer->lookahead == ' ') { + advance(lexer); + } + if (eat_count(lexer, '\'', 3) != 3) { + // treat less than 3 `'`s as normal characters + continue; + } + if (eat_count(lexer, '#', hash_count) == hash_count) { + // end of string + return true; + } else { + // treat as normal characters + continue; + } + } else { + // skip not new lines + advance(lexer); + } + } + } else if (lexer->lookahead == '"') { + advance(lexer); + while (!lexer->eof(lexer)) { + if (lexer->lookahead == '\\') { + advance(lexer); + if (eat_count(lexer, '#', hash_count) == hash_count) { + // treat next character as not special + // but \n is not allowed in simple string + if (lexer->lookahead != '\n') { + advance(lexer); + } + } else { + // not an escape + continue; + } + } else if (lexer->lookahead == '"') { + advance(lexer); + if (eat_count(lexer, '#', hash_count) == hash_count) { + // end of string + return true; + } else { + continue; + } + } else if (lexer->lookahead == '\n') { + // new line is not allowed in simple string + return false; + } else { + advance(lexer); + } + } + } else { + // # not followed by ' or " + return false; + } } - return true; + return false; } diff --git a/utils/treesitter/test_runner.cpp b/utils/treesitter/test_runner.cpp index e70b5b151443..eb3751dd26ec 100644 --- a/utils/treesitter/test_runner.cpp +++ b/utils/treesitter/test_runner.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -35,7 +36,9 @@ auto main(int argc, char** argv) -> int { auto* parser = ts_parser_new(); ts_parser_set_language(parser, tree_sitter_carbon()); - std::vector failed; + bool fail_tests = std::getenv("FAIL_TESTS") != nullptr; + + std::vector incorrect; for (int i = 1; i < argc; i++) { std::string file_path = argv[i]; std::string source = ReadFile(file_path); @@ -48,19 +51,27 @@ auto main(int argc, char** argv) -> int { char* node_debug = ts_node_string(root); std::cout << file_path << ":\n" << node_debug << "\n"; - if (has_error) { - failed.push_back(file_path); + if (has_error ^ fail_tests) { + incorrect.push_back(file_path); } free(node_debug); ts_tree_delete(tree); } ts_parser_delete(parser); - for (const auto& file : failed) { - std::cout << "FAILED " << file << "\n"; + for (const auto& file : incorrect) { + if (fail_tests) { + std::cout << "INCORRECTLY PASSING " << file << "\n"; + } else { + std::cout << "FAILED " << file << "\n"; + } } - if (!failed.empty()) { - std::cout << failed.size() << " tests failing.\n"; + if (!incorrect.empty()) { + if (fail_tests) { + std::cout << incorrect.size() << " tests incorrectly passing.\n"; + } else { + std::cout << incorrect.size() << " tests failing.\n"; + } return 1; } } diff --git a/utils/treesitter/testdata/string/block_inside_quotes.carbon b/utils/treesitter/testdata/string/block_inside_quotes.carbon new file mode 100644 index 000000000000..0052e9776cb4 --- /dev/null +++ b/utils/treesitter/testdata/string/block_inside_quotes.carbon @@ -0,0 +1,8 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = ''' + '' '' + ' ' ' +'''; diff --git a/utils/treesitter/testdata/string/fail_block_not_enough_quotes.carbon b/utils/treesitter/testdata/string/fail_block_not_enough_quotes.carbon new file mode 100644 index 000000000000..f23ea1f67f8e --- /dev/null +++ b/utils/treesitter/testdata/string/fail_block_not_enough_quotes.carbon @@ -0,0 +1,7 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = '' + hello + ''; diff --git a/utils/treesitter/testdata/string/fail_block_single_quote.carbon b/utils/treesitter/testdata/string/fail_block_single_quote.carbon new file mode 100644 index 000000000000..d5fd8aa99b3c --- /dev/null +++ b/utils/treesitter/testdata/string/fail_block_single_quote.carbon @@ -0,0 +1,7 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = ' + single quote +'; diff --git a/utils/treesitter/testdata/string/fail_block_unclosed_string.carbon b/utils/treesitter/testdata/string/fail_block_unclosed_string.carbon new file mode 100644 index 000000000000..f9edc323fe73 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_block_unclosed_string.carbon @@ -0,0 +1,6 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var x: String = ''' +unclosed string diff --git a/utils/treesitter/testdata/string/fail_newline.carbon b/utils/treesitter/testdata/string/fail_newline.carbon new file mode 100644 index 000000000000..55b78ee2fa00 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_newline.carbon @@ -0,0 +1,6 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var x: String = "new +line"; diff --git a/utils/treesitter/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon b/utils/treesitter/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon new file mode 100644 index 000000000000..a1b2d5df668b --- /dev/null +++ b/utils/treesitter/testdata/string/fail_raw_block_more_hash_tags_on_left.carbon @@ -0,0 +1,7 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = ##''' + error: there are more #s on the left than the right. +'''#; diff --git a/utils/treesitter/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon b/utils/treesitter/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon new file mode 100644 index 000000000000..48c4950d35d1 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_raw_block_more_hash_tags_on_right.carbon @@ -0,0 +1,7 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = #''' + error: there are more #s on the right than the left. +'''##; diff --git a/utils/treesitter/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon b/utils/treesitter/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon new file mode 100644 index 000000000000..534a0e8f5bc0 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_raw_block_quotes_not_on_own_line.carbon @@ -0,0 +1,6 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = #''' + error: closing not on own line '''#; diff --git a/utils/treesitter/testdata/string/fail_raw_more_hash_tags_on_left.carbon b/utils/treesitter/testdata/string/fail_raw_more_hash_tags_on_left.carbon new file mode 100644 index 000000000000..b9b140edb2b5 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_raw_more_hash_tags_on_left.carbon @@ -0,0 +1,5 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = ##"str"#; diff --git a/utils/treesitter/testdata/string/fail_raw_more_hash_tags_on_right.carbon b/utils/treesitter/testdata/string/fail_raw_more_hash_tags_on_right.carbon new file mode 100644 index 000000000000..ddc5b2697ab4 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_raw_more_hash_tags_on_right.carbon @@ -0,0 +1,5 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = "str"#; diff --git a/utils/treesitter/testdata/string/fail_simple_escaped_newline.carbon b/utils/treesitter/testdata/string/fail_simple_escaped_newline.carbon new file mode 100644 index 000000000000..ad6f07452724 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_simple_escaped_newline.carbon @@ -0,0 +1,6 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: String = "hello\ +world"; diff --git a/utils/treesitter/testdata/string/fail_unclosed_string.carbon b/utils/treesitter/testdata/string/fail_unclosed_string.carbon new file mode 100644 index 000000000000..b4d274f282a4 --- /dev/null +++ b/utils/treesitter/testdata/string/fail_unclosed_string.carbon @@ -0,0 +1,5 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +var s: "