mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-10-04 08:41:03 +01:00
Chandler CarruthandJon Meow
3512c2218f
Only change is to update the path to the fuzzer build extension. Original main commit message: > Add an initial parser library. (#30) > > This library builds a parse tree, very similar to a concrete syntax > tree. There are no semantics here, simply introducing the basic > syntactic structure. > > The current focus has been on the APIs and the data structures used to > represent the parse tree, and not on the actual code doing the > parsing. The code doing the parsing tries to be reasonably efficient > and reasonably easy to understand recursive descent parser. But there > is likely much that can be done to improve this code path. A notable > area where very little thought has been given yet are emitting good > diagnostics and doing good recovery in the event of parse errors. > > Also, this code does not try to match the current under-discussion > grammar closely. It is only partial and reflects discussions from some > time ago. It should be updated incrementally to reflect the current > expected grammar. > > The data structure used for the parse tree is unusual. The first > constraint is that there is a precise one-to-one correspondence > between the tokens produced by the lexer and the nodes in the parse > tree. Every token results in exactly one node. In that way, the parse > tree can be thought of as merely shaping the token stream into a tree. > > Each node is also represented with a fixed set of data that is densely > packed. Combined with the exact relationship to tokens, this allows us > to fully allocate the parse tree's storage, and to use a dense array > rather than a pointer-based tree structure. > > The tree structure itself is implicitly defined by tracking the size > of each subtree rooted at a particular node. See the code comments for > more details (and I'm happy to add more comments where necessary). The > goal is to minimize both the allocations (one), the working set size > of the tree as a whole, and optimize common iteration patterns. The > tree is stored in postorder. This allows depth-first postorder > iteration as well as topological iteration by walking in reverse. > > Building the parse tree in postorder is a natural consequence of the > grammar being LR rather than LL, which is a consequence of supporting > infix operators. > > As with the Lexer, the parser supports an API for operating on the > parse tree, as well as the ability to print the tree in both > a human-readable and machine-readable format (YAML-based). It includes > significant unit tests and a fuzz tester. The fuzzer's corpus will be > in a follow-up commit. > > This is the largest chunk of code already written by several of us > prior to open sourcing. (There are a few more pieces, but they are > significantly smaller and less interesting.) If there are major things > that folks would like to see happen here, it may make sense to move > them into issues for tracking. I have tried to update the code to > follow the style guidelines, but apologies if I missed anything, just > let me know. We also have issues #19 and #29 to track things that > already came up with the lexer. Co-authored-by: Jon Meow <46229924+jonmeow@users.noreply.github.com>
87 lines
2.2 KiB
Python
87 lines
2.2 KiB
Python
# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
# Exceptions. See /LICENSE for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
|
|
load("//bazel/fuzzing:rules.bzl", "cc_fuzz_test")
|
|
|
|
package(default_visibility = ["//visibility:public"])
|
|
|
|
cc_library(
|
|
name = "parse_node_kind",
|
|
srcs = ["parse_node_kind.cpp"],
|
|
hdrs = ["parse_node_kind.h"],
|
|
textual_hdrs = ["parse_node_kind.def"],
|
|
deps = ["@llvm-project//llvm:Support"],
|
|
)
|
|
|
|
cc_test(
|
|
name = "parse_node_kind_test",
|
|
srcs = ["parse_node_kind_test.cpp"],
|
|
deps = [
|
|
":parse_node_kind",
|
|
"@llvm-project//llvm:Support",
|
|
"@llvm-project//llvm:gtest",
|
|
"@llvm-project//llvm:gtest_main",
|
|
],
|
|
)
|
|
|
|
cc_library(
|
|
name = "parse_tree",
|
|
srcs = [
|
|
"parse_tree.cpp",
|
|
"parser_impl.cpp",
|
|
"parser_impl.h",
|
|
],
|
|
hdrs = ["parse_tree.h"],
|
|
deps = [
|
|
":parse_node_kind",
|
|
"@llvm-project//llvm:Support",
|
|
"//diagnostics:diagnostic_emitter",
|
|
"//lexer:token_kind",
|
|
"//lexer:tokenized_buffer",
|
|
],
|
|
)
|
|
|
|
cc_library(
|
|
name = "parse_test_helpers",
|
|
testonly = 1,
|
|
hdrs = ["parse_test_helpers.h"],
|
|
deps = [
|
|
":parse_node_kind",
|
|
":parse_tree",
|
|
"@llvm-project//llvm:Support",
|
|
"@llvm-project//llvm:gmock",
|
|
"//lexer:tokenized_buffer",
|
|
],
|
|
)
|
|
|
|
cc_test(
|
|
name = "parse_tree_test",
|
|
srcs = ["parse_tree_test.cpp"],
|
|
deps = [
|
|
":parse_node_kind",
|
|
":parse_test_helpers",
|
|
":parse_tree",
|
|
"@llvm-project//llvm:Support",
|
|
"@llvm-project//llvm:gmock",
|
|
"@llvm-project//llvm:gtest",
|
|
"@llvm-project//llvm:gtest_main",
|
|
"//diagnostics:diagnostic_emitter",
|
|
"//lexer:tokenized_buffer",
|
|
"//lexer:tokenized_buffer_test_helpers",
|
|
],
|
|
)
|
|
|
|
cc_fuzz_test(
|
|
name = "parse_tree_fuzzer",
|
|
srcs = ["parse_tree_fuzzer.cpp"],
|
|
corpus = glob(["fuzzer_corpus/*"]),
|
|
deps = [
|
|
":parse_tree",
|
|
"@llvm-project//llvm:Support",
|
|
"//diagnostics:diagnostic_emitter",
|
|
"//lexer:tokenized_buffer",
|
|
],
|
|
)
|