mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 22:02:23 +01:00
Jon Ross-Perkins
f67791cfee
Move subtree sizes over to TreeAndSubtrees, using the different structure to represent the additional parse work that occurs, as well as making it clear which functions require the extra information. My intent is to make it hard to use this by accident. The subtree size is still tracked during Parse::Tree construction. I think a lot of that can be cleaned up, although we use it during placeholder assignment so it may take some work. I wanted to see what people thought about this before taking action on such a change. I'm using a 1m line source file generated by #4124 for testing. Command is `time bazel-bin/toolchain/install/prefix_root/bin/carbon compile --phase=check --dump-mem-usage ~/tmp/data.carbon` At head, what I'm seeing is: ``` ... parse_tree_.node_impls_: used_bytes: 61516116 reserved_bytes: 61516116 ... Total: used_bytes: 447814230 reserved_bytes: 551663894 ... 1.43s user 0.14s system 99% cpu 1.565 total ``` With `Tree::Verify` disabled completely, it looks like: ``` parse_tree_.node_impls_: used_bytes: 41010744 reserved_bytes: 41010744 ... Total: used_bytes: 427308858 reserved_bytes: 531158522 ... 1.20s user 0.13s system 99% cpu 1.332 total ``` Re-enabling just the basic verification (what is now `Tree::Verify`), I'm seeing maybe 0.05s slower, but that's within noise for my system. I do see variability in my timing results, and overall I think this is a 0.2s +/- 0.1s improvement versus the earlier (always testing `Extract` code) implementation. That's opt; debug builds will be unaffected, because the same checking occurs as before. Note, the subtree size is a third of the node representation, which is why I'm showing the decrease in memory usage here.
190 lines
7.3 KiB
C++
190 lines
7.3 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "toolchain/parse/tree.h"
|
|
|
|
#include <gmock/gmock.h>
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <forward_list>
|
|
|
|
#include "testing/base/test_raw_ostream.h"
|
|
#include "toolchain/base/value_store.h"
|
|
#include "toolchain/diagnostics/diagnostic_emitter.h"
|
|
#include "toolchain/diagnostics/mocks.h"
|
|
#include "toolchain/lex/lex.h"
|
|
#include "toolchain/lex/tokenized_buffer.h"
|
|
#include "toolchain/parse/parse.h"
|
|
#include "toolchain/parse/tree_and_subtrees.h"
|
|
#include "toolchain/testing/yaml_test_helpers.h"
|
|
|
|
namespace Carbon::Parse {
|
|
namespace {
|
|
|
|
using ::Carbon::Testing::TestRawOstream;
|
|
using ::testing::ElementsAre;
|
|
using ::testing::Pair;
|
|
|
|
namespace Yaml = ::Carbon::Testing::Yaml;
|
|
|
|
class TreeTest : public ::testing::Test {
|
|
protected:
|
|
auto GetSourceBuffer(llvm::StringRef t) -> SourceBuffer& {
|
|
CARBON_CHECK(fs_.addFile("test.carbon", /*ModificationTime=*/0,
|
|
llvm::MemoryBuffer::getMemBuffer(t)));
|
|
source_storage_.push_front(
|
|
std::move(*SourceBuffer::MakeFromFile(fs_, "test.carbon", consumer_)));
|
|
return source_storage_.front();
|
|
}
|
|
|
|
auto GetTokenizedBuffer(llvm::StringRef t) -> Lex::TokenizedBuffer& {
|
|
token_storage_.push_front(
|
|
Lex::Lex(value_stores_, GetSourceBuffer(t), consumer_));
|
|
return token_storage_.front();
|
|
}
|
|
|
|
SharedValueStores value_stores_;
|
|
llvm::vfs::InMemoryFileSystem fs_;
|
|
std::forward_list<SourceBuffer> source_storage_;
|
|
std::forward_list<Lex::TokenizedBuffer> token_storage_;
|
|
DiagnosticConsumer& consumer_ = ConsoleDiagnosticConsumer();
|
|
};
|
|
|
|
TEST_F(TreeTest, IsValid) {
|
|
Lex::TokenizedBuffer& tokens = GetTokenizedBuffer("");
|
|
Tree tree = Parse(tokens, consumer_, /*vlog_stream=*/nullptr);
|
|
EXPECT_TRUE((*tree.postorder().begin()).is_valid());
|
|
}
|
|
|
|
TEST_F(TreeTest, AsAndTryAs) {
|
|
Lex::TokenizedBuffer& tokens = GetTokenizedBuffer("fn F();");
|
|
Tree tree = Parse(tokens, consumer_, /*vlog_stream=*/nullptr);
|
|
ASSERT_FALSE(tree.has_errors());
|
|
TreeAndSubtrees tree_and_subtrees(tokens, tree);
|
|
auto it = tree_and_subtrees.roots().begin();
|
|
// A FileEnd node, so won't match.
|
|
NodeId n = *it;
|
|
|
|
// NodeIdForKind
|
|
std::optional<FunctionDeclId> fn_decl_id = tree.TryAs<FunctionDeclId>(n);
|
|
EXPECT_FALSE(fn_decl_id.has_value());
|
|
// NodeIdOneOf
|
|
std::optional<AnyFunctionDeclId> any_fn_decl_id =
|
|
tree.TryAs<AnyFunctionDeclId>(n);
|
|
EXPECT_FALSE(any_fn_decl_id.has_value());
|
|
// NodeIdInCategory
|
|
std::optional<AnyDeclId> any_decl_id = tree.TryAs<AnyDeclId>(n);
|
|
EXPECT_FALSE(any_decl_id.has_value());
|
|
|
|
++it;
|
|
n = *it;
|
|
// A FunctionDecl node, so will match.
|
|
fn_decl_id = tree.TryAs<FunctionDeclId>(n);
|
|
ASSERT_TRUE(fn_decl_id.has_value());
|
|
EXPECT_TRUE(*fn_decl_id == n);
|
|
// Under normal usage, this function should be used with `auto`, but for
|
|
// a test it is nice to verify that it is returning the expected type.
|
|
// NOLINTNEXTLINE(modernize-use-auto).
|
|
FunctionDeclId fn_decl_id2 = tree.As<FunctionDeclId>(n);
|
|
EXPECT_TRUE(*fn_decl_id == fn_decl_id2);
|
|
|
|
any_fn_decl_id = tree.TryAs<AnyFunctionDeclId>(n);
|
|
ASSERT_TRUE(any_fn_decl_id.has_value());
|
|
EXPECT_TRUE(*any_fn_decl_id == n);
|
|
// NOLINTNEXTLINE(modernize-use-auto).
|
|
AnyFunctionDeclId any_fn_decl_id2 = tree.As<AnyFunctionDeclId>(n);
|
|
EXPECT_TRUE(*any_fn_decl_id == any_fn_decl_id2);
|
|
|
|
any_decl_id = tree.TryAs<AnyDeclId>(n);
|
|
ASSERT_TRUE(any_decl_id.has_value());
|
|
EXPECT_TRUE(*any_decl_id == n);
|
|
// NOLINTNEXTLINE(modernize-use-auto).
|
|
AnyDeclId any_decl_id2 = tree.As<AnyDeclId>(n);
|
|
EXPECT_TRUE(*any_decl_id == any_decl_id2);
|
|
}
|
|
|
|
TEST_F(TreeTest, PrintPostorderAsYAML) {
|
|
Lex::TokenizedBuffer& tokens = GetTokenizedBuffer("fn F();");
|
|
Tree tree = Parse(tokens, consumer_, /*vlog_stream=*/nullptr);
|
|
EXPECT_FALSE(tree.has_errors());
|
|
TestRawOstream print_stream;
|
|
tree.Print(print_stream);
|
|
|
|
auto file = Yaml::Sequence(ElementsAre(
|
|
Yaml::Mapping(ElementsAre(Pair("kind", "FileStart"), Pair("text", ""))),
|
|
Yaml::Mapping(
|
|
ElementsAre(Pair("kind", "FunctionIntroducer"), Pair("text", "fn"))),
|
|
Yaml::Mapping(
|
|
ElementsAre(Pair("kind", "IdentifierName"), Pair("text", "F"))),
|
|
Yaml::Mapping(
|
|
ElementsAre(Pair("kind", "TuplePatternStart"), Pair("text", "("))),
|
|
Yaml::Mapping(ElementsAre(Pair("kind", "TuplePattern"), Pair("text", ")"),
|
|
Pair("subtree_size", "2"))),
|
|
Yaml::Mapping(ElementsAre(Pair("kind", "FunctionDecl"), Pair("text", ";"),
|
|
Pair("subtree_size", "5"))),
|
|
Yaml::Mapping(ElementsAre(Pair("kind", "FileEnd"), Pair("text", "")))));
|
|
|
|
auto root = Yaml::Sequence(ElementsAre(Yaml::Mapping(
|
|
ElementsAre(Pair("filename", "test.carbon"), Pair("parse_tree", file)))));
|
|
|
|
EXPECT_THAT(Yaml::Value::FromText(print_stream.TakeStr()),
|
|
IsYaml(ElementsAre(root)));
|
|
}
|
|
|
|
TEST_F(TreeTest, PrintPreorderAsYAML) {
|
|
Lex::TokenizedBuffer& tokens = GetTokenizedBuffer("fn F();");
|
|
Tree tree = Parse(tokens, consumer_, /*vlog_stream=*/nullptr);
|
|
EXPECT_FALSE(tree.has_errors());
|
|
TreeAndSubtrees tree_and_subtrees(tokens, tree);
|
|
TestRawOstream print_stream;
|
|
tree_and_subtrees.PrintPreorder(print_stream);
|
|
|
|
auto param_list = Yaml::Sequence(ElementsAre(Yaml::Mapping(
|
|
ElementsAre(Pair("node_index", "3"), Pair("kind", "TuplePatternStart"),
|
|
Pair("text", "(")))));
|
|
|
|
auto function_decl = Yaml::Sequence(ElementsAre(
|
|
Yaml::Mapping(ElementsAre(Pair("node_index", "1"),
|
|
Pair("kind", "FunctionIntroducer"),
|
|
Pair("text", "fn"))),
|
|
Yaml::Mapping(ElementsAre(Pair("node_index", "2"),
|
|
Pair("kind", "IdentifierName"),
|
|
Pair("text", "F"))),
|
|
Yaml::Mapping(ElementsAre(Pair("node_index", "4"),
|
|
Pair("kind", "TuplePattern"), Pair("text", ")"),
|
|
Pair("subtree_size", "2"),
|
|
Pair("children", param_list)))));
|
|
|
|
auto file = Yaml::Sequence(ElementsAre(
|
|
Yaml::Mapping(ElementsAre(Pair("node_index", "0"),
|
|
Pair("kind", "FileStart"), Pair("text", ""))),
|
|
Yaml::Mapping(ElementsAre(Pair("node_index", "5"),
|
|
Pair("kind", "FunctionDecl"), Pair("text", ";"),
|
|
Pair("subtree_size", "5"),
|
|
Pair("children", function_decl))),
|
|
Yaml::Mapping(ElementsAre(Pair("node_index", "6"),
|
|
Pair("kind", "FileEnd"), Pair("text", "")))));
|
|
|
|
auto root = Yaml::Sequence(ElementsAre(Yaml::Mapping(
|
|
ElementsAre(Pair("filename", "test.carbon"), Pair("parse_tree", file)))));
|
|
|
|
EXPECT_THAT(Yaml::Value::FromText(print_stream.TakeStr()),
|
|
IsYaml(ElementsAre(root)));
|
|
}
|
|
|
|
TEST_F(TreeTest, HighRecursion) {
|
|
std::string code = "fn Foo() { return ";
|
|
code.append(10000, '(');
|
|
code.append(10000, ')');
|
|
code += "; }";
|
|
Lex::TokenizedBuffer& tokens = GetTokenizedBuffer(code);
|
|
ASSERT_FALSE(tokens.has_errors());
|
|
Testing::MockDiagnosticConsumer consumer;
|
|
Tree tree = Parse(tokens, consumer, /*vlog_stream=*/nullptr);
|
|
EXPECT_FALSE(tree.has_errors());
|
|
}
|
|
|
|
} // namespace
|
|
} // namespace Carbon::Parse
|