mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 20:30:14 +01:00
Chandler Carruth
577fda1ca2
This rejects type literals with more digits than we can lex without APInt's help, and using a custom diagnostic. This is a pretty arbitrary implementation limit, I'm wide open to even more strict rules here. Despite no special casing and a very simplistic approach, by not using APInt this completely eliminates the lexing overhead for `i32` in the generated compilation benchmark where that specific type literal is very common. We see a 10% improvement in lexing there: ``` BM_CompileAPIFileDenseDecls<Phase::Lex>/256 39.0µs ± 4% 34.8µs ± 2% -10.86% (p=0.000 n=19+20) BM_CompileAPIFileDenseDecls<Phase::Lex>/1024 180µs ± 1% 158µs ± 2% -12.22% (p=0.000 n=18+20) BM_CompileAPIFileDenseDecls<Phase::Lex>/4096 731µs ± 2% 641µs ± 1% -12.31% (p=0.000 n=18+20) BM_CompileAPIFileDenseDecls<Phase::Lex>/16384 3.20ms ± 2% 2.86ms ± 2% -10.47% (p=0.000 n=18+19) BM_CompileAPIFileDenseDecls<Phase::Lex>/65536 13.8ms ± 1% 12.4ms ± 2% -9.78% (p=0.000 n=18+19) BM_CompileAPIFileDenseDecls<Phase::Lex>/262144 64.0ms ± 2% 58.4ms ± 2% -8.70% (p=0.000 n=19+18) ``` This starts to fix a TODO in the diagnostic for these by giving a reasonably good diagnostic about a very large type literal. However, in practice it regresses the diagnostics because error tokens produce noisy extraneous diagnostics from parse and check currently. Leaving the TODO there, and I have a follow-up PR to start improving the extraneous diagnostics.
58 lines
2.2 KiB
C++
58 lines
2.2 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "toolchain/testing/compile_helper.h"
|
|
|
|
namespace Carbon::Testing {
|
|
|
|
auto CompileHelper::GetTokenizedBuffer(llvm::StringRef text,
|
|
DiagnosticConsumer* consumer)
|
|
-> Lex::TokenizedBuffer& {
|
|
auto& source = GetSourceBuffer(text);
|
|
|
|
value_store_storage_.emplace_front();
|
|
token_storage_.push_front(Lex::Lex(value_store_storage_.front(), source,
|
|
consumer ? *consumer : consumer_));
|
|
return token_storage_.front();
|
|
}
|
|
|
|
auto CompileHelper::GetTokenizedBufferWithSharedValueStore(
|
|
llvm::StringRef text, DiagnosticConsumer* consumer)
|
|
-> std::pair<Lex::TokenizedBuffer&, SharedValueStores&> {
|
|
auto& tokens = GetTokenizedBuffer(text, consumer);
|
|
return {tokens, value_store_storage_.front()};
|
|
}
|
|
|
|
auto CompileHelper::GetTree(llvm::StringRef text) -> Parse::Tree& {
|
|
auto& tokens = GetTokenizedBuffer(text);
|
|
tree_storage_.push_front(Parse::Parse(tokens, consumer_,
|
|
/*vlog_stream=*/nullptr));
|
|
return tree_storage_.front();
|
|
}
|
|
|
|
auto CompileHelper::GetTreeAndSubtrees(llvm::StringRef text)
|
|
-> Parse::TreeAndSubtrees& {
|
|
auto& tree = GetTree(text);
|
|
tree_and_subtrees_storage_.push_front(
|
|
Parse::TreeAndSubtrees(token_storage_.front(), tree));
|
|
return tree_and_subtrees_storage_.front();
|
|
}
|
|
|
|
auto CompileHelper::GetTokenizedBufferWithTreeAndSubtrees(llvm::StringRef text)
|
|
-> std::pair<Lex::TokenizedBuffer&, Parse::TreeAndSubtrees&> {
|
|
auto& tree_and_subtrees = GetTreeAndSubtrees(text);
|
|
return {token_storage_.front(), tree_and_subtrees};
|
|
}
|
|
|
|
auto CompileHelper::GetSourceBuffer(llvm::StringRef text) -> SourceBuffer& {
|
|
std::string filename = llvm::formatv("test{0}.carbon", ++file_index_);
|
|
CARBON_CHECK(fs_.addFile(filename, /*ModificationTime=*/0,
|
|
llvm::MemoryBuffer::getMemBuffer(text)));
|
|
source_storage_.push_front(
|
|
std::move(*SourceBuffer::MakeFromFile(fs_, filename, consumer_)));
|
|
return source_storage_.front();
|
|
}
|
|
|
|
} // namespace Carbon::Testing
|