From 16c6ba6bd1467f6a9567acd7ed6072c35e3eab1e Mon Sep 17 00:00:00 2001 From: Jon Meow Date: Tue, 15 Mar 2022 13:02:44 -0700 Subject: [PATCH] Accessor renames on lexer (#1134) --- toolchain/driver/driver.cpp | 4 +- toolchain/lexer/numeric_literal.h | 6 +- toolchain/lexer/numeric_literal_test.cpp | 2 +- toolchain/lexer/tokenized_buffer.cpp | 16 +-- toolchain/lexer/tokenized_buffer.h | 20 +-- toolchain/lexer/tokenized_buffer_fuzzer.cpp | 4 +- toolchain/lexer/tokenized_buffer_test.cpp | 128 +++++++++--------- .../lexer/tokenized_buffer_test_helpers.h | 8 +- toolchain/parser/parse_tree_fuzzer.cpp | 4 +- toolchain/parser/parse_tree_test.cpp | 8 +- toolchain/parser/parser_impl.cpp | 8 +- 11 files changed, 104 insertions(+), 104 deletions(-) diff --git a/toolchain/driver/driver.cpp b/toolchain/driver/driver.cpp index 8d8f02c4871b..d62d6ba72f35 100644 --- a/toolchain/driver/driver.cpp +++ b/toolchain/driver/driver.cpp @@ -120,7 +120,7 @@ auto Driver::RunDumpTokensSubcommand(llvm::ArrayRef args) auto tokenized_source = TokenizedBuffer::Lex(*source, ConsoleDiagnosticConsumer()); tokenized_source.Print(output_stream_); - return !tokenized_source.HasErrors(); + return !tokenized_source.has_errors(); } auto Driver::RunDumpParseTreeSubcommand(llvm::ArrayRef args) @@ -152,7 +152,7 @@ auto Driver::RunDumpParseTreeSubcommand(llvm::ArrayRef args) auto parse_tree = ParseTree::Parse(tokenized_source, ConsoleDiagnosticConsumer()); parse_tree.Print(output_stream_); - return !tokenized_source.HasErrors() && !parse_tree.HasErrors(); + return !tokenized_source.has_errors() && !parse_tree.HasErrors(); } auto Driver::ReportExtraArgs(llvm::StringRef subcommand_text, diff --git a/toolchain/lexer/numeric_literal.h b/toolchain/lexer/numeric_literal.h index 5226a4368dd5..52076a4713d4 100644 --- a/toolchain/lexer/numeric_literal.h +++ b/toolchain/lexer/numeric_literal.h @@ -46,13 +46,13 @@ class LexedNumericLiteral { static auto Lex(llvm::StringRef source_text) -> llvm::Optional; - // Get the text corresponding to this literal. - [[nodiscard]] auto Text() const -> llvm::StringRef { return text_; } - // Compute the value of the token, if possible. Emit diagnostics to the given // emitter if the token is not valid. auto ComputeValue(DiagnosticEmitter& emitter) const -> Value; + // Get the text corresponding to this literal. + [[nodiscard]] auto text() const -> llvm::StringRef { return text_; } + private: class Parser; diff --git a/toolchain/lexer/numeric_literal_test.cpp b/toolchain/lexer/numeric_literal_test.cpp index df5bf5ea54d6..d4b4734ca39c 100644 --- a/toolchain/lexer/numeric_literal_test.cpp +++ b/toolchain/lexer/numeric_literal_test.cpp @@ -33,7 +33,7 @@ class NumericLiteralTest : public ::testing::Test { auto Lex(llvm::StringRef text) -> LexedNumericLiteral { llvm::Optional result = LexedNumericLiteral::Lex(text); CHECK(result); - EXPECT_EQ(result->Text(), text); + EXPECT_EQ(result->text(), text); return *result; } diff --git a/toolchain/lexer/tokenized_buffer.cpp b/toolchain/lexer/tokenized_buffer.cpp index ecb3ebe14f48..52764419c47d 100644 --- a/toolchain/lexer/tokenized_buffer.cpp +++ b/toolchain/lexer/tokenized_buffer.cpp @@ -223,7 +223,7 @@ class TokenizedBuffer::Lexer { } int int_column = current_column_; - int token_size = literal->Text().size(); + int token_size = literal->text().size(); current_column_ += token_size; source_text = source_text.drop_front(token_size); @@ -440,8 +440,8 @@ class TokenizedBuffer::Lexer { open_groups_.pop_back(); token_emitter_.EmitError(opening_token); - CHECK(!buffer_.Tokens().empty()) << "Must have a prior opening token!"; - Token prev_token = buffer_.Tokens().end()[-1]; + CHECK(!buffer_.tokens().empty()) << "Must have a prior opening token!"; + Token prev_token = buffer_.tokens().end()[-1]; // TODO: do a smarter backwards scan for where to put the closing // token. @@ -644,7 +644,7 @@ auto TokenizedBuffer::GetTokenText(Token token) const -> llvm::StringRef { llvm::Optional relexed_token = LexedNumericLiteral::Lex(source_->Text().substr(token_start)); CHECK(relexed_token) << "Could not reform numeric literal token."; - return relexed_token->Text(); + return relexed_token->text(); } // Refer back to the source text to find the original spelling, including @@ -741,7 +741,7 @@ auto TokenizedBuffer::GetMatchedOpeningToken(Token closing_token) const auto TokenizedBuffer::HasLeadingWhitespace(Token token) const -> bool { auto it = TokenIterator(token); - return it == Tokens().begin() || GetTokenInfo(*(it - 1)).has_trailing_space; + return it == tokens().begin() || GetTokenInfo(*(it - 1)).has_trailing_space; } auto TokenizedBuffer::HasTrailingWhitespace(Token token) const -> bool { @@ -799,17 +799,17 @@ auto TokenizedBuffer::GetTokenPrintWidths(Token token) const -> PrintWidths { } auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void { - if (Tokens().begin() == Tokens().end()) { + if (tokens().begin() == tokens().end()) { return; } PrintWidths widths = {}; widths.index = ComputeDecimalPrintedWidth((token_infos_.size())); - for (Token token : Tokens()) { + for (Token token : tokens()) { widths.Widen(GetTokenPrintWidths(token)); } - for (Token token : Tokens()) { + for (Token token : tokens()) { PrintToken(output_stream, token, widths); output_stream << "\n"; } diff --git a/toolchain/lexer/tokenized_buffer.h b/toolchain/lexer/tokenized_buffer.h index 7bb01b73f0b2..a42b66222f32 100644 --- a/toolchain/lexer/tokenized_buffer.h +++ b/toolchain/lexer/tokenized_buffer.h @@ -272,16 +272,6 @@ class TokenizedBuffer { static auto Lex(SourceBuffer& source, DiagnosticConsumer& consumer) -> TokenizedBuffer; - // Returns true if the buffer has errors that are detectable at lexing time. - [[nodiscard]] auto HasErrors() const -> bool { return has_errors_; } - - [[nodiscard]] auto Tokens() const -> llvm::iterator_range { - return llvm::make_range(TokenIterator(Token(0)), - TokenIterator(Token(token_infos_.size()))); - } - - [[nodiscard]] auto Size() const -> int { return token_infos_.size(); } - [[nodiscard]] auto GetKind(Token token) const -> TokenKind; [[nodiscard]] auto GetLine(Token token) const -> Line; @@ -365,6 +355,16 @@ class TokenizedBuffer { // format. auto PrintToken(llvm::raw_ostream& output_stream, Token token) const -> void; + // Returns true if the buffer has errors that are detectable at lexing time. + [[nodiscard]] auto has_errors() const -> bool { return has_errors_; } + + [[nodiscard]] auto tokens() const -> llvm::iterator_range { + return llvm::make_range(TokenIterator(Token(0)), + TokenIterator(Token(token_infos_.size()))); + } + + [[nodiscard]] auto size() const -> int { return token_infos_.size(); } + private: // Implementation detail struct implementing the actual lexer logic. class Lexer; diff --git a/toolchain/lexer/tokenized_buffer_fuzzer.cpp b/toolchain/lexer/tokenized_buffer_fuzzer.cpp index e1122ea841a2..10524200b952 100644 --- a/toolchain/lexer/tokenized_buffer_fuzzer.cpp +++ b/toolchain/lexer/tokenized_buffer_fuzzer.cpp @@ -29,7 +29,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, llvm::StringRef(reinterpret_cast(data), size)); auto buffer = TokenizedBuffer::Lex(*source, NullDiagnosticConsumer()); - if (buffer.HasErrors()) { + if (buffer.has_errors()) { return 0; } @@ -37,7 +37,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, // // TODO: We should enhance this to do more sanity checks on the resulting // token stream. - for (TokenizedBuffer::Token token : buffer.Tokens()) { + for (TokenizedBuffer::Token token : buffer.tokens()) { int line_number = buffer.GetLineNumber(token); CHECK(line_number > 0) << "Invalid line number!"; CHECK(line_number < INT_MAX) << "Invalid line number!"; diff --git a/toolchain/lexer/tokenized_buffer_test.cpp b/toolchain/lexer/tokenized_buffer_test.cpp index 81c294e7ade6..04704bb17dab 100644 --- a/toolchain/lexer/tokenized_buffer_test.cpp +++ b/toolchain/lexer/tokenized_buffer_test.cpp @@ -48,7 +48,7 @@ class LexerTest : public ::testing::Test { TEST_F(LexerTest, HandlesEmptyBuffer) { auto buffer = Lex(""); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); @@ -56,7 +56,7 @@ TEST_F(LexerTest, HandlesEmptyBuffer) { TEST_F(LexerTest, TracksLinesAndColumns) { auto buffer = Lex("\n ;;\n ;;;\n x\"foo\" \"\"\"baz\n a\n \"\"\" y"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Semi(), @@ -103,7 +103,7 @@ TEST_F(LexerTest, TracksLinesAndColumns) { TEST_F(LexerTest, HandlesNumericLiteral) { auto buffer = Lex("12-578\n 1 2\n0x12_3ABC\n0b10_10_11\n1_234_567\n1.5e9"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::IntegerLiteral(), @@ -152,21 +152,21 @@ TEST_F(LexerTest, HandlesNumericLiteral) { .text = "1.5e9"}, {.kind = TokenKind::EndOfFile(), .line = 6, .column = 6}, })); - auto token_12 = buffer.Tokens().begin(); + auto token_12 = buffer.tokens().begin(); EXPECT_EQ(buffer.GetIntegerLiteral(*token_12), 12); - auto token_578 = buffer.Tokens().begin() + 2; + auto token_578 = buffer.tokens().begin() + 2; EXPECT_EQ(buffer.GetIntegerLiteral(*token_578), 578); - auto token_1 = buffer.Tokens().begin() + 3; + auto token_1 = buffer.tokens().begin() + 3; EXPECT_EQ(buffer.GetIntegerLiteral(*token_1), 1); - auto token_2 = buffer.Tokens().begin() + 4; + auto token_2 = buffer.tokens().begin() + 4; EXPECT_EQ(buffer.GetIntegerLiteral(*token_2), 2); - auto token_0x12_3abc = buffer.Tokens().begin() + 5; + auto token_0x12_3abc = buffer.tokens().begin() + 5; EXPECT_EQ(buffer.GetIntegerLiteral(*token_0x12_3abc), 0x12'3abc); - auto token_0b10_10_11 = buffer.Tokens().begin() + 6; + auto token_0b10_10_11 = buffer.tokens().begin() + 6; EXPECT_EQ(buffer.GetIntegerLiteral(*token_0b10_10_11), 0b10'10'11); - auto token_1_234_567 = buffer.Tokens().begin() + 7; + auto token_1_234_567 = buffer.tokens().begin() + 7; EXPECT_EQ(buffer.GetIntegerLiteral(*token_1_234_567), 1'234'567); - auto token_1_5e9 = buffer.Tokens().begin() + 8; + auto token_1_5e9 = buffer.tokens().begin() + 8; auto value_1_5e9 = buffer.GetRealLiteral(*token_1_5e9); EXPECT_EQ(value_1_5e9.Mantissa().getZExtValue(), 15); EXPECT_EQ(value_1_5e9.Exponent().getSExtValue(), 8); @@ -175,7 +175,7 @@ TEST_F(LexerTest, HandlesNumericLiteral) { TEST_F(LexerTest, HandlesInvalidNumericLiterals) { auto buffer = Lex("14x 15_49 0x3.5q 0x3_4.5_6 0ops"); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Error(), @@ -224,7 +224,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) { 13._ )"; auto buffer = Lex(source_text); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::IntegerLiteral(), .text = "1"}, @@ -282,7 +282,7 @@ TEST_F(LexerTest, SplitsNumericLiteralsProperly) { TEST_F(LexerTest, HandlesGarbageCharacters) { constexpr char GarbageText[] = "$$💩-$\n$\0$12$\n\\\"\\\n\"x"; auto buffer = Lex(llvm::StringRef(GarbageText, sizeof(GarbageText) - 1)); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ @@ -320,7 +320,7 @@ TEST_F(LexerTest, Symbols) { // common code, but we want to check specific patterns to verify things like // max-munch rule and handling of interesting symbols. auto buffer = Lex("<<<"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::LessLess()}, {TokenKind::Less()}, @@ -328,7 +328,7 @@ TEST_F(LexerTest, Symbols) { })); buffer = Lex("<<=>>"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::LessLessEqual()}, {TokenKind::GreaterGreater()}, @@ -336,7 +336,7 @@ TEST_F(LexerTest, Symbols) { })); buffer = Lex("< <=> >"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::Less()}, {TokenKind::LessEqualGreater()}, @@ -345,7 +345,7 @@ TEST_F(LexerTest, Symbols) { })); buffer = Lex("\\/?@&^!"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::Backslash()}, {TokenKind::Slash()}, @@ -360,7 +360,7 @@ TEST_F(LexerTest, Symbols) { TEST_F(LexerTest, Parens) { auto buffer = Lex("()"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::OpenParen()}, {TokenKind::CloseParen()}, @@ -368,7 +368,7 @@ TEST_F(LexerTest, Parens) { })); buffer = Lex("((()()))"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::OpenParen()}, {TokenKind::OpenParen()}, @@ -384,7 +384,7 @@ TEST_F(LexerTest, Parens) { TEST_F(LexerTest, CurlyBraces) { auto buffer = Lex("{}"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::OpenCurlyBrace()}, {TokenKind::CloseCurlyBrace()}, @@ -392,7 +392,7 @@ TEST_F(LexerTest, CurlyBraces) { })); buffer = Lex("{{{}{}}}"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::OpenCurlyBrace()}, {TokenKind::OpenCurlyBrace()}, @@ -409,8 +409,8 @@ TEST_F(LexerTest, CurlyBraces) { TEST_F(LexerTest, MatchingGroups) { { TokenizedBuffer buffer = Lex("(){}"); - ASSERT_FALSE(buffer.HasErrors()); - auto it = buffer.Tokens().begin(); + ASSERT_FALSE(buffer.has_errors()); + auto it = buffer.tokens().begin(); auto open_paren_token = *it++; auto close_paren_token = *it++; EXPECT_EQ(close_paren_token, @@ -425,13 +425,13 @@ TEST_F(LexerTest, MatchingGroups) { buffer.GetMatchedOpeningToken(close_curly_token)); auto eof_token = *it++; EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile()); - EXPECT_EQ(buffer.Tokens().end(), it); + EXPECT_EQ(buffer.tokens().end(), it); } { TokenizedBuffer buffer = Lex("({x}){(y)} {{((z))}}"); - ASSERT_FALSE(buffer.HasErrors()); - auto it = buffer.Tokens().begin(); + ASSERT_FALSE(buffer.has_errors()); + auto it = buffer.tokens().begin(); auto open_paren_token = *it++; auto open_curly_token = *it++; ASSERT_EQ("x", buffer.GetIdentifierText(buffer.GetIdentifier(*it++))); @@ -488,13 +488,13 @@ TEST_F(LexerTest, MatchingGroups) { auto eof_token = *it++; EXPECT_EQ(buffer.GetKind(eof_token), TokenKind::EndOfFile()); - EXPECT_EQ(buffer.Tokens().end(), it); + EXPECT_EQ(buffer.tokens().end(), it); } } TEST_F(LexerTest, MismatchedGroups) { auto buffer = Lex("{"); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::OpenCurlyBrace()}, @@ -503,14 +503,14 @@ TEST_F(LexerTest, MismatchedGroups) { })); buffer = Lex("}"); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Error(), .text = "}"}, {TokenKind::EndOfFile()}, })); buffer = Lex("{(}"); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ @@ -522,7 +522,7 @@ TEST_F(LexerTest, MismatchedGroups) { })); buffer = Lex(")({)"); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{ @@ -559,7 +559,7 @@ TEST_F(LexerTest, Whitespace) { // EOF false}; int pos = 0; - for (TokenizedBuffer::Token token : buffer.Tokens()) { + for (TokenizedBuffer::Token token : buffer.tokens()) { ASSERT_LT(pos, std::size(space)); EXPECT_THAT(buffer.HasLeadingWhitespace(token), Eq(space[pos])); ++pos; @@ -571,7 +571,7 @@ TEST_F(LexerTest, Whitespace) { TEST_F(LexerTest, Keywords) { auto buffer = Lex(" fn"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Fn(), .column = 4, .indent_column = 4}, @@ -579,7 +579,7 @@ TEST_F(LexerTest, Keywords) { })); buffer = Lex("and or not if else for return var break continue _"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {TokenKind::And()}, {TokenKind::Or()}, @@ -598,7 +598,7 @@ TEST_F(LexerTest, Keywords) { TEST_F(LexerTest, Comments) { auto buffer = Lex(" ;\n // foo\n ;\n"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Semi(), @@ -613,21 +613,21 @@ TEST_F(LexerTest, Comments) { })); buffer = Lex("// foo\n//\n// bar"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); // Make sure weird characters aren't a problem. buffer = Lex(" // foo#$!^?@-_💩🍫⃠ [̲̅$̲̅(̲̅ ͡° ͜ʖ ͡°̲̅)̲̅$̲̅]"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); // Make sure we can lex a comment at the end of the input. buffer = Lex("//"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT( buffer, HasTokens(llvm::ArrayRef{{TokenKind::EndOfFile()}})); @@ -642,13 +642,13 @@ TEST_F(LexerTest, InvalidComments) { }; for (llvm::StringLiteral testcase : testcases) { auto buffer = Lex(testcase); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); } } TEST_F(LexerTest, Identifiers) { auto buffer = Lex(" foobar"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Identifier(), .column = 4, @@ -659,14 +659,14 @@ TEST_F(LexerTest, Identifiers) { // Check different kinds of identifier character sequences. buffer = Lex("_foo_bar"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Identifier(), .text = "_foo_bar"}, {TokenKind::EndOfFile()}, })); buffer = Lex("foo2bar00"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Identifier(), .text = "foo2bar00"}, @@ -675,7 +675,7 @@ TEST_F(LexerTest, Identifiers) { // Check that we can parse identifiers that start with a keyword. buffer = Lex("fnord"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Identifier(), .text = "fnord"}, {TokenKind::EndOfFile()}, @@ -683,7 +683,7 @@ TEST_F(LexerTest, Identifiers) { // Check multiple identifiers with indent and interning. buffer = Lex(" foo;bar\nbar \n foo\tfoo"); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Identifier(), @@ -735,7 +735,7 @@ TEST_F(LexerTest, StringLiterals) { )"; auto buffer = Lex(testcase); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); EXPECT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::StringLiteral(), @@ -811,11 +811,11 @@ TEST_F(LexerTest, InvalidStringLiterals) { for (llvm::StringLiteral test : invalid) { SCOPED_TRACE(test); auto buffer = Lex(test); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); // We should have formed at least one error token. bool found_error = false; - for (TokenizedBuffer::Token token : buffer.Tokens()) { + for (TokenizedBuffer::Token token : buffer.tokens()) { if (buffer.GetKind(token) == TokenKind::Error()) { found_error = true; break; @@ -834,7 +834,7 @@ TEST_F(LexerTest, TypeLiterals) { )"; auto buffer = Lex(testcase); - EXPECT_FALSE(buffer.HasErrors()); + EXPECT_FALSE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Identifier(), @@ -914,21 +914,21 @@ TEST_F(LexerTest, TypeLiterals) { {.kind = TokenKind::EndOfFile(), .line = 6, .column = 3}, })); - auto token_i1 = buffer.Tokens().begin() + 1; + auto token_i1 = buffer.tokens().begin() + 1; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_i1), 1); - auto token_i20 = buffer.Tokens().begin() + 2; + auto token_i20 = buffer.tokens().begin() + 2; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_i20), 20); - auto token_i999999999999 = buffer.Tokens().begin() + 3; + auto token_i999999999999 = buffer.tokens().begin() + 3; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_i999999999999), 999999999999ull); - auto token_u1 = buffer.Tokens().begin() + 6; + auto token_u1 = buffer.tokens().begin() + 6; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_u1), 1); - auto token_u64 = buffer.Tokens().begin() + 7; + auto token_u64 = buffer.tokens().begin() + 7; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_u64), 64); - auto token_f32 = buffer.Tokens().begin() + 9; + auto token_f32 = buffer.tokens().begin() + 9; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_f32), 32); - auto token_f80 = buffer.Tokens().begin() + 10; + auto token_f80 = buffer.tokens().begin() + 10; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_f80), 80); - auto token_f1 = buffer.Tokens().begin() + 11; + auto token_f1 = buffer.tokens().begin() + 11; EXPECT_EQ(buffer.GetTypeLiteralSize(*token_f1), 1); } @@ -943,7 +943,7 @@ TEST_F(LexerTest, TypeLiteralTooManyDigits) { DiagnosticAt(1, 2), DiagnosticMessage(HasSubstr("Found a sequence of 10000 digits"))))); auto buffer = Lex(code, consumer); - EXPECT_TRUE(buffer.HasErrors()); + EXPECT_TRUE(buffer.has_errors()); ASSERT_THAT(buffer, HasTokens(llvm::ArrayRef{ {.kind = TokenKind::Error(), @@ -1040,7 +1040,7 @@ auto GetAndDropLine(llvm::StringRef& text) -> std::string { TEST_F(LexerTest, PrintingBasic) { auto buffer = Lex(";"); - ASSERT_FALSE(buffer.HasErrors()); + ASSERT_FALSE(buffer.has_errors()); std::string print_storage; llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); @@ -1056,7 +1056,7 @@ TEST_F(LexerTest, PrintingBasic) { TEST_F(LexerTest, PrintingInteger) { auto buffer = Lex("123"); - ASSERT_FALSE(buffer.HasErrors()); + ASSERT_FALSE(buffer.has_errors()); std::string print_storage; llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); @@ -1071,7 +1071,7 @@ TEST_F(LexerTest, PrintingInteger) { TEST_F(LexerTest, PrintingReal) { auto buffer = Lex("2.5"); - ASSERT_FALSE(buffer.HasErrors()); + ASSERT_FALSE(buffer.has_errors()); std::string print_storage; llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); @@ -1088,7 +1088,7 @@ TEST_F(LexerTest, PrintingReal) { TEST_F(LexerTest, PrintingPadding) { // Test kind padding. auto buffer = Lex("(;foo;)"); - ASSERT_FALSE(buffer.HasErrors()); + ASSERT_FALSE(buffer.has_errors()); std::string print_storage; llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); @@ -1118,7 +1118,7 @@ TEST_F(LexerTest, PrintingPadding) { TEST_F(LexerTest, PrintingPaddingDigits) { // Test digit padding with max values of 9, 10, and 11. auto buffer = Lex(";\n\n\n\n\n\n\n\n\n\n ;;"); - ASSERT_FALSE(buffer.HasErrors()); + ASSERT_FALSE(buffer.has_errors()); std::string print_storage; llvm::raw_string_ostream print_stream(print_storage); buffer.Print(print_stream); @@ -1145,7 +1145,7 @@ TEST_F(LexerTest, PrintingPaddingDigits) { TEST_F(LexerTest, PrintingAsYaml) { // Test that we can parse this into YAML and verify line and indent data. auto buffer = Lex("\n ;\n\n\n; ;\n\n\n\n\n\n\n\n\n\n\n"); - ASSERT_FALSE(buffer.HasErrors()); + ASSERT_FALSE(buffer.has_errors()); std::string print_output; llvm::raw_string_ostream print_stream(print_output); buffer.Print(print_stream); diff --git a/toolchain/lexer/tokenized_buffer_test_helpers.h b/toolchain/lexer/tokenized_buffer_test_helpers.h index d46beec3ad3f..cee2cb9e3e8a 100644 --- a/toolchain/lexer/tokenized_buffer_test_helpers.h +++ b/toolchain/lexer/tokenized_buffer_test_helpers.h @@ -70,14 +70,14 @@ MATCHER_P(HasTokens, raw_all_expected, "") { llvm::ArrayRef all_expected = raw_all_expected; bool matches = true; - auto buffer_it = buffer.Tokens().begin(); + auto buffer_it = buffer.tokens().begin(); for (const ExpectedToken& expected : all_expected) { - if (buffer_it == buffer.Tokens().end()) { + if (buffer_it == buffer.tokens().end()) { // The size check outside the loop will fail and print useful info. break; } - int index = buffer_it - buffer.Tokens().begin(); + int index = buffer_it - buffer.tokens().begin(); auto token = *buffer_it++; TokenKind actual_kind = buffer.GetKind(token); @@ -144,7 +144,7 @@ MATCHER_P(HasTokens, raw_all_expected, "") { } } - int actual_size = buffer.Tokens().end() - buffer.Tokens().begin(); + int actual_size = buffer.tokens().end() - buffer.tokens().begin(); if (static_cast(all_expected.size()) != actual_size) { *result_listener << "\nExpected " << all_expected.size() << " tokens but found " << actual_size << "."; diff --git a/toolchain/parser/parse_tree_fuzzer.cpp b/toolchain/parser/parse_tree_fuzzer.cpp index 01093dcb9139..59dc9e8a29c2 100644 --- a/toolchain/parser/parse_tree_fuzzer.cpp +++ b/toolchain/parser/parse_tree_fuzzer.cpp @@ -29,7 +29,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, // Lex the input. auto tokens = TokenizedBuffer::Lex(*source, NullDiagnosticConsumer()); - if (tokens.HasErrors()) { + if (tokens.has_errors()) { return 0; } @@ -42,7 +42,7 @@ extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, // In the absence of parse errors, we should have exactly as many nodes as // tokens. - CHECK(tree.Size() == tokens.Size()) << "Unexpected number of tree nodes!"; + CHECK(tree.Size() == tokens.size()) << "Unexpected number of tree nodes!"; return 0; } diff --git a/toolchain/parser/parse_tree_test.cpp b/toolchain/parser/parse_tree_test.cpp index 856671e14e71..c94fce50f349 100644 --- a/toolchain/parser/parse_tree_test.cpp +++ b/toolchain/parser/parse_tree_test.cpp @@ -75,8 +75,8 @@ TEST_F(ParseTreeTest, EmptyDeclaration) { EXPECT_THAT(tree.GetNodeKind(eof), Eq(ParseNodeKind::FileEnd())); auto t = tree.GetNodeToken(n); - ASSERT_THAT(tokens.Tokens().begin(), Ne(tokens.Tokens().end())); - EXPECT_THAT(t, Eq(*tokens.Tokens().begin())); + ASSERT_THAT(tokens.tokens().begin(), Ne(tokens.tokens().end())); + EXPECT_THAT(t, Eq(*tokens.tokens().begin())); EXPECT_THAT(tokens.GetTokenText(t), Eq(";")); EXPECT_THAT(tree.Children(n).begin(), Eq(tree.Children(n).end())); @@ -1252,7 +1252,7 @@ TEST_F(ParseTreeTest, RecursionLimit) { code.append(10000, ')'); code += "; }"; TokenizedBuffer tokens = GetTokenizedBuffer(code); - ASSERT_FALSE(tokens.HasErrors()); + ASSERT_FALSE(tokens.has_errors()); Testing::MockDiagnosticConsumer consumer; // Recursion might be exceeded multiple times due to quirks in parse tree // handling; we only need to be sure it's hit at least once for test @@ -1275,7 +1275,7 @@ TEST_F(ParseTreeTest, ParsePostfixExpressionRegression) { code.append(ParseTree::StackDepthLimit - n, '*'); code += "(z);"; TokenizedBuffer tokens = GetTokenizedBuffer(code); - ASSERT_FALSE(tokens.HasErrors()); + ASSERT_FALSE(tokens.has_errors()); ParseTree tree = ParseTree::Parse(tokens, consumer); EXPECT_TRUE(tree.HasErrors()); } diff --git a/toolchain/parser/parser_impl.cpp b/toolchain/parser/parser_impl.cpp index 64ddfdbef449..5001d972ee65 100644 --- a/toolchain/parser/parser_impl.cpp +++ b/toolchain/parser/parser_impl.cpp @@ -242,8 +242,8 @@ ParseTree::Parser::Parser(ParseTree& tree_arg, TokenizedBuffer& tokens_arg, : tree_(tree_arg), tokens_(tokens_arg), emitter_(emitter), - position_(tokens_.Tokens().begin()), - end_(tokens_.Tokens().end()) { + position_(tokens_.tokens().begin()), + end_(tokens_.tokens().end()) { CHECK(std::find_if(position_, end_, [&](TokenizedBuffer::Token t) { return tokens_.GetKind(t) == TokenKind::EndOfFile(); @@ -258,7 +258,7 @@ auto ParseTree::Parser::Parse(TokenizedBuffer& tokens, // We expect to have a 1:1 correspondence between tokens and tree nodes, so // reserve the space we expect to need here to avoid allocation and copying // overhead. - tree.node_impls_.reserve(tokens.Size()); + tree.node_impls_.reserve(tokens.size()); Parser parser(tree, tokens, emitter); while (!parser.AtEndOfFile()) { @@ -982,7 +982,7 @@ auto ParseTree::Parser::IsLexicallyValidInfixOperator() -> bool { // Otherwise, for an infix operator, the preceding token must be any close // bracket, identifier, or literal and the next token must be an open paren, // identifier, or literal. - if (position_ == tokens_.Tokens().begin() || + if (position_ == tokens_.tokens().begin() || !IsAssumedEndOfOperand(tokens_.GetKind(*(position_ - 1))) || !IsAssumedStartOfOperand(tokens_.GetKind(*(position_ + 1)))) { return false;