Files
carbon-lang/toolchain/lexer/token_kind_test.cpp
T
Jon Ross-Perkins b9df8ca765 Manual cleanup of toolchain clang-tidy/clangd warnings. (#3157)
A lot of this is more boring "remove unused header", plus some other
minor cleanups. I think the most significant changes were:

- yaml_test_helpers.cpp is doing a switch on an unsigned int, comparing
to enum values.
-
[bugprone-switch-missing-default-case](https://clang.llvm.org/extra/clang-tidy/checks/bugprone/switch-missing-default-case.html)
is unhappy with EnumBase, but correctly identified
yaml_test_helpers.cpp, so I'm opting to address it rather than disabling
it even though it needs NOLINT in several locations as a result, in
addition to what I think are some low-value `default` cases. I'd be fine
going the other way with this too and disabling it globally (I could see
it being noisier in the explorer).
- MarkInitializerFor swaps the argument names between the .h and .cpp. I
think the .cpp had the order as intended.
- There's a new-ish
[performance-enum-size](https://clang.llvm.org/extra/clang-tidy/checks/performance/enum-size.html)
which I'm basically treating as "add int8_t to enums".

My main motivation here is to just clean up as many of these as I can so
that I stop seeing them in vscode.
2023-08-25 22:45:57 +00:00

88 lines
4.3 KiB
C++

// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
// Exceptions. See /LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "toolchain/lexer/token_kind.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "llvm/ADT/StringRef.h"
namespace Carbon::Testing {
namespace {
using ::testing::MatchesRegex;
// We restrict symbols to punctuation characters that are expected to be widely
// available on modern keyboards used for programming.
constexpr llvm::StringLiteral SymbolRegex =
R"([\[\]{}!@#%^&*()/?\\|;:.,<>=+~-]+)";
// We restrict keywords to be lowercase ASCII letters and underscores with a few
// specific exceptions.
constexpr llvm::StringLiteral KeywordRegex = "[a-z_]+|Self|String";
#define CARBON_TOKEN(TokenName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ("", TokenKind::TokenName.fixed_spelling()); \
}
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \
TEST(TokenKindTest, TokenName) { \
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_grouping_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
}
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_opening_symbol()); \
EXPECT_EQ(TokenKind::ClosingName, TokenKind::TokenName.closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
}
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
TEST(TokenKindTest, TokenName) { \
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_closing_symbol()); \
EXPECT_EQ(TokenKind::OpeningName, TokenKind::TokenName.opening_symbol()); \
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
}
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \
TEST(TokenKindTest, TokenName) { \
EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \
EXPECT_TRUE(TokenKind::TokenName.is_keyword()); \
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \
}
#include "toolchain/lexer/token_kind.def"
// Verify that the symbol tokens are sorted from longest to shortest. This is
// important to ensure that simply in-order testing will identify tokens
// following the max-munch rule.
TEST(TokenKindTest, SymbolsInDescendingLength) {
int previous_length = INT_MAX;
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \
EXPECT_LE(llvm::StringRef(Spelling).size(), previous_length) \
<< "Symbol token not in descending length order: " << #TokenName; \
previous_length = llvm::StringRef(Spelling).size();
#include "toolchain/lexer/token_kind.def"
EXPECT_GT(previous_length, 0);
}
} // namespace
} // namespace Carbon::Testing