mirror of
https://github.com/carbon-language/carbon-lang.git
synced 2026-09-24 22:02:23 +01:00
Jon Ross-Perkins
22d7cd19ed
Reals were mostly handled, but this PR adds storage of them. It also switches a little towards the FloatingPointType semantic from TokenizedBuffer. While real literals like `1.0` were handled, the type literals were not. This just adds `f64`, similar to how I also only support `i32`. The String type literal wasn't used, so I've added support in lexer and parser. Per discussion with @zygoloid String might be renamed based on the newer type literal plan, but it's still String in explorer and the design, so this is just consistent. The builtin_types.carbon tests the three basic types that are there right now. The test is added to both parser and semantics so that it's clear what the state is in both stages.
90 lines
4.3 KiB
C++
90 lines
4.3 KiB
C++
// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
|
|
// Exceptions. See /LICENSE for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
#include "toolchain/lexer/token_kind.h"
|
|
|
|
#include <gmock/gmock.h>
|
|
#include <gtest/gtest.h>
|
|
|
|
#include <cstring>
|
|
|
|
#include "llvm/ADT/StringRef.h"
|
|
|
|
namespace Carbon::Testing {
|
|
namespace {
|
|
|
|
using ::testing::MatchesRegex;
|
|
|
|
// We restrict symbols to punctuation characters that are expected to be widely
|
|
// available on modern keyboards used for programming.
|
|
constexpr llvm::StringLiteral SymbolRegex =
|
|
R"([\[\]{}!@#%^&*()/?\\|;:.,<>=+~-]+)";
|
|
|
|
// We restrict keywords to be lowercase ASCII letters and underscores with a few
|
|
// specific exceptions.
|
|
constexpr llvm::StringLiteral KeywordRegex = "[a-z_]+|Self|String";
|
|
|
|
#define CARBON_TOKEN(TokenName) \
|
|
TEST(TokenKindTest, TokenName) { \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
|
|
EXPECT_EQ("", TokenKind::TokenName.fixed_spelling()); \
|
|
}
|
|
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \
|
|
TEST(TokenKindTest, TokenName) { \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_grouping_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
|
|
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
|
|
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
|
|
}
|
|
#define CARBON_OPENING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, ClosingName) \
|
|
TEST(TokenKindTest, TokenName) { \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_opening_symbol()); \
|
|
EXPECT_EQ(TokenKind::ClosingName, TokenKind::TokenName.closing_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_closing_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
|
|
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
|
|
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
|
|
}
|
|
#define CARBON_CLOSING_GROUP_SYMBOL_TOKEN(TokenName, Spelling, OpeningName) \
|
|
TEST(TokenKindTest, TokenName) { \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_symbol()); \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_grouping_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_opening_symbol()); \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_closing_symbol()); \
|
|
EXPECT_EQ(TokenKind::OpeningName, TokenKind::TokenName.opening_symbol()); \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_keyword()); \
|
|
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
|
|
EXPECT_THAT(Spelling, MatchesRegex(SymbolRegex.str())); \
|
|
}
|
|
#define CARBON_KEYWORD_TOKEN(TokenName, Spelling) \
|
|
TEST(TokenKindTest, TokenName) { \
|
|
EXPECT_FALSE(TokenKind::TokenName.is_symbol()); \
|
|
EXPECT_TRUE(TokenKind::TokenName.is_keyword()); \
|
|
EXPECT_EQ(Spelling, TokenKind::TokenName.fixed_spelling()); \
|
|
EXPECT_THAT(Spelling, MatchesRegex(KeywordRegex.str())); \
|
|
}
|
|
#include "toolchain/lexer/token_kind.def"
|
|
|
|
// Verify that the symbol tokens are sorted from longest to shortest. This is
|
|
// important to ensure that simply in-order testing will identify tokens
|
|
// following the max-munch rule.
|
|
TEST(TokenKindTest, SymbolsInDescendingLength) {
|
|
int previous_length = INT_MAX;
|
|
#define CARBON_SYMBOL_TOKEN(TokenName, Spelling) \
|
|
EXPECT_LE(llvm::StringRef(Spelling).size(), previous_length) \
|
|
<< "Symbol token not in descending length order: " << #TokenName; \
|
|
previous_length = llvm::StringRef(Spelling).size();
|
|
#include "toolchain/lexer/token_kind.def"
|
|
EXPECT_GT(previous_length, 0);
|
|
}
|
|
|
|
} // namespace
|
|
} // namespace Carbon::Testing
|