// Part of the Carbon Language project, under the Apache License v2.0 with LLVM // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include #include "common/check.h" #include "testing/fuzzing/libfuzzer.h" #include "toolchain/lex/mismatched_brackets.h" namespace Carbon::Lex::Testing { namespace { struct Insertion { TokenIndex anchor; bool is_after; size_t order; TokenKind kind; }; // Verifies that applying all corrections to the input token sequence results in // a correctly bracket-balanced stream. auto VerifyBracketBalance(llvm::ArrayRef tokens, llvm::ArrayRef corrections) -> void { llvm::SmallVector is_replaced_with_error(tokens.size(), false); llvm::SmallVector insertions; for (size_t order = 0; order < corrections.size(); ++order) { const auto& corr = corrections[order]; if (corr.fix_action == BracketFixAction::ReplaceWithError) { is_replaced_with_error[corr.fix_token_index.index] = true; } else if (corr.fix_action == BracketFixAction::InsertBefore) { insertions.push_back({ .anchor = corr.fix_token_index, .is_after = false, .order = order, .kind = corr.fix_token_kind, }); } else if (corr.fix_action == BracketFixAction::InsertAfter) { insertions.push_back({ .anchor = corr.fix_token_index, .is_after = true, .order = order, .kind = corr.fix_token_kind, }); } } // This must match `ErrorRecoveryBuffer::Apply` in lex.cpp, which is how the // corrections are actually applied to the token stream. In particular, at a // shared insertion point, closing brackets are inserted before opening // brackets, so that closing an outer group and opening a new one land in // the correct order. llvm::stable_sort(insertions, [](const Insertion& a, const Insertion& b) { TokenIndex a_target = a.is_after ? TokenIndex(a.anchor.index + 1) : a.anchor; TokenIndex b_target = b.is_after ? TokenIndex(b.anchor.index + 1) : b.anchor; if (a_target != b_target) { return a_target < b_target; } if (a.is_after != b.is_after) { return a.is_after; } bool a_is_closing = a.kind.is_closing_symbol(); bool b_is_closing = b.kind.is_closing_symbol(); if (a_is_closing != b_is_closing) { return a_is_closing; } if (a.is_after) { return a.order < b.order; } else { return a.order > b.order; } }); llvm::SmallVector resulting_stream; size_t ins_idx = 0; for (int32_t i = 0; i <= static_cast(tokens.size()); ++i) { while (ins_idx < insertions.size()) { TokenIndex target = insertions[ins_idx].is_after ? TokenIndex(insertions[ins_idx].anchor.index + 1) : insertions[ins_idx].anchor; if (target.index != i) { break; } resulting_stream.push_back(insertions[ins_idx].kind); ++ins_idx; } if (i < static_cast(tokens.size())) { if (!is_replaced_with_error[i]) { resulting_stream.push_back(ToTokenKind(tokens[i].kind)); } } } llvm::SmallVector stack; for (TokenKind kind : resulting_stream) { if (kind.is_opening_symbol()) { stack.push_back(kind); } else if (kind.is_closing_symbol()) { CARBON_CHECK(!stack.empty(), "Unmatched closing bracket in fixed stream!"); TokenKind top = stack.pop_back_val(); CARBON_CHECK(top.closing_symbol() == kind, "Mismatched bracket pair in fixed stream!"); } } CARBON_CHECK(stack.empty(), "Unclosed opening brackets remaining in fixed stream!"); } } // namespace // Fuzz tester for mismatched bracket recovery. // NOLINTNEXTLINE: Match the documented fuzzer entry point declaration style. extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, size_t size) { if (size > 2000) { return 0; } // Every kind except `FileEnd`, which only ever appears as the final token. constexpr auto NumGeneratedKinds = static_cast(BracketTokenKind::Other); static_assert(static_cast(BracketTokenKind::FileEnd) == NumGeneratedKinds - 1); // Bytes per generated token: kind, indentation, line advance, and flags. Most // recovery cues come from the kind and the flags, so leaving either coarse // would make most of the rules unreachable. constexpr size_t BytesPerToken = 4; llvm::SmallVector tokens; tokens.reserve(size / BytesPerToken); size_t i = 0; int32_t token_idx = 0; int32_t current_line = 1; while (i + BytesPerToken <= size) { uint8_t kind_byte = data[i++]; uint8_t indent_byte = data[i++]; uint8_t line_delta = data[i++]; uint8_t flags_byte = data[i++]; auto kind = static_cast(kind_byte % NumGeneratedKinds); if (kind == BracketTokenKind::FileEnd) { kind = BracketTokenKind::Other; } int32_t indent = (indent_byte % 32) * 2; current_line += line_delta % 3; tokens.push_back(MismatchedBracketToken{ .token_index = TokenIndex(token_idx++), .kind = kind, .line = current_line, .line_indent = indent, .is_at_end_of_line = (flags_byte & 1) != 0, .is_struct_brace = (flags_byte & 2) != 0, .is_paren_keyword = (flags_byte & 8) != 0, .is_else_keyword = (flags_byte & 16) != 0, .has_leading_space = (flags_byte & 32) != 0, .has_wide_leading_space = (flags_byte & 64) != 0, }); } tokens.push_back(MismatchedBracketToken{ .token_index = TokenIndex(token_idx++), .kind = BracketTokenKind::FileEnd, .line = current_line, .line_indent = 0, .is_at_end_of_line = true, }); auto corrections = FixMismatchedBrackets(tokens); // Invariant verification: all indices must be valid. for (const auto& corr : corrections) { CARBON_CHECK(corr.diagnostic_token_index.index >= 0 && corr.diagnostic_token_index.index < token_idx, "Invalid diag token index!"); CARBON_CHECK(corr.fix_token_index.index >= 0 && corr.fix_token_index.index < token_idx, "Invalid fix token index!"); } // Verification: applying fixes must result in a balanced bracket sequence. VerifyBracketBalance(tokens, corrections); return 0; } } // namespace Carbon::Lex::Testing