diff --git a/toolchain/format/BUILD b/toolchain/format/BUILD index 5e01d0a087de..4655645fb0ec 100644 --- a/toolchain/format/BUILD +++ b/toolchain/format/BUILD @@ -13,11 +13,16 @@ filegroup( cc_library( name = "format", - srcs = ["format.cpp"], + srcs = [ + "format.cpp", + "formatter.cpp", + "formatter.h", + ], hdrs = ["format.h"], deps = [ "//common:ostream", "//toolchain/lex:token_index", + "//toolchain/lex:token_kind", "//toolchain/lex:tokenized_buffer", ], ) diff --git a/toolchain/format/format.cpp b/toolchain/format/format.cpp index 827a287e60e8..11f3c4398a68 100644 --- a/toolchain/format/format.cpp +++ b/toolchain/format/format.cpp @@ -4,45 +4,13 @@ #include "toolchain/format/format.h" -#include "toolchain/lex/token_index.h" -#include "toolchain/lex/tokenized_buffer.h" +#include "toolchain/format/formatter.h" namespace Carbon::Format { -// TODO: Add support for formatting line ranges (will need flags too). auto Format(const Lex::TokenizedBuffer& tokens, llvm::raw_ostream& out) -> bool { - if (tokens.has_errors()) { - // TODO: Error recovery. - return false; - } - - auto comments = tokens.comments(); - auto comment_it = comments.begin(); - - llvm::ListSeparator sep(" "); - - for (auto token : tokens.tokens()) { - while (comment_it != comments.end() && - tokens.IsAfterComment(token, *comment_it)) { - // TODO: Fix newlines and indent. - out << "\n" << tokens.GetCommentText(*comment_it) << "\n"; - ++comment_it; - } - - switch (tokens.GetKind(token)) { - case Lex::TokenKind::FileStart: - break; - case Lex::TokenKind::FileEnd: - out << "\n"; - break; - default: - // TODO: More dependent formatting. - out << sep << tokens.GetTokenText(token); - break; - } - } - return true; + return Formatter(&tokens, &out).Run(); } } // namespace Carbon::Format diff --git a/toolchain/format/formatter.cpp b/toolchain/format/formatter.cpp new file mode 100644 index 000000000000..9503e53a16b5 --- /dev/null +++ b/toolchain/format/formatter.cpp @@ -0,0 +1,110 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "toolchain/format/formatter.h" + +namespace Carbon::Format { + +auto Formatter::Run() -> bool { + if (tokens_->has_errors()) { + // TODO: Error recovery. + return false; + } + + auto comments = tokens_->comments(); + auto comment_it = comments.begin(); + + // If there are no tokens or comments, format as empty. + if (tokens_->size() == 0 && comment_it == comments.end()) { + *out_ << "\n"; + return true; + } + + for (auto token : tokens_->tokens()) { + auto token_kind = tokens_->GetKind(token); + + while (comment_it != comments.end() && + tokens_->IsAfterComment(token, *comment_it)) { + RequireEmptyLine(); + PrepareForSpacedContent(); + // TODO: We do need to adjust the indent of multi-line comments. + *out_ << tokens_->GetCommentText(*comment_it); + // Comment text includes a terminating newline, so just update the state. + line_state_ = LineState::Empty; + ++comment_it; + } + + switch (token_kind) { + case Lex::TokenKind::FileStart: + break; + + case Lex::TokenKind::FileEnd: + RequireEmptyLine(); + break; + + case Lex::TokenKind::OpenCurlyBrace: + PrepareForSpacedContent(); + *out_ << "{"; + // Check for `{}`. + if (NextToken(token) != tokens_->GetMatchedClosingToken(token)) { + RequireEmptyLine(); + } + indent_ += 2; + break; + + case Lex::TokenKind::CloseCurlyBrace: + indent_ -= 2; + PrepareForPackedContent(); + *out_ << "}"; + RequireEmptyLine(); + break; + + case Lex::TokenKind::Semi: + PrepareForPackedContent(); + *out_ << ";"; + RequireEmptyLine(); + break; + + default: + if (token_kind.IsOneOf( + {Lex::TokenKind::CloseParen, Lex::TokenKind::Colon, + Lex::TokenKind::ColonExclaim, Lex::TokenKind::Comma})) { + PrepareForPackedContent(); + } else { + PrepareForSpacedContent(); + } + *out_ << tokens_->GetTokenText(token); + line_state_ = token_kind.is_opening_symbol() + ? LineState::HasSeparator + : LineState::NeedsSeparator; + break; + } + } + return true; +} + +auto Formatter::PrepareForPackedContent() -> void { + if (line_state_ == LineState::Empty) { + out_->indent(indent_); + line_state_ = LineState::HasSeparator; + } +} + +auto Formatter::RequireEmptyLine() -> void { + if (line_state_ != LineState::Empty) { + *out_ << "\n"; + line_state_ = LineState::Empty; + } +} + +auto Formatter::PrepareForSpacedContent() -> void { + if (line_state_ == LineState::NeedsSeparator) { + *out_ << " "; + line_state_ = LineState::HasSeparator; + } else { + PrepareForPackedContent(); + } +} + +} // namespace Carbon::Format diff --git a/toolchain/format/formatter.h b/toolchain/format/formatter.h new file mode 100644 index 000000000000..89f3c9419774 --- /dev/null +++ b/toolchain/format/formatter.h @@ -0,0 +1,83 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef CARBON_TOOLCHAIN_FORMAT_FORMATTER_H_ +#define CARBON_TOOLCHAIN_FORMAT_FORMATTER_H_ + +#include + +#include "common/ostream.h" +#include "toolchain/lex/tokenized_buffer.h" + +namespace Carbon::Format { + +// Implements Format(); see format.h. It's intended to be constructed and +// `Run()` once, then destructed. +// +// TODO: This will probably need to work less linearly in the future, for +// example to handle smart wrapping of arguments. This is a simple +// implementation that only handles simple code. Before adding too much more +// complexity, it should be rewritten. +// +// TODO: Add retention of blank lines between original code. +// +// TODO: Add support for formatting line ranges (will need flags too). +class Formatter { + public: + explicit Formatter(const Lex::TokenizedBuffer* tokens, llvm::raw_ostream* out) + : tokens_(tokens), out_(out) {} + + // See class comments. + auto Run() -> bool; + + private: + // Tracks the status of the current line of output. + enum class LineState : uint8_t { + // There is no output for the current line. + Empty, + // The current line has content (possibly just an indent), and does not need + // a separator added. + HasSeparator, + // The current line has content, and will need a separator, typically a + // single space or newline. + NeedsSeparator, + }; + + // Ensure output is on an empty line, setting line_state_ to Empty. May output + // a newline, dependent on line state. Does not indent, allowing blank lines. + auto RequireEmptyLine() -> void; + + // Ensures there is a separator before adding new content. May do + // `PrepareForPackedContent` or output a separator space, dependent on line + // state. Always results in line_state_ being HasSeparator; the caller is + // responsible for adjusting state if needed. + auto PrepareForSpacedContent() -> void; + + // Requires that the current line is indented, but not necessarily a separator + // space. May output spaces for `indent_`, dependent on line state. Only + // guarantees the line_state_ is not Empty; the caller is responsible for + // adjusting state if needed. + auto PrepareForPackedContent() -> void; + + // Returns the next token index. + static auto NextToken(Lex::TokenIndex token) -> Lex::TokenIndex { + return *(Lex::TokenIterator(token) + 1); + } + + // The tokens being formatted. + const Lex::TokenizedBuffer* tokens_; + + // The output stream for formatted content. + llvm::raw_ostream* out_; + + // The state of the line currently written to output. + LineState line_state_ = LineState::Empty; + + // The current code indent level, to be added to new lines. + int indent_ = 0; +}; + +} // namespace Carbon::Format + +#endif // CARBON_TOOLCHAIN_FORMAT_FORMATTER_H_ diff --git a/toolchain/format/testdata/basics/braces.carbon b/toolchain/format/testdata/basics/braces.carbon new file mode 100644 index 000000000000..142bffab25fb --- /dev/null +++ b/toolchain/format/testdata/basics/braces.carbon @@ -0,0 +1,51 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/format/testdata/basics/braces.carbon +// TIP: To dump output, run: +// TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/format/testdata/basics/braces.carbon + +// --- test.carbon + +fn F() { + +} + +fn G() -> i32 { + +return 3; + +} + + +fn H(x: i32, y: i32) -> i32 { + +var z: i32 = x + y; +return z; + +} + +class C { + class D { + class E {} + } +} + +// --- AUTOUPDATE-SPLIT + +// CHECK:STDOUT: fn F () {} +// CHECK:STDOUT: fn G () -> i32 { +// CHECK:STDOUT: return 3; +// CHECK:STDOUT: } +// CHECK:STDOUT: fn H (x: i32, y: i32) -> i32 { +// CHECK:STDOUT: var z: i32 = x + y; +// CHECK:STDOUT: return z; +// CHECK:STDOUT: } +// CHECK:STDOUT: class C { +// CHECK:STDOUT: class D { +// CHECK:STDOUT: class E {} +// CHECK:STDOUT: } +// CHECK:STDOUT: } diff --git a/toolchain/format/testdata/basics/comments.carbon b/toolchain/format/testdata/basics/comments.carbon index 4b218a3d61f8..f12706a0f90a 100644 --- a/toolchain/format/testdata/basics/comments.carbon +++ b/toolchain/format/testdata/basics/comments.carbon @@ -31,23 +31,15 @@ class C { // --- AUTOUPDATE-SPLIT -// CHECK:STDOUT: // CHECK:STDOUT: // A comment -// CHECK:STDOUT: -// CHECK:STDOUT: fn F ( ) { } +// CHECK:STDOUT: fn F () {} // CHECK:STDOUT: // Another comment -// CHECK:STDOUT: -// CHECK:STDOUT: // CHECK:STDOUT: // Block // CHECK:STDOUT: // comment -// CHECK:STDOUT: -// CHECK:STDOUT: class C { -// CHECK:STDOUT: // Internal comment -// CHECK:STDOUT: -// CHECK:STDOUT: } +// CHECK:STDOUT: class C { +// CHECK:STDOUT: // Internal comment +// CHECK:STDOUT: } // CHECK:STDOUT: // Another // CHECK:STDOUT: // Block // CHECK:STDOUT: // // CHECK:STDOUT: // Comment -// CHECK:STDOUT: -// CHECK:STDOUT: diff --git a/toolchain/format/testdata/basics/empty.carbon b/toolchain/format/testdata/basics/empty.carbon index 91a70c24a444..6af90bcb0f0c 100644 --- a/toolchain/format/testdata/basics/empty.carbon +++ b/toolchain/format/testdata/basics/empty.carbon @@ -10,5 +10,3 @@ // --- test.carbon // --- AUTOUPDATE-SPLIT - -// CHECK:STDOUT: diff --git a/toolchain/format/testdata/basics/simple.carbon b/toolchain/format/testdata/basics/simple.carbon index 913e934c9832..de8093296977 100644 --- a/toolchain/format/testdata/basics/simple.carbon +++ b/toolchain/format/testdata/basics/simple.carbon @@ -10,8 +10,10 @@ // --- test.carbon -fn F(x: i32) -> i32 { return x; } +fn F ( x : i32 ) -> i32 { return x ; } // --- AUTOUPDATE-SPLIT -// CHECK:STDOUT: fn F ( x : i32 ) -> i32 { return x ; } +// CHECK:STDOUT: fn F (x: i32) -> i32 { +// CHECK:STDOUT: return x; +// CHECK:STDOUT: }