diff --git a/toolchain/format/BUILD b/toolchain/format/BUILD index 5f4ca1631c07..9af74da771ff 100644 --- a/toolchain/format/BUILD +++ b/toolchain/format/BUILD @@ -21,6 +21,7 @@ cc_library( hdrs = ["format.h"], deps = [ "//common:ostream", + "//toolchain/lex:token_kind", "//toolchain/lex:tokenized_buffer", ], ) diff --git a/toolchain/format/formatter.cpp b/toolchain/format/formatter.cpp index f368120bbfbd..e01aa4f7f61d 100644 --- a/toolchain/format/formatter.cpp +++ b/toolchain/format/formatter.cpp @@ -4,6 +4,10 @@ #include "toolchain/format/formatter.h" +#include + +#include "toolchain/lex/token_kind.h" + namespace Carbon::Format { auto Formatter::Run() -> bool { @@ -29,6 +33,7 @@ auto Formatter::Run() -> bool { EmitComment(); } + int token_start_line = tokens_->GetLine(token).index; switch (token_kind) { case Lex::TokenKind::FileStart: break; @@ -38,7 +43,7 @@ auto Formatter::Run() -> bool { break; case Lex::TokenKind::OpenCurlyBrace: - PrepareForSpacedContent(); + PrepareForSpacedContent(token_start_line); *out_ << "{"; // Check for `{}`. if (NextToken(token) != tokens_->GetMatchedClosingToken(token)) { @@ -49,24 +54,55 @@ auto Formatter::Run() -> bool { case Lex::TokenKind::CloseCurlyBrace: indent_ -= 2; - PrepareForPackedContent(); + PrepareForPackedContent(token_start_line); *out_ << "}"; RequireEmptyLine(); break; + case Lex::TokenKind::Else: + // `else` token should be placed on same line as `}` + if (line_state_ == LineState::EndOfLine) { + line_state_ = LineState::NeedsSeparator; + } + PrepareForSpacedContent(token_start_line); + *out_ << "else"; + line_state_ = LineState::NeedsSeparator; + break; + + case Lex::TokenKind::Period: + PrepareForPackedContent(token_start_line); + *out_ << "."; + line_state_ = LineState::HasSeparator; + break; + + case Lex::TokenKind::PlusPlus: + case Lex::TokenKind::MinusMinus: + PrepareForSpacedContent(token_start_line); + *out_ << tokens_->GetTokenText(token); + line_state_ = LineState::HasSeparator; + break; + case Lex::TokenKind::Semi: - PrepareForPackedContent(); + PrepareForPackedContent(token_start_line); *out_ << ";"; RequireEmptyLine(); break; default: - if (token_kind.IsOneOf({Lex::TokenKind::CloseParen, - Lex::TokenKind::Colon, - Lex::TokenKind::Comma})) { - PrepareForPackedContent(); + if (token_kind.IsOneOf( + {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseSquareBracket, + Lex::TokenKind::Colon, Lex::TokenKind::Comma})) { + PrepareForPackedContent(token_start_line); + } else if (token_kind.IsOneOf({Lex::TokenKind::OpenParen, + Lex::TokenKind::OpenSquareBracket}) && + (prev_token_kind_.IsOneOf( + {Lex::TokenKind::Identifier, Lex::TokenKind::Array, + Lex::TokenKind::CloseParen, + Lex::TokenKind::CloseSquareBracket}) || + prev_token_kind_.is_sized_type_literal())) { + PrepareForPackedContent(token_start_line); } else { - PrepareForSpacedContent(); + PrepareForSpacedContent(token_start_line); } *out_ << tokens_->GetTokenText(token); line_state_ = token_kind.is_opening_symbol() @@ -74,6 +110,8 @@ auto Formatter::Run() -> bool { : LineState::NeedsSeparator; break; } + prev_token_kind_ = token_kind; + prev_end_line_ = tokens_->GetEndLoc(token).first.index; } // Materialize any newline deferred by the final line. @@ -93,23 +131,45 @@ auto Formatter::EmitComment() -> void { // line still has content because its newline was deferred (`EndOfLine`) or // not yet required. *out_ << " " << tokens_->GetCommentText(comment); + prev_end_line_ = tokens_->GetLine(comment).index; } else { // A full-line comment (or a trailing comment with nothing left to attach // to) is emitted on its own line. - RequireEmptyLine(); - PrepareForSpacedContent(); - // TODO: We do need to adjust the indent of multi-line comments. - *out_ << tokens_->GetCommentText(comment); + int comment_start_line = tokens_->GetLine(comment).index; + if (line_state_ != LineState::Empty) { + EmitNewLine(comment_start_line); + } + + int line_count = 0; + // Split comment lines so we can re-apply indent. + for (auto line : + llvm::split(tokens_->GetCommentText(comment).rtrim(), '\n')) { + out_->indent(indent_) << line.trim() << '\n'; + line_count++; + } + prev_end_line_ = comment_start_line + line_count - 1; } // Comment text includes a terminating newline, so just update the state. line_state_ = LineState::Empty; } -auto Formatter::PrepareForPackedContent() -> void { +auto Formatter::EmitNewLine(int start_line) -> void { + *out_ << "\n"; + + // If source code chose to have an empty line + int source_code_gap = start_line - prev_end_line_; + if (source_code_gap > 1 && + prev_token_kind_.IsOneOf( + {Lex::TokenKind::Semi, Lex::TokenKind::CloseCurlyBrace})) { + *out_ << "\n"; + } + line_state_ = LineState::Empty; +} + +auto Formatter::PrepareForPackedContent(int start_line) -> void { // Materialize a deferred newline before starting to fill a fresh line. if (line_state_ == LineState::EndOfLine) { - *out_ << "\n"; - line_state_ = LineState::Empty; + EmitNewLine(start_line); } if (line_state_ == LineState::Empty) { out_->indent(indent_); @@ -125,12 +185,12 @@ auto Formatter::RequireEmptyLine() -> void { } } -auto Formatter::PrepareForSpacedContent() -> void { +auto Formatter::PrepareForSpacedContent(int start_line) -> void { if (line_state_ == LineState::NeedsSeparator) { *out_ << " "; line_state_ = LineState::HasSeparator; } else { - PrepareForPackedContent(); + PrepareForPackedContent(start_line); } } diff --git a/toolchain/format/formatter.h b/toolchain/format/formatter.h index 9de39feb10f6..a82d34949050 100644 --- a/toolchain/format/formatter.h +++ b/toolchain/format/formatter.h @@ -65,17 +65,21 @@ class Formatter { // line. auto EmitComment() -> void; + // Emits a new line before next token. If the source code contained multiple + // newlines, will emit up to 2 new lines. + auto EmitNewLine(int start_line) -> void; + // Ensures there is a separator before adding new content. May do // `PrepareForPackedContent` or output a separator space, dependent on line // state. Always results in line_state_ being HasSeparator; the caller is // responsible for adjusting state if needed. - auto PrepareForSpacedContent() -> void; + auto PrepareForSpacedContent(int start_line = 0) -> void; // Requires that the current line is indented, but not necessarily a separator // space. May output spaces for `indent_`, dependent on line state. Only // guarantees the line_state_ is not Empty; the caller is responsible for // adjusting state if needed. - auto PrepareForPackedContent() -> void; + auto PrepareForPackedContent(int start_line = 0) -> void; // Returns the next token index. static auto NextToken(Lex::TokenIndex token) -> Lex::TokenIndex { @@ -97,6 +101,12 @@ class Formatter { // The current code indent level, to be added to new lines. int indent_ = 0; + + // The 0-based end line in original source of the previous token or comment. + int prev_end_line_ = 0; + + // Kind of the last token before current one. + Lex::TokenKind prev_token_kind_ = Lex::TokenKind::FileStart; }; } // namespace Carbon::Format diff --git a/toolchain/format/testdata/basics/braces.carbon b/toolchain/format/testdata/basics/braces.carbon index 142bffab25fb..ff0f9ed5c413 100644 --- a/toolchain/format/testdata/basics/braces.carbon +++ b/toolchain/format/testdata/basics/braces.carbon @@ -36,14 +36,19 @@ class C { // --- AUTOUPDATE-SPLIT -// CHECK:STDOUT: fn F () {} -// CHECK:STDOUT: fn G () -> i32 { +// CHECK:STDOUT: fn F() {} +// CHECK:STDOUT: +// CHECK:STDOUT: fn G() -> i32 { // CHECK:STDOUT: return 3; +// CHECK:STDOUT: // CHECK:STDOUT: } -// CHECK:STDOUT: fn H (x: i32, y: i32) -> i32 { +// CHECK:STDOUT: +// CHECK:STDOUT: fn H(x: i32, y: i32) -> i32 { // CHECK:STDOUT: var z: i32 = x + y; // CHECK:STDOUT: return z; +// CHECK:STDOUT: // CHECK:STDOUT: } +// CHECK:STDOUT: // CHECK:STDOUT: class C { // CHECK:STDOUT: class D { // CHECK:STDOUT: class E {} diff --git a/toolchain/format/testdata/basics/comments.carbon b/toolchain/format/testdata/basics/comments.carbon index e8aabf1ba217..0e7f12dbb2e8 100644 --- a/toolchain/format/testdata/basics/comments.carbon +++ b/toolchain/format/testdata/basics/comments.carbon @@ -53,23 +53,25 @@ var poem: String = ''' // No file type indicator, just a comment. // --- AUTOUPDATE-SPLIT // CHECK:STDOUT: // A comment -// CHECK:STDOUT: fn F () {} +// CHECK:STDOUT: fn F() {} +// CHECK:STDOUT: // CHECK:STDOUT: // Another comment // CHECK:STDOUT: // Block -// CHECK:STDOUT: // comment +// CHECK:STDOUT: // comment // CHECK:STDOUT: class C { // CHECK:STDOUT: // Internal comment // CHECK:STDOUT: } +// CHECK:STDOUT: // CHECK:STDOUT: // Another -// CHECK:STDOUT: // Block -// CHECK:STDOUT: // -// CHECK:STDOUT: // Comment +// CHECK:STDOUT: // Block +// CHECK:STDOUT: // +// CHECK:STDOUT: // Comment // CHECK:STDOUT: // A trailing comment may follow a variable, a function call, or a closing // CHECK:STDOUT: // brace, and the formatter keeps it on the line it annotates. // CHECK:STDOUT: var count: i32 = 0; // a) A local variable, -// CHECK:STDOUT: fn Render (frame: Frame) { -// CHECK:STDOUT: Draw (frame); // b) A function call, -// CHECK:STDOUT: Flush (); +// CHECK:STDOUT: fn Render(frame: Frame) { +// CHECK:STDOUT: Draw(frame); // b) A function call, +// CHECK:STDOUT: Flush(); // CHECK:STDOUT: } // c) And a closing brace. // CHECK:STDOUT: // A trailing comment on a block string literal's introducer line is carried // CHECK:STDOUT: // within the literal token and is preserved, with or without a file type @@ -78,6 +80,7 @@ var poem: String = ''' // No file type indicator, just a comment. // CHECK:STDOUT: var query: String = '''sql // TODO: switch to a prepared statement // CHECK:STDOUT: SELECT * FROM t // CHECK:STDOUT: '''; +// CHECK:STDOUT: // CHECK:STDOUT: var poem: String = ''' // No file type indicator, just a comment. // CHECK:STDOUT: Roses are red. // CHECK:STDOUT: '''; // A trailing comment after the closing line. diff --git a/toolchain/format/testdata/basics/simple.carbon b/toolchain/format/testdata/basics/simple.carbon index de8093296977..2dc16b84914a 100644 --- a/toolchain/format/testdata/basics/simple.carbon +++ b/toolchain/format/testdata/basics/simple.carbon @@ -8,12 +8,128 @@ // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/format/testdata/basics/simple.carbon -// --- test.carbon +// --- basic.carbon fn F ( x : i32 ) -> i32 { return x ; } +// --- member_access.carbon + +fn MemberAccess() { + var a: auto = s . is_prime [ n ] ; + var b: auto = Core . Range ( 1000 ) ; + var c: auto = Sieve . Make ( ) ; + var d: auto = m [ i ] [ j ] ; +} + +// --- prefix_operators.carbon + +fn PrefixOperators() { + var x: i32 = 0; + ++ x; + -- x; + var y: i32 = ++ x; +} + +// --- calls_and_types.carbon + +fn CallsAndTypes ( a : i32 , b : i32 ) -> i32 { + var v: array ( bool , 1000 ) ; + if ( a < b ) { + for ( x : i32 in Core . Range ( 10 ) ) { + Print ( x ) ; + } + } else if ( a == b ) { + Print ( 0 ) ; + } else { + Print ( 1 ) ; + } + while ( a > 0 ) { + return CallsAndTypes ( a - 1 , b ) ; + } + return 0 ; +} + +// --- todo_column_limit.carbon + +// TODO: Long declarations and statements exceeding the 80 column limit should +// be broken across multiple lines. +fn FunctionWithVeryLongParameterList(first_argument_name: i32, second_argument_name: i32, third_argument_name: i32) -> i32 { + var very_long_variable_name_with_a_long_initialization: i32 = first_argument_name + second_argument_name + third_argument_name; + return very_long_variable_name_with_a_long_initialization; +} + +// --- todo_dereference.carbon + +// TODO: Dereference operator `*p` requires context of whether there is a +// preceding expression, to distinguish from binary multiplication `a * b` and +// pointer type expression `i32*`. +fn Dereference(p: i32*) -> i32 { + var a: i32 = *p; + var b: i32 = a * *p; + return b; +} + +// --- todo_pointer_member_accessor.carbon + +// TODO: Pointer member access `obj->property` should be packed without spaces, +// while function return type `fn F() -> T` should have spaces around `->`. +fn PointerMemberAccessor(p: Point*) -> i32 { + var x: i32 = p -> x; + var y: i32 = p -> y; + return x + y; +} + // --- AUTOUPDATE-SPLIT -// CHECK:STDOUT: fn F (x: i32) -> i32 { +// CHECK:STDOUT: fn F(x: i32) -> i32 { // CHECK:STDOUT: return x; // CHECK:STDOUT: } +// CHECK:STDOUT: fn MemberAccess() { +// CHECK:STDOUT: var a: auto = s.is_prime[n]; +// CHECK:STDOUT: var b: auto = Core.Range(1000); +// CHECK:STDOUT: var c: auto = Sieve.Make(); +// CHECK:STDOUT: var d: auto = m[i][j]; +// CHECK:STDOUT: } +// CHECK:STDOUT: fn PrefixOperators() { +// CHECK:STDOUT: var x: i32 = 0; +// CHECK:STDOUT: ++x; +// CHECK:STDOUT: --x; +// CHECK:STDOUT: var y: i32 = ++x; +// CHECK:STDOUT: } +// CHECK:STDOUT: fn CallsAndTypes(a: i32, b: i32) -> i32 { +// CHECK:STDOUT: var v: array(bool, 1000); +// CHECK:STDOUT: if (a < b) { +// CHECK:STDOUT: for (x: i32 in Core.Range(10)) { +// CHECK:STDOUT: Print(x); +// CHECK:STDOUT: } +// CHECK:STDOUT: } else if (a == b) { +// CHECK:STDOUT: Print(0); +// CHECK:STDOUT: } else { +// CHECK:STDOUT: Print(1); +// CHECK:STDOUT: } +// CHECK:STDOUT: while (a > 0) { +// CHECK:STDOUT: return CallsAndTypes(a - 1, b); +// CHECK:STDOUT: } +// CHECK:STDOUT: return 0; +// CHECK:STDOUT: } +// CHECK:STDOUT: // TODO: Long declarations and statements exceeding the 80 column limit should +// CHECK:STDOUT: // be broken across multiple lines. +// CHECK:STDOUT: fn FunctionWithVeryLongParameterList(first_argument_name: i32, second_argument_name: i32, third_argument_name: i32) -> i32 { +// CHECK:STDOUT: var very_long_variable_name_with_a_long_initialization: i32 = first_argument_name + second_argument_name + third_argument_name; +// CHECK:STDOUT: return very_long_variable_name_with_a_long_initialization; +// CHECK:STDOUT: } +// CHECK:STDOUT: // TODO: Dereference operator `*p` requires context of whether there is a +// CHECK:STDOUT: // preceding expression, to distinguish from binary multiplication `a * b` and +// CHECK:STDOUT: // pointer type expression `i32*`. +// CHECK:STDOUT: fn Dereference(p: i32 *) -> i32 { +// CHECK:STDOUT: var a: i32 = * p; +// CHECK:STDOUT: var b: i32 = a * * p; +// CHECK:STDOUT: return b; +// CHECK:STDOUT: } +// CHECK:STDOUT: // TODO: Pointer member access `obj->property` should be packed without spaces, +// CHECK:STDOUT: // while function return type `fn F() -> T` should have spaces around `->`. +// CHECK:STDOUT: fn PointerMemberAccessor(p: Point *) -> i32 { +// CHECK:STDOUT: var x: i32 = p -> x; +// CHECK:STDOUT: var y: i32 = p -> y; +// CHECK:STDOUT: return x + y; +// CHECK:STDOUT: } diff --git a/toolchain/language_server/testdata/formatting/basic.carbon b/toolchain/language_server/testdata/formatting/basic.carbon index 11ee85eeb27d..02c13e92f272 100644 --- a/toolchain/language_server/testdata/formatting/basic.carbon +++ b/toolchain/language_server/testdata/formatting/basic.carbon @@ -64,14 +64,14 @@ // CHECK:STDOUT: "diagnostics": [], // CHECK:STDOUT: "uri": "file:///unformatted.carbon" // CHECK:STDOUT: } -// CHECK:STDOUT: }Content-Length: 284{{\r}} +// CHECK:STDOUT: }Content-Length: 283{{\r}} // CHECK:STDOUT: {{\r}} // CHECK:STDOUT: { // CHECK:STDOUT: "id": 2, // CHECK:STDOUT: "jsonrpc": "2.0", // CHECK:STDOUT: "result": [ // CHECK:STDOUT: { -// CHECK:STDOUT: "newText": "fn F () {\n return;\n}\n", +// CHECK:STDOUT: "newText": "fn F() {\n return;\n}\n", // CHECK:STDOUT: "range": { // CHECK:STDOUT: "end": { // CHECK:STDOUT: "character": 0, @@ -93,26 +93,12 @@ // CHECK:STDOUT: "diagnostics": [], // CHECK:STDOUT: "uri": "file:///formatted.carbon" // CHECK:STDOUT: } -// CHECK:STDOUT: }Content-Length: 284{{\r}} +// CHECK:STDOUT: }Content-Length: 49{{\r}} // CHECK:STDOUT: {{\r}} // CHECK:STDOUT: { // CHECK:STDOUT: "id": 3, // CHECK:STDOUT: "jsonrpc": "2.0", -// CHECK:STDOUT: "result": [ -// CHECK:STDOUT: { -// CHECK:STDOUT: "newText": "fn F () {\n return;\n}\n", -// CHECK:STDOUT: "range": { -// CHECK:STDOUT: "end": { -// CHECK:STDOUT: "character": 0, -// CHECK:STDOUT: "line": 4 -// CHECK:STDOUT: }, -// CHECK:STDOUT: "start": { -// CHECK:STDOUT: "character": 0, -// CHECK:STDOUT: "line": 0 -// CHECK:STDOUT: } -// CHECK:STDOUT: } -// CHECK:STDOUT: } -// CHECK:STDOUT: ] +// CHECK:STDOUT: "result": [] // CHECK:STDOUT: }Content-Length: 145{{\r}} // CHECK:STDOUT: {{\r}} // CHECK:STDOUT: { diff --git a/toolchain/lex/tokenized_buffer.cpp b/toolchain/lex/tokenized_buffer.cpp index d3b283f9ecb6..24570843a2ca 100644 --- a/toolchain/lex/tokenized_buffer.cpp +++ b/toolchain/lex/tokenized_buffer.cpp @@ -32,6 +32,10 @@ auto TokenizedBuffer::GetLineNumber(TokenIndex token) const -> int { return GetLine(token).index + 1; } +auto TokenizedBuffer::GetLine(CommentIndex comment) const -> LineIndex { + return FindLineIndex(comments_.Get(comment).start); +} + auto TokenizedBuffer::GetColumnNumber(TokenIndex token) const -> int { const auto& token_info = token_infos_.Get(token); const auto& line_info = diff --git a/toolchain/lex/tokenized_buffer.h b/toolchain/lex/tokenized_buffer.h index fe02d6bd1ab5..9e278e6a06e0 100644 --- a/toolchain/lex/tokenized_buffer.h +++ b/toolchain/lex/tokenized_buffer.h @@ -209,6 +209,9 @@ class TokenizedBuffer : public Printable { // content on its line. auto IsTrailingComment(CommentIndex comment_index) const -> bool; + // Returns the line the comment begins on. + auto GetLine(CommentIndex comment) const -> LineIndex; + // Returns tokens as YAML. This prints the tracked token information on a // single line for each token. We use the single-line format so that output is // compact, and so that tools like `grep` are compatible.