Update Carbon::Format to produce semi-reasonable output. (#7687)

Makes following changes to Carbon::Format()

- TokenKind::Period (i.e. `.`) should never have a space before or after
it.
- TokenKind::CloseSquareParen (i.e. `]`) should be treated as packed
content (no space preceeding it)
  - Only exception I can think of is `impl forall [...]`
- Remove preceeding space from `[` and `(` if previous token was an
identifier (or identifier-ish token)
- Remove seperator following `++` / `--` unary operators.
- Explicit gaps in source code should be retained, up to 2 new lines.

Multiple test files were added to test formatting. 

I imagine eventually this will need to be updated to read parse tree to
gather more context but this atleast lets us get a decent-ish format for
many of our current sample files (e.g. sieve.carbon)

Assisted-With: Gemini / Antigravity

---------

Co-authored-by: David Blaikie <dblaikie@gmail.com>

toolchain/format/BUILD

@@ -21,6 +21,7 @@ cc_library(
     hdrs = ["format.h"],
     deps = [
         "//common:ostream",
+        "//toolchain/lex:token_kind",
         "//toolchain/lex:tokenized_buffer",
     ],
 )

toolchain/format/formatter.cpp

@@ -4,6 +4,10 @@
 
 #include "toolchain/format/formatter.h"
 
+#include <algorithm>
+
+#include "toolchain/lex/token_kind.h"
+
 namespace Carbon::Format {
 
 auto Formatter::Run() -> bool {
@@ -29,6 +33,7 @@ auto Formatter::Run() -> bool {
       EmitComment();
     }
 
+    int token_start_line = tokens_->GetLine(token).index;
     switch (token_kind) {
       case Lex::TokenKind::FileStart:
         break;
@@ -38,7 +43,7 @@ auto Formatter::Run() -> bool {
         break;
 
       case Lex::TokenKind::OpenCurlyBrace:
-        PrepareForSpacedContent();
+        PrepareForSpacedContent(token_start_line);
         *out_ << "{";
         // Check for `{}`.
         if (NextToken(token) != tokens_->GetMatchedClosingToken(token)) {
@@ -49,24 +54,55 @@ auto Formatter::Run() -> bool {
 
       case Lex::TokenKind::CloseCurlyBrace:
         indent_ -= 2;
-        PrepareForPackedContent();
+        PrepareForPackedContent(token_start_line);
         *out_ << "}";
         RequireEmptyLine();
         break;
 
+      case Lex::TokenKind::Else:
+        // `else` token should be placed on same line as `}`
+        if (line_state_ == LineState::EndOfLine) {
+          line_state_ = LineState::NeedsSeparator;
+        }
+        PrepareForSpacedContent(token_start_line);
+        *out_ << "else";
+        line_state_ = LineState::NeedsSeparator;
+        break;
+
+      case Lex::TokenKind::Period:
+        PrepareForPackedContent(token_start_line);
+        *out_ << ".";
+        line_state_ = LineState::HasSeparator;
+        break;
+
+      case Lex::TokenKind::PlusPlus:
+      case Lex::TokenKind::MinusMinus:
+        PrepareForSpacedContent(token_start_line);
+        *out_ << tokens_->GetTokenText(token);
+        line_state_ = LineState::HasSeparator;
+        break;
+
       case Lex::TokenKind::Semi:
-        PrepareForPackedContent();
+        PrepareForPackedContent(token_start_line);
         *out_ << ";";
         RequireEmptyLine();
         break;
 
       default:
-        if (token_kind.IsOneOf({Lex::TokenKind::CloseParen,
-                                Lex::TokenKind::Colon,
-                                Lex::TokenKind::Comma})) {
-          PrepareForPackedContent();
+        if (token_kind.IsOneOf(
+                {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseSquareBracket,
+                 Lex::TokenKind::Colon, Lex::TokenKind::Comma})) {
+          PrepareForPackedContent(token_start_line);
+        } else if (token_kind.IsOneOf({Lex::TokenKind::OpenParen,
+                                       Lex::TokenKind::OpenSquareBracket}) &&
+                   (prev_token_kind_.IsOneOf(
+                        {Lex::TokenKind::Identifier, Lex::TokenKind::Array,
+                         Lex::TokenKind::CloseParen,
+                         Lex::TokenKind::CloseSquareBracket}) ||
+                    prev_token_kind_.is_sized_type_literal())) {
+          PrepareForPackedContent(token_start_line);
         } else {
-          PrepareForSpacedContent();
+          PrepareForSpacedContent(token_start_line);
         }
         *out_ << tokens_->GetTokenText(token);
         line_state_ = token_kind.is_opening_symbol()
@@ -74,6 +110,8 @@ auto Formatter::Run() -> bool {
                           : LineState::NeedsSeparator;
         break;
     }
+    prev_token_kind_ = token_kind;
+    prev_end_line_ = tokens_->GetEndLoc(token).first.index;
   }
 
   // Materialize any newline deferred by the final line.
@@ -93,23 +131,45 @@ auto Formatter::EmitComment() -> void {
     // line still has content because its newline was deferred (`EndOfLine`) or
     // not yet required.
     *out_ << " " << tokens_->GetCommentText(comment);
+    prev_end_line_ = tokens_->GetLine(comment).index;
   } else {
     // A full-line comment (or a trailing comment with nothing left to attach
     // to) is emitted on its own line.
-    RequireEmptyLine();
-    PrepareForSpacedContent();
-    // TODO: We do need to adjust the indent of multi-line comments.
-    *out_ << tokens_->GetCommentText(comment);
+    int comment_start_line = tokens_->GetLine(comment).index;
+    if (line_state_ != LineState::Empty) {
+      EmitNewLine(comment_start_line);
+    }
+
+    int line_count = 0;
+    // Split comment lines so we can re-apply indent.
+    for (auto line :
+         llvm::split(tokens_->GetCommentText(comment).rtrim(), '\n')) {
+      out_->indent(indent_) << line.trim() << '\n';
+      line_count++;
+    }
+    prev_end_line_ = comment_start_line + line_count - 1;
   }
   // Comment text includes a terminating newline, so just update the state.
   line_state_ = LineState::Empty;
 }
 
-auto Formatter::PrepareForPackedContent() -> void {
+auto Formatter::EmitNewLine(int start_line) -> void {
+  *out_ << "\n";
+
+  // If source code chose to have an empty line
+  int source_code_gap = start_line - prev_end_line_;
+  if (source_code_gap > 1 &&
+      prev_token_kind_.IsOneOf(
+          {Lex::TokenKind::Semi, Lex::TokenKind::CloseCurlyBrace})) {
+    *out_ << "\n";
+  }
+  line_state_ = LineState::Empty;
+}
+
+auto Formatter::PrepareForPackedContent(int start_line) -> void {
   // Materialize a deferred newline before starting to fill a fresh line.
   if (line_state_ == LineState::EndOfLine) {
-    *out_ << "\n";
-    line_state_ = LineState::Empty;
+    EmitNewLine(start_line);
   }
   if (line_state_ == LineState::Empty) {
     out_->indent(indent_);
@@ -125,12 +185,12 @@ auto Formatter::RequireEmptyLine() -> void {
   }
 }
 
-auto Formatter::PrepareForSpacedContent() -> void {
+auto Formatter::PrepareForSpacedContent(int start_line) -> void {
   if (line_state_ == LineState::NeedsSeparator) {
     *out_ << " ";
     line_state_ = LineState::HasSeparator;
   } else {
-    PrepareForPackedContent();
+    PrepareForPackedContent(start_line);
   }
 }
 

toolchain/format/formatter.h

@@ -65,17 +65,21 @@ class Formatter {
   // line.
   auto EmitComment() -> void;
 
+  // Emits a new line before next token. If the source code contained multiple
+  // newlines, will emit up to 2 new lines.
+  auto EmitNewLine(int start_line) -> void;
+
   // Ensures there is a separator before adding new content. May do
   // `PrepareForPackedContent` or output a separator space, dependent on line
   // state. Always results in line_state_ being HasSeparator; the caller is
   // responsible for adjusting state if needed.
-  auto PrepareForSpacedContent() -> void;
+  auto PrepareForSpacedContent(int start_line = 0) -> void;
 
   // Requires that the current line is indented, but not necessarily a separator
   // space. May output spaces for `indent_`, dependent on line state. Only
   // guarantees the line_state_ is not Empty; the caller is responsible for
   // adjusting state if needed.
-  auto PrepareForPackedContent() -> void;
+  auto PrepareForPackedContent(int start_line = 0) -> void;
 
   // Returns the next token index.
   static auto NextToken(Lex::TokenIndex token) -> Lex::TokenIndex {
@@ -97,6 +101,12 @@ class Formatter {
 
   // The current code indent level, to be added to new lines.
   int indent_ = 0;
+
+  // The 0-based end line in original source of the previous token or comment.
+  int prev_end_line_ = 0;
+
+  // Kind of the last token before current one.
+  Lex::TokenKind prev_token_kind_ = Lex::TokenKind::FileStart;
 };
 
 }  // namespace Carbon::Format

toolchain/format/testdata/basics/braces.carbon

@@ -36,14 +36,19 @@ class C {
 
 // --- AUTOUPDATE-SPLIT
 
-// CHECK:STDOUT: fn F () {}
-// CHECK:STDOUT: fn G () -> i32 {
+// CHECK:STDOUT: fn F() {}
+// CHECK:STDOUT:
+// CHECK:STDOUT: fn G() -> i32 {
 // CHECK:STDOUT:   return 3;
+// CHECK:STDOUT:
 // CHECK:STDOUT: }
-// CHECK:STDOUT: fn H (x: i32, y: i32) -> i32 {
+// CHECK:STDOUT:
+// CHECK:STDOUT: fn H(x: i32, y: i32) -> i32 {
 // CHECK:STDOUT:   var z: i32 = x + y;
 // CHECK:STDOUT:   return z;
+// CHECK:STDOUT:
 // CHECK:STDOUT: }
+// CHECK:STDOUT:
 // CHECK:STDOUT: class C {
 // CHECK:STDOUT:   class D {
 // CHECK:STDOUT:     class E {}

toolchain/format/testdata/basics/comments.carbon

@@ -53,23 +53,25 @@ var poem: String = ''' // No file type indicator, just a comment.
 // --- AUTOUPDATE-SPLIT
 
 // CHECK:STDOUT: // A comment
-// CHECK:STDOUT: fn F () {}
+// CHECK:STDOUT: fn F() {}
+// CHECK:STDOUT:
 // CHECK:STDOUT: // Another comment
 // CHECK:STDOUT: // Block
-// CHECK:STDOUT:   // comment
+// CHECK:STDOUT: // comment
 // CHECK:STDOUT: class C {
 // CHECK:STDOUT:   // Internal comment
 // CHECK:STDOUT: }
+// CHECK:STDOUT:
 // CHECK:STDOUT: // Another
-// CHECK:STDOUT:   // Block
-// CHECK:STDOUT:   //
-// CHECK:STDOUT:   // Comment
+// CHECK:STDOUT: // Block
+// CHECK:STDOUT: //
+// CHECK:STDOUT: // Comment
 // CHECK:STDOUT: // A trailing comment may follow a variable, a function call, or a closing
 // CHECK:STDOUT: // brace, and the formatter keeps it on the line it annotates.
 // CHECK:STDOUT: var count: i32 = 0; // a) A local variable,
-// CHECK:STDOUT: fn Render (frame: Frame) {
-// CHECK:STDOUT:   Draw (frame); // b) A function call,
-// CHECK:STDOUT:   Flush ();
+// CHECK:STDOUT: fn Render(frame: Frame) {
+// CHECK:STDOUT:   Draw(frame); // b) A function call,
+// CHECK:STDOUT:   Flush();
 // CHECK:STDOUT: } // c) And a closing brace.
 // CHECK:STDOUT: // A trailing comment on a block string literal's introducer line is carried
 // CHECK:STDOUT: // within the literal token and is preserved, with or without a file type
@@ -78,6 +80,7 @@ var poem: String = ''' // No file type indicator, just a comment.
 // CHECK:STDOUT: var query: String = '''sql // TODO: switch to a prepared statement
 // CHECK:STDOUT:   SELECT * FROM t
 // CHECK:STDOUT:   ''';
+// CHECK:STDOUT:
 // CHECK:STDOUT: var poem: String = ''' // No file type indicator, just a comment.
 // CHECK:STDOUT:   Roses are red.
 // CHECK:STDOUT:   '''; // A trailing comment after the closing line.

toolchain/format/testdata/basics/simple.carbon

@@ -8,12 +8,128 @@
 // TIP: To dump output, run:
 // TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/format/testdata/basics/simple.carbon
 
-// --- test.carbon
+// --- basic.carbon
 
 fn  F  (  x  :  i32  )  ->  i32  {  return  x  ;  }
 
+// --- member_access.carbon
+
+fn MemberAccess() {
+  var a: auto = s . is_prime [ n ] ;
+  var b: auto = Core . Range ( 1000 ) ;
+  var c: auto = Sieve . Make ( ) ;
+  var d: auto = m [ i ] [ j ] ;
+}
+
+// --- prefix_operators.carbon
+
+fn PrefixOperators() {
+  var x: i32 = 0;
+  ++ x;
+  -- x;
+  var y: i32 = ++ x;
+}
+
+// --- calls_and_types.carbon
+
+fn CallsAndTypes ( a : i32 , b : i32 ) -> i32 {
+  var v: array ( bool , 1000 ) ;
+  if ( a < b ) {
+    for ( x : i32 in Core . Range ( 10 ) ) {
+      Print ( x ) ;
+    }
+  } else if ( a == b ) {
+    Print ( 0 ) ;
+  } else {
+    Print ( 1 ) ;
+  }
+  while ( a > 0 ) {
+    return CallsAndTypes ( a - 1 , b ) ;
+  }
+  return 0 ;
+}
+
+// --- todo_column_limit.carbon
+
+// TODO: Long declarations and statements exceeding the 80 column limit should
+// be broken across multiple lines.
+fn FunctionWithVeryLongParameterList(first_argument_name: i32, second_argument_name: i32, third_argument_name: i32) -> i32 {
+  var very_long_variable_name_with_a_long_initialization: i32 = first_argument_name + second_argument_name + third_argument_name;
+  return very_long_variable_name_with_a_long_initialization;
+}
+
+// --- todo_dereference.carbon
+
+// TODO: Dereference operator `*p` requires context of whether there is a
+// preceding expression, to distinguish from binary multiplication `a * b` and
+// pointer type expression `i32*`.
+fn Dereference(p: i32*) -> i32 {
+  var a: i32 = *p;
+  var b: i32 = a * *p;
+  return b;
+}
+
+// --- todo_pointer_member_accessor.carbon
+
+// TODO: Pointer member access `obj->property` should be packed without spaces,
+// while function return type `fn F() -> T` should have spaces around `->`.
+fn PointerMemberAccessor(p: Point*) -> i32 {
+  var x: i32 = p -> x;
+  var y: i32 = p -> y;
+  return x + y;
+}
+
 // --- AUTOUPDATE-SPLIT
 
-// CHECK:STDOUT: fn F (x: i32) -> i32 {
+// CHECK:STDOUT: fn F(x: i32) -> i32 {
 // CHECK:STDOUT:   return x;
 // CHECK:STDOUT: }
+// CHECK:STDOUT: fn MemberAccess() {
+// CHECK:STDOUT:   var a: auto = s.is_prime[n];
+// CHECK:STDOUT:   var b: auto = Core.Range(1000);
+// CHECK:STDOUT:   var c: auto = Sieve.Make();
+// CHECK:STDOUT:   var d: auto = m[i][j];
+// CHECK:STDOUT: }
+// CHECK:STDOUT: fn PrefixOperators() {
+// CHECK:STDOUT:   var x: i32 = 0;
+// CHECK:STDOUT:   ++x;
+// CHECK:STDOUT:   --x;
+// CHECK:STDOUT:   var y: i32 = ++x;
+// CHECK:STDOUT: }
+// CHECK:STDOUT: fn CallsAndTypes(a: i32, b: i32) -> i32 {
+// CHECK:STDOUT:   var v: array(bool, 1000);
+// CHECK:STDOUT:   if (a < b) {
+// CHECK:STDOUT:     for (x: i32 in Core.Range(10)) {
+// CHECK:STDOUT:       Print(x);
+// CHECK:STDOUT:     }
+// CHECK:STDOUT:   } else if (a == b) {
+// CHECK:STDOUT:     Print(0);
+// CHECK:STDOUT:   } else {
+// CHECK:STDOUT:     Print(1);
+// CHECK:STDOUT:   }
+// CHECK:STDOUT:   while (a > 0) {
+// CHECK:STDOUT:     return CallsAndTypes(a - 1, b);
+// CHECK:STDOUT:   }
+// CHECK:STDOUT:   return 0;
+// CHECK:STDOUT: }
+// CHECK:STDOUT: // TODO: Long declarations and statements exceeding the 80 column limit should
+// CHECK:STDOUT: // be broken across multiple lines.
+// CHECK:STDOUT: fn FunctionWithVeryLongParameterList(first_argument_name: i32, second_argument_name: i32, third_argument_name: i32) -> i32 {
+// CHECK:STDOUT:   var very_long_variable_name_with_a_long_initialization: i32 = first_argument_name + second_argument_name + third_argument_name;
+// CHECK:STDOUT:   return very_long_variable_name_with_a_long_initialization;
+// CHECK:STDOUT: }
+// CHECK:STDOUT: // TODO: Dereference operator `*p` requires context of whether there is a
+// CHECK:STDOUT: // preceding expression, to distinguish from binary multiplication `a * b` and
+// CHECK:STDOUT: // pointer type expression `i32*`.
+// CHECK:STDOUT: fn Dereference(p: i32 *) -> i32 {
+// CHECK:STDOUT:   var a: i32 = * p;
+// CHECK:STDOUT:   var b: i32 = a * * p;
+// CHECK:STDOUT:   return b;
+// CHECK:STDOUT: }
+// CHECK:STDOUT: // TODO: Pointer member access `obj->property` should be packed without spaces,
+// CHECK:STDOUT: // while function return type `fn F() -> T` should have spaces around `->`.
+// CHECK:STDOUT: fn PointerMemberAccessor(p: Point *) -> i32 {
+// CHECK:STDOUT:   var x: i32 = p -> x;
+// CHECK:STDOUT:   var y: i32 = p -> y;
+// CHECK:STDOUT:   return x + y;
+// CHECK:STDOUT: }

toolchain/language_server/testdata/formatting/basic.carbon

@@ -64,14 +64,14 @@
 // CHECK:STDOUT:     "diagnostics": [],
 // CHECK:STDOUT:     "uri": "file:///unformatted.carbon"
 // CHECK:STDOUT:   }
-// CHECK:STDOUT: }Content-Length: 284{{\r}}
+// CHECK:STDOUT: }Content-Length: 283{{\r}}
 // CHECK:STDOUT: {{\r}}
 // CHECK:STDOUT: {
 // CHECK:STDOUT:   "id": 2,
 // CHECK:STDOUT:   "jsonrpc": "2.0",
 // CHECK:STDOUT:   "result": [
 // CHECK:STDOUT:     {
-// CHECK:STDOUT:       "newText": "fn F () {\n  return;\n}\n",
+// CHECK:STDOUT:       "newText": "fn F() {\n  return;\n}\n",
 // CHECK:STDOUT:       "range": {
 // CHECK:STDOUT:         "end": {
 // CHECK:STDOUT:           "character": 0,
@@ -93,26 +93,12 @@
 // CHECK:STDOUT:     "diagnostics": [],
 // CHECK:STDOUT:     "uri": "file:///formatted.carbon"
 // CHECK:STDOUT:   }
-// CHECK:STDOUT: }Content-Length: 284{{\r}}
+// CHECK:STDOUT: }Content-Length: 49{{\r}}
 // CHECK:STDOUT: {{\r}}
 // CHECK:STDOUT: {
 // CHECK:STDOUT:   "id": 3,
 // CHECK:STDOUT:   "jsonrpc": "2.0",
-// CHECK:STDOUT:   "result": [
-// CHECK:STDOUT:     {
-// CHECK:STDOUT:       "newText": "fn F () {\n  return;\n}\n",
-// CHECK:STDOUT:       "range": {
-// CHECK:STDOUT:         "end": {
-// CHECK:STDOUT:           "character": 0,
-// CHECK:STDOUT:           "line": 4
-// CHECK:STDOUT:         },
-// CHECK:STDOUT:         "start": {
-// CHECK:STDOUT:           "character": 0,
-// CHECK:STDOUT:           "line": 0
-// CHECK:STDOUT:         }
-// CHECK:STDOUT:       }
-// CHECK:STDOUT:     }
-// CHECK:STDOUT:   ]
+// CHECK:STDOUT:   "result": []
 // CHECK:STDOUT: }Content-Length: 145{{\r}}
 // CHECK:STDOUT: {{\r}}
 // CHECK:STDOUT: {

toolchain/lex/tokenized_buffer.cpp

@@ -32,6 +32,10 @@ auto TokenizedBuffer::GetLineNumber(TokenIndex token) const -> int {
   return GetLine(token).index + 1;
 }
 
+auto TokenizedBuffer::GetLine(CommentIndex comment) const -> LineIndex {
+  return FindLineIndex(comments_.Get(comment).start);
+}
+
 auto TokenizedBuffer::GetColumnNumber(TokenIndex token) const -> int {
   const auto& token_info = token_infos_.Get(token);
   const auto& line_info =

toolchain/lex/tokenized_buffer.h

@@ -209,6 +209,9 @@ class TokenizedBuffer : public Printable<TokenizedBuffer> {
   // content on its line.
   auto IsTrailingComment(CommentIndex comment_index) const -> bool;
 
+  // Returns the line the comment begins on.
+  auto GetLine(CommentIndex comment) const -> LineIndex;
+
   // Returns tokens as YAML. This prints the tracked token information on a
   // single line for each token. We use the single-line format so that output is
   // compact, and so that tools like `grep` are compatible.