Update Carbon::Format to produce semi-reasonable output. (#7687)

Makes following changes to Carbon::Format()

- TokenKind::Period (i.e. `.`) should never have a space before or after
it.
- TokenKind::CloseSquareParen (i.e. `]`) should be treated as packed
content (no space preceeding it)
  - Only exception I can think of is `impl forall [...]`
- Remove preceeding space from `[` and `(` if previous token was an
identifier (or identifier-ish token)
- Remove seperator following `++` / `--` unary operators.
- Explicit gaps in source code should be retained, up to 2 new lines.

Multiple test files were added to test formatting. 

I imagine eventually this will need to be updated to read parse tree to
gather more context but this atleast lets us get a decent-ish format for
many of our current sample files (e.g. sieve.carbon)

Assisted-With: Gemini / Antigravity

---------

Co-authored-by: David Blaikie <dblaikie@gmail.com>

toolchain/format/formatter.cpp

@@ -4,6 +4,10 @@
 
 #include "toolchain/format/formatter.h"
 
+#include <algorithm>
+
+#include "toolchain/lex/token_kind.h"
+
 namespace Carbon::Format {
 
 auto Formatter::Run() -> bool {
@@ -29,6 +33,7 @@ auto Formatter::Run() -> bool {
       EmitComment();
     }
 
+    int token_start_line = tokens_->GetLine(token).index;
     switch (token_kind) {
       case Lex::TokenKind::FileStart:
         break;
@@ -38,7 +43,7 @@ auto Formatter::Run() -> bool {
         break;
 
       case Lex::TokenKind::OpenCurlyBrace:
-        PrepareForSpacedContent();
+        PrepareForSpacedContent(token_start_line);
         *out_ << "{";
         // Check for `{}`.
         if (NextToken(token) != tokens_->GetMatchedClosingToken(token)) {
@@ -49,24 +54,55 @@ auto Formatter::Run() -> bool {
 
       case Lex::TokenKind::CloseCurlyBrace:
         indent_ -= 2;
-        PrepareForPackedContent();
+        PrepareForPackedContent(token_start_line);
         *out_ << "}";
         RequireEmptyLine();
         break;
 
+      case Lex::TokenKind::Else:
+        // `else` token should be placed on same line as `}`
+        if (line_state_ == LineState::EndOfLine) {
+          line_state_ = LineState::NeedsSeparator;
+        }
+        PrepareForSpacedContent(token_start_line);
+        *out_ << "else";
+        line_state_ = LineState::NeedsSeparator;
+        break;
+
+      case Lex::TokenKind::Period:
+        PrepareForPackedContent(token_start_line);
+        *out_ << ".";
+        line_state_ = LineState::HasSeparator;
+        break;
+
+      case Lex::TokenKind::PlusPlus:
+      case Lex::TokenKind::MinusMinus:
+        PrepareForSpacedContent(token_start_line);
+        *out_ << tokens_->GetTokenText(token);
+        line_state_ = LineState::HasSeparator;
+        break;
+
       case Lex::TokenKind::Semi:
-        PrepareForPackedContent();
+        PrepareForPackedContent(token_start_line);
         *out_ << ";";
         RequireEmptyLine();
         break;
 
       default:
-        if (token_kind.IsOneOf({Lex::TokenKind::CloseParen,
-                                Lex::TokenKind::Colon,
-                                Lex::TokenKind::Comma})) {
-          PrepareForPackedContent();
+        if (token_kind.IsOneOf(
+                {Lex::TokenKind::CloseParen, Lex::TokenKind::CloseSquareBracket,
+                 Lex::TokenKind::Colon, Lex::TokenKind::Comma})) {
+          PrepareForPackedContent(token_start_line);
+        } else if (token_kind.IsOneOf({Lex::TokenKind::OpenParen,
+                                       Lex::TokenKind::OpenSquareBracket}) &&
+                   (prev_token_kind_.IsOneOf(
+                        {Lex::TokenKind::Identifier, Lex::TokenKind::Array,
+                         Lex::TokenKind::CloseParen,
+                         Lex::TokenKind::CloseSquareBracket}) ||
+                    prev_token_kind_.is_sized_type_literal())) {
+          PrepareForPackedContent(token_start_line);
         } else {
-          PrepareForSpacedContent();
+          PrepareForSpacedContent(token_start_line);
         }
         *out_ << tokens_->GetTokenText(token);
         line_state_ = token_kind.is_opening_symbol()
@@ -74,6 +110,8 @@ auto Formatter::Run() -> bool {
                           : LineState::NeedsSeparator;
         break;
     }
+    prev_token_kind_ = token_kind;
+    prev_end_line_ = tokens_->GetEndLoc(token).first.index;
   }
 
   // Materialize any newline deferred by the final line.
@@ -93,23 +131,45 @@ auto Formatter::EmitComment() -> void {
     // line still has content because its newline was deferred (`EndOfLine`) or
     // not yet required.
     *out_ << " " << tokens_->GetCommentText(comment);
+    prev_end_line_ = tokens_->GetLine(comment).index;
   } else {
     // A full-line comment (or a trailing comment with nothing left to attach
     // to) is emitted on its own line.
-    RequireEmptyLine();
-    PrepareForSpacedContent();
-    // TODO: We do need to adjust the indent of multi-line comments.
-    *out_ << tokens_->GetCommentText(comment);
+    int comment_start_line = tokens_->GetLine(comment).index;
+    if (line_state_ != LineState::Empty) {
+      EmitNewLine(comment_start_line);
+    }
+
+    int line_count = 0;
+    // Split comment lines so we can re-apply indent.
+    for (auto line :
+         llvm::split(tokens_->GetCommentText(comment).rtrim(), '\n')) {
+      out_->indent(indent_) << line.trim() << '\n';
+      line_count++;
+    }
+    prev_end_line_ = comment_start_line + line_count - 1;
   }
   // Comment text includes a terminating newline, so just update the state.
   line_state_ = LineState::Empty;
 }
 
-auto Formatter::PrepareForPackedContent() -> void {
+auto Formatter::EmitNewLine(int start_line) -> void {
+  *out_ << "\n";
+
+  // If source code chose to have an empty line
+  int source_code_gap = start_line - prev_end_line_;
+  if (source_code_gap > 1 &&
+      prev_token_kind_.IsOneOf(
+          {Lex::TokenKind::Semi, Lex::TokenKind::CloseCurlyBrace})) {
+    *out_ << "\n";
+  }
+  line_state_ = LineState::Empty;
+}
+
+auto Formatter::PrepareForPackedContent(int start_line) -> void {
   // Materialize a deferred newline before starting to fill a fresh line.
   if (line_state_ == LineState::EndOfLine) {
-    *out_ << "\n";
-    line_state_ = LineState::Empty;
+    EmitNewLine(start_line);
   }
   if (line_state_ == LineState::Empty) {
     out_->indent(indent_);
@@ -125,12 +185,12 @@ auto Formatter::RequireEmptyLine() -> void {
   }
 }
 
-auto Formatter::PrepareForSpacedContent() -> void {
+auto Formatter::PrepareForSpacedContent(int start_line) -> void {
   if (line_state_ == LineState::NeedsSeparator) {
     *out_ << " ";
     line_state_ = LineState::HasSeparator;
   } else {
-    PrepareForPackedContent();
+    PrepareForPackedContent(start_line);
   }
 }