diff --git a/.vscode/lldb_launch.json b/.vscode/lldb_launch.json
index f5001d7e41f9..821350165ba3 100644
--- a/.vscode/lldb_launch.json
+++ b/.vscode/lldb_launch.json
@@ -12,6 +12,7 @@
         "command script import external/+llvm_project+llvm-project/llvm/utils/lldbDataFormatters.py",
         "settings append target.source-map \".\" \"${workspaceFolder}\"",
         "settings append target.source-map \"/proc/self/cwd\" \"${workspaceFolder}\"",
+        "settings set escape-non-printables false",
         "env TEST_TARGET=//toolchain/testing:file_test",
         "env TEST_TMPDIR=/tmp"
       ]
@@ -32,7 +33,8 @@
       "initCommands": [
         "command script import external/+llvm_project+llvm-project/llvm/utils/lldbDataFormatters.py",
         "settings append target.source-map \".\" \"${workspaceFolder}\"",
-        "settings append target.source-map \"/proc/self/cwd\" \"${workspaceFolder}\""
+        "settings append target.source-map \"/proc/self/cwd\" \"${workspaceFolder}\"",
+        "settings set escape-non-printables false"
       ]
     }
   ]
diff --git a/common/ostream.h b/common/ostream.h
index 8897f017fca1..56c06a74652f 100644
--- a/common/ostream.h
+++ b/common/ostream.h
@@ -22,9 +22,11 @@ namespace Carbon {
 template <typename DerivedT>
 class Printable {
   // Provides simple printing for debuggers.
-  LLVM_DUMP_METHOD void Dump() const {
-    static_cast<const DerivedT*>(this)->Print(llvm::errs());
-    llvm::errs() << '\n';
+  LLVM_DUMP_METHOD auto Dump() const -> std::string {
+    std::string buffer;
+    llvm::raw_string_ostream stream(buffer);
+    static_cast<const DerivedT*>(this)->Print(stream);
+    return buffer;
   }
 
   // Supports printing to llvm::raw_ostream.
diff --git a/toolchain/check/BUILD b/toolchain/check/BUILD
index ca72ba77c7a1..e9aba18ad6bc 100644
--- a/toolchain/check/BUILD
+++ b/toolchain/check/BUILD
@@ -130,6 +130,7 @@ cc_library(
         ":context",
         "//common:check",
         "//common:ostream",
+        "//common:raw_string_ostream",
         "//toolchain/lex:dump",
         "//toolchain/lex:tokenized_buffer",
         "//toolchain/parse:dump",
diff --git a/toolchain/check/dump.cpp b/toolchain/check/dump.cpp
index 3abeea2c0381..1f83f1bfa19b 100644
--- a/toolchain/check/dump.cpp
+++ b/toolchain/check/dump.cpp
@@ -10,16 +10,13 @@
 //
 // - lldb: `expr Dump(context, id)`
 // - gdb: `call Dump(context, id)`
-//
-// The `DumpNoNewline` functions are helpers that exclude a trailing newline.
-// They're intended to be composed by `Dump` function implementations.
 
 #ifndef NDEBUG
 
 #include "toolchain/lex/dump.h"
 
 #include "common/check.h"
-#include "common/ostream.h"
+#include "common/raw_string_ostream.h"
 #include "toolchain/check/context.h"
 #include "toolchain/lex/tokenized_buffer.h"
 #include "toolchain/parse/dump.h"
@@ -29,10 +26,91 @@
 
 namespace Carbon::Check {
 
-static auto DumpNoNewline(const Context& context, SemIR::LocId loc_id) -> void {
+static auto Dump(const Context& context, SemIR::LocId loc_id) -> std::string;
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context, Lex::TokenIndex token)
+    -> std::string {
+  return Parse::Dump(context.parse_tree(), token);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context, Parse::NodeId node_id)
+    -> std::string {
+  return Parse::Dump(context.parse_tree(), node_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::ClassId class_id) -> std::string {
+  return SemIR::Dump(context.sem_ir(), class_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::ConstantId const_id) -> std::string {
+  return SemIR::Dump(context.sem_ir(), const_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::EntityNameId entity_name_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), entity_name_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::FacetTypeId facet_type_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), facet_type_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::FunctionId function_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), function_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::GenericId generic_id) -> std::string {
+  return SemIR::Dump(context.sem_ir(), generic_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::ImplId impl_id)
+    -> std::string {
+  RawStringOstream out;
+  out << SemIR::Dump(context.sem_ir(), impl_id);
+  if (!impl_id.has_value()) {
+    return out.TakeStr();
+  }
+  const auto& impl = context.sem_ir().impls().Get(impl_id);
+  auto loc_id = context.sem_ir().insts().GetLocId(impl.witness_id);
+  out << "\nwitness loc: " << Dump(context, loc_id);
+  return out.TakeStr();
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::InstBlockId inst_block_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), inst_block_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::InstId inst_id)
+    -> std::string {
+  RawStringOstream out;
+  auto loc_id = context.sem_ir().insts().GetLocId(inst_id);
+  out << SemIR::Dump(context.sem_ir(), inst_id) << '\n'
+      << "  - " << Dump(context, loc_id);
+  return out.TakeStr();
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context,
+                                  SemIR::InterfaceId interface_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), interface_id);
+}
+
+LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::LocId loc_id)
+    -> std::string {
+  RawStringOstream out;
   if (!loc_id.has_value()) {
-    llvm::errs() << "LocId(<none>)";
-    return;
+    out << "LocId(<none>)";
+    return out.TakeStr();
   }
 
   if (loc_id.is_node_id()) {
@@ -40,8 +118,8 @@ static auto DumpNoNewline(const Context& context, SemIR::LocId loc_id) -> void {
     auto line = context.tokens().GetLineNumber(token);
     auto col = context.tokens().GetColumnNumber(token);
     const char* implicit = loc_id.is_implicit() ? " implicit" : "";
-    llvm::errs() << "LocId(" << FormatEscaped(context.sem_ir().filename())
-                 << ":" << line << ":" << col << implicit << ")";
+    out << "LocId(" << FormatEscaped(context.sem_ir().filename()) << ":" << line
+        << ":" << col << implicit << ")";
   } else {
     CARBON_CHECK(loc_id.is_import_ir_inst_id());
 
@@ -51,129 +129,56 @@ static auto DumpNoNewline(const Context& context, SemIR::LocId loc_id) -> void {
                             .ir_id;
     const auto* import_file =
         context.sem_ir().import_irs().Get(import_ir_id).sem_ir;
-    llvm::errs() << "LocId(import from \""
-                 << FormatEscaped(import_file->filename()) << "\")";
+    out << "LocId(import from \"" << FormatEscaped(import_file->filename())
+        << "\")";
   }
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context, Lex::TokenIndex token)
-    -> void {
-  Parse::Dump(context.parse_tree(), token);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context, Parse::NodeId node_id)
-    -> void {
-  Parse::Dump(context.parse_tree(), node_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::ClassId class_id) -> void {
-  SemIR::Dump(context.sem_ir(), class_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::ConstantId const_id) -> void {
-  SemIR::Dump(context.sem_ir(), const_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::EntityNameId entity_name_id) -> void {
-  SemIR::Dump(context.sem_ir(), entity_name_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::FacetTypeId facet_type_id) -> void {
-  SemIR::Dump(context.sem_ir(), facet_type_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::FunctionId function_id) -> void {
-  SemIR::Dump(context.sem_ir(), function_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::GenericId generic_id) -> void {
-  SemIR::Dump(context.sem_ir(), generic_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::ImplId impl_id)
-    -> void {
-  SemIR::Dump(context.sem_ir(), impl_id);
-  if (impl_id.has_value()) {
-    const auto& impl = context.sem_ir().impls().Get(impl_id);
-    auto loc_id = context.sem_ir().insts().GetLocId(impl.witness_id);
-    llvm::errs() << "witness loc: ";
-    DumpNoNewline(context, loc_id);
-    llvm::errs() << '\n';
-  }
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::InstBlockId inst_block_id) -> void {
-  SemIR::Dump(context.sem_ir(), inst_block_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::InstId inst_id)
-    -> void {
-  SemIR::Dump(context.sem_ir(), inst_id);
-  auto loc_id = context.sem_ir().insts().GetLocId(inst_id);
-  llvm::errs() << "  - ";
-  DumpNoNewline(context, loc_id);
-  llvm::errs() << '\n';
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::InterfaceId interface_id) -> void {
-  SemIR::Dump(context.sem_ir(), interface_id);
-}
-
-LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::LocId loc_id)
-    -> void {
-  DumpNoNewline(context, loc_id);
-  llvm::errs() << '\n';
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::NameId name_id)
-    -> void {
-  SemIR::Dump(context.sem_ir(), name_id);
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), name_id);
 }
 
 LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::NameScopeId name_scope_id) -> void {
-  SemIR::Dump(context.sem_ir(), name_scope_id);
+                                  SemIR::NameScopeId name_scope_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), name_scope_id);
 }
 
 LLVM_DUMP_METHOD static auto Dump(
     const Context& context, SemIR::CompleteFacetTypeId complete_facet_type_id)
-    -> void {
-  SemIR::Dump(context.sem_ir(), complete_facet_type_id);
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), complete_facet_type_id);
 }
 
 LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::SpecificId specific_id) -> void {
-  SemIR::Dump(context.sem_ir(), specific_id);
+                                  SemIR::SpecificId specific_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), specific_id);
 }
 
 LLVM_DUMP_METHOD static auto Dump(
     const Context& context, SemIR::SpecificInterfaceId specific_interface_id)
-    -> void {
-  SemIR::Dump(context.sem_ir(), specific_interface_id);
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), specific_interface_id);
 }
 
 LLVM_DUMP_METHOD static auto Dump(
     const Context& context, SemIR::StructTypeFieldsId struct_type_fields_id)
-    -> void {
-  SemIR::Dump(context.sem_ir(), struct_type_fields_id);
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), struct_type_fields_id);
 }
 
 LLVM_DUMP_METHOD static auto Dump(const Context& context,
-                                  SemIR::TypeBlockId type_block_id) -> void {
-  SemIR::Dump(context.sem_ir(), type_block_id);
+                                  SemIR::TypeBlockId type_block_id)
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), type_block_id);
 }
 
 LLVM_DUMP_METHOD static auto Dump(const Context& context, SemIR::TypeId type_id)
-    -> void {
-  SemIR::Dump(context.sem_ir(), type_id);
+    -> std::string {
+  return SemIR::Dump(context.sem_ir(), type_id);
 }
 
 }  // namespace Carbon::Check
diff --git a/toolchain/lex/BUILD b/toolchain/lex/BUILD
index 5b2f57da508a..3c8616cd8ee7 100644
--- a/toolchain/lex/BUILD
+++ b/toolchain/lex/BUILD
@@ -207,6 +207,7 @@ cc_library(
     deps = [
         ":tokenized_buffer",
         "//common:ostream",
+        "//common:raw_string_ostream",
     ],
     # Always link dump methods.
     alwayslink = 1,
diff --git a/toolchain/lex/dump.cpp b/toolchain/lex/dump.cpp
index 53899a85127d..a38eced70458 100644
--- a/toolchain/lex/dump.cpp
+++ b/toolchain/lex/dump.cpp
@@ -6,29 +6,26 @@
 
 #include "toolchain/lex/dump.h"
 
-#include "common/ostream.h"
+#include "common/raw_string_ostream.h"
 
 namespace Carbon::Lex {
 
-auto DumpNoNewline(const TokenizedBuffer& tokens, TokenIndex token) -> void {
+LLVM_DUMP_METHOD auto Dump(const TokenizedBuffer& tokens, TokenIndex token)
+    -> std::string {
+  RawStringOstream out;
   if (!token.has_value()) {
-    llvm::errs() << "TokenIndex(<none>)";
-    return;
+    out << "TokenIndex(<none>)";
+    return out.TakeStr();
   }
 
   auto kind = tokens.GetKind(token);
   auto line = tokens.GetLineNumber(token);
   auto col = tokens.GetColumnNumber(token);
 
-  llvm::errs() << "TokenIndex(kind: " << kind << ", loc: ";
-  llvm::errs().write_escaped(tokens.source().filename());
-  llvm::errs() << ":" << line << ":" << col << ")";
-}
-
-LLVM_DUMP_METHOD auto Dump(const TokenizedBuffer& tokens, TokenIndex token)
-    -> void {
-  DumpNoNewline(tokens, token);
-  llvm::errs() << '\n';
+  out << "TokenIndex(kind: " << kind
+      << ", loc: " << FormatEscaped(tokens.source().filename()) << ":" << line
+      << ":" << col << ")";
+  return out.TakeStr();
 }
 
 }  // namespace Carbon::Lex
diff --git a/toolchain/lex/dump.h b/toolchain/lex/dump.h
index f38f5598b271..70827d813745 100644
--- a/toolchain/lex/dump.h
+++ b/toolchain/lex/dump.h
@@ -23,9 +23,7 @@
 
 namespace Carbon::Lex {
 
-auto DumpNoNewline(const TokenizedBuffer& tokens, TokenIndex token) -> void;
-
-auto Dump(const TokenizedBuffer& tokens, TokenIndex token) -> void;
+auto Dump(const TokenizedBuffer& tokens, TokenIndex token) -> std::string;
 
 }  // namespace Carbon::Lex
 
diff --git a/toolchain/parse/BUILD b/toolchain/parse/BUILD
index 3b10e24fc7c2..331aba3735de 100644
--- a/toolchain/parse/BUILD
+++ b/toolchain/parse/BUILD
@@ -123,6 +123,7 @@ cc_library(
     deps = [
         ":tree",
         "//common:ostream",
+        "//common:raw_string_ostream",
         "//toolchain/lex:dump",
     ],
     # Always link dump methods.
diff --git a/toolchain/parse/dump.cpp b/toolchain/parse/dump.cpp
index de03db9484f3..ef3b71e3293c 100644
--- a/toolchain/parse/dump.cpp
+++ b/toolchain/parse/dump.cpp
@@ -6,32 +6,29 @@
 
 #include "toolchain/parse/dump.h"
 
-#include "common/ostream.h"
+#include "common/raw_string_ostream.h"
 #include "toolchain/lex/dump.h"
 
 namespace Carbon::Parse {
 
-auto DumpNoNewline(const Tree& tree, NodeId node_id) -> void {
+LLVM_DUMP_METHOD auto Dump(const Tree& tree, Lex::TokenIndex token)
+    -> std::string {
+  return Lex::Dump(tree.tokens(), token);
+}
+
+LLVM_DUMP_METHOD auto Dump(const Tree& tree, NodeId node_id) -> std::string {
+  RawStringOstream out;
   if (!node_id.has_value()) {
-    llvm::errs() << "NodeId(invalid)";
-    return;
+    out << "NodeId(<none>)";
+    return out.TakeStr();
   }
 
   auto kind = tree.node_kind(node_id);
   auto token = tree.node_token(node_id);
 
-  llvm::errs() << "NodeId(kind: " << kind << ", token: ";
-  Lex::DumpNoNewline(tree.tokens(), token);
-  llvm::errs() << ")";
-}
-
-LLVM_DUMP_METHOD auto Dump(const Tree& tree, Lex::TokenIndex token) -> void {
-  Lex::Dump(tree.tokens(), token);
-}
-
-LLVM_DUMP_METHOD auto Dump(const Tree& tree, NodeId node_id) -> void {
-  DumpNoNewline(tree, node_id);
-  llvm::errs() << '\n';
+  out << "NodeId(kind: " << kind
+      << ", token: " << Lex::Dump(tree.tokens(), token) << ")";
+  return out.TakeStr();
 }
 
 }  // namespace Carbon::Parse
diff --git a/toolchain/parse/dump.h b/toolchain/parse/dump.h
index 4c3d90377c3b..33c8baeaaf0b 100644
--- a/toolchain/parse/dump.h
+++ b/toolchain/parse/dump.h
@@ -23,10 +23,8 @@
 
 namespace Carbon::Parse {
 
-auto DumpNoNewline(const Tree& tree, NodeId node_id) -> void;
-
-auto Dump(const Tree& tree, Lex::TokenIndex token) -> void;
-auto Dump(const Tree& tree, NodeId node_id) -> void;
+auto Dump(const Tree& tree, Lex::TokenIndex token) -> std::string;
+auto Dump(const Tree& tree, NodeId node_id) -> std::string;
 
 }  // namespace Carbon::Parse
 
diff --git a/toolchain/sem_ir/BUILD b/toolchain/sem_ir/BUILD
index b38f972d9ddb..9f7488111c8a 100644
--- a/toolchain/sem_ir/BUILD
+++ b/toolchain/sem_ir/BUILD
@@ -220,6 +220,7 @@ cc_library(
         ":file",
         ":stringify_type",
         "//common:ostream",
+        "//common:raw_string_ostream",
     ],
     # Always link dump methods so they are callable from a debugger
     # even though they are never called.
diff --git a/toolchain/sem_ir/dump.cpp b/toolchain/sem_ir/dump.cpp
index 5cc8eb3690cc..714728209a4c 100644
--- a/toolchain/sem_ir/dump.cpp
+++ b/toolchain/sem_ir/dump.cpp
@@ -6,324 +6,330 @@
 
 #include "toolchain/sem_ir/dump.h"
 
-#include "common/ostream.h"
+#include "common/raw_string_ostream.h"
 #include "toolchain/sem_ir/stringify_type.h"
 
 namespace Carbon::SemIR {
 
-static auto DumpNameIfValid(const File& file, NameId name_id) -> void {
+static auto DumpNameIfValid(const File& file, NameId name_id) -> std::string {
+  RawStringOstream out;
   if (name_id.has_value()) {
-    llvm::errs() << " `" << file.names().GetFormatted(name_id) << "`";
+    out << " `" << file.names().GetFormatted(name_id) << "`";
   }
+  return out.TakeStr();
 }
 
-static auto DumpNoNewline(const File& file, ConstantId const_id) -> void {
-  llvm::errs() << const_id;
-  if (!const_id.has_value()) {
-    return;
-  }
-  if (const_id.is_symbolic()) {
-    llvm::errs() << ": "
-                 << file.constant_values().GetSymbolicConstant(const_id);
-  } else if (const_id.is_concrete()) {
-    llvm::errs() << ": "
-                 << file.insts().Get(
-                        file.constant_values().GetInstId(const_id));
-  }
-}
-
-static auto DumpNoNewline(const File& file, InstId inst_id) -> void {
-  llvm::errs() << inst_id;
+static auto DumpInstSummary(const File& file, InstId inst_id) -> std::string {
+  RawStringOstream out;
+  out << inst_id;
   if (inst_id.has_value()) {
-    llvm::errs() << ": " << file.insts().Get(inst_id);
+    out << ": " << file.insts().Get(inst_id);
   }
+  return out.TakeStr();
 }
 
-static auto DumpNoNewline(const File& file, InterfaceId interface_id) -> void {
-  llvm::errs() << interface_id;
-  if (interface_id.has_value()) {
-    const auto& interface = file.interfaces().Get(interface_id);
-    llvm::errs() << ": " << interface;
-    DumpNameIfValid(file, interface.name_id);
-  }
-}
-
-static auto DumpNoNewline(const File& file, SpecificId specific_id) -> void {
-  llvm::errs() << specific_id;
+static auto DumpSpecificSummary(const File& file, SpecificId specific_id)
+    -> std::string {
+  RawStringOstream out;
+  out << specific_id;
   if (specific_id.has_value()) {
-    llvm::errs() << ": " << file.specifics().Get(specific_id);
+    out << ": " << file.specifics().Get(specific_id);
   }
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, ClassId class_id) -> void {
-  llvm::errs() << class_id;
+LLVM_DUMP_METHOD auto Dump(const File& file, ClassId class_id) -> std::string {
+  RawStringOstream out;
+  out << class_id;
   if (class_id.has_value()) {
     const auto& class_obj = file.classes().Get(class_id);
-    llvm::errs() << ": " << class_obj;
-    DumpNameIfValid(file, class_obj.name_id);
+    out << ": " << class_obj << DumpNameIfValid(file, class_obj.name_id);
   }
-  llvm::errs() << '\n';
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, ConstantId const_id) -> void {
-  DumpNoNewline(file, const_id);
-  llvm::errs() << '\n';
+LLVM_DUMP_METHOD auto Dump(const File& file, ConstantId const_id)
+    -> std::string {
+  RawStringOstream out;
+  out << const_id;
+  if (!const_id.has_value()) {
+    return out.TakeStr();
+  }
+  if (const_id.is_symbolic()) {
+    out << ": " << file.constant_values().GetSymbolicConstant(const_id);
+  } else if (const_id.is_concrete()) {
+    out << ": " << file.insts().Get(file.constant_values().GetInstId(const_id));
+  }
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file, EntityNameId entity_name_id)
-    -> void {
-  llvm::errs() << entity_name_id;
+    -> std::string {
+  RawStringOstream out;
+  out << entity_name_id;
   if (entity_name_id.has_value()) {
     auto entity_name = file.entity_names().Get(entity_name_id);
-    llvm::errs() << ": " << entity_name;
-    DumpNameIfValid(file, entity_name.name_id);
+    out << ": " << entity_name << DumpNameIfValid(file, entity_name.name_id);
   }
-  llvm::errs() << '\n';
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file, FacetTypeId facet_type_id)
-    -> void {
-  llvm::errs() << facet_type_id;
-  if (facet_type_id.has_value()) {
-    const auto& facet_type = file.facet_types().Get(facet_type_id);
-    llvm::errs() << ": " << facet_type << '\n';
-    for (auto impls : facet_type.impls_constraints) {
-      llvm::errs() << "  - ";
-      DumpNoNewline(file, impls.interface_id);
-      if (impls.specific_id.has_value()) {
-        llvm::errs() << "; ";
-        DumpNoNewline(file, impls.specific_id);
-      }
-      llvm::errs() << '\n';
-    }
-    for (auto rewrite : facet_type.rewrite_constraints) {
-      llvm::errs() << "  - ";
-      Dump(file, rewrite.lhs_const_id);
-      llvm::errs() << "  - ";
-      Dump(file, rewrite.rhs_const_id);
-    }
-    if (auto complete_id = file.complete_facet_types().TryGetId(facet_type_id);
-        complete_id.has_value()) {
-      llvm::errs() << "complete: ";
-      Dump(file, complete_id);
-    }
-  } else {
-    llvm::errs() << '\n';
+    -> std::string {
+  RawStringOstream out;
+  out << facet_type_id;
+  if (!facet_type_id.has_value()) {
+    return out.TakeStr();
   }
+
+  const auto& facet_type = file.facet_types().Get(facet_type_id);
+  out << ": " << facet_type;
+  for (auto impls : facet_type.impls_constraints) {
+    out << "\n  - " << Dump(file, impls.interface_id);
+    if (impls.specific_id.has_value()) {
+      out << "; " << DumpSpecificSummary(file, impls.specific_id);
+    }
+  }
+  for (auto rewrite : facet_type.rewrite_constraints) {
+    out << "\n"
+        << "  - " << Dump(file, rewrite.lhs_const_id) << "\n"
+        << "  - " << Dump(file, rewrite.rhs_const_id);
+  }
+  if (auto complete_id = file.complete_facet_types().TryGetId(facet_type_id);
+      complete_id.has_value()) {
+    out << "\ncomplete: " << Dump(file, complete_id);
+  }
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, FunctionId function_id) -> void {
-  llvm::errs() << function_id;
+LLVM_DUMP_METHOD auto Dump(const File& file, FunctionId function_id)
+    -> std::string {
+  RawStringOstream out;
+  out << function_id;
   if (function_id.has_value()) {
     const auto& function = file.functions().Get(function_id);
-    llvm::errs() << ": " << function;
-    DumpNameIfValid(file, function.name_id);
+    out << ": " << function << DumpNameIfValid(file, function.name_id);
   }
-  llvm::errs() << '\n';
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, GenericId generic_id) -> void {
-  llvm::errs() << generic_id;
+LLVM_DUMP_METHOD auto Dump(const File& file, GenericId generic_id)
+    -> std::string {
+  RawStringOstream out;
+  out << generic_id;
   if (!generic_id.has_value()) {
-    llvm::errs() << '\n';
-    return;
+    return out.TakeStr();
   }
-  llvm::errs() << ": " << file.generics().Get(generic_id) << '\n';
-  Dump(file, file.generics().Get(generic_id).bindings_id);
+  out << ": " << file.generics().Get(generic_id) << '\n'
+      << Dump(file, file.generics().Get(generic_id).bindings_id);
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, ImplId impl_id) -> void {
-  llvm::errs() << impl_id;
+LLVM_DUMP_METHOD auto Dump(const File& file, ImplId impl_id) -> std::string {
+  RawStringOstream out;
+  out << impl_id;
   if (!impl_id.has_value()) {
-    llvm::errs() << '\n';
-    return;
+    return out.TakeStr();
   }
-
   const auto& impl = file.impls().Get(impl_id);
-  llvm::errs() << ": " << impl << '\n';
-  llvm::errs() << "  - interface_id: ";
-  DumpNoNewline(file, impl.interface.interface_id);
-  llvm::errs() << '\n';
-  llvm::errs() << "  - specific_id: ";
-  DumpNoNewline(file, impl.interface.specific_id);
-  llvm::errs() << '\n';
+  out << ": " << impl << '\n'
+      << "  - interface_id: " << Dump(file, impl.interface.interface_id) << '\n'
+      << "  - specific_id: "
+      << DumpSpecificSummary(file, impl.interface.specific_id);
   if (impl.interface.specific_id.has_value()) {
     auto inst_block_id =
         file.specifics().Get(impl.interface.specific_id).args_id;
-    Dump(file, inst_block_id);
+    out << '\n' << Dump(file, inst_block_id);
   }
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file, InstBlockId inst_block_id)
-    -> void {
-  llvm::errs() << inst_block_id;
+    -> std::string {
+  RawStringOstream out;
+  out << inst_block_id;
   if (inst_block_id.has_value()) {
-    llvm::errs() << ":";
+    out << ":";
     auto inst_block = file.inst_blocks().Get(inst_block_id);
     for (auto inst_id : inst_block) {
-      llvm::errs() << "\n  - ";
-      DumpNoNewline(file, inst_id);
+      out << "\n  - " << DumpInstSummary(file, inst_id);
     }
   }
-  llvm::errs() << '\n';
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, InstId inst_id) -> void {
-  DumpNoNewline(file, inst_id);
-  llvm::errs() << '\n';
-  if (inst_id.has_value()) {
-    Inst inst = file.insts().Get(inst_id);
-    if (inst.type_id().has_value()) {
-      llvm::errs() << "  - type ";
-      Dump(file, inst.type_id());
-    }
-    ConstantId const_id = file.constant_values().Get(inst_id);
-    if (const_id.has_value()) {
-      InstId const_inst_id = file.constant_values().GetInstId(const_id);
-      llvm::errs() << "  - value ";
-      if (const_inst_id == inst_id) {
-        llvm::errs() << const_id << '\n';
-      } else {
-        Dump(file, const_id);
-      }
+LLVM_DUMP_METHOD auto Dump(const File& file, InstId inst_id) -> std::string {
+  RawStringOstream out;
+  out << DumpInstSummary(file, inst_id);
+  if (!inst_id.has_value()) {
+    return out.TakeStr();
+  }
+
+  Inst inst = file.insts().Get(inst_id);
+  if (inst.type_id().has_value()) {
+    out << "\n  - type: " << Dump(file, inst.type_id());
+  }
+  ConstantId const_id = file.constant_values().Get(inst_id);
+  if (const_id.has_value()) {
+    InstId const_inst_id = file.constant_values().GetInstId(const_id);
+    out << "\n  - value: ";
+    if (const_inst_id == inst_id) {
+      out << const_id;
+    } else {
+      out << Dump(file, const_id);
     }
   }
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, InterfaceId interface_id) -> void {
-  DumpNoNewline(file, interface_id);
-  llvm::errs() << '\n';
+LLVM_DUMP_METHOD auto Dump(const File& file, InterfaceId interface_id)
+    -> std::string {
+  RawStringOstream out;
+  out << interface_id;
+  if (interface_id.has_value()) {
+    const auto& interface = file.interfaces().Get(interface_id);
+    out << ": " << interface << DumpNameIfValid(file, interface.name_id);
+  }
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, NameId name_id) -> void {
-  llvm::errs() << name_id;
-  DumpNameIfValid(file, name_id);
-  llvm::errs() << '\n';
+LLVM_DUMP_METHOD auto Dump(const File& file, NameId name_id) -> std::string {
+  RawStringOstream out;
+  out << name_id << DumpNameIfValid(file, name_id);
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file, NameScopeId name_scope_id)
-    -> void {
-  llvm::errs() << name_scope_id;
-  if (name_scope_id.has_value()) {
-    const auto& name_scope = file.name_scopes().Get(name_scope_id);
-    llvm::errs() << ": " << name_scope;
-    if (name_scope.inst_id().has_value()) {
-      llvm::errs() << " " << file.insts().Get(name_scope.inst_id());
-    }
-    DumpNameIfValid(file, name_scope.name_id());
-    llvm::errs() << '\n';
-    for (const auto& entry : name_scope.entries()) {
-      llvm::errs() << "  - " << entry.name_id;
-      DumpNameIfValid(file, entry.name_id);
-      llvm::errs() << ": ";
-      if (entry.result.is_poisoned()) {
-        llvm::errs() << "<poisoned>\n";
-      } else if (entry.result.is_found()) {
-        switch (entry.result.access_kind()) {
-          case AccessKind::Public:
-            llvm::errs() << "public ";
-            break;
-          case AccessKind::Protected:
-            llvm::errs() << "protected ";
-            break;
-          case AccessKind::Private:
-            llvm::errs() << "private ";
-            break;
-        }
-        DumpNoNewline(file, entry.result.target_inst_id());
-        llvm::errs() << "\n";
-      } else {
-        llvm::errs() << "<not-found>\n";
-      }
-    }
-  } else {
-    llvm::errs() << '\n';
+    -> std::string {
+  RawStringOstream out;
+  out << name_scope_id;
+  if (!name_scope_id.has_value()) {
+    return out.TakeStr();
   }
+
+  const auto& name_scope = file.name_scopes().Get(name_scope_id);
+  out << ": " << name_scope;
+  if (name_scope.inst_id().has_value()) {
+    out << " " << file.insts().Get(name_scope.inst_id());
+  }
+  out << DumpNameIfValid(file, name_scope.name_id());
+  for (const auto& entry : name_scope.entries()) {
+    out << "\n  - " << entry.name_id << DumpNameIfValid(file, entry.name_id)
+        << ": ";
+    if (entry.result.is_poisoned()) {
+      out << "<poisoned>";
+    } else if (entry.result.is_found()) {
+      switch (entry.result.access_kind()) {
+        case AccessKind::Public:
+          out << "public ";
+          break;
+        case AccessKind::Protected:
+          out << "protected ";
+          break;
+        case AccessKind::Private:
+          out << "private ";
+          break;
+      }
+      out << DumpInstSummary(file, entry.result.target_inst_id());
+    } else {
+      out << "<not-found>";
+    }
+  }
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file,
-                           CompleteFacetTypeId complete_facet_type_id) -> void {
-  llvm::errs() << complete_facet_type_id << "\n";
-  if (complete_facet_type_id.has_value()) {
-    const auto& complete_facet_type =
-        file.complete_facet_types().Get(complete_facet_type_id);
-    for (auto [i, req_interface] :
-         llvm::enumerate(complete_facet_type.required_interfaces)) {
-      llvm::errs() << "  - ";
-      DumpNoNewline(file, req_interface.interface_id);
-      if (req_interface.specific_id.has_value()) {
-        llvm::errs() << "; ";
-        DumpNoNewline(file, req_interface.specific_id);
-      }
-      if (static_cast<int>(i) < complete_facet_type.num_to_impl) {
-        llvm::errs() << " (to impl)";
-      }
-      llvm::errs() << '\n';
+                           CompleteFacetTypeId complete_facet_type_id)
+    -> std::string {
+  RawStringOstream out;
+  out << complete_facet_type_id;
+  if (!complete_facet_type_id.has_value()) {
+    return out.TakeStr();
+  }
+
+  const auto& complete_facet_type =
+      file.complete_facet_types().Get(complete_facet_type_id);
+  for (auto [i, req_interface] :
+       llvm::enumerate(complete_facet_type.required_interfaces)) {
+    out << "\n  - " << Dump(file, req_interface.interface_id);
+    if (req_interface.specific_id.has_value()) {
+      out << "; " << DumpSpecificSummary(file, req_interface.specific_id);
+    }
+    if (static_cast<int>(i) < complete_facet_type.num_to_impl) {
+      out << " (to impl)";
     }
   }
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, SpecificId specific_id) -> void {
-  DumpNoNewline(file, specific_id);
-  llvm::errs() << '\n';
+LLVM_DUMP_METHOD auto Dump(const File& file, SpecificId specific_id)
+    -> std::string {
+  RawStringOstream out;
+  out << DumpSpecificSummary(file, specific_id);
   if (specific_id.has_value()) {
-    Dump(file, file.specifics().Get(specific_id).args_id);
+    out << "\n" << Dump(file, file.specifics().Get(specific_id).args_id);
   }
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file,
-                           SpecificInterfaceId specific_interface_id) -> void {
+                           SpecificInterfaceId specific_interface_id)
+    -> std::string {
+  RawStringOstream out;
   const auto& interface = file.specific_interfaces().Get(specific_interface_id);
-  llvm::errs() << specific_interface_id << '\n';
-  llvm::errs() << "  - interface: ";
-  DumpNoNewline(file, interface.interface_id);
-  llvm::errs() << '\n';
-  llvm::errs() << "  - specific_id: ";
-  Dump(file, interface.specific_id);
+  out << specific_interface_id << "\n"
+      << "  - interface: " << Dump(file, interface.interface_id) << "\n"
+      << "  - specific_id: " << Dump(file, interface.specific_id);
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file,
-                           StructTypeFieldsId struct_type_fields_id) -> void {
-  llvm::errs() << struct_type_fields_id;
+                           StructTypeFieldsId struct_type_fields_id)
+    -> std::string {
+  RawStringOstream out;
+  out << struct_type_fields_id;
   if (struct_type_fields_id.has_value()) {
-    llvm::errs() << ":";
+    out << ":";
     auto block = file.struct_type_fields().Get(struct_type_fields_id);
     for (auto field : block) {
-      llvm::errs() << "\n  - " << field;
-      DumpNameIfValid(file, field.name_id);
+      out << "\n  - " << field << DumpNameIfValid(file, field.name_id);
       if (field.type_id.has_value()) {
         InstId inst_id =
             file.constant_values().GetInstId(field.type_id.AsConstantId());
-        llvm::errs() << ": " << StringifyTypeExpr(file, inst_id);
+        out << ": " << StringifyTypeExpr(file, inst_id);
       }
     }
   }
-  llvm::errs() << '\n';
+  return out.TakeStr();
 }
 
 LLVM_DUMP_METHOD auto Dump(const File& file, TypeBlockId type_block_id)
-    -> void {
-  llvm::errs() << type_block_id;
-  if (type_block_id.has_value()) {
-    llvm::errs() << ":\n";
-    auto type_block = file.type_blocks().Get(type_block_id);
-    for (auto type_id : type_block) {
-      llvm::errs() << "  - ";
-      Dump(file, type_id);
-    }
-  } else {
-    llvm::errs() << '\n';
+    -> std::string {
+  RawStringOstream out;
+  out << type_block_id;
+  if (!type_block_id.has_value()) {
+    return out.TakeStr();
   }
+
+  out << ":";
+  auto type_block = file.type_blocks().Get(type_block_id);
+  for (auto type_id : type_block) {
+    out << "\n  - " << Dump(file, type_id);
+  }
+  return out.TakeStr();
 }
 
-LLVM_DUMP_METHOD auto Dump(const File& file, TypeId type_id) -> void {
-  llvm::errs() << type_id;
-  if (type_id.has_value()) {
-    InstId inst_id = file.constant_values().GetInstId(type_id.AsConstantId());
-    llvm::errs() << ": " << StringifyTypeExpr(file, inst_id) << "; "
-                 << file.insts().Get(inst_id);
+LLVM_DUMP_METHOD auto Dump(const File& file, TypeId type_id) -> std::string {
+  RawStringOstream out;
+  out << type_id;
+  if (!type_id.has_value()) {
+    return out.TakeStr();
   }
-  llvm::errs() << '\n';
+
+  InstId inst_id = file.constant_values().GetInstId(type_id.AsConstantId());
+  out << ": " << StringifyTypeExpr(file, inst_id) << "; "
+      << file.insts().Get(inst_id);
+  return out.TakeStr();
 }
 
 // Functions that can be used instead of the corresponding constructor, which is
diff --git a/toolchain/sem_ir/dump.h b/toolchain/sem_ir/dump.h
index 6080498f20b5..82e1b7a750b6 100644
--- a/toolchain/sem_ir/dump.h
+++ b/toolchain/sem_ir/dump.h
@@ -20,24 +20,27 @@
 
 namespace Carbon::SemIR {
 
-auto Dump(const File& file, ClassId class_id) -> void;
-auto Dump(const File& file, ConstantId const_id) -> void;
-auto Dump(const File& file, EntityNameId entity_name_id) -> void;
-auto Dump(const File& file, FacetTypeId facet_type_id) -> void;
-auto Dump(const File& file, FunctionId function_id) -> void;
-auto Dump(const File& file, GenericId generic_id) -> void;
-auto Dump(const File& file, ImplId impl_id) -> void;
-auto Dump(const File& file, InstBlockId inst_block_id) -> void;
-auto Dump(const File& file, InstId inst_id) -> void;
-auto Dump(const File& file, InterfaceId interface_id) -> void;
-auto Dump(const File& file, NameId name_id) -> void;
-auto Dump(const File& file, NameScopeId name_scope_id) -> void;
-auto Dump(const File& file, CompleteFacetTypeId complete_facet_type_id) -> void;
-auto Dump(const File& file, SpecificId specific_id) -> void;
-auto Dump(const File& file, SpecificInterfaceId specific_interface_id) -> void;
-auto Dump(const File& file, StructTypeFieldsId struct_type_fields_id) -> void;
-auto Dump(const File& file, TypeBlockId type_block_id) -> void;
-auto Dump(const File& file, TypeId type_id) -> void;
+auto Dump(const File& file, ClassId class_id) -> std::string;
+auto Dump(const File& file, ConstantId const_id) -> std::string;
+auto Dump(const File& file, EntityNameId entity_name_id) -> std::string;
+auto Dump(const File& file, FacetTypeId facet_type_id) -> std::string;
+auto Dump(const File& file, FunctionId function_id) -> std::string;
+auto Dump(const File& file, GenericId generic_id) -> std::string;
+auto Dump(const File& file, ImplId impl_id) -> std::string;
+auto Dump(const File& file, InstBlockId inst_block_id) -> std::string;
+auto Dump(const File& file, InstId inst_id) -> std::string;
+auto Dump(const File& file, InterfaceId interface_id) -> std::string;
+auto Dump(const File& file, NameId name_id) -> std::string;
+auto Dump(const File& file, NameScopeId name_scope_id) -> std::string;
+auto Dump(const File& file, CompleteFacetTypeId complete_facet_type_id)
+    -> std::string;
+auto Dump(const File& file, SpecificId specific_id) -> std::string;
+auto Dump(const File& file, SpecificInterfaceId specific_interface_id)
+    -> std::string;
+auto Dump(const File& file, StructTypeFieldsId struct_type_fields_id)
+    -> std::string;
+auto Dump(const File& file, TypeBlockId type_block_id) -> std::string;
+auto Dump(const File& file, TypeId type_id) -> std::string;
 
 }  // namespace Carbon::SemIR