diff --git a/toolchain/check/call.cpp b/toolchain/check/call.cpp
index dcfcf561da15..2df0874efe34 100644
--- a/toolchain/check/call.cpp
+++ b/toolchain/check/call.cpp
@@ -229,8 +229,8 @@ static auto CheckCalleeFunctionReturnType(Context& context, SemIR::LocId loc_id,
 auto PerformCallToFunction(Context& context, SemIR::LocId loc_id,
                            SemIR::InstId callee_id,
                            const SemIR::CalleeFunction& callee_function,
-                           llvm::ArrayRef<SemIR::InstId> arg_ids)
-    -> SemIR::InstId {
+                           llvm::ArrayRef<SemIR::InstId> arg_ids,
+                           bool is_operator_syntax) -> SemIR::InstId {
   // If the callee is a generic function, determine the generic argument values
   // for the call.
   auto callee_specific_id = ResolveCalleeInCall(
@@ -280,9 +280,9 @@ auto PerformCallToFunction(Context& context, SemIR::LocId loc_id,
   auto& callee = context.functions().Get(callee_function.function_id);
 
   // Convert the arguments to match the parameters.
-  auto converted_args_id =
-      ConvertCallArgs(context, loc_id, callee_function.self_id, arg_ids,
-                      return_slot_arg_id, callee, *callee_specific_id);
+  auto converted_args_id = ConvertCallArgs(
+      context, loc_id, callee_function.self_id, arg_ids, return_slot_arg_id,
+      callee, *callee_specific_id, is_operator_syntax);
 
   switch (callee.special_function_kind) {
     case SemIR::Function::SpecialFunctionKind::Thunk: {
@@ -355,7 +355,8 @@ static auto PerformCallToNonFunction(Context& context, SemIR::LocId loc_id,
 }
 
 auto PerformCall(Context& context, SemIR::LocId loc_id, SemIR::InstId callee_id,
-                 llvm::ArrayRef<SemIR::InstId> arg_ids) -> SemIR::InstId {
+                 llvm::ArrayRef<SemIR::InstId> arg_ids, bool is_operator_syntax)
+    -> SemIR::InstId {
   // Try treating the callee as a function first.
   auto callee = GetCallee(context.sem_ir(), callee_id);
   CARBON_KIND_SWITCH(callee) {
@@ -363,18 +364,17 @@ auto PerformCall(Context& context, SemIR::LocId loc_id, SemIR::InstId callee_id,
       return SemIR::ErrorInst::InstId;
     }
     case CARBON_KIND(SemIR::CalleeFunction fn): {
-      context.ref_tags().Insert(fn.self_id, Context::RefTag::NotRequired);
-      return PerformCallToFunction(context, loc_id, callee_id, fn, arg_ids);
+      return PerformCallToFunction(context, loc_id, callee_id, fn, arg_ids,
+                                   is_operator_syntax);
     }
     case CARBON_KIND(SemIR::CalleeNonFunction _): {
       return PerformCallToNonFunction(context, loc_id, callee_id, arg_ids);
     }
 
     case CARBON_KIND(SemIR::CalleeCppOverloadSet overload): {
-      context.ref_tags().Insert(overload.self_id, Context::RefTag::NotRequired);
-      return PerformCallToCppFunction(context, loc_id,
-                                      overload.cpp_overload_set_id,
-                                      overload.self_id, arg_ids);
+      return PerformCallToCppFunction(
+          context, loc_id, overload.cpp_overload_set_id, overload.self_id,
+          arg_ids, is_operator_syntax);
     }
   }
 }
diff --git a/toolchain/check/call.h b/toolchain/check/call.h
index a42a0d9ebb08..a119851f10c3 100644
--- a/toolchain/check/call.h
+++ b/toolchain/check/call.h
@@ -11,16 +11,22 @@
 namespace Carbon::Check {
 
 // Checks and builds SemIR for a call to `callee_id` with arguments `args_id`,
-// where the callee is a function.
+// where the callee is a function. `is_operator_syntax` indicates that this call
+// was generated from an operator rather than from function call syntax, so
+// arguments to `ref` parameters aren't required to have `ref` tags.
 auto PerformCallToFunction(Context& context, SemIR::LocId loc_id,
                            SemIR::InstId callee_id,
                            const SemIR::CalleeFunction& callee_function,
-                           llvm::ArrayRef<SemIR::InstId> arg_ids)
-    -> SemIR::InstId;
+                           llvm::ArrayRef<SemIR::InstId> arg_ids,
+                           bool is_operator_syntax) -> SemIR::InstId;
 
 // Checks and builds SemIR for a call to `callee_id` with arguments `args_id`.
+// `is_operator_syntax` indicates that this call
+// was generated from an operator rather than from function call syntax, so
+// arguments to `ref` parameters aren't required to have `ref` tags.
 auto PerformCall(Context& context, SemIR::LocId loc_id, SemIR::InstId callee_id,
-                 llvm::ArrayRef<SemIR::InstId> arg_ids) -> SemIR::InstId;
+                 llvm::ArrayRef<SemIR::InstId> arg_ids,
+                 bool is_operator_syntax = false) -> SemIR::InstId;
 
 }  // namespace Carbon::Check
 
diff --git a/toolchain/check/context.cpp b/toolchain/check/context.cpp
index 7b55d8a9c68f..5767ad7782d0 100644
--- a/toolchain/check/context.cpp
+++ b/toolchain/check/context.cpp
@@ -80,19 +80,6 @@ auto Context::VerifyOnFinish() const -> void {
     CARBON_FATAL("{0}Built invalid semantics IR: {1}\n", sem_ir_,
                  verify.error());
   }
-
-  if (!sem_ir_->has_errors()) {
-    auto ref_tags_needed = sem_ir_->CollectRefTagsNeeded();
-
-    ref_tags_.ForEach([&ref_tags_needed](SemIR::InstId inst_id, RefTag kind) {
-      CARBON_CHECK(
-          ref_tags_needed.Erase(inst_id) || kind == RefTag::NotRequired,
-          "Inst has unnecessary `ref` tag: {0}", inst_id);
-    });
-    ref_tags_needed.ForEach([this](SemIR::InstId inst_id) {
-      CARBON_FATAL("Inst missing `ref` tag: {0}", insts().Get(inst_id));
-    });
-  }
 #endif
 }
 
diff --git a/toolchain/check/context.h b/toolchain/check/context.h
index e23d5af8d23e..dac1529e3ed0 100644
--- a/toolchain/check/context.h
+++ b/toolchain/check/context.h
@@ -198,13 +198,6 @@ class Context {
     return var_storage_map_;
   }
 
-  enum class RefTag { Present, NotRequired };
-
-  auto ref_tags() -> Map<SemIR::InstId, RefTag>& { return ref_tags_; }
-  auto ref_tags() const -> const Map<SemIR::InstId, RefTag>& {
-    return ref_tags_;
-  }
-
   // During Choice typechecking, each alternative turns into a name binding on
   // the Choice type, but this can't be done until the full Choice type is
   // known. This represents each binding to be done at the end of checking the
@@ -481,13 +474,6 @@ class Context {
   // processing the enclosing full-pattern.
   Map<SemIR::InstId, SemIR::InstId> var_storage_map_;
 
-  // Insts in this map are syntactically permitted to be bound to a reference
-  // parameter, either because they've been explicitly tagged with `ref` in the
-  // source code, or because they appear in a position where that tag is not
-  // required, such as an operator operand (the RefTag value indicates which
-  // of those is the case).
-  Map<SemIR::InstId, RefTag> ref_tags_;
-
   // Each alternative in a Choice gets an entry here, they are stored in
   // declaration order. The vector is consumed and emptied at the end of the
   // Choice definition.
diff --git a/toolchain/check/convert.cpp b/toolchain/check/convert.cpp
index 4719f124ba84..61e5c7a01644 100644
--- a/toolchain/check/convert.cpp
+++ b/toolchain/check/convert.cpp
@@ -795,6 +795,7 @@ static auto IsValidExprCategoryForConversionTarget(
              category == SemIR::ExprCategory::EphemeralRef ||
              category == SemIR::ExprCategory::Initializing;
     case ConversionTarget::RefParam:
+    case ConversionTarget::UnmarkedRefParam:
       return category == SemIR::ExprCategory::DurableRef ||
              category == SemIR::ExprCategory::EphemeralRef ||
              category == SemIR::ExprCategory::Initializing;
@@ -1428,28 +1429,6 @@ auto PerformAction(Context& context, SemIR::LocId loc_id,
                       action.target_type_inst_id)});
 }
 
-// Diagnoses a missing or unnecessary `ref` tag when converting `expr_id` to
-// `target`, and returns whether a `ref` tag is present.
-static auto CheckRefTag(Context& context, SemIR::InstId expr_id,
-                        ConversionTarget target) -> bool {
-  if (auto lookup_result = context.ref_tags().Lookup(expr_id)) {
-    if (lookup_result.value() == Context::RefTag::Present &&
-        target.kind != ConversionTarget::RefParam) {
-      CARBON_DIAGNOSTIC(RefTagNoRefParam, Error,
-                        "`ref` tag is not an argument to a `ref` parameter");
-      context.emitter().Emit(expr_id, RefTagNoRefParam);
-    }
-    return true;
-  } else {
-    if (target.kind == ConversionTarget::RefParam) {
-      CARBON_DIAGNOSTIC(RefParamNoRefTag, Error,
-                        "argument to `ref` parameter not marked with `ref`");
-      context.emitter().Emit(expr_id, RefParamNoRefTag);
-    }
-    return false;
-  }
-}
-
 // State machine for performing category conversions.
 class CategoryConverter {
  public:
@@ -1558,8 +1537,41 @@ auto CategoryConverter::DoStep(const SemIR::InstId expr_id,
                         .category = SemIR::ExprCategory::EphemeralRef};
       }
 
+    case SemIR::ExprCategory::RefTagged: {
+      auto tagged_expr_id =
+          sem_ir_.insts().GetAs<SemIR::RefTagExpr>(expr_id).expr_id;
+      auto tagged_expr_category =
+          SemIR::GetExprCategory(sem_ir_, tagged_expr_id);
+      if (target_.diagnose &&
+          tagged_expr_category != SemIR::ExprCategory::DurableRef) {
+        CARBON_DIAGNOSTIC(
+            RefTagNotDurableRef, Error,
+            "expression tagged with `ref` is not a durable reference");
+        context_.emitter().Emit(tagged_expr_id, RefTagNotDurableRef);
+      }
+
+      if (target_.kind == ConversionTarget::RefParam) {
+        return Done{expr_id};
+      }
+
+      // If the target isn't a reference parameter, ignore the `ref` tag.
+      // Unnecessary `ref` tags are diagnosed earlier.
+      return NextStep{.expr_id = tagged_expr_id,
+                      .category = tagged_expr_category};
+    }
+
     case SemIR::ExprCategory::DurableRef:
-      if (target_.kind == ConversionTarget::DurableRef) {
+      if (target_.kind == ConversionTarget::DurableRef ||
+          target_.kind == ConversionTarget::UnmarkedRefParam) {
+        return Done{expr_id};
+      }
+      if (target_.kind == ConversionTarget::RefParam) {
+        if (target_.diagnose) {
+          CARBON_DIAGNOSTIC(
+              RefParamNoRefTag, Error,
+              "argument to `ref` parameter not marked with `ref`");
+          context_.emitter().Emit(expr_id, RefParamNoRefTag);
+        }
         return Done{expr_id};
       }
       [[fallthrough]];
@@ -1569,7 +1581,8 @@ auto CategoryConverter::DoStep(const SemIR::InstId expr_id,
       if (target_.kind == ConversionTarget::ValueOrRef ||
           target_.kind == ConversionTarget::Discarded ||
           target_.kind == ConversionTarget::CppThunkRef ||
-          target_.kind == ConversionTarget::RefParam) {
+          target_.kind == ConversionTarget::RefParam ||
+          target_.kind == ConversionTarget::UnmarkedRefParam) {
         return Done{expr_id};
       }
 
@@ -1592,17 +1605,12 @@ auto CategoryConverter::DoStep(const SemIR::InstId expr_id,
         }
         return Done{SemIR::ErrorInst::InstId};
       }
-      if (target_.kind == ConversionTarget::RefParam) {
-        // Don't diagnose a non-reference scrutinee if it has a user-written
-        // `ref` tag, because that's diagnosed in `CheckRefTag`.
+      if (target_.kind == ConversionTarget::RefParam ||
+          target_.kind == ConversionTarget::UnmarkedRefParam) {
         if (target_.diagnose) {
-          if (auto lookup_result = context_.ref_tags().Lookup(expr_id);
-              !lookup_result ||
-              lookup_result.value() != Context::RefTag::Present) {
-            CARBON_DIAGNOSTIC(ValueForRefParam, Error,
-                              "value expression passed to reference parameter");
-            context_.emitter().Emit(loc_id_, ValueForRefParam);
-          }
+          CARBON_DIAGNOSTIC(ValueForRefParam, Error,
+                            "value expression passed to reference parameter");
+          context_.emitter().Emit(loc_id_, ValueForRefParam);
         }
         return Done{SemIR::ErrorInst::InstId};
       }
@@ -1635,7 +1643,8 @@ auto Convert(Context& context, SemIR::LocId loc_id, SemIR::InstId expr_id,
     return SemIR::ErrorInst::InstId;
   }
 
-  if (SemIR::GetExprCategory(sem_ir, expr_id) == SemIR::ExprCategory::NotExpr) {
+  auto starting_category = SemIR::GetExprCategory(sem_ir, expr_id);
+  if (starting_category == SemIR::ExprCategory::NotExpr) {
     // TODO: We currently encounter this for use of namespaces and functions.
     // We should provide a better diagnostic for inappropriate use of
     // namespace names, and allow use of functions as values.
@@ -1647,7 +1656,14 @@ auto Convert(Context& context, SemIR::LocId loc_id, SemIR::InstId expr_id,
     return SemIR::ErrorInst::InstId;
   }
 
-  bool has_ref_tag = CheckRefTag(context, expr_id, target);
+  // Diagnose unnecessary `ref` tags early, so that they're not obscured by
+  // conversions.
+  if (starting_category == SemIR::ExprCategory::RefTagged &&
+      target.kind != ConversionTarget::RefParam && target.diagnose) {
+    CARBON_DIAGNOSTIC(RefTagNoRefParam, Error,
+                      "`ref` tag is not an argument to a `ref` parameter");
+    context.emitter().Emit(expr_id, RefTagNoRefParam);
+  }
 
   // We can only perform initialization for complete, non-abstract types. Note
   // that `RequireConcreteType` returns true for facet types, since their
@@ -1781,9 +1797,6 @@ auto Convert(Context& context, SemIR::LocId loc_id, SemIR::InstId expr_id,
                                         {.type_id = target.type_id,
                                          .original_id = orig_expr_id,
                                          .result_id = expr_id});
-    if (has_ref_tag) {
-      context.ref_tags().Insert(expr_id, Context::RefTag::NotRequired);
-    }
   }
 
   // For `as`, don't perform any value category conversions. In particular, an
@@ -1882,8 +1895,8 @@ auto ConvertCallArgs(Context& context, SemIR::LocId call_loc_id,
                      llvm::ArrayRef<SemIR::InstId> arg_refs,
                      SemIR::InstId return_slot_arg_id,
                      const SemIR::Function& callee,
-                     SemIR::SpecificId callee_specific_id)
-    -> SemIR::InstBlockId {
+                     SemIR::SpecificId callee_specific_id,
+                     bool is_operator_syntax) -> SemIR::InstBlockId {
   auto param_patterns =
       context.inst_blocks().GetOrEmpty(callee.param_patterns_id);
   auto return_patterns_id = callee.return_patterns_id;
@@ -1904,7 +1917,8 @@ auto ConvertCallArgs(Context& context, SemIR::LocId call_loc_id,
 
   return CallerPatternMatch(context, callee_specific_id, callee.self_param_id,
                             callee.param_patterns_id, return_patterns_id,
-                            self_id, arg_refs, return_slot_arg_id);
+                            self_id, arg_refs, return_slot_arg_id,
+                            is_operator_syntax);
 }
 
 auto TypeExpr::ForUnsugared(Context& context, SemIR::TypeId type_id)
diff --git a/toolchain/check/convert.h b/toolchain/check/convert.h
index 1a7a4ab29b4e..cffb8e21ac01 100644
--- a/toolchain/check/convert.h
+++ b/toolchain/check/convert.h
@@ -26,6 +26,10 @@ struct ConversionTarget {
     // only a `ref self` parameter can bind to an ephemeral reference is
     // enforced separately when handling `ref` tags on call arguments.
     RefParam,
+    // Equivalent to RefParam, except that the source expression is not required
+    // to be marked with a `ref` tag, such as an argument to a `ref self`
+    // parameter or an operator operand.
+    UnmarkedRefParam,
     // Convert to a reference of type `type_id`, for use as the argument to a
     // C++ thunk.
     CppThunkRef,
@@ -126,14 +130,16 @@ auto ConvertForExplicitAs(Context& context, Parse::NodeId as_node,
 
 // Implicitly converts a set of arguments to match the parameter types in a
 // function call. Returns a block containing the converted implicit and explicit
-// argument values for runtime parameters.
+// argument values for runtime parameters. `is_operator_syntax` indicates that
+// this call was generated from an operator rather than from function call
+// syntax, so arguments to `ref` parameters aren't required to have `ref` tags.
 auto ConvertCallArgs(Context& context, SemIR::LocId call_loc_id,
                      SemIR::InstId self_id,
                      llvm::ArrayRef<SemIR::InstId> arg_refs,
                      SemIR::InstId return_slot_arg_id,
                      const SemIR::Function& callee,
-                     SemIR::SpecificId callee_specific_id)
-    -> SemIR::InstBlockId;
+                     SemIR::SpecificId callee_specific_id,
+                     bool is_operator_syntax) -> SemIR::InstBlockId;
 
 // A type that has been converted for use as a type expression.
 struct TypeExpr {
diff --git a/toolchain/check/cpp/call.cpp b/toolchain/check/cpp/call.cpp
index 23bbc5a94c1e..df3eeff9c23a 100644
--- a/toolchain/check/cpp/call.cpp
+++ b/toolchain/check/cpp/call.cpp
@@ -22,8 +22,8 @@ namespace Carbon::Check {
 auto PerformCallToCppFunction(Context& context, SemIR::LocId loc_id,
                               SemIR::CppOverloadSetId overload_set_id,
                               SemIR::InstId self_id,
-                              llvm::ArrayRef<SemIR::InstId> arg_ids)
-    -> SemIR::InstId {
+                              llvm::ArrayRef<SemIR::InstId> arg_ids,
+                              bool is_operator_syntax) -> SemIR::InstId {
   SemIR::InstId callee_id = PerformCppOverloadResolution(
       context, loc_id, overload_set_id, self_id, arg_ids);
   SemIR::Callee callee = GetCallee(context.sem_ir(), callee_id);
@@ -37,7 +37,8 @@ auto PerformCallToCppFunction(Context& context, SemIR::LocId loc_id,
         // Preserve the `self` argument from the original callee.
         fn.self_id = self_id;
       }
-      return PerformCallToFunction(context, loc_id, callee_id, fn, arg_ids);
+      return PerformCallToFunction(context, loc_id, callee_id, fn, arg_ids,
+                                   is_operator_syntax);
     }
     case CARBON_KIND(SemIR::CalleeCppOverloadSet _): {
       CARBON_FATAL("overloads can't be recursive");
diff --git a/toolchain/check/cpp/call.h b/toolchain/check/cpp/call.h
index 940c732e1417..0fa49283f4ab 100644
--- a/toolchain/check/cpp/call.h
+++ b/toolchain/check/cpp/call.h
@@ -11,7 +11,10 @@
 namespace Carbon::Check {
 
 // Checks and builds SemIR for a call to a C++ function in the given overload
-// set with self `self_id` and arguments `arg_ids`.
+// set with self `self_id` and arguments `arg_ids`. `is_operator_syntax`
+// indicates that this call was generated from an operator rather than from
+// function call syntax, so arguments to `ref` parameters aren't required to
+// have `ref` tags.
 //
 // Chooses the best viable C++ function by performing Clang overloading
 // resolution over the overload set.
@@ -28,8 +31,8 @@ namespace Carbon::Check {
 auto PerformCallToCppFunction(Context& context, SemIR::LocId loc_id,
                               SemIR::CppOverloadSetId overload_set_id,
                               SemIR::InstId self_id,
-                              llvm::ArrayRef<SemIR::InstId> arg_ids)
-    -> SemIR::InstId;
+                              llvm::ArrayRef<SemIR::InstId> arg_ids,
+                              bool is_operator_syntax) -> SemIR::InstId;
 
 // Checks and builds SemIR for a call to a C++ template name with arguments
 // `arg_ids`.
diff --git a/toolchain/check/cpp/type_mapping.cpp b/toolchain/check/cpp/type_mapping.cpp
index 76798c83a753..b099541135af 100644
--- a/toolchain/check/cpp/type_mapping.cpp
+++ b/toolchain/check/cpp/type_mapping.cpp
@@ -318,6 +318,7 @@ auto InventClangArg(Context& context, SemIR::InstId arg_id) -> clang::Expr* {
       CARBON_FATAL("Passing a pattern as a function argument");
 
     case SemIR::ExprCategory::DurableRef:
+    case SemIR::ExprCategory::RefTagged:
       value_kind = clang::ExprValueKind::VK_LValue;
       break;
 
diff --git a/toolchain/check/handle_operator.cpp b/toolchain/check/handle_operator.cpp
index 328356b10b90..d3116e93901e 100644
--- a/toolchain/check/handle_operator.cpp
+++ b/toolchain/check/handle_operator.cpp
@@ -359,16 +359,11 @@ auto HandleParseNode(Context& context, Parse::PrefixOperatorPlusPlusId node_id)
 
 auto HandleParseNode(Context& context, Parse::PrefixOperatorRefId node_id)
     -> bool {
-  auto expr_id = context.node_stack().Peek<Parse::NodeCategory::Expr>();
-
-  if (SemIR::GetExprCategory(context.sem_ir(), expr_id) !=
-      SemIR::ExprCategory::DurableRef) {
-    CARBON_DIAGNOSTIC(
-        RefTagNotDurableRef, Error,
-        "expression tagged with `ref` is not a durable reference");
-    context.emitter().Emit(node_id, RefTagNotDurableRef);
-  }
-  context.ref_tags().Insert(expr_id, Context::RefTag::Present);
+  auto expr_id = context.node_stack().PopExpr();
+  auto ref_id = AddInst<SemIR::RefTagExpr>(
+      context, node_id,
+      {.type_id = context.insts().Get(expr_id).type_id(), .expr_id = expr_id});
+  context.node_stack().Push(node_id, ref_id);
   return true;
 }
 
diff --git a/toolchain/check/operator.cpp b/toolchain/check/operator.cpp
index 317611cd9f96..af0a4c271f2a 100644
--- a/toolchain/check/operator.cpp
+++ b/toolchain/check/operator.cpp
@@ -64,9 +64,6 @@ auto BuildUnaryOperator(Context& context, SemIR::LocId loc_id, Operator op,
     return SemIR::ErrorInst::InstId;
   }
 
-  // Operator operands don't require `ref` tags.
-  context.ref_tags().Insert(operand_id, Context::RefTag::NotRequired);
-
   SemIR::InstId op_fn_id = SemIR::InstId::None;
 
   // For unary operators with a C++ class as the operand, try to import and call
@@ -79,7 +76,8 @@ auto BuildUnaryOperator(Context& context, SemIR::LocId loc_id, Operator op,
     // If C++ operator lookup found a non-method operator, call it with one call
     // argument. Otherwise fall through to call it with a self argument.
     if (op_fn_id.has_value() && !IsCppOperatorMethod(context, op_fn_id)) {
-      return PerformCall(context, loc_id, op_fn_id, {operand_id});
+      return PerformCall(context, loc_id, op_fn_id, {operand_id},
+                         /*is_operator_syntax=*/true);
     }
   }
 
@@ -96,7 +94,8 @@ auto BuildUnaryOperator(Context& context, SemIR::LocId loc_id, Operator op,
   }
 
   // Form `bound_op()`.
-  return PerformCall(context, loc_id, bound_op_id, {});
+  return PerformCall(context, loc_id, bound_op_id, {},
+                     /*is_operator_syntax=*/true);
 }
 
 auto BuildBinaryOperator(Context& context, SemIR::LocId loc_id, Operator op,
@@ -108,10 +107,6 @@ auto BuildBinaryOperator(Context& context, SemIR::LocId loc_id, Operator op,
     return SemIR::ErrorInst::InstId;
   }
 
-  // Operator operands don't require `ref` tags.
-  context.ref_tags().Insert(lhs_id, Context::RefTag::NotRequired);
-  context.ref_tags().Insert(rhs_id, Context::RefTag::NotRequired);
-
   SemIR::InstId op_fn_id = SemIR::InstId::None;
 
   // For binary operators with a C++ class as at least one of the operands, try
@@ -129,7 +124,8 @@ auto BuildBinaryOperator(Context& context, SemIR::LocId loc_id, Operator op,
     // arguments. Otherwise fall through to call it with a self argument and one
     // call argument.
     if (op_fn_id.has_value() && !IsCppOperatorMethod(context, op_fn_id)) {
-      return PerformCall(context, loc_id, op_fn_id, {lhs_id, rhs_id});
+      return PerformCall(context, loc_id, op_fn_id, {lhs_id, rhs_id},
+                         /*is_operator_syntax=*/true);
     }
   }
 
@@ -146,7 +142,8 @@ auto BuildBinaryOperator(Context& context, SemIR::LocId loc_id, Operator op,
   }
 
   // Form `bound_op(rhs)`.
-  return PerformCall(context, loc_id, bound_op_id, {rhs_id});
+  return PerformCall(context, loc_id, bound_op_id, {rhs_id},
+                     /*is_operator_syntax=*/true);
 }
 
 }  // namespace Carbon::Check
diff --git a/toolchain/check/pattern_match.cpp b/toolchain/check/pattern_match.cpp
index 7ffc95d57096..ce8aefa72de3 100644
--- a/toolchain/check/pattern_match.cpp
+++ b/toolchain/check/pattern_match.cpp
@@ -49,10 +49,12 @@ class MatchContext {
     // `None` when processing the callee side.
     SemIR::InstId scrutinee_id;
 
-    bool is_self = false;
+    // If true, disables diagnostics that would otherwise require scrutinee_id
+    // to be tagged with `ref`. Only affects caller pattern matching.
+    bool allow_unmarked_ref = false;
     auto Print(llvm::raw_ostream& out) const -> void {
       out << "{pattern_id: " << pattern_id << ", scrutinee_id: " << scrutinee_id
-          << ", is_self = " << is_self << "}";
+          << ", allow_unmarked_ref = " << allow_unmarked_ref << "}";
     }
   };
 
@@ -309,7 +311,9 @@ auto MatchContext::DoEmitPatternMatch(Context& context,
                                          entry.pattern_id));
       results_.push_back(Convert(
           context, SemIR::LocId(entry.scrutinee_id), entry.scrutinee_id,
-          {.kind = ConversionTarget::RefParam, .type_id = scrutinee_type_id}));
+          {.kind = entry.allow_unmarked_ref ? ConversionTarget::UnmarkedRefParam
+                                            : ConversionTarget::RefParam,
+           .type_id = scrutinee_type_id}));
       // Do not traverse farther, because the caller side of the pattern
       // ends here.
       break;
@@ -625,8 +629,8 @@ auto CallerPatternMatch(Context& context, SemIR::SpecificId specific_id,
                         SemIR::InstBlockId return_patterns_id,
                         SemIR::InstId self_arg_id,
                         llvm::ArrayRef<SemIR::InstId> arg_refs,
-                        SemIR::InstId return_slot_arg_id)
-    -> SemIR::InstBlockId {
+                        SemIR::InstId return_slot_arg_id,
+                        bool is_operator_syntax) -> SemIR::InstBlockId {
   MatchContext match(MatchKind::Caller, specific_id);
 
   auto return_patterns = context.inst_blocks().GetOrEmpty(return_patterns_id);
@@ -641,13 +645,15 @@ auto CallerPatternMatch(Context& context, SemIR::SpecificId specific_id,
   // Check type conversions per-element.
   for (auto [arg_id, param_pattern_id] : llvm::reverse(llvm::zip_equal(
            arg_refs, context.inst_blocks().GetOrEmpty(param_patterns_id)))) {
-    match.AddWork({.pattern_id = param_pattern_id, .scrutinee_id = arg_id});
+    match.AddWork({.pattern_id = param_pattern_id,
+                   .scrutinee_id = arg_id,
+                   .allow_unmarked_ref = is_operator_syntax});
   }
 
   if (self_pattern_id.has_value()) {
     match.AddWork({.pattern_id = self_pattern_id,
                    .scrutinee_id = self_arg_id,
-                   .is_self = true});
+                   .allow_unmarked_ref = true});
   }
 
   return match.DoWork(context);
diff --git a/toolchain/check/pattern_match.h b/toolchain/check/pattern_match.h
index e18fb3188ff9..f783223d86b6 100644
--- a/toolchain/check/pattern_match.h
+++ b/toolchain/check/pattern_match.h
@@ -34,14 +34,17 @@ auto CalleePatternMatch(Context& context,
 
 // Emits the pattern-match IR for matching the given arguments with the given
 // parameter patterns, and returns an inst block of the arguments that should
-// be passed to the `Call` inst.
+// be passed to the `Call` inst. `is_operator_syntax` indicates that this call
+// was generated from an operator rather than from function call syntax, so
+// arguments to `ref` parameters aren't required to have `ref` tags.
 auto CallerPatternMatch(Context& context, SemIR::SpecificId specific_id,
                         SemIR::InstId self_pattern_id,
                         SemIR::InstBlockId param_patterns_id,
                         SemIR::InstBlockId return_patterns_id,
                         SemIR::InstId self_arg_id,
                         llvm::ArrayRef<SemIR::InstId> arg_refs,
-                        SemIR::InstId return_slot_arg_id) -> SemIR::InstBlockId;
+                        SemIR::InstId return_slot_arg_id,
+                        bool is_operator_syntax) -> SemIR::InstBlockId;
 
 // Emits the pattern-match IR for a local pattern matching operation with the
 // given pattern and scrutinee.
diff --git a/toolchain/check/testdata/builtins/int/and_assign.carbon b/toolchain/check/testdata/builtins/int/and_assign.carbon
index 2e293051107f..95f9a2c5321d 100644
--- a/toolchain/check/testdata/builtins/int/and_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/and_assign.carbon
@@ -71,8 +71,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.and_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/left_shift_assign.carbon b/toolchain/check/testdata/builtins/int/left_shift_assign.carbon
index 7922537049ef..a883b6bf7da8 100644
--- a/toolchain/check/testdata/builtins/int/left_shift_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/left_shift_assign.carbon
@@ -71,8 +71,9 @@ fn NotRef(a: i32, b: i32) = "int.left_shift_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
@@ -80,8 +81,9 @@ fn NotRef(a: i32, b: i32) = "int.left_shift_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %MixedTypes.ref: %MixedTypes.type = name_ref MixedTypes, file.%MixedTypes.decl [concrete = constants.%MixedTypes]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc16: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i64 = name_ref b, %b
-// CHECK:STDOUT:   %MixedTypes.call: init %empty_tuple.type = call %MixedTypes.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %MixedTypes.call: init %empty_tuple.type = call %MixedTypes.ref(%.loc16, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/or_assign.carbon b/toolchain/check/testdata/builtins/int/or_assign.carbon
index ff30c7b9bb87..59dd3079c324 100644
--- a/toolchain/check/testdata/builtins/int/or_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/or_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.or_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/right_shift_assign.carbon b/toolchain/check/testdata/builtins/int/right_shift_assign.carbon
index 9c2cf5d529e5..1ef1bd312797 100644
--- a/toolchain/check/testdata/builtins/int/right_shift_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/right_shift_assign.carbon
@@ -71,8 +71,9 @@ fn NotRef(a: i32, b: i32) = "int.right_shift_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
@@ -80,8 +81,9 @@ fn NotRef(a: i32, b: i32) = "int.right_shift_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %MixedTypes.ref: %MixedTypes.type = name_ref MixedTypes, file.%MixedTypes.decl [concrete = constants.%MixedTypes]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc16: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i64 = name_ref b, %b
-// CHECK:STDOUT:   %MixedTypes.call: init %empty_tuple.type = call %MixedTypes.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %MixedTypes.call: init %empty_tuple.type = call %MixedTypes.ref(%.loc16, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/sdiv_assign.carbon b/toolchain/check/testdata/builtins/int/sdiv_assign.carbon
index e1caba7c2176..2de8167b26cf 100644
--- a/toolchain/check/testdata/builtins/int/sdiv_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/sdiv_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.sdiv_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/smod_assign.carbon b/toolchain/check/testdata/builtins/int/smod_assign.carbon
index 76a88a8ca7a8..eeed753adc7c 100644
--- a/toolchain/check/testdata/builtins/int/smod_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/smod_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.smod_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/smul_assign.carbon b/toolchain/check/testdata/builtins/int/smul_assign.carbon
index 69f8c66bc006..55e201f7f300 100644
--- a/toolchain/check/testdata/builtins/int/smul_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/smul_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.smul_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/ssub_assign.carbon b/toolchain/check/testdata/builtins/int/ssub_assign.carbon
index 11754c732ad8..6b89ed1106e5 100644
--- a/toolchain/check/testdata/builtins/int/ssub_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/ssub_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.ssub_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/uadd_assign.carbon b/toolchain/check/testdata/builtins/int/uadd_assign.carbon
index 97d295076515..655dca412218 100644
--- a/toolchain/check/testdata/builtins/int/uadd_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/uadd_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.uadd_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/udiv_assign.carbon b/toolchain/check/testdata/builtins/int/udiv_assign.carbon
index b30889555ed4..7ed7f7f304ca 100644
--- a/toolchain/check/testdata/builtins/int/udiv_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/udiv_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.udiv_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/umod_assign.carbon b/toolchain/check/testdata/builtins/int/umod_assign.carbon
index 6f3babb37c5c..4bcd417be6bf 100644
--- a/toolchain/check/testdata/builtins/int/umod_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/umod_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.umod_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/umul_assign.carbon b/toolchain/check/testdata/builtins/int/umul_assign.carbon
index f833405af91e..1adb10d2bc17 100644
--- a/toolchain/check/testdata/builtins/int/umul_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/umul_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.umul_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/usub_assign.carbon b/toolchain/check/testdata/builtins/int/usub_assign.carbon
index b4badd4821f8..f3a94a1f4751 100644
--- a/toolchain/check/testdata/builtins/int/usub_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/usub_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.usub_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/builtins/int/xor_assign.carbon b/toolchain/check/testdata/builtins/int/xor_assign.carbon
index aec730c5a1de..460cd0a42043 100644
--- a/toolchain/check/testdata/builtins/int/xor_assign.carbon
+++ b/toolchain/check/testdata/builtins/int/xor_assign.carbon
@@ -65,8 +65,9 @@ fn MixedTypes(ref a: i32, b: i64) = "int.xor_assign";
 // CHECK:STDOUT: !entry:
 // CHECK:STDOUT:   %Builtin.ref: %Builtin.type = name_ref Builtin, file.%Builtin.decl [concrete = constants.%Builtin]
 // CHECK:STDOUT:   %a.ref: ref %i32 = name_ref a, %a
+// CHECK:STDOUT:   %.loc8: %i32 = ref_tag %a.ref
 // CHECK:STDOUT:   %b.ref: %i32 = name_ref b, %b
-// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%a.ref, %b.ref)
+// CHECK:STDOUT:   %Builtin.call: init %empty_tuple.type = call %Builtin.ref(%.loc8, %b.ref)
 // CHECK:STDOUT:   <elided>
 // CHECK:STDOUT: }
 // CHECK:STDOUT:
diff --git a/toolchain/check/testdata/function/call/ref.carbon b/toolchain/check/testdata/function/call/ref.carbon
index 603e75d157a1..1384cc6d7209 100644
--- a/toolchain/check/testdata/function/call/ref.carbon
+++ b/toolchain/check/testdata/function/call/ref.carbon
@@ -49,9 +49,12 @@ fn F(ref x: i32);
 
 fn G() {
   let y: i32 = 0;
-  // CHECK:STDERR: fail_ref_not_durable_ref.carbon:[[@LINE+4]]:5: error: expression tagged with `ref` is not a durable reference [RefTagNotDurableRef]
+  // CHECK:STDERR: fail_ref_not_durable_ref.carbon:[[@LINE+7]]:9: error: expression tagged with `ref` is not a durable reference [RefTagNotDurableRef]
   // CHECK:STDERR:   F(ref y);
-  // CHECK:STDERR:     ^~~~~
+  // CHECK:STDERR:         ^
+  // CHECK:STDERR: fail_ref_not_durable_ref.carbon:[[@LINE-7]]:6: note: initializing function parameter [InCallToFunctionParam]
+  // CHECK:STDERR: fn F(ref x: i32);
+  // CHECK:STDERR:      ^~~~~~~~~~
   // CHECK:STDERR:
   F(ref y);
 }
@@ -83,14 +86,14 @@ fn F(x: i32);
 fn G() {
   var y: i32 = 0;
 
-  // CHECK:STDERR: fail_unnecessary_ref.carbon:[[@LINE+4]]:24: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
+  // CHECK:STDERR: fail_unnecessary_ref.carbon:[[@LINE+4]]:20: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
   // CHECK:STDERR:   var z: (i32,) = (ref y,);
-  // CHECK:STDERR:                        ^
+  // CHECK:STDERR:                    ^~~~~
   // CHECK:STDERR:
   var z: (i32,) = (ref y,);
-  // CHECK:STDERR: fail_unnecessary_ref.carbon:[[@LINE+7]]:9: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
+  // CHECK:STDERR: fail_unnecessary_ref.carbon:[[@LINE+7]]:5: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
   // CHECK:STDERR:   F(ref y);
-  // CHECK:STDERR:         ^
+  // CHECK:STDERR:     ^~~~~
   // CHECK:STDERR: fail_unnecessary_ref.carbon:[[@LINE-13]]:6: note: initializing function parameter [InCallToFunctionParam]
   // CHECK:STDERR: fn F(x: i32);
   // CHECK:STDERR:      ^~~~~~
@@ -172,7 +175,8 @@ fn G() {
 // CHECK:STDOUT:   %y: ref %i32 = ref_binding y, %y.var
 // CHECK:STDOUT:   %F.ref: %F.type = name_ref F, file.%F.decl [concrete = constants.%F]
 // CHECK:STDOUT:   %y.ref: ref %i32 = name_ref y, %y
-// CHECK:STDOUT:   %F.call: init %empty_tuple.type = call %F.ref(%y.ref)
+// CHECK:STDOUT:   %.loc10: %i32 = ref_tag %y.ref
+// CHECK:STDOUT:   %F.call: init %empty_tuple.type = call %F.ref(%.loc10)
 // CHECK:STDOUT:   %DestroyOp.bound: <bound method> = bound_method %y.var, constants.%DestroyOp
 // CHECK:STDOUT:   %DestroyOp.call: init %empty_tuple.type = call %DestroyOp.bound(%y.var)
 // CHECK:STDOUT:   return
diff --git a/toolchain/check/testdata/interop/cpp/function/reference.carbon b/toolchain/check/testdata/interop/cpp/function/reference.carbon
index 7e2a6da86b7d..9ea345e36829 100644
--- a/toolchain/check/testdata/interop/cpp/function/reference.carbon
+++ b/toolchain/check/testdata/interop/cpp/function/reference.carbon
@@ -364,7 +364,8 @@ fn F() {
 // CHECK:STDOUT:   %Cpp.ref.loc9: <namespace> = name_ref Cpp, imports.%Cpp [concrete = imports.%Cpp]
 // CHECK:STDOUT:   %TakesLValue.ref: %TakesLValue.cpp_overload_set.type = name_ref TakesLValue, imports.%TakesLValue.cpp_overload_set.value [concrete = constants.%TakesLValue.cpp_overload_set.value]
 // CHECK:STDOUT:   %s.ref: ref %S = name_ref s, %s
-// CHECK:STDOUT:   %TakesLValue.call: init %empty_tuple.type = call imports.%TakesLValue.decl(%s.ref)
+// CHECK:STDOUT:   %.loc9: %S = ref_tag %s.ref
+// CHECK:STDOUT:   %TakesLValue.call: init %empty_tuple.type = call imports.%TakesLValue.decl(%.loc9)
 // CHECK:STDOUT:   %S.cpp_destructor.bound: <bound method> = bound_method %s.var, constants.%S.cpp_destructor
 // CHECK:STDOUT:   %S.cpp_destructor.call: init %empty_tuple.type = call %S.cpp_destructor.bound(%s.var)
 // CHECK:STDOUT:   <elided>
diff --git a/toolchain/check/testdata/operators/builtin/ref.carbon b/toolchain/check/testdata/operators/builtin/ref.carbon
index 0e8d08e4179c..035c44209de2 100644
--- a/toolchain/check/testdata/operators/builtin/ref.carbon
+++ b/toolchain/check/testdata/operators/builtin/ref.carbon
@@ -17,9 +17,9 @@ library "[[@TEST_NAME]]";
 
 fn F() {
   var x: () = ();
-  // CHECK:STDERR: fail_discarded_ref.carbon:[[@LINE+4]]:7: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
+  // CHECK:STDERR: fail_discarded_ref.carbon:[[@LINE+4]]:3: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
   // CHECK:STDERR:   ref x;
-  // CHECK:STDERR:       ^
+  // CHECK:STDERR:   ^~~~~
   // CHECK:STDERR:
   ref x;
 }
@@ -30,14 +30,14 @@ library "[[@TEST_NAME]]";
 
 fn F() {
   var x: () = ();
-  // CHECK:STDERR: fail_ref_out_of_place.carbon:[[@LINE+4]]:19: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
+  // CHECK:STDERR: fail_ref_out_of_place.carbon:[[@LINE+4]]:15: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
   // CHECK:STDERR:   let y: () = ref x;
-  // CHECK:STDERR:                   ^
+  // CHECK:STDERR:               ^~~~~
   // CHECK:STDERR:
   let y: () = ref x;
-  // CHECK:STDERR: fail_ref_out_of_place.carbon:[[@LINE+4]]:20: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
+  // CHECK:STDERR: fail_ref_out_of_place.carbon:[[@LINE+4]]:16: error: `ref` tag is not an argument to a `ref` parameter [RefTagNoRefParam]
   // CHECK:STDERR:   let z: () = (ref x);
-  // CHECK:STDERR:                    ^
+  // CHECK:STDERR:                ^~~~~
   // CHECK:STDERR:
   let z: () = (ref x);
 }
diff --git a/toolchain/lower/file_context.cpp b/toolchain/lower/file_context.cpp
index c47187aa90ba..2ee5d367175a 100644
--- a/toolchain/lower/file_context.cpp
+++ b/toolchain/lower/file_context.cpp
@@ -219,6 +219,7 @@ auto FileContext::GetConstant(SemIR::ConstantId const_id,
     case SemIR::ExprCategory::Error:
     case SemIR::ExprCategory::Pattern:
     case SemIR::ExprCategory::Mixed:
+    case SemIR::ExprCategory::RefTagged:
       CARBON_FATAL("Unexpected category {0} for lowered constant {1}", cat,
                    sem_ir().insts().Get(const_inst_id));
   };
diff --git a/toolchain/lower/handle.cpp b/toolchain/lower/handle.cpp
index a8e3259990aa..3d7331ccdde1 100644
--- a/toolchain/lower/handle.cpp
+++ b/toolchain/lower/handle.cpp
@@ -240,6 +240,11 @@ auto HandleInst(FunctionContext& /*context*/, SemIR::InstId /*inst_id*/,
   // Parameters are lowered by `BuildFunctionDefinition`.
 }
 
+auto HandleInst(FunctionContext& context, SemIR::InstId inst_id,
+                SemIR::RefTagExpr inst) -> void {
+  context.SetLocal(inst_id, context.GetValue(inst.expr_id));
+}
+
 auto HandleInst(FunctionContext& context, SemIR::InstId inst_id,
                 SemIR::ReturnSlot inst) -> void {
   context.SetLocal(inst_id, context.GetValue(inst.storage_id));
diff --git a/toolchain/lower/handle_aggregates.cpp b/toolchain/lower/handle_aggregates.cpp
index c1a1a8143212..45a98a13367a 100644
--- a/toolchain/lower/handle_aggregates.cpp
+++ b/toolchain/lower/handle_aggregates.cpp
@@ -62,6 +62,7 @@ static auto GetAggregateElement(FunctionContext& context,
   auto* aggr_value = context.GetValue(aggr_inst_id);
 
   switch (SemIR::GetExprCategory(context.sem_ir(), aggr_inst_id)) {
+    case SemIR::ExprCategory::RefTagged:
     case SemIR::ExprCategory::Error:
     case SemIR::ExprCategory::NotExpr:
     case SemIR::ExprCategory::Pattern:
diff --git a/toolchain/sem_ir/file.cpp b/toolchain/sem_ir/file.cpp
index 3b758db92ea1..3901c87dd0f4 100644
--- a/toolchain/sem_ir/file.cpp
+++ b/toolchain/sem_ir/file.cpp
@@ -157,43 +157,6 @@ auto File::OutputYaml(bool include_singletons) const -> Yaml::OutputMapping {
   });
 }
 
-auto File::CollectRefTagsNeeded() const -> Set<SemIR::InstId> {
-  CARBON_CHECK(!has_errors_);
-  Set<SemIR::InstId> ref_tags_needed;
-  for (auto [id, inst] : insts_.enumerate()) {
-    if (inst.kind() != SemIR::InstKind::Call) {
-      continue;
-    }
-    auto call_inst = inst.As<SemIR::Call>();
-    auto callee = SemIR::GetCallee(*this, call_inst.callee_id);
-    CARBON_KIND_SWITCH(callee) {
-      case CARBON_KIND(SemIR::CalleeError _):
-        break;
-      case CARBON_KIND(SemIR::CalleeNonFunction _):
-        break;
-      case CARBON_KIND(SemIR::CalleeCppOverloadSet _): {
-        // TODO: Perform validation here once we model C++ ref parameters as
-        // Carbon ref parameters.
-        break;
-      }
-      case CARBON_KIND(SemIR::CalleeFunction fn): {
-        auto function = functions_.Get(fn.function_id);
-        auto args = inst_blocks_.GetOrEmpty(call_inst.args_id);
-        for (auto param_id : llvm::concat<const InstId>(
-                 inst_blocks_.GetOrEmpty(function.implicit_param_patterns_id),
-                 inst_blocks_.GetOrEmpty(function.param_patterns_id))) {
-          if (auto ref_param_pattern =
-                  insts_.TryGetAs<SemIR::RefParamPattern>(param_id)) {
-            ref_tags_needed.Insert(args[ref_param_pattern->index.index]);
-          }
-        }
-        break;
-      }
-    }
-  }
-  return ref_tags_needed;
-}
-
 auto File::CollectMemUsage(MemUsage& mem_usage, llvm::StringRef label) const
     -> void {
   mem_usage.Collect(MemUsage::ConcatLabel(label, "allocator_"), allocator_);
diff --git a/toolchain/sem_ir/file.h b/toolchain/sem_ir/file.h
index 997103f2185b..2630ee6e8807 100644
--- a/toolchain/sem_ir/file.h
+++ b/toolchain/sem_ir/file.h
@@ -94,12 +94,6 @@ class File : public Printable<File> {
   }
   auto OutputYaml(bool include_singletons) const -> Yaml::OutputMapping;
 
-  // Returns the set of all insts corresponding to expressions that are used
-  // in positions where a `ref` tag is needed. Should only be called if
-  // has_errors is false. This is intended for validation purposes, and should
-  // only be called if !NDEBUG, because it walks the entire IR.
-  auto CollectRefTagsNeeded() const -> Set<SemIR::InstId>;
-
   // Collects memory usage of members.
   auto CollectMemUsage(MemUsage& mem_usage, llvm::StringRef label) const
       -> void;
diff --git a/toolchain/sem_ir/formatter.cpp b/toolchain/sem_ir/formatter.cpp
index a69fdd96361e..44b49563ccb4 100644
--- a/toolchain/sem_ir/formatter.cpp
+++ b/toolchain/sem_ir/formatter.cpp
@@ -1005,6 +1005,7 @@ auto Formatter::FormatInstLhs(InstId inst_id, Inst inst) -> void {
       case ExprCategory::Value:
       case ExprCategory::Pattern:
       case ExprCategory::Mixed:
+      case ExprCategory::RefTagged:
         break;
       case ExprCategory::DurableRef:
       case ExprCategory::EphemeralRef:
diff --git a/toolchain/sem_ir/inst_kind.def b/toolchain/sem_ir/inst_kind.def
index 70a51f3b57b3..e85df792061c 100644
--- a/toolchain/sem_ir/inst_kind.def
+++ b/toolchain/sem_ir/inst_kind.def
@@ -109,6 +109,7 @@ CARBON_SEM_IR_INST_KIND(RefBinding)
 CARBON_SEM_IR_INST_KIND(RefBindingPattern)
 CARBON_SEM_IR_INST_KIND(RefParam)
 CARBON_SEM_IR_INST_KIND(RefParamPattern)
+CARBON_SEM_IR_INST_KIND(RefTagExpr)
 CARBON_SEM_IR_INST_KIND(RefineTypeAction)
 CARBON_SEM_IR_INST_KIND(RequireCompleteType)
 CARBON_SEM_IR_INST_KIND(RequireImplsDecl)
diff --git a/toolchain/sem_ir/inst_kind.h b/toolchain/sem_ir/inst_kind.h
index e83ee0566f14..d3974782d2f1 100644
--- a/toolchain/sem_ir/inst_kind.h
+++ b/toolchain/sem_ir/inst_kind.h
@@ -42,7 +42,10 @@ enum class ExprCategory : int8_t {
   // and struct literals, where the subexpressions for different elements can
   // have different categories.
   Mixed,
-  Last = Mixed
+  // This instruction is a `RefTagExpr`, and so its semantics (including its
+  // expression category) depends on the usage context.
+  RefTagged,
+  Last = RefTagged
 };
 
 // The computation used to determine the expression category for an instruction,
diff --git a/toolchain/sem_ir/typed_insts.h b/toolchain/sem_ir/typed_insts.h
index c20fc0b95f19..5d66e2b85233 100644
--- a/toolchain/sem_ir/typed_insts.h
+++ b/toolchain/sem_ir/typed_insts.h
@@ -1368,6 +1368,26 @@ struct RefParamPattern {
   CallParamIndex index;
 };
 
+// A `ref x` expression. The semantics of this instruction depend on the usage
+// context:
+// - As an argument to a `ref` parameter, it evaluates to `x`, but requires
+//   `x` to be a durable reference expression.
+// - In a return type expression or form literal, it evaluates to a `Core.Form`
+//   value representing a reference to `x`, which must be a type.
+// - In any other context, it's an error.
+//
+// See issue #6342 for background.
+struct RefTagExpr {
+  static constexpr auto Kind =
+      InstKind::RefTagExpr.Define<Parse::PrefixOperatorRefId>(
+          {.ir_name = "ref_tag",
+           .expr_category = ExprCategory::RefTagged,
+           .constant_kind = InstConstantKind::Never});
+
+  TypeId type_id;
+  InstId expr_id;
+};
+
 // Requires a type to be complete. This is only created for generic types and
 // produces a witness that the type is complete.
 //