Fix ASan CI failures due to preserve_none (#8045)

These only show up post-merge as we're only running ASan there.

This disables the recent `[[clang::preserve_none]]` optimization when
ASan is enabled as there is an LLVM bug on AArch64 that causes register
corruption with that calling convention. This is factored into a common
helper library so we can use it in other places as well.

Assisted-by: Antigravity with Gemini

.clang-format

@@ -7,6 +7,7 @@ BasedOnStyle: Google
 AllowShortBlocksOnASingleLine: 'false'
 AllowShortIfStatementsOnASingleLine: Never
 AllowShortLoopsOnASingleLine: 'false'
+AttributeMacros: ['CARBON_PRESERVE_NONE']
 DerivePointerAlignment: 'false'
 ExperimentalAutoDetectBinPacking: 'false'
 FixNamespaceComments: 'true'

common/BUILD

@@ -546,6 +546,25 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "preserve_none",
+    hdrs = ["preserve_none.h"],
+    deps = [
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_test(
+    name = "preserve_none_test",
+    size = "small",
+    srcs = ["preserve_none_test.cpp"],
+    deps = [
+        ":preserve_none",
+        "//testing/base:gtest_main",
+        "@googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "pretty_stack_trace_function",
     hdrs = ["pretty_stack_trace_function.h"],

common/preserve_none.h

@@ -0,0 +1,25 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_COMMON_PRESERVE_NONE_H_
+#define CARBON_COMMON_PRESERVE_NONE_H_
+
+#include "llvm/Support/Compiler.h"
+
+// Applies `[[clang::preserve_none]]` when supported and safe.
+//
+// Disabled when building with AddressSanitizer on AArch64 to work around an
+// LLVM backend bug: when sanitizer stack instrumentation combines dynamic stack
+// allocation with stack realignment, AArch64 reserves `x19` as the stack base
+// pointer (`hasBasePointer`) and fails to save and restore `x19` across calls
+// to `[[clang::preserve_none]]` callees that clobber `x19`.
+#if __has_cpp_attribute(clang::preserve_none) &&     \
+    !((defined(__aarch64__) || defined(_M_ARM64)) && \
+      (LLVM_ADDRESS_SANITIZER_BUILD || LLVM_HWADDRESS_SANITIZER_BUILD))
+#define CARBON_PRESERVE_NONE [[clang::preserve_none]]
+#else
+#define CARBON_PRESERVE_NONE
+#endif
+
+#endif  // CARBON_COMMON_PRESERVE_NONE_H_

common/preserve_none_test.cpp

@@ -0,0 +1,48 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "common/preserve_none.h"
+
+#include <gtest/gtest.h>
+
+#include <array>
+
+namespace Carbon {
+namespace {
+
+using DispatchFn = auto(int value, int remaining) CARBON_PRESERVE_NONE->int;
+
+CARBON_PRESERVE_NONE static auto DispatchDone(int value, int /*remaining*/)
+    -> int {
+  return value;
+}
+
+CARBON_PRESERVE_NONE static auto DispatchStep(int value, int remaining) -> int;
+
+constexpr std::array<DispatchFn*, 2> DispatchTable = {
+    &DispatchDone,
+    &DispatchStep,
+};
+
+CARBON_PRESERVE_NONE static auto DispatchStep(int value, int remaining) -> int {
+  // Touch enough temporaries that register allocation exercises caller-saved
+  // registers under `preserve_none`.
+  int next = (value * 3 + 1) ^ (remaining * 7);
+  int next_remaining = remaining - 1;
+  [[clang::musttail]] return DispatchTable[next_remaining != 0 ? 1 : 0](
+      next, next_remaining);
+}
+
+TEST(PreserveNoneTest, TailCallDispatchWithAlignedCallerStack) {
+  // Overalign a local buffer so sanitizer stack instrumentation exercises stack
+  // realignment alongside `CARBON_PRESERVE_NONE` calls.
+  alignas(64) std::array<int, 8> scratch = {1, 2, 3, 4, 5, 6, 7, 8};
+  DispatchFn* entry = &DispatchStep;
+  int result = entry(scratch[0], 16);
+  EXPECT_EQ(scratch[7], 8);
+  EXPECT_NE(result, 0);
+}
+
+}  // namespace
+}  // namespace Carbon

toolchain/lex/BUILD

@@ -197,6 +197,7 @@ cc_library(
         ":token_kind",
         ":tokenized_buffer",
         "//common:check",
+        "//common:preserve_none",
         "//common:vlog",
         "//toolchain/base:kind_switch",
         "//toolchain/base:shared_value_stores",

toolchain/lex/lex.cpp

@@ -12,6 +12,7 @@
 #include <utility>
 
 #include "common/check.h"
+#include "common/preserve_none.h"
 #include "common/vlog.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -536,11 +537,6 @@ static auto ScanForIdentifierPrefix(llvm::StringRef text) -> llvm::StringRef {
   return ScanForIdentifierPrefixScalar(text, 0);
 }
 
-using DispatchFunctionT = auto(Lexer& lexer, llvm::StringRef source_text,
-                               ssize_t position) [[clang::preserve_none]]
--> void;
-using DispatchTableT = std::array<DispatchFunctionT*, 256>;
-
 static constexpr std::array<TokenKind, 256> OneCharTokenKindTable = [] {
   std::array<TokenKind, 256> table = {};
 #define CARBON_ONE_CHAR_SYMBOL_TOKEN(TokenName, Spelling) \
@@ -561,12 +557,16 @@ static constexpr std::array<TokenKind, 256> OneCharTokenKindTable = [] {
 // build efficient dispatch tables out of them. All of them end by doing a
 // must-tail return call to this routine. It handles continuing the dispatch
 // chain.
-[[clang::preserve_none]] static auto DispatchNext(Lexer& lexer,
+CARBON_PRESERVE_NONE static auto DispatchNext(Lexer& lexer,
-                                                  llvm::StringRef source_text,
+                                              llvm::StringRef source_text,
-                                                  ssize_t position) -> void;
+                                              ssize_t position) -> void;
 
-[[clang::preserve_none]] static auto DispatchLexError(
+using DispatchFunctionT = decltype(DispatchNext);
-    Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void {
+using DispatchTableT = std::array<DispatchFunctionT*, 256>;
+
+CARBON_PRESERVE_NONE static auto DispatchLexError(Lexer& lexer,
+                                                  llvm::StringRef source_text,
+                                                  ssize_t position) -> void {
   Lexer::LexResult result = lexer.LexError(source_text, position);
   CARBON_CHECK(result, "Failed to form a token!");
   // NOLINTNEXTLINE(readability-avoid-return-with-void-value): For musttail.
@@ -577,7 +577,7 @@ static constexpr std::array<TokenKind, 256> OneCharTokenKindTable = [] {
 // lexes a token. If no token is matched, they tail-dispatch to the error token
 // handler; otherwise they continue dispatching the next token.
 #define CARBON_DISPATCH_LEX_TOKEN(LexMethod)                                   \
-  [[clang::preserve_none]] static auto Dispatch##LexMethod(                    \
+  CARBON_PRESERVE_NONE static auto Dispatch##LexMethod(                        \
       Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void {   \
     if (LLVM_LIKELY(lexer.LexMethod(source_text, position))) {                 \
       /* NOLINTNEXTLINE(readability-avoid-return-with-void-value) */           \
@@ -591,8 +591,9 @@ CARBON_DISPATCH_LEX_TOKEN(LexKeywordOrIdentifier)
 CARBON_DISPATCH_LEX_TOKEN(LexNumericLiteral)
 CARBON_DISPATCH_LEX_TOKEN(LexStringLiteral)
 
-[[clang::preserve_none]] static auto DispatchLexHash(
+CARBON_PRESERVE_NONE static auto DispatchLexHash(Lexer& lexer,
-    Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void {
+                                                 llvm::StringRef source_text,
+                                                 ssize_t position) -> void {
   if (lexer.LexHash(source_text, position)) {
     // NOLINTNEXTLINE(readability-avoid-return-with-void-value): For musttail.
     [[clang::musttail]] return DispatchNext(lexer, source_text, position);
@@ -604,7 +605,7 @@ CARBON_DISPATCH_LEX_TOKEN(LexStringLiteral)
 
 // A set of custom dispatch functions that preselect the symbol token to lex.
 #define CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexMethod)                          \
-  [[clang::preserve_none]] static auto Dispatch##LexMethod##SymbolToken(     \
+  CARBON_PRESERVE_NONE static auto Dispatch##LexMethod##SymbolToken(         \
       Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void { \
     Lexer::LexResult result = lexer.LexMethod##SymbolToken(                  \
         source_text,                                                         \
@@ -621,7 +622,7 @@ CARBON_DISPATCH_LEX_SYMBOL_TOKEN(LexClosing)
 // Define a set of non-token dispatch functions that handle things like
 // whitespace and comments.
 #define CARBON_DISPATCH_LEX_NON_TOKEN(LexMethod)                             \
-  [[clang::preserve_none]] static auto Dispatch##LexMethod(                  \
+  CARBON_PRESERVE_NONE static auto Dispatch##LexMethod(                      \
       Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void { \
     lexer.LexMethod(source_text, position);                                  \
     [[clang::musttail]] return DispatchNext(lexer, source_text, position);   \
@@ -630,7 +631,7 @@ CARBON_DISPATCH_LEX_NON_TOKEN(LexHorizontalWhitespace)
 CARBON_DISPATCH_LEX_NON_TOKEN(LexVerticalWhitespace)
 CARBON_DISPATCH_LEX_NON_TOKEN(LexCR)
 
-[[clang::preserve_none]] static auto DispatchLexCommentOrSlash(
+CARBON_PRESERVE_NONE static auto DispatchLexCommentOrSlash(
     Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void {
   CARBON_DCHECK(source_text[position] == '/');
 
@@ -753,8 +754,9 @@ static constexpr auto MakeDispatchTable() -> DispatchTableT {
 
 static constexpr DispatchTableT DispatchTable = MakeDispatchTable();
 
-[[clang::preserve_none]] static auto DispatchFileEnd(
+CARBON_PRESERVE_NONE static auto DispatchFileEnd(Lexer& lexer,
-    Lexer& lexer, llvm::StringRef source_text, ssize_t position) -> void {
+                                                 llvm::StringRef source_text,
+                                                 ssize_t position) -> void {
   if (lexer.has_dump_sem_ir_ranges()) {
     // Incomplete ranges will use the next token for their end; we want that to
     // be `FileEnd` in this case, so check before adding `FileEnd`. The argument
@@ -770,9 +772,9 @@ static constexpr DispatchTableT DispatchTable = MakeDispatchTable();
   lexer.LexFileEnd(source_text, position);
 }
 
-[[clang::preserve_none]] static auto DispatchNext(Lexer& lexer,
+CARBON_PRESERVE_NONE static auto DispatchNext(Lexer& lexer,
-                                                  llvm::StringRef source_text,
+                                              llvm::StringRef source_text,
-                                                  ssize_t position) -> void {
+                                              ssize_t position) -> void {
   if (LLVM_LIKELY(position < static_cast<ssize_t>(source_text.size()))) {
     // The common case is to tail recurse based on the next character. Note
     // that because this is a must-tail return, this cannot fail to tail-call