From 8c85fa2744f91ce1dee56c1625625af8ba247c43 Mon Sep 17 00:00:00 2001
From: Vineeth B V <37930821+vinsdragonis@users.noreply.github.com>
Date: Sat, 20 Aug 2022 02:09:11 +0530
Subject: [PATCH] Added unicode support (#2027)

Can parse Unicode literals following the format \u{HHHH...} in a string
---
 common/string_helpers.cpp               | 37 +++++++++++++++++++++++--
 common/string_helpers_test.cpp          | 27 ++++++++++++++++++
 explorer/testdata/string/unicode.carbon | 23 +++++++++++++++
 3 files changed, 85 insertions(+), 2 deletions(-)
 create mode 100644 explorer/testdata/string/unicode.carbon

diff --git a/common/string_helpers.cpp b/common/string_helpers.cpp
index 15f1b3a2357b..7548d574c407 100644
--- a/common/string_helpers.cpp
+++ b/common/string_helpers.cpp
@@ -10,6 +10,7 @@
 #include "common/check.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ConvertUTF.h"
 
 namespace Carbon {
 
@@ -81,8 +82,40 @@ auto UnescapeStringLiteral(llvm::StringRef source, const int hashtag_num,
           ret.push_back(16 * *c1 + *c2);
           break;
         }
-        case 'u':
-          CARBON_FATAL() << "\\u is not yet supported in string literals";
+        case 'u': {
+          ++i;
+          if (i >= source.size() || source[i] != '{') {
+            return std::nullopt;
+          }
+          unsigned int unicode_int = 0;
+          ++i;
+          int original_i = i;
+          while (i < source.size() && source[i] != '}') {
+            std::optional<char> hex_val = FromHex(source[i]);
+            if (hex_val == std::nullopt) {
+              return std::nullopt;
+            }
+            unicode_int = unicode_int << 4;
+            unicode_int += hex_val.value();
+            ++i;
+            if (i - original_i > 8) {
+              return std::nullopt;
+            }
+          }
+          if (i >= source.size()) {
+            return std::nullopt;
+          }
+          if (i - original_i == 0) {
+            return std::nullopt;
+          }
+          char utf8_buf[4];
+          char* utf8_end = &utf8_buf[0];
+          if (!llvm::ConvertCodePointToUTF8(unicode_int, utf8_end)) {
+            return std::nullopt;
+          }
+          ret.append(utf8_buf, utf8_end - utf8_buf);
+          break;
+        }
         case '\n':
           if (!is_block_string) {
             return std::nullopt;
diff --git a/common/string_helpers_test.cpp b/common/string_helpers_test.cpp
index c7251528cf57..1854035936c8 100644
--- a/common/string_helpers_test.cpp
+++ b/common/string_helpers_test.cpp
@@ -30,6 +30,9 @@ TEST(UnescapeStringLiteral, Valid) {
   EXPECT_THAT(UnescapeStringLiteral("\\x12"), Optional(Eq("\x12")));
   EXPECT_THAT(UnescapeStringLiteral("test", 1), Optional(Eq("test")));
   EXPECT_THAT(UnescapeStringLiteral("test\\#n", 1), Optional(Eq("test\n")));
+  EXPECT_THAT(UnescapeStringLiteral(
+                  "r\\u{000000E9}al \\u{2764}\\u{FE0F}\\u{1F50A}!\\u{10FFFF}"),
+              Optional(Eq("réal ❤️🔊!􏿿")));
 }
 
 TEST(UnescapeStringLiteral, Invalid) {
@@ -48,6 +51,30 @@ TEST(UnescapeStringLiteral, Invalid) {
   EXPECT_THAT(UnescapeStringLiteral("\\#00", 1), Eq(std::nullopt));
 }
 
+TEST(UnescapeStringLiteral, InvalidUnicodes) {
+  // Various incomplete Unicode specifiers
+  EXPECT_THAT(UnescapeStringLiteral("\\u"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\u1"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\uz"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\u{"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\u{z"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\u{E9"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\u{E9z"), Eq(std::nullopt));
+  EXPECT_THAT(UnescapeStringLiteral("\\u{}"), Eq(std::nullopt));
+
+  // invalid characters in unicode
+  EXPECT_THAT(UnescapeStringLiteral("\\u{z}"), Eq(std::nullopt));
+
+  // lowercase hexadecimal
+  EXPECT_THAT(UnescapeStringLiteral("\\u{e9}"), Eq(std::nullopt));
+
+  // Codepoint number too high
+  EXPECT_THAT(UnescapeStringLiteral("\\u{110000}"), Eq(std::nullopt));
+
+  // codepoint more than 8 hex digits
+  EXPECT_THAT(UnescapeStringLiteral("\\u{FF000000E9}"), Eq(std::nullopt));
+}
+
 TEST(UnescapeStringLiteral, Nul) {
   std::optional<std::string> str = UnescapeStringLiteral("a\\0b");
   ASSERT_NE(str, std::nullopt);
diff --git a/explorer/testdata/string/unicode.carbon b/explorer/testdata/string/unicode.carbon
new file mode 100644
index 000000000000..d4cd296e3a4f
--- /dev/null
+++ b/explorer/testdata/string/unicode.carbon
@@ -0,0 +1,23 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// RUN: %{explorer} %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes=false %s
+// RUN: %{explorer} --parser_debug --trace_file=- %s 2>&1 | \
+// RUN:   %{FileCheck} --match-full-lines --allow-unused-prefixes %s
+// AUTOUPDATE: %{explorer} %s
+// CHECK: result: 0
+
+package ExplorerTest api;
+
+fn CompareStr(s: String) -> i32 {
+  if (s == "HELLO WORLD!!") {
+    return 0;
+  }
+  return 1;
+}
+
+fn Main() -> i32 {
+  return CompareStr("\u{0048}\u{0045}\u{004C}\u{004C}\u{004F} \u{0057}\u{004F}\u{0052}\u{004C}\u{0044}\u{0021}\u{21}");
+}