[highlightjs] Add support for var and let. (#1366)

This required reworking how `=` and `;` were handled as well as more
general surgery on patterns.

This looks for initializers at the top level of parameters and at the
end of `var` or `let` declarations. It supports fancy nested `var`
patterns within `let` declarations, etc.

Co-authored-by: josh11b <josh11b@users.noreply.github.com>

highlighting/highlightjs_carbon_lang.js

@@ -9,7 +9,6 @@
  * Category: common, system
  * Website: https://github.com/carbon-language/carbon-lang/
  *
- * TODO: local `var` and `let` declarations.
  * TODO: struct literals.
  * TODO: abstract, virtual, impl on methods.
  * TODO: private, protected.
@@ -88,7 +87,7 @@ export default function (hljs) {
   // But it is harmless to give them nicer syntax highlighting. Eventually, as
   // the language settles, we can explore specifically highlighting operators
   // that aren't valid as errors.
-  const PUNCTUATION = [/->/, /\./, /:!?/, /;/];
+  const PUNCTUATION = [/->/, /\./];
   const OPERATORS = [
     />>=/,
     /<=>/,
@@ -117,7 +116,6 @@ export default function (hljs) {
     /&/,
     /\\/,
     /\^/,
-    /=/,
     /!/,
     />/,
     /</,
@@ -131,6 +129,13 @@ export default function (hljs) {
     /~/,
   ];
 
+  const COMMA_SEPARATOR = {
+    begin: [/,/, /\s*/],
+    beginScope: {
+      1: 'punctuation',
+    },
+  };
+
   // The core expression patterns.
   const NUMBER_LITERAL = {
     scope: 'number',
@@ -204,15 +209,25 @@ export default function (hljs) {
     beginScope: 'punctuation',
     end: /\)/,
     endScope: 'punctuation',
-    contains: ['self', ...UNPARENTHESIZED_EXPRESSION],
+    contains: [
+      'self',
+      // Tuple literals are comma-separated parenthesized expressions.
+      COMMA_SEPARATOR,
+      ...UNPARENTHESIZED_EXPRESSION,
+    ],
   };
   const EXPRESSION = [PARENTHESIZED_EXPRESSION, ...UNPARENTHESIZED_EXPRESSION];
 
   // The pattern patterns, including comma-separated sequences.
   const VALUE_PATTERN = {
     scope: 'carbon-value-pattern',
-    begin: /(?!,|\)|\])/,
-    end: /,|\)|\]/,
+    // This begins to match on *any* (zero-length) input that *isn't* one of the
+    // terminating sequences. This is important so that once we end one of these
+    // patterns we don't immediately begin it again. The mode that introduces
+    // this has already determined that a value pattern can start immediately,
+    // so we don't need other characters in the beginning.
+    begin: /(?![=,;\)\]\}])/,
+    end: /\s*[=,;\)\]\}]/,
     returnEnd: true,
     contains: [...EXPRESSION],
   };
@@ -236,7 +251,7 @@ export default function (hljs) {
         },
       },
     ],
-    end: /,|\)|\]/,
+    end: /\s*[=,;\)\]\}]/,
     returnEnd: true,
     contains: [...EXPRESSION],
   };
@@ -248,7 +263,7 @@ export default function (hljs) {
       3: 'keyword',
       4: 'punctuation',
     },
-    end: /,|\)|\]/,
+    end: /[=,;\)\]\}]/,
     returnEnd: true,
     contains: [...EXPRESSION],
   };
@@ -259,28 +274,44 @@ export default function (hljs) {
     beginScope: 'punctuation',
     end: /\)/,
     endScope: 'punctuation',
-    contains: [
-      'self',
-      {
-        begin: [/,/, /\s*/],
-        beginScope: {
-          1: 'punctuation',
-        },
-      },
-      ...UNPARENTHESIZED_PATTERNS,
-    ],
+    contains: ['self', ...UNPARENTHESIZED_PATTERNS],
   };
   const PATTERN_SEQUENCE = [
-    {
-      begin: [/,/, /\s*/],
-      beginScope: {
-        1: 'punctuation',
-      },
-    },
+    COMMA_SEPARATOR,
     PARENTHESIZED_PATTERN,
     ...UNPARENTHESIZED_PATTERNS,
   ];
 
+  // Both variable and `let` value declarations.
+  const INITIALIZER = {
+    scope: 'carbon-initializer',
+    begin: [/\s*/, /=/, /\s*/],
+    beginScope: {
+      2: 'punctuation',
+    },
+    end: /[,;\)\]\}]/,
+    returnEnd: true,
+    contains: [...EXPRESSION],
+  };
+  const VARIABLE = {
+    variants: [
+      {
+        scope: 'carbon-var-declaration',
+        begin: [/\bvar/, /\s+/],
+      },
+      {
+        scope: 'carbon-let-declaration',
+        begin: [/\blet/, /\s+/],
+      },
+    ],
+    beginScope: {
+      1: 'keyword',
+    },
+    end: /;/,
+    endScope: 'punctuation',
+    contains: [PARENTHESIZED_PATTERN, BINDING_PATTERN, INITIALIZER],
+  };
+
   // Parameters.
   const PARAMETER_LIST = {
     variants: [
@@ -297,7 +328,7 @@ export default function (hljs) {
     ],
     beginScope: 'punctuation',
     endScope: 'punctuation',
-    contains: [...PATTERN_SEQUENCE],
+    contains: [INITIALIZER, ...PATTERN_SEQUENCE],
   };
 
   // Functions.
@@ -384,8 +415,14 @@ export default function (hljs) {
   // Statements -- very loosely. We bundle together top-level declaration
   // constructs, blocks, and statements.
   const STATEMENTS = [
+    VARIABLE,
     FUNCTION,
     CLASS,
+    {
+      // We only match the assignment operator at the statement level.
+      scope: 'operator',
+      match: /=/,
+    },
     {
       scope: 'punctuation',
       match: /;/,

highlighting/highlightjs_example.html

@@ -23,6 +23,7 @@ import Carbon library "Printing";
 fn Partition[T:! Comparable & Movable](s: Slice(T))
       -> i64 {
   var i: i64 = -1;
+  let (x: i64, var y: i64) = (1, 2);
 
   for (e: T in s) {
     if (e <= s.Last()) {
@@ -49,7 +50,7 @@ constraint WidgetC {
 }
 
 base class Widget {
-  fn Print[addr me: Self*](var i: i64, x: f32, _: bool);
+  fn Print[addr me: Self*](var i: i64, x: f32 = 4.2, _: bool);
 }
 
 fn Widget.Print[addr me: Self*](var i: i64, x: f32, _: bool, 47) {