[JavaScript] Basic support for syntax highlighting in tagged templates (

#4019)

* [JavaScript] Syntax highlighting in tagged templates

This commit adds support for syntax highlighting of CSS/JS/JSON/HTML
within tagged template strings.

* [JavaScript] Add fallback for unknown tags

* [JavaScript] Fix unknown tagged templates

This commit...

1. adds a `literal-string-templates` context to include templates
   into contexts without popping it off stack.
2. adjusts tag scopes to already existing `variable.function.tagged-template`.
3. adds support for whitespace between template tag and punctuation
4. drops supportt for block quotes between template tag and punctuation
   for simplicity reasons. It's jugdged unlikely enough to appear in real
   world code.

* [JavaScript] Auto-indent tagged template string content

This commit adjusts syntax definition and indentation rules to ensure
content of tagged templates is indented, automatically.

It includes consuming all trailing/leading whitespace after opening and
in front of closing string punctuation, to ensure JavaScript indentation
rules being applied, when hitting enter after an opening template punctuation.

* [JavaScript] Improve embedded scope source boundaries.

This commit strips embedded source scope after opening and before closing
string punctuation only, if those are located on a separate line.

In single line tagged template strings, embedded scope is applied to leading
and trailing whitespace as well.

JavaScript/Embeddings/CSS (for JS template).sublime-syntax

@@ -0,0 +1,23 @@
+%YAML 1.2
+---
+# http://www.sublimetext.com/docs/syntax.html
+# highlight tagged template strings
+scope: source.css.js-template
+version: 2
+hidden: true
+
+extends: Packages/CSS/CSS.sublime-syntax
+
+variables:
+
+    ident_start: (?:{{nmstart}}|\${)
+
+contexts:
+
+  prototype:
+    - meta_prepend: true
+    - include: scope:source.js#text-interpolations
+
+  strings-content:
+    - meta_prepend: true
+    - include: scope:source.js#string-interpolations

JavaScript/Embeddings/HTML (for JS template).sublime-syntax

@@ -0,0 +1,122 @@
+%YAML 1.2
+---
+# http://www.sublimetext.com/docs/syntax.html
+# highlight tagged template strings
+scope: text.html.js-template
+version: 2
+hidden: true
+
+extends: Packages/HTML/HTML.sublime-syntax
+
+variables:
+
+  tag_name_start: (?:[A-Za-z]|\${)
+
+contexts:
+
+  prototype:
+    - meta_prepend: true
+    - include: scope:source.js#text-interpolations
+
+  cdata-content:
+    - meta_prepend: true
+    - include: scope:source.js#string-interpolations
+
+  script-javascript-content:
+    - meta_include_prototype: false
+    - match: '{{script_content_begin}}'
+      captures:
+        1: comment.block.html punctuation.definition.comment.begin.html
+      pop: 1  # make sure to match only once
+      embed: scope:source.js.js-template
+      embed_scope: source.js.embedded.html
+      escape: '{{script_content_end}}'
+      escape_captures:
+        1: source.js.embedded.html
+        2: comment.block.html punctuation.definition.comment.end.html
+        3: source.js.embedded.html
+        4: comment.block.html punctuation.definition.comment.end.html
+
+  script-json-content:
+    - meta_include_prototype: false
+    - match: '{{script_content_begin}}'
+      captures:
+        1: comment.block.html punctuation.definition.comment.begin.html
+      pop: 1  # make sure to match only once
+      embed: scope:source.json.js-template
+      embed_scope: source.json.embedded.html
+      escape: '{{script_content_end}}'
+      escape_captures:
+        1: source.json.embedded.html
+        2: comment.block.html punctuation.definition.comment.end.html
+        3: source.json.embedded.html
+        4: comment.block.html punctuation.definition.comment.end.html
+
+  style-css-content:
+    - meta_include_prototype: false
+    - match: '{{style_content_begin}}'
+      captures:
+        1: comment.block.html punctuation.definition.comment.begin.html
+      pop: 1  # make sure to match only once
+      embed: scope:source.css.js-template
+      embed_scope: source.css.embedded.html
+      escape: '{{style_content_end}}'
+      escape_captures:
+        1: source.css.embedded.html
+        2: comment.block.html punctuation.definition.comment.end.html
+        3: source.css.embedded.html
+        4: comment.block.html punctuation.definition.comment.end.html
+
+  tag-event-attribute-value:
+    - match: \"
+      scope:
+        meta.string.html string.quoted.double.html
+        punctuation.definition.string.begin.html
+      embed: scope:source.js.js-template
+      embed_scope: meta.string.html meta.embedded.html source.js.embedded.html
+      escape: \"
+      escape_captures:
+        0: meta.string.html string.quoted.double.html
+           punctuation.definition.string.end.html
+    - match: \'
+      scope:
+        meta.string.html string.quoted.single.html
+        punctuation.definition.string.begin.html
+      embed: scope:source.js.js-template
+      embed_scope: meta.string.html meta.embedded.html source.js.embedded.html
+      escape: \'
+      escape_captures:
+        0: meta.string.html string.quoted.single.html
+           punctuation.definition.string.end.html
+    - include: else-pop
+
+  tag-style-attribute-value:
+    - match: \"
+      scope:
+        meta.string.html string.quoted.double.html
+        punctuation.definition.string.begin.html
+      embed: scope:source.css.js-template#rule-list-body
+      embed_scope: meta.string.html meta.embedded.html source.css.embedded.html
+      escape: \"
+      escape_captures:
+        0: meta.string.html string.quoted.double.html
+           punctuation.definition.string.end.html
+    - match: \'
+      scope:
+        meta.string.html string.quoted.single.html
+        punctuation.definition.string.begin.html
+      embed: scope:source.css.js-template#rule-list-body
+      embed_scope: meta.string.html meta.embedded.html source.css.embedded.html
+      escape: \'
+      escape_captures:
+        0: meta.string.html string.quoted.single.html
+           punctuation.definition.string.end.html
+    - include: else-pop
+
+  tag-attribute-value-content:
+    - meta_prepend: true
+    - include: scope:source.js#string-interpolations
+
+  strings-common-content:
+    - meta_prepend: true
+    - include: scope:source.js#string-interpolations

JavaScript/Embeddings/JSON (JS template).sublime-syntax

@@ -0,0 +1,18 @@
+%YAML 1.2
+---
+# http://www.sublimetext.com/docs/syntax.html
+# highlight tagged template strings
+scope: source.json.js-template
+version: 2
+hidden: true
+
+extends: Packages/JSON/JSON.sublime-syntax
+
+contexts:
+  prototype:
+    - meta_prepend: true
+    - include: scope:source.js#text-interpolations
+
+  string-prototype:
+    - meta_prepend: true
+    - include: scope:source.js#string-interpolations

JavaScript/Embeddings/JavaScript (JS template).sublime-syntax

@@ -0,0 +1,19 @@
+%YAML 1.2
+---
+# http://www.sublimetext.com/docs/syntax.html
+# highlight tagged template strings
+scope: source.js.js-template
+version: 2
+hidden: true
+
+extends: Packages/JavaScript/JavaScript.sublime-syntax
+
+contexts:
+
+  prototype:
+    - meta_prepend: true
+    - include: scope:source.js#text-interpolations
+
+  string-content:
+    - meta_prepend: true
+    - include: scope:source.js#string-interpolations

JavaScript/Indentation Rules.tmPreferences

@@ -12,6 +12,8 @@
 			(?:
 			# dedent closing braces
 			  \}
+			# dedent closing tagged templates
+			| `
 			# detent `case ... :`
 			| case\b
 			# detent `default:`
@@ -27,6 +29,8 @@
 			# indent after opening braces (may be followed by whitespace or comments)
 			# but exclude lines such as `extern "C" {`
 			  .* \{ (?: \s* /\*.*\*/ )* \s* (?: //.* )? $
+			# indent after opening tagged template: e.g.: "css`"
+			| .* \w+ \s* `
 			# indent after `case ... :`
 			| case\b
 			# indent after `default:`

JavaScript/JavaScript.sublime-syntax

@@ -107,6 +107,9 @@ variables:
   # '@' followed by a pattern like \S but excluding literal '*' and '@'.
   jsdoc_block_tag: \@[^\n\t\f\v *@]+
 
+  leading_wspace: (?:^\s*)
+  trailing_wspace: (?:\s*$\n?)
+
 contexts:
   main:
     - meta_include_prototype: false # don't match comments before shebang
@@ -1011,8 +1014,7 @@ contexts:
         - include: decorator-name
         - include: object-property
 
-    - match: (?=`)
-      push: literal-string-template
+    - include: literal-string-templates
 
     - match: (?={{function_call_after_lookahead}})
       push: function-call-arguments
@@ -1078,8 +1080,7 @@ contexts:
   left-expression-end:
     - include: expression-break
 
-    - match: (?=`)
-      push: literal-string-template
+    - include: literal-string-templates
 
     - match: '{{function_call_after_lookahead}}'
       push: function-call-arguments
@@ -1223,16 +1224,76 @@ contexts:
       pop: 1
     - include: string-content
 
+  literal-string-templates:
+    - match: (?=(?:{{identifier_name}}\s*)?`)
+      push: literal-string-template
+
   literal-string-template:
-    - match: \`
-      scope: punctuation.definition.string.begin.js
+    # Notes:
+    #  Consume trailing whitespace after opening punctuation
+    #  and leading whitespace in front of closing punctuation
+    #  to maintain JavaScript indentation rules until embedded
+    #  code really begins/ends. It's required for embedded code
+    #  to be indented using global JavaScript indentation rules.
+    - match: (css)\s*((\`){{trailing_wspace}}?)
+      captures:
+        1: variable.function.tagged-template.js
+        2: meta.string.js string.quoted.other.js
+        3: punctuation.definition.string.begin.js
+      embed: scope:source.css.js-template
+      embed_scope: meta.string.js source.css.embedded.js
+      escape: '{{leading_wspace}}?(\`)'
+      escape_captures:
+        0: meta.string.js string.quoted.other.js
+        1: punctuation.definition.string.end.js
+      pop: 1
+    - match: (html)\s*((\`){{trailing_wspace}}?)
+      captures:
+        1: variable.function.tagged-template.js
+        2: meta.string.js string.quoted.other.js
+        3: punctuation.definition.string.begin.js
+      embed: scope:text.html.js-template
+      embed_scope: meta.string.js text.html.embedded.js
+      escape: '{{leading_wspace}}?(\`)'
+      escape_captures:
+        0: meta.string.js string.quoted.other.js
+        1: punctuation.definition.string.end.js
+      pop: 1
+    - match: (js)\s*((\`){{trailing_wspace}}?)
+      captures:
+        1: variable.function.tagged-template.js
+        2: meta.string.js string.quoted.other.js
+        3: punctuation.definition.string.begin.js
+      embed: scope:source.js.js-template
+      embed_scope: meta.string.js source.js.embedded.js
+      escape: '{{leading_wspace}}?(\`)'
+      escape_captures:
+        0: meta.string.js string.quoted.other.js
+        1: punctuation.definition.string.end.js
+      pop: 1
+    - match: (json)\s*((\`){{trailing_wspace}}?)
+      captures:
+        1: variable.function.tagged-template.js
+        2: meta.string.js string.quoted.other.js
+        3: punctuation.definition.string.begin.js
+      embed: scope:source.json.js-template
+      embed_scope: meta.string.js source.json.embedded.js
+      escape: '{{leading_wspace}}?(\`)'
+      escape_captures:
+        0: meta.string.js string.quoted.other.js
+        1: punctuation.definition.string.end.js
+      pop: 1
+    - match: (?:({{identifier_name}})\s*)?(\`)
+      captures:
+        1: variable.function.tagged-template.js
+        2: meta.string.js string.quoted.other.js punctuation.definition.string.begin.js
       set: literal-string-template-content
 
   literal-string-template-content:
     - meta_include_prototype: false
-    - meta_scope: meta.string.js string.quoted.other.js
+    - meta_content_scope: meta.string.js string.quoted.other.js
     - match: \`
-      scope: punctuation.definition.string.end.js
+      scope: meta.string.js string.quoted.other.js punctuation.definition.string.end.js
       pop: 1
     - include: string-interpolations
     - include: string-content
@@ -1250,13 +1311,22 @@ contexts:
 
   string-interpolation-content:
     - clear_scopes: 1
+    - meta_scope: meta.interpolation.js
+    - meta_content_scope: source.js.embedded
+    - include: text-interpolation-content
+
+  text-interpolations:
+    - match: \$\{
+      scope: punctuation.section.interpolation.begin.js
+      push: text-interpolation-content
+
+  text-interpolation-content:
     - meta_scope: meta.interpolation.js
     - meta_content_scope: source.js.embedded
     - match: \}
       scope: punctuation.section.interpolation.end.js
       pop: 1
-    - match: (?=\S)
-      push: expression
+    - include: expressions
 
   regexp-complete:
     - match: '/'
@@ -2088,9 +2158,7 @@ contexts:
         - function-name-meta
         - literal-variable-base
 
-    - match: '{{identifier_name}}(?={{nothing}}`)'
-      scope: variable.function.tagged-template.js
-      pop: 1
+    - include: literal-string-template
 
     - match: '{{constant_identifier}}(?=\s*(?:{{dot_accessor}}|\[))'
       scope: support.class.js
@@ -2523,9 +2591,7 @@ contexts:
     - match: '(?=#?{{identifier_name}}{{function_call_after_lookahead}})'
       set: call-method-name
 
-    - match: '{{identifier_name}}(?={{nothing}}`)'
-      scope: variable.function.tagged-template.js
-      pop: 1
+    - include: literal-string-template
 
     - include: object-property-base
     - include: else-pop

JavaScript/tests/syntax_test_js.js

@@ -273,9 +273,6 @@ tag `template`;
 // <- variable.function.tagged-template
 //  ^^^^^^^^^^ meta.string string.quoted.other
 
-tag/**/`template`;
-// <- variable.function.tagged-template
-
 x ? y // y is a template tag!
 `template` : z;
 //         ^ keyword.operator.ternary
@@ -1051,7 +1048,7 @@ foo
 //   ^^^ variable.function.tagged-template
 //      ^^ meta.string string.quoted.other punctuation.definition.string
 
-foo.tag/**/``;
+foo.tag ``;
 //  ^^^ variable.function.tagged-template
 
 return new Promise(resolve => preferenceObject.set({value}, resolve));