From 9c91811ca97670fb072e2c6210c9b47b1338f2c5 Mon Sep 17 00:00:00 2001
From: Evan Welsh <contact@evanwelsh.com>
Date: Sat, 27 Feb 2021 18:31:25 -0800
Subject: [PATCH] Support template strings with substitutions.

---
 gettext-tools/src/x-javascript.c          | 98 ++++++++++++++++++-----
 gettext-tools/tests/xgettext-javascript-7 | 34 +++++---
 2 files changed, 104 insertions(+), 28 deletions(-)

diff --git a/gettext-tools/src/x-javascript.c b/gettext-tools/src/x-javascript.c
index 3f2e2ac7e..6967a756e 100644
--- a/gettext-tools/src/x-javascript.c
+++ b/gettext-tools/src/x-javascript.c
@@ -1302,7 +1302,8 @@ phase5_get (token_ty *tp)
 
                 if (uc == P7_TEMPLATE_START_OF_EXPRESSION)
                   {
-                    mixed_string_buffer_destroy (&msb);
+                    tp->mixed_string = mixed_string_buffer_result (&msb);
+                    tp->comment = add_reference (savable_comment);
                     tp->type = last_token_type = token_type_ltemplate;
                     template_literal_depth++;
                     new_brace_depth_level ();
@@ -1449,6 +1450,9 @@ phase5_get (token_ty *tp)
             brace_depths[template_literal_depth]--;
           else if (template_literal_depth > 0)
             {
+              struct mixed_string_buffer msb;
+              mixed_string_buffer_init (&msb, lexical_context,
+                                        logical_file_name, line_number);  
               /* Middle or right part of template literal.  */
               for (;;)
                 {
@@ -1457,6 +1461,7 @@ phase5_get (token_ty *tp)
                   if (uc == P7_EOF || uc == P7_STRING_END)
                     {
                       tp->type = last_token_type = token_type_rtemplate;
+                      tp->mixed_string = mixed_string_buffer_result (&msb);
                       template_literal_depth--;
                       break;
                     }
@@ -1464,8 +1469,19 @@ phase5_get (token_ty *tp)
                   if (uc == P7_TEMPLATE_START_OF_EXPRESSION)
                     {
                       tp->type = last_token_type = token_type_mtemplate;
+                      tp->mixed_string = mixed_string_buffer_result (&msb);
                       break;
                     }
+
+                  if (IS_UNICODE (uc))
+                    {
+                      assert (UNICODE_VALUE (uc) >= 0
+                              && UNICODE_VALUE (uc) < 0x110000);
+                      mixed_string_buffer_append_unicode (&msb,
+                                                          UNICODE_VALUE (uc));
+                    }
+                  else
+                    mixed_string_buffer_append_char (&msb, uc);
                 }
               return;
             }
@@ -1519,10 +1535,45 @@ phase5_unget (token_ty *tp)
    Handling of tagged template literals.  */
 
 static void
-x_javascript_lex (token_ty *tp)
+x_javascript_lex (token_ty *tp, int tag_state)
 {
   phase5_get (tp);
-  if (tp->type == token_type_string || tp->type == token_type_template)
+  if (tag_state > 0 && tp->type == token_type_ltemplate)
+    {
+      token_ty token2;
+      phase5_get(&token2);
+      mixed_string_ty* ms = tp->mixed_string;
+
+      int expr_count = 0;
+      for (;;)
+        {
+          if (token2.type == token_type_mtemplate ||
+              token2.type == token_type_rtemplate)
+            {
+              char* expr = xasprintf("${%i}", expr_count);
+              mixed_string_ty* expr_ms = mixed_string_alloc_utf8(expr, lc_comment,
+                                                      logical_file_name, line_number);
+              free(expr);
+
+              ms = mixed_string_concat_free1(ms, expr_ms);
+              mixed_string_free(expr_ms);
+
+              ms = mixed_string_concat_free1(ms, token2.mixed_string);
+            }
+     
+          if (token2.type == token_type_mtemplate)
+            expr_count++;
+
+          if (token2.type == token_type_rtemplate)
+            break;
+
+          phase5_get(&token2);
+        }
+
+      tp->type = token_type_template;
+      tp->mixed_string = ms;
+    } 
+  else if (tp->type == token_type_string || tp->type == token_type_template)
     {
       mixed_string_ty *sum = tp->mixed_string;
 
@@ -1540,7 +1591,6 @@ x_javascript_lex (token_ty *tp)
                   || token3.type == token_type_template)
                 {
                   sum = mixed_string_concat_free1 (sum, token3.mixed_string);
-
                   free_token (&token3);
                   free_token (&token2);
                   continue;
@@ -1559,13 +1609,17 @@ x_javascript_lex (token_ty *tp)
       phase5_get (&token2);
       if (token2.type == token_type_template)
         {
+          if (strcmp(tp->string, "msgid") != 0 && strcmp(tp->string, "_") != 0)
           /* The value of
                tag `abc`
              is the value of the function call
                tag (["abc"])
              We don't know anything about this value.  Therefore, don't
-             let the extractor see this template literal.  */
-          free_token (&token2);
+             let the extractor see this template literal.
+             In the case of msgid or _, preserve them for libraries which handle them.*/
+            free_token (&token2);
+          else
+            phase5_unget (&token2);
         }
       else
         phase5_unget (&token2);
@@ -1611,6 +1665,7 @@ extract_balanced (message_list_ty *mlp,
   int arg = 1;
   /* 0 when no keyword has been seen.  1 right after a keyword is seen.  */
   int state;
+  int tag_state;
   /* Parameters of the keyword just seen.  Defined only in state 1.  */
   const struct callshapes *next_shapes = NULL;
   /* Context iterator that will be used if the next token is a '('.  */
@@ -1623,18 +1678,25 @@ extract_balanced (message_list_ty *mlp,
 
   /* Start state is 0.  */
   state = 0;
+  /* Normal state is 0, _ is 1, msgid is 2.  */
+  tag_state = 0;
 
   for (;;)
     {
       token_ty token;
 
-      x_javascript_lex (&token);
+      x_javascript_lex (&token, tag_state);
       switch (token.type)
         {
         case token_type_symbol:
           {
             void *keyword_value;
 
+            if (strcmp(token.string, "_") == 0)
+              tag_state = 1;
+            if (strcmp(token.string, "msgid") == 0)
+              tag_state = 2;
+            
             if (hash_find_entry (&keywords, token.string, strlen (token.string),
                                  &keyword_value)
                 == 0)
@@ -1663,7 +1725,7 @@ extract_balanced (message_list_ty *mlp,
               return true;
             }
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_rparen:
@@ -1673,7 +1735,7 @@ extract_balanced (message_list_ty *mlp,
               return false;
             }
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_comma:
@@ -1683,7 +1745,7 @@ extract_balanced (message_list_ty *mlp,
                                flag_context_list_iterator_advance (
                                  &context_iter));
           next_context_iter = passthrough_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_lbracket:
@@ -1695,7 +1757,7 @@ extract_balanced (message_list_ty *mlp,
               return true;
             }
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_rbracket:
@@ -1717,7 +1779,7 @@ extract_balanced (message_list_ty *mlp,
               return true;
             }
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_rbrace:
@@ -1727,7 +1789,7 @@ extract_balanced (message_list_ty *mlp,
               return false;
             }
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_string:
@@ -1738,7 +1800,7 @@ extract_balanced (message_list_ty *mlp,
             pos.file_name = logical_file_name;
             pos.line_number = token.line_number;
 
-            if (extract_all)
+            if (extract_all || tag_state == 1)
               {
                 char *string = mixed_string_contents (token.mixed_string);
                 mixed_string_free (token.mixed_string);
@@ -1754,7 +1816,7 @@ extract_balanced (message_list_ty *mlp,
           }
           drop_reference (token.comment);
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_xml_element_start:
@@ -1766,7 +1828,7 @@ extract_balanced (message_list_ty *mlp,
               return true;
             }
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_xml_element_end:
@@ -1776,7 +1838,7 @@ extract_balanced (message_list_ty *mlp,
               return false;
             }
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         case token_type_eof:
@@ -1797,7 +1859,7 @@ extract_balanced (message_list_ty *mlp,
         case token_type_xml_empty_element:
         case token_type_other:
           next_context_iter = null_context_list_iterator;
-          state = 0;
+          state = tag_state = 0;
           continue;
 
         default:
diff --git a/gettext-tools/tests/xgettext-javascript-7 b/gettext-tools/tests/xgettext-javascript-7
index c9d1f7eba..48f99f90e 100755
--- a/gettext-tools/tests/xgettext-javascript-7
+++ b/gettext-tools/tests/xgettext-javascript-7
@@ -10,19 +10,22 @@ embedded
 newlines`);
 var s2 = _(`A template literal with ${n} substitutions`);
 var s3 = _(`A template literal with several substitutions: ${a} and ${b} and ${c} and so on`);
-var s4 = `/${looks_like_regex}`;
-var s5 = _('not part of a regex');
-var s6 = `that's a valid string. ` + _('This too');
-var s7 = _(tag`A template literal with a tag`);
-var s8 = `a${`b${`c`+d}`}e`;
-var s9 = _("a normal string");
-var s10 = `abc${foo({},_('should be extracted'))}xyz`;
+var s4 = _(msgid`A tagged template literal with several substitutions: ${a} and ${b} and ${c} and so on`);
+var s5 = `/${looks_like_regex}`;
+var s6 = _('not part of a regex');
+var s7 = `that's a valid string. ` + _('This too');
+var s8 = _(tag`A template literal with a tag`);
+var s9 = _(msgid`A template literal with a msgid tag`);
+var s10 = _(_`A template literal with a _ tag`);
+var s11 = `a${`b${`c`+d}`}e`;
+var s12 = _("a normal string");
+var s13 = `abc${foo({},_('should be extracted'))}xyz`;
 var f1 = function () {
   return _("first normal string") + `${foo}` + _("second normal string");
 };
-var s11 = _("another normal string");
-var s12 = { property: `A template literal with ${n} substitution` };
-var s13 = _("yet another normal string");
+var s14 = _("another normal string");
+var s15 = { property: `A template literal with ${n} substitution` };
+var s16 = _("yet another normal string");
 
 EOF
 
@@ -59,12 +62,23 @@ msgid ""
 "newlines"
 msgstr ""
 
+msgid ""
+"A tagged template literal with several substitutions: ${0} and ${1} and ${2} "
+"and so on"
+msgstr ""
+
 msgid "not part of a regex"
 msgstr ""
 
 msgid "This too"
 msgstr ""
 
+msgid "A template literal with a msgid tag"
+msgstr ""
+
+msgid "A template literal with a _ tag"
+msgstr ""
+
 msgid "a normal string"
 msgstr ""
 
-- 
2.30.1

