commit 5d7bc40199782d9e685cfe633e76152d7bcf62e2
Author: Dmitry Goncharov <dgoncharov@users.sf.net>
Date:   Fri Mar 29 14:49:21 2024 -0400

    [sv 65588] Fix a buffer overrun in handling of .SHELLFLAGS.
    
    A buffer overflow occurs in oneshell mode when shellflags contains characters
    special to shell.
    
    $ cat makefile
    .ONESHELL:
    .SHELLFLAGS:=hello'
    all:;:
    $ make
    Aborted (core dumped)
    
    In oneshell mode construct_command_argv_internal wants to store in argv[0]
    the value of shell, followed by shellflags, followed by the command line.
    construct_command_argv_internal preallocates memory with
    
    nextp = new_argv[0] = xmalloc (shell_len + sflags_len + line_len + 3);
    
    construct_command_argv_internal is then used recursively to construct argv from
    shellflags.  If shellflags contains characters special to shell, then this
    recursive call does the so-called slow mode and constructs argv of "/bin/sh",
    "-c", and the value of shellflags.
    One problem is that the top construct_command_argv_internal never allocated
    room for the "/bin/sh" and "-c".
    
    Aside from the overflow, it is incorrect to prepend "/bin/sh" and "-c" to
    shellflags. Preallocaing more room in top construct_command_argv_internal can
    solve the overflow, but the resultant command line stays incorrect, because
    shellflags were not supposed to carry "/bin/sh" and "-c".
    
    This commit parses shellflags without help from construct_command_argv_internal
    and lets shellflags carry quoted tokens in oneshell mode, even when special
    characters are present.
    
    * src/job.c (construct_command_argv_internal): Fix a buffer overrun in
    construct_command_argv_internal by using next_dequoted_token to parse
    .SHELLFLAGS.
    * src/token.h: Declare utilities to break up a C string to tokens.
    * src/token.c: Implement utilities to break up a C string to tokens.
    * src/token.t.c: Test the implementation in token.c.
    * Makefile.am: Add token.h, token.c to make.
    Add token.t.tsk as a check_PROGRAM.
    Have rule 'check-regression' run token.t.tsk.
    On test failure, once the makeerror tarball is ready, remove the temporary
    directory that contains test logs.
    * tests/scripts/variables/SHELLFLAGS: Add tests.

diff --git a/Makefile.am b/Makefile.am
index 096ff661..e2206d3e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -37,7 +37,8 @@ make_SRCS =	src/ar.c src/arscan.c src/commands.c src/commands.h \
 		src/mkcustom.h src/os.h src/output.c src/output.h src/read.c \
 		src/remake.c src/rule.c src/rule.h src/shuffle.h src/shuffle.c \
 		src/signame.c src/strcache.c src/variable.c src/variable.h \
-		src/version.c src/vpath.c src/warning.c src/warning.h
+		src/version.c src/vpath.c src/warning.c src/warning.h \
+		src/token.c src/token.h
 
 w32_SRCS =	src/w32/pathstuff.c src/w32/w32os.c src/w32/compat/dirent.c \
 		src/w32/compat/posixfcn.c src/w32/include/dirent.h \
@@ -59,6 +60,8 @@ loadavg_SRCS =	lib/getloadavg.c
 
 make_SOURCES =	$(make_SRCS)
 EXTRA_make_SOURCES = $(vms_SRCS)
+check_PROGRAMS = token.t.tsk
+token_t_tsk_SOURCES = src/token.t.c
 
 if HAVE_GUILE
   _GUILE_CFLAGS = $(GUILE_CFLAGS)
@@ -126,7 +129,7 @@ EXTRA_DIST =	ChangeLog INSTALL README build.sh build.cfg.in $(man_MANS) \
 		README.W32 build_w32.bat src/config.h.W32 \
 		README.VMS makefile.com src/config.h-vms src/vmsjobs.c \
 		vms_export_symbol_test.com \
-		src/gmk-default.scm src/gmk-default.h \
+		src/gmk-default.scm src/gmk-default.h src/token.t.c \
 		$(mk_FILES) $(m4_FILES) $(test_FILES) $(w32_utf8_SRCS)
 
 # --------------- Generate the Guile default module content
@@ -171,43 +174,55 @@ MAKETESTFLAGS =
 GMK_OUTDIR = ..
 
 testlog = test-suite.log
+token_testlog = token_test.log
 testresult = tests/.test-result
+token_testresult = tests/.token_test-result
 errorpre = makeerror-$(PACKAGE_VERSION)-$(host_triplet)
 
-testfiles = $(testlog) $(testresult) $(errorfile)
+testfiles = $(testlog) $(token_testlog) $(testresult) $(token_testresult) $(errorfile)
 MOSTLYCLEANFILES = $(testfiles)
 
 errordetails =	config.status config.log src/config.h src/mkconfig.h \
-		$(testlog) tests/work
+		$(testlog) $(token_testlog)
 
 # Create a 4-letter random sequence
 rand_value = c = "abcdefghijklmnopqrstuvwxyz0123456789"
 rand_char  = substr(c,int(rand()*36),1)
 rand_string = $(AWK) 'BEGIN{srand(); $(rand_value); print $(rand_char) "" $(rand_char) "" $(rand_char) "" $(rand_char);}'
 
-check-regression: tests/config-flags.pm
+check-regression: token.t.tsk$(EXEEXT) tests/config-flags.pm
 	$(AM_V_at) rm -f $(testfiles)
+	$(AM_V_at) (./token.t.tsk$(EXEEXT); echo $$? >$(token_testresult)) 2>&1 | tee $(token_testlog)
 	$(AM_V_at) if test -f '$(top_srcdir)/tests/run_make_tests.pl'; then \
 	  ulimit -n 128; \
 	  if $(PERL) -v >/dev/null 2>&1; then \
 	    echo "cd tests && $(PERL) $(PERLFLAGS) $(abs_top_srcdir)/tests/run_make_tests.pl -make $(GMK_OUTDIR)/make$(EXEEXT) $(MAKETESTFLAGS)"; \
 	    (cd tests && $(PERL) $(PERLFLAGS) '$(abs_top_srcdir)/tests/run_make_tests.pl' -make '$(GMK_OUTDIR)/make$(EXEEXT)' $(MAKETESTFLAGS); echo $$? >.test-result) 2>&1 | tee $(testlog); \
-	    export TAR_OPTIONS='$(filter-out --sort%,$(TAR_OPTIONS))'; \
-	    er=$$(cat $(testresult)); if test "$$er" -ne 0; then \
-	      dirnm="$(errorpre)-$$($(rand_string))"; fnm="$$dirnm.tar.gz"; \
-	      rm -rf "$$dirnm"; mkdir "$$dirnm"; \
-	      $(AMTAR) chf - $(errordetails) | (cd "$$dirnm"; $(AMTAR) xf -); \
-	      $(AMTAR) chf - "$$dirnm" | eval GZIP= gzip $(GZIP_ENV) -c >"$$fnm"; \
-	      echo "*** Testing FAILED!  Details: $$fnm"; \
-	      echo '*** Please report to <$(PACKAGE_BUGREPORT)>'; echo; \
-	      exit $$er; \
-	    fi; \
 	  else \
 	    echo "Can't find a working Perl ($(PERL)); the test suite requires Perl."; \
 	  fi; \
 	else \
 	  echo "Can't find the $(PACKAGE_NAME) test suite ($(top_srcdir)/tests)."; \
 	fi
+	$(AM_V_at) \
+    export TAR_OPTIONS='$(filter-out --sort%,$(TAR_OPTIONS))'; \
+    er=$$(cat $(testresult)); \
+    if test "$$er" -eq 0; then \
+        errdetails='$(errordetails)'; \
+        er=$$(cat $(token_testresult)); \
+    else \
+        errdetails='$(errordetails) tests/work'; \
+    fi;\
+    if test "$$er" -ne 0; then \
+      dirnm="$(errorpre)-$$($(rand_string))"; fnm="$$dirnm.tar.gz"; \
+      rm -rf "$$dirnm"; mkdir "$$dirnm"; \
+      $(AMTAR) chf - $$errdetails | (cd "$$dirnm"; $(AMTAR) xf -); \
+      $(AMTAR) chf - "$$dirnm" | eval GZIP= gzip $(GZIP_ENV) -c >"$$fnm"; \
+      rm -r "$$dirnm"; \
+      echo "*** Testing FAILED!  Details: $$fnm"; \
+      echo '*** Please report to <$(PACKAGE_BUGREPORT)>'; echo; \
+      exit $$er; \
+    fi
 
 
 # --------------- Maintainer's Section
diff --git a/src/job.c b/src/job.c
index 9f6395e4..c6f00746 100644
--- a/src/job.c
+++ b/src/job.c
@@ -15,6 +15,7 @@ You should have received a copy of the GNU General Public License along with
 this program.  If not, see <https://www.gnu.org/licenses/>.  */
 
 #include "makeint.h"
+#include "token.h"
 
 #include <assert.h>
 #include <string.h>
@@ -3339,39 +3340,38 @@ construct_command_argv_internal (char *line, char **restp, const char *shell,
 #endif /* MK_OS_W32 */
         /* Create an argv list for the shell command line.  */
         {
-          int n = 1;
-          char *nextp;
-
-          new_argv = xmalloc ((4 + sflags_len/2) * sizeof (char *));
-
-          nextp = new_argv[0] = xmalloc (shell_len + sflags_len + line_len + 3);
+          int n = 1, malformed = 0;
+          char *nextp, *t;
+          size_t defsh_len = strlen (default_shell);
+          size_t defshflags_len = strlen (defsh_flags);
+          /* new_argv has to have room for at least 4 elements in the case of
+             an unterminated quote.  */
+          size_t new_argv_len = (4 + sflags_len/2) * sizeof (char *);
+          size_t argv0_len = defsh_len + defshflags_len + shell_len +
+                             sflags_len + line_len + 8;
+          size_t tlen;
+
+          new_argv = xmalloc (new_argv_len);
+
+          nextp = new_argv[0] = xmalloc (argv0_len);
           nextp = mempcpy (nextp, shell, shell_len + 1);
-
-          /* Chop up the shellflags (if any) and assign them.  */
-          {
-            /* Parse shellflags using construct_command_argv_internal to
-               handle quotes. */
-            char **argv;
-            char *f = alloca (sflags_len + 1);
-            memcpy (f, shellflags, sflags_len + 1);
-            argv = construct_command_argv_internal (f, 0, 0, 0, 0, flags, 0);
-            if (argv)
-              {
-                char **a;
-                for (a = argv; *a; ++a)
-                  {
-                    new_argv[n++] = nextp;
-                    nextp = stpcpy (nextp, *a) + 1;
-                  }
-                free (argv[0]);
-                free (argv);
-              }
-          }
-
-          /* Set the command to invoke.  */
-          new_argv[n++] = nextp;
+          /* Preserve the null terminator.  */
+          ++nextp;
+
+          /* Populate new_argv with tokens from shellflags using
+             next_dequoted_token to let shellflags carry quoted tokens.  */
+          memcpy (nextp, shellflags, sflags_len + 1);
+          new_argv[n] = nextp;
+          while ((t = next_dequoted_token (&nextp, &tlen, &malformed)))
+            new_argv[n++] = t;
+          /* Preserve the null terminator.  */
+          ++nextp;
+          /* new_argv contains shell, followed by tokenized shellflags.  */
+
+          /* append the command line.  */
           memcpy(nextp, line, line_len + 1);
-          new_argv[n++] = NULL;
+          new_argv[n++] = nextp;
+          new_argv[n] = NULL;
         }
         return new_argv;
       }
diff --git a/src/token.c b/src/token.c
new file mode 100644
index 00000000..95cd26f2
--- /dev/null
+++ b/src/token.c
@@ -0,0 +1,534 @@
+/* Utilities to tokenize a C string.
+Copyright (C) 2024 Free Software Foundation, Inc.
+Written by Dmitry Goncharov <dgoncharov@users.sf.net>.
+This file is part of GNU Make.
+
+GNU Make is free software; you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation; either version 3 of the License, or (at your option) any later
+version.
+
+GNU Make is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program.  If not, see <https://www.gnu.org/licenses/>.  */
+
+#include "token.h"
+#include <string.h>
+#include <stddef.h>
+#include <assert.h>
+
+static const char quotes_and_separators[] = "'\" \t\n";
+static const char *separators = quotes_and_separators + 2;
+static const char *next_token (const char **s, size_t *tokenlen, int *status);
+static char *dequote (char *s, size_t *slen);
+static char *quote_removal_in_double_quotes (char *s, size_t *slen);
+static char *quote_removal_in_single_quotes (char *s, size_t *slen);
+static int whitespace (char c);
+static const char *skip_separators (const char *s);
+static const char *skip_until_separator (const char *s);
+static size_t strecspn (const char *s, const char *reject);
+static size_t memecspn (const char *s, const char *reject, size_t slen);
+static char *collapse_escaped_newlines (char *s, char *beg, const char **endp);
+
+
+/* 's' is mutable, because next_dequoted_token dequotes the token in place.
+  An alternative interface would be to have next_dequoted_token allocate the
+  buffer on heap and return it.  That would require a malloc per token and
+  cause the caller to free each token.
+  Yet another interface would be to have next_dequoted_token take a buffer
+  from the caller, copy a token to the provided buffer and modify the token in
+  the buffer. Such interface requires next_dequoted_token to perform a memcpy
+  per token.  Also, such interface requires that the caller allocates a buffer
+  of sufficient size.  Since the caller does not know the length of a token,
+  the caller would have to allocate a buffer sufficient to hold the whole
+  input string. If a buffer sufficient to hold the whole string is allocated
+  anyway the caller can as well copy the input string to the buffer to relieve
+  next_dequoted_token from doing memcpy per token.
+  The current interface, on the other hand, does not force either malloc/free
+  or memcpy per token.
+  Another nice property of this mutable 's' is that it serves as a storage for
+  all dequoted tokens. This relieves the caller from keeping each token
+  somewhere individually.
+  For example, to populate an argv, the caller can copy the address of each
+  token to the related argv element.  The tokens themself stay in 's'.
+
+  while ((token = next_dequoted_token (&s, &len, &malformed)))
+    argv[argc++] = token;
+  argv[argc] = 0;  */
+char *
+next_dequoted_token (char **s, size_t *tokenlen, int *status)
+{
+  int st = 0;
+  char *token = (char*) next_token ((const char **)s, tokenlen, &st);
+  *status |= st;
+  if (token && st == 0)
+    dequote (token, tokenlen);
+  return token;
+}
+
+/* A not escaped and not quoted space, tab or newline character serves as a
+   token separator.
+   A not escaped backslash serves as an escape character.
+
+   In the absence of quotes, a token is delimited by token separators.
+   Beginning of a quoted token is delimited by a pair of a token separator
+   followed, immediately or not, by a not escaped single or double quote. The
+   end of a quoted token is delimited by the same single or not escaped double
+   quote followed, immediately or not, by a token separator.  A space, tab or
+   newline inside the quoted token does not serve as a token separator.
+   Multiple adjacent quoted tokens, in other words, quoted tokens without
+   interleaving separators, are treated as one token.
+   E.g. hello'world'of'many'tokens is treated as one token.  */
+
+/* Return the beginning of the first token in the specified string 's'.
+   Set '*tokenlen' to the length of the token.  The quotes that delimit a
+   quoted token are themself a part of the token and included to '*tokenlen'.
+   Set '*s' to the beginning of the next token.
+   If an opening quote is present and the closing quote is missing, set
+   '*status' to 1.  Otherwise, keep '*status' intact.
+   If no token is found, return 0.
+
+   'status' is a bitmask.
+   The current implementation only sets bit 0. However, it is possible to
+   modify next_token to detect $ or ` and set other bits in 'status'.  */
+static const char *
+next_token (const char **input, size_t *tokenlen, int *status)
+{
+  const char *token, *s;
+
+  assert (*input);
+
+  *tokenlen = 0;
+  *input = skip_separators (*input);
+  if (**input == '\0')
+    /* Only separators in this input.  */
+    return 0;
+
+  /* Point 'token' at the beginning of the token.  */
+  token = s = *input;
+
+  /* An escaped newline is not treated like some other escaped space.
+     An escaped newline is substituted with a space during quote removal phase,
+     which serves as a token separator.  */
+
+  for (;;)
+    {
+      /* Skip until a separator or quote.  */
+      s = skip_until_separator (s);
+      if (*s == '\0')
+        /* Found the end of the token.  */
+        break;
+
+      if (*s == '\\' && s[1] == '\n')
+        break;
+
+      if (*s == '\'')
+        {
+          /* Advance past the opening quote.  */
+          ++s;
+          /* Skip until the next single quote, which is the closing quote.  */
+          s += strcspn (s, "'");
+          if (*s == '\0')
+            {
+              /* Closing quote is missing.  */
+              *status |= 1;
+              break;
+            }
+          assert (*s == '\'');
+          /* Advance past the closing quote.  */
+          ++s;
+          if (whitespace (*s))
+            /* Found the end of the token.  */
+            break;
+          continue;
+        }
+
+      if (*s == '"')
+        {
+          /* Advance past the opening quote.  */
+          ++s;
+          /* Skip until a not escaped double quote, which is the closing quote.
+           */
+          s += strecspn (s, "\"");
+          if (*s == '\0')
+            {
+              /* Closing quote is missing.  */
+              *status |= 1;
+              break;
+            }
+          assert (*s == '"');
+          /* Advance past the closing quote.  */
+          ++s;
+          if (whitespace (*s))
+            /* Found the end of the token.  */
+            break;
+          continue;
+        }
+
+      assert (strchr (separators, *s));
+      /* Found the end of the token.  */
+      break;
+    }
+
+  /* 'token' points at the begininnig of the token. 's' points immediately
+     after the end of the token.  */
+  *tokenlen = s - token;
+
+  /* Move 's' to the beginning of the next token.  */
+  s = skip_separators (s);
+  *input = s;
+
+  return token;
+}
+
+/* Dequote the initial portion of '*slen' characters of string 's'.
+   Store the lenght of the dequoted portion to '*slen'.
+   Null terminate the dequoted portion.
+   Return 's'.  */
+static char *
+dequote (char *s, size_t *slen)
+{
+  char *beg = s;
+  const char *end = s + *slen;
+  size_t len = *slen;
+
+  s += memecspn (s, "\\'\"", *slen);
+  for (s = beg; s < end; )
+    {
+      len = end - s;
+      switch (*s)
+        {
+        case '\'':
+          s = quote_removal_in_single_quotes (s, &len);
+          end = s + len;
+          break;
+        case '"':
+          s = quote_removal_in_double_quotes (s, &len);
+          end = s + len;
+          break;
+        case '\\':
+          if (s[1] == '\n')
+            {
+              /* This backslash escapes the newline in 's[1]'.
+                 Remove the backslash and the newline.  */
+              memmove (s, s + 2, len - 2);
+              end -= 2;
+              break;
+            }
+          if (strchr ("'\"\\ \t", s[1]))
+            {
+              /* This backslash escapes 's[1]'.
+                 Remove the backslash.  */
+              memmove (s, s + 1, len - 1);
+              --end;
+
+              /* Keep intact the escaped character.  */
+              ++s;
+              break;
+            }
+
+          /* This is a lone backslash that does not escape anything
+             interesting. Remove the backslash.  */
+          memmove (s, s + 1, len - 1);
+          --end;
+          break;
+        default:
+          /* Keep intact a regular character.  */
+          ++s;
+          break;
+        }
+    }
+
+  assert (beg <= end);
+  *slen = end - beg;
+  beg[*slen] = '\0';
+
+  return beg;
+}
+
+/* Within substring '[s, s + slen)' remove the opening and closing single
+   quotes.
+
+  Store the number of characters between the end of the
+  token and the end of the substring, remaining after quote removal.
+  Return the end of the token.  */
+static char *
+quote_removal_in_single_quotes (char *s, size_t *slen)
+{
+  const char *end = s + *slen;
+  char *close;
+
+  /* These assertions are corrects, because next_dequoted_token calls dequote
+     only when the token is well formed.  */
+  assert (*slen > 1);
+  assert (*s == '\'');
+
+  close = (char*)  memchr (s + 1, '\'', *slen - 1);
+  assert (close);
+  assert (close > s);
+
+  /* Remove closing single quote.  */
+  memmove (close, close + 1, end - close);
+  --end;
+
+  /* Remove opening single quote.  */
+  memmove (s, s + 1, end - s);
+  --end;
+  --close;
+  *slen = end - close;
+
+  return close;
+}
+
+/* Within substring '[s, s + slen)'
+   Remove the opening and closing double quotes.
+   Remove each backslash which escapes a double quote or backslash.
+   Replace each group of consecutive backslash-newline pairs along with
+   surrounding space with a single space.
+
+   Store the number of characters between the end of the token and the end of
+   the substring, remaining after quote removal.
+   Return the end of the token.
+
+   Because '[s, s + slen)' is one token, there is no need to care about single
+   quotes.  */
+static char *
+quote_removal_in_double_quotes (char *s, size_t *slen)
+{
+  const char qe[] = {'\\', '"', '\0'};
+  char *beg = s;
+  const char *end = s + *slen;
+
+  /* These assertions are corrects, because next_dequoted_token calls dequote
+     only when the token is well formed.  */
+  assert (*slen > 1);
+  assert (*s == '"');
+
+  /* Remove the opening quote.  */
+  memmove (s, s + 1, end - s);
+  --end;
+
+  for (;;)
+    {
+      s += memecspn (s, qe, end - s);
+      if (s >= end)
+        break;
+
+      if (*s == '"')
+        {
+          /* Remove the closing quote.  */
+          memmove (s, s + 1, end - s);
+          --end;
+          /* A not escaped double quote is the end of the double quoted
+             substring.  */
+          break;
+        }
+
+      assert (*s == '\\');
+
+      if (s + 1 >= end)
+        /* There is nothing to escape.  */
+        break;
+
+      if (s[1] == '\\' || s[1] == '"')
+        {
+          /* A backslash is escaping either a backslash or a quote.  */
+
+          /* Remove the backslash.  */
+          memmove (s, s + 1, end - s);
+          --end;
+
+          /* Keep intact the escaped character.  */
+          ++s;
+
+          continue;
+        }
+
+      if (s[1] == '\n')
+        {
+          s = collapse_escaped_newlines (s, beg, &end);
+          continue;
+        }
+
+        /* Keep intact a lone backslash.  */
+        ++s;
+    }
+
+  *slen = end - s;
+
+  return s;
+}
+
+/* Return 1 if 'c' is a newline, space or tab.
+   Various shells, posix and GNU Make do not honor form-feed, carriage return
+   or vertical tab as token delimiters, even though isspace returns true for
+   those characters.  */
+static int
+whitespace (char c)
+{
+    return c == '\n' || c == ' ' || c == '\t';
+}
+
+/* Skip separators and escaped newlines.  */
+static const char *
+skip_separators (const char *s)
+{
+  const char *beg;
+  for (beg = 0; beg != s; )
+    {
+      beg = s;
+      /* Skip separators.  */
+      s += strspn (s, separators);
+
+      /* Skip an escaped newline.  */
+      if (*s == '\\' && s[1] == '\n')
+        s += 2;
+    }
+  return s;
+}
+
+/* Skip until a separator or a newline or a not escaped newline.  */
+static const char *
+skip_until_separator (const char *s)
+{
+  const char escape = '\\';
+  int n;
+
+  for (n = 0; *s; ++s)
+    {
+      /* A backslash followed by newline is replaced with a space and thus
+         serves as a separator. Therefore, any newline, escaped or not is a
+         separator.  */
+      if (*s == '\n')
+        {
+          s -= n % 2;
+          break;
+        }
+      if (*s == escape)
+        {
+          ++n;
+          continue;
+        }
+      if ((n % 2) == 0 && strchr (quotes_and_separators, (unsigned char) *s))
+        break;
+      n = 0;
+    }
+
+  return s;
+}
+
+/* Return the index of the first character in 's' that is present in 'reject'
+   and not escaped with a backslash.
+   If backslash itself is present in 'reject', then nothing can be escaped,
+   because lookup returns the index of that very backslash and strecspn behaves
+   as strcspn.  */
+static size_t
+strecspn (const char *s, const char *reject)
+{
+  size_t result;
+  int n = 0;
+  const char escape = '\\';
+  const char *escape_rejected = strchr (reject, (unsigned char) escape);
+
+  for (result = 0; *s; ++s, ++result)
+    {
+      if (*s == escape)
+        {
+          ++n;
+          if (escape_rejected)
+            break;
+          else
+            continue;
+        }
+
+      if ((n % 2) == 0 && strchr (reject, (unsigned char) *s))
+        break;
+
+      n = 0;
+    }
+
+  return result;
+}
+
+/* Same as strecspn within the first 'slen' characters of 's'.  */
+static size_t
+memecspn (const char *s, const char *reject, size_t slen)
+{
+  size_t result;
+  int n = 0;
+  const char escape = '\\';
+  /* memchr along with 'rlen', rather than strchr, is used to ensure
+     that a \0 in '[s, s+slen)' does not match the null terminator in 'reject'.
+   */
+  size_t rlen = strlen (reject);
+  const char *escape_rejected =
+                         (char*) memchr (reject, (unsigned char) escape, rlen);
+
+  for (result = 0; slen && *s; ++s, ++result, --slen)
+    {
+      if (*s == escape)
+        {
+          ++n;
+          if (escape_rejected)
+            break;
+          else
+            continue;
+        }
+
+      if ((n % 2) == 0 && memchr (reject, (unsigned char) *s, rlen))
+        break;
+
+      n = 0;
+    }
+
+  return result;
+}
+
+/* Within substring '[beg, *endp)' replace all consecutive backslash-newline
+   pairs along with surrounding space with a single space.
+   Return the address immediately after the newly inserted space.
+   Store the new end of the substring in '*endp'.  */
+static char *
+collapse_escaped_newlines (char *s, char *beg, const char **endp)
+{
+  char *p;
+  const char *end = *endp;
+
+  assert (beg < end);
+  assert (beg <= s);
+  assert (s < end);
+  assert (*s == '\\');
+  assert (s[1] == '\n');
+
+  /* Convert all consecutive backslash-newline pairs along with
+     surrounding space to a single space.  */
+
+  /* Walk back optional leading space.  */
+  for (p = s - 1; beg < p && (*p == '\t' || *p == ' '); --p)
+    ;
+  ++p;
+
+  /* Walk forward optional trailing space along with more
+     backslash-newline pairs.  */
+  while (s < end && ((*s == '\t' || *s == ' ') || (*s == '\\' && s[1] == '\n')))
+    {
+      for (; s < end && (*s == '\t' || *s == ' '); ++s)
+        ;
+      for (; s < end && (*s == '\\' && s[1] == '\n'); s += 2)
+        ;
+    }
+
+  /* 'p' points at the beginning of the leading space.
+     's' points immediately after trailing space and all consecutive
+     backslash-newline pairs.
+     Replace '[p, s)' with a single space.  */
+  *p = ' ';
+  ++p;
+  memmove (p, s, end - s);
+  *endp -= s - p;
+
+  return p;
+}
diff --git a/src/token.h b/src/token.h
new file mode 100644
index 00000000..3643f7fc
--- /dev/null
+++ b/src/token.h
@@ -0,0 +1,86 @@
+/* Utilities to tokenize a C string.
+Copyright (C) 2024 Free Software Foundation, Inc.
+Written by Dmitry Goncharov <dgoncharov@users.sf.net>.
+This file is part of GNU Make.
+
+GNU Make is free software; you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation; either version 3 of the License, or (at your option) any later
+version.
+
+GNU Make is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program.  If not, see <https://www.gnu.org/licenses/>.  */
+
+#ifndef _TOKEN_H_
+#define _TOKEN_H_
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Find, validate and dequote the first token.
+
+   Tokenizing rules.
+
+   A not escaped and not quoted space, tab or newline character serves as a
+   token separator.
+   A not escaped backslash serves as an escape character.
+
+   In the absence of quotes, a token is delimited by token separators.
+   Beginning of a quoted token is delimited by a pair of a token separator
+   followed, immediately or not, by a not escaped single or double quote. The
+   end of a quoted token is delimited by the same single or not escaped double
+   quote followed, immediately or not, by a token separator.  A space, tab or
+   newline inside the quoted token does not serve as a token separator.
+   Multiple adjacent quoted tokens, in other words, quoted tokens without
+   interleaving separators, are treated as one token.
+   E.g. hello'world'of'many'tokens is treated as one token.
+
+   Dequoting rules.
+
+   Outside of quotes
+     Remove each not escaped backslash.
+     Remove each backslash that escapes a backslash.
+     Replace one or multiple consecutive backslash-newline pairs with a single
+     space.
+
+   Within double quotes
+     Remove opening and closing double quotes.
+     Remove each backslash escaping a backslash or double quote.
+     Replace one or multiple consecutive backslash-newline pairs with a single
+     space.
+
+   Within single quotes.
+     Remove opening and closing single quotes.
+
+   If no token is found,
+     Keep '*status' intact.
+     Point '*s' to the null terminator.
+     Set '*tokenlen' to 0.
+     Return 0.
+
+   Otherwise, if a not escaped opening quote is present in the first token, but
+   a corresponding closing quote is missing, then,
+     Set '*status' to 1.
+     Point '*s' to the null terminator.
+     Set '*tokenlen' to the length of first token.
+     Return the beginning of the first token.
+
+   Otherwise,
+     Keep '*status' intact.
+     Point '*s' to the next token.
+     Dequote S in place.
+     Set '*tokenlen' to the length of the dequoted first token.
+     Return the beginning of the first token.  */
+char *next_dequoted_token (char **s, size_t *tokenlen, int *status);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/src/token.t.c b/src/token.t.c
new file mode 100644
index 00000000..e6c84010
--- /dev/null
+++ b/src/token.t.c
@@ -0,0 +1,2167 @@
+/* Unit test of the impl in token.c.
+Copyright (C) 2024 Free Software Foundation, Inc.
+Written by Dmitry Goncharov <dgoncharov@users.sf.net>.
+This file is part of GNU Make.
+
+GNU Make is free software; you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation; either version 3 of the License, or (at your option) any later
+version.
+
+GNU Make is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program.  If not, see <https://www.gnu.org/licenses/>.  */
+
+/* include token.c, rather than token.h, to test static functions.  */
+#include "token.c"
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <stdarg.h>
+
+static const char quotes[] = "'\"";
+static const char *dquote_and_separators = quotes_and_separators + 1;
+static int test (long, int, char *[]);
+static void test_memecspn (int, const char *, const char *, size_t);
+static void test_memecspn_impl (int, const char *, size_t, const char *,
+                                size_t);
+static void test_strecspn (int, const char *, const char *, size_t);
+static void test_next_token (int, const char *, const char *, ...);
+static void test_next_token_ws (int, const char *, const char *, ...);
+static void test_next_token_ws_impl (int, const char *, const char *, va_list,
+                                     char);
+static void test_next_token_impl (int, const char *, const char *, va_list,
+                                  char, char);
+static void test_next_dequoted_token (int, const char *, const char *, ...);
+static void test_next_dequoted_token_ws (int, const char *, const char *, ...);
+static void test_next_dequoted_token_i (int, const char *, const char *, ...);
+
+static void test_next_dequoted_token_ws_impl (int, const char *, const char *,
+                                              va_list, char);
+static void test_next_dequoted_token_impl (int, const char *, const char *,
+                                           va_list, char, char);
+static char *strdup_ (const char *s);
+static char *subchr (char *input, char x, char y);
+static char *swapchr (char *input, char x, char y);
+
+static int test_status = 0;
+static int test_ntotal = 0;
+static int test_nfailed = 0;
+
+#define ASSERT(cond, line) \
+do {\
+  ++test_ntotal;\
+  if (!(cond))\
+    {\
+      ++test_nfailed;\
+      if (test_status < 64)\
+        ++test_status;\
+      fprintf (stderr, "%s:%d: assertion `%s' failed, line = %d\n",\
+                        __FILE__, __LINE__, #cond, line);\
+    }\
+} while (0)
+
+int main (int argc, char *argv[])
+{
+  long n = 0;
+
+  /* Disable buffering of stdout to have stdout and stderr output synchronized
+     in the log file.  */
+  n = setvbuf (stdout, NULL, _IONBF, 0);
+  ASSERT (n == 0, __LINE__);
+
+  if (argc >= 2)
+    {
+      /* Run the specified test.  */
+      char *r;
+      errno = 0;
+      n = strtol (argv[1], &r, 0);
+      if (errno || r == argv[1])
+        {
+          fprintf (stderr, "usage: %s [test] [test arg]...\n", argv[0]);
+          return 1;
+        }
+    }
+  test_nfailed = test_ntotal = 0;
+  test (n, argc, argv);
+  fprintf (test_status > 0 ? stderr : stdout,
+           "%d failed tests, %d total tests\n", test_nfailed, test_ntotal);
+  return test_status;
+}
+
+/* Test the following functions
+memecspn
+strecspn
+next_token
+next_dequoted_token
+
+These characters have special powers.
+backslash, space, newline, single quote, double quote.
+
+Test parsing and dequoting
+
+1. Outside of quotes.
+  empty
+  end of token
+    backslash immediately followed by the null terminator
+    space immediately followed by the null terminator
+    newline immediately followed by the null terminator
+    single quote immediately followed by the null terminator
+    double quote immediately followed by the null terminator
+
+  newline
+  other space
+  backslash escapes newline,
+  backslash escapes space,
+  backslash escapes single quote,
+  backslash escapes double quote,
+  backslash escapes backslash,
+
+2. Inside single quotes.
+  empty
+  end of single quoted input
+    single quote immediately followed by the null terminator
+    even number of single quotes embedded w/o adjucent spaces.
+    even number of double quotes embedded with adjucent spaces.
+    odd number of double quotes embedded w/o adjucent spaces.
+
+  Space (including newline) has no special powers.
+  double quotes have no special powers.
+  backslash has no special powers.
+  Missing closing quote.
+
+3. Inside double quotes.
+  empty
+  end of double quoted input
+    double quote immediately followed by the null terminator
+    even number of double quotes embedded w/o adjucent spaces.
+    even number of single quotes embedded with adjucent spaces.
+    odd number of single quotes embedded w/o adjucent spaces.
+
+  Space (including newline) has no special powers.
+  Single quotes have no special powers.
+
+  A backslash immediately followed by a backslash or a double quote escapes
+  immediately following backslash or double quote and is removed.
+  A backslash-newline pair is replaced with a space.
+  A backslash immediately followed by any other character is left intact.
+
+  Missing closing quote.  */
+static int
+test (long n, int argc, char *argv[])
+{
+    int retcode;
+
+    (void) argc;
+    (void) argv;
+
+    retcode = 0;
+
+    switch (n)
+      {
+      case 0:
+      /* Test memecspn.  */
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\", "\\", 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\\\", "\\", 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "", separators, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, " ", separators, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello", separators, 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello ", separators, 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "\\", separators, 1);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "\\ ", separators, 2);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "\\\t ", separators, 2);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "\\\\\t ", separators, 2);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "\\\\\\\t ", separators, 4);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\ ", separators, 7);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\ \t", separators, 7);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\ world", separators, 12);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\\\ ", separators, 7);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\\\\\ ", separators, 9);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\\\\\ \t", separators, 9);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn (__LINE__ - 1, "hello\\\\\\ world", separators, 14);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_memecspn_impl (__LINE__ - 1, "hello\\", 4, separators, 4);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Test strecspn.  */
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "", separators, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, " ", separators, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello", separators, 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello ", separators, 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\\", separators, 1);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\\ ", separators, 2);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\\\t ", separators, 2);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\\\\\t ", separators, 2);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\\\\\\\t ", separators, 4);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\ ", separators, 7);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\ \t", separators, 7);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\ world", separators, 12);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\\\ ", separators, 7);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\\\\\ ", separators, 9);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\\\\\ \t", separators, 9);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\\\\\ world", separators, 14);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\", "\\", 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "hello\\\\", "\\", 5);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\"", dquote_and_separators, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash works as escape.  */
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\\\"", "\" \t\n", 2);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash itself is rejected.  */
+      case __LINE__:
+        test_strecspn (__LINE__ - 1, "\\\"", "\" \t\n\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+
+      /* No token. */
+      /* Empty.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A lone separator.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, " ", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, " ", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Multiple separators.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "   ", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "   ", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Multiple distinct separators.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, " \t\n  ", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, " \t\n  ", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Test next_token and next_dequoted_token outside of quotes.
+          end of token
+            backslash immediately followed by the null terminator
+            space immediately followed by the null terminator
+            single quote immediately followed by the null terminator
+            double quote immediately followed by the null terminator
+
+          newline
+          other space
+          backslash not escaping anything,
+          backslash escapes newline,
+          backslash escapes space,
+          backslash escapes single quote,
+          backslash escapes double quote,
+          backslash escapes backslash.  */
+
+      /* Backslash not escaping anything.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\world", "hello\\world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello\\world", "helloworld", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A not escaped backslash, followed by null terminator.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\", "\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\", "", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes backslash.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\\", "\\\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\\", "\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes newline.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\n", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\n", 0, 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes newline, followed by a token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\nhello", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\nhello", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes newline, followed by a token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\n\\\nhello", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\n\\\nhello", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\n \\\nhello", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\n \\\nhello", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1, " hello, world", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  " hello, world", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes space.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\\ hello, world", "\\ hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "\\ hello, world", " hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes backslash.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\\\\ hello, world", "\\\\", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "\\\\ hello, world", "\\",
+                                  "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\\\nhello\\\n \\\nworld\\\n", "hello", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "\\\nhello\\\n \\\nworld\\\n ",
+                                  "hello", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\\\n \\\nhello\\\n \\\nworld", "hello", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "\\\n \\\nhello\\\n \\\nworld",
+                                  "hello", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A token, followed by a space  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello ", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello ", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A token, followed by a backslash escaping newline.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\n", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\\n", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A token, followed by a backslash escaping backslash, followed by
+         newline.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\\\n", "hello\\\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\\\\n", "hello\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A lone escaped newline is not considered a token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\n \\\n", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\\n \\\n", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A lone escaped newline is not considered a token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\\\n \\\n", "hello\\\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello\\\\\n \\\n", "hello\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A token, followed by a backslash escaping backslash.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\\", "hello\\\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\\\", "hello\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A lone escaped newline is not considered a token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\\ \\\n", "hello\\\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\\\ \\\n", "hello\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A lone escaped newline is not considered a token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\ \\\n", "hello\\ ", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\ \\\n", "hello ", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash, followed by newline. Backslash-newline is replaced with a
+         space. Two tokens.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello,\\\nworld", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello,\\\nworld", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash, escapes backslash, followed by newline. Backslash is
+         removed and not escaped newline is removed. Two tokens.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "hello,\\\\\nworld", "hello,\\\\", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello,\\\\\nworld", "hello,\\", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "hello,\\\\\\\nworld", "hello,\\\\", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello,\\\\\\\nworld",
+                                  "hello,\\", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Two tokens.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello, world", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello, world", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello, world\n", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello, world\n", "hello,", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes space. Backslash removed, single token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello,\\ world", "hello,\\ world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello,\\ world", "hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash, followed by two spaces. Backslash removed, two tokens.
+         First token contains the escaped space.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "hello,\\  world", "hello,\\ ", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello,\\  world", "hello, ", "world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes white space.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "a\\ hello, world\\ b",
+                         "a\\ hello,", "world\\ b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "a\\ hello, world\\ b",
+                                  "a hello,", "world b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\'", "\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\'", "'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes backslash, followed by a quote.
+         Malformed token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\\'", "\\\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\\'", "\\\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\\\\'", "\\\\\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\\\\'", "\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\'hello", "\\'hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\'hello", "'hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "a \\'hello, world\\' b",
+                         "a", "\\'hello,", "world\\'", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "a \\'hello, world\\' b",
+                                  "a", "'hello,", "world'", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "a 'hello one\" \"two world' b",
+                         "a", "'hello one\" \"two world'", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "a 'hello one\" \"two world' b",
+                                  "a", "hello one\" \"two world", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "a 'hello one\\' \\'two world' b",
+                            "a", "'hello one\\'", "\\'two", "world' b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "a \"hello one\\\" \\\"two world\" b",
+                            "a", "\"hello one\\\" \\\"two world\"", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "a 'hello one\\' \\'two world b",
+                                     "a", "hello one\\", "'two", "world", "b",
+                                     0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "a \"hello one\\\" \\\"two world\" b",
+                                     "a", "hello one\" \"two world", "b",
+                                     0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "a \\\"hello one\\\" \\\"two world b",
+                                  "a", "\"hello", "one\"", "\"two", "world",
+                                  "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes backslash, followed by a quote.
+         Malformed token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\\'hello", "\\\\'hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\\'hello", "\\\\'hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\\\\\\'hello", "\\\\\\'hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "\\\\\\'hello", "\\'hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\'", "hello\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\'", "hello'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes backslash, followed by a quote.
+         Malformed token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\\'", "hello\\\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\\\'", "hello\\\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\\\\'", "hello\\\\\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "hello\\\\\\'", "hello\\'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\'world", "hello\\'world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello\\'world", "hello'world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes backslash, followed by a quote.
+         Malformed token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello\\\\'world", "hello\\\\'world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello\\\\'world", "hello\\\\'world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes a quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "hello\\\\\\'world", "hello\\\\\\'world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello\\\\\\'world", "hello\\'world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash escapes backslash, followed by a quote.
+         Malformed token.  */
+       case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "hello\\\\' world", "hello\\\\' world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello\\\\' world", "hello\\\\' world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+
+      /* Quotes.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "''", "''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "''", "", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+
+
+
+      /* Malformed token. A quote immediately followed by the null terminator.
+         Closing quote is missing.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'", "'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'", "'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Malformed token. A quote immediately followed by the null terminator.
+         Closing quote is missing.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'''", "'''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'''", "'''", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Malformed token. Leading separators are skipped.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, " '", "'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, " '", "'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Malformed token. Trailing separators stay intact.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "' ", "' ", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "' ", "' ", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'  ", "'  ", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'  ", "'  ", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Malformed token. Trailing separators stay intact.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'\\\n", "'\\\n", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'\\\n", "'\\\n", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Even number of quotes.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "''''", "''''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "''''", "", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A wellformed token in quotes.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'hello, world'", "'hello, world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello, world'", "hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Missing closing quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'hello, world", "'hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello, world", "'hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Missing opening quote.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "hello, world'", "hello,", "world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "hello, world'", "hello,", "world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash in quotes.  */
+      /* Backslash in single or double quotes does not escape a space or tab.
+         It is simply a lone backslash and it stays.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'hello,\\ world'", "'hello,\\ world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello,\\ world'", "hello,\\ world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash in single quotes does not escape a newline.  It is simply a
+         lone backslash and it stays.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'hello,\\\nworld'", "'hello,\\\nworld'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "'hello,\\\nworld'", "hello,\\\nworld", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash in double quotes escapes a newline. A backslash newline pair
+         is replaced with a space.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\"hello,\\\nworld\"", "\"hello,\\\nworld\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "\"hello,\\\nworld\"", "hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Multiple consecutive backslash newline pairs along with leading and
+         trailng space are all replaced with a single space .  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld\"",
+                         "\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_i (
+            __LINE__ - 1,
+            "\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld\"",
+            "hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "\"a\\\nb\\\nc\"", "\"a\\\nb\\\nc\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_i (__LINE__ - 1,
+                                    "\"a\\\nb\\\nc\"", "a b c", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld "
+                         " bye,\\\n \t \\\n \t \\\n \t   \\\n  \\\nmoon\"",
+                         "\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld "
+                         " bye,\\\n \t \\\n \t \\\n \t   \\\n  \\\nmoon\"",
+                         0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_i (
+            __LINE__ - 1,
+            "\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld "
+            " bye,\\\n \t \\\n \t \\\n \t   \\\n  \\\nmoon\"",
+            "hello, world  bye, moon", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash in double quotes escapes a backslash.
+         Backslash in single quotes has no special powers at all.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'hello,\\\\ world'", "'hello,\\\\ world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "'hello,\\\\ world'",
+                                     "hello,\\\\ world", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "\"hello,\\\\ world\"",
+                                     "hello,\\ world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* The backslash does not escape the quote and this string is split into
+         a wellformed token and a malformed token.*/
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "'hello,\\' world'", "'hello,\\'", "world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "'hello,\\' world'",
+                                     "hello,\\", "world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash in double quotes escapes a double quote. */
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "\"hello,\\\" world\"", "\"hello,\\\" world\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "\"hello,\\\" world\"",
+                                     "hello,\" world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash in double quotes escapes a double quote, which leaves a
+         dangling closing quote and makes a malformed second token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\"hello,\\\\\" world\"",
+                         "\"hello,\\\\\"", "world\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "\"hello,\\\\\" world\"",
+                                     "hello,\\", "world\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Backslash in double quotes escapes a double quote. */
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "\"hello,\\\\\\\" world\"",
+                            "\"hello,\\\\\\\" world\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "\"hello,\\\\\\\" world\"",
+                                     "hello,\\\" world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Test that outside quotes a backslash-newline pair is replaced with a
+         space, each not escaped backslash is removed and each backslash that
+         escapes a backslash is removed.  */
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "one\\\\two\\ tree\\\" four\\\nfive",
+                            "one\\\\two\\ tree\\\"", "four", "five", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_i (__LINE__ - 1,
+                                    "one\\\\two\\ tree\\\" four\\\nfive",
+                                    "one\\two tree\"", "four", "five", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+
+      /* Test that within double quotes a backslash that escapes a backslash or
+         double quote is removed, a backslash-newline pair is replaced with a
+         space and other backslashes stay.  */
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "\"one\\\\two\\ tree\\\" four\\\nfive\"",
+                            "\"one\\\\two\\ tree\\\" four\\\nfive\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_i (__LINE__ - 1,
+                                    "\"one\\\\two\\ tree\\\" four\\\nfive\"",
+                                    "one\\two\\ tree\" four five", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Test that within single quotes only the opening and closing quotes are
+         removed.  */
+      case __LINE__:
+        test_next_token_ws (__LINE__ - 1,
+                            "'one\\\\two\\ tree\\\" four\\\nfive'",
+                            "'one\\\\two\\ tree\\\" four\\\nfive'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "'one\\\\two\\ tree\\\" four\\\nfive'",
+                                     "one\\\\two\\ tree\\\" four\\\nfive", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Quotes within qoutes.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'hello, \"one two\" world'",
+                         "'hello, \"one two\" world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello, \"one two\" world'",
+                                  "hello, \"one two\" world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "a 'hello one\\\\' \\\\'two world' b",
+                         "a", "'hello one\\\\'", "\\\\'two world'", "b",
+                         0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                  "a 'hello one\\\\' \\\\'two world' b",
+                                  "a", "hello one\\\\", "\\two world", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                  "a \"hello one\\\\\" \\\\\"two world\" b",
+                                  "a", "hello one\\", "\\two world", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "a 'hello one\\\\\\\\' \\\\\\\\'two world' b",
+                         "a", "'hello one\\\\\\\\'",
+                         "\\\\\\\\'two world'", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (
+            __LINE__ - 1,
+            "a 'hello one\\\\\\\\' \\\\\\\\'two world' b",
+            "a", "hello one\\\\\\\\", "\\\\two world", "b",
+            0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_dequoted_token_ws (
+            __LINE__ - 1,
+            "a \"hello one\\\\\\\\\" \\\\\\\\\"two world\" b",
+            "a", "hello one\\\\", "\\\\two world", "b",
+            0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'a \"one 'cd' two\" b'",
+                         "'a \"one 'cd' two\" b'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'a \"one 'cd' two\" b'",
+                                  "a \"one cd two\" b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'a \"one 'c d' two\" b'",
+                         "'a \"one 'c", "d' two\" b'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'a \"one 'c d' two\" b'",
+                                  "a \"one c", "d two\" b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "a 'hell\"o''w\"orld' b",
+                         "a", "'hell\"o''w\"orld'", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "a 'hell\"o''w\"orld' b",
+                                  "a", "hell\"ow\"orld", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "a \\'hell\"o\\'\\'w\"orld\\' b",
+                         "a", "\\'hell\"o\\'\\'w\"orld\\'", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "a \\'hell\"o\\'\\'w\"orld\\' b",
+                                  "a", "'hello\\'\\'world'", "b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "\\'a \"one 'cd' two\" \\'b",
+                         "\\'a", "\"one 'cd' two\"", "\\'b", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "\\'a \"one 'cd' two\" \\'b",
+                                  "'a", "one 'cd' two", "'b", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Trailing separator.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'hello, world' ", "'hello, world'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello, world' ", "hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Trailing escaped separator.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'hello'\\ ", "'hello'\\ ", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'hello'\\ ", "hello ", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'hello'\\\n", "hello", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'hello, world'\\ ", "'hello, world'\\ ", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello, world'\\ ", "hello, world ", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello, world'\\\n", "hello, world", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A malformed token.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'a'b'c", "'a'b'c", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'a'b'c", "'a'b'c", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Multiple adjacent quoted tokens are treated as one.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'a'b'c'", "'a'b'c'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "'a'b'c'", "abc", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A malformed token ajacent to a good one.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "a'b'c'd", "a'b'c'd", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "a'b'c'd", "a'b'c'd", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Multiple adjacent quoted tokens are treated as one.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "a'b'c'd'", "a'b'c'd'", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1, "a'b'c'd'", "abcd", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'\\\\'abc''", "'\\\\'abc''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1, "'\\\\'abc''", "\\\\abc", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "\"\\\\\"abc\"\"", "\\abc", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'\\'\\''", "'\\'\\''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      /* A malformed token. Backslash has no special powers within single
+         quotes.  */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1, "'\\'\\''", "'\\'\\''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      /* A wellformed token. A backslash escapes a double quote within double
+         quotes.  */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1, "\"\\\"\\\"\"", "\"\"", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1, "'\\\\'\\\\''", "'\\\\'\\\\''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      /* A wellformed token. Backslash has no special powers within single
+         quotes. The third backslash is outside single quotes and it escapes
+         the following (forth) backslash.  */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1, "'\\\\'\\\\''", "\\\\\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+      /* A wellformed token. A backslash escapes a backslash within double
+         quotes.  */
+      case __LINE__:
+        test_next_dequoted_token_ws (__LINE__ - 1,
+                                     "\"\\\\\"\\\\\"\"", "\\\\", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "  a  '  b  '  c  '  d  '  ",
+                         "a",  "'  b  '",  "c",  "'  d  '" , 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "  a  '  b  '  c  '  d  '  ",
+                                  "a", "  b  ", "c", "  d  ", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Multiple adjacent quoted tokens are treated as one.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "'hello'world\\'12''", "'hello'world\\'12''", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "'hello'world\\'12''", "helloworld'12", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* Multiple adjacent quoted tokens are treated as one.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+                         "01'2 3'45\"67 89\"ab\\ cd",
+                         "01'2 3'45\"67 89\"ab\\ cd", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+                                  "01'2 3'45\"67 89\"ab\\ cd",
+                                  "012 34567 89ab cd", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* An example from gnu make test suite.  */
+      case __LINE__:
+        test_next_token (__LINE__ - 1,
+            "-w -E 'use warnings FATAL => \"all\";' -E",
+            "-w", "-E", "'use warnings FATAL => \"all\";'", "-E", 0);
+        if (n)
+          break;
+        /* Fall through */
+      case __LINE__:
+        test_next_dequoted_token (__LINE__ - 1,
+            "-w -E 'use warnings FATAL => \"all\";' -E",
+            "-w", "-E", "use warnings FATAL => \"all\";", "-E", 0);
+        if (n)
+          break;
+        /* Fall through */
+
+      /* A very long input.  */
+      case __LINE__:
+        {
+          int k;
+          char *input = (char*) malloc (1024 * 65 * 4);
+          char *result;
+          ASSERT (input, __LINE__);
+          for (k = 0; k < 1024 * 65 * 2; ++k)
+            input[k] = 'a' + k % 9;
+          input[1024 * 65 * 2 - 1] = ' ';
+          for (k = 1024 * 65 * 2; k < 1024 * 65 * 4; ++k)
+            input[k] = 'f' + k % 7;
+          input[1024 * 65 * 4 - 1] = '\0';
+          result = strdup_ (input);
+          result[1024 * 65 * 2 - 1] = '\0';
+          test_next_token (__LINE__, input, result, result + 1024 * 65 * 2, 0);
+          free (result);
+          free (input);
+          if (n)
+            break;
+        }
+        /* Fall through */
+
+      /* Allocate a long string.
+         Populate with printable chars.
+         Insert multiple token separators every 128 chars.
+         Border with quotes.  */
+      case __LINE__:
+        {
+          int k;
+          char *input = (char*) malloc (1024 * 65 * 4);
+          char *dequoted;
+          ASSERT (input, __LINE__);
+          for (k = 0; k < 1024 * 65 * 2; ++k)
+            input[k] = 'a' + k % 9;
+          for (k = 1; k < 1024 * 65 * 2; k += 128)
+            input[k] = ' ';
+          input[0] = '\'';
+          input[1024 * 65 * 2 - 2] = '\'';
+          input[1024 * 65 * 2 - 1] = ' ';
+
+          for (k = 1024 * 65 * 2; k < 1024 * 65 * 4; ++k)
+            input[k] = 'f' + k % 7;
+          for (k = 1024 * 65 * 2 + 1; k < 1024 * 65 * 4; k += 128)
+            input[k] = ' ';
+          input[1024 * 65 * 2] = '\'';
+          input[1024 * 65 * 4 - 2] = '\'';
+          input[1024 * 65 * 4 - 1] = '\0';
+          dequoted = strdup_ (input);
+          dequoted[1024 * 65 * 2 - 2] = '\0';
+          dequoted[1024 * 65 * 4 - 2] = '\0';
+          test_next_dequoted_token (__LINE__, input, dequoted + 1,
+                                    dequoted + 1024 * 65 * 2 + 1, 0);
+          free (dequoted);
+          free (input);
+          if (n)
+            break;
+        }
+        /* Fall through */
+
+      default:
+        retcode = -1;
+        break;
+      }
+    return retcode;
+}
+
+static void
+test_memecspn (int line, const char *s, const char *reject, size_t expected)
+{
+  test_memecspn_impl (line, s, strlen (s), reject, expected);
+}
+
+static void
+test_memecspn_impl (int line, const char *s, size_t slen, const char *reject,
+                    size_t expected)
+{
+  size_t r;
+
+  printf ("token test %d\n", line);
+  r = memecspn (s, reject, slen);
+  ASSERT (r == expected, line);
+}
+
+static void
+test_strecspn (int line, const char *s, const char *reject, size_t expected)
+{
+  size_t r;
+
+  printf ("token test %d\n", line);
+  r = strecspn (s, reject);
+  ASSERT (r == expected, line);
+}
+
+/* Test next_token with the specified 'input', as well as modified
+   'input' where each single quote is replaced with a double quote and each
+   space is replaced with a tab and a newline.  */
+static void
+test_next_token (int line, const char *input, const char *expected, ...)
+{
+  const char *q;
+  char *in;
+  va_list ap;
+
+  for (q = quotes; *q; ++q)
+    {
+      in = strdup_ (input);
+      swapchr (in, '\'', *q);
+      if ('\'' == *q || strcmp (in, input))
+        {
+          va_start (ap, expected);
+          test_next_token_ws_impl (line, in, expected, ap, *q);
+          va_end (ap);
+        }
+      free (in);
+    }
+}
+
+/* Test next_token with the specified 'input', as well as modified
+   'input' where each space is replaced with a tab and a newline.  */
+static void
+test_next_token_ws (int line, const char *input, const char *expected, ...)
+{
+  va_list ap;
+  va_start (ap, expected);
+  test_next_token_ws_impl (line, input, expected, ap, '\'');
+  va_end (ap);
+}
+
+/* Test next_token with the specified 'input', as well as modified
+   'input' where each space is replaced with a tab and a newline.  */
+static void
+test_next_token_ws_impl (int line, const char *input, const char *expected,
+                         va_list ap, char c)
+{
+  const char *w;
+  char *in;
+  va_list aq;
+
+  for (w = separators; *w; ++w)
+    {
+      /* An escaped space and and escaped newline are handled
+         differently. */
+      if (strstr (input, "\\ ") && *w == '\n')
+        continue;
+      in = strdup_ (input);
+      subchr (in, ' ', *w);
+      /* Avoid running the test again with the same input.  */
+      if (' ' == *w || strcmp (in, input))
+        {
+          va_copy (aq, ap);
+          test_next_token_impl (line, in, expected, aq, c, *w);
+          va_end (aq);
+        }
+      free (in);
+    }
+}
+
+static void
+test_next_token_impl (int line, const char *input, const char *expected,
+                      va_list ap, char c, char c2)
+{
+  const char *s = input, *t;
+  size_t tlen, elen;
+  int status = 0;
+
+  printf ("token test %d\n", line);
+  do
+    {
+      elen = expected ? strlen (expected) : 0;
+      t = next_token (&s, &tlen, &status);
+      ASSERT (tlen == elen, line);
+
+      if (expected && *expected)
+        {
+          char *e;
+          /* An escaped space and and escaped newline are handled
+             differently. */
+          if (strstr (expected, "\\ ") && c2 == '\n')
+            continue;
+          e = strdup_ (expected);
+          swapchr (e, '\'', c);
+          subchr (e, ' ', c2);
+          ASSERT (memcmp (t, e, tlen) == 0, line);
+          free (e);
+        }
+      else
+        ASSERT (t == 0, line);
+      expected = va_arg (ap, const char *);
+    } while (expected);
+    /* After went through all expected tokens, next next_token call
+       should return 0.  This tests that all calls of test_next_token pass all
+       expected tokens.  */
+    t = next_token (&s, &tlen, &status);
+    ASSERT (t == 0, line);
+}
+
+/* Test next_dequoted_token with the specified 'input', as well as modified
+   'input' where each single quote is replaced with a double quote and each
+   space is replaced with a tab and a newline.  */
+static void
+test_next_dequoted_token (int line, const char *input,
+                          const char *expected, ...)
+{
+  const char *q;
+  char *i;
+  va_list ap;
+
+  for (q = quotes; *q; ++q)
+    {
+      i = strdup_ (input);
+      swapchr (i, '\'', *q);
+      if ('\'' == *q || strcmp (i, input))
+        {
+          va_start (ap, expected);
+          test_next_dequoted_token_ws_impl (line, i, expected, ap, *q);
+          va_end (ap);
+        }
+      free (i);
+    }
+}
+
+/* Test next_dequoted_token with the specified 'input', as well as modified
+   'input' where each space is replaced with a tab and a newline.  */
+static void
+test_next_dequoted_token_ws (int line, const char *input,
+                             const char *expected, ...)
+{
+  va_list ap;
+  va_start (ap, expected);
+  test_next_dequoted_token_ws_impl (line, input, expected, ap, '\'');
+  va_end (ap);
+}
+
+/* Test next_dequoted_token with the specified 'input'. Make no modifications
+   to 'input'.  */
+static void
+test_next_dequoted_token_i (int line, const char *input,
+                            const char *expected, ...)
+{
+  va_list ap;
+  va_start (ap, expected);
+  test_next_dequoted_token_impl (line, input, expected, ap, '\'', ' ');
+  va_end (ap);
+}
+
+/* Test next_dequoted_token with the specified 'input', as well as modified
+   'input' where each space is replaced with each character from the specified
+   'ws'.  */
+static void
+test_next_dequoted_token_ws_impl (int line, const char *input,
+                                  const char *expected, va_list ap,
+                                  char c)
+{
+  const char *w;
+  char *i;
+  va_list aq;
+
+  for (w = separators; *w; ++w)
+    {
+      /* An escaped space and and escaped newline are handled
+         differently. */
+      if (strstr (input, "\\ ") && *w == '\n')
+        continue;
+
+      i = strdup_ (input);
+      subchr (i, ' ', *w);
+      /* Avoid running the test again with the same input.  */
+      if (' ' == *w || strcmp (i, input))
+        {
+          va_copy (aq, ap);
+          test_next_dequoted_token_impl (line, i, expected, aq, c, *w);
+          va_end (aq);
+        }
+      free (i);
+    }
+}
+
+/* Test next_dequoted_token with the specified 'input'.  */
+static void
+test_next_dequoted_token_impl (int line, const char *input,
+                               const char *expected, va_list ap, char c,
+                               char c2)
+{
+  char *beg, *s, *t;
+  size_t tlen, elen;
+  int status = 0;
+
+  printf ("token test %d\n", line);
+
+  beg = s = strdup_ (input);
+  do
+    {
+      elen = expected ? strlen (expected) : 0;
+      t = next_dequoted_token (&s, &tlen, &status);
+      ASSERT (tlen == elen, line);
+      if (expected && *expected)
+        {
+          char *e = strdup_ (expected);
+          assert (*input);
+          swapchr (e, '\'', c);
+          subchr (e, ' ', c2);
+          ASSERT (memcmp (t, e, tlen) == 0, line);
+          free (e);
+        }
+      else if (*input && expected && !*expected)
+        ASSERT (t && *t == '\0', line);
+      else
+        ASSERT (t == 0, line);
+      expected = va_arg (ap, const char *);
+    } while (expected);
+  free (beg);
+}
+
+/* Duplicate a string and assert on success.  */
+static char *
+strdup_ (const char *s)
+{
+  char *r = strdup (s);
+  assert (r);
+  return r;
+}
+
+/* Substitute all instances of 'x' in 'input' with 'y'.  */
+static char *
+subchr (char *input, char x, char y)
+{
+  for (; *input; ++input)
+    if (*input == x)
+      *input = y;
+  return input;
+}
+
+/* Substitute all instances of 'x' in 'input' with 'y'.
+   Substitute all instances of 'y' in 'input' with 'x'.  */
+static char *
+swapchr (char *input, char x, char y)
+{
+  for (; *input; ++input)
+    if (*input == x)
+      *input = y;
+    else if (*input == y)
+      *input = x;
+  return input;
+}
diff --git a/tests/scripts/variables/SHELLFLAGS b/tests/scripts/variables/SHELLFLAGS
new file mode 100644
index 00000000..3ce22890
--- /dev/null
+++ b/tests/scripts/variables/SHELLFLAGS
@@ -0,0 +1,197 @@
+#                                                                    -*-perl-*-
+$description = "Test behaviour of variable .SHELLFLAGS.";
+
+$details = "";
+
+# Only bother with this on UNIX systems.
+$port_type eq 'UNIX' or return -1;
+$^O =~ /cygwin/ and return -1;
+
+create_file('args.pl',
+"#!$perl_name" .
+'
+foreach my $a (@ARGV) {
+  print "$a\n";
+}
+');
+
+chmod(0755, 'args.pl');
+
+# args.pl prints each argument followed by a newline.
+# This allows to test how make chops up shellflags to tokens.
+# In this hash table a key is the input value of shellflags and the value is
+# the output from args.pl, which is shellflags chopped up by make to tokens,
+# followed by the recipe, which is :.
+my %input;
+
+# Valid inputs.
+$input{""} = ":";
+$input{" "} = ":";
+$input{"   "} = ":";
+$input{" \t\n  "} = ":";
+$input{"hello\\world"} = "helloworld\n:";
+$input{"\\\\"} = "\\\n:";
+$input{"\\\n"} = ":";
+$input{"\\\nhello"} = "hello\n:";
+$input{"\\\n\\\nhello"} = "hello\n:";
+$input{"\\\n \\\nhello"} = "hello\n:";
+$input{" hello, world"} = "hello,\nworld\n:";
+$input{"\\ hello, world"} = " hello,\nworld\n:";
+$input{"\\\\ hello, world"} = "\\\nhello,\nworld\n:";
+$input{"\\\nhello\\\n \\\nworld\\\n "} = "hello\nworld\n:";
+$input{"\\\n \\\nhello\\\n \\\nworld"} = "hello\nworld\n:";
+$input{"hello "} = "hello\n:";
+$input{"hello\\\n"} = "hello\n:";
+$input{"hello\\\\\n"} = "hello\\\n:";
+$input{"hello\\\n \\\n"} = "hello\n:";
+$input{"hello\\\\\n \\\n"} = "hello\\\n:";
+$input{"hello\\\\"} = "hello\\\n:";
+$input{"hello\\\\ \\\n"} = "hello\\\n:";
+$input{"hello\\ \\\n"} = "hello \n:";
+$input{"hello,\\\nworld"} = "hello,\nworld\n:";
+$input{"hello, world"} = "hello,\nworld\n:";
+$input{"hello, world\n"} = "hello,\nworld\n:";
+$input{"hello,\\ world"} = "hello, world\n:";
+$input{"hello,\\  world"} = "hello, \nworld\n:";
+$input{"a\\ hello, world\\ b"} = "a hello,\nworld b\n:";
+$input{"\\'"} = "'\n:";
+$input{"\\\\\\'"} = "\\'\n:";
+$input{"\\'hello"} = "'hello\n:";
+$input{"a \\'hello, world\\' b"} = "a\n'hello,\nworld'\nb\n:";
+$input{"a 'hello one\" \"two world' b"} = "a\nhello one\" \"two world\nb\n:";
+$input{"a 'hello one\\' \\'two world b"} = "a\nhello one\\\n'two\nworld\nb\n:";
+$input{"a \"hello one\\\" \\\"two world\" b"} = "a\nhello one\" \"two world\nb\n:";
+$input{"a \\\"hello one\\\" \\\"two world b"} = "a\n\"hello\none\"\n\"two\nworld\nb\n:";
+$input{"\\\\\\'hello"} = "\\'hello\n:";
+$input{"hello\\'"} = "hello'\n:";
+$input{"hello\\\\\\'"} = "hello\\'\n:";
+$input{"hello\\'world"} = "hello'world\n:";
+$input{"hello\\\\\\'world"} = "hello\\'world\n:";
+$input{"''"} = "\n:";
+$input{"''''"} = "\n:";
+$input{"'hello, world'"} = "hello, world\n:";
+$input{"'hello,\\ world'"} = "hello,\\ world\n:";
+$input{"'hello,\\\nworld'"} = "hello, world\n:";
+$input{"\"hello,\\\nworld\""} = "hello, world\n:";
+$input{"\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld\""} = "hello, world\n:";
+$input{"\"a\\\nb\\\nc\""} = "a b c\n:";
+$input{"\"hello, \t \\\n \t \\\n\t \t\\\n  \t\tworld  bye,\\\n \t \\\n \t \\\n \t   \\\n  \\\nmoon\""} = "hello, world  bye, moon\n:";
+$input{"'hello,\\\\ world'"} = "hello,\\\\ world\n:";
+$input{"\"hello,\\\\ world\""} = "hello,\\ world\n:";
+$input{"\"hello,\\\" world\""} = "hello,\" world\n:";
+$input{"\"hello,\\\\\\\" world\""} = "hello,\\\" world\n:";
+$input{"one\\\\two\\ tree\\\" four\\\nfive"} = "one\\two tree\"\nfour\nfive\n:";
+$input{"\"one\\\\two\\ tree\\\" four\\\nfive\""} = "one\\two\\ tree\" four five\n:";
+$input{"'hello, \"one two\" world'"} = "hello, \"one two\" world\n:";
+$input{"a 'hello one\\\\' \\\\'two world' b"} = "a\nhello one\\\\\n\\two world\nb\n:";
+$input{"a \"hello one\\\\\" \\\\\"two world\" b"} = "a\nhello one\\\n\\two world\nb\n:";
+$input{"a 'hello one\\\\\\\\' \\\\\\\\'two world' b"} = "a\nhello one\\\\\\\\\n\\\\two world\nb\n:";
+$input{"a \"hello one\\\\\\\\\" \\\\\\\\\"two world\" b"} = "a\nhello one\\\\\n\\\\two world\nb\n:";
+$input{"'a \"one 'cd' two\" b'"} = "a \"one cd two\" b\n:";
+$input{"'a \"one 'c d' two\" b'"} = "a \"one c\nd two\" b\n:";
+$input{"a 'hell\"o''w\"orld' b"} = "a\nhell\"ow\"orld\nb\n:";
+$input{"a \\'hell\"o\\'\\'w\"orld\\' b"} = "a\n'hello\\'\\'world'\nb\n:";
+$input{"\\'a \"one 'cd' two\" \\'b"} = "'a\none 'cd' two\n'b\n:";
+$input{"'hello, world' "} = "hello, world\n:";
+$input{"'hello'\\ "} = "hello \n:";
+$input{"'hello'\\\n"} = "hello\n:";
+$input{"'hello, world'\\ "} = "hello, world \n:";
+$input{"'hello, world'\\\n"} = "hello, world\n:";
+$input{"'a'b'c'"} = "abc\n:";
+$input{"a'b'c'd'"} = "abcd\n:";
+$input{"'\\\\'abc''"} = "\\\\abc\n:";
+$input{"\"\\\\\"abc\"\""} = "\\abc\n:";
+$input{"\"\\\"\\\"\""} = "\"\"\n:";
+$input{"'\\\\'\\\\''"} = "\\\\\\\n:";
+$input{"\"\\\\\"\\\\\"\""} = "\\\\\n:";
+$input{"  a  '  b  '  c  '  d  '  "} = "a\n  b  \nc\n  d  \n:";
+$input{"'hello'world\\'12''"} = "helloworld'12\n:";
+$input{"01'2 3'45\"67 89\"ab\\ cd"} = "012 34567 89ab cd\n:";
+$input{"-w -E 'use warnings FATAL => \"all\";' -E"} = "-w\n-E\nuse warnings FATAL => \"all\";\n-E\n:";
+$input{"\\"} = "#MAKE#: *** No targets.  Stop.";
+$input{"hello,\\\\\nworld"} = "#MAKEFILE#:5: *** missing separator.  Stop.";
+$input{"\"hello,\\\\\nworld\""} = "#MAKEFILE#:5: *** missing separator.  Stop.";
+
+# The following is a single quoted newline. The parser in read.c goes line by
+# line and ignores the single quotes and reads line 'hello,\\
+# and believes it is missing a separator.
+$input{"'hello,\\\\\nworld'"} = "#MAKEFILE#:5: *** missing separator.  Stop.";
+
+# make is supposed to preserve a single quoted backslash-newline combination
+# intact.
+# However, in the current impl collapse_continuations replaces each such
+# backslash-newline combination with a space, whether this combination is
+# single quoted or not.
+#
+# On top of that, collapse_continuations halfs the number consecutive
+# backslashes and then construct_command_argv_internal halfs the remaining
+# backslashes again. This behavior causes two consecutive backslashes to be
+# removed, rather than replaced with one backslash. So, given input like
+# hello,\\\
+# world
+# (three backslashes, followed by a newline)
+# collapse_continuations replaces two leading backslashes with one backslash
+# and replaces the combination of backslash-newline with a space and produces
+# hello,\ world
+# Then construct_command_argv_internal removes the lone backslash and produces
+# hello, world.
+# next_dequoted_token mimicks the behavior of construct_command_argv_internal
+# and removes double quoted backslash-newline combinations and those outside of
+# quotes.
+# next_dequoted_token also halfs backslashes.
+# The following tests just test that the current incorrect behavior is the
+# same whether .ONESHELL is active or not.
+
+# The following result is wrong due to consecutive backslashes.
+$input{"hello,\\\\\\\nworld"} = "hello, world\n:";
+# The following result is wrong due to a backslash-newline combination within
+# single quotes.
+$input{"'one\\\\two\\ tree\\\" four\\\nfive'"} = "one\\\\two\\ tree\\\" four five\n:";
+
+# This string is 120K bytes long.
+$input{'hello world ' x 10000} = ("hello\nworld\n" x 10000) . ":";
+# This string is 104K bytes long and contains quotes.
+$input{'"hello world" ' x 8000} = ("hello world\n" x 8000) . ":";
+# This string is 104K bytes long and contains quotes.
+$input{"'hello world' " x 8000} = ("hello world\n" x 8000) . ":";
+
+my @oneshell = ('', '.ONESHELL:');
+foreach my $sflags (keys %input) {
+my $answer = $input{$sflags};
+my $code = $answer =~ "Stop" || $answer =~ "Error 1" ? 512 : 0;
+for my $osh (@oneshell) {
+    run_make_test("
+$osh
+SHELL:=./args.pl
+.SHELLFLAGS:=$sflags
+all:;\@:
+    ", '', "$answer", $code);
+}
+}
+
+# Shellflags with characters of special interest to shell.
+# Run tests for special characters only in the oneshell mode, because in the
+# oneshell mode make execs ./args.pl. In the no-oneshell mode make execs sh -c
+# when the value of $SHELL is different from /bin/sh. These characters
+# cause /bin/sh to produce different messages that are harder to match than a
+# simple loop here.
+%input = ();
+for my $c (split //, ';*?[]&|<>(){}`^~!') {
+    my $k = "hello $c world";
+    my $v = "hello\n$c\nworld\n:";
+    $input{$k} = $v;
+}
+
+foreach my $sflags (keys %input) {
+my $answer = $input{$sflags};
+    run_make_test("
+.ONESHELL:
+SHELL:=./args.pl
+.SHELLFLAGS:=$sflags
+all:;\@:
+    ", '', "$answer");
+}
+
+unlink('args.pl') unless $keep;
+# This tells the test driver that the perl test script executed properly.
+1;
