# HG changeset patch
# User Michael C. Grant <mcg@cvxr.com>
# Date 1388957507 21600
#      Sun Jan 05 15:31:47 2014 -0600
# Node ID 4836f3a19e129e16051559135583a29be7b314a2
# Parent  cda4bd2fd0c09e1a3287cf990d991488c045404b
Attempts to bring the parsing of command syntax in line with MATLAB, including the funky brace matching approach. Even though it is not documented behavior, it is relied upon by MATLAB’s Symbolic Math Toolbox, so we have good reason to believe it is stable.

diff -r cda4bd2fd0c0 -r 4836f3a19e12 libinterp/parse-tree/lex.h
--- a/libinterp/parse-tree/lex.h	Fri Jan 03 20:55:33 2014 -0500
+++ b/libinterp/parse-tree/lex.h	Sun Jan 05 15:31:47 2014 -0600
@@ -275,7 +275,8 @@
       string_text (), string_line (0), string_column (0),
       fcn_file_name (), fcn_file_full_name (), looking_at_object_index (),
       parsed_function_name (), pending_local_variables (),
-      symtab_context (), nesting_level (), tokens ()
+      symtab_context (), nesting_level (), tokens (), 
+      command_arg_paren_count(0)
   {
     init ();
   }
@@ -445,6 +446,9 @@
   // Tokens generated by the lexer.
   token_cache tokens;
 
+  // Parenthesis count for command argument parsing.
+  int command_arg_paren_count;
+
 private:
 
   // No copying!
diff -r cda4bd2fd0c0 -r 4836f3a19e12 libinterp/parse-tree/lex.ll
--- a/libinterp/parse-tree/lex.ll	Fri Jan 03 20:55:33 2014 -0500
+++ b/libinterp/parse-tree/lex.ll	Sun Jan 05 15:31:47 2014 -0600
@@ -232,6 +232,17 @@
     } \
   while (0)
 
+// When a command argument boundary is detected, push out the
+// current argument being built. This one seems like a good
+// candidate for a function call.
+#define COMMAND_ARG_FINISH do { \
+  if ( !curr_lexer->string_text.empty() ) break; \
+  int retval = curr_lexer->handle_token( curr_lexer->string_text, SQ_STRING ); \
+  curr_lexer->string_text = ""; \
+  curr_lexer->command_arg_paren_count = 0; \
+  yyless(0); \
+  return retval; \
+} while(0)
 
 static bool Vdisplay_tokens = false;
 
@@ -283,54 +294,132 @@
 // Help and other command-style functions.
 %}
 
-<COMMAND_START>{NL} {
-    curr_lexer->lexer_debug ("<COMMAND_START>{NL}");
+%{
+// Commands can be continued on a second line using the ellipsis.
+// If an argument is in construction, it is completed.
+%}
+
+<COMMAND_START>(\.\.\.)[^\r\n]*{NL} {
+    curr_lexer->lexer_debug( "<COMMAND_START>(\\.\\.\\.)[^\\r\\n]*{NL}" );
+
+    COMMAND_ARG_FINISH;
 
     curr_lexer->input_line_number++;
     curr_lexer->current_input_column = 1;
 
+    HANDLE_STRING_CONTINUATION;
+  }
+
+%{
+// Commands normally end at the end of a line or a semicolon.
+%}
+
+<COMMAND_START>({CCHAR}[^\r\n]*)?{NL} {
+    curr_lexer->lexer_debug( "<COMMAND_START>({CCHAR}[^\\r\\n]*)?{NL}" );
+
+    COMMAND_ARG_FINISH;
+
+    curr_lexer->input_line_number++;
+    curr_lexer->current_input_column = 1;
+
     curr_lexer->looking_for_object_index = false;
     curr_lexer->at_beginning_of_statement = true;
 
     curr_lexer->pop_start_state ();
 
-    return curr_lexer->count_token ('\n');
+    return curr_lexer->handle_token('\n');
   }
 
-<COMMAND_START>[\;\,] {
-    curr_lexer->lexer_debug ("<COMMAND_START>[\\;\\,]");
-
-    curr_lexer->looking_for_object_index = false;
-    curr_lexer->at_beginning_of_statement = true;
-
-    curr_lexer->pop_start_state ();
-
-    if (strcmp (yytext, ",") == 0)
-      return curr_lexer->handle_token (',');
-    else
-      return curr_lexer->handle_token (';');
+<COMMAND_START>[\,\;] {
+    curr_lexer->lexer_debug( "<COMMAND_START>[\\,\\;]" );
+
+    if ( yytext[0] != ',' || curr_lexer->command_arg_paren_count == 0 ) {
+
+      COMMAND_ARG_FINISH;
+
+      curr_lexer->looking_for_object_index = false;
+      curr_lexer->at_beginning_of_statement = true;
+
+      curr_lexer->pop_start_state ();
+
+      return curr_lexer->handle_token(yytext[0]);
+
+    } else
+
+      curr_lexer->string_text += yytext;
+
+    curr_lexer->current_input_column += yyleng;
   }
 
+%{
+// Unbalanced parentheses serve as pseudo-quotes: they are included in the final
+// argument string, but they cause parentheses and quotes to be slurped into that
+// argument as well.
+%}
+
+<COMMAND_START>[\(\[\{]+ {
+    curr_lexer->lexer_debug("<COMMAND_START>[\\(\\[\\{]+");
+
+    curr_lexer->command_arg_paren_count += yyleng;
+
+    curr_lexer->string_text += yytext;
+    curr_lexer->current_input_column += yyleng;
+  }
+
+<COMMAND_START>[\)\]\}]+ {
+    curr_lexer->lexer_debug("<COMMAND_START>[\\)\\]\\}]+");
+
+    curr_lexer->command_arg_paren_count -= yyleng;
+
+    curr_lexer->string_text += yytext;
+    curr_lexer->current_input_column += yyleng;
+  }
+
+%{
+// Handle quoted strings. Quoted strings that are not separated by whitespace
+// from other argument text are combined with that previous text. For instance,
+//     command 'text1'"text2"
+// has a single argument text1text2, not two separate arguments. That's why 
+// we must test to see if we are in command argument mode when processing
+// the end of a string.
+%}
+
 <COMMAND_START>[\"\'] {
     curr_lexer->lexer_debug ("<COMMAND_START>[\\\"\\']");
 
-    curr_lexer->at_beginning_of_statement = false;
-
-    curr_lexer->current_input_column++;
-
-    curr_lexer->begin_string (yytext[0] == '"'
-                              ? DQ_STRING_START : SQ_STRING_START);
+    if ( curr_lexer->command_arg_paren_count == 0 )
+        curr_lexer->begin_string( yytext[0] == '"' ? DQ_STRING_START : SQ_STRING_START );
+    else
+        curr_lexer->string_text += yytext;
+
+    curr_lexer->current_input_column += yyleng;
   }
 
-<COMMAND_START>[^#% \t\r\n\;\,\"\'][^ \t\r\n\;\,]*{S}* {
-    curr_lexer->lexer_debug ("<COMMAND_START>[^#% \\t\\r\\n\\;\\,\\\"\\'][^ \\t\\r\\n\\;\\,]*{S}*");
-
-    std::string tok = strip_trailing_whitespace (yytext);
-
-    curr_lexer->looking_for_object_index = false;
-    curr_lexer->at_beginning_of_statement = false;
-
-    return curr_lexer->handle_token (tok, SQ_STRING);
+%{
+// In standard command argument processing, whitespace separates arguments. 
+// In the presence of unbalanced parentheses, it is incorporated into the argument.
+%}
+
+<COMMAND_START>{S}+ {
+    curr_lexer->lexer_debug ("<COMMAND_START>{S}+");
+
+    if ( curr_lexer->command_arg_paren_count == 0 )
+  	    COMMAND_ARG_FINISH;
+    else
+    	  curr_lexer->string_text += yytext;
+
+    curr_lexer->current_input_column += yyleng;
+  }
+
+%{
+// Everything else is slurped into the command arguments.
+%}
+
+<COMMAND_START>([\.]|[^#% \t\r\n\,\;\"\'\(\[\{\}\]\)]+) {
+    curr_lexer->lexer_debug("<COMMAND_START>[^#% \\t\\r\\n\\.\\,\\;\\\"\\'\\(\\[\\{\\}\\]\\)]+");
+
+    curr_lexer->string_text += yytext;
+    curr_lexer->current_input_column += yyleng;
   }
 
 <MATRIX_START>{S}* {
@@ -678,17 +767,23 @@
 
     curr_lexer->pop_start_state ();
 
-    curr_lexer->looking_for_object_index = true;
-    curr_lexer->at_beginning_of_statement = false;
-
-    curr_lexer->push_token (new token (DQ_STRING,
-                                       curr_lexer->string_text,
-                                       curr_lexer->string_line,
-                                       curr_lexer->string_column));
-
-    curr_lexer->string_text = "";
-
-    return curr_lexer->count_token_internal (DQ_STRING);
+    // Do not output the string if we are in a command
+    // context; let the command state handle that. This is
+    // because command arguments can combine multiple quoted
+    // and un-quoted strings together.
+    if ( curr_lexer->start_state() != COMMAND_START ) {
+      curr_lexer->looking_for_object_index = true;
+      curr_lexer->at_beginning_of_statement = false;
+
+      curr_lexer->push_token (new token (DQ_STRING,
+                                         curr_lexer->string_text,
+                                         curr_lexer->string_line,
+                                         curr_lexer->string_column));
+
+      curr_lexer->string_text = "";
+
+      return curr_lexer->count_token_internal (DQ_STRING);
+    }
   }
 
 <DQ_STRING_START>\\[0-7]{1,3} {
@@ -861,17 +956,23 @@
 
     curr_lexer->pop_start_state ();
 
-    curr_lexer->looking_for_object_index = true;
-    curr_lexer->at_beginning_of_statement = false;
-
-    curr_lexer->push_token (new token (SQ_STRING,
-                                       curr_lexer->string_text,
-                                       curr_lexer->string_line,
-                                       curr_lexer->string_column));
-
-    curr_lexer->string_text = "";
-
-    return curr_lexer->count_token_internal (SQ_STRING);
+    // Do not output the string if we are in a command
+    // context; let the command state handle that. This is
+    // because command arguments can combine multiple quoted
+    // and un-quoted strings together.
+    if ( curr_lexer->start_state() != COMMAND_START ) {
+      curr_lexer->looking_for_object_index = true;
+      curr_lexer->at_beginning_of_statement = false;
+
+      curr_lexer->push_token (new token (SQ_STRING,
+                                         curr_lexer->string_text,
+                                         curr_lexer->string_line,
+                                         curr_lexer->string_column));
+
+      curr_lexer->string_text = "";
+
+      return curr_lexer->count_token_internal (SQ_STRING);
+    }
   }
 
 <SQ_STRING_START>[^\'\n\r]+ {
@@ -1851,6 +1952,7 @@
   fcn_file_full_name = "";
   looking_at_object_index.clear ();
   looking_at_object_index.push_front (false);
+  command_arg_paren_count = 0;
 
   while (! parsed_function_name.empty ())
     parsed_function_name.pop ();
@@ -2604,7 +2706,7 @@
     }
 
   push_token (new token (SUPERCLASSREF, meth, pkg, cls,
-                         input_line_number, current_input_column));
+               input_line_number, current_input_column));
 
   current_input_column += flex_yyleng ();
 
@@ -2627,7 +2729,7 @@
   int kw_token = is_keyword_token (cls) || is_keyword_token (pkg);
   if (kw_token)
     {
-      error ("class and package names may not be keywords");
+       error ("class and package names may not be keywords");
       return LEXICAL_ERROR;
     }
 
