# HG changeset patch
# User Philip Nienhuis <prnienhuis@users.sf.net>
# Date 1338233613 -7200
# Node ID f48f40118d000bc56aa41f2a5cee5a0870a52738
# Parent  c2411bff11c6339a3660e7102ed468140cfc7867
Add support for pattern matching and bit width conversion specifiers (bug #33875)
**** strread.m: fix for bugs #36356, #36392, 36398 included;
      Fix for when specified width specifiers (%5s, %8n) exactly matched words;
      Added support for pattern matching (%[ ], %*[ ], %[^ ], %*[^ ]), given
      current strread setup this is as best we can get;
      Added support for bit width numerical conversion specifiers (%u8, %u16,
      %u32, %u64, %d8, %d16, %d32, %d64, %f32, %f64);
      Updated texinfo header;
      Catch remaining ML-compatible but not supported conversion specifiers
      (bug #36464);
      Several tests added.
**** textscan: fix for bugs #35356, #36392

diff -r c2411bff11c6 -r f48f40118d00 scripts/io/strread.m
--- a/scripts/io/strread.m	Mon May 28 09:40:03 2012 -0700
+++ b/scripts/io/strread.m	Mon May 28 21:33:33 2012 +0200
@@ -47,12 +47,35 @@
 ## @itemx %u
 ## The word is parsed as a number and converted to int32.
 ##
-## @item %*', '%*f', '%*s
+## @item %*, %*f, %*s
 ## The word is skipped.
 ##
 ## For %s and %d, %f, %n, %u and the associated %*s @dots{} specifiers an
-## optional width can be specified as %Ns, etc. where N is an integer > 1.
-## For %f, format specifiers like %N.Mf are allowed.
+## optional character width can be specified as %Ns, etc. where N is an
+## integer > 1. For %f, format specifiers like %N.Mf are allowed.
+##
+## For %f, %u, and %d specifiers, a bit width can be specified and the
+## relating read numbers will be casted to that bit width: %f32 to single,
+## %f64 to double, %d8, %d16, %d32 and %d64 to int8, int16, int32, and int64,
+## respectively; same for %u8 ... %u64 (uint8 ... uint64).
+##
+## @item %[ ]
+## Pattern matching: read only characters matching those enclosed between the
+## brackets til the first non-matching character or delimiter.
+## @item %[^ ]
+## Read only characters NOT matching those between %[^ and ] til the first
+## matching character.
+##
+## @item %*[ ], %*[^ ]
+## Skip matching or non-matching characters, respectively.
+##
+## If ']' should be part of the pattern it must be the first character of the
+## actual pattern after the starting [ or [^: %[] ] or %[^] ]
+##
+## NOTE: patterns containing whitespace may not work as expected due to
+## the way strread currently works. Patterns containing delimiters may not
+## work at all, especially if the number of enclosed delimiters differs from 
+## line to line. endofline (EOL) characters have a better chance to work.
 ##
 ## @item literals
 ## In addition the format may contain literal character strings; these will be
@@ -102,7 +125,6 @@
 ## @item "commentstyle"
 ## Parts of @var{str} are considered comments and will be skipped.
 ## @var{value} is the comment style and can be any of the following.
-##
 ## @itemize
 ## @item "shell"
 ## Everything from @code{#} characters to the nearest end-of-line is skipped.
@@ -154,6 +176,20 @@
 ##
 ## @end table
 ##
+## When the number of words in @var{str} doesn't match an exact multiple
+## of the number of format conversion specifiers, strread's behavior
+## depends on the last character of @var{str}:
+##
+## @table @asis
+## @item last character = "\n"
+## Data columns are padded with empty fields or Nan so that all columns
+## have equal length 
+##
+## @item last character is not "\n"
+## Data columns are not padded; strread returns columns of uneven length
+##
+## @end table
+##
 ## @seealso{textscan, textread, load, dlmread, fscanf}
 ## @end deftypefn
 
@@ -163,12 +199,6 @@
   if (nargin < 1)
     print_usage ();
   endif
-  
-  if (isempty (str))
-    ## Return empty args (no match), rather than raising an error
-    varargout = cell (1, nargout);
-    return;
-  endif
 
   if (isempty (format))
     format = "%f";
@@ -279,21 +309,93 @@
     ## Default format specified.  Expand it (to desired nargout)
     fmt_words = cell (nargout, 1);
     fmt_words (1:nargout) = format;
+    idw = zeros (1, numel (fmt_words));
   else
     ## Determine the number of words per line as a first guess.  Forms
     ## like %f<literal>) (w/o delimiter in between) are fixed further on
     format = strrep (format, "%", " %");
     fmt_words = regexp (format, '[^ ]+', 'match');
+
+    ## Find position of conversion specifiers (they start with %)
+    idy2 = find (! cellfun("isempty", regexp(fmt_words, '^%')));
+
+    ## Check for unsupported format specifiers
+    errpat = '(^%\*?[cq])';
+    if (! all (cellfun ("isempty", regexp (fmt_words(idy2), errpat))))
+      error ("strread: %q and %c format specifiers are not supported (yet)");
+    endif
+
+    ## Check for whitespace or delimiters in %[] style fmt_words.
+    ## After all, patterns like "%[] [%ab\ncdefg]" are allowed...
+    ii = 1; num_words_per_line = numel (fmt_words);
+    ## idw keeps track of which %[] fmt_words contain whitespace/delims
+    idw = zeros (1, num_words_per_line);
+    while ii <= num_words_per_line
+      if (! isempty (regexp (fmt_words{ii}, '%\*?\[')))
+        ## %[] style conv specifier. Find begin pos of actual pattern chars
+        ## 1. Check if pattern characters contains ']' 
+        es = regexp (fmt_words{ii}, '%(\*?\[\]|\*?\[\^\])', 'end');
+        if (isempty (es))
+          ## 2. No, assume pattern w/o ']'
+          es = regexp (fmt_words{ii}, '%(\*?\[|\*?\[\^)', 'end');
+        endif
+        ## Check if closing ']' of pattern is in fmt_words{ii}.
+        ## If fmt_words{i} ends after es we can skip searching ']'
+        no_close_brckt = (es == length (fmt_words{ii}));
+        if (no_close_brckt || isempty (strfind (fmt_words{ii}(es+1:end), ']')))
+          ## Apparently one or more spaces / delimiters in pattern as fmt_words
+          ## got split up there. Check next fmt_words where matching ] is
+          jj = ii + 1; mtch_ptrn = 0;
+          while jj <= num_words_per_line
+            ## Even "%", "*", "[", "^" and "]" (in that order) could be in pattern
+            if (! isempty (regexp (fmt_words{ii+1}, '%\*?\[\^')))
+              if (mtch_ptrn)
+                error ("strread - ambiguous format specifier # %d", ii);
+              else
+                mtch_ptrn = 1;
+              endif
+            endif
+            if (isempty (regexp (fmt_words{jj}, '\]')))
+              ## Found no matching ] yet, search next fmt_words
+              ++jj;
+            else 
+              ## Found the missing ]. Concatenate adjacent fmt_words
+              fmt_words{ii} = [fmt_words{ii:jj}];
+              fmt_words(ii+1:jj) = [];
+              num_words_per_line -= jj - ii;
+              ## For %[^..], the delim/whitesp behavior is inverted
+              idw(ii) = 1;
+              idw(ii+1:jj) = [];
+              jj = num_words_per_line + 1;
+            endif
+          endwhile
+        else
+          idw(ii) = ! isempty (regexp (fmt_words{ii}, '%\*?\[\^?'));
+        endif
+      endif
+      ++ii;
+    endwhile
+    ## Just renew idy2
+    idy2 = find (! cellfun ("isempty", regexp (fmt_words, '^%')));
+
+    allwptrn = '([nfdus]8|[nfdus]16|[nfdus]32|[nfdus]64)';
     ## Format conversion specifiers following literals w/o space/delim
     ## in between are separate now.  Separate those w trailing literals
-    idy2 = find (! cellfun ("isempty", strfind (fmt_words, "%")));
     a = strfind (fmt_words(idy2), "%");
-    b = regexp (fmt_words(idy2), '[nfdus]', 'end');
+    b = regexp (fmt_words(idy2), '(\[.*\]|[nfdusq])', 'end');
     for jj = 1:numel (a)
+      ## From right to left to avoid losing track
       ii = numel (a) - jj + 1;
+      [re] = regexp (fmt_words{ii}, allwptrn, 'end');
+      if (!isempty (re))
+        ## Add bit width specifier to conv. specifier
+        b{ii}(1) = re;
+      endif
       if (! (length (fmt_words{idy2(ii)}) == b{ii}(1)))
-        ## Fix format_words
+        ## Split fmt_words(ii) into % conv specifier and trailing literal
         fmt_words(idy2(ii)+1 : end+1) = fmt_words(idy2(ii) : end);
+        idw(idy2(ii)+1 : end+1) = idw(idy2(ii) : end);
+        idw(idy2(ii)+1) = 0;
         fmt_words{idy2(ii)} = fmt_words{idy2(ii)}(a{ii} : b{ii}(1));
         fmt_words{idy2(ii)+1} = fmt_words{idy2(ii)+1}(b{ii}+1:end);
       endif
@@ -342,7 +444,8 @@
   endif
 
   if (! isempty (white_spaces))
-    ## For numeric fields, whitespace is always a delimiter, but not for text fields
+    ## For numeric fields, whitespace is always a delimiter, but not for text
+    ## fields
     if (isempty (strfind (format, "%s")))
       ## Add whitespace to delimiter set
       delimiter_str = unique ([white_spaces delimiter_str]);
@@ -361,6 +464,25 @@
     white_spaces = strrep (white_spaces, eol_char, '');
   endif
 
+  ## Check if pattern contains EOL (a special one as EOL can be used
+  ## to reading until EOL while skipping other delimiters
+  ideol = zeros (1, num_words_per_line);
+  if (! isempty (eol_char))
+    for ii=1:num_words_per_line
+      if (idw(ii))
+        ideol(ii) = ! isempty (strfind (fmt_words{ii}, eol_char));
+      endif
+    endfor
+  endif
+  if (sum(ideol))
+    ## Prepend char(253) to all EOL in str so some trace of EOL is left
+    ## FIXME this assumes char(253) doesn't occur in str
+    str = strrep (str, eol_char, [char(253) eol_char]);
+    ## Ensure last char was eol_char
+    if (str(end) != char(253)); str(end+1) = char(253); endif
+    fmt_words = strrep (fmt_words, eol_char, char(253));
+  endif
+
   pad_out = 0;
   ## Trim whitespace if needed
   if (! isempty (white_spaces))
@@ -378,8 +500,6 @@
        str = str(2:end);
     endif
     ## Check for single delimiter followed/preceded by whitespace
-    ## FIXME: Double strrep on str is enormously expensive of CPU time.
-    ## Can this be eliminated
     if (! isempty (delimiter_str))
       dlmstr = setdiff (delimiter_str, " ");
       rxp_dlmwsp = sprintf ("( [%s]|[%s] )", dlmstr, dlmstr);
@@ -387,7 +507,8 @@
     endif
     ## FIXME: Double strrep on str is enormously expensive of CPU time.
     ## Can this be eliminated
-    ## Wipe leading and trailing whitespace on each line (it may be delimiter too)
+    ## Wipe leading and trailing whitespace on each line (it may be
+    ## delimiter too)
     if (! isempty (eol_char))
       str = strrep (str, [eol_char " "], eol_char);
       str = strrep (str, [" " eol_char], eol_char);
@@ -403,13 +524,13 @@
     words = strtrim (words);
   endif
   num_words = numel (words);
-  ## First guess at number of lines in file (ignoring leading/trailing literals)
+  ## First guess at nr. of lines in file (ignoring leading/trailing literals)
   num_lines = ceil (num_words / num_words_per_line);
 
   ## Replace TreatAsEmpty char sequences by empty strings
   if (! isempty (empty_str))
     for ii = 1:numel (empty_str)
-      idz = strncmp (empty_str{ii}, words, length (empty_str{ii}));
+      idz = strmatch (empty_str{ii}, words, "exact");
       words(idz) = {""};
     endfor
   endif
@@ -420,6 +541,8 @@
   ## needed.
   ## We also don't know the number of lines (as EndOfLine may have been set to
   ## "" (empty) by the caller).
+  ## In addition, words may have been split within %[ ] matches if the matched
+  ## string contained delimiters.
   ##
   ## We also may have to cope with 3 cases as far as literals go:
   ## A: Trailing literals (%f<literal>) w/o delimiter in between.
@@ -431,7 +554,11 @@
   idf = cellfun ("isempty", strfind (fmt_words, "%"));
   ## Find indices and pointers to conversion specifiers with fixed width
   idg = ! cellfun ("isempty", regexp (fmt_words, '%\*?\d'));
-  idy = find (idf | idg);
+  ## Find indices and pointers to regexp-like conversion specifiers
+  idr = ! cellfun ("isempty", regexp (fmt_words, '%\*?\[.*\]', 'end'));
+  ## Check which [] fmt specifiers (im/explicitly) contain delimiters
+  
+  idy = find (idf | idg | idr);
   ## Find indices to numeric conversion specifiers
   idn = ! cellfun ("isempty", regexp (fmt_words, "%[dnfu]"));
 
@@ -445,11 +572,66 @@
       ## Alternative below goes by simply parsing a first grab of words
       ## and counting words until the fmt_words array is exhausted:
       iwrd = 1; iwrdp = 0; iwrdl = length (words{iwrd});
-      for ii = 1:numel (fmt_words)
+      sttc = endc = fwptr = zeros (1, numel (fmt_words));
+      ii = 1;
+      while ii <= numel (fmt_words)
 
         nxt_wrd = 0;
+        ## Keep track of which words nr. every fmt_words{} is (starts) in.
+        fwptr(ii) = iwrd;
 
-        if (idf(ii))
+        if (idr(ii))
+          ## Regexp-like specifier. Search current + next words until satisfied
+          ## sttc & endc keep track of columns to be concatenated later on
+          sttc(ii) = jj = iwrd;
+          ptrn = fmt_words{ii}(index (fmt_words{ii}, '['): end);
+          while jj <= numel (words)
+            [rs, re, ~, ~, ~, ~, nm] = regexp (words{jj}(iwrdp+1:end), ptrn);
+            if (isempty (rs))
+              ## No match at all, stop search
+              iwrd = jj;
+              iwrdl = length (words{jj});
+              jj = numel(words) + 1;
+            else
+              if (! length ([nm{:}]) == 0)
+                ## Match ends within word. Find where & update iwrdp
+                if (ptrn(2) == '^')
+                  invptrn = ['[' ptrn(3:end)];
+                else
+                  invptrn = ['[^' ptrn(2:end)];
+                endif
+                iwrdp = regexp (words{jj}(iwrdp+1:end), invptrn, 'once') - 1;
+                iwrd = jj;
+                if (iwrdl - iwrdp == 1)
+                  ## Match at last char
+                  nxt_wrd = 1;
+                endif
+                jj = numel (words) + 1;
+              else
+                ## Match ends at last char
+                if (idw(ii))
+                  ## Pattern contains whitespace/delims
+                  ## => next word may contain more matches. Continue there
+                  ++jj;
+                  if (jj <= numel (words))
+                    iwrdl = length (words{jj});
+                    iwrdp = 0;
+                  else
+                    iwrd = jj;
+                    nxt_wrd = 1;
+                  endif
+                else
+                  ## Match should end at delimiter
+                  iwrd = jj;
+                  jj = numel (words) + 1;
+                  nxt_wrd = 1;
+                endif
+              endif
+            endif
+          endwhile
+          endc(ii) = min (iwrd, numel (words));
+
+        elseif (idf(ii))
           ## Literal expected
           if (isempty (strfind (fmt_words{ii}, words(iwrd))))
             ## Not found in current word; supposed to be in next word
@@ -471,8 +653,9 @@
 
         elseif (idg(ii))
           ## Fixed width specifier (%N or %*N): read just a part of word
-          iwrdp += floor ...
-           (str2double (fmt_words{ii}(regexp(fmt_words{ii}, '\d') : end-1)));
+          sw = regexp (fmt_words{ii}, '\d', 'once');
+          ew = regexp (fmt_words{ii}, '[nfudsq]') - 1;
+          iwrdp += floor (str2double (fmt_words{ii}(sw:ew)));
           if (iwrdp > iwrdl)
             ## Match error. Field extends beyond word boundary.
             warning  ...
@@ -495,7 +678,7 @@
               ## ...but not found in current word => field uses rest of word
               nxt_wrd = 1;
             else
-              ## ..or it IS found.  Add inferred width of current conversion field
+              ## ...it IS found. Add inferred width of current conversion field
               iwrdp += index (words{iwrd}(iwrdp+1:end), fmt_words{ii+1}) - 1;
             endif
           elseif (iwrdp < iwrdl)
@@ -507,25 +690,52 @@
 
         if (nxt_wrd)
           ++iwrd; iwrdp = 0;
-          if (ii < numel (fmt_words))
+          if (iwrd > numel (words))
+            ## Apparently EOF; assume incomplete row already at L.1 of data
+            ii = numel (fmt_words);
+          elseif (ii < numel (fmt_words) && iwrd < numel (words))
             iwrdl = length (words{iwrd});
           endif
         endif
 
-      endfor
+      endwhile
       ## Done
       words_period = max (iwrd - 1, 1);
       num_lines = ceil (num_words / words_period);
 
       ## 2. Pad words array so that it can be reshaped
-      tmp_lines = ceil (num_words / words_period);
-      num_words_padded = tmp_lines * words_period - num_words;
+      num_words_padded = num_lines * words_period - num_words;
       if (num_words_padded)
         words = [words'; cell(num_words_padded, 1)];
       endif
-      words = reshape (words, words_period, tmp_lines);
+      words = reshape (words, words_period, num_lines);
 
-      ## 3. Do the column splitting on rectangular words array
+      ## 3. Concatenate columns contained within single %q or %[] specifiers
+      ## Proceed backward (right to left) to avoid mixing up pointers
+      icol = length (fmt_words);
+      while (icol > 0)
+        if (sttc(icol) > 0)
+          ## For this fmt specifier some columns must be combined
+          ncols = endc(icol) - sttc(icol) + 1;
+          if (ncols > 1)
+            ## Combine columns. Transpose as combining later on is columnwise
+            tmpc = words(sttc(icol):endc(icol), :);
+            ## Add split to row below
+            ## FIXME: this assumes data doesn't contain char(255) 
+            tmpc(ncols+1, :) = char(255);
+            ## Assign combined columns to start column
+            words(sttc(icol), :) = strsplit ([tmpc{1:end-1}], char (255));
+            words(sttc(icol)+1:endc(icol), :) = [];
+            ## Update fmt_words pointers to words
+            if (icol < length (fmt_words))
+              fwptr(icol+1:end) = fwptr(icol+1:end) - ncols + 1;
+            endif
+          endif
+        endif
+        --icol;
+      endwhile
+
+      ## 4. Do the column splitting on rectangular words array
       icol = 1; ii = 1;    # icol = current column, ii = current fmt_word
       while (ii <= num_words_per_line)
 
@@ -556,6 +766,7 @@
                 (@(x) substr(x, e(1)+1, length(x)-e(1)), words(icol, jptr), ...
                 "UniformOutput", false);
               words(icol, jptr) = fmt_words{ii};
+              fwptr = [fwptr(1:ii) (++fwptr(ii+1:end))];
 
             else
               if (! idg(ii) && ! isempty (strfind (fmt_words{ii-1}, "%s")))
@@ -563,11 +774,19 @@
                 warning ("Ambiguous '%s' specifier next to literal in column %d", icol);
               elseif (idg(ii))
                 ## Current field = fixed width. Strip into icol, rest in icol+1
-                wdth = floor (str2double (fmt_words{ii}(regexp(fmt_words{ii}, ...
-                              '\d') : end-1)));
+                sw = regexp (fmt_words{ii}, '\d', 'once');
+                ew = regexp (fmt_words{ii}, '[nfudsq]') - 1;
+                wdth = floor (str2double (fmt_words{ii}(sw:ew)));
                 words(icol+1, jptr) = cellfun (@(x) x(wdth+1:end),
                      words(icol,jptr), "UniformOutput", false);
-                words(icol, jptr) = strtrunc (words(icol, jptr), wdth);
+                if (isempty ([words(icol+1, :){:}]))
+                  ## Apparently split wasn't needed as turns out to cover
+                  ## entire column. So delete column again
+                  words(icol+1, :) = [];
+                else
+                  words(icol, jptr) = strtrunc (words(icol, jptr), wdth);
+                  fwptr = [fwptr(1:ii) (++fwptr(ii+1:end))];
+                endif
               else
                 ## FIXME: this assumes char(254)/char(255) won't occur in input!
                 clear wrds;
@@ -584,16 +803,42 @@
                    char(254), fmt_words{ii}), char(255));
                 ## Former trailing literal may now be leading for next specifier
                 --ii;
+                fwptr = [fwptr(1:ii) (++fwptr(ii+1:end))];
               endif
             endif
           endif
 
+        elseif (idr(ii) && ii < num_words_per_line)
+          ## Leading %[..] type specifier. Split into two columns. This split
+          ## is simply unavoidable as we don't know what is left to follow %[]
+          ## 1. Insert a column after present column
+          words(icol+1:end+1, :) = words(icol:end, :);
+          ## 2. Amend pattern
+          ptrn = ['^' fmt_words{ii}(index (fmt_words{ii}, '['): end) '*'];
+          ## 3. Extract first column and remove optional trailing char(253)
+          words(icol, :) = regexp (words(icol, :), ptrn, 'match', 'once');
+          if (words{icol, 1}(end) == char(253))
+            words(icol, :) = strrep (words(icol, :), char(253), '');
+          endif
+          ## 4. Extract second column
+          words(icol+1, :) = regexprep (words(icol+1, :), ptrn, '');
+          ## 5. This split may not have been needed; delete 2nd col if empty
+          if (isempty ([words(icol+1, :){:}]))
+            words(icol+1, :) = [];
+          else
+            fwptr = [fwptr(1:ii) (++fwptr(ii+1:end))];
+          endif
+
+        elseif (idr(ii))
+          ## Just check for trailing char(253) and if present remove it
+          if (words{icol, 1}(end) == char(253))
+            words(icol, :) = strrep (words(icol, :), char(253), '');
+          endif
+
         else
           ## Conv. specifier.  Peek if next fmt_word needs split from current column
           if (ii < num_words_per_line)
-            if (idf(ii+1) && (! isempty (strfind (words{icol, 1}, fmt_words{ii+1}))))
-              --icol;
-            elseif (idg(ii+1))
+            if (fwptr(ii) == fwptr(ii+1))
               --icol;
             endif
           endif
@@ -630,7 +875,8 @@
       ## FIXME - add support for formats like "<%s>", "%[a-zA-Z]"
       ##         Someone with regexp experience is needed.
       switch fmt_words{m}(1:min (2, length (fmt_words{m})))
-        case "%s"
+        case {"%s", "%["}
+          ## for %[] column has been singled out above; just return as string
           if (pad_out)
             data(end+1:num_lines) = {""};
           endif
@@ -643,7 +889,26 @@
           if (! isempty (regexp (fmt_words{m}, "%[du]")))
             ## Cast to integer
             ## FIXME: NaNs will be transformed into zeros
-            data = int32 (data);
+            switch fmt_words{m}
+              case '%d8'
+                data = int8 (data);
+              case '%d16'
+                  data = int16 (data);
+              case '%d64'
+                  data = int64 (data);
+              case '%u8'
+                data = uint8 (data);
+              case '%u16'
+                  data = uint16 (data);
+              case {'%u32', '%u'}
+                  data = uint32 (data);
+              case '%u64'
+                  data = uint64 (data);
+              otherwise
+                data = int32 (data);
+            endswitch
+          elseif (regexp (fmt_words{m}, '%f32'))
+            data = single (data);
           endif
           data(n) = numeric_fill_value;
           if (pad_out)
@@ -652,26 +917,60 @@
           varargout{k} = data.';
           k++;
         case {"%0", "%1", "%2", "%3", "%4", "%5", "%6", "%7", "%8", "%9"}
-          nfmt = strsplit (fmt_words{m}(2:end-1), '.');
+          sw = regexp (fmt_words{m}, '\d', 'once');
+          ew = regexp (fmt_words{m}, '[nfudsq]') - 1;
+          nfmt = strsplit (fmt_words{m}(2:ew), '.');
           swidth = str2double (nfmt{1});
-          switch fmt_words{m}(end)
-            case {"d", "u", "f", "n%"}
+          switch fmt_words{m}(ew+1)
+            case {"d", "u", "f", "n"}
               n = cellfun ("isempty", data);
               ### FIXME - erroneously formatted data lead to NaN, not an error
+              data = str2double (data);
+                if (! isempty (regexp (fmt_words{m}, "f32")))
+                  data = single (data);
+                endif
+              ### FIXME - erroneously formatted data lead to NaN, not an error
               ###         => ReturnOnError can't be implemented for numeric data
-              data = str2double (strtrunc (data, swidth));
-              data(n) = numeric_fill_value;
               if (pad_out)
                 data(end+1:num_lines) = numeric_fill_value;
               endif
-              if (numel (nfmt) > 1)
-                sprec = str2double (nfmt{2});
-                data = 10^-sprec * round (10^sprec * data);
-              elseif (! isempty (regexp (fmt_words{m}, "[du]")))
+              if (! isempty (regexp (fmt_words{m}, "[du]")))
                 ## Cast to integer
                 ## FIXME: NaNs will be transformed into zeros
-                data = int32 (data);
+                s = regexp (fmt_words{m}, "[du]", 'start');
+                switch fmt_words{m}(s : end)
+                  case 'd8'
+                    data = int8 (data);
+                  case 'd16'
+                    data = int16 (data);
+                  case 'd64'
+                    data = int64 (data);
+                  case 'u8'
+                    data = uint8 (data);
+                  case 'u16'
+                    data = uint16 (data);
+                  case {'u32', 'u'}
+                    data = uint32 (data);
+                  case 'u64'
+                    data = uint64 (data);
+                  otherwise
+                    data = int32 (data);
+                endswitch
+              else
+                if (! isempty (strfind (fmt_words{m}, "f32")))
+                  data = single (data);
+                  if (numel (nfmt) > 1)
+                    sprec = str2double (nfmt{2});
+                    data = 10^-sprec * single (round (10^sprec * data));
+                  endif
+                else
+                  if (numel (nfmt) > 1)
+                    sprec = str2double (nfmt{2});
+                    data = 10^-sprec * double (round (10^sprec * data));
+                  endif
+                endif
               endif
+              data(n) = numeric_fill_value;
               varargout{k} = data.';
               k++;
             case "s"
@@ -896,9 +1195,95 @@
 %! assert (c, [0.94; 0.87], 0.01)
 
 %!test
 %! # Bug #35999
 %! [a, b, c] = strread ("", "%f");
 %! assert (isempty (a));
 %! assert (isempty (b));
 %! assert (isempty (c));
 
+%!test
+%! [a, b] = strread (["Empty 1" char(10)], "Empty%s %f");
+%! assert (a{1}, "1");
+%! assert (b, NaN);
 
+%!test
+%! [a, b] = strread (["Empty" char(10)], "Empty%f %f");
+%! assert (a, NaN);
+%! assert (b, NaN);
+
+%% %[] tests
+%% Plain [..] and *[..]
+%!test
+%! ar = "abcdefguvwxAny\nacegxyzTrailing\nJunk";
+%! [a, b] = strread (ar, "%[abcdefg] %*[uvwxyz] %s");
+%! assert (a, {"abcdefg"; "aceg"; ""});
+%! assert (b, {"Any"; "Trailing"; "Junk"});
+
+%% [^..] and *[^..]
+%!test
+%! br = "abcdefguvwx1Any\nacegxyz2Trailing\n3Junk";
+%! [a, b] = strread (br, "%[abcdefg] %*[^0123456789] %s");
+%! assert (a, {"abcdefg"; "aceg"; ""});
+%! assert (b, {"1Any"; "2Trailing"; "3Junk"});
+
+%% [..] and [^..] crossing delimiters
+%!test
+%! cr = "ab cd efguv wx1Any\na ce gx yz2Trailing\n3   Junk";
+%! [a, b] = strread (cr, "%[ abcdefg] %*[^0123456789] %s", "delimiter", " \n", "whitespace", "");
+%! assert (a, {"abcdefg"; "aceg"; ""});
+%! assert (b, {"1Any"; "2Trailing"; "3Junk"});
+
+%% Bug #36464
+%!test
+%! assert (strread ("1 2 3 4 5 6", "%*n%n%*[^\n]"), 2);
+
+%% Textscan example
+%!test
+%! st4 = {"1st string  123.45 MoreText 1953-05-14 60 82+i15 abcdefg", "\n";
+%! "2nd string  678.90 LessText 1961-02-01 52 53-i03 aceg", "\n"
+%! "3rd string  246.08 SameText 1973-12-13 40 12+i66 bdfh", "\n"
+%! "4th string  135.79 NoText 1982-09-27 29 13+i81 iiiiii", "\n"};
+%! C = textscan ([st4'{:}], "%s %*[^\n]");
+%! D = textscan ([st4'{:}], "%s string %n %*[^\n]");
+%! E = textscan ([st4'{:}], "%s string %n %[^T]Text %*[^\n]");
+%! assert (C{1}, {"1st"; "2nd"; "3rd"; "4th"});
+%! assert (D{1}, {"1st"; "2nd"; "3rd"; "4th"});
+%! assert (D{2}, [123.45; 678.90; 246.08; 135.79], 1e-10);
+%! assert (E{1}, {"1st"; "2nd"; "3rd"; "4th"});
+%! assert (E{2}, [123.45; 678.90; 246.08; 135.79], 1e-10);
+%! assert (E{3}, {"More"; "Less"; "Same"; "No"});
+
+%% test %[]] and %[^]]
+%!test
+%! assert (strread ('345]', "%*[123456]%[]]"){1}, "]");
+%! assert (strread ('345]', "%*[^]]%s"){1}, "]");
+
+%% Bit width
+%!test
+%! str = "500.3 Inf NaN 1e200 1e10 2e10 3e10 4e10 5e11 6e11 7e11 8e11";
+%! [a, b, c, d, e, f, g, h, i, j, k, l] = strread (str, "%n %n %f %f32 %u8 %u16 %u32 %u64 %d8 %d16 %d32 %d64");
+%! assert (a, double (500.3), 1e-15);
+%! assert (isinf (b), true);
+%! assert (isnan (c), true);
+%! assert (d, single (Inf));
+%! assert (e, uint8 (255));
+%! assert (f, uint16 (65535));
+%! assert (g, uint32 (4294967295));
+%! assert (h, uint64 (4e10));
+%! assert (i, int8 (127), 1e-15);
+%! assert (j, int16 (32767), 1e-15);
+%! assert (k, int32 (2147483647), 1e-15);
+%! assert (l, int64 (8e11), 1e-15);
+
+%% Bit width, fixed width conv. specifiers
+%!test
+%! str2 = "123456789012345 ";
+%! str2 = [str2 str2 str2 str2 str2 str2 str2 str2];
+%! str2 = [str2 " 123456789.01234 1234567890.1234 12345.678901234 12345.678901234"];
+%! pttrn = "%3u8%*s %5u16%*s %10u32%*s %15u64 %3d8%*s %5d16%*s %10d32%*s %15d64 %9f32%*s %14f64%*s %10.2f32%*s %12.2f64%*s";
+%! [a b c d e f g h i j k l] = strread (str2, pttrn, "delimiter", " ");
+%! assert (a, uint8 (123));
+%! assert (b, uint16 (12345));
+%! assert (c, uint32 (1234567890));
+%! assert (d, uint64 (123456789012345));
+%! assert (e, int8 (123));
+%! assert (f, int16 (12345));
+%! assert (g, int32 (1234567890));
+%! assert (h, int64 (123456789012345));
+%! assert (i, single (123456789), 1e-12);
+%! assert (j, double (1234567890.123), 1e-15);
+%! assert (k, single (12345.68), 1e-5);
+%! assert (l, double (12345.68), 1e-15);
diff -r c2411bff11c6 -r f48f40118d00 scripts/io/textscan.m
--- a/scripts/io/textscan.m	Mon May 28 09:40:03 2012 -0700
+++ b/scripts/io/textscan.m	Mon May 28 21:33:33 2012 +0200
@@ -25,13 +25,13 @@
 ## @deftypefnx {Function File} {[@var{C}, @var{position}] =} textscan (@var{fid}, @dots{})
 ## Read data from a text file or string.
 ##
-## The file associated with @var{fid} is read and parsed according to
-## @var{format}.  The function behaves like @code{strread} except it works by
-## parsing a file instead of a string.  See the documentation of
-## @code{strread} for details.
+## The string @var{str} or file associated with @var{fid} is read from and
+## parsed according to @var{format}. The function behaves like @code{strread}
+## except it can also read from file instead of a string. See the documentation
+## of @code{strread} for details.
 ##
-## In addition to the options supported by
-## @code{strread}, this function supports a few more:
+## In addition to the options supported by @code{strread}, this function
+## supports a few more:
 ##
 ## @itemize
 ## @item "collectoutput":
@@ -50,16 +50,19 @@
 ## @item "returnonerror":
 ## If set to numerical 1 or true (default), return normally when read errors
 ## have been encountered.  If set to 0 or false, return an error and no data.
+## As the string or file is read by columns rather than by rows, and because
+## textscan is fairly forgiving as regards read errors, setting this option
+## may have little or no actual effect.
 ## @end itemize
 ##
 ## When reading from a character string, optional input argument @var{n}
 ## the amount of data read).
-## When reading fro file, @var{n} specifies the number of data lines to read;
+## When reading from file, @var{n} specifes the number of data lines to read;
 ## in this sense it differs slightly from the format repeat count in strread.
 ##
-## The output @var{C} is a cell array whose length is given by the number
-## of format specifiers.
+## The output @var{C} is a cell array whose second dimension is determined
+## by the number of format specifiers.
 ##
 ## The second output, @var{position}, provides the position, in characters,
 ## from the beginning of the file.
@@ -80,14 +83,18 @@
     format = "%f";
   endif
 
+  if (! ischar (format))
+    error ("textscan: FORMAT must be a string");
+  endif
+
+  ## Determine the number of data fields & initialize output array
+  num_fields = numel (strfind (format, "%")) - numel (strfind (format, "%*"));
+  C = cell (1, num_fields);
+
   if (! (isa (fid, "double") && fid > 0) && ! ischar (fid))
     error ("textscan: first argument must be a file id or character string");
   endif
 
-  if (! ischar (format))
-    error ("textscan: FORMAT must be a string");
-  endif
-
   args = varargin;
   if (nargin > 2 && isnumeric (args{1}))
     nlines = args{1};
@@ -96,7 +103,6 @@
   endif
   if (nlines < 1)
     printf ("textscan: N = 0, no data read\n");
-    C = [];
     return
   endif
 
@@ -174,7 +180,6 @@
   ## Check for empty result
   if (isempty (str))
     warning ("textscan: no data read");
-    C = [];
     return;
   endif
 
@@ -193,10 +198,10 @@
     ## Determine EOL from file.  Search for EOL candidates in first BUFLENGTH chars
     eol_srch_len = min (length (str), BUFLENGTH);
     ## First try DOS (CRLF)
-    if (! isempty (strfind ("\r\n", str(1 : eol_srch_len))))
+    if (! isempty (strfind (str(1 : eol_srch_len), "\r\n")))
       eol_char = "\r\n";
     ## Perhaps old Macintosh? (CR)
-    elseif (! isempty (strfind ("\r", str(1 : eol_srch_len))))
+    elseif (! isempty (strfind (str(1 : eol_srch_len), "\r")))
       eol_char = "\r";
     ## Otherwise, use plain UNIX (LF)
     else
@@ -246,16 +251,20 @@
     endif
   endif
 
-  ## Determine the number of data fields
-  num_fields = numel (strfind (format, "%")) - numel (strfind (format, "%*"));
-
-  ## Strip trailing EOL to avoid returning stray missing values (f. strread)
-  if (strcmp (str(end-length (eol_char) + 1 : end), eol_char));
+  ## Strip trailing EOL to avoid returning stray missing values (f. strread).
+  ## However, in case of CollectOutput request, presence of EOL is required
+  eol_at_end = strcmp (str(end-length (eol_char) + 1 : end), eol_char);
+  if (collop)
+    if (! eol_at_end)
+      str(end+1 : end+length (eol_char)) = eol_char;
+    endif
+  elseif (eol_at_end)
     str(end-length (eol_char) + 1 : end) = "";
+    ## A corner case: str may now be empty....
+    if (isempty (str)); return; endif
   endif
 
   ## Call strread to make it do the real work
-  C = cell (1, num_fields);
   [C{:}] = strread (str, format, args{:});
 
   ## If requested, collect output columns of same class
@@ -362,6 +371,21 @@
 %! assert (size(c{3}), [10, 2]);
 %! assert (size(c{2}), [10, 2]);
 
+%!test
+%% CollectOutput test with uneven column length files
+%! b = [10:10:100];
+%! b = [b; 8*b/5; 8*b*1000/5];
+%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
+%! str = [str "110 miles/hr"];
+%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
+%! c = textscan (str, fmt, "collectoutput", 1);
+%! assert (size(c{1}), [11, 1]);
+%! assert (size(c{3}), [11, 2]);
+%! assert (size(c{2}), [11, 2]);
+%! assert (c{3}(end), NaN);
+%! assert (c{2}{11, 1}, "/hr");
+%! assert (isempty (c{2}{11, 2}), true);
+
 %% Test input validation
 %!error textscan ()
 %!error textscan (single (4))
@@ -370,3 +394,7 @@
 %!error <cannot provide position information> [C, pos] = textscan ("Hello World")
 %!error <character value required> textscan ("Hello World", '%s', 'EndOfLine', 3)
 
+%! Test incomplete first data line
+%! R = textscan (['Empty1' char(10)], 'Empty%d %f');
+%! assert (R{1}, int32(1));
+%! assert (isempty(R{2}), true);
