diff -r dde2d050dab2 libinterp/corefcn/c-file-ptr-stream.cc
--- a/libinterp/corefcn/c-file-ptr-stream.cc Tue Dec 15 21:30:25 2015 +1100
+++ b/libinterp/corefcn/c-file-ptr-stream.cc Wed Dec 16 09:32:27 2015 +1100
@@ -127,12 +127,10 @@ seekdir_to_whence (std::ios::seekdir dir
}
std::streampos
-c_file_ptr_buf::seekoff (std::streamoff /* offset */,
- std::ios::seekdir /* dir */,
+c_file_ptr_buf::seekoff (std::streamoff offset,
+ std::ios::seekdir dir,
std::ios::openmode)
{
- // FIXME
-#if 0
if (f)
{
fseek (f, offset, seekdir_to_whence (dir));
@@ -141,15 +139,11 @@ c_file_ptr_buf::seekoff (std::streamoff
}
else
return 0;
-#endif
- return -1;
}
std::streampos
-c_file_ptr_buf::seekpos (std::streampos /* offset */, std::ios::openmode)
+c_file_ptr_buf::seekpos (std::streampos offset, std::ios::openmode)
{
- // FIXME
-#if 0
if (f)
{
fseek (f, offset, SEEK_SET);
@@ -158,8 +152,6 @@ c_file_ptr_buf::seekpos (std::streampos
}
else
return 0;
-#endif
- return -1;
}
int
diff -r dde2d050dab2 libinterp/corefcn/oct-stream.cc
--- a/libinterp/corefcn/oct-stream.cc Tue Dec 15 21:30:25 2015 +1100
+++ b/libinterp/corefcn/oct-stream.cc Wed Dec 16 09:32:27 2015 +1100
@@ -277,7 +277,7 @@ scanf_format_list::add_elt_to_list (int
}
}
-static std::string
+std::string
expand_char_class (const std::string& s)
{
std::string retval;
diff -r dde2d050dab2 libinterp/corefcn/oct-stream.h
--- a/libinterp/corefcn/oct-stream.h Tue Dec 15 21:30:25 2015 +1100
+++ b/libinterp/corefcn/oct-stream.h Wed Dec 16 09:32:27 2015 +1100
@@ -747,4 +747,6 @@ private:
int do_get_file_number (const octave_value& fid) const;
};
+std::string expand_char_class (const std::string& s);
+
#endif
diff -r dde2d050dab2 libinterp/dldfcn/module-files
--- a/libinterp/dldfcn/module-files Tue Dec 15 21:30:25 2015 +1100
+++ b/libinterp/dldfcn/module-files Wed Dec 16 09:32:27 2015 +1100
@@ -1,4 +1,5 @@
# FILE|CPPFLAGS|LDFLAGS|LIBRARIES
+xtextscan.cc|-I../../liboctave/array||
__delaunayn__.cc|$(QHULL_CPPFLAGS)|$(QHULL_LDFLAGS)|$(QHULL_LIBS)
__eigs__.cc|$(ARPACK_CPPFLAGS) $(SPARSE_XCPPFLAGS)|$(ARPACK_LDFLAGS) $(SPARSE_XLDFLAGS)|$(ARPACK_LIBS) $(SPARSE_XLIBS) $(LAPACK_LIBS) $(BLAS_LIBS)
__fltk_uigetfile__.cc|$(FLTK_CPPFLAGS) $(FT2_CPPFLAGS)|$(FLTK_LDFLAGS) $(FT2_LDFLAGS)|$(FLTK_LIBS) $(FT2_LIBS)
diff -r dde2d050dab2 libinterp/dldfcn/xtextscan.cc
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/libinterp/dldfcn/xtextscan.cc Wed Dec 16 09:32:27 2015 +1100
@@ -0,0 +1,2890 @@
+/*
+Copyright (C) 2015 Lachlan Andrew
+
+This file is part of Octave.
+
+Octave is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 3 of the License, or (at your
+option) any later version.
+
+Octave is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with Octave; see the file COPYING. If not, see
+.
+
+*/
+
+/* Implement the 'xtextscan' octave command */
+
+#ifdef HAVE_CONFIG_H
+#include
+#endif
+
+#include "list"
+
+#include "Array.cc"
+
+#include "Cell.h"
+#include "xtextscan.h"
+#include "oct-stream.h"
+#include "ovl.h"
+#include "utils.h"
+#include "defun-dld.h"
+#include "ov-re-mat.h"
+
+DEFUN_DLD (xtextscan, args, ,
+ "-*- texinfo -*-\n\
+@deftypefn {Function File} {@var{C} =} xtextscan (@var{fid}, @var{format})\n\
+@deftypefnx {Function File} {@var{C} =} xtextscan (@var{fid}, @var{format}, @var{repeat})\n\
+@deftypefnx {Function File} {@var{C} =} xtextscan (@var{fid}, @var{format}, @var{param}, @var{value}, @dots{})\n\
+@deftypefnx {Function File} {@var{C} =} xtextscan (@var{fid}, @var{format}, @var{repeat}, @var{param}, @var{value}, @dots{})\n\
+@deftypefnx {Function File} {@var{C} =} xtextscan (@var{str}, @dots{})\n\
+@deftypefnx {Function File} {[@var{C}, @var{position}] =} xtextscan (@dots{})\n\
+Read data from a text file or string.\n\
+\n\
+The string @var{str} or file associated with @var{fid} is read from and\n\
+parsed according to @var{format}. The function is an extension\n\
+of @code{strread} and @code{textread}. Differences include: the ability to\n\
+read from either a file or a string, additional options, and additional\n\
+format specifiers.\n\
+\n\
+The input is interpreted as a sequence of \"words\", delimiters (such as\n\
+whitespace) and literals. The characters that form delimiters and whitespace\n\
+are determined by the options. The format consists of format specifiers\n\
+interspersed between literals. In the format, whitespace forms a delimiter\n\
+between consecutive literals, but is otherwise ignored.\n\
+\n\
+The output @var{C} is a cell array whose second dimension is determined\n\
+by the number of format specifiers.\n\
+\n\
+The first word of the input is matched to the first specifier of the\n\
+format and placed in the first column of the output; the second is matched\n\
+to the second specifier and placed in the second column and so forth.\n\
+If there are more words than specifiers, the process is repeated until all\n\
+words have been processed or the limit imposed by @var{repeat} has been met\n\
+(see below).\n\
+\n\
+The string @var{format} describes how the words in @var{str} should be\n\
+parsed. As in @var{fscanf},\n\
+any (non-whitespace) text in the format that is not one of these specifiers\n\
+is considered a literal; if there is a literal between two format specifiers\n\
+then that same literal must appear in the input stream between the matching\n\
+words.\n\
+\n\
+The following specifiers are valid:\n\
+\n\
+@table @code\n\
+@item %f\n\
+@itemx %f64\n\
+@itemx %n\n\
+The word is parsed as a number and converted to double.\n\
+\n\
+@item %f32\n\
+The word is parsed as a number and converted to single (float).\n\
+\n\
+@item %d\n\
+@itemx %d8\n\
+@itemx %d16\n\
+@itemx %d32\n\
+@itemx %d64\n\
+The word is parsed as a number and converted to int8, int16, int32 or int64.\n\
+If not size is specified, int32 is used.\n\
+\n\
+@item %u\n\
+@itemx %u8\n\
+@itemx %u16\n\
+@itemx %u32\n\
+@itemx %u64\n\
+The word is parsed as a number and converted to uint8, uint16, uint32 or\n\
+uint64. If not size is specified, uint32 is used.\n\
+\n\
+@item %s\n\
+The word is parsed as a string, ending at the last character before\n\
+whitespace, an end-of-line or a delimiter specified in the options.\n\
+\n\
+@item %q\n\
+The word is parsed as a \"quoted string\". If the first character of the\n\
+string is a double quote (\") then the string includes everything until a\n\
+matching double quote, including whitespace, delimiters and end of line\n\
+characters. If a pair of consecutive double quotes appears in the input,\n\
+it is replaced in the output by a single double quote. That is, the input\n\
+\"He said \"\"Hello\"\"\" would return the value 'He said \"Hello\"'.\n\
+\n\
+@item %c\n\
+The next character of the input is read. This includes delimiters,\n\
+whitespace and end of line characters.\n\
+\n\
+@item %[...]\n\
+@itemx %[^...]\n\
+In the first form, the word consists of the longest run consisting of only\n\
+characters between the brackets. Ranges of characters can be specified by\n\
+a hyphen; for example, %[0-9a-zA-Z] matches all alphanumeric characters\n\
+(if the underlying character set is ASCII). To include '-' in the set,\n\
+it should appear first or last in the brackets; to include ']', it should\n\
+be the first character. If the first character is '^' then the word\n\
+consists of characters NOT listed.\n\
+\n\
+For %s, %c %d, %f, %n, %u, an\n\
+optional width can be specified as %Ns, etc. where N is an integer > 1.\n\
+For %c, this causes exactly the next N characters to be read instead of\n\
+a single character. For the other specifiers, it is an upper bound on the\n\
+number of characters read; normal delimiters can cause fewer characters to\n\
+be read. For complex numbers, this limit applies to the real and imaginary\n\
+components individually\n\
+For %f and %n, format specifiers like %N.Mf are allowed, where M is an upper\n\
+bound on number of characters after the decimal point to be considered;\n\
+subsequent digits are skipped. For example, the specifier %8.2f would\n\
+read 12.345e6 as 1.234e7.\n\
+\n\
+@item %*...\n\
+The word specified by the remainder of the conversion specifier is skipped.\n\
+\n\
+@item literals\n\
+In addition the format may contain literal character strings; these will be\n\
+skipped during reading. If the input string does not match this literal,\n\
+the processing terminates.\n\
+@end table\n\
+\n\
+Parsed words corresponding to the first specifier are returned in the first\n\
+output argument and likewise for the rest of the specifiers.\n\
+\n\
+By default, @var{format} is @t{\"%f\"}, meaning that numbers are read from\n\
+@var{str}. This will do if @var{str} contains only numeric fields.\n\
+\n\
+For example, the string\n\
+\n\
+@example\n\
+@group\n\
+@var{str} = \"\\\n\
+Bunny Bugs 5.5\n\\\n\
+Duck Daffy -7.5e-5\n\\\n\
+Penguin Tux 6\"\n\
+@end group\n\
+@end example\n\
+\n\
+@noindent\n\
+can be read using\n\
+\n\
+@example\n\
+[@var{a}] = xtextscan (@var{str}, \"%s %s %f\");\n\
+@end example\n\
+\n\
+Optional numeric argument @var{repeat} can be used for limiting the\n\
+number of items read:\n\
+\n\
+@table @asis\n\
+@item -1\n\
+(default) read all of the string or file until the end.\n\
+\n\
+@item N\n\
+Read until the first of two conditions occurs: the format has been processed\n\
+N times, or N lines of the input have been processed. Zero (0) is an\n\
+acceptable value for @var{repeat}. Currently, end-of-line characters\n\
+inside %q, %c, and %[...]$ conversions do not contribute to the line count.\n\
+This is incompatible with Matlab and may change in future.\n\
+@end table\n\
+\n\
+The behavior of @code{xtextscan} can be changed via property-value pairs. The\n\
+following properties are recognized:\n\
+\n\
+@table @asis\n\
+@item @qcode{\"CollectOutput\"}\n\
+A value of 1 or true instructs xtextscan to concatenate consecutive columns\n\
+of the same class in the output cell array. A value of 0 or false (default)\n\
+leaves output in distinct columns.\n\
+\n\
+@item @qcode{\"CommentStyle\"}\n\
+Parts of @var{str} are considered comments and will be skipped.\n\
+@var{value} is the comment style and can be\n\
+either\n\
+@c any of the following.\n\
+@c \n\
+@c @itemize\n\
+@c @item @qcode{\"shell\"}\n\
+@c Everything from @code{#} characters to the nearest end-of-line is skipped.\n\
+@c \n\
+@c @item @qcode{\"c\"}\n\
+@c Everything between @code{/*} and @code{*/} is skipped.\n\
+@c \n\
+@c @item @qcode{\"c++\"}\n\
+@c Everything from @code{//} characters to the nearest end-of-line is\n\
+@c skipped.\n\
+@c \n\
+@c @item @qcode{\"matlab\"}\n\
+@c Everything from @code{%} characters to the nearest end-of-line is skipped.\n\
+@c \n\
+@c @item user-supplied. Two options:\n\
+(1) One string, or 1x1 cell string, to skip everything to the right of it;\n\
+(2) A cell array of two strings, to skip everything between the first and\n\
+second strings.\n\
+@c @end itemize\n\
+Comments are only parsed where whitespace is accepted, and do not act as\n\
+delimiters.\n\
+\n\
+@item @qcode{\"Delimiter\"}\n\
+If @var{value} is a string, any character in @var{value} will be used to\n\
+split @var{str} into words. If @var{value} is a cell array of strings,\n\
+any string in the array will be used to split @var{str} into words.\n\
+(default value = any whitespace).\n\
+\n\
+@item @qcode{\"EmptyValue\"}\n\
+Value to return for empty numeric values in non-whitespace delimited data.\n\
+The default is NaN@. When the data type does not support NaN (int32 for\n\
+example), then default is zero.\n\
+\n\
+@item @qcode{\"EndOfLine\"}\n\
+@var{value} can be either a emtpy or one character specifying the end of line\n\
+character, or the pair\n\
+@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} (CRLF).\n\
+In the latter case, any of\n\
+@qcode{\"@xbackslashchar{}r\"}, @qcode{\"@xbackslashchar{}n\"} or\n\
+@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is counted as a (single)\n\
+newline.\n\
+If no value is given, @qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is\n\
+used.\n\
+@c If set to \"\" (empty string) EOLs are ignored as delimiters and added\n\
+@c to whitespace.\n\
+\n\
+@c When reading from a character string, optional input argument @var{n}\n\
+@c specifies the number of times @var{format} should be used (i.e., to limit\n\
+@c the amount of data read).\n\
+@c When reading from file, @var{n} specifies the number of data lines to read;\n\
+@c in this sense it differs slightly from the format repeat count in strread.\n\
+\n\
+@item @qcode{\"HeaderLines\"}\n\
+The first @var{value} number of lines of @var{fid} are skipped.\n\
+\n\
+@item @qcode{\"MultipleDelimsAsOne\"}\n\
+If @var{value} is non-zero,\n\
+treat a series of consecutive delimiters, without whitespace in between,\n\
+as a single delimiter. Consecutive delimiter series need not be vertically\n\
+@qcode{\"aligned\"}.\n\
+Without this option, a single delimiter before the end of the line does\n\
+not cause the line to be considered to start with an empty value, but a\n\
+single delimiter at the start of a line causes the line to be considered\n\
+to start with an empty value.\n\
+\n\
+@item @qcode{\"TreatAsEmpty\"}\n\
+Treat single occurrences (surrounded by delimiters or whitespace) of the\n\
+string(s) in @var{value} as missing values.\n\
+\n\
+@item @qcode{\"ReturnOnError\"}\n\
+If set to numerical 1 or true (default), return normally when read errors\n\
+have been encountered. If set to 0 or false, return an error and no data.\n\
+As the string or file is read by columns rather than by rows, and because\n\
+xtextscan is fairly forgiving as regards read errors, setting this option\n\
+may have little or no actual effect.\n\
+\n\
+@item @qcode{\"Whitespace\"}\n\
+Any character in @var{value} will be interpreted as whitespace and trimmed;\n\
+The default value for whitespace is\n\
+@c Note: the next line specifically has a newline which generates a space\n\
+@c in the output of qcode, but keeps the next line < 80 characters.\n\
+@qcode{\"\n\
+@xbackslashchar{}b@xbackslashchar{}r@xbackslashchar{}n@xbackslashchar{}t\"}\n\
+(note the space). Unless whitespace is set to @qcode{\"\"} (empty) AND at\n\
+least one @qcode{\"%s\"} format conversion specifier is supplied, a space is\n\
+always part of whitespace.\n\
+\n\
+@end table\n\
+\n\
+When the number of words in @var{str} or @var{fid} doesn't match an exact\n\
+multiple of\n\
+the number of format conversion specifiers,\n\
+xtextscan fills the last line with empty strings, NaNs or 0s (for integer\n\
+fields). In the case that the last line does not end with an end-of-line\n\
+character, this is incompatible with Matlab and subject to change.\n\
+@c xtextscan's behavior depends on\n\
+@c whether the last line of the string or file is an end-of-line as specified\n\
+@c by the EndOfLine option:\n\
+@c \n\
+@c @table @asis\n\
+@c @item last character = end-of-line\n\
+@c Data columns are padded with empty fields, NaN or 0 so that all columns have\n\
+@c equal length\n\
+@c \n\
+@c @item last character is not end-of-line\n\
+@c Data columns are not padded; xtextscan returns columns of unequal length\n\
+@c @end table\n\
+\n\
+If the format string is explicitly empty (not omitted) and the file contains\n\
+only\n\
+numeric data (excluding headerlines), xtextscan will return data in a number\n\
+of columns matching the number of fields on the first data line of\n\
+the input.\n\
+\n\
+The second output, @var{position}, provides the position, in characters\n\
+from the beginning of the file or string, at which the processing stopped.\n\
+\n\
+@seealso{dlmread, fscanf, load, strread, textread}\n\
+@end deftypefn")
+{
+ octave_value_list retval;
+ std::string format;
+ int params = 1;
+
+ if (args.length () < 1)
+ {
+ print_usage ();
+ error_state = true;
+ }
+ else if (args.length () == 1)
+ format = "%f"; // ommited format = %f. explicit "" = width from file
+ else if (args(1).is_string ())
+ {
+ format = args(1).string_value ();
+ if (args(1).is_sq_string ())
+ format = do_string_escapes (format);
+ params++;
+ }
+ else
+ error ("text:scan: FORMAT must be a string, not <%s>",
+ args(1).class_name ().c_str ());
+
+ if (! error_state)
+ {
+ octave_idx_type ntimes = -1;
+ xtextscan tscanner;
+
+ if (args.length () >= 3)
+ {
+ if (args(2).is_numeric_type ())
+ {
+ ntimes = args(2).idx_type_value ();
+ params = 3;
+ }
+ }
+ xtextscan_format_list fmt_list (format);
+
+ tscanner.parse_options (args, params, fmt_list);
+
+ fmt_list.consider_options (tscanner);
+
+ if (! error_state)
+ {
+ if (args(0).is_string ())
+ {
+ std::istringstream is (args(0).string_value ());
+ octave_value tmp = tscanner.scan (&is, fmt_list, ntimes);
+ if (! error_state)
+ {
+ retval(0) = tmp;
+
+ std::ios::iostate state = is.rdstate ();
+ is.clear ();
+ retval(1) = octave_value (static_cast(is.tellg ()));
+ is.setstate (state);
+ }
+ }
+ else
+ {
+ octave_stream os = octave_stream_list::lookup (args(0),
+ "xtextscan");
+ octave_value tmp = tscanner.scan (os.input_stream (),
+ fmt_list, ntimes);
+
+ if (! error_state)
+ {
+ retval(0) = tmp;
+ // FIXME -- warn if stream is not opened in binary mode?
+ std::ios::iostate state = os.input_stream ()->rdstate ();
+ os.input_stream ()->clear ();
+ retval(1) = os.tell ();
+ os.input_stream ()->setstate (state);
+ }
+ }
+ }
+ }
+
+ return retval;
+}
+
+/********************/
+/* Calculate x^n */
+/* Used for ...e+nn so that, for example, 1e2 is exactly 100 and 5e-1 is 1/2 */
+double pown (double x, unsigned int n)
+{
+ double retval = 1;
+
+ for (unsigned int d = n; d; d >>= 1)
+ {
+ if (d & 1)
+ retval *= x;
+ x *= x;
+ }
+
+ return retval;
+}
+
+/********************/
+/* Read a double considering the "precision" field of fmt and the */
+/* exp_chars option of options. */
+double
+xtextscan::formatted_double (std::istream& is, const xtextscan_format_elt& fmt)
+ const
+{
+ int ch = is.peek ();
+ int sign = 1;
+ unsigned int width_left = fmt.width;
+ double retval = 0;
+ bool valid = false; // syntactically correct double?
+
+ if (ch == '+')
+ {
+ is.get ();
+ ch = is.peek ();
+ if (width_left)
+ width_left--;
+ }
+ else if (ch == '-')
+ {
+ sign = -1;
+ is.get ();
+ ch = is.peek ();
+ if (width_left)
+ width_left--;
+ }
+
+ // Read integer part
+ if (ch != '.')
+ {
+ if (ch >= '0' && ch <= '9') // valid if at least one digit
+ valid = true;
+ while (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9')
+ retval = retval * 10 + (ch - '0');
+ width_left++;
+ }
+ // Read fractional part, up to specified precision
+ if (ch == '.' && width_left)
+ {
+ double multiplier = 1;
+ int precision = fmt.prec;
+ int i;
+
+ if (width_left)
+ width_left--; // Consider width of '.'
+
+ if (precision == -1)
+ precision = 1<<30; // FIXME Should be MAXINT
+ if (!valid) // if there was nothing before '.'...
+ is.get (); // ...ch was a "peek", not "get".
+
+ for (i = 0; i < precision; i++)
+ {
+ if (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9')
+ retval += (ch - '0') * (multiplier *= 0.1);
+ else
+ {
+ width_left++;
+ break;
+ }
+ }
+
+ // round up if we truncated and the next digit is >= 5
+ if ((i == precision || !width_left) && (ch = is.peek ()) >= '5'
+ && ch <= '9')
+ retval += multiplier;
+
+ if (i > 0)
+ valid = true; // valid if at least one digit after '.'
+
+ // skip remainder after '.', to field width, to look for exponent
+ while (width_left-- && is && (ch = is.get ()) >= '0' && ch <= '9')
+ ; /* discard */
+ width_left++;
+ }
+
+ // look for exponent part in, e.g., 6.023E+23
+ const char *ec = exp_chars.c_str ();
+ bool used_exp = false;
+ if (valid && width_left > 1 && strchr (ec, ch))
+ {
+ int ch1 = is.peek ();
+ if (ch1 == '-' || ch1 == '+' || (ch1 >= '0' && ch1 <= '9'))
+ { // if 1.0e+$ or some such, this will set failbit, as we want
+ width_left--; // count "E"
+ int exp = 0;
+ int exp_sign = 1;
+ if (ch1 == '+')
+ {
+ if (width_left)
+ width_left--;
+ is.get ();
+ }
+ else if (ch1 == '-')
+ {
+ exp_sign = -1;
+ is.get ();
+ if (width_left)
+ width_left--;
+ }
+ valid = false;
+ while (width_left-- && is && (ch = is.get ()) >= '0' && ch1 <= '9')
+ {
+ exp = exp*10 + ch - '0';
+ valid = true;
+ }
+ width_left++;
+ if (ch != EOF && width_left)
+ is.putback (ch);
+
+ double multiplier = pown (10, exp);
+ if (exp_sign > 0)
+ retval *= multiplier;
+ else
+ retval /= multiplier;
+
+ used_exp = true;
+ }
+ }
+ is.clear ();
+ if (!used_exp && ch != EOF && width_left)
+ is.putback (ch);
+
+ // Check for +/- inf and NaN
+ if (!valid && width_left >= 3)
+ {
+ int i = lookahead (is, inf_nan, 3, false); // false -> case insensitive
+ if (i == 0)
+ {
+ retval = octave_Inf;
+ valid = true;
+ }
+ else if (i == 1)
+ {
+ retval = octave_NaN;
+ valid = true;
+ }
+ }
+
+ // Check for +/- inf and NaN
+ if (!valid && width_left >= 3)
+ {
+ int i = lookahead (is, inf_nan, 3, false); // false -> case insensitive
+ if (i == 0)
+ {
+ retval = octave_Inf;
+ valid = true;
+ }
+ else if (i == 1)
+ {
+ retval = octave_NaN;
+ valid = true;
+ }
+ }
+
+ if (!valid)
+ is.setstate (std::ios_base::failbit);
+ else
+ is.setstate (is.rdstate () & ~std::ios_base::failbit);
+
+ return retval * sign;
+}
+
+/********************/
+std::istream&
+xtextscan::scan_complex (std::istream& is, const xtextscan_format_elt& fmt,
+ Complex& val) const
+{
+ double im = 0;
+ double re = 0;
+ bool as_empty = false; // did we fail but match a "treat_as_empty" string?
+ bool inf = false;
+
+ int ch = is.peek ();
+
+ if (ch == '+' || ch == '-') // check for [+-][ij] with no coefficients
+ {
+ ch = is.get ();
+ int ch2 = is.peek ();
+ if (ch2 == 'i' || ch2 == 'j')
+ {
+ double value = 1;
+ is.get ();
+ // Check not -inf
+ if (is.peek () == 'n')
+ {
+ std::streampos pos = is.tellg ();
+ std::ios::iostate state = is.rdstate ();
+
+ is.get ();
+ ch2 = is.get ();
+ if (ch2 == 'f')
+ {
+ inf = true;
+ re = (ch == '+') ? octave_Inf : -octave_Inf;
+ value = 0;
+ }
+ else
+ {
+ is.clear (state);
+ is.seekg (pos); // reset to position before look-ahead
+ }
+ }
+
+ im = (ch == '+') ? value : -value;
+ }
+ else
+ is.putback (ch);
+ }
+
+ if (!im && !inf) // if not [+-][ij] or [+-]inf, read real normally
+ {
+ std::streampos pos = is.tellg ();
+ std::ios::iostate state = is.rdstate ();
+ //re = octave_read_value (is);
+ re = read_double (is, fmt);
+
+ // check for "treat as empty" string
+ if (treat_as_empty.numel ()
+ && (is.fail () || octave_is_NaN_or_NA (Complex (re))
+ || re == octave_Inf))
+ {
+
+ for (int i = 0; i < treat_as_empty.numel (); i++)
+ {
+ if (ch == treat_as_empty (i).string_value ()[0])
+ {
+ as_empty = true; // first char matches, so read the lot
+ break;
+ }
+ }
+ if (as_empty) // if first char matched...
+ {
+ as_empty = false; // ...look for the whole string
+
+ is.clear (state); // treat_as_empty "-" causes partial read
+ is.seekg (pos); // reset to position before failed read, since
+
+ // treat_as_empty strings may be different sizes.
+ // Read ahead longest, put it all back, then re-read the string
+ // that matches.
+ char look [treat_as_empty_len + 1];
+ // prefill, in case EOF means part-filled.
+ memset (look, '\0', treat_as_empty_len);
+ is.read (look, treat_as_empty_len);
+
+ is.clear (state);
+ is.seekg (pos); // reset to position before look-ahead
+
+ for (int i = 0; i < treat_as_empty.numel (); i++)
+ {
+ std::string s = treat_as_empty (i).string_value ();
+ if (!strncmp (s.c_str (), look, s.size ()))
+ {
+ as_empty = true;
+ is.read (look, s.size ()); // read just the right amount
+ // FIXME Could this be combined with the above seek?
+ break;
+ }
+ }
+ }
+ }
+
+ if (!is.eof () && !as_empty)
+ {
+ state = is.rdstate (); // before tellg, since that fails at EOF
+ pos = is.tellg ();
+ ch = is.peek (); // ch == EOF if read failed; no need to chk fail
+ if (ch == 'i' || ch == 'j') // pure imaginary
+ {
+ is.get ();
+ im = re;
+ re = 0;
+ }
+ else if (ch == '+' || ch == '-') // see if it is real+imag[ij]
+ {
+ // save stream state in case we have to restore it
+ pos = is.tellg ();
+ state = is.rdstate ();
+
+ //im = octave_read_value (is);
+ im = read_double (is, fmt);
+ if (is.fail ())
+ im = 1;
+
+ if (is.peek () == 'i' || is.peek () == 'j')
+ is.get ();
+ else
+ {
+ im = 0; // no valid imaginary part. Restore state
+ is.clear (state);
+ is.seekg (pos);
+ }
+ }
+ else if (ch == EOF) // we've read enough to be a "valid" read
+ is.clear (state); // failed peek shouldn't cause fail
+ }
+ }
+ if (as_empty)
+ val = empty_value.scalar_value ();
+ else
+ val = Complex (re, im);
+
+ return is;
+}
+
+/********************/
+// Return in val the run of characters from is NOT contained in pattern.
+int
+xtextscan::scan_caret (std::istream& is, const char *pattern, std::string& val)
+ const
+{
+ int c1 = EOF;
+ std::ostringstream obuf; // Is this optimised for growing?
+
+ while (is && (c1 = (is && !is.eof ()) ? is.get () : EOF) != EOF
+ && !strchr (pattern, c1))
+ obuf << static_cast (c1);
+
+ val = obuf.str ();
+
+ // can only push back if not failed.
+ is.clear (is.rdstate () & ~std::ios_base::failbit);
+ if (c1 != EOF)
+ is.putback (c1);
+
+ return c1;
+}
+
+/********************/
+// Read until one of the strings in delimiters is found.
+// For efficiency, ends is a list of the last character of each delimiter
+std::string
+xtextscan::read_until (std::istream& is, const Cell& delimiters,
+ const std::string& ends) const
+{
+ std::string retval ("");
+ bool done = false;
+ do
+ { // find sequence ending with an ending char
+ std::string next;
+ scan_caret (is, ends.c_str (), next);
+ retval = retval + next;
+
+ int last = (!is.eof ()) ? is.get () : EOF;
+ if (last != EOF)
+ {
+ retval = retval + static_cast (last);
+ for (int i = 0; i < delimiters.numel (); i++)
+ {
+ std::string delim = delimiters(i).string_value ();
+ int start = retval.length () - delim.length ();
+ if (start < 0)
+ start = 0;
+ std::string may_match = retval.substr (start);
+ if (may_match == delim)
+ {
+ done = true;
+ retval = retval.substr (0, start);
+ break;
+ }
+ }
+ }
+ }
+ while (!done && is && !is.eof ());
+
+ return retval;
+}
+
+
+/********************/
+// Read stream until either fmt.width chars have been read, or options.delimiter
+// has been found
+// Does *not* rely on fmt being 's' -- used by formats like %6f to limit to 6
+std::istream&
+xtextscan::scan_string (std::istream& is, const xtextscan_format_elt& fmt, std::string& val) const
+{
+ if (delim_list.numel () == 0)
+ {
+ unsigned int i = 0;
+ unsigned int width = fmt.width;
+
+ for (i = 0; i < width; i++)
+ {
+ if (i+1 > val.length ())
+ val = val + val + ' '; // grow even if empty
+ int ch = is.get ();
+ if (is_delim (ch) || ch == EOF)
+ {
+ is.putback (ch);
+ break;
+ }
+ else
+ val[i] = ch;
+ }
+ val = val.substr (0, i); // trim pre-allocation
+ }
+ else // Cell array of multi-character delimiters
+ {
+ std::string ends ("");
+ for (int i = 0; i < delim_list.numel (); i++)
+ {
+ std::string tmp = delim_list(i).string_value ();
+ ends += tmp.substr (tmp.length () - 1);
+ }
+ val = xtextscan::read_until (is, delim_list, ends);
+ }
+
+ return is;
+}
+
+/********************/
+// Return in val the run of characters from is contained in pattern.
+int
+xtextscan::scan_bracket (std::istream& is, const char *pattern, std::string& val)
+ const
+{
+ int c1 = EOF;
+ std::ostringstream obuf; // Is this optimised for growing?
+
+ while (is && strchr (pattern, (c1 = is.get ())))
+ obuf << static_cast (c1);
+
+ val = obuf.str ();
+ if (c1 != EOF)
+ is.putback (c1);
+ return c1;
+}
+
+/********************/
+// Return in val a string, either delimited by whitespace/delimiters, or
+// enclosed in a pair of double quotes ("..."). Enclosing quotes are removed.
+// A consecutive pair "" is inserted into val as a single ".
+std::istream&
+xtextscan::scan_qstring (std::istream& is, const xtextscan_format_elt& fmt,
+ std::string& val)
+{
+ skip_whitespace (is);
+
+ if (is.peek () != '\"')
+ scan_string (is, fmt, val);
+ else
+ {
+ is.get ();
+ scan_caret (is, "\"", val); // read everything until "
+ is.get (); // swallow "
+
+ while (is && is.peek () == '\"') // if double ", insert one in stream,
+ { // and keep looking for single "
+ is.get ();
+ std::string val1;
+ scan_caret (is, "\"", val1);
+ val = val + "\"" + val1;
+ is.get ();
+ }
+ }
+ return is;
+}
+
+/********************/
+// Read from is into val a string of the next fmt.width characters,
+// including any whitespace or delimiters.
+std::istream&
+xtextscan::scan_cstring (std::istream& is, const xtextscan_format_elt& fmt,
+ std::string& val) const
+{
+ int ch;
+ val.resize (fmt.width);
+ for (unsigned int i = 0; is && i < fmt.width; i++)
+ if ((ch = is.get ()) != EOF)
+ val[i] = ch;
+ else
+ {
+ val.resize (i);
+ break;
+ }
+
+ return is;
+}
+
+
+/********************/
+std::istream&
+xtextscan::scan_one (std::istream& is, const xtextscan_format_elt& fmt,
+ octave_value& ov, Array row)
+{
+ skip_whitespace (is);
+
+ is.clear ();
+
+ octave_value val;
+ if (fmt.numeric)
+ {
+ if (fmt.type == 'f' || fmt.type == 'n')
+ {
+ Complex v;
+ scan_complex (is, fmt, v);
+
+ if (!fmt.discard && !is.fail ())
+ {
+ if (fmt.bitwidth == 64)
+ {
+ if (ov.is_real_type () && v.imag () == 0)
+ ov.internal_rep ()->fast_elem_insert (row(0), v.real ());
+ else
+ {
+ if (ov.is_real_type ()) // cat does type conversion
+ ov = do_cat_op (ov, octave_value (v), row);
+ else
+ ov.internal_rep ()->fast_elem_insert (row(0), v);
+ }
+ }
+ else
+ {
+ if (ov.is_real_type () && v.imag () == 0)
+ ov.internal_rep ()->fast_elem_insert (row(0),
+ float (v.real ()));
+ else
+ {
+ if (ov.is_real_type ()) // cat does type conversion
+ ov = do_cat_op (ov, octave_value (v), row);
+ else
+ ov.internal_rep ()->fast_elem_insert (row(0),
+ FloatComplex (v));
+ }
+ }
+ }
+ }
+ else
+ {
+ double v; // Matlab docs say 1e30 etc should be valid for %d and
+ // 1000 as a %d8 should be 127, so read as double.
+ // Some loss of precision for d64 and u64.
+ v = read_double (is, fmt);
+ if (!fmt.discard && !is.fail ())
+ switch (fmt.bitwidth)
+ {
+ case 64:
+ switch (fmt.type)
+ {
+ case 'd':
+ {
+ octave_int64 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ case 'u':
+ {
+ octave_uint64 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ }
+ break;
+ case 32:
+ switch (fmt.type)
+ {
+ case 'd':
+ {
+ octave_int32 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ case 'u':
+ {
+ octave_uint32 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ }
+ break;
+ case 16:
+ if (fmt.type == 'd')
+ {
+ octave_int16 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ else
+ {
+ octave_uint16 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ case 8:
+ if (fmt.type == 'd')
+ {
+ octave_int8 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ else
+ {
+ octave_uint8 vv = v;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ }
+ }
+ if (is.fail ())
+ {
+ if (!fmt.discard)
+ ov = do_cat_op (ov, empty_value, row);
+ std::string dummy; // FIXME: should fail instead of skipping
+
+ std::ios::iostate state = is.rdstate ();
+ is.clear (); // clear to allow read pointer to advance
+ scan_string (is, fmt, dummy);
+ is.setstate (state);
+ }
+
+ }
+ else
+ {
+ std::string vv (" "); // initial buffer. Grows as needed
+ switch (fmt.type)
+ {
+ case 's':
+ scan_string (is, fmt, vv);
+ break;
+ case 'q':
+ scan_qstring (is, fmt, vv);
+ break;
+ case 'c':
+ scan_cstring (is, fmt, vv);
+ break;
+ case '[':
+ scan_bracket (is, fmt.char_class.c_str (), vv);
+ break;
+ case '^':
+ scan_caret (is, fmt.char_class.c_str (), vv);
+ break;
+ }
+ if (!fmt.discard)
+ ov.internal_rep ()->fast_elem_insert (row (0),
+ Cell (octave_value (vv)));
+ // fixme: why does failbit get set at EOF, instead of eofbit??
+ if (vv.length () != 0)
+ is.clear (is.rdstate () & ~std::ios_base::failbit);
+ }
+
+ return is;
+}
+
+/********************/
+int
+xtextscan::read_format_once (std::istream* isp,
+ xtextscan_format_list& fmt_list,
+ std::list & retval,
+ Array row)
+{
+ const xtextscan_format_elt *elem = fmt_list.first ();
+ std::list::iterator out=retval.begin ();
+ bool no_conversions = true;
+ bool done = false;
+ int i;
+
+ for (i = 0; i < fmt_list.numel (); i++)
+ {
+ isp->clear (); // clear fail of previous numeric conversions
+ switch (elem->type)
+ {
+ case 'C':
+ case 'D':
+ std::cerr << "xtextscan: Conversion %" << elem->type
+ << " not yet implemented\n";
+ break;
+
+ case 'u':
+ case 'd':
+ case 'f':
+ case 'n':
+ case 's':
+ case '[':
+ case '^':
+ case 'q':
+ case 'c':
+ scan_one (*isp, *elem, *out, row);
+ break;
+
+ case xtextscan_format_elt::literal_conversion :
+ match_literal (*isp, *elem);
+ break;
+ }
+
+ if (!isp->fail () && !elem->discard)
+ no_conversions = false;
+ if (isp->eof ())
+ done = true; // note EOF, but process others to get empty_val
+ else
+ isp->clear (isp->rdstate () & ~std::ios::failbit);
+ if (!elem->discard)
+ out++;
+
+ elem = fmt_list.next ();
+ // fixme: these conversions "ignore delimiters". Should they include
+ // delimiters at the start of the conversion, or can those be skipped?
+ if (// elem->type != '[' && elem->type != '^' && elem->type != 'c'
+ /*&&*/ elem->type != xtextscan_format_elt::literal_conversion)
+ skip_delim (*isp);
+
+ if (error_state)
+ break;
+ }
+ if (done)
+ isp->setstate (std::ios::eofbit);
+
+ // err = -2 means "error, and no columns read this row
+ if (error_state || isp->eof ())
+ return (-1 - no_conversions);
+
+ return 1;
+}
+
+/********************/
+void
+xtextscan::parse_options (const octave_value_list& args, int first_param,
+ xtextscan_format_list& fmt_list)
+{
+ int last = args.length ();
+ int n = last - first_param;
+
+ if (n & 1)
+ error ("xtextscan: %d parameters given, but only %d values", n-n/2, n/2);
+ else
+ {
+ std::string delims;
+ bool have_delims = false;
+ for (int i = first_param; i < last; i += 2)
+ {
+ if (!args(i).is_string ())
+ {
+ error ("xtextscan: Invalid paramter type <%s> for parameter %d",
+ args(i).type_name ().c_str (), (i-first_param)/2 + 1);
+ }
+ else
+ {
+ std::string param = args(i).string_value ();
+ std::transform (param.begin (), param.end (),
+ param.begin (), ::tolower);
+ if (param == "delimiter")
+ {
+ bool invalid = true;
+ if (args(i+1).is_string ())
+ {
+ invalid = false;
+ have_delims = true;
+ delims = args(i+1).string_value ();
+ }
+ else if (args(i+1).is_cell ())
+ {
+ invalid = false;
+ delim_list = args(i+1).cell_value ();
+ delim_table = " "; // non-empty, to flag non-default delim
+
+ // Check that all elements are strings
+ for (int j = 0; j < delim_list.numel (); j++)
+ if (!delim_list (j).is_string ())
+ invalid = true;
+ }
+ if (invalid)
+ error ("xtextscan: Delimimters must be either a string"
+ " or cell array of strings");
+ }
+ else if (param == "commentstyle")
+ {
+ if (args(i+1).is_string ())
+ { // check here for names like "C++", "C", "shell", ...?
+ comment_style = Cell (args(i+1));
+ }
+ else if (args(i+1).is_cell ())
+ {
+ comment_style = args(i+1).cell_value ();
+ int len = comment_style.numel ();
+ if ((len >= 1 && !comment_style (0).is_string ())
+ || (len >= 2 && !comment_style (1).is_string ())
+ || (len >= 3))
+ error ("xtextscan: CommentStyle must be either a string"
+ " or cell array of one or two strings");
+ }
+ else
+ error ("xtextscan: CommentStyle must be either a string"
+ " or cell array of one or two strings, not <%s>",
+ args(i+1).class_name ().c_str ());
+
+ // How far ahead do we need to look to detect an open comment
+ // and which character do we look for?
+ if (comment_style.numel () >= 1)
+ {
+ comment_len = comment_style (0).string_value ().size ();
+ comment_char = comment_style (0).string_value ()[0];
+ }
+ }
+ else if (param == "treatasempty")
+ {
+ bool invalid = false;
+ if (args(i+1).is_string ())
+ {
+ treat_as_empty = Cell (args(i+1));
+ treat_as_empty_len = args(i+1).string_value ().size ();
+ }
+ else if (args(i+1).is_cell ())
+ {
+ treat_as_empty = args(i+1).cell_value ();
+ for (int j = 0; j < treat_as_empty.numel (); j++)
+ if (!treat_as_empty (j).is_string ())
+ invalid = true;
+ else
+ {
+ int k = treat_as_empty (j).string_value ().size ();
+ if (k > treat_as_empty_len)
+ treat_as_empty_len = k;
+ }
+ }
+ if (invalid)
+ error ("xtextscan: TreatAsEmpty must be either a string"
+ " or cell array of one or two strings");
+ else
+ {
+ // FIXME Ensure none is a prefix of a later one. sort len?
+ }
+ }
+ else if (param == "collectoutput")
+ {
+ if (args(i+1).is_numeric_type ())
+ collect_output = args(i+1).bool_value ();
+ else
+ error ("xtextscan: CollectOutput must be logical or numeric");
+ }
+ else if (param == "emptyvalue")
+ {
+ if (args(i+1).is_numeric_type ())
+ empty_value = args(i+1).scalar_value ();
+ else
+ error ("xtextscan: EmptyValue must be numeric, not <%s>",
+ args(i+1).class_name ().c_str ());
+ }
+ else if (param == "headerlines")
+ {
+ if (args(i+1).is_numeric_type ())
+ header_lines = args(i+1).scalar_value ();
+ else
+ error ("xtextscan: HeaderLines must be numeric");
+ }
+ else if (param == "multipledelimsasone")
+ {
+ if (args(i+1).is_numeric_type ())
+ {
+ if (args(i+1).bool_value ())
+ multiple_delims_as_one = true;
+ }
+ else
+ error ("xtextscan: MultipleDimsAsOne must be logical or numeric");
+ }
+ else if (param == "returnonerror")
+ {
+ if (args(i+1).is_numeric_type ())
+ {
+ if (!args(i+1).bool_value ())
+ return_on_error = false;
+ }
+ else
+ error ("xtextscan: ReturnOnError must be logical or numeric");
+ }
+ else if (param == "whitespace")
+ {
+ if (args(i+1).is_string ())
+ whitespace = args(i+1).string_value ();
+ else
+ error ("xtextscan: Whitespace must be a character string");
+ }
+ else if (param == "expchars")
+ {
+ if (args(i+1).is_string ())
+ {
+ exp_chars = args(i+1).string_value ();
+ default_exp = false;
+ }
+ else
+ error ("xtextscan: ExpChars must be a character string");
+ }
+ else if (param == "endofline")
+ {
+ bool valid = true;
+ if (args(i+1).is_string ())
+ {
+ std::string s = args(i+1).string_value ();
+ if (args(i+1).is_sq_string ())
+ s = do_string_escapes (s);
+ int l = s.length ();
+ if (l == 0)
+ eol1 = eol2 = -2;
+ else if (l == 1)
+ eol1 = eol2 = s.c_str ()[0];
+ else if (l == 2)
+ {
+ eol1 = s.c_str ()[0];
+ eol2 = s.c_str ()[1];
+ if (eol1 != '\r' || eol2 != '\n') // Why limit it?
+ valid = false;
+ }
+ else
+ valid = false;
+ }
+ else
+ valid = false;
+ if (!valid)
+ error ("xtextscan: EndOfLine must be at most one character or '\\r\\n'");
+ }
+ else
+ {
+ error ("xtextscan: Unrecognised option '%s'", param.c_str ());
+ }
+ }
+ }
+
+ whitespace_table = std::string (256, '\0');
+ for (unsigned int i = 0; i < whitespace.length (); i++)
+ whitespace_table[whitespace[i]] = '1';
+
+ // For Matlab compatibility, add 0x20 to whitespace, unless
+ // whitespace is explicitly ignored.
+ if (! (whitespace.length () == 0 && fmt_list.has_string))
+ whitespace_table[' '] = '1';
+
+ // Create look-up table of delimiters, based on 'delimiter'
+ delim_table = std::string (256, '\0');
+ delim_table[eol1] = '1'; // EOL is always a delimiter
+ delim_table[eol2] = '1'; // EOL is always a delimiter
+ if (!have_delims)
+ for (unsigned int i = 0; i < 256; i++)
+ {
+ if (isspace (i))
+ delim_table[i] = '1';
+ }
+ else
+ for (unsigned int i = 0; i < delims.length (); i++)
+ delim_table[delims[i]] = '1';
+
+// // Add CR and LF to whitespace if they are not end-of-line.
+// if (!strchr (end_of_line.c_str (), '\r'))
+// whitespace_table['\r'] = '1';
+// if (!strchr (end_of_line.c_str (), '\n'))
+// whitespace_table['\n'] = '1';
+ }
+}
+
+/********************/
+// skip comments, and characters specified by the "Whitespace" option
+// If EOLstop==true, doesn't skip end of line
+int
+xtextscan::skip_whitespace (std::istream& is, bool EOLstop)
+{
+ int c1 = EOF;
+ bool found_comment = false;
+
+ do
+ {
+ found_comment = false;
+ int prev = -1;
+ while (is && (c1 = is.get ()) != EOF
+ && ( ( (c1 == eol1 || c1 == eol2) && ++lines && !EOLstop)
+ || isspace (c1)))
+ {
+ if (prev == eol1 && eol1 != eol2 && c1 == eol2)
+ lines--;
+ prev = c1;
+ }
+
+ if (c1 == comment_char) // see if we match an open comment
+ {
+ // save stream state in case we have to restore it
+ std::streampos pos = is.tellg ();
+ std::ios::iostate state = is.rdstate ();
+
+ char look [comment_len];
+ is.read (look, comment_len-1); // already read first char
+ look[comment_len-1] = '\0';
+ if (is && comment_style(0).string_value ().substr (1) == look)
+ {
+ found_comment = true;
+
+ std::string dummy;
+ char eol [3] = {static_cast (eol1),
+ static_cast (eol2),
+ '\0'};
+ if (comment_style.numel () == 1) // skip to end of line
+ {
+ scan_caret (is, eol, dummy);
+ c1 = is.get ();
+ if (c1 == eol1 && eol1 != eol2 && is.peek () == eol2)
+ is.get ();
+ lines++;
+ }
+ else // matching pair
+ {
+ std::string end_c = comment_style(1).string_value ();
+ // last char of end-comment sequence
+ char last = *(end_c.substr (end_c.length ()-1).c_str ());
+ std::string may_match ("");
+ do
+ { // find sequence ending with last char
+ scan_caret (is, &last, dummy);
+ is.get (); // (read last itself)
+
+ may_match = may_match + dummy + last;
+ int start = may_match.length () - end_c.length ();
+ if (start < 0)
+ start = 0;
+ may_match = may_match.substr (start);
+ }
+ while (may_match != end_c && is && !is.eof ());
+ }
+ }
+ else // wasn't really a comment; restore state
+ {
+ is.clear (state);
+ is.seekg (pos);
+ }
+ }
+ }
+ while (found_comment);
+
+ if (c1 != EOF)
+ is.putback (c1);
+ return c1;
+}
+
+/********************/
+// See if the next few characters match one of the strings in target.
+// For efficiency, max_len is the cached longest length of any target.
+// Returns -1 if none is found, or the inde of the match.
+int
+xtextscan::lookahead (std::istream& is, const Cell& targets, int max_len,
+ bool case_sensitive) const
+{
+ // target strings may be different sizes.
+ // Read ahead longest, put it all back, then re-read the string
+ // that matches.
+
+ std::streampos pos = is.tellg ();
+ std::ios::iostate state = is.rdstate ();
+
+ char look [max_len + 1];
+
+ memset (look, '\0', max_len); // prefill, in case EOF means part-filled.
+ is.read (look, max_len);
+
+ is.clear (state);
+ is.seekg (pos); // reset to position before look-ahead
+
+ int i;
+ int (*compare)(const char *, const char *, size_t);
+ compare = case_sensitive ? strncmp : strncasecmp;
+
+ for (i = 0; i < targets.numel (); i++)
+ {
+ std::string s = targets (i).string_value ();
+ if (!(*compare) (s.c_str (), look, s.size ()))
+ {
+ is.read (look, s.size ()); // read just the right amount
+ break;
+ }
+ }
+
+ if (i == targets.numel ())
+ i = -1;
+
+ return i;
+}
+
+/********************/
+// Skip delimiters -- multiple if MultipleDelimsAsOne specified.
+int
+xtextscan::skip_delim (std::istream& is)
+{
+ int c1 = skip_whitespace (is, true); // 'true': stop once EOL is read
+ if (delim_list.numel () == 0) // single character delimiter
+ {
+ if (is_delim (c1) || c1 == eol1 || c1 == eol2)
+ {
+ is.get ();
+ if (c1 == eol1 && is.peek () == eol2)
+ is.get (); // if \r\n, skip the \n too.
+
+ if (multiple_delims_as_one)
+ {
+ int prev = -1;
+ // skip multiple delims.
+ // Increment lines for each end-of-line seen; for \r\n, decrement
+ while (is && (c1 = is.get ()) != EOF
+ && (((c1 == eol1 || c1 == eol2) && ++lines)
+ || isspace (c1) || is_delim (c1)))
+ {
+ if (prev == eol1 && eol1 != eol2 && c1 == eol2)
+ lines--;
+ prev = c1;
+ }
+ if (c1 != EOF)
+ is.putback (c1);
+ }
+ }
+ }
+ else // multi-character delimiter
+ {
+ int first_match;
+ int max_len = 0;
+ for (int i = 0; i < delim_list.numel (); i++)
+ {
+ int len = delim_list(i).numel ();
+ if (len > max_len)
+ max_len = len;
+ }
+
+ if (c1 == eol1 || c1 == eol2
+ || (-1 != (first_match = lookahead (is, delim_list, max_len))))
+ {
+ if (c1 == eol1)
+ {
+ is.get ();
+ if (is.peek () == eol2)
+ is.get ();
+ }
+ else if (c1 == eol2)
+ {
+ is.get ();
+ }
+
+ if (multiple_delims_as_one)
+ {
+ int prev = -1;
+ // skip multiple delims.
+ // Increment lines for each end-of-line seen; for \r\n, decrement
+ while (is && (c1 == skip_whitespace (is, true)) != EOF
+ && (((c1 == eol1 || c1 == eol2) && ++lines)
+ || -1 != lookahead (is, delim_list, max_len)))
+ {
+ if (prev == eol1 && eol1 != eol2 && c1 == eol2)
+ lines--;
+ prev = c1;
+ }
+ }
+ }
+ }
+
+ return c1;
+}
+
+/********************/
+// Read in as much of the input as coincides with the literal in
+// the format string.
+// Return "true" if the entire literal is matched, else false (and set failbit)
+bool
+xtextscan::match_literal (std::istream& is, const xtextscan_format_elt& fmt)
+{
+ // "false" -> treat EOL as normal space
+ // since a delimiter at the start of a line is a mismatch, not empty field
+ skip_whitespace (is, false);
+
+ for (unsigned int i = 0; i < fmt.width; i++)
+ {
+ int ch = is.get ();
+ if (ch != fmt.text[i])
+ {
+ if (ch != EOF)
+ is.putback (ch);
+ is.setstate (std::ios_base::failbit);
+ return false;
+ }
+ }
+ return true;
+}
+
+
+/********************/
+xtextscan_format_list::xtextscan_format_list (const std::string& s)
+ : set_from_first (false), has_string (false), nconv (0), curr_idx (0),
+ list (dim_vector (16, 1)), buf (0)
+{
+ octave_idx_type num_elts = 0;
+
+ size_t n = s.length ();
+
+ size_t i = 0;
+
+ unsigned int width = -1; // Unspecified width = max (except %c)
+ int prec = -1;
+ int bitwidth = 0;
+ bool discard = false;
+ char type = '\0';
+
+ bool have_more = true;
+
+ if (s.length () == 0)
+ {
+ buf = new std::ostringstream ("%f");
+ bitwidth = 64;
+ type = 'f';
+ add_elt_to_list (width, prec, bitwidth, octave_value (NDArray ()),
+ discard, type, num_elts);
+ have_more = false;
+ set_from_first = true;
+ nconv = 1;
+ }
+ else
+ {
+ set_from_first = false;
+ while (i < n)
+ {
+ have_more = true;
+
+ if (! buf)
+ buf = new std::ostringstream ();
+
+ if (s[i] == '%' && (i+1 == n || s[i+1] != '%'))
+ {
+ // Process percent-escape conversion type.
+
+ process_conversion (s, i, n, num_elts);
+
+ have_more = (buf != 0);
+ }
+ else if (isspace (s[i]))
+ {
+ while (++i < n && isspace (s[i]))
+ /* skip whitespace */;
+
+ have_more = false;
+ }
+ else
+ {
+ type = xtextscan_format_elt::literal_conversion;
+
+ width = 0;
+ prec = -1;
+ bitwidth = 0;
+ discard = true;
+
+ while (i < n && ! isspace (s[i])
+ && (s[i] != '%' || (i+1 < n && s[i+1] == '%')))
+ {
+ if (s[i] == '%') // if double %, skip one
+ i++;
+ *buf << s[i++];
+ width++;
+ }
+
+ add_elt_to_list (width, prec, bitwidth, octave_value (), discard,
+ type, num_elts);
+
+ have_more = false;
+ }
+
+ if (nconv < 0)
+ {
+ have_more = false;
+ break;
+ }
+ }
+ }
+
+ if (have_more)
+ add_elt_to_list (width, prec, bitwidth, octave_value (), discard, type, num_elts);
+
+ list.resize (dim_vector (num_elts, 1));
+
+ delete buf;
+}
+
+/********************/
+xtextscan_format_list::~xtextscan_format_list (void)
+{
+ octave_idx_type n = list.numel ();
+
+ for (octave_idx_type i = 0; i < n; i++)
+ {
+ xtextscan_format_elt *elt = list(i);
+ delete elt;
+ }
+}
+
+/********************/
+void
+xtextscan_format_list::add_elt_to_list (unsigned int width, int prec,
+ int bitwidth, octave_value val_type,
+ bool discard, char type,
+ octave_idx_type& num_elts,
+ const std::string& char_class)
+{
+ if (buf)
+ {
+ std::string text = buf->str ();
+
+ if (! text.empty ())
+ {
+ xtextscan_format_elt *elt
+ = new xtextscan_format_elt (text.c_str (), width, prec, bitwidth,
+ discard, type, char_class);
+
+ if (num_elts == list.numel ())
+ {
+ list.resize (dim_vector (2 * num_elts, 1));
+ }
+
+ if (!discard)
+ {
+ output_container.push_back (val_type);
+ }
+ list(num_elts++) = elt;
+ }
+
+ delete buf;
+ buf = 0;
+ }
+}
+
+/********************/
+void
+xtextscan_format_list::process_conversion (const std::string& s, size_t& i,
+ size_t n, octave_idx_type& num_elts)
+{
+ unsigned width = 0;
+ int prec = -1;
+ int bitwidth = 0;
+ bool discard = false;
+ octave_value val_type;
+ char type = '\0';
+
+ *buf << s[i++];
+
+ bool have_width = false;
+
+ while (i < n)
+ {
+ switch (s[i])
+ {
+ case '*':
+ if (discard)
+ nconv = -1;
+ else
+ {
+ discard = true;
+ *buf << s[i++];
+ }
+ break;
+
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ if (have_width)
+ nconv = -1;
+ else
+ {
+ char c = s[i++];
+ width = width * 10 + c - '0';
+ have_width = true;
+ *buf << c;
+ while (i < n && isdigit (s[i]))
+ {
+ c = s[i++];
+ width = width * 10 + c - '0';
+ *buf << c;
+ }
+
+ if (i < n && s[i] == '.')
+ {
+ *buf << s[i++];
+ prec = 0;
+ while (i < n && isdigit (s[i]))
+ {
+ c = s[i++];
+ prec = prec * 10 + c - '0';
+ *buf << c;
+ }
+ }
+ }
+ break;
+
+ case 'd': case 'u':
+ {
+ bool done = true;
+ *buf << (type = s[i++]);
+ if (i < n)
+ {
+ if (s[i] == '8')
+ {
+ bitwidth = 8;
+ if (type == 'd')
+ val_type = octave_value (int8NDArray ());
+ else
+ val_type = octave_value (uint8NDArray ());
+ *buf << s[i++];
+ }
+ else if (s[i] == '1' && i+1 < n && s[i+1] == '6')
+ {
+ bitwidth = 16;
+ if (type == 'd')
+ val_type = octave_value (int16NDArray ());
+ else
+ val_type = octave_value (uint16NDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else if (s[i] == '3' && i+1 < n && s[i+1] == '2')
+ {
+ done = false; // use default size below
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else if (s[i] == '6' && i+1 < n && s[i+1] == '4')
+ {
+ bitwidth = 64;
+ if (type == 'd')
+ val_type = octave_value (int64NDArray ());
+ else
+ val_type = octave_value (uint64NDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else
+ done = false;
+ }
+ else
+ done = false;
+
+ if (!done)
+ {
+ bitwidth = 32;
+ if (type == 'd')
+ val_type = octave_value (int32NDArray ());
+ else
+ val_type = octave_value (uint32NDArray ());
+ }
+ goto fini;
+ }
+
+ case 'f':
+ *buf << (type = s[i++]);
+ bitwidth = 64;
+ if (i < n)
+ {
+ if (s[i] == '3' && i+1 < n && s[i+1] == '2')
+ {
+ bitwidth = 32;
+ val_type = octave_value (FloatNDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else if (s[i] == '6' && i+1 < n && s[i+1] == '4')
+ {
+ val_type = octave_value (NDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else
+ val_type = octave_value (NDArray ());
+ }
+ else
+ val_type = octave_value (NDArray ());
+ goto fini;
+
+ case 'n':
+ *buf << (type = s[i++]);
+ bitwidth = 64;
+ val_type = octave_value (NDArray ());
+ goto fini;
+
+ case 's': case 'q': case '[': case 'c':
+ if (!discard)
+ val_type = octave_value (Cell ());
+ *buf << (type = s[i++]);
+ has_string = true;
+ goto fini;
+
+ fini:
+ {
+ if (!have_width)
+ {
+ if (type == 'c') // %c defaults to one character
+ width = 1;
+ else
+ width = static_cast (-1); // others: unlimited
+ }
+
+ if (finish_conversion (s, i, n, width, prec, bitwidth, val_type,
+ discard, type, num_elts) == 0)
+ return;
+ }
+ break;
+
+ default:
+ error ("xtextscan: '%%%c' is not a valid format specifier", s[i]);
+ nconv = -1;
+ break;
+ }
+
+ if (nconv < 0)
+ break;
+ }
+
+ nconv = -1;
+}
+
+/********************/
+int
+xtextscan_format_list::finish_conversion (const std::string& s, size_t& i,
+ size_t n, unsigned int& width,
+ int& prec, int& bitwidth,
+ octave_value& val_type, bool discard,
+ char& type, octave_idx_type& num_elts)
+{
+ int retval = 0;
+
+ std::string char_class;
+
+ size_t beg_idx = std::string::npos;
+ size_t end_idx = std::string::npos;
+
+ if (type != '%')
+ {
+ nconv++;
+ if (type == '[')
+ {
+ if (i < n)
+ {
+ beg_idx = i;
+
+ if (s[i] == '^')
+ {
+ type = '^';
+ *buf << s[i++];
+
+ if (i < n)
+ {
+ beg_idx = i;
+
+ if (s[i] == ']')
+ *buf << s[i++];
+ }
+ }
+ else if (s[i] == ']')
+ *buf << s[i++];
+ }
+
+ while (i < n && s[i] != ']')
+ *buf << s[i++];
+
+ if (i < n && s[i] == ']')
+ {
+ end_idx = i-1;
+ *buf << s[i++];
+ }
+
+ if (s[i-1] != ']')
+ retval = nconv = -1;
+ }
+ }
+
+ if (nconv >= 0)
+ {
+ if (beg_idx != std::string::npos && end_idx != std::string::npos)
+ char_class = expand_char_class (s.substr (beg_idx,
+ end_idx - beg_idx + 1));
+
+ add_elt_to_list (width, prec, bitwidth, val_type, discard, type,
+ num_elts, char_class);
+ }
+
+ return retval;
+}
+
+/********************/
+// Apply global options to individual conversion specifiers.
+// Currently, records whether or not we need the custom parser for doubles,
+// or if octave_read_value is sufficient.
+void
+xtextscan_format_list::consider_options (const xtextscan& ts)
+{
+ // We need to use custom number parser if the list of exponential signifiers
+ // (the "e" in 1e20) is non-default
+ bool all_need_custom = !ts.default_exp;
+
+ // Check if any delimiter starts with an exponential signifier.
+ // If so, we need the custom number parser.
+ if (!all_need_custom)
+ {
+ for (int i = 0; i < ts.delim_list.numel (); i++)
+ {
+ std::string s = ts.delim_list(i).string_value ();
+ if (s.length () > 0)
+ {
+ char last = s[s.length () - 1];
+ if (strchr (ts.exp_chars.c_str (), last))
+ {
+ all_need_custom = true;
+ break;
+ }
+ }
+ }
+ }
+
+ // Need custom parser if options mean that all do, or if this format
+ // is something like %10f or %10.4f.
+ for (int i = 0; i < numel (); i++)
+ {
+ list(i)->custom_number = (all_need_custom || list(i)->prec != -1
+ || list(i)->width != static_cast (-1));
+ }
+}
+
+/********************
+void
+xtextscan_format_list::printme (void) const
+{
+ octave_idx_type n = list.numel ();
+
+ for (octave_idx_type i = 0; i < n; i++)
+ {
+ xtextscan_format_elt *elt = list(i);
+
+ std::cerr
+ << "width: " << elt->width << "\n"
+ << "digits " << elt->prec << "\n"
+ << "bitwidth: " << elt->bitwidth << "\n"
+ << "discard: " << elt->discard << "\n"
+ << "type: ";
+
+ if (elt->type == xtextscan_format_elt::literal_conversion)
+ std::cerr << "literal text\n";
+ else if (elt->type == xtextscan_format_elt::whitespace_conversion)
+ std::cerr << "whitespace\n";
+ else
+ std::cerr << elt->type << "\n";
+
+ std::cerr
+ << "char_class: `" << undo_string_escapes (elt->char_class) << "'\n"
+ << "text: `" << undo_string_escapes (elt->text) << "'\n\n";
+ }
+}
+* */
+
+/********************/
+// If FORMAT is explicitly "", it is assumed to be "%f" repeated enough times
+// to read the first row of the file. Set it now.
+void
+xtextscan_format_list::read_first_row (std::istream& is, xtextscan& ts)
+{
+ std::string first_line;
+ getline (is, first_line, static_cast (ts.eol2));
+ std::istringstream strstr (first_line);
+
+ dim_vector dv (1,1); // initial size of each output_container
+ Complex val;
+ octave_value val_type;
+ nconv = 0;
+ int max_empty = 1000; // failsafe, if strstr fails but not with eof
+ long int pos;
+
+ // read line, creating output_container as we go
+ while (!strstr.eof ())
+ {
+ pos = strstr.tellg ();
+ ts.scan_complex (strstr, *list(0), val);
+ if (strstr.fail ()) // allow missing values in first row
+ {
+ strstr.clear (strstr.rdstate () & ~std::ios_base::failbit);
+ val = ts.empty_value.scalar_value ();
+ if (!--max_empty)
+ break;
+ }
+ if (val.imag () == 0)
+ val_type = octave_value (NDArray (dv, val.real ()));
+ else
+ val_type = octave_value (ComplexNDArray (dv, val));
+ output_container.push_back (val_type);
+ ts.skip_delim (strstr);
+ if (strstr.tellg () == pos) // Stop if we're not making progress
+ break;
+ nconv++;
+ }
+ output_container.pop_front (); // discard empty element from constructor
+
+ //Create fmt now that the size is known
+ list.resize (dim_vector (nconv, 1));
+ for (octave_idx_type i = 1; i < nconv; i++)
+ list(i) = new xtextscan_format_elt (*list(0));
+}
+
+/********************/
+// Perform actual xtextscan: read data from stream, and create cell array
+octave_value
+xtextscan::scan (std::istream *isp, xtextscan_format_list& fmt_list,
+ octave_idx_type ntimes)
+{
+ octave_value retval = Cell ();
+
+ if (!isp)
+ error ("internal error: xtextscan called with invalid istream");
+ else if (fmt_list.num_conversions () == -1)
+ ::error ("xtextscan: invalid format specified");
+ else if (! error_state)
+ {
+ if (fmt_list.num_conversions () == 0)
+ {
+ error ("xtextscan: no valid format conversion specifiers\n");
+ return Cell ();
+ }
+
+ // skip the first header_lines
+ std::string dummy;
+ for (int i = 0; i < header_lines && *isp; i++)
+ getline (*isp, dummy, static_cast (eol2));
+
+ // Grow retval dynamically. (If ntimes large, start smaller?)
+ // This is half the initial size
+ octave_idx_type size = ((ntimes < 8 && ntimes >= 0) ? ntimes : 1);
+ Array row_idx (dim_vector (1,2));
+ row_idx(1) = 0;
+
+ int err = 0;
+ octave_idx_type row = 0;
+
+ if (multiple_delims_as_one) // bug #44750?
+ skip_delim (*isp);
+
+ // If FORMAT explicitly "", read first line and see how many "%f" match
+ if (fmt_list.set_from_first)
+ {
+ row = 1; // the above puts the first line into fmt_list.out_buf ()
+ fmt_list.read_first_row (*isp, *this);
+ lines = 1;
+ }
+ std::list out = fmt_list.out_buf ();
+
+ // Check which columns to merge. Reals may become complex, and so can't
+ // trust types after reading in data.
+ bool merge_with_prev[fmt_list.numel ()];
+ int conv = 0;
+ if (collect_output)
+ {
+ int prev_type = -1;
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++)
+ {
+ if (col->type_id () == prev_type)
+ merge_with_prev [conv++] = true;
+ else
+ merge_with_prev [conv++] = false;
+ prev_type = col->type_id ();
+ }
+ }
+
+ // Read the data
+ for (/* row set ~25 lines above */; row < ntimes || ntimes == -1; row++)
+ {
+ if (row == 0 || row >= size)
+ {
+ size += size+1;
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++)
+ *col = (*col).resize (dim_vector (size, 1), 0);
+ }
+ row_idx(0) = row;
+ err = read_format_once (isp, fmt_list, out, row_idx);
+ if (err < 0 || !*isp || error_state
+ || (lines >= ntimes && ntimes > -1))
+ break;
+ }
+
+ if (return_on_error)
+ error_state = false;
+
+ // convert return value to Cell array
+ int i = 0;
+ Array ra_idx (dim_vector (1,2));
+ // err = -2 means "error, and no columns read this row
+ int valid_rows = (row == ntimes) ? ntimes : ((err == -2) ? row : row+1);
+ dim_vector dv (valid_rows, 1);
+ ra_idx(0) = 0;
+ if (!collect_output)
+ {
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++, i++)
+ {
+ // resize col to size row
+ ra_idx(1) = i;
+ retval = do_cat_op (retval,
+ octave_value (Cell (col->resize (dv,0))),
+ ra_idx);
+ }
+ }
+ else // group adjacent cells of the same type into a single cell
+ {
+ octave_value cur; // current cell, accumulating columns
+ octave_idx_type group_size = 0; // columns in this cell
+ int prev_type = -1;
+
+ conv = 0;
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++)
+ {
+ if (!merge_with_prev [conv++]) // including first time
+ {
+ if (prev_type != -1)
+ {
+ ra_idx(1) = i++;
+ retval = do_cat_op (retval, octave_value (Cell(cur)),
+ ra_idx);
+ }
+ cur = octave_value (col->resize (dv,0));
+ group_size = 1;
+ prev_type = col->type_id ();
+ }
+ else
+ {
+ ra_idx(1) = group_size++;
+ cur = do_cat_op (cur, octave_value (col->resize (dv,0)),
+ ra_idx);
+ }
+ }
+ ra_idx(1) = i;
+ retval = do_cat_op (retval, octave_value (Cell (cur)), ra_idx);
+ }
+ }
+ return retval;
+}
+
+/*
+%!test
+%! str = "1, 2, 3, 4\n 5, , , 8\n 9, 10, 11, 12";
+%! fmtstr = "%f %d %f %s";
+%! c = xtextscan (str, fmtstr, 2, "delimiter", ",", "emptyvalue", -Inf);
+%! assert (isequal (c{1}, [1;5]));
+%! assert (length (c{1}), 2);
+%! assert (iscellstr (c{4}));
+%! assert (isequal (c{3}, [3; -Inf]));
+
+%!test
+%! b = [10:10:100];
+%! b = [b; 8*b/5];
+%! str = sprintf ("%g miles/hr = %g kilometers/hr\n", b);
+%! fmt = "%f miles/hr = %f kilometers/hr";
+%! c = xtextscan (str, fmt);
+%! assert (c{1}, b(1,:)', 1e-5);
+%! assert (c{2}, b(2,:)', 1e-5);
+
+%!test
+%! str = "13, -, NA, str1, -25\r\n// Middle line\r\n36, na, 05, str3, 6";
+%! a = xtextscan (str, "%d %n %f %s %n", "delimiter", ",",
+%! "treatAsEmpty", {"NA", "na", "-"},"commentStyle", "//");
+%! assert (a{1}, int32 ([13; 36]));
+%! assert (a{2}, [NaN; NaN]);
+%! assert (a{3}, [NaN; 5]);
+%! assert (a{4}, {"str1"; "str3"});
+%! assert (a{5}, [-25; 6]);
+
+%!test
+%! str = "Km:10 = hhhBjjj miles16hour\r\n";
+%! str = [str "Km:15 = hhhJjjj miles241hour\r\n"];
+%! str = [str "Km:2 = hhhRjjj miles3hour\r\n"];
+%! str = [str "Km:25 = hhhZ\r\n"];
+%! fmt = "Km:%d = hhh%1sjjj miles%dhour";
+%! a = xtextscan (str, fmt, "delimiter", " ");
+%! assert (a{1}', int32 ([10 15 2 25]));
+%! assert (a{2}', {'B' 'J' 'R' 'Z'});
+%! assert (a{3}', int32 ([16 241 3 0]));
+
+## Test with default endofline parameter
+%!test
+%! c = xtextscan ("L1\nL2", "%s");
+%! assert (c{:}, {"L1"; "L2"});
+
+## Test with endofline parameter set to "" (empty) - newline should be in word
+%!test
+%! c = xtextscan ("L1\nL2", "%s", "endofline", "");
+%! assert (int8 ([c{:}{:}]), int8 ([ 76, 49, 10, 76, 50 ]));
+
+### Matlab fails this test. A literal after a conversion is not a delimiter
+#%!test
+#%! ## No delimiters at all besides EOL. Skip fields, even empty fields
+#%! str = "Text1Text2Text\nTextText4Text\nText57Text";
+#%! c = xtextscan (str, "Text%*dText%dText");
+#%! assert (c{1}, int32 ([2; 4; 0]));
+
+## CollectOutput test
+%!test
+%! b = [10:10:100];
+%! b = [b; 8*b/5; 8*b*1000/5];
+%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
+%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
+%! c = xtextscan (str, fmt, "collectoutput", 1);
+%! assert (size (c{3}), [10, 2]);
+%! assert (size (c{2}), [10, 2]);
+
+## CollectOutput test with uneven column length files
+%!test
+%! b = [10:10:100];
+%! b = [b; 8*b/5; 8*b*1000/5];
+%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
+%! str = [str "110 miles/hr"];
+%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
+%! c = xtextscan (str, fmt, "collectoutput", 1);
+%! assert (size (c{1}), [11, 1]);
+%! assert (size (c{3}), [11, 2]);
+%! assert (size (c{2}), [11, 2]);
+%! assert (c{3}(end), NaN);
+%! assert (c{2}{11, 1}, "/hr");
+%! assert (isempty (c{2}{11, 2}), true);
+
+## Double quoted string
+%!test
+%! str = 'First "the second called ""the middle""" third';
+%! fmt = "%q";
+%! c = xtextscan (str, fmt);
+%! assert (c{1}, {"First"; 'the second called "the middle"'; "third"});
+
+## Arbitrary character
+%!test
+%! c = xtextscan ("a first, \n second, third", "%s %c %11c", 'delimiter', ' ,');
+%! assert (c{1}, {"a"; "ond"});
+%! assert (c{2}, {"f"; "t"});
+%! assert (c{3}, {"irst, \n sec"; "hird"});
+
+## Field width and non-standard delimiters
+%!test
+%! str = "12;34;123456789;7";
+%! c = xtextscan (str, "%4d %4d", "delimiter", ";", "collectOutput", 1);
+%! assert (c, {[12 34; 1234 5678; 9 7]});
+
+## Field width and non-standard delimiters
+%!test
+%! str = "12;34;123456789;7";
+%! c = xtextscan (str, "%4f %f", "delimiter", ";", "collectOutput", 1);
+%! assert (c, {[12 34; 1234 56789; 7 NaN]});
+
+## Ignore trailing delimiter, but use leading one
+%!test
+%! str = "12.234e+2,34, \n12345.789-9876j,78\n,10|3";
+%! c = xtextscan (str, "%10.2f %f", "delimiter", ",", "collectOutput", 1,
+%! "expChars", "e|");
+%! assert (c, {[1223 34; 12345.79-9876j 78; NaN 10000]}, 1e-6);
+
+%!test
+%! ## Multi-character delimiter
+%! str = "99end2 space88gap 4564";
+%! c = xtextscan (str, "%d %s", "delimiter", {"end", "gap", "space"});
+%! assert (c{1}, int32([99; 88]));
+%! assert (c{2}, {"2 "; "4564"});
+
+### Delimiters as part of literals, and following literals
+#%!test
+#%! str = "12 R&D & 7";
+#%! c = xtextscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1, "EmptyValue", -99);
+#%! assert (c, {[12 -99; 7 -99]});
+#
+### Delimiters as part of literals, and before literals
+#%!test
+#%! str = "12 & R&D 7";
+#%! c = xtextscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1);
+#%! assert (c, {[12 7]});
+
+%!test
+%! ## Check number of lines read, not number of passes through format string
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, "1\n2\n3\n4\n5\n6");
+%! fseek (fid, 0, "bof");
+%! c = xtextscan (fid, "%f %f", 2);
+%! E = feof (fid);
+%! fclose (fid);
+%! unlink (f);
+%! assert (c, {1, 2});
+%! assert (!E);
+
+%!test
+%! ## Check number of lines read, not number of passes through format string
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, "1\r\n2\r3\n4\r\n5\n6");
+%! fseek (fid, 0, "bof");
+%! c = xtextscan (fid, "%f %f", 4);
+%! fclose (fid);
+%! unlink (f);
+%! assert (c, {[1;3], [2;4]})
+
+%!test
+%! ## Check number of lines read, with multiple delimiters
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, "1-\r\n-2\r3-\n-4\r\n5\n6");
+%! fseek (fid, 0, "bof");
+%! c = xtextscan (fid, "%f %f", 4, "delimiter", "-", "multipleDelimsAsOne", 1);
+%! fclose (fid);
+%! unlink (f);
+%! assert (c, {[1;3], [2;4]})
+
+## Test input validation
+%!error xtextscan ()
+%!error xtextscan (single (40))
+%!error xtextscan ({40})
+%!error xtextscan ("Hello World", 2)
+#%!error [C, pos] = xtextscan ("Hello World")
+%!error xtextscan ("Hello World", '%s', 'EndOfLine', 3)
+%!error <'%z' is not a valid format specifier> xtextscan ("1.0", "%z");
+%!error xtextscan ("1.0", "foo");
+
+## Test incomplete first data line
+%! R = xtextscan (['Empty1' char(10)], 'Empty%d %f');
+%! assert (R{1}, int32 (1));
+%! assert (isempty (R{2}), true);
+
+## bug #37023
+%!test
+%! data = xtextscan(" 1. 1 \n 2 3\n", '%f %f');
+%! assert (data{1}, [1; 2], 1e-15);
+%! assert (data{2}, [1; 3], 1e-15);
+
+## Whitespace test (bug #37333) using delimiter ";"
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = "C:/code/meas;";
+%! tc{1, end+1} = " C:/code/sim;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = xtextscan (string, "%s", "delimiter", ";");
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333), adding multipleDelimsAsOne true arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = xtextscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 1);
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333), adding multipleDelimsAsOne false arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;;";
+%! tc{1, end+1} = "";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = xtextscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 0);
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333) whitespace "" arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = xtextscan (string, "%s", "delimiter", ";", "whitespace", "");
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333), whitespace " " arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = xtextscan (string, "%s", "delimiter", ";", "whitespace", " ");
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+## Tests reading with empty format, should return proper nr of columns
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, " 1 2 3 4\n5 6 7 8");
+%! fseek (fid, 0, "bof");
+%! A = xtextscan (fid, "");
+%! E = feof (fid);
+%! fclose (fid);
+%! unlink (f);
+%! assert (A{1}, [1 ; 5], 1e-6);
+%! assert (A{2}, [2 ; 6], 1e-6);
+%! assert (A{3}, [3 ; 7], 1e-6);
+%! assert (A{4}, [4 ; 8], 1e-6);
+%! assert (E);
+
+## Tests reading with empty format; empty fields & incomplete lower row
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, " ,2,,4\n5,6");
+%! fseek (fid, 0, "bof");
+%! A = xtextscan (fid, "", "delimiter", ",", "EmptyValue", 999, "CollectOutput" , 1);
+%! fclose (fid);
+%! unlink (f);
+%! assert (A{1}, [999, 2, 999, 4; 5, 6, 999, 999], 1e-6);
+
+## Error message tests
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! msg1 = "xtextscan: 1 parameters given, but only 0 values";
+%! try
+%! A = xtextscan (fid, "", "headerlines");
+%! end_try_catch;
+%! assert (!feof (fid));
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! msg1 = "xtextscan: HeaderLines must be numeric";
+%! try
+%! A = xtextscan (fid, "", "headerlines", "hh");
+%! end_try_catch;
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+%!test
+%! ## Skip headerlines
+%! A = xtextscan ("field 1 field2\n 1 2\n3 4", "", "headerlines", 1, "collectOutput", 1);
+%! assert (A, {[1 2; 3 4]});
+
+%!test
+%! ## Skip headerlines with non-default EOL
+%! A = xtextscan ("field 1 field2\r 1 2\r3 4", "", "headerlines", 2, "collectOutput", 1, "EndOfLine", '\r');
+%! assert (A, {[3 4]});
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid,"some_string");
+%! fseek (fid, 0, "bof");
+%! msg1 = "xtextscan: EndOfLine must be at most one character or '\\r\\n'";
+%! try
+%! A = xtextscan (fid, "%f", "EndOfLine", "\n\r");
+%! end_try_catch;
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid,"some_string");
+%! fseek (fid, 0, "bof");
+%! msg1 = "xtextscan: EndOfLine must be at most one character or '\\r\\n'";
+%! try
+%! A = xtextscan (fid, "%f", "EndOfLine", 33);
+%! end_try_catch;
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+## Bug #41824
+%!test
+%! assert (xtextscan ("123", "", "whitespace", " "){:}, 123);
+
+## Bug #42343-1, just test supplied emptyvalue
+%!test
+%! assert (xtextscan (",NaN", "", "delimiter", "," ,"emptyValue" ,Inf), {Inf, NaN});
+
+## Bug #42343-2, test padding with supplied emptyvalue
+%!test
+%! a = xtextscan (",1,,4\nInf, ,NaN", "", "delimiter", ",", "emptyvalue", -10);
+%! assert (cell2mat (a), [-10, 1, -10, 4; Inf, -10, NaN, -10]);
+
+## Bug #42528
+%!test
+%! assert (xtextscan ("1i", ""){1}, 0+1i);
+%! assert (cell2mat (xtextscan ("3, 2-4i, NaN\n -i, 1, 23.4+2.2i\n 1+1 1+1j", "", "delimiter", ",")), [3+0i, 2-4i, NaN+0i; 0-i, 1+0i, 23.4+2.2i; 1 1 1+1i]);
+
+%!test
+%! ## TreatAsEmpty
+%! C = xtextscan ("1,2,3,NN,5,6\n", "%d%d%d%f", "delimiter", ",", "TreatAsEmpty", "NN");
+%! assert (C{3}(1), int32 (3));
+%! assert (C{4}(1), NaN);
+
+## MultipleDelimsAsOne
+%!test
+%! str = "11, 12, 13,, 15\n21,, 23, 24, 25\n,, 33, 34, 35";
+%! C = xtextscan (str, "%f %f %f %f", "delimiter", ",", "multipledelimsasone", 1, "endofline", "\n");
+%! assert (C{1}', [11, 21, 33]);
+%! assert (C{2}', [12, 23, 34]);
+%! assert (C{3}', [13, 24, 35]);
+%! assert (C{4}', [15, 25, NaN]);
+
+## Bug #44750
+%!test
+%! assert (xtextscan ("/home/foo/", "%s", "delimiter", "/", "MultipleDelimsAsOne", 1){1}, ...
+%! {"home"; "foo"});
+
+### Allow cuddling %sliteral but warn it is ambiguous
+#%!test
+#%! C = xtextscan ("abcxyz51\nxyz83\n##xyz101", "%s xyz %d");
+#%! assert (C{1}([1 3]), {"abc"; "##"});
+#%! assert (isempty (C{1}{2}), true);
+#%! assert (C{2}, int32([51; 83; 101]));
+### Literals are not delimiters.
+
+## Test for false positives in check for non-supported format specifiers
+%!test
+%! assert (xtextscan ("Total: 32.5 % (of cm values)", "Total: %f %% (of cm values)"){1}, 32.5, 1e-5);
+
+## Test various forms of string format specifiers (bug #45712)
+%!test
+%! str = "14 :1 z:2 z:3 z:5 z:11";
+%! C = xtextscan (str, "%f %s %*s %3s %*3s %f", "delimiter", ":");
+%! assert (C, {14, {"1 z"}, {"3 z"}, 11});
+
+%% Bit width, fixed width conv. specifiers
+%!test
+%! str2 = "123456789012345 ";
+%! str2 = [str2 str2 str2 str2 str2 str2 str2 str2];
+%! str2 = [str2 "123456789.01234 1234567890.1234 12345.678901234 12345.678901234"];
+%! pttrn = "%3u8%*s %5u16%*s %10u32%*s %15u64 %3d8%*s %5d16%*s %10d32%*s %15d64 %9f32%*s %14f64%*s %10.2f32%*s %12.2f64%*s";
+%! C = xtextscan (str2, pttrn, "delimiter", " ");
+%! assert (C{1}, uint8 (123));
+%! assert (C{2}, uint16 (12345));
+%! assert (C{3}, uint32 (1234567890));
+%! assert (C{4}, uint64 (123456789012345));
+%! assert (C{5}, int8 (123));
+%! assert (C{6}, int16 (12345));
+%! assert (C{7}, int32 (1234567890));
+%! assert (C{8}, int64 (123456789012345));
+%! assert (C{9}, single (123456789), 1e-12);
+%! assert (C{10}, double (1234567890.123), 1e-15);
+%! assert (C{11}, single (12345.68), 1e-5);
+%! assert (C{12}, double (12345.68), 1e-11);
+
+%% Bit width, fixed width conv. specifiers -- check the right amount is left
+%!test
+%! str2 = "123456789012345 ";
+%! str2 = [str2 str2 "123456789.01234"];
+%! pttrn = "%3u8 %5u16 %10u32 %3d8 %5d16 %10d32 %9f32 %9f";
+%! C = xtextscan (str2, pttrn, "delimiter", " ");
+%! assert (C{1}, uint8 (123));
+%! assert (C{2}, uint16 (45678));
+%! assert (C{3}, uint32 (9012345));
+%! assert (C{4}, int8 (123));
+%! assert (C{5}, int16 (45678));
+%! assert (C{6}, int32 (9012345));
+%! assert (C{7}, single (123456789), 1e-12);
+%! assert (C{8}, double (0.01234), 1e-12);
+
+%!test
+%! C = xtextscan ("123.123", "%2f %3f %3f");
+%! assert (C{1}, 12);
+%! assert (C{2}, 3.1, 1e-11);
+%! assert (C{3}, 23);
+
+%!test
+%! C = xtextscan ("123.123", "%3f %3f %3f");
+%! assert (C{1}, 123);
+%! assert (C{2}, 0.12, 1e-11);
+%! assert (C{3}, 3);
+
+%!test
+%! C = xtextscan ("123.123", "%4f %3f");
+%! assert (C{1}, 123);
+%! assert (C{2}, 123);
+
+%% field width interrupts exponent. (Matlab incorrectly gives [12, 2e12])
+%!test
+%! assert (xtextscan ("12e12", "%4f"), {[120; 2]});
+%! assert (xtextscan ("12e+12", "%5f"), {[120; 2]});
+%! assert (xtextscan ("125e-12","%6f"), {[12.5; 2]});
+
+%% %[] tests
+%% Plain [..] and *[..]
+%!test
+%! ar = "abcdefguvwxAny\nacegxyzTrailing\nJunk";
+%! C = xtextscan (ar, "%[abcdefg] %*[uvwxyz] %s");
+%! assert (C{1}, {"abcdefg"; "aceg"; ""});
+%! assert (C{2}, {"Any"; "Trailing"; "Junk"});
+
+%!test
+%! assert (xtextscan ("A2 B2 C3", "%*[ABC]%d", 3), {int32([2; 2; 3])});
+
+%% [^..] and *[^..]
+%!test
+%! br = "abcdefguvwx1Any\nacegxyz2Trailing\n3Junk";
+%! C = xtextscan (br, "%[abcdefg] %*[^0123456789] %s");
+%! assert (C{1}, {"abcdefg"; "aceg"; ""});
+%! assert (C{2}, {"1Any"; "2Trailing"; "3Junk"});
+
+%% [..] and [^..] containing delimiters
+%!test
+%! cr = "ab cd efguv wx1Any\na ce gx yz2Trailing\n 3Junk";
+%! C = xtextscan (cr, "%[ abcdefg] %*[^0123456789] %s", "delimiter", " \n", "whitespace", "");
+%! assert (C{1}, {"ab cd efg"; "a ce g"; " "});
+%! assert (C{2}, {"1Any"; "2Trailing"; "3Junk"});
+
+%% Bug #36464
+%!test
+%! assert (xtextscan ('1 2 3 4 5 6', "%*n%n%*[^\n]"){1}, 2);
+
+%% test %[]] and %[^]]
+%!test
+%! assert (xtextscan ('345]', "%*[123456]%[]]"){1}{1}, "]");
+%! assert (xtextscan ('345]', "%*[^]]%s"){1}{1}, "]");
+
+%% Test that "-i" checks the next two characters
+%!test
+%! C = xtextscan ("-i -in -inf -infinity", "%f %f%s %f %f %s");
+%! assert (C, {-i, -i, {"n"}, -Inf, -Inf, {"inity"}});
+
+%% Again for "+i", this time with custom parser
+%!test
+%! C = xtextscan ("+i +in +inf +infinity", "%f %f%s %f %f %s", "ExpChars", "eE");
+%! assert (C, {i, i, {"n"}, Inf, Inf, {"inity"}});
+
+%% Check single quoted format interprets control sequences
+%!test
+%! C = xtextscan ("1 2\t3 4", '%f %[^\t] %f %f');
+%! assert (C, {1, {"2"}, 3, 4});
+
+%% Check overflow and underflow of integer types
+%!test
+%! a = "-1e90 ";
+%! b = "1e90 ";
+%! fmt = "%d8 %d16 %d32 %d64 %u8 %u16 %u32 %u64 ";
+%! C = xtextscan ([a a a a a a a a b b b b b b b b], fmt);
+%! assert (C{1}, int8 ([-128; 127]));
+%! assert (C{2}, int16([-32768; 32767]));
+%! assert (C{3}, int32([-2147483648; 2147483647]));
+%! assert (C{4}, int64([-9223372036854775808; 9223372036854775807]));
+%! assert (C{5}, uint8 ([0; 255]));
+%! assert (C{6}, uint16([0; 65535]));
+%! assert (C{7}, uint32([0; 4294967295]));
+%! assert (C{8}, uint64([0; 18446744073709551615]));
+
+%% Tests from Matlab (does The MathWorks have any copyright over the input?)
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid,"09/12/2005 Level1 12.34 45 1.23e10 inf Nan Yes 5.1+3i\n");
+%! fprintf (fid,"10/12/2005 Level2 23.54 60 9e19 -inf 0.001 No 2.2-.5i\n");
+%! fprintf (fid,"11/12/2005 Level3 34.90 12 2e5 10 100 No 3.1+.1i\n");
+%! fseek (fid, 0, "bof");
+%! C = xtextscan(fid,"%s %s %f32 %d8 %u %f %f %s %f");
+%! %assert (C{1}, {"09/12/2005";"10/12/2005";"11/12/2005"});
+%! assert (C{2}, {"Level1";"Level2";"Level3"});
+%! assert (C{3}, [single(12.34);single(23.54);single(34.90)]);
+%! assert (C{4}, [int8(45);int8(60);int8(12)]);
+%! assert (C{5}, [uint32(4294967295);uint32(4294967295);uint32(200000)]);
+%! assert (C{6}, [inf;-inf;10]);
+%! assert (C{7}, [NaN;0.001;100]);
+%! assert (C{8}, {"Yes";"No";"No"});
+%! assert (C{9}, [5.1+3i;2.2-0.5i;3.1+0.1i]);
+%! fseek (fid, 0, "bof");
+%! C = xtextscan(fid,"%s Level%d %f32 %d8 %u %f %f %s %f");
+%! assert (C{2}, [int32(1);int32(2);int32(3)]);
+%! assert (C{3}, [single(12.34);single(23.54);single(34.90)]);
+%! fseek (fid, 0, "bof");
+%! C = xtextscan(fid,'%s %*[^\n]');
+%! fclose (fid);
+%! unlink (f);
+%! assert (C, {{"09/12/2005";"10/12/2005";"11/12/2005"}});
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid,"1, 2, 3, 4, , 6\n");
+%! fprintf (fid,"7, 8, 9, , 11, 12\n");
+%! fseek (fid, 0, "bof");
+%! C = xtextscan(fid,"%f %f %f %f %u8 %f", "Delimiter",",","EmptyValue",-Inf);
+%! fclose (fid);
+%! unlink (f);
+%! assert (C{4}, [4; -Inf]);
+%! assert (C{5}, uint8 ([0; 11]));
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid,"abc, 2, NA, 3, 4\n");
+%! fprintf (fid,"// Comment Here\n");
+%! fprintf (fid,"def, na, 5, 6, 7\n");
+%! fseek (fid, 0, "bof");
+%! C = xtextscan(fid,"%s %n %n %n %n","Delimiter",",","TreatAsEmpty",{"NA","na"},"CommentStyle","//");
+%! fclose (fid);
+%! unlink (f);
+%! assert (C{1}, {"abc";"def"});
+%! assert (C{2}, [2; NaN]);
+%! assert (C{3}, [NaN; 5]);
+%! assert (C{4}, [3; 6]);
+%! assert (C{5}, [4; 7]);
+
+%!test
+%!## Test start of comment as string
+%! c = xtextscan ("1 / 2 // 3", "%n %s %u8", "CommentStyle", {"//"});
+%! assert (c, {1, "/", 2});
+*/
+
+// These tests have end-comment sequences, so can't just be in a comment
+#if 0
+%!test
+%!## Test unfinished comment
+%! c = xtextscan ("1 2 /* half comment", "%n %u8", "CommentStyle", {"/*", "*/});
+%! assert (c, {1, 2});
+
+## Test reading from a real file
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! d = rand (1, 4);
+%! fprintf (fid, " %f %f /* comment */ %f %f ", d);
+%! fseek (fid, 0, "bof");
+%! A = xtextscan (fid, "%f %f", "CommentStyle", {"/*", "*/"});
+%! E = feof (fid);
+%! fclose (fid);
+%! unlink (f);
+%! assert (A{1}, [d(1); d(3)], 1e-6);
+%! assert (A{2}, [d(2); d(4)], 1e-6);
+%! assert (E);
+#endif
+
+
+/*
+## Needed tests:
+# 1. Check that it fails at an appropriate point
+# 2. Checks for file vs string
+# 3. Checks for ReturnOnError
+# 4. Checks for all types of invalid parameter
+
+#ToDO:
+# 1. Fail when the first mismatch occurs. Tests for this
+# 2. Treatment of missing final end-of-line -- unequal line lengths.
+# 3. Minimise duplication with scanf and printf? Use as basis for textread?
+# 4. Implement ReturnOnError. Perhaps:
+# omitted = permissive, present = Matlab behaviour.
+*/
diff -r dde2d050dab2 libinterp/dldfcn/xtextscan.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/libinterp/dldfcn/xtextscan.h Wed Dec 16 09:32:27 2015 +1100
@@ -0,0 +1,289 @@
+
+#if !defined (octave_xtextscan_h)
+#define octave_xtextscan_h 1
+
+// For Inf and NaN
+#include "lo-ieee.h"
+
+class
+OCTINTERP_API
+xtextscan_format_elt
+{
+public:
+
+ enum special_conversion
+ {
+ whitespace_conversion = 1,
+ literal_conversion = 2
+ };
+
+ xtextscan_format_elt (const char *txt = 0, int w = 0, int p = -1,
+ int bw = 0, bool dis = false, char typ = '\0',
+ const std::string& ch_class = std::string ())
+ : text (strsave (txt)), width (w), prec (p), bitwidth (bw),
+ char_class (ch_class), type (typ), discard (dis),
+ numeric(typ == 'd' || typ == 'u' || type == 'f' || type == 'n')
+ { }
+
+ xtextscan_format_elt (const xtextscan_format_elt& e)
+ : text (strsave (e.text)), width (e.width), prec (e.prec),
+ bitwidth (e.bitwidth), char_class (e.char_class), type (e.type),
+ discard (e.discard), numeric (e.numeric), custom_number (e.custom_number)
+ { }
+
+ xtextscan_format_elt& operator = (const xtextscan_format_elt& e)
+ {
+ if (this != &e)
+ {
+ text = strsave (e.text);
+ width = e.width;
+ prec = e.prec;
+ bitwidth = e.bitwidth;
+ discard = e.discard;
+ type = e.type;
+ numeric = e.numeric;
+ custom_number = e.custom_number;
+ char_class = e.char_class;
+ }
+
+ return *this;
+ }
+
+ ~xtextscan_format_elt (void) { delete [] text; }
+
+ // The C-style format string.
+ const char *text;
+
+ // The maximum field width.
+ unsigned int width;
+
+ // The maximum number of digits to read after the decimal in a
+ // floating point conversion.
+ int prec;
+
+ // The size of the result. For integers, bitwidth may be 8, 16, 34,
+ // or 64. For floating point values, bitwidth may be 32 or 64.
+ int bitwidth;
+
+ // The class of characters in a `[' or `^' format.
+ std::string char_class;
+
+ // Type of conversion
+ // -- `d', `u', `f', `n', `s', `q', `c', `%', `C', `D', `[' or `^'.
+ char type;
+
+ // TRUE if we are not storing the result of this conversion.
+ bool discard;
+
+ // TRUE if the type is 'd', 'u', 'f', 'n'
+ bool numeric;
+
+ // TRUE if width or precision is specified, or options.exp_chars is set
+ bool custom_number;
+};
+
+class xtextscan;
+
+class
+OCTINTERP_API
+xtextscan_format_list
+{
+public:
+
+ xtextscan_format_list (const std::string& fmt = std::string ());
+
+ ~xtextscan_format_list (void);
+
+ void consider_options (const xtextscan& options);
+
+ octave_idx_type num_conversions (void) const { return nconv; }
+
+ // The length can be different than the number of conversions.
+ // For example, "x %d y %d z" has 2 conversions but the length of
+ // the list is 3 because of the characters that appear after the
+ // last conversion.
+
+ octave_idx_type numel (void) const { return list.numel (); }
+
+ const xtextscan_format_elt *first (void)
+ {
+ curr_idx = 0;
+ return current ();
+ }
+
+ const xtextscan_format_elt *current (void) const
+ { return list.numel () > 0 ? list.elem (curr_idx) : 0; }
+
+ const xtextscan_format_elt *next (bool cycle = true)
+ {
+ curr_idx++;
+
+ if (curr_idx >= list.numel ())
+ {
+ if (cycle)
+ curr_idx = 0;
+ else
+ return 0;
+ }
+ return current ();
+ }
+
+ void printme (void) const;
+
+ bool ok (void) const { return (nconv >= 0); }
+
+ operator bool () const { return ok (); }
+
+ bool set_from_first; // true if number of %f to be set from data file
+ bool has_string; // at least one conversion specifier is s,q,c, or [...]
+ void read_first_row (std::istream& is, xtextscan& ts);
+
+ std::list out_buf () const { return (output_container); }
+
+private:
+
+ // Number of conversions specified by this format string, or -1 if
+ // invalid conversions have been found.
+ octave_idx_type nconv;
+
+ // Index to current element;
+ octave_idx_type curr_idx;
+
+ // FIXME -- maybe LIST should be a std::list object?
+ // List of format elements.
+ Array list;
+
+ // list holding column arrays of types specified by conversions
+ std::list output_container;
+
+ // Temporary buffer.
+ std::ostringstream *buf;
+
+ void add_elt_to_list (unsigned int width, int prec, int bitwidth,
+ octave_value val_type, bool discard,
+ char type, octave_idx_type& num_elts,
+ const std::string& char_class = std::string ());
+
+ void process_conversion (const std::string& s, size_t& i, size_t n,
+ octave_idx_type& num_elts);
+
+ int finish_conversion (const std::string& s, size_t& i, size_t n,
+ unsigned int& width, int& prec, int& bitwidth,
+ octave_value& val_type,
+ bool discard, char& type,
+ octave_idx_type& num_elts);
+ // No copying!
+
+ xtextscan_format_list (const xtextscan_format_list&);
+
+ xtextscan_format_list& operator = (const xtextscan_format_list&);
+};
+
+class
+xtextscan
+{
+ friend xtextscan_format_list;
+
+ std::string buf;
+ // Three cases for delim_table and delim_list
+ // 1. delim_table empty, delim_list empty: whitespace delimiters
+ // 2. delim_table = look-up table of delim chars, delim_list empty.
+ // 3. delim_table non-empty, delim_list = Cell array of delim strings
+ std::string delim_table; // delim_table[i]=='\0' if i is not a delimiter,
+ std::string whitespace_table;
+ Cell comment_style;
+ int comment_len; // How far ahead to look to detect an open comment
+ int comment_char; // first character of open comment
+
+ std::string date_locale;
+
+ Cell delim_list; // Array of strings of delimiters
+ Cell inf_nan; // 'inf' and 'nan' for formatted_double
+
+ octave_value empty_value;
+ std::string exp_chars;
+ int header_lines;
+ Cell treat_as_empty;
+ int treat_as_empty_len; // longest string to treat as "N/A"
+ std::string whitespace;
+ short eol1, eol2;
+
+ bool collect_output;
+ bool multiple_delims_as_one;
+ bool return_on_error;
+
+ bool default_exp;
+ bool numeric_delim;
+
+ octave_idx_type lines;
+
+ char *get_field (std::istream& isp, unsigned int width) const;
+ bool match_literal (std::istream& isp, const xtextscan_format_elt& elem);
+ int skip_whitespace (std::istream& is, bool EOLstop = false);
+ int skip_delim (std::istream& is);
+ bool is_delim (unsigned char ch) const
+ {
+ return (delim_table.length () == 0
+ && (isspace (ch) || ch == eol1 || ch == eol2))
+ || delim_table[ch] != '\0';
+ }
+ // the only delimiter is whitespace
+ bool whitespace_delim () const { return delim_table.length () == 0; }
+ bool isspace (unsigned int ch) const { return whitespace_table[ch & 0xff]; }
+
+ std::istream& scan_string (std::istream& is, const xtextscan_format_elt& fmt,
+ std::string& val) const;
+ std::string read_until (std::istream& is, const Cell& delimiters,
+ const std::string& ends) const;
+ std::istream& scan_cstring (std::istream& is, const xtextscan_format_elt& fmt,
+ std::string& val) const;
+ std::istream& scan_qstring (std::istream& is, const xtextscan_format_elt& fmt,
+ std::string& val);
+ int read_format_once (std::istream* isp, xtextscan_format_list& fmt_list,
+ std::list & retval,
+ Array row);
+ std::istream& scan_one (std::istream& is, const xtextscan_format_elt& fmt,
+ octave_value& ov, Array row);
+ int scan_bracket (std::istream& is, const char *pattern, std::string& val)
+ const;
+ int scan_caret (std::istream& is, const char *, std::string& val) const;
+ int lookahead (std::istream& is, const Cell& targets, int max_len,
+ bool case_sensitive = true) const;
+ int read_format_once(std::istream* isp, xtextscan_format_list& fmt_list,
+ Cell& retval, octave_idx_type row) const;
+
+ double formatted_double (std::istream& is, const xtextscan_format_elt& fmt)
+ const;
+ double read_double (std::istream& is, const xtextscan_format_elt& fmt) const
+ {
+ if (fmt.custom_number)
+ return formatted_double (is, fmt);
+ else
+ return octave_read_value (is);
+ }
+ std::istream& scan_complex (std::istream& is, const xtextscan_format_elt& fmt,
+ Complex& val) const;
+ public:
+
+ xtextscan ()
+ : buf (""), delim_table (""), comment_char(-2), empty_value (octave_NaN),
+ exp_chars ("edED"), header_lines (0), treat_as_empty_len (0),
+ whitespace (" \b\t"),
+ eol1('\r'), eol2('\n'),
+ collect_output (false), multiple_delims_as_one (false),
+ return_on_error (true), default_exp (true), numeric_delim (false),
+ lines (0)
+ {
+ inf_nan = Cell (dim_vector (1,2));
+ inf_nan(0) = Cell (octave_value ("inf"));
+ inf_nan(1) = Cell (octave_value ("nan"));
+ };
+
+ octave_value scan (std::istream* isp, xtextscan_format_list& fmt_list,
+ octave_idx_type ntimes);
+ void parse_options (const octave_value_list& args, int first_param,
+ xtextscan_format_list& formats);
+};
+
+
+#endif
diff -r dde2d050dab2 libinterp/module.mk
--- a/libinterp/module.mk Tue Dec 15 21:30:25 2015 +1100
+++ b/libinterp/module.mk Wed Dec 16 09:32:27 2015 +1100
@@ -68,6 +68,7 @@ LIBINTERP_BUILT_DISTFILES = \
$(GENERATED_PARSER_FILES) \
libinterp/corefcn/oct-tex-parser.h \
libinterp/corefcn/oct-tex-symbols.cc \
+ libinterp/dldfcn/xtextscan.h \
libinterp/parse-tree/oct-gperf.h \
libinterp/parse-tree/oct-parse.h