diff --git a/libinterp/builtin-defun-decls.h b/libinterp/builtin-defun-decls.h
index 0611b6f..353f953 100644
--- a/libinterp/builtin-defun-decls.h
+++ b/libinterp/builtin-defun-decls.h
@@ -525,6 +525,7 @@ DEFUN_DECL (Fhistory_size);
DEFUN_DECL (Fhistory_file);
DEFUN_DECL (Fhistory_timestamp_format_string);
DEFUN_DECL (Fhistory_save);
+DEFUN_DECL (Ftextscan);
DEFUN_DECL (F__octave_link_enabled__);
DEFUN_DECL (F__octave_link_edit_file__);
DEFUN_DECL (F__octave_link_message_dialog__);
diff --git a/libinterp/corefcn/c-file-ptr-stream.cc b/libinterp/corefcn/c-file-ptr-stream.cc
index 447178e..f5b6191 100644
--- a/libinterp/corefcn/c-file-ptr-stream.cc
+++ b/libinterp/corefcn/c-file-ptr-stream.cc
@@ -127,12 +127,10 @@ seekdir_to_whence (std::ios::seekdir dir)
}
std::streampos
-c_file_ptr_buf::seekoff (std::streamoff /* offset */,
- std::ios::seekdir /* dir */,
+c_file_ptr_buf::seekoff (std::streamoff offset,
+ std::ios::seekdir dir,
std::ios::openmode)
{
- // FIXME
-#if 0
if (f)
{
fseek (f, offset, seekdir_to_whence (dir));
@@ -141,15 +139,11 @@ c_file_ptr_buf::seekoff (std::streamoff /* offset */,
}
else
return 0;
-#endif
- return -1;
}
std::streampos
-c_file_ptr_buf::seekpos (std::streampos /* offset */, std::ios::openmode)
+c_file_ptr_buf::seekpos (std::streampos offset, std::ios::openmode)
{
- // FIXME
-#if 0
if (f)
{
fseek (f, offset, SEEK_SET);
@@ -158,8 +152,6 @@ c_file_ptr_buf::seekpos (std::streampos /* offset */, std::ios::openmode)
}
else
return 0;
-#endif
- return -1;
}
int
diff --git a/libinterp/corefcn/oct-stream.cc b/libinterp/corefcn/oct-stream.cc
index 0f0d6dc..80c4e64 100644
--- a/libinterp/corefcn/oct-stream.cc
+++ b/libinterp/corefcn/oct-stream.cc
@@ -4522,3 +4522,10 @@ octave_stream_list::do_get_file_number (const octave_value& fid) const
return retval;
}
+
+
+/*
+// include this string to get the build system to put xtextscan in builtins.cc
+DEFUN (textscan, args, ,
+*/
+#include "textscan.cc"
diff --git a/libinterp/corefcn/textscan.cc b/libinterp/corefcn/textscan.cc
new file mode 100644
index 0000000..2629bd9
--- /dev/null
+++ b/libinterp/corefcn/textscan.cc
@@ -0,0 +1,2133 @@
+/*
+Copyright (C) 2015 Lachlan Andrew
+
+This file is part of Octave.
+
+Octave is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 3 of the License, or (at your
+option) any later version.
+
+Octave is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with Octave; see the file COPYING. If not, see
+.
+
+*/
+
+/* Implement the 'textscan' octave command */
+
+#ifdef HAVE_CONFIG_H
+#include
+#endif
+
+#include "Array.cc"
+
+#include "textscan.h"
+#include "oct-stream.h"
+#include "oct-obj.h"
+#include "utils.h"
+#include "defun.h"
+#include "ov-re-mat.h"
+
+DEFUN (textscan, args, ,
+ "-*- texinfo -*-\n\
+@deftypefn {Function File} {@var{C} =} textscan (@var{fid}, @var{format})\n\
+@deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{repeat})\n\
+@deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{param}, @var{value}, @dots{})\n\
+@deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{repeat}, @var{param}, @var{value}, @dots{})\n\
+@deftypefnx {Function File} {@var{C} =} textscan (@var{str}, @dots{})\n\
+@deftypefnx {Function File} {[@var{C}, @var{position}] =} textscan (@dots{})\n\
+Read data from a text file or string.\n\
+\n\
+The string @var{str} or file associated with @var{fid} is read from and\n\
+parsed according to @var{format}. The function is an extension\n\
+of @code{strread} and @code{textread}. Differences include: the ability to\n\
+read from either a file or a string, additional options, and additional\n\
+format specifiers.\n\
+\n\
+The input is interpreted as a sequence of \"words\", delimiters (such as\n\
+whitespace) and literals. The characters that form delimiters and whitespace\n\
+are determined by the options. The format consists of format specifiers\n\
+interspersed between literals. In the format, whitespace forms a delimiter\n\
+between consecutive literals, but is otherwise ignored.\n\
+\n\
+The output @var{C} is a cell array whose second dimension is determined\n\
+by the number of format specifiers.\n\
+\n\
+The first word of the input is matched to the first specifier of the\n\
+format and placed in the first column of the output; the second is matched\n\
+to the second specifier and placed in the second column and so forth.\n\
+If there are more words than specifiers, the process is repeated until all\n\
+words have been processed or the limit imposed by @var{repeat} has been met\n\
+(see below).\n\
+\n\
+The string @var{format} describes how the words in @var{str} should be\n\
+parsed. As in @var{fscanf},\n\
+any (non-whitespace) text in the format that is not one of these specifiers\n\
+is considered a literal; if there is a literal between two format specifiers\n\
+then that same literal must appear in the input stream between the matching\n\
+words.\n\
+\n\
+The following specifiers are valid:\n\
+\n\
+@table @code\n\
+@item %f\n\
+@itemx %f64\n\
+@itemx %n\n\
+The word is parsed as a number and converted to double.\n\
+\n\
+@item %f32\n\
+The word is parsed as a number and converted to single (float).\n\
+\n\
+@item %d\n\
+@itemx %d8\n\
+@itemx %d16\n\
+@itemx %d32\n\
+@itemx %d64\n\
+The word is parsed as a number and converted to int8, int16, int32 or int64.\n\
+If not size is specified, int32 is used.\n\
+\n\
+@item %u\n\
+@itemx %u8\n\
+@itemx %u16\n\
+@itemx %u32\n\
+@itemx %u64\n\
+The word is parsed as a number and converted to uint8, uint16, uint32 or\n\
+uint64. If not size is specified, uint32 is used.\n\
+\n\
+@item %s\n\
+The word is parsed as a string, ending at the last character before\n\
+whitespace, an end-of-line or a delimiter specified in the options.\n\
+\n\
+@item %q\n\
+The word is parsed as a \"quoted string\". If the first character of the\n\
+string is a double quote (\") then the string includes everything until a\n\
+matching double quote, including whitespace, delimiters and end of line\n\
+characters. If a pair of consecutive double quotes appears in the input,\n\
+it is replaced in the output by a single double quote. That is, the input\n\
+\"He said \"\"Hello\"\"\" would return the value 'He said \"Hello\"'.\n\
+\n\
+@item %c\n\
+The next character of the input is read. This includes delimiters,\n\
+whitespace and end of line characters.\n\
+\n\
+@item %[...]\n\
+@itemx %[^...]\n\
+In the first form, the word consists of the longest run consisting of only\n\
+characters between the brackets. Ranges of characters can be specified by\n\
+a hyphen; for example, %[0-9a-zA-Z] matches all alphanumeric characters\n\
+(if the underlying character set is ASCII). To include '-' in the set,\n\
+it should appear first or last in the brackets; to include ']', it should\n\
+be the first character. If the first character is '^' then the word\n\
+consists of characters NOT listed.\n\
+\n\
+For %s, %c %d, %f, %n, %u, an\n\
+optional width can be specified as %Ns, etc. where N is an integer > 1.\n\
+For %c, this causes exactly the next N characters to be read instead of\n\
+a single character. For the other specifiers, it is an upper bound on the\n\
+number of characters read; normal delimiters can cause fewer characters to\n\
+be read. For complex numbers, this limit applies to the real and imaginary\n\
+components individually\n\
+For %f and %n, format specifiers like %N.Mf are allowed, where M is an upper\n\
+bound on number of characters after the decimal point to be considered;\n\
+subsequent digits are skipped. For example, the specifier %8.2f would\n\
+read 12.345e6 as 1.234e7.\n\
+\n\
+@item %*...\n\
+The word specified by the remainder of the conversion specifier is skipped.\n\
+\n\
+@item literals\n\
+In addition the format may contain literal character strings; these will be\n\
+skipped during reading. If the input string does not match this literal,\n\
+the processing terminates.\n\
+@end table\n\
+\n\
+Parsed words corresponding to the first specifier are returned in the first\n\
+output argument and likewise for the rest of the specifiers.\n\
+\n\
+By default, @var{format} is @t{\"%f\"}, meaning that numbers are read from\n\
+@var{str}. This will do if @var{str} contains only numeric fields.\n\
+\n\
+For example, the string\n\
+\n\
+@example\n\
+@group\n\
+@var{str} = \"\\\n\
+Bunny Bugs 5.5\n\\\n\
+Duck Daffy -7.5e-5\n\\\n\
+Penguin Tux 6\"\n\
+@end group\n\
+@end example\n\
+\n\
+@noindent\n\
+can be read using\n\
+\n\
+@example\n\
+[@var{a}] = textscan (@var{str}, \"%s %s %f\");\n\
+@end example\n\
+\n\
+Optional numeric argument @var{repeat} can be used for limiting the\n\
+number of items read:\n\
+\n\
+@table @asis\n\
+@item -1\n\
+(default) read all of the string or file until the end.\n\
+\n\
+@item N\n\
+Read until the first of two conditions occurs: the format has been processed\n\
+N times, or N lines of the input have been processed. Zero (0) is an\n\
+acceptable value for @var{format_repeat}. Currently, end-of-line characters\n\
+inside %q, %c, and %[...]$ conversions do not contribute to the line count.\n\
+This is incompatible with Matlab and may change in future.\n\
+@end table\n\
+\n\
+The behavior of @code{textscan} can be changed via property-value pairs. The\n\
+following properties are recognized:\n\
+\n\
+@table @asis\n\
+@item @qcode{\"CollectOutput\"}\n\
+A value of 1 or true instructs textscan to concatenate consecutive columns\n\
+of the same class in the output cell array. A value of 0 or false (default)\n\
+leaves output in distinct columns.\n\
+\n\
+@item @qcode{\"CommentStyle\"}\n\
+Parts of @var{str} are considered comments and will be skipped.\n\
+@var{value} is the comment style and can be\n\
+either\n\
+@c any of the following.\n\
+@c \n\
+@c @itemize\n\
+@c @item @qcode{\"shell\"}\n\
+@c Everything from @code{#} characters to the nearest end-of-line is skipped.\n\
+@c \n\
+@c @item @qcode{\"c\"}\n\
+@c Everything between @code{/*} and @code{*/} is skipped.\n\
+@c \n\
+@c @item @qcode{\"c++\"}\n\
+@c Everything from @code{//} characters to the nearest end-of-line is\n\
+@c skipped.\n\
+@c \n\
+@c @item @qcode{\"matlab\"}\n\
+@c Everything from @code{%} characters to the nearest end-of-line is skipped.\n\
+@c \n\
+@c @item user-supplied. Two options:\n\
+(1) One string, or 1x1 cell string, to skip everything to the right of it;\n\
+(2) A cell array of two strings, to skip everything between the first and\n\
+second strings.\n\
+@c @end itemize\n\
+Comments are only parsed where whitespace is accepted, and do not act as\n\
+delimiters.\n\
+\n\
+@item @qcode{\"Delimiter\"}\n\
+If @var{value} is a string, any character in @var{value} will be used to\n\
+split @var{str} into words. If @var{value} is a cell array of strings,\n\
+any string in the array will be used to split @var{str} into words.\n\
+(default value = any whitespace).\n\
+\n\
+@item @qcode{\"EmptyValue\"}\n\
+Value to return for empty numeric values in non-whitespace delimited data.\n\
+The default is NaN@. When the data type does not support NaN (int32 for\n\
+example), then default is zero.\n\
+\n\
+@item @qcode{\"EndOfLine\"}\n\
+@var{value} can be either a emtpy or one character specifying the end of line\n\
+character, or the pair\n\
+@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} (CRLF).\n\
+In the latter case, any of\n\
+@qcode{\"@xbackslashchar{}r\"}, @qcode{\"@xbackslashchar{}n\"} or\n\
+@qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is counted as a (single)\n\
+newline.\n\
+If no value is given, @qcode{\"@xbackslashchar{}r@xbackslashchar{}n\"} is\n\
+used.\n\
+@c If set to \"\" (empty string) EOLs are ignored as delimiters and added\n\
+@c to whitespace.\n\
+\n\
+@c When reading from a character string, optional input argument @var{n}\n\
+@c specifies the number of times @var{format} should be used (i.e., to limit\n\
+@c the amount of data read).\n\
+@c When reading from file, @var{n} specifies the number of data lines to read;\n\
+@c in this sense it differs slightly from the format repeat count in strread.\n\
+\n\
+@item @qcode{\"HeaderLines\"}\n\
+The first @var{value} number of lines of @var{fid} are skipped.\n\
+\n\
+@item @qcode{\"MultipleDelimsAsOne\"}\n\
+If @var{value} is non-zero,\n\
+treat a series of consecutive delimiters, without whitespace in between,\n\
+as a single delimiter. Consecutive delimiter series need not be vertically\n\
+@qcode{\"aligned\"}.\n\
+Without this option, a single delimiter before the end of the line does\n\
+not cause the line to be considered to start with an empty value, but a\n\
+single delimiter at the start of a line causes the line to be considered\n\
+to start with an empty value.\n\
+\n\
+@item @qcode{\"TreatAsEmpty\"}\n\
+Treat single occurrences (surrounded by delimiters or whitespace) of the\n\
+string(s) in @var{value} as missing values.\n\
+\n\
+@item @qcode{\"ReturnOnError\"}\n\
+If set to numerical 1 or true (default), return normally when read errors\n\
+have been encountered. If set to 0 or false, return an error and no data.\n\
+As the string or file is read by columns rather than by rows, and because\n\
+textscan is fairly forgiving as regards read errors, setting this option\n\
+may have little or no actual effect.\n\
+\n\
+@item @qcode{\"Whitespace\"}\n\
+Any character in @var{value} will be interpreted as whitespace and trimmed;\n\
+The default value for whitespace is\n\
+@c Note: the next line specifically has a newline which generates a space\n\
+@c in the output of qcode, but keeps the next line < 80 characters.\n\
+@qcode{\"\n\
+@xbackslashchar{}b@xbackslashchar{}r@xbackslashchar{}n@xbackslashchar{}t\"}\n\
+(note the space). Unless whitespace is set to @qcode{\"\"} (empty) AND at\n\
+least one @qcode{\"%s\"} format conversion specifier is supplied, a space is\n\
+always part of whitespace.\n\
+\n\
+@end table\n\
+\n\
+When the number of words in @var{str} or @var{fid} doesn't match an exact\n\
+multiple of\n\
+the number of format conversion specifiers,\n\
+textscan fills the last line with empty strings, NaNs or 0s (for integer\n\
+fields). In the case that the last line does not end with an end-of-line\n\
+character, this is incompatible with Matlab and subject to change.\n\
+@c textscan's behavior depends on\n\
+@c whether the last line of the string or file is an end-of-line as specified\n\
+@c by the EndOfLine option:\n\
+@c \n\
+@c @table @asis\n\
+@c @item last character = end-of-line\n\
+@c Data columns are padded with empty fields, NaN or 0 so that all columns have\n\
+@c equal length\n\
+@c \n\
+@c @item last character is not end-of-line\n\
+@c Data columns are not padded; textscan returns columns of unequal length\n\
+@c @end table\n\
+\n\
+If the format string is explicitly empty (not omitted) and the file contains\n\
+only\n\
+numeric data (excluding headerlines), textscan will return data in a number\n\
+of columns matching the number of fields on the first data line of\n\
+the input.\n\
+\n\
+The second output, @var{position}, provides the position, in characters\n\
+from the beginning of the file or string, at which the processing stopped.\n\
+\n\
+@seealso{dlmread, fscanf, load, strread, textread}\n\
+@end deftypefn")
+{
+ octave_value_list retval;
+ std::string format;
+ int params = 1;
+
+ if (args.length () < 1)
+ {
+ print_usage ();
+ error_state = true;
+ }
+ else if (args.length () == 1)
+ format = "%f"; // ommited format = %f. explicit "" = width from file
+ else if (args(1).is_string ())
+ {
+ format = args(1).string_value ();
+ params++;
+ }
+ else
+ error ("text:scan: FORMAT must be a string, not <%s>",
+ args(1).class_name ().c_str ());
+
+ if (! error_state)
+ {
+ octave_idx_type ntimes = -1;
+ textscan tscanner;
+
+ if (args.length () >= 3)
+ {
+ if (args(2).is_numeric_type ())
+ {
+ ntimes = args(2).idx_type_value ();
+ params = 3;
+ }
+ }
+ textscan_format_list fmt_list (format);
+
+ tscanner.parse_options (args, params, fmt_list);
+
+ fmt_list.consider_options (tscanner);
+
+ if (! error_state)
+ {
+ if (args(0).is_string ())
+ {
+ std::istringstream is (args(0).string_value ());
+ octave_value tmp = tscanner.scan (&is, fmt_list, ntimes);
+ if (! error_state)
+ {
+ is.clear ();
+ retval(1) = octave_value (static_cast(is.tellg ()));
+ retval(0) = tmp;
+ }
+ }
+ else
+ {
+ octave_stream os = octave_stream_list::lookup (args(0),
+ "textscan");
+ octave_value tmp = tscanner.scan (os.input_stream (),
+ fmt_list, ntimes);
+
+ if (! error_state)
+ {
+ // FIXME -- warn if stream is not opened in binary mode?
+ os.input_stream ()->clear ();
+ retval(1) = os.tell ();
+ retval(0) = tmp;
+ }
+ }
+ }
+ }
+
+ return retval;
+}
+
+
+/********************/
+/* Read a double considering the "precision" field of fmt and the */
+/* exp_chars option of options. */
+double
+textscan::formatted_double (std::istream& is, const textscan_format_elt& fmt)
+ const
+{
+ int ch = is.peek ();
+ int sign = 1;
+ unsigned int width_left = fmt.width;
+ double retval = 0;
+ bool valid = false; // syntactically correct double?
+
+ if (ch == '+')
+ {
+ is.get ();
+ ch = is.peek ();
+ }
+ else if (ch == '-')
+ {
+ sign = -1;
+ is.get ();
+ ch = is.peek ();
+ }
+
+ // Read integer part
+ if (ch != '.')
+ {
+ if (ch >= '0' && ch <= '9') // valid if at least one digit
+ valid = true;
+ while (is && (ch = is.get ()) >= '0' && ch <= '9' && width_left--)
+ retval = retval * 10 + (ch - '0');
+ }
+
+ // Read fractional part, up to specified precision
+ if (ch == '.' && width_left)
+ {
+ double multiplier = 1;
+ int precision = fmt.prec;
+ int i;
+
+ if (precision == 0)
+ precision = 1<<30; // FIXME Should be MAXINT
+ if (!valid) // if there was nothing before '.'...
+ {
+ is.get (); // ...ch was a "peek", not "get".
+ width_left--;
+ }
+ for (i = 0; i < precision; i++)
+ if (is && (ch = is.get ()) >= '0' && ch <= '9' && width_left--)
+ retval += (ch - '0') * (multiplier *= 0.1);
+ else
+ break;
+ if (i > 0)
+ valid = true; // valid if at least one digit after '.'
+ while (is && (ch = is.get ()) >= '0' && ch <= '9' && width_left--)
+ ; /* discard */
+ is.clear ();
+ }
+
+ // look for exponent part in 6.023E+23
+ const char *ec = exp_chars.c_str ();
+ bool used_exp = false;
+ if (valid && width_left && strchr (ec, ch))
+ {
+ int ch1 = is.peek ();
+ if (ch1 == '-' || ch1 == '+' || (ch1 >= '0' && ch1 <= '9'))
+ {
+ // if 1.0e+$ or some such, this will set failbit, as we want
+ //double exp = octave_read_value (is);
+ int exp = 0;
+ int exp_sign = 1;
+ if (ch1 == '+')
+ is.get ();
+ else if (ch1 == '-')
+ {
+ exp_sign = -1;
+ is.get ();
+ }
+ valid = false;
+ while (is && (ch = is.get ()) >= '0' && ch1 <= '9' && width_left--)
+ {
+ exp = exp*10 + ch - '0';
+ valid = true;
+ }
+ is.putback (ch);
+ retval *= pow10f (exp * exp_sign);
+ used_exp = true;
+ }
+ }
+ is.clear ();
+ if (!used_exp && ch != EOF)
+ is.putback (ch);
+
+ if (!valid)
+ is.setstate (std::fstream::failbit);
+ else
+ is.setstate (is.rdstate () & ~std::fstream::failbit);
+
+ return retval * sign;
+}
+
+/********************/
+std::istream&
+textscan::scan_complex (std::istream& is, const textscan_format_elt& fmt,
+ Complex& val) const
+{
+ double im = 0;
+ double re = 0;
+ bool as_empty = false; // did we fail but match a "treat_as_empty" string?
+
+ int ch = is.peek ();
+
+ if (ch == '+' || ch == '-') // check for [+-][ij] with no coefficients
+ {
+ ch = is.get ();
+ int ch2 = is.peek ();
+ if (ch2 == 'i' || ch2 == 'j')
+ {
+ is.get ();
+ im = (ch == '+') ? 1 : -1;
+ }
+ else
+ is.putback (ch);
+ }
+
+ if (!im) // if not [+-][ij], read real normally
+ {
+ std::streampos pos = is.tellg ();
+ std::ios::iostate state = is.rdstate ();
+ //re = octave_read_value (is);
+ re = read_double (is, fmt);
+
+ // check for "treat as empty" string
+ if (treat_as_empty.numel ()
+ && (is.fail () || octave_is_NaN_or_NA (Complex (re))
+ || re == octave_Inf))
+ {
+ for (int i = 0; treat_as_empty.numel (); i++)
+ {
+ if (ch == treat_as_empty (i).string_value ()[0])
+ {
+ as_empty = true; // first char matches, so read the lot
+ break;
+ }
+ }
+ if (as_empty) // if first char matched...
+ {
+ as_empty = false; // ...look for the whole string
+
+ is.clear (state); // treat_as_empty "-" causes partial read
+ is.seekg (pos); // reset to position before failed read, since
+
+ // treat_as_empty strings may be different sizes.
+ // Read ahead longest, put it all back, then re-read the string
+ // that matches.
+ char look [treat_as_empty_len + 1];
+ // prefill, in case EOF means part-filled.
+ memset (look, '\0', treat_as_empty_len);
+ is.read (look, treat_as_empty_len);
+
+ is.clear (state);
+ is.seekg (pos); // reset to position before look-ahead
+
+ for (int i = 0; i < treat_as_empty.numel (); i++)
+ {
+ std::string s = treat_as_empty (i).string_value ();
+ if (!strncmp (s.c_str (), look, s.size ()))
+ {
+ as_empty = true;
+ is.read (look, s.size ()); // read just the right amount
+ // FIXME Could this be combined with the above seek?
+ break;
+ }
+ }
+ }
+ }
+
+ if (!is.eof () && !as_empty)
+ {
+ state = is.rdstate (); // before tellg, since that fails at EOF
+ pos = is.tellg ();
+ ch = is.peek (); // ch == EOF if read failed; no need to chk fail
+ if (ch == 'i' || ch == 'j') // pure imaginary
+ {
+ is.get ();
+ im = re;
+ re = 0;
+ }
+ else if (ch == '+' || ch == '-') // see if it is real+imag[ij]
+ {
+ // save stream state in case we have to restore it
+ pos = is.tellg ();
+ state = is.rdstate ();
+
+ //im = octave_read_value (is);
+ im = read_double (is, fmt);
+ if (is.fail ())
+ im = 1;
+
+ if (is.peek () == 'i' || is.peek () == 'j')
+ is.get ();
+ else
+ {
+ im = 0; // no valid imaginary part. Restore state
+ is.clear (state);
+ is.seekg (pos);
+ }
+ }
+ else if (ch == EOF) // we've read enough to be a "valid" read
+ is.clear (state); // failed peek shouldn't cause fail
+ }
+ }
+ if (as_empty)
+ val = empty_value.scalar_value ();
+ else
+ val = Complex (re, im);
+
+ return is;
+}
+
+/********************/
+// Return in val the run of characters from is NOT contained in pattern.
+int
+textscan::scan_caret (std::istream& is, const char *pattern, std::string& val)
+ const
+{
+ int c1 = EOF;
+ std::ostringstream obuf; // Is this optimised for growing?
+
+ while (is && (c1 = (is && !is.eof ()) ? is.get () : EOF) != EOF
+ && !strchr (pattern, c1))
+ obuf << static_cast (c1);
+
+ val = obuf.str ();
+
+ // can only push back if not failed.
+ is.clear (is.rdstate () & ~std::fstream::failbit);
+ if (c1 != EOF)
+ is.putback (c1);
+
+ return c1;
+}
+
+/********************/
+// Read until one of the strings in delimiters is found.
+// For efficiency, ends is a list of the last character of each delimiter
+std::string
+textscan::read_until (std::istream& is, const Cell& delimiters,
+ const std::string& ends) const
+{
+ std::string retval ("");
+ bool done = false;
+ do
+ { // find sequence ending with an ending char
+ std::string next;
+ scan_caret (is, ends.c_str (), next);
+ retval = retval + next;
+
+ int last = (!is.eof ()) ? is.get () : EOF;
+ if (last != EOF)
+ {
+ retval = retval + static_cast (last);
+ for (int i = 0; i < delimiters.numel (); i++)
+ {
+ std::string delim = delimiters(i).string_value ();
+ int start = retval.length () - delim.length ();
+ if (start < 0)
+ start = 0;
+ std::string may_match = retval.substr (start);
+ if (may_match == delim)
+ {
+ done = true;
+ retval = retval.substr (0, start);
+ break;
+ }
+ }
+ }
+ }
+ while (!done && is && !is.eof ());
+
+ return retval;
+}
+
+
+/********************/
+// Read stream until either fmt.width chars have been read, or options.delimiter
+// has been found
+// Does *not* rely on fmt being 's' -- used by formats like %6f to limit to 6
+std::istream&
+textscan::scan_string (std::istream& is, const textscan_format_elt& fmt, std::string& val) const
+{
+ if (delim_list.length () == 0)
+ {
+ unsigned int i = 0;
+ unsigned int width = fmt.width;
+
+ for (i = 0; i < width; i++)
+ {
+ if (i+1 > val.length ())
+ val = val + val + ' '; // grow even if empty
+ int ch = is.get ();
+ if (is_delim (ch) || ch == EOF)
+ {
+ is.putback (ch);
+ break;
+ }
+ else
+ val[i] = ch;
+ }
+ val = val.substr (0, i); // trim pre-allocation
+ }
+ else // Cell array of multi-character delimiters
+ {
+ //std::cerr << "textscan: multi-character delimiters not yet implemented\n";
+ std::string ends ("");
+ for (int i = 0; i < delim_list.length (); i++)
+ {
+ std::string tmp = delim_list(i).string_value ();
+ ends += tmp.substr (tmp.length () - 1);
+ }
+ val = textscan::read_until (is, delim_list, ends);
+ }
+
+ return is;
+}
+
+/********************/
+// Return in val the run of characters from is contained in pattern.
+int
+textscan::scan_bracket (std::istream& is, const char *pattern, std::string& val)
+ const
+{
+ int c1 = EOF;
+ std::ostringstream obuf; // Is this optimised for growing?
+
+ while (is && strchr (pattern, (c1 = is.get ())))
+ obuf << static_cast (c1);
+
+ val = obuf.str ();
+ if (c1 != EOF)
+ is.putback (c1);
+ return c1;
+}
+
+/********************/
+// Return in val a string, either delimited by whitespace/delimiters, or
+// enclosed in a pair of double quotes ("..."). Enclosing quotes are removed.
+// A consecutive pair "" is inserted into val as a single ".
+std::istream&
+textscan::scan_qstring (std::istream& is, const textscan_format_elt& fmt,
+ std::string& val)
+{
+ skip_whitespace (is);
+
+ if (is.peek () != '\"')
+ scan_string (is, fmt, val);
+ else
+ {
+ is.get ();
+ scan_caret (is, "\"", val); // read everything until "
+ is.get (); // swallow "
+
+ while (is && is.peek () == '\"') // if double ", insert one in stream,
+ { // and keep looking for single "
+ is.get ();
+ std::string val1;
+ scan_caret (is, "\"", val1);
+ val = val + "\"" + val1;
+ is.get ();
+ }
+ }
+ return is;
+}
+
+/********************/
+// Read from is into val a string of the next fmt.width characters,
+// including any whitespace or delimiters.
+std::istream&
+textscan::scan_cstring (std::istream& is, const textscan_format_elt& fmt,
+ std::string& val) const
+{
+ int ch;
+ val.resize (fmt.width);
+ for (unsigned int i = 0; is && i < fmt.width; i++)
+ if ((ch = is.get ()) != EOF)
+ val[i] = ch;
+ else
+ {
+ val.resize (i);
+ break;
+ }
+
+ return is;
+}
+
+
+/********************/
+std::istream&
+textscan::scan_one (std::istream& is, const textscan_format_elt& fmt,
+ octave_value& ov, Array row)
+{
+ skip_whitespace (is);
+
+ is.clear ();
+ int pos = is.tellg ();
+
+ octave_value val;
+ if (fmt.numeric)
+ {
+ if (fmt.type == 'f' || fmt.type == 'n')
+ {
+ Complex v;
+ scan_complex (is, fmt, v);
+
+ if (!fmt.discard)
+ {
+ if (fmt.bitwidth == 64)
+ {
+ if (ov.is_real_type () && v.imag () == 0)
+ ov.internal_rep ()->fast_elem_insert (row(0), v.real ());
+ else
+ {
+ if (ov.is_real_type ()) // cat does type conversion
+ ov = do_cat_op (ov, octave_value (v), row);
+ else
+ ov.internal_rep ()->fast_elem_insert (row(0), v);
+ }
+ }
+ else
+ {
+ if (ov.is_real_type () && v.imag () == 0)
+ ov.internal_rep ()->fast_elem_insert (row(0),
+ float (v.real ()));
+ else
+ {
+ if (ov.is_real_type ()) // cat does type conversion
+ ov = do_cat_op (ov, octave_value (v), row);
+ else
+ ov.internal_rep ()->fast_elem_insert (row(0),
+ FloatComplex (v));
+ }
+ }
+ }
+ }
+ else
+ {
+ double v; // Matlab docs say 1e30 etc should be valid for %d and
+ // 1000 as a %d8 should be 127, so read as double.
+ // Some loss of precision for d64 and u64.
+ v = read_double (is, fmt);
+ if (!fmt.discard)
+ switch (fmt.bitwidth)
+ {
+ case 64:
+ switch (fmt.type)
+ {
+ case 'd':
+ ov.internal_rep ()->fast_elem_insert (row(0),
+ octave_int64 (v));
+ break;
+ case 'u':
+ ov.internal_rep ()->fast_elem_insert (row(0),
+ octave_uint64 (v));
+ break;
+ }
+ break;
+ case 32:
+ switch (fmt.type)
+ {
+ case 'd':
+ {
+ octave_int32 vv = v;
+ if (vv < v)
+ vv = (1L<<31)-1;
+ else if (vv > v)
+ vv = (1<<31);
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ case 'u':
+ {
+ octave_uint32 vv = v;
+ if (vv < v)
+ vv = -1;
+ else if (vv > v)
+ vv = 0;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ }
+ break;
+ case 16:
+ if (fmt.type == 'd')
+ {
+ octave_int16 vv = v;
+ if (vv < v)
+ vv = (1<<15)-1;
+ else if (vv > v)
+ vv = (1<<15);
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ else
+ {
+ octave_uint16 vv = v;
+ if (vv < v)
+ vv = -1;
+ else if (vv > v)
+ vv = 0;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ case 8:
+ if (fmt.type == 'd')
+ {
+ octave_int8 vv = v;
+ if (vv < v)
+ vv = (1<<7)-1;
+ else if (vv > v)
+ vv = (1<<7);
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ else
+ {
+ octave_uint8 vv = v;
+ if (vv < v)
+ vv = -1;
+ else if (vv > v)
+ vv = 0;
+ ov.internal_rep ()->fast_elem_insert (row(0), vv);
+ }
+ break;
+ }
+ }
+ }
+ else
+ {
+ std::string vv (" "); // initial buffer. Grows as needed
+ switch (fmt.type)
+ {
+ case 's':
+ scan_string (is, fmt, vv);
+ break;
+ case 'q':
+ scan_qstring (is, fmt, vv);
+ break;
+ case 'c':
+ scan_cstring (is, fmt, vv);
+ break;
+ case '[':
+ scan_bracket (is, fmt.char_class.c_str (), vv);
+ break;
+ case '^':
+ scan_caret (is, fmt.char_class.c_str (), vv);
+ break;
+ }
+ if (!fmt.discard)
+ ov.internal_rep ()->fast_elem_insert (row (0),
+ Cell (octave_value (vv)));
+ // fixme: why does failbit get set at EOF, instead of eofbit??
+ if (vv.length () != 0)
+ is.clear (is.rdstate () & ~std::ifstream::failbit);
+ }
+
+ if (is.fail () && fmt.numeric)
+ {
+ if (!fmt.discard)
+ ov = do_cat_op (ov, empty_value, row);
+ is.clear (); // clear so we can check tellg; failbit set again below
+ if (is.tellg () == pos)
+ {
+ std::string dummy; // FIXME: should fail instead of skipping
+ scan_string (is, fmt, dummy);
+ }
+ is.setstate (std::ifstream::failbit);
+ }
+
+ return is;
+}
+
+/********************/
+int
+textscan::read_format_once (std::istream* isp,
+ textscan_format_list& fmt_list,
+ std::list & retval,
+ Array row)
+{
+ const textscan_format_elt *elem = fmt_list.first ();
+ std::list::iterator out=retval.begin ();
+ bool no_conversions = true;
+ bool done = false;
+ int i;
+
+ for (i = 0; i < fmt_list.length (); i++)
+ {
+ isp->clear (); // clear fail of previous numeric conversions
+ switch (elem->type)
+ {
+ case 'C':
+ case 'D':
+ std::cerr << "textscan: Conversion %" << elem->type << " not yet implemented\n";
+ break;
+
+ case 'u':
+ case 'd':
+ case 'f':
+ case 'n':
+ case 's':
+ case '[':
+ case '^':
+ case 'q':
+ case 'c':
+ scan_one (*isp, *elem, *out, row);
+ break;
+
+ case textscan_format_elt::literal_conversion :
+ match_literal (*isp, *elem);
+ break;
+ }
+
+ if (!isp->fail () && !elem->discard)
+ no_conversions = false;
+ if (isp->eof ())
+ done = true; // note EOF, but process others to get empty_val
+ else
+ isp->clear (isp->rdstate () & ~std::ios::failbit);
+ if (!elem->discard)
+ out++;
+
+ elem = fmt_list.next ();
+ // fixme: these conversions "ignore delimiters". Should they include
+ // delimiters at the start of the conversion, or can those be skipped?
+ if (// elem->type != '[' && elem->type != '^' && elem->type != 'c'
+ /*&&*/ elem->type != textscan_format_elt::literal_conversion)
+ skip_delim (*isp);
+
+ if (error_state)
+ break;
+ }
+ // err = -2 means "error, and no columns read this row
+ if (error_state || done || isp->eof ())
+ return (-1 - no_conversions);
+
+ return 1;
+}
+
+/********************/
+void
+textscan::parse_options (const octave_value_list& args, int first_param,
+ textscan_format_list& fmt_list)
+{
+ int last = args.length ();
+ int n = last - first_param;
+
+ if (n & 1)
+ error ("textscan: %d parameters given, but only %d values", n-n/2, n/2);
+ else
+ {
+ std::string delims;
+ bool have_delims = false;
+ for (int i = first_param; i < last; i += 2)
+ {
+ if (!args(i).is_string ())
+ {
+ error ("textscan: Invalid paramter type <%s> for parameter %d",
+ args(i).type_name ().c_str (), (i-first_param)/2 + 1);
+ }
+ else
+ {
+ std::string param = args(i).string_value ();
+ std::transform (param.begin (), param.end (),
+ param.begin (), ::tolower);
+ if (param == "delimiter")
+ {
+ bool invalid = true;
+ if (args(i+1).is_string ())
+ {
+ invalid = false;
+ have_delims = true;
+ delims = args(i+1).string_value ();
+ }
+ else if (args(i+1).is_cell ())
+ {
+ invalid = false;
+ delim_list = args(i+1).cell_value ();
+ delim_table = " "; // non-empty, to flag non-default delim
+
+ // Check that all elements are strings
+ for (int j = 0; j < delim_list.numel (); j++)
+ if (!delim_list (j).is_string ())
+ invalid = true;
+ }
+ if (invalid)
+ error ("textscan: Delimimters must be either a string"
+ " or cell array of strings");
+ }
+ else if (param == "commentstyle")
+ {
+ if (args(i+1).is_string ())
+ { // check here for names like "C++", "C", "shell", ...?
+ comment_style = Cell (args(i+1));
+ }
+ else if (args(i+1).is_cell ())
+ {
+ comment_style = args(i+1).cell_value ();
+ int len = comment_style.numel ();
+ if ((len >= 1 && !comment_style (0).is_string ())
+ || (len >= 2 && !comment_style (1).is_string ())
+ || (len >= 3))
+ error ("textscan: CommentStyle must be either a string"
+ " or cell array of one or two strings");
+ }
+ else
+ error ("textscan: CommentStyle must be either a string"
+ " or cell array of one or two strings, not <%s>",
+ args(i+1).class_name ().c_str ());
+
+ // How far ahead do we need to look to detect an open comment
+ // and which character do we look for?
+ if (comment_style.numel () >= 1)
+ {
+ comment_len = comment_style (0).string_value ().size ();
+ comment_char = comment_style (0).string_value ()[0];
+ }
+ }
+ else if (param == "treatasempty")
+ {
+ bool invalid = false;
+ if (args(i+1).is_string ())
+ {
+ treat_as_empty = Cell (args(i+1));
+ treat_as_empty_len = args(i+1).string_value ().size ();
+ }
+ else if (args(i+1).is_cell ())
+ {
+ treat_as_empty = args(i+1).cell_value ();
+ for (int j = 0; j < treat_as_empty.numel (); j++)
+ if (!treat_as_empty (j).is_string ())
+ invalid = true;
+ else
+ {
+ int k = treat_as_empty (j).string_value ().size ();
+ if (k > treat_as_empty_len)
+ treat_as_empty_len = k;
+ }
+ }
+ if (invalid)
+ error ("textscan: TreatAsEmpty must be either a string"
+ " or cell array of one or two strings");
+ else
+ {
+ // FIXME Ensure none is a prefix of a later one. sort len?
+ }
+ }
+ else if (param == "collectoutput")
+ {
+ if (args(i+1).is_numeric_type ())
+ collect_output = args(i+1).bool_value ();
+ else
+ error ("textscan: CollectOutput must be logical or numeric");
+ }
+ else if (param == "emptyvalue")
+ {
+ if (args(i+1).is_numeric_type ())
+ empty_value = args(i+1).scalar_value ();
+ else
+ error ("textscan: EmptyValue must be numeric, not <%s>",
+ args(i+1).class_name ().c_str ());
+ }
+ else if (param == "headerlines")
+ {
+ if (args(i+1).is_numeric_type ())
+ header_lines = args(i+1).scalar_value ();
+ else
+ error ("textscan: HeaderLines must be numeric");
+ }
+ else if (param == "multipledelimsasone")
+ {
+ if (args(i+1).is_numeric_type ())
+ {
+ if (args(i+1).bool_value ())
+ multiple_delims_as_one = true;
+ }
+ else
+ error ("textscan: MultipleDimsAsOne must be logical or numeric");
+ }
+ else if (param == "returnonerror")
+ {
+ if (args(i+1).is_numeric_type ())
+ {
+ if (!args(i+1).bool_value ())
+ return_on_error = false;
+ }
+ else
+ error ("textscan: ReturnOnError must be logical or numeric");
+ }
+ else if (param == "whitespace")
+ {
+ if (args(i+1).is_string ())
+ whitespace = args(i+1).string_value ();
+ else
+ error ("textscan: Whitespace must be a character string");
+ }
+ else if (param == "expchars")
+ {
+ if (args(i+1).is_string ())
+ {
+ exp_chars = args(i+1).string_value ();
+ default_exp = false;
+ }
+ else
+ error ("textscan: ExpChars must be a character string");
+ }
+ else if (param == "endofline")
+ {
+ bool valid = true;
+ if (args(i+1).is_string ())
+ {
+ std::string s = args(i+1).string_value ();
+ if (args(i+1).is_sq_string ())
+ s = do_string_escapes (s);
+ int l = s.length ();
+ if (l == 0)
+ eol1 = eol2 = -2;
+ else if (l == 1)
+ eol1 = eol2 = s.c_str ()[0];
+ else if (l == 2)
+ {
+ eol1 = s.c_str ()[0];
+ eol2 = s.c_str ()[1];
+ if (eol1 != '\r' || eol2 != '\n') // Why limit it?
+ valid = false;
+ }
+ else
+ valid = false;
+ }
+ else
+ valid = false;
+ if (!valid)
+ error ("textscan: EndOfLine must be at most one character or '\\r\\n'");
+ }
+ else
+ {
+ error ("textscan: Unrecognised option '%s'", param.c_str ());
+ }
+ }
+ }
+
+ // Create look-up table of delimiters, based on 'delimiter'
+ if (have_delims)
+ {
+ delim_table = std::string (256, '\0');
+ for (unsigned int i = 0; i < delims.length (); i++)
+ delim_table[delims[i]] = '1';
+ delim_table[eol1] = '1'; // EOL is always a delimiter
+ delim_table[eol2] = '1'; // EOL is always a delimiter
+ }
+
+ whitespace_table = std::string (256, '\0');
+ for (unsigned int i = 0; i < whitespace.length (); i++)
+ whitespace_table[whitespace[i]] = '1';
+
+ // For Matlab compatibility, add 0x20 to whitespace, unless
+ // whitespace is explicitly ignored.
+ if (! (whitespace.length () == 0 && fmt_list.has_string))
+ whitespace_table[' '] = '1';
+
+// // Add CR and LF to whitespace if they are not end-of-line.
+// if (!strchr (end_of_line.c_str (), '\r'))
+// whitespace_table['\r'] = '1';
+// if (!strchr (end_of_line.c_str (), '\n'))
+// whitespace_table['\n'] = '1';
+ }
+}
+
+/********************/
+// skip comments, and characters specified by the "Whitespace" option
+// If EOLstop==true, doesn't skip end of line
+int
+textscan::skip_whitespace (std::istream& is, bool EOLstop)
+{
+ int c1 = EOF;
+ bool found_comment = false;
+
+ do
+ {
+ found_comment = false;
+ int prev = -1;
+ while (is && (c1 = is.get ()) != EOF
+ && ( ( (c1 == eol1 || c1 == eol2) && ++lines && !EOLstop)
+ || isspace (c1)))
+ {
+ if (prev == eol1 && eol1 != eol2 && c1 == eol2)
+ lines--;
+ prev = c1;
+ }
+
+ if (c1 == comment_char) // see if we match an open comment
+ {
+ // save stream state in case we have to restore it
+ std::streampos pos = is.tellg ();
+ std::ios::iostate state = is.rdstate ();
+
+ char look [comment_len];
+ is.read (look, comment_len-1); // already read first char
+ look[comment_len-1] = '\0';
+ if (is && comment_style(0).string_value ().substr (1) == look)
+ {
+ found_comment = true;
+
+ std::string dummy;
+ char eol [3] = {static_cast (eol1),
+ static_cast (eol2),
+ '\0'};
+ if (comment_style.numel () == 1) // skip to end of line
+ {
+ scan_caret (is, eol, dummy);
+ c1 = is.get ();
+ if (c1 == eol1 && eol1 != eol2 && is.peek () == eol2)
+ is.get ();
+ lines++;
+ }
+ else // matching pair
+ {
+ std::string end_c = comment_style(1).string_value ();
+ // last char of end-comment sequence
+ char last = *(end_c.substr (end_c.length ()-1).c_str ());
+ std::string may_match ("");
+ do
+ { // find sequence ending with last char
+ scan_caret (is, &last, dummy);
+ is.get (); // (read last itself)
+
+ may_match = may_match + dummy + last;
+ int start = may_match.length () - end_c.length ();
+ if (start < 0)
+ start = 0;
+ may_match = may_match.substr (start);
+ }
+ while (may_match != end_c && is && !is.eof ());
+ }
+ }
+ else // wasn't really a comment; restore state
+ {
+ is.clear (state);
+ is.seekg (pos);
+ }
+ }
+ }
+ while (found_comment);
+
+ if (c1 != EOF)
+ is.putback (c1);
+ return c1;
+}
+
+/********************/
+// See if the next few characters match one of the strings in target.
+// For efficiency, max_len is the cached longest length of any target.
+// Returns -1 if none is found, or the inde of the match.
+int
+textscan::lookahead (std::istream& is, const Cell& targets, int max_len) const
+{
+ // target strings may be different sizes.
+ // Read ahead longest, put it all back, then re-read the string
+ // that matches.
+
+ std::streampos pos = is.tellg ();
+ std::ios::iostate state = is.rdstate ();
+
+ char look [max_len + 1];
+
+ memset (look, '\0', max_len); // prefill, in case EOF means part-filled.
+ is.read (look, max_len);
+
+ is.clear (state);
+ is.seekg (pos); // reset to position before look-ahead
+
+ int i;
+ for (i = 0; i < targets.numel (); i++)
+ {
+ std::string s = targets (i).string_value ();
+ if (!strncmp (s.c_str (), look, s.size ()))
+ {
+ is.read (look, s.size ()); // read just the right amount
+ break;
+ }
+ }
+
+ if (i == targets.numel ())
+ i = -1;
+
+ return i;
+}
+
+/********************/
+// Skip delimiters -- multiple if MultipleDelimsAsOne specified.
+// FIXME For now, just handles single character delimiters
+int
+textscan::skip_delim (std::istream& is)
+{
+ int c1 = skip_whitespace (is, true); // 'true': stop once EOL is read
+ if (delim_list.length () == 0)
+ {
+ if (is_delim (c1) || c1 == eol1 || c1 == eol2)
+ {
+ is.get ();
+ if (c1 == eol1 && is.peek () == eol2)
+ is.get (); // if \r\n, skip the \n too.
+
+ if (multiple_delims_as_one)
+ {
+ int prev = -1;
+ while (is && (c1 = is.get ()) != EOF
+ && (((c1 == eol1 || c1 == eol2) && ++lines)
+ || isspace (c1) || is_delim (c1)))
+ {
+ if (prev == eol1 && eol1 != eol2 && c1 == eol2)
+ lines--;
+ prev = c1;
+ }
+ if (c1 != EOF)
+ is.putback (c1);
+ }
+ }
+ }
+ else
+ {
+ int first_match;
+ int max_len = 0;
+ for (int i = 0; i < delim_list.numel (); i++)
+ {
+ int len = delim_list(i).length ();
+ if (len > max_len)
+ max_len = len;
+ }
+
+ if (c1 == eol1 || c1 == eol2
+ || (-1 != (first_match = lookahead (is, delim_list, max_len))))
+ {
+ if (c1 == eol1)
+ {
+ is.get ();
+ if (is.peek () == eol2)
+ is.get ();
+ }
+ else if (c1 == eol2)
+ {
+ is.get ();
+ }
+
+ if (multiple_delims_as_one)
+ {
+ int prev = -1;
+ while (is && (c1 == skip_whitespace (is, true)) != EOF
+ && (((c1 == eol1 || c1 == eol2) && ++lines)
+ || -1 != lookahead (is, delim_list, max_len)))
+ {
+ if (prev == eol1 && eol1 != eol2 && c1 == eol2)
+ lines--;
+ prev = c1;
+ }
+ }
+ }
+ }
+
+ return c1;
+}
+
+/********************/
+// Skip delimiters -- multiple if MultipleDelimsAsOne specified.
+// FIXME For now, just handles single character delimiters
+bool
+textscan::match_literal (std::istream& is, const textscan_format_elt& fmt)
+{
+ // "false" -> treat EOL as normal space
+ // since a delimiter at the start of a line is a mismatch, not empty field
+ skip_whitespace (is, false);
+
+ for (unsigned int i = 0; i < fmt.width; i++)
+ {
+ int ch = is.get ();
+ if (ch != fmt.text[i])
+ {
+ if (ch != EOF)
+ is.putback (ch);
+ is.setstate (std::ifstream::failbit);
+ return false;
+ }
+ }
+ return true;
+}
+
+
+/********************/
+textscan_format_list::textscan_format_list (const std::string& s)
+ : set_from_first (false), has_string (false), nconv (0), curr_idx (0),
+ list (dim_vector (16, 1)), buf (0)
+{
+ octave_idx_type num_elts = 0;
+
+ size_t n = s.length ();
+
+ size_t i = 0;
+
+ unsigned int width = -1; // Unspecified width = max (except %c)
+ int prec = -1;
+ int bitwidth = 0;
+ bool discard = false;
+ char type = '\0';
+
+ bool have_more = true;
+
+ if (s.length () == 0)
+ {
+ buf = new std::ostringstream ("%f");
+ bitwidth = 64;
+ type = 'f';
+ add_elt_to_list (width, prec, bitwidth, octave_value (NDArray ()),
+ discard, type, num_elts);
+ have_more = false;
+ set_from_first = true;
+ nconv = 1;
+ }
+ else
+ {
+ set_from_first = false;
+ while (i < n)
+ {
+ have_more = true;
+
+ if (! buf)
+ buf = new std::ostringstream ();
+
+ if (s[i] == '%') // FIXME: %% should be part of literal_conversion
+ {
+ // Process percent-escape conversion type.
+
+ process_conversion (s, i, n, num_elts);
+
+ have_more = (buf != 0);
+ }
+ else if (isspace (s[i]))
+ {
+ while (++i < n && isspace (s[i]))
+ /* skip whitespace */;
+
+ have_more = false;
+ }
+ else
+ {
+ type = textscan_format_elt::literal_conversion;
+
+ width = 0;
+ prec = -1;
+ bitwidth = 0;
+ discard = true;
+
+ while (i < n && ! isspace (s[i]) && s[i] != '%')
+ {
+ *buf << s[i++];
+ width++;
+ }
+
+ add_elt_to_list (width, prec, bitwidth, octave_value (), discard, type,
+ num_elts);
+
+ have_more = false;
+ }
+
+ if (nconv < 0)
+ {
+ have_more = false;
+ break;
+ }
+ }
+ }
+
+ if (have_more)
+ add_elt_to_list (width, prec, bitwidth, octave_value (), discard, type, num_elts);
+
+ list.resize (dim_vector (num_elts, 1));
+
+ delete buf;
+}
+
+/********************/
+textscan_format_list::~textscan_format_list (void)
+{
+ octave_idx_type n = list.length ();
+
+ for (octave_idx_type i = 0; i < n; i++)
+ {
+ textscan_format_elt *elt = list(i);
+ delete elt;
+ }
+}
+
+/********************/
+void
+textscan_format_list::add_elt_to_list (unsigned int width, int prec, int bitwidth,
+ octave_value val_type,
+ bool discard, char type,
+ octave_idx_type& num_elts,
+ const std::string& char_class)
+{
+ if (buf)
+ {
+ std::string text = buf->str ();
+
+ if (! text.empty ())
+ {
+ textscan_format_elt *elt
+ = new textscan_format_elt (text.c_str (), width, prec, bitwidth,
+ discard, type, char_class);
+
+ if (num_elts == list.length ())
+ {
+ list.resize (dim_vector (2 * num_elts, 1));
+ }
+
+ if (!discard)
+ {
+ output_container.push_back (val_type);
+ }
+ list(num_elts++) = elt;
+ }
+
+ delete buf;
+ buf = 0;
+ }
+}
+
+/********************/
+void
+textscan_format_list::process_conversion (const std::string& s, size_t& i,
+ size_t n, octave_idx_type& num_elts)
+{
+ unsigned width = 0;
+ int prec = -1;
+ int bitwidth = 0;
+ bool discard = false;
+ octave_value val_type;
+ char type = '\0';
+
+ *buf << s[i++];
+
+ bool have_width = false;
+
+ while (i < n)
+ {
+ switch (s[i])
+ {
+ case '*':
+ if (discard)
+ nconv = -1;
+ else
+ {
+ discard = true;
+ *buf << s[i++];
+ }
+ break;
+
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ if (have_width)
+ nconv = -1;
+ else
+ {
+ char c = s[i++];
+ width = width * 10 + c - '0';
+ have_width = true;
+ *buf << c;
+ while (i < n && isdigit (s[i]))
+ {
+ c = s[i++];
+ width = width * 10 + c - '0';
+ *buf << c;
+ }
+
+ if (i < n && s[i] == '.')
+ {
+ *buf << s[i++];
+ prec = 0;
+ while (i < n && isdigit (s[i]))
+ {
+ c = s[i++];
+ prec = prec * 10 + c - '0';
+ *buf << c;
+ }
+ }
+ }
+ break;
+
+ case 'd': case 'u':
+ {
+ bool done = true;
+ *buf << (type = s[i++]);
+ if (i < n)
+ {
+ if (s[i] == '8')
+ {
+ bitwidth = 8;
+ if (type == 'd')
+ val_type = octave_value (int8NDArray ());
+ else
+ val_type = octave_value (uint8NDArray ());
+ *buf << s[i++];
+ }
+ else if (s[i] == '1' && i+1 < n && s[i+1] == '6')
+ {
+ bitwidth = 16;
+ if (type == 'd')
+ val_type = octave_value (int16NDArray ());
+ else
+ val_type = octave_value (uint16NDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else if (s[i] == '3' && i+1 < n && s[i+1] == '2')
+ {
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else if (s[i] == '6' && i+1 < n && s[i+1] == '4')
+ {
+ bitwidth = 64;
+ if (type == 'd')
+ val_type = octave_value (int64NDArray ());
+ else
+ val_type = octave_value (uint64NDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else
+ done = false;
+ }
+ else
+ done = false;
+
+ if (!done) // FIXME use type_id ()
+ {
+ bitwidth = 32;
+ if (type == 'd')
+ val_type = octave_value (int32NDArray ());
+ else
+ val_type = octave_value (uint32NDArray ());
+ }
+ goto fini;
+ }
+
+ case 'f':
+ *buf << (type = s[i++]);
+ bitwidth = 64;
+ if (i < n)
+ {
+ if (s[i] == '3' && i+1 < n && s[i+1] == '2')
+ {
+ bitwidth = 32;
+ val_type = octave_value (FloatNDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else if (s[i] == '6' && i+1 < n && s[i+1] == '4')
+ {
+ val_type = octave_value (NDArray ());
+ *buf << s[i++];
+ *buf << s[i++];
+ }
+ else
+ val_type = octave_value (NDArray ());
+ }
+ else
+ val_type = octave_value (NDArray ());
+ goto fini;
+
+ case 'n':
+ *buf << (type = s[i++]);
+ bitwidth = 64;
+ val_type = octave_value (NDArray ());
+ goto fini;
+
+ case 's': case 'q': case '%': case '[': case 'c':
+ if (!discard)
+ val_type = octave_value (Cell ());
+ *buf << (type = s[i++]);
+ has_string = true;
+ goto fini;
+
+ fini:
+ {
+ if (!have_width)
+ {
+ if (type == 'c')
+ width = 1;
+ else
+ width = static_cast (-1);
+ }
+
+ if (finish_conversion (s, i, n, width, prec, bitwidth, val_type,
+ discard, type, num_elts) == 0)
+ return;
+ }
+ break;
+
+ default:
+ error ("textscan: '%%%c' is not a valid format specifier", s[i]);
+ nconv = -1;
+ break;
+ }
+
+ if (nconv < 0)
+ break;
+ }
+
+ nconv = -1;
+}
+
+/********************/
+int
+textscan_format_list::finish_conversion (const std::string& s, size_t& i,
+ size_t n, unsigned int& width, int& prec,
+ int& bitwidth, octave_value& val_type,
+ bool discard,
+ char& type, octave_idx_type& num_elts)
+{
+ int retval = 0;
+
+ std::string char_class;
+
+ size_t beg_idx = std::string::npos;
+ size_t end_idx = std::string::npos;
+
+ if (type != '%')
+ {
+ nconv++;
+ if (type == '[')
+ {
+ if (i < n)
+ {
+ beg_idx = i;
+
+ if (s[i] == '^')
+ {
+ type = '^';
+ *buf << s[i++];
+
+ if (i < n)
+ {
+ beg_idx = i;
+
+ if (s[i] == ']')
+ *buf << s[i++];
+ }
+ }
+ else if (s[i] == ']')
+ *buf << s[i++];
+ }
+
+ while (i < n && s[i] != ']')
+ *buf << s[i++];
+
+ if (i < n && s[i] == ']')
+ {
+ end_idx = i-1;
+ *buf << s[i++];
+ }
+
+ if (s[i-1] != ']')
+ retval = nconv = -1;
+ }
+ }
+
+ if (nconv >= 0)
+ {
+ if (beg_idx != std::string::npos && end_idx != std::string::npos)
+ char_class = expand_char_class (s.substr (beg_idx,
+ end_idx - beg_idx + 1));
+
+ add_elt_to_list (width, prec, bitwidth, val_type, discard, type,
+ num_elts, char_class);
+ }
+
+ return retval;
+}
+
+/********************/
+// Apply global options to individual conversion specifiers.
+// Currently, records whether or not exp_chars is the default.
+void
+textscan_format_list::consider_options (const textscan& ts)
+{
+ // We need to use custom number parser if the list of exponential signifiers
+ // (the "e" in 1e20) is non-default
+ bool all_need_custom = !ts.default_exp;
+
+ // Check if any delimiter starts with an exponential signifier.
+ // If so, we need the custom number parser.
+ if (!all_need_custom)
+ {
+ for (int i = 0; i < ts.delim_list.length (); i++)
+ {
+ std::string s = ts.delim_list(i).string_value ();
+ if (s.length () > 0)
+ {
+ char last = s[s.length () - 1];
+ if (strchr (ts.exp_chars.c_str (), last))
+ {
+ all_need_custom = true;
+ break;
+ }
+ }
+ }
+ }
+
+ // Need custom parser if options mean that all do, or if this format
+ // is something like %10f or %10.4f.
+ for (int i = 0; i < length (); i++)
+ {
+ list(i)->custom_number = (all_need_custom || list(i)->prec != -1
+ || list(i)->width != static_cast (-1));
+ }
+}
+
+/********************
+void
+textscan_format_list::printme (void) const
+{
+ octave_idx_type n = list.length ();
+
+ for (octave_idx_type i = 0; i < n; i++)
+ {
+ textscan_format_elt *elt = list(i);
+
+ std::cerr
+ << "width: " << elt->width << "\n"
+ << "digits " << elt->prec << "\n"
+ << "bitwidth: " << elt->bitwidth << "\n"
+ << "discard: " << elt->discard << "\n"
+ << "type: ";
+
+ if (elt->type == textscan_format_elt::literal_conversion)
+ std::cerr << "literal text\n";
+ else if (elt->type == textscan_format_elt::whitespace_conversion)
+ std::cerr << "whitespace\n";
+ else
+ std::cerr << elt->type << "\n";
+
+ std::cerr
+ << "char_class: `" << undo_string_escapes (elt->char_class) << "'\n"
+ << "text: `" << undo_string_escapes (elt->text) << "'\n\n";
+ }
+}
+* */
+
+/********************/
+// If FORMAT is explicitly "", it is assumed to be "%f" repeated enough times
+// to read the first row of the file. Set it now.
+void
+textscan_format_list::read_first_row (std::istream& is, textscan& ts)
+{
+ std::string first_line;
+ getline (is, first_line, static_cast (ts.eol2));
+ std::istringstream strstr (first_line);
+
+ dim_vector dv (1,1); // initial size of each output_container
+ Complex val;
+ octave_value val_type;
+ nconv = 0;
+ int max_empty = 1000; // failsafe, if strstr fails but not with eof
+ long int pos;
+
+ // read line, creating output_container as we go
+ while (!strstr.eof ())
+ {
+ pos = strstr.tellg ();
+ ts.scan_complex (strstr, *list(0), val);
+ if (strstr.fail ()) // allow missing values in first row
+ {
+ strstr.clear (strstr.rdstate () & ~std::ifstream::failbit);
+ val = ts.empty_value.scalar_value ();
+ if (!--max_empty)
+ break;
+ }
+ if (val.imag () == 0)
+ val_type = octave_value (NDArray (dv, val.real ()));
+ else
+ val_type = octave_value (ComplexNDArray (dv, val));
+ output_container.push_back (val_type);
+ ts.skip_delim (strstr);
+ if (strstr.tellg () == pos) // Stop if we're not making progress
+ break;
+ nconv++;
+ }
+ output_container.pop_front (); // discard empty element from constructor
+
+ //Create fmt now that the size is known
+ list.resize (dim_vector (nconv, 1));
+ for (octave_idx_type i = 1; i < nconv; i++)
+ list(i) = new textscan_format_elt (*list(0));
+}
+
+/********************/
+// Perform actual textscan: read data from stream, and create cell array
+octave_value
+textscan::scan (std::istream *isp, textscan_format_list& fmt_list,
+ octave_idx_type ntimes)
+{
+ octave_value retval = Cell ();
+
+ if (!isp)
+ error ("internal error: textscan called with invalid istream");
+ else if (fmt_list.num_conversions () == -1)
+ ::error ("textscan: invalid format specified");
+ else if (! error_state)
+ {
+ if (fmt_list.num_conversions () == 0)
+ {
+ error ("textscan: no valid format conversion specifiers\n");
+ return Cell ();
+ }
+
+ // skip the first header_lines
+ std::string dummy;
+ for (int i = 0; i < header_lines && *isp; i++)
+ getline (*isp, dummy, static_cast (eol2));
+
+ // Grow retval dynamically. (If ntimes large, start smaller?)
+ // This is half the initial size
+ octave_idx_type size = ((ntimes < 8 && ntimes >= 0) ? ntimes : 1);
+ Array row_idx (dim_vector (1,2));
+ row_idx(1) = 0;
+
+ int err = 0;
+ octave_idx_type row = 0;
+ // If FORMAT explicitly "", read first line and see how many "%f" match
+ if (fmt_list.set_from_first)
+ {
+ row = 1; // the above puts the first line into fmt_list.out_buf ()
+ fmt_list.read_first_row (*isp, *this);
+ lines = 1;
+ }
+ std::list out = fmt_list.out_buf ();
+
+ // Check which columns to merge. Reals may become complex, and so can't
+ // trust types after reading in data.
+ bool merge_with_prev[fmt_list.length ()];
+ int conv = 0;
+ if (collect_output)
+ {
+ int prev_type = -1;
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++)
+ {
+ if (col->type_id () == prev_type)
+ merge_with_prev [conv++] = true;
+ else
+ merge_with_prev [conv++] = false;
+ prev_type = col->type_id ();
+ }
+ }
+
+ // Read the data
+ for (/* row set ~25 lines above */; row < ntimes || ntimes == -1; row++)
+ {
+ if (row == 0 || row >= size)
+ {
+ size += size+1;
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++)
+ *col = (*col).resize (dim_vector (size, 1), 0);
+ }
+ row_idx(0) = row;
+ err = read_format_once (isp, fmt_list, out, row_idx);
+ if (err < 0 || !*isp || error_state
+ || (lines >= ntimes && ntimes > -1))
+ break;
+ }
+
+ if (return_on_error)
+ error_state = false;
+
+ // convert return value to Cell array
+ int i = 0;
+ Array ra_idx (dim_vector (1,2));
+ // err = -2 means "error, and no columns read this row
+ int valid_rows = (row == ntimes) ? ntimes : ((err == -2) ? row : row+1);
+ dim_vector dv (valid_rows, 1);
+ ra_idx(0) = 0;
+ if (!collect_output)
+ {
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++, i++)
+ {
+ // resize col to size row
+ ra_idx(1) = i;
+ retval = do_cat_op (retval,
+ octave_value (Cell (col->resize (dv,0))),
+ ra_idx);
+ }
+ }
+ else // group adjacent cells of the same type into a single cell
+ {
+ octave_value cur; // current cell, accumulating columns
+ octave_idx_type group_size = 0; // columns in this cell
+ int prev_type = -1;
+
+ conv = 0;
+ for (std::list::iterator col = out.begin ();
+ col != out.end (); col++)
+ {
+ if (!merge_with_prev [conv++]) // including first time
+ {
+ if (prev_type != -1)
+ {
+ ra_idx(1) = i++;
+ retval = do_cat_op (retval, octave_value (Cell(cur)),
+ ra_idx);
+ }
+ cur = octave_value (col->resize (dv,0));
+ group_size = 1;
+ prev_type = col->type_id ();
+ }
+ else
+ {
+ ra_idx(1) = group_size++;
+ cur = do_cat_op (cur, octave_value (col->resize (dv,0)),
+ ra_idx);
+ }
+ }
+ ra_idx(1) = i;
+ retval = do_cat_op (retval, octave_value (Cell (cur)), ra_idx);
+ }
+ }
+ return retval;
+}
diff --git a/libinterp/corefcn/textscan.h b/libinterp/corefcn/textscan.h
new file mode 100644
index 0000000..5d17f6d
--- /dev/null
+++ b/libinterp/corefcn/textscan.h
@@ -0,0 +1,283 @@
+
+#if !defined (octave_textscan_h)
+#define octave_textscan_h 1
+
+// For Inf and NaN
+#include "lo-ieee.h"
+
+class
+OCTINTERP_API
+textscan_format_elt
+{
+public:
+
+ enum special_conversion
+ {
+ whitespace_conversion = 1,
+ literal_conversion = 2
+ };
+
+ textscan_format_elt (const char *txt = 0, int w = 0, int p = -1,
+ int bw = 0, bool dis = false, char typ = '\0',
+ const std::string& ch_class = std::string ())
+ : text (strsave (txt)), width (w), prec (p), bitwidth (bw),
+ char_class (ch_class), type (typ), discard (dis),
+ numeric(typ == 'd' || typ == 'u' || type == 'f' || type == 'n')
+ { }
+
+ textscan_format_elt (const textscan_format_elt& e)
+ : text (strsave (e.text)), width (e.width), prec (e.prec),
+ bitwidth (e.bitwidth), char_class (e.char_class), type (e.type),
+ discard (e.discard), numeric (e.numeric), custom_number (e.custom_number)
+ { }
+
+ textscan_format_elt& operator = (const textscan_format_elt& e)
+ {
+ if (this != &e)
+ {
+ text = strsave (e.text);
+ width = e.width;
+ prec = e.prec;
+ bitwidth = e.bitwidth;
+ discard = e.discard;
+ type = e.type;
+ numeric = e.numeric;
+ custom_number = e.custom_number;
+ char_class = e.char_class;
+ }
+
+ return *this;
+ }
+
+ ~textscan_format_elt (void) { delete [] text; }
+
+ // The C-style format string.
+ const char *text;
+
+ // The maximum field width.
+ unsigned int width;
+
+ // The maximum number of digits to read after the decimal in a
+ // floating point conversion.
+ int prec;
+
+ // The size of the result. For integers, bitwidth may be 8, 16, 34,
+ // or 64. For floating point values, bitwidth may be 32 or 64.
+ int bitwidth;
+
+ // The class of characters in a `[' or `^' format.
+ std::string char_class;
+
+ // Type of conversion
+ // -- `d', `u', `f', `n', `s', `q', `c', `%', `C', `D', `[' or `^'.
+ char type;
+
+ // TRUE if we are not storing the result of this conversion.
+ bool discard;
+
+ // TRUE if the type is 'd', 'u', 'f', 'n'
+ bool numeric;
+
+ // TRUE if width or precision is specified, or options.exp_chars is set
+ bool custom_number;
+};
+
+class textscan;
+
+class
+OCTINTERP_API
+textscan_format_list
+{
+public:
+
+ textscan_format_list (const std::string& fmt = std::string ());
+
+ ~textscan_format_list (void);
+
+ void consider_options (const textscan& options);
+
+ octave_idx_type num_conversions (void) const { return nconv; }
+
+ // The length can be different than the number of conversions.
+ // For example, "x %d y %d z" has 2 conversions but the length of
+ // the list is 3 because of the characters that appear after the
+ // last conversion.
+
+ octave_idx_type length (void) const { return list.length (); }
+
+ const textscan_format_elt *first (void)
+ {
+ curr_idx = 0;
+ return current ();
+ }
+
+ const textscan_format_elt *current (void) const
+ { return list.length () > 0 ? list.elem (curr_idx) : 0; }
+
+ const textscan_format_elt *next (bool cycle = true)
+ {
+ curr_idx++;
+
+ if (curr_idx >= list.length ())
+ {
+ if (cycle)
+ curr_idx = 0;
+ else
+ return 0;
+ }
+ return current ();
+ }
+
+ void printme (void) const;
+
+ bool ok (void) const { return (nconv >= 0); }
+
+ operator bool () const { return ok (); }
+
+ bool set_from_first; // true if number of %f to be set from data file
+ bool has_string; // at least one conversion specifier is s,q,c, or [...]
+ void read_first_row (std::istream& is, textscan& ts);
+
+ std::list out_buf () const { return (output_container); }
+
+private:
+
+ // Number of conversions specified by this format string, or -1 if
+ // invalid conversions have been found.
+ octave_idx_type nconv;
+
+ // Index to current element;
+ octave_idx_type curr_idx;
+
+ // FIXME -- maybe LIST should be a std::list object?
+ // List of format elements.
+ Array list;
+
+ // list holding column arrays of types specified by conversions
+ std::list output_container;
+
+ // Temporary buffer.
+ std::ostringstream *buf;
+
+ void add_elt_to_list (unsigned int width, int prec, int bitwidth,
+ octave_value val_type, bool discard,
+ char type, octave_idx_type& num_elts,
+ const std::string& char_class = std::string ());
+
+ void process_conversion (const std::string& s, size_t& i, size_t n,
+ octave_idx_type& num_elts);
+
+ int finish_conversion (const std::string& s, size_t& i, size_t n,
+ unsigned int& width, int& prec, int& bitwidth,
+ octave_value& val_type,
+ bool discard, char& type,
+ octave_idx_type& num_elts);
+ // No copying!
+
+ textscan_format_list (const textscan_format_list&);
+
+ textscan_format_list& operator = (const textscan_format_list&);
+};
+
+class
+textscan
+{
+ friend textscan_format_list;
+
+ std::string buf;
+ // Three cases for delim_table and delim_list
+ // 1. delim_table empty, delim_list empty: whitespace delimiters
+ // 2. delim_table = look-up table of delim chars, delim_list empty.
+ // 3. delim_table non-empty, delim_list = Cell array of delim strings
+ std::string delim_table; // delim_table[i]=='\0' if i is not a delimiter,
+ std::string whitespace_table;
+ Cell comment_style;
+ int comment_len; // How far ahead to look to detect an open comment
+ int comment_char; // first character of open comment
+
+ std::string date_locale;
+
+ Cell delim_list; // Array of strings of delimiters
+
+ octave_value empty_value;
+ std::string exp_chars;
+ int header_lines;
+ Cell treat_as_empty;
+ int treat_as_empty_len; // longest string to treat as "N/A"
+ std::string whitespace;
+ short eol1, eol2;
+
+ bool collect_output;
+ bool multiple_delims_as_one;
+ bool return_on_error;
+
+ bool default_exp;
+ bool numeric_delim;
+
+ octave_idx_type lines;
+
+ char *get_field (std::istream& isp, unsigned int width) const;
+ bool match_literal (std::istream& isp, const textscan_format_elt& elem);
+ int skip_whitespace (std::istream& is, bool EOLstop = false);
+ int skip_delim (std::istream& is);
+ bool is_delim (unsigned char ch) const
+ {
+ return (delim_table.length() == 0
+ && (isspace (ch) || ch == eol1 || ch == eol2))
+ || delim_table[ch] != '\0';
+ }
+ // the only delimiter is whitespace
+ bool whitespace_delim () const { return delim_table.length() == 0; }
+ bool isspace (unsigned int ch) const { return whitespace_table[ch & 0xff]; }
+
+ std::istream& scan_string (std::istream& is, const textscan_format_elt& fmt,
+ std::string& val) const;
+ std::string read_until (std::istream& is, const Cell& delimiters,
+ const std::string& ends) const;
+ std::istream& scan_cstring (std::istream& is, const textscan_format_elt& fmt,
+ std::string& val) const;
+ std::istream& scan_qstring (std::istream& is, const textscan_format_elt& fmt,
+ std::string& val);
+ int read_format_once (std::istream* isp, textscan_format_list& fmt_list,
+ std::list & retval,
+ Array row);
+ std::istream& scan_one (std::istream& is, const textscan_format_elt& fmt,
+ octave_value& ov, Array row);
+ int scan_bracket (std::istream& is, const char *pattern, std::string& val)
+ const;
+ int scan_caret (std::istream& is, const char *, std::string& val) const;
+ int lookahead (std::istream& is, const Cell& targets, int max_len) const;
+ int read_format_once(std::istream* isp, textscan_format_list& fmt_list,
+ Cell& retval, octave_idx_type row) const;
+
+ double formatted_double (std::istream& is, const textscan_format_elt& fmt)
+ const;
+ double read_double (std::istream& is, const textscan_format_elt& fmt) const
+ {
+ if (fmt.custom_number)
+ return formatted_double (is, fmt);
+ else
+ return octave_read_value (is);
+ }
+ std::istream& scan_complex (std::istream& is, const textscan_format_elt& fmt,
+ Complex& val) const;
+ public:
+
+ textscan ()
+ : buf (""), delim_table (""), comment_char(-2), empty_value (octave_NaN),
+ exp_chars ("edED"), header_lines (0), treat_as_empty_len (0),
+ whitespace (" \b\t"),
+ eol1('\r'), eol2('\n'),
+ collect_output (false), multiple_delims_as_one (false),
+ return_on_error (true), default_exp (true), numeric_delim (false),
+ lines (0)
+ { };
+
+ octave_value scan (std::istream* isp, textscan_format_list& fmt_list,
+ octave_idx_type ntimes);
+ void parse_options (const octave_value_list& args, int first_param,
+ textscan_format_list& formats);
+};
+
+
+#endif
diff --git a/scripts/Makefile.in b/scripts/Makefile.in
index e513116..93515dc 100644
--- a/scripts/Makefile.in
+++ b/scripts/Makefile.in
@@ -2229,7 +2229,6 @@ io_FCN_FILES = \
io/importdata.m \
io/is_valid_file_id.m \
io/strread.m \
- io/textscan.m \
io/textread.m
java_FCN_FILES = \
diff --git a/scripts/io/textscan.m b/scripts/io/textscan.m
deleted file mode 100644
index 32f597c..0000000
--- a/scripts/io/textscan.m
+++ /dev/null
@@ -1,698 +0,0 @@
-## Copyright (C) 2010-2014 Ben Abbott
-##
-## This file is part of Octave.
-##
-## Octave is free software; you can redistribute it and/or modify it
-## under the terms of the GNU General Public License as published by
-## the Free Software Foundation; either version 3 of the License, or (at
-## your option) any later version.
-##
-## Octave is distributed in the hope that it will be useful, but
-## WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with Octave; see the file COPYING. If not, see
-## .
-
-## -*- texinfo -*-
-## @deftypefn {Function File} {@var{C} =} textscan (@var{fid}, @var{format})
-## @deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{n})
-## @deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{param}, @var{value}, @dots{})
-## @deftypefnx {Function File} {@var{C} =} textscan (@var{fid}, @var{format}, @var{n}, @var{param}, @var{value}, @dots{})
-## @deftypefnx {Function File} {@var{C} =} textscan (@var{str}, @dots{})
-## @deftypefnx {Function File} {[@var{C}, @var{position}] =} textscan (@var{fid}, @dots{})
-## Read data from a text file or string.
-##
-## The string @var{str} or file associated with @var{fid} is read from and
-## parsed according to @var{format}. The function behaves like @code{strread}
-## except it can also read from file instead of a string. See the documentation
-## of @code{strread} for details.
-##
-## In addition to the options supported by @code{strread}, this function
-## supports a few more:
-##
-## @itemize
-## @item @qcode{"collectoutput"}:
-## A value of 1 or true instructs textscan to concatenate consecutive columns
-## of the same class in the output cell array. A value of 0 or false (default)
-## leaves output in distinct columns.
-##
-## @item @qcode{"endofline"}:
-## Specify @qcode{"@xbackslashchar{}r"}, @qcode{"@xbackslashchar{}n"} or
-## @qcode{"@xbackslashchar{}r@xbackslashchar{}n"} (for CR, LF, or CRLF). If no
-## value is given, it will be inferred from the file. If set to "" (empty
-## string) EOLs are ignored as delimiters and added to whitespace.
-##
-## @item @qcode{"headerlines"}:
-## The first @var{value} number of lines of @var{fid} are skipped.
-##
-## @item @qcode{"returnonerror"}:
-## If set to numerical 1 or true (default), return normally when read errors
-## have been encountered. If set to 0 or false, return an error and no data.
-## As the string or file is read by columns rather than by rows, and because
-## textscan is fairly forgiving as regards read errors, setting this option
-## may have little or no actual effect.
-## @end itemize
-##
-## When reading from a character string, optional input argument @var{n}
-## specifies the number of times @var{format} should be used (i.e., to limit
-## the amount of data read).
-## When reading from file, @var{n} specifies the number of data lines to read;
-## in this sense it differs slightly from the format repeat count in strread.
-##
-## The output @var{C} is a cell array whose second dimension is determined
-## by the number of format specifiers.
-##
-## The second output, @var{position}, provides the position, in characters,
-## from the beginning of the file.
-##
-## If the format string is empty (not: omitted) and the file contains only
-## numeric data (excluding headerlines), textscan will return data in a number
-## of columns matching the number of numeric fields on the first data line of
-## the file.
-##
-## @seealso{dlmread, fscanf, load, strread, textread}
-## @end deftypefn
-
-## Author: Ben Abbott
-
-function [C, position] = textscan (fid, format = "%f", varargin)
-
- BUFLENGTH = 4096; # Read buffer
- emptfmt = 0; # Signals deliberately empty format string
-
- ## Check input
- if (nargin < 1)
- print_usage ();
- endif
-
- if (isempty (format))
- emptfmt = 1;
- format = "%f";
- endif
-
- if (! ischar (format))
- error ("textscan: FORMAT must be a string");
- endif
-
- ## Determine the number of data fields & initialize output array
- num_fields = numel (regexp (format, '(%(\d*|\d*\.\d*)?[nfduscq]|%\[)', "match"));
- if (! num_fields)
- error ("textscan.m: no valid format conversion specifiers found\n");
- endif
- C = cell (1, num_fields);
-
- if (! (isa (fid, "double") && fid > 0) && ! ischar (fid))
- error ("textscan: first argument must be a file id or character string");
- endif
-
- args = varargin;
- if (nargin > 2 && isnumeric (args{1}))
- nlines = args{1};
- else
- nlines = Inf;
- endif
- if (nlines < 1)
- printf ("textscan: N = 0, no data read\n");
- return; endif
-
- if (! any (strcmpi (args, "emptyvalue")))
- ## Matlab returns NaNs for missing values
- args(end+1:end+2) = {'emptyvalue', NaN};
- endif
-
- ## Check default parameter values that differ for strread & textread
-
- ipos = find (strcmpi (args, "whitespace"));
- if (isempty (ipos))
- ## Matlab default whitespace = " \b\t"
- args(end+1:end+2) = {'whitespace', " \b\t"};
- whitespace = " \b\t";
- else
- ## Check if there's at least one string format specifier
- has_str_fmt = regexp (format, '%[*]?\d*s', "once");
- ## If there is a string format AND whitespace value = empty,
- ## don't add a space (char(32)) to whitespace
- if (! (isempty (args{ipos+1}) && has_str_fmt))
- args{ipos+1} = unique ([" ", args{ipos+1}]);
- endif
- whitespace = args{ipos+1};
- endif
-
- if (! any (strcmpi (args, "delimiter")))
- ## Matlab says default delimiter = whitespace.
- ## strread() will pick this up further
- args(end+1:end+2) = {'delimiter', ""};
- delimiter = "";
- else
- delimiter = args{find (strcmpi (args, "delimiter")) + 1};
- endif
-
- collop = false;
- ipos = find (strcmpi (args, "collectoutput"));
- if (! isempty (ipos))
- ## Search & concatenate consecutive columns of same class requested
- if (isscalar (args{ipos+1})
- && (islogical (args{ipos+1}) || isnumeric (args{ipos+1})))
- collop = args{ipos+1};
- else
- warning ("textscan: illegal value for CollectOutput parameter - ignored");
- endif
- ## Remove argument before call to strread() below
- args(ipos:ipos+1) = [];
- endif
-
- if (any (strcmpi (args, "returnonerror")))
- ## Because of the way strread() reads data (columnwise) this parameter
- ## can't be neatly implemented. strread() will pick it up anyway
- warning ('textscan: ReturnOnError is not fully implemented');
- else
- ## Set default value (=true)
- args(end+1:end+2) = {"returnonerror", 1};
- endif
-
- ## Check if a headerlines argument is specified
- headerlines = find (strcmpi (args, "headerlines"), 1);
- if (! isempty (headerlines))
- ## Yep. But it is stray when reading from strings...
- if (ischar (fid))
- warning ("textscan: 'headerlines' ignored when reading from strings");
- endif
- endif
-
- if (ischar (fid))
- ## Read from a text string
- if (nargout == 2)
- error ("textscan: cannot provide position information for character input");
- endif
- str = fid;
- else
- st_pos = ftell (fid);
- ## Skip header lines if requested
- if (! isempty (headerlines))
- ## Beware of missing or wrong headerline value
- if (headerlines == numel (args)
- || ! isnumeric (args{headerlines + 1}))
- error ("Missing or illegal value for 'headerlines'" );
- endif
- ## Avoid conveying floats to fskipl
- args{headerlines + 1} = round (args{headerlines + 1});
- if (args{headerlines + 1} > 0)
- ## Beware of zero valued headerline, fskipl would skip to EOF
- fskipl (fid, args{headerlines + 1});
- args(headerlines:headerlines+1) = [];
- st_pos = ftell (fid);
- elseif (args{headerlines + 1} < 0)
- warning ("textscan.m: negative headerline value ignored");
- endif
- endif
- ## Read a first file chunk. Rest follows after endofline processing
- [str, count] = fscanf (fid, "%c", BUFLENGTH);
-
- endif
-
- ## Check for empty result
- if (isempty (str))
- warning ("textscan: no data read");
- return;
- endif
-
- ## Check value of 'endofline'. String or file doesn't seem to matter
- endofline = find (strcmpi (args, "endofline"), 1);
- if (! isempty (endofline))
- if (ischar (args{endofline + 1}))
- eol_char = args{endofline + 1};
- if (! any (strcmp (eol_char, {"", "\n", "\r", "\r\n"})))
- error ("textscan: illegal EndOfLine character value specified");
- endif
- else
- error ("textscan: character value required for EndOfLine");
- endif
- else
- if (! ischar (fid))
- ## Determine EOL from file.
- ## Search for EOL candidates in the first BUFLENGTH chars
- eol_srch_len = min (length (str), BUFLENGTH);
- ## First try DOS (CRLF)
- if (! isempty (strfind (str(1 : eol_srch_len), "\r\n")))
- eol_char = "\r\n";
- ## Perhaps old Macintosh? (CR)
- elseif (! isempty (strfind (str(1 : eol_srch_len), "\r")))
- eol_char = "\r";
- ## Otherwise, use plain UNIX (LF)
- else
- eol_char = "\n";
- endif
- else
- eol_char = "\n";
- endif
- ## Set up the default endofline param value
- args(end+1:end+2) = {"endofline", eol_char};
- endif
-
- if (! ischar (fid))
- ## Now that we know what EOL looks like, we can process format_repeat_count.
- ## FIXME The below isn't ML-compatible: counts lines, not format string uses
- if (isfinite (nlines) && (nlines >= 0))
- l_eol_char = length (eol_char);
- eoi = findstr (str, eol_char);
- n_eoi = length (eoi);
- nblks = 0;
- ## Avoid slow repeated str concatenation, first seek requested end of data
- while (n_eoi < nlines && count == BUFLENGTH)
- [nstr, count] = fscanf (fid, "%c", BUFLENGTH);
- if (count > 0)
- ## Watch out for multichar EOL being missed across buffer boundaries
- if (l_eol_char > 1)
- str = [str(end - length (eol_char) + 2 : end) nstr];
- else
- str = nstr;
- endif
- eoi = findstr (str, eol_char);
- n_eoi += numel (eoi);
- ++nblks;
- endif
- endwhile
- ## Handle case of missing trailing EOL
- if (! strcmp (str(end - length (eol_char) + 1 : end), eol_char))
- eoi = [ eoi (length (str)) ];
- ++n_eoi;
- endif
- ## OK, found EOL delimiting last requested line. Compute ptr (incl. EOL)
- if (isempty (eoi))
- disp ("textscan: format repeat count specified but no endofline found");
- data_size = nblks * BUFLENGTH + count;
- else
- ## Compute data size to read incl complete EOL
- data_size = (nblks * BUFLENGTH) ...
- + eoi(end + min (nlines, n_eoi) - n_eoi) ...
- + l_eol_char - 1;
- endif
- fseek (fid, st_pos, "bof");
- str = fscanf (fid, "%c", data_size);
- args{1} = Inf;
- else
- fseek (fid, st_pos, "bof");
- str = fread (fid, "char=>char").';
- endif
- endif
-
- ## Strip trailing EOL to avoid returning stray missing values (f. strread).
- ## However, in case of CollectOutput request, presence of EOL is required;
- ## also in case of deliberately entered empty format string
- eol_at_end = strcmp (str(end-length (eol_char) + 1 : end), eol_char);
- if (collop || emptfmt)
- if (! eol_at_end)
- str(end+1 : end+length (eol_char)) = eol_char;
- endif
- elseif (eol_at_end)
- str(end-length (eol_char) + 1 : end) = "";
- ## A corner case: str may now be empty....
- if (isempty (str)); return; endif
- endif
-
- ## Call strread to make it do the real work
- C = cell (1, num_fields);
- [C{:}] = strread (str, format, args{:});
-
- ## I.c.o. empty format, match nr. of cols to nr. of fields on first read line
- if (emptfmt)
- ## Find end of first line
- eoi = index (str, eol_char);
- if (eoi)
- ## str contains an EOL, proceed with assessing nr. of columns
- ncols = countcols (C, str(1 : eoi-1), delimiter, whitespace);
- ## See if lowermost data row must be completed
- pad = mod (numel (C{1}), ncols);
- if (pad)
- ## Pad output with emptyvalues (rest has been done by stread.m)
- emptv = find (strcmpi (args, "emptyvalue"));
- if (isempty (emptv))
- ## By default textscan returns NaNs for empty fields
- C(1) = [C{1}; NaN(ncols - pad, 1)];
- else
- ## Otherwise return supplied emptyvalue. Pick last occurrence
- C(1) = [C{1}; repmat(args{emptv(end)+1}, ncols - pad, 1)];
- endif
- endif
- ## Compute nr. of rows
- nrows = floor (numel (C{1}) / ncols);
- ## Reshape C; watch out, transpose needed
- C(1) = reshape (C{1}, ncols, numel (C{1}) / ncols).';
- ## Distribute columns over C and wipe cols 2:end of C{1}
- for ii=2:ncols
- C(ii) = C{1}(:, ii);
- endfor
- C{1} = C{1}(:, 1);
- endif
- endif
-
- ## If requested, collect output columns of same class
- if (collop)
- C = colloutp (C);
- endif
-
- if (nargout == 2)
- ## Remember file position (persistent var)
- position = ftell (fid);
- endif
-
-endfunction
-
-
-## Assess nr of data fields on first line of data
-function ncols = countcols (C, str, dlm, wsp)
-
- if (isempty (dlm))
- ## Field separator = whitespace. Fold multiple whitespace into one
- str = regexprep (str, sprintf ("[%s]", wsp), " ");
- str = strtrim (str);
- ncols = numel (strfind (str, " ")) + 1;
- else
- ncols = numel (regexp (str, sprintf ("[%s]", dlm))) + 1;
- endif
-
-endfunction
-
-
-## Collect consecutive columns of same class into one cell column
-function C = colloutp (C)
-
- ## Start at rightmost column and work backwards to avoid ptr mixup
- ii = numel (C);
- while (ii > 1)
- clss1 = class (C{ii});
- jj = ii;
- while (jj > 1 && strcmp (clss1, class (C{jj - 1})))
- ## Column to the left is still same class; check next column to the left
- --jj;
- endwhile
- if (jj < ii)
- ## Concatenate columns into current column
- C{jj} = [C{jj : ii}];
- ## Wipe concatenated columns to the right, resume search to the left
- C(jj+1 : ii) = [];
- ii = jj - 1;
- else
- ## No similar class in column to the left, search from there
- --ii;
- endif
- endwhile
-
-endfunction
-
-
-%!test
-%! str = "1, 2, 3, 4\n 5, , , 8\n 9, 10, 11, 12";
-%! fmtstr = "%f %d %f %s";
-%! c = textscan (str, fmtstr, 2, "delimiter", ",", "emptyvalue", -Inf);
-%! assert (isequal (c{1}, [1;5]));
-%! assert (length (c{1}), 2);
-%! assert (iscellstr (c{4}));
-%! assert (isequal (c{3}, [3; -Inf]));
-
-%!test
-%! b = [10:10:100];
-%! b = [b; 8*b/5];
-%! str = sprintf ("%g miles/hr = %g kilometers/hr\n", b);
-%! fmt = "%f miles/hr = %f kilometers/hr";
-%! c = textscan (str, fmt);
-%! assert (b(1,:)', c{1}, 1e-5);
-%! assert (b(2,:)', c{2}, 1e-5);
-
-%!test
-%! str = "13, 72, NA, str1, 25\r\n// Middle line\r\n36, na, 05, str3, 6";
-%! a = textscan (str, "%d %n %f %s %n", "delimiter", ",","treatAsEmpty", {"NA", "na"},"commentStyle", "//");
-%! assert (a{1}, int32 ([13; 36]));
-%! assert (a{2}, [72; NaN]);
-%! assert (a{3}, [NaN; 5]);
-%! assert (a{4}, {"str1"; "str3"});
-%! assert (a{5}, [25; 6]);
-
-%!test
-%! str = "Km:10 = hhhBjjj miles16hour\r\n";
-%! str = [str "Km:15 = hhhJjjj miles241hour\r\n"];
-%! str = [str "Km:2 = hhhRjjj miles3hour\r\n"];
-%! str = [str "Km:25 = hhhZ\r\n"];
-%! fmt = "Km:%d = hhh%1sjjj miles%dhour";
-%! a = textscan (str, fmt, "delimiter", " ");
-%! assert (a{1}', int32 ([10 15 2 25]));
-%! assert (a{2}', {'B' 'J' 'R' 'Z'});
-%! assert (a{3}', int32 ([16 241 3 0]));
-
-## Test with default endofline parameter
-%!test
-%! c = textscan ("L1\nL2", "%s");
-%! assert (c{:}, {"L1"; "L2"});
-
-## Test with endofline parameter set to "" (empty) - newline should be in word
-%!test
-%! c = textscan ("L1\nL2", "%s", "endofline", "");
-%! assert (int8 (c{:}{:}), int8 ([ 76, 49, 10, 76, 50 ]));
-
-%!test
-%! ## No delimiters at all besides EOL. Skip fields, even empty fields
-%! str = "Text1Text2Text\nTextText4Text\nText57Text";
-%! c = textscan (str, "Text%*dText%dText");
-%! assert (c{1}, int32 ([2; 4; 0]));
-
-## CollectOutput test
-%!test
-%! b = [10:10:100];
-%! b = [b; 8*b/5; 8*b*1000/5];
-%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
-%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
-%! c = textscan (str, fmt, "collectoutput", 1);
-%! assert (size (c{3}), [10, 2]);
-%! assert (size (c{2}), [10, 2]);
-
-## CollectOutput test with uneven column length files
-%!test
-%! b = [10:10:100];
-%! b = [b; 8*b/5; 8*b*1000/5];
-%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
-%! str = [str "110 miles/hr"];
-%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
-%! c = textscan (str, fmt, "collectoutput", 1);
-%! assert (size (c{1}), [11, 1]);
-%! assert (size (c{3}), [11, 2]);
-%! assert (size (c{2}), [11, 2]);
-%! assert (c{3}(end), NaN);
-%! assert (c{2}{11, 1}, "/hr");
-%! assert (isempty (c{2}{11, 2}), true);
-
-## Test input validation
-%!error textscan ()
-%!error textscan (single (4))
-%!error textscan ({4})
-%!error textscan ("Hello World", 2)
-%!error [C, pos] = textscan ("Hello World")
-%!error textscan ("Hello World", '%s', 'EndOfLine', 3)
-
-%! Test incomplete first data line
-%! R = textscan (['Empty1' char(10)], 'Empty%d %f');
-%! assert (R{1}, int32 (1));
-%! assert (isempty (R{2}), true);
-
-## bug #37023 (actually a strread test)
-%!test
-%! data = textscan(" 1. 1 \n 2 3\n", '%f %f');
-%! assert (data{1}, [1; 2], 1e-15);
-%! assert (data{2}, [1; 3], 1e-15);
-
-## Whitespace test (bug #37333) using delimiter ";"
-%!test
-%! tc = [];
-%! tc{1, 1} = "C:/code;";
-%! tc{1, end+1} = "C:/code/meas;";
-%! tc{1, end+1} = " C:/code/sim;";
-%! tc{1, end+1} = "C:/code/utils;";
-%! string = [tc{:}];
-%! c = textscan (string, "%s", "delimiter", ";");
-%! for k = 1:numel (c{1})
-%! lh = c{1}{k};
-%! rh = tc{k};
-%! rh(rh == ";") = "";
-%! rh = strtrim (rh);
-%! assert (strcmp (lh, rh));
-%! end
-
-## Whitespace test (bug #37333), adding multipleDelimsAsOne true arg
-%!test
-%! tc = [];
-%! tc{1, 1} = "C:/code;";
-%! tc{1, end+1} = " C:/code/meas;";
-%! tc{1, end+1} = "C:/code/sim;;";
-%! tc{1, end+1} = "C:/code/utils;";
-%! string = [tc{:}];
-%! c = textscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 1);
-%! for k = 1:numel (c{1})
-%! lh = c{1}{k};
-%! rh = tc{k};
-%! rh(rh == ";") = "";
-%! rh = strtrim (rh);
-%! assert (strcmp (lh, rh));
-%! end
-
-## Whitespace test (bug #37333), adding multipleDelimsAsOne false arg
-%!test
-%! tc = [];
-%! tc{1, 1} = "C:/code;";
-%! tc{1, end+1} = " C:/code/meas;";
-%! tc{1, end+1} = "C:/code/sim;;";
-%! tc{1, end+1} = "";
-%! tc{1, end+1} = "C:/code/utils;";
-%! string = [tc{:}];
-%! c = textscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 0);
-%! for k = 1:numel (c{1})
-%! lh = c{1}{k};
-%! rh = tc{k};
-%! rh(rh == ";") = "";
-%! rh = strtrim (rh);
-%! assert (strcmp (lh, rh));
-%! end
-
-## Whitespace test (bug #37333) whitespace "" arg
-%!test
-%! tc = [];
-%! tc{1, 1} = "C:/code;";
-%! tc{1, end+1} = " C:/code/meas;";
-%! tc{1, end+1} = "C:/code/sim;";
-%! tc{1, end+1} = "C:/code/utils;";
-%! string = [tc{:}];
-%! c = textscan (string, "%s", "delimiter", ";", "whitespace", "");
-%! for k = 1:numel (c{1})
-%! lh = c{1}{k};
-%! rh = tc{k};
-%! rh(rh == ";") = "";
-%! assert (strcmp (lh, rh));
-%! end
-
-## Whitespace test (bug #37333), whitespace " " arg
-%!test
-%! tc = [];
-%! tc{1, 1} = "C:/code;";
-%! tc{1, end+1} = " C:/code/meas;";
-%! tc{1, end+1} = "C:/code/sim;";
-%! tc{1, end+1} = "C:/code/utils;";
-%! string = [tc{:}];
-%! c = textscan (string, "%s", "delimiter", ";", "whitespace", " ");
-%! for k = 1:numel (c{1})
-%! lh = c{1}{k};
-%! rh = tc{k};
-%! rh(rh == ";") = "";
-%! rh = strtrim (rh);
-%! assert (strcmp (lh, rh));
-%! end
-
-## Test reading from a real file
-%!test
-%! f = tempname ();
-%! fid = fopen (f, "w+");
-%! d = rand (1, 4);
-%! fprintf (fid, " %f %f %f %f ", d);
-%! fseek (fid, 0, "bof");
-%! A = textscan (fid, "%f %f");
-%! fclose (fid);
-%! unlink (f);
-%! assert (A{1}, [d(1); d(3)], 1e-6);
-%! assert (A{2}, [d(2); d(4)], 1e-6);
-
-## Tests reading with empty format, should return proper nr of columns
-%!test
-%! f = tempname ();
-%! fid = fopen (f, "w+");
-%! fprintf (fid, " 1 2 3 4\n5 6 7 8");
-%! fseek (fid, 0, "bof");
-%! A = textscan (fid, "");
-%! fclose (fid);
-%! unlink (f);
-%! assert (A{1}, [1 ; 5], 1e-6);
-%! assert (A{2}, [2 ; 6], 1e-6);
-%! assert (A{3}, [3 ; 7], 1e-6);
-%! assert (A{4}, [4 ; 8], 1e-6);
-
-## Tests reading with empty format; empty fields & incomplete lower row
-%!test
-%! f = tempname ();
-%! fid = fopen (f, "w+");
-%! fprintf (fid, " ,2,,4\n5,6");
-%! fseek (fid, 0, "bof");
-%! A = textscan (fid, "", "delimiter", ",", "EmptyValue", 999, "CollectOutput" , 1);
-%! fclose (fid);
-%! unlink (f);
-%! assert (A{1}, [999, 2, 999, 4; 5, 6, 999, 999], 1e-6);
-
-## Error message tests
-
-%!test
-%! f = tempname ();
-%! fid = fopen (f, "w+");
-%! msg1 = "Missing or illegal value for 'headerlines'";
-%! try
-%! A = textscan (fid, "", "headerlines");
-%! end_try_catch;
-%! fclose (fid);
-%! unlink (f);
-%! assert (msg1, lasterr);
-
-%!test
-%! f = tempname ();
-%! fid = fopen (f, "w+");
-%! msg1 = "Missing or illegal value for 'headerlines'";
-%! try
-%! A = textscan (fid, "", "headerlines", "hh");
-%! end_try_catch;
-%! fclose (fid);
-%! unlink (f);
-%! assert (msg1, lasterr);
-
-%!test
-%! f = tempname ();
-%! fid = fopen (f, "w+");
-%! fprintf (fid,"some_string");
-%! fseek (fid, 0, "bof");
-%! msg1 = "textscan: illegal EndOfLine character value specified";
-%! try
-%! A = textscan (fid, "%f", "EndOfLine", "\n\r");
-%! end_try_catch;
-%! fclose (fid);
-%! unlink (f);
-%! assert (msg1, lasterr);
-
-%!test
-%! f = tempname ();
-%! fid = fopen (f, "w+");
-%! fprintf (fid,"some_string");
-%! fseek (fid, 0, "bof");
-%! msg1 = "textscan: character value required for EndOfLine";
-%! try
-%! A = textscan (fid, "%f", "EndOfLine", 33);
-%! end_try_catch;
-%! fclose (fid);
-%! unlink (f);
-%! assert (msg1, lasterr);
-
-## Bug #41824
-%!test
-%! assert (textscan ("123", "", "whitespace", " "){:}, 123);
-
-## Bug #42343-1, just test supplied emptyvalue (actually done by strread.m)
-%!test
-%! assert (textscan (",NaN", "", "delimiter", "," ,"emptyValue" ,Inf), {Inf, NaN});
-
-## Bug #42343-2, test padding with supplied emptyvalue (done by textscan.m)
-%!test
-%! a = textscan (",1,,4\nInf, ,NaN", "", "delimiter", ",", "emptyvalue", -10);
-%! assert (cell2mat (a), [-10, 1, -10, 4; Inf, -10, NaN, -10]);
-
-## Bug #42528
-%!test
-%! assert (textscan ("1i", ""){1}, 0+1i);
-%! assert (cell2mat (textscan ("3, 2-4i, NaN\n -i, 1, 23.4+2.2i", "")), [3+0i, 2-4i, NaN+0i; 0-i, 1+0i, 23.4+2.2i]);
-
-## Illegal format specifiers
-%!test
-%!error textscan ("1.0", "%z");
diff --git a/test/textscan.tst b/test/textscan.tst
new file mode 100644
index 0000000..8d738d1
--- /dev/null
+++ b/test/textscan.tst
@@ -0,0 +1,433 @@
+
+
+%!test
+%! str = "1, 2, 3, 4\n 5, , , 8\n 9, 10, 11, 12";
+%! fmtstr = "%f %d %f %s";
+%! c = textscan (str, fmtstr, 2, "delimiter", ",", "emptyvalue", -Inf);
+%! assert (isequal (c{1}, [1;5]));
+%! assert (length (c{1}), 2);
+%! assert (iscellstr (c{4}));
+%! assert (isequal (c{3}, [3; -Inf]));
+
+%!test
+%! b = [10:10:100];
+%! b = [b; 8*b/5];
+%! str = sprintf ("%g miles/hr = %g kilometers/hr\n", b);
+%! fmt = "%f miles/hr = %f kilometers/hr";
+%! c = textscan (str, fmt);
+%! assert (c{1}, b(1,:)', 1e-5);
+%! assert (c{2}, b(2,:)', 1e-5);
+
+%!test
+%! str = "13, -, NA, str1, -25\r\n// Middle line\r\n36, na, 05, str3, 6";
+%! a = textscan (str, "%d %n %f %s %n", "delimiter", ",",
+%! "treatAsEmpty", {"NA", "na", "-"},"commentStyle", "//");
+%! assert (a{1}, int32 ([13; 36]));
+%! assert (a{2}, [NaN; NaN]);
+%! assert (a{3}, [NaN; 5]);
+%! assert (a{4}, {"str1"; "str3"});
+%! assert (a{5}, [-25; 6]);
+
+%!test
+%! str = "Km:10 = hhhBjjj miles16hour\r\n";
+%! str = [str "Km:15 = hhhJjjj miles241hour\r\n"];
+%! str = [str "Km:2 = hhhRjjj miles3hour\r\n"];
+%! str = [str "Km:25 = hhhZ\r\n"];
+%! fmt = "Km:%d = hhh%1sjjj miles%dhour";
+%! a = textscan (str, fmt, "delimiter", " ");
+%! assert (a{1}', int32 ([10 15 2 25]));
+%! assert (a{2}', {'B' 'J' 'R' 'Z'});
+%! assert (a{3}', int32 ([16 241 3 0]));
+
+## Test with default endofline parameter
+%!test
+%! c = textscan ("L1\nL2", "%s");
+%! assert (c{:}, {"L1"; "L2"});
+
+## Test with endofline parameter set to "" (empty) - newline should be in word
+%!test
+%! c = textscan ("L1\nL2", "%s", "endofline", "");
+%! assert (int8 (c{:}{:}), int8 ([ 76, 49, 10, 76, 50 ]));
+
+### Matlab fails this test. A literal after a conversion is not a delimiter
+#%!test
+#%! ## No delimiters at all besides EOL. Skip fields, even empty fields
+#%! str = "Text1Text2Text\nTextText4Text\nText57Text";
+#%! c = textscan (str, "Text%*dText%dText");
+#%! assert (c{1}, int32 ([2; 4; 0]));
+
+## CollectOutput test
+%!test
+%! b = [10:10:100];
+%! b = [b; 8*b/5; 8*b*1000/5];
+%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
+%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
+%! c = textscan (str, fmt, "collectoutput", 1);
+%! assert (size (c{3}), [10, 2]);
+%! assert (size (c{2}), [10, 2]);
+
+## CollectOutput test with uneven column length files
+%!test
+%! b = [10:10:100];
+%! b = [b; 8*b/5; 8*b*1000/5];
+%! str = sprintf ("%g miles/hr = %g (%g) kilometers (meters)/hr\n", b);
+%! str = [str "110 miles/hr"];
+%! fmt = "%f miles%s %s %f (%f) kilometers %*s";
+%! c = textscan (str, fmt, "collectoutput", 1);
+%! assert (size (c{1}), [11, 1]);
+%! assert (size (c{3}), [11, 2]);
+%! assert (size (c{2}), [11, 2]);
+%! assert (c{3}(end), NaN);
+%! assert (c{2}{11, 1}, "/hr");
+%! assert (isempty (c{2}{11, 2}), true);
+
+## Double quoted string
+%!test
+%! str = 'First "the second called ""the middle""" third';
+%! fmt = "%q";
+%! c = textscan (str, fmt);
+%! assert (c{1}, {"First"; 'the second called "the middle"'; "third"});
+
+## Arbitrary character
+%!test
+%! c = textscan ("a first, \n second, third", "%s %c %11c", 'delimiter', ' ,');
+%! assert (c{1}, {"a"; "ond"});
+%! assert (c{2}, {"f"; "t"});
+%! assert (c{3}, {"irst, \n sec"; "hird"});
+
+## Field width and non-standard delimiters
+%!test
+%! str = "12;34;123456789;7";
+%! c = textscan (str, "%4d %4d", "delimiter", ";", "collectOutput", 1);
+%! assert (c, {[12 34; 1234 5678; 9 7]});
+
+## Field width and non-standard delimiters
+%!test
+%! str = "12;34;123456789;7";
+%! c = textscan (str, "%4f %f", "delimiter", ";", "collectOutput", 1);
+%! assert (c, {[12 34; 1234 56789; 7 NaN]});
+
+## Ignore trailing delimiter, but use leading one
+%!test
+%! str = "12.235e+2,34, \n12345.789-9876j,78\n,10|3";
+%! c = textscan (str, "%10.2f %f", "delimiter", ",", "collectOutput", 1,
+%! "expChars", "e|");
+%! assert (c, {[1223 34; 12345.78-9876j 78; NaN 10000]}, 1e-6);
+
+%!test
+%! ## Multi-character delimiter
+%! str = "99end2 space88gap 4564";
+%! c = textscan (str, "%d %s", "delimiter", {"end", "gap", "space"});
+%! assert (c{1}, int32([99; 88]));
+%! assert (c{2}, {"2 "; "4564"});
+
+### Delimiters as part of literals, and following literals
+#%!test
+#%! str = "12 R&D & 7";
+#%! c = textscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1, "EmptyValue", -99);
+#%! assert (c, {[12 -99; 7 -99]});
+#
+### Delimiters as part of literals, and before literals
+#%!test
+#%! str = "12 & R&D 7";
+#%! c = textscan (str, "%f R&D %f", "delimiter", "&", "collectOutput", 1);
+#%! assert (c, {[12 7]});
+
+%!test
+%! ## Check number of lines read, not number of passes through format string
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, "1\n2\n3\n4\n5\n6");
+%! fseek (fid, 0, "bof");
+%! c = textscan (fid, "%f %f", 2);
+%! fclose (fid);
+%! unlink (f);
+%! assert (c, {1, 2})
+
+%!test
+%! ## Check number of lines read, not number of passes through format string
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, "1\r\n2\r3\n4\r\n5\n6");
+%! fseek (fid, 0, "bof");
+%! c = textscan (fid, "%f %f", 4);
+%! fclose (fid);
+%! unlink (f);
+%! assert (c, {[1;3], [2;4]})
+
+%!test
+%! ## Check number of lines read, with multiple delimiters
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, "1-\r\n-2\r3-\n-4\r\n5\n6");
+%! fseek (fid, 0, "bof");
+%! c = textscan (fid, "%f %f", 4, "delimiter", "-", "multipleDelimsAsOne", 1);
+%! fclose (fid);
+%! unlink (f);
+%! assert (c, {[1;3], [2;4]})
+
+## Test input validation
+%!error textscan ()
+%!error textscan (single (40))
+%!error textscan ({40})
+%!error textscan ("Hello World", 2)
+#%!error [C, pos] = textscan ("Hello World")
+%!error textscan ("Hello World", '%s', 'EndOfLine', 3)
+%!error <'%z' is not a valid format specifier> textscan ("1.0", "%z");
+%!error textscan ("1.0", "foo");
+
+## Test incomplete first data line
+%! R = textscan (['Empty1' char(10)], 'Empty%d %f');
+%! assert (R{1}, int32 (1));
+%! assert (isempty (R{2}), true);
+
+## bug #37023
+%!test
+%! data = textscan(" 1. 1 \n 2 3\n", '%f %f');
+%! assert (data{1}, [1; 2], 1e-15);
+%! assert (data{2}, [1; 3], 1e-15);
+
+## Whitespace test (bug #37333) using delimiter ";"
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = "C:/code/meas;";
+%! tc{1, end+1} = " C:/code/sim;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = textscan (string, "%s", "delimiter", ";");
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333), adding multipleDelimsAsOne true arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = textscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 1);
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333), adding multipleDelimsAsOne false arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;;";
+%! tc{1, end+1} = "";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = textscan (string, "%s", "delimiter", ";", "multipleDelimsAsOne", 0);
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333) whitespace "" arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = textscan (string, "%s", "delimiter", ";", "whitespace", "");
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! assert (strcmp (lh, rh));
+%! end
+
+## Whitespace test (bug #37333), whitespace " " arg
+%!test
+%! tc = [];
+%! tc{1, 1} = "C:/code;";
+%! tc{1, end+1} = " C:/code/meas;";
+%! tc{1, end+1} = "C:/code/sim;";
+%! tc{1, end+1} = "C:/code/utils;";
+%! string = [tc{:}];
+%! c = textscan (string, "%s", "delimiter", ";", "whitespace", " ");
+%! for k = 1:max (numel (c{1}), numel (tc))
+%! lh = c{1}{k};
+%! rh = tc{k};
+%! rh(rh == ";") = "";
+%! rh = strtrim (rh);
+%! assert (strcmp (lh, rh));
+%! end
+
+%!test
+%!## Test unfinished comment
+%! c = textscan ("1 2 /* half comment", "%n %u8", "CommentStyle", {"/*", "*/});
+%! assert (c, {1, 2});
+
+%!test
+%!## Test start of comment as string
+%! c = textscan ("1 / 2 // 3", "%n %s %u8", "CommentStyle", {"//"});
+%! assert (c, {1, "/", 2});
+
+## Test reading from a real file
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! d = rand (1, 4);
+%! fprintf (fid, " %f %f /* comment */ %f %f ", d);
+%! fseek (fid, 0, "bof");
+%! A = textscan (fid, "%f %f", "CommentStyle", {"/*", "*/"});
+%! fclose (fid);
+%! unlink (f);
+%! assert (A{1}, [d(1); d(3)], 1e-6);
+%! assert (A{2}, [d(2); d(4)], 1e-6);
+
+## Tests reading with empty format, should return proper nr of columns
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, " 1 2 3 4\n5 6 7 8");
+%! fseek (fid, 0, "bof");
+%! A = textscan (fid, "");
+%! fclose (fid);
+%! unlink (f);
+%! assert (A{1}, [1 ; 5], 1e-6);
+%! assert (A{2}, [2 ; 6], 1e-6);
+%! assert (A{3}, [3 ; 7], 1e-6);
+%! assert (A{4}, [4 ; 8], 1e-6);
+
+## Tests reading with empty format; empty fields & incomplete lower row
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid, " ,2,,4\n5,6");
+%! fseek (fid, 0, "bof");
+%! A = textscan (fid, "", "delimiter", ",", "EmptyValue", 999, "CollectOutput" , 1);
+%! fclose (fid);
+%! unlink (f);
+%! assert (A{1}, [999, 2, 999, 4; 5, 6, 999, 999], 1e-6);
+
+## Error message tests
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! msg1 = "textscan: 1 parameters given, but only 0 values";
+%! try
+%! A = textscan (fid, "", "headerlines");
+%! end_try_catch;
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! msg1 = "textscan: HeaderLines must be numeric";
+%! try
+%! A = textscan (fid, "", "headerlines", "hh");
+%! end_try_catch;
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+%!test
+%! ## Skip headerlines
+%! A = textscan ("field 1 field2\n 1 2\n3 4", "", "headerlines", 1, "collectOutput", 1);
+%! assert (A, {[1 2; 3 4]});
+
+%!test
+%! ## Skip headerlines with non-default EOL
+%! A = textscan ("field 1 field2\r 1 2\r3 4", "", "headerlines", 2, "collectOutput", 1, "EndOfLine", '\r');
+%! assert (A, {[3 4]});
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid,"some_string");
+%! fseek (fid, 0, "bof");
+%! msg1 = "textscan: EndOfLine must be at most one character or '\\r\\n'";
+%! try
+%! A = textscan (fid, "%f", "EndOfLine", "\n\r");
+%! end_try_catch;
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+%!test
+%! f = tempname ();
+%! fid = fopen (f, "w+");
+%! fprintf (fid,"some_string");
+%! fseek (fid, 0, "bof");
+%! msg1 = "textscan: EndOfLine must be at most one character or '\\r\\n'";
+%! try
+%! A = textscan (fid, "%f", "EndOfLine", 33);
+%! end_try_catch;
+%! fclose (fid);
+%! unlink (f);
+%! assert (msg1, lasterr);
+
+## Bug #41824
+%!test
+%! assert (textscan ("123", "", "whitespace", " "){:}, 123);
+
+## Bug #42343-1, just test supplied emptyvalue
+%!test
+%! assert (textscan (",NaN", "", "delimiter", "," ,"emptyValue" ,Inf), {Inf, NaN});
+
+## Bug #42343-2, test padding with supplied emptyvalue
+%!test
+%! a = textscan (",1,,4\nInf, ,NaN", "", "delimiter", ",", "emptyvalue", -10);
+%! assert (cell2mat (a), [-10, 1, -10, 4; Inf, -10, NaN, -10]);
+
+## Bug #42528
+%!test
+%! assert (textscan ("1i", ""){1}, 0+1i);
+%! assert (cell2mat (textscan ("3, 2-4i, NaN\n -i, 1, 23.4+2.2i\n 1+1 1+1j", "", "delimiter", ",")), [3+0i, 2-4i, NaN+0i; 0-i, 1+0i, 23.4+2.2i; 1 1 1+1i]);
+
+
+## Needed tests:
+# 1. Check that it fails at an appropriate point
+# 3. Checks for %[...] and %[^...]
+# 4. Checks for file vs string
+# 5. Checks for mixing headerlines and comments
+# 7. Checks for ReturnOnError
+# 8. Checks for all types of invalid parameter
+# 9. Checks derived from searching the internet for "= textscan"
+
+## Check of what Matlab does:
+# 1. mixing headerlines and comments (just skip lines, even if comments)
+# 2. Empty EndOfLine (acceptable)
+# 3. Complex NaN (NaN+0i. However 1+NaNi gives 1+NaNi)
+# 4. delimiters at the start of lines with "multiple delimiters as one"
+ (delim merges with EOL. Without multiple...one, a trailing delimiter
+ before a newline is ignored)
+
+# 5. If N not specified, are parameters still checked?
+
+
+#ToDO:
+# 1. Common skip whitespace for everything except %[...] and %[^...]
+# 3. Do everything by reading an entire line into a string. Trouble if
+# %[...] contains end-of-line, %[^...] doesn't contain end-of-line, or
+# \r\n can't be handled, or the line is too long.
+# 4. Refactor octave_scan to separate integers from floating point?
+# 5. Fail when the first mismatch occurs. Tests for this
+# 6. Remove commented out code
+# 7. Document: [^...].
+# 8. Count newlines in [^...], [...], %c and %q.
+# 10. Treatment of missing final end-of-line -- unequal line lengths.
+# 11. String delimiter as multichar, not as list of delims. (Check Matlab.)