# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1601582805 -7200
#      Thu Oct 01 22:06:45 2020 +0200
# Node ID b82e04b900bcf548d92022d69ea1179d61e64d11
# Parent  98f9112537657957f03518dd57a139aa672697ff
Fix reading non-ASCII Latin-1 characters with COM interface (bug #59203).

* xls2oct.m: Default to convert_utf. The COM interface reads Latin-1(?); convert
  it to UTF-8. The OCT interface already reads UTF-8; don't break it with
  manipulations.
* utf82unicode.m: Delete file.

diff -r 98f911253765 -r b82e04b900bc inst/utf82unicode.m
--- a/inst/utf82unicode.m	Fri Sep 04 22:41:33 2020 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,65 +0,0 @@
-## Copyright (C) 2018-2020 Markus Mützel
-## 
-## This program is free software; you can redistribute it and/or modify it
-## under the terms of the GNU General Public License as published by
-## the Free Software Foundation; either version 3 of the License, or
-## (at your option) any later version.
-## 
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-## 
-## You should have received a copy of the GNU General Public License
-## along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-## -*- texinfo -*- 
-## @deftypefn {} {[@var{ustr}, @var{error_flag}] =} utf82unicode (@var{istr})
-## Convert UTF-8 encoded strings @var{istr} to (1-byte) Unicode @var{ustr}.
-##
-## UTF-8 characters with more than 2 bytes are dropped since Octave does not
-## support characters >255.
-## If an error occured @var{error_flag} is set to true.
-## @end deftypefn
-
-## Author: Markus Mützel <markus.muetzel@gmx.de>
-## Created: 2016-10-12
-
-function [ustr, error_flag] = utf82unicode (istr="")
-
-  error_flag = false;
-  istr = uint8 (istr);
-  ibyte = 1;
-  ustr = uint8 ([]);
-  while (true)
-    if (isequal (bitget (istr(ibyte), 8), 0))
-      ## Single byte character
-      ustr(end+1) = istr(ibyte);
-      ibyte += 1;
-    elseif (isequal (bitget (istr(ibyte), 6:8), [0 1 1]))
-      ## Start of double-byte char
-      if (isequal (bitget (istr(ibyte+1), 7:8), [0 1]))
-        ## Decode byte if it is valid UTF-8
-        ustr(end+1) = bitand (31, istr(ibyte))*64 + bitand (63, istr(ibyte+1));
-      else
-        error_flag = true;
-      endif
-      ibyte += 2;
-    elseif (isequal (bitget (istr(ibyte), 6:8), [1 1 1]))
-      ## Drop this character (Octave does not support chars > 255).
-      error_flag = true;
-      ## Detect how many bytes to drop
-      ibyte += find (bitget (istr(ibyte), 8:-1:1) == 0, 1, "first") - 1;
-    elseif (isequal (bitget (istr(ibyte), 7:8), [0 1]))
-      ## Drop this character (must follow a start byte)
-      error_flag = true;
-      ibyte += 1;
-    else
-      ## Should not reach here but maybe for safety?
-      error_flag = true;
-      ibyte += 1;
-    endif
-    if (ibyte > numel (istr))
-      break
-    endif
-  endwhile
diff -r 98f911253765 -r b82e04b900bc inst/xls2oct.m
--- a/inst/xls2oct.m	Fri Sep 04 22:41:33 2020 +0200
+++ b/inst/xls2oct.m	Thu Oct 01 22:06:45 2020 +0200
@@ -157,7 +157,7 @@
   if (nargin < 4 || isempty (spsh_opts))
     spsh_opts.formulas_as_text = 0;
     spsh_opts.strip_array = 1;
-    spsh_opts.convert_utf = 0; 
+    spsh_opts.convert_utf = 1; 
     ## Future options:
 
   elseif (isstruct (spsh_opts))
@@ -168,7 +168,7 @@
       spsh_opts.strip_array = 1; 
     endif
     if (! isfield (spsh_opts, "convert_utf"))
-      spsh_opts.convert_utf = 0; 
+      spsh_opts.convert_utf = 1; 
     endif
     ## Future options:
 
@@ -237,13 +237,12 @@
 
   ## Convert from UTF-8 and strip characters that are not supported by Octave
   ## (any chars < 32 or > 255).
-  if (! strcmp (xls.xtype, "COM") && (spsh_opts.convert_utf))
-    if (exist ("native2unicode", "file"))
-      conv_fcn = @(str) unicode2native (native2unicode (str, "UTF-8"));
-    else
-      conv_fcn = @utf82unicode;
-    endif
-    rawarr = tidyxml (rawarr, conv_fcn);
+  if (strcmp (xls.xtype, "COM") && spsh_opts.convert_utf && ...
+      ! isempty (which ("native2unicode")))
+    ## FIXME: The COM interface seems to map all character values outside the
+    ##        range of 0-255 to 32.  So higher Unicode characters are lost.
+    conv_fcn = @(str) native2unicode (uint8 (str), "iso8859-1");
+    rawarr = cellfun (conv_fcn, rawarr, "uniformoutput", false);
   endif
 
   ## Optionally strip empty outer rows and columns & keep track of original data location
