# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1601582805 -7200
#      Thu Oct 01 22:06:45 2020 +0200
# Node ID 03bf8153a7ae42a9cd2a43fd31280aa9d84ac100
# Parent  ba25187de84a57436925b9f292ebf88fe305267c
Try to read and write Unicode characters with COM interface (bug #59203).

* xls2oct.m, oct2xls.m: Remove logic that converts encoding from here. It
  should be handled for each interface as needed.
* __COM_spsh2oct__.m: The COM interface in "of-windows" until version 1.5.0
  reads strings in the system locale encoding. Convert it to UTF-8. Fall back
  to conversion from ISO 8859-1 (Latin-1) if the locale is unknown.
* __COM_oct2spsh__.m: The COM interface in "of-windows" until version 1.5.0
  expected strings to be encoded in the system locale encoding. Try to convert
  to that encoding. Fall back to conversion to ISO 8859-1 (Latin-1) if the
  locale is unknown.
* utf82unicode.m, unicode2utf8.m: Clarify docstring.
* tidyxml.m: Remove file.

diff -r ba25187de84a -r 03bf8153a7ae inst/oct2xls.m
--- a/inst/oct2xls.m	Sun Oct 04 15:58:05 2020 +0200
+++ b/inst/oct2xls.m	Thu Oct 01 22:06:45 2020 +0200
@@ -194,16 +194,6 @@
     printf ("oct2xls: warning: no output spreadsheet file pointer specified.\n");
   endif
 
-  ## Convert to UTF-8
-  if (! strcmp (xls.xtype, "COM") && (spsh_opts.convert_utf))
-    if (exist ("native2unicode", "file"))
-      conv_fcn = @(str) unicode2native (native2unicode (uint8 (str)), "UTF-8");
-    else
-      conv_fcn = @unicode2utf8;
-    endif
-    obj = tidyxml (obj, conv_fcn);
-  endif
-  
   ## Select interface to be used
   switch upper (xls.xtype)
     case, "COM"
diff -r ba25187de84a -r 03bf8153a7ae inst/private/__COM_oct2spsh__.m
--- a/inst/private/__COM_oct2spsh__.m	Sun Oct 04 15:58:05 2020 +0200
+++ b/inst/private/__COM_oct2spsh__.m	Thu Oct 01 22:06:45 2020 +0200
@@ -197,6 +197,28 @@
     xls.changed = 2;      ## 3 => 2
   endif
 
+  if (spsh_opts.convert_utf) ...
+      && (compare_versions (ver ("windows").Version, "1.5.0", "<="))
+    ## The COM interface in of-windows until version 1.5.0 expected strings
+    ## that were encoded in the system locale.  Not all Unicode characters can
+    ## be encoded in a non-Unicode locale.  Try to convert as much as possible.
+    if (isempty (which ("unicode2native")))
+      ## Not available before Octave 4.4
+      ## Use fallback function. (Assumes the locale is ISO 8859-1.)
+      conv_fcn = @utf82unicode;
+    else
+      if (isempty (which ("__locale_charset__")))
+        ## Not available before Octave 6
+        ## Assume ISO 8859-1
+        enc = "iso8859-1";
+      else
+        enc = __locale_charset__ ();
+      endif
+      conv_fcn = @(str) char (unicode2native (str, enc));
+    endif
+    obj = cellfun (conv_fcn, obj, "uniformoutput", false);
+  endif
+
   ## MG's original part.
   ## Save object in Excel sheet, starting at cell top_left_cell
   if (! isempty(obj))
diff -r ba25187de84a -r 03bf8153a7ae inst/private/__COM_spsh2oct__.m
--- a/inst/private/__COM_spsh2oct__.m	Sun Oct 04 15:58:05 2020 +0200
+++ b/inst/private/__COM_spsh2oct__.m	Thu Oct 01 22:06:45 2020 +0200
@@ -130,6 +130,31 @@
       rawarr = {rawarr};
     endif
 
+    if (spsh_opts.convert_utf) ...
+        && (compare_versions (ver ("windows").Version, "1.5.0", "<="))
+      ## The COM interface in of-windows until version 1.5.0 returns strings
+      ## that are encoded in the system locale.  All character values not
+      ## contained in the locale encoding are mapped to codepoint 32 (space
+      ## character).  Try to recover the remainder.
+      if (isempty (which ("native2unicode")))
+        ## Not available before Octave 4.4
+        ## Use fallback function. (Assumes the locale is ISO 8859-1.)
+        conv_fcn = @unicode2utf8;
+      else
+        if (isempty (which ("__locale_charset__")))
+          ## Not available before Octave 6
+          ## Assume ISO 8859-1 (Latin-1)
+          enc = "iso8859-1";
+        else
+          enc = __locale_charset__ ();
+        endif
+        conv_fcn = @(str) native2unicode (uint8 (str), enc);
+      endif
+
+      idx = cellfun (@ischar, rawarr);
+      rawarr(idx) = cellfun (conv_fcn, rawarr(idx), "uniformoutput", false);
+    endif
+    
     ## If we get here, all seems to have gone OK
     rstatus = 1;
     ## Keep track of data rectangle limits
diff -r ba25187de84a -r 03bf8153a7ae inst/tidyxml.m
--- a/inst/tidyxml.m	Sun Oct 04 15:58:05 2020 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,64 +0,0 @@
-## Copyright (C) 2016-2020 Philip Nienhuis
-##
-## This program is free software; you can redistribute it and/or modify it
-## under the terms of the GNU General Public License as published by
-## the Free Software Foundation; either version 3 of the License, or
-## (at your option) any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-## -*- texinfo -*-
-## @deftypefn {} {@var{ostr} =} tidyxml (@var{istr}, @var{conv_fcn})
-## Optionally convert character using the function handle in @var{conv_fcn},
-## remove characters (<32 >255) from text string or cell array @var{istr}
-## and return the result in @var{ostr}.
-##
-## tidyxml is useful for converting strings in XML that have been partly
-## or wholly encoded as double-byte characters.  Such strings occur when
-## dealing with a.o., spreadsheet programs reading/writing from/to
-## XML-based formats and cannot be processed by Octave as Octave doesn't
-## support unicode.  For (optionally: nested) nested cell arrays tidyxml
-## is called recursively and only processes cells containing text strings.
-## @end deftypefn
-
-## Author: Philip Nienhuis <prnienhuis@users.sf.net>
-## Created: 2016-01-13
-
-function [ostr] = tidyxml (istr="", conv_fcn=[])
-
-  if (iscell (istr))
-    idx = find (cellfun (@ischar, istr));
-    ostr = istr;
-    ostr(idx) = cellfun (@(instr) tidyxml (instr, conv_fcn), istr(idx), "uni", 0);
-  elseif (! ischar (istr))
-    print_usage ();
-  elseif (isempty (istr))
-    ostr = "";
-  else
-    if (isempty (conv_fcn))
-      ustr = uint8 (istr);
-    else
-      if (nargout (conv_fcn) > 1)
-        [ustr, error_flag] = conv_fcn (istr);
-        if (error_flag)
-          warning ("Encoding conversion failed; some characters might be lost");
-        endif
-      else
-        try
-          ustr = conv_fcn (istr);
-        catch err
-          warning ("Encoding conversion failed; some characters might be lost");
-          ustr = istr;
-        end_try_catch
-      endif
-    endif
-    ostr = char (ustr(ustr > 31 & ustr < 256));
-  endif
-
-endfunction
diff -r ba25187de84a -r 03bf8153a7ae inst/unicode2utf8.m
--- a/inst/unicode2utf8.m	Sun Oct 04 15:58:05 2020 +0200
+++ b/inst/unicode2utf8.m	Thu Oct 01 22:06:45 2020 +0200
@@ -15,10 +15,12 @@
 
 ## -*- texinfo -*- 
 ## @deftypefn {} {[@var{ostr}, @var{error_flag}] =} unicode2utf8 (@var{ustr})
-## Encode (1-byte) Unicode string @var{ustr} into UTF-8 @var{ostr}.
+## Convert from Unicode code points @var{ustr} to UTF-8 encoded string @var{ostr}.
 ##
-## UTF-8 characters with more than 2 bytes are dropped since Octave does not
-## support characters >255.
+## It is not possible to convert Unicode characters with a codepoint higher
+## than 255 because Octave's @chode{char} type is 8-bit wide.  This means that
+## only the ISO 8859-1 (Latin-1) subset of Unicode can be mapped correctly.
+##
 ## If an error occured @var{error_flag} is set to true.
 ## @end deftypefn
 
@@ -44,3 +46,5 @@
       error_flag = true;
     endif
   endfor
+
+endfunction
diff -r ba25187de84a -r 03bf8153a7ae inst/utf82unicode.m
--- a/inst/utf82unicode.m	Sun Oct 04 15:58:05 2020 +0200
+++ b/inst/utf82unicode.m	Thu Oct 01 22:06:45 2020 +0200
@@ -15,10 +15,12 @@
 
 ## -*- texinfo -*- 
 ## @deftypefn {} {[@var{ustr}, @var{error_flag}] =} utf82unicode (@var{istr})
-## Convert UTF-8 encoded strings @var{istr} to (1-byte) Unicode @var{ustr}.
+## Convert from UTF-8 encoded strings @var{istr} to Unicode codepoints @var{ustr}.
 ##
 ## UTF-8 characters with more than 2 bytes are dropped since Octave does not
-## support characters >255.
+## support characters >255.  This means that only the ISO 8859-1 (Latin-1)
+## subset of Unicode can be mapped correctly.
+##
 ## If an error occured @var{error_flag} is set to true.
 ## @end deftypefn
 
@@ -63,3 +65,5 @@
       break
     endif
   endwhile
+
+endfunction
diff -r ba25187de84a -r 03bf8153a7ae inst/xls2oct.m
--- a/inst/xls2oct.m	Sun Oct 04 15:58:05 2020 +0200
+++ b/inst/xls2oct.m	Thu Oct 01 22:06:45 2020 +0200
@@ -157,18 +157,18 @@
   if (nargin < 4 || isempty (spsh_opts))
     spsh_opts.formulas_as_text = 0;
     spsh_opts.strip_array = 1;
-    spsh_opts.convert_utf = 0; 
+    spsh_opts.convert_utf = 0;
     ## Future options:
 
   elseif (isstruct (spsh_opts))
     if (! isfield (spsh_opts, "formulas_as_text"))
-      spsh_opts.formulas_as_text = 0; 
+      spsh_opts.formulas_as_text = 0;
     endif
     if (! isfield (spsh_opts, "strip_array"))
-      spsh_opts.strip_array = 1; 
+      spsh_opts.strip_array = 1;
     endif
     if (! isfield (spsh_opts, "convert_utf"))
-      spsh_opts.convert_utf = 0; 
+      spsh_opts.convert_utf = 0;
     endif
     ## Future options:
 
@@ -235,17 +235,6 @@
       error (sprintf ("xls2oct: unknown Excel .xls interface - %s.\n", xls.xtype));
   endswitch
 
-  ## Convert from UTF-8 and strip characters that are not supported by Octave
-  ## (any chars < 32 or > 255).
-  if (! strcmp (xls.xtype, "COM") && (spsh_opts.convert_utf))
-    if (exist ("native2unicode", "file"))
-      conv_fcn = @(str) unicode2native (native2unicode (str, "UTF-8"));
-    else
-      conv_fcn = @utf82unicode;
-    endif
-    rawarr = tidyxml (rawarr, conv_fcn);
-  endif
-
   ## Optionally strip empty outer rows and columns & keep track of original data location
   if (spsh_opts.strip_array)
     emptr = cellfun ("isempty", rawarr);
