# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1476343465 -7200
#      Thu Oct 13 09:24:25 2016 +0200
# Node ID 5dba058baa25886ff6150dc0f272937938e79553
# Parent  cddd085dbf8c4bc60479e08c06af22b53ee75a64
Add functions for conversion of UTF-8 to Unicode (up to char(255)) and back (bug #49222)

diff -r cddd085dbf8c -r 5dba058baa25 inst/oct2xls.m
--- a/inst/oct2xls.m	Wed Oct 12 21:31:15 2016 +0200
+++ b/inst/oct2xls.m	Thu Oct 13 09:24:25 2016 +0200
@@ -179,7 +179,7 @@
   elseif (strcmpi (xls.xtype, "OCT"))
     ## Clean up obj from characters that'll be converted to double-byte
     ## by Excel/LibreOffice (any chars < 32 or > 126)
-    obj = tidyxml (obj);
+    obj = tidyxml (obj, @unicode2utf8);
     ## Replace illegal XML characters by XML escape sequences
     idx = cellfun (@ischar, obj);
     obj(idx) = strrep (obj(idx), "&", "&amp;" );
diff -r cddd085dbf8c -r 5dba058baa25 inst/tidyxml.m
--- a/inst/tidyxml.m	Wed Oct 12 21:31:15 2016 +0200
+++ b/inst/tidyxml.m	Thu Oct 13 09:24:25 2016 +0200
@@ -14,9 +14,10 @@
 ## along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 ## -*- texinfo -*- 
-## @deftypefn {} {@var{ostr} =} tidyxml (@var{istr})
-## Remove characters (<32 >126) from text string or cell array @var{istr}
-## and return the result in @var{ostr}.
+## @deftypefn {} {@var{ostr} =} tidyxml (@var{istr}, @var{conv_fcn})
+## Convert character using the function handle in @var[conv_fcn}, remove
+## characters (<32 >256) from text string or cell array @var{istr} and return
+## the result in @var{ostr}.
 ##
 ## tidyxml is useful for cleaning up strings in xml that have been partly
 ## encoded as double-byte characters.  Such strings occur when dealing
@@ -29,45 +30,23 @@
 ## Author: Philip Nienhuis <prnienhuis@users.sf.net>
 ## Created: 2016-01-13
 
-function [ostr] = tidyxml (istr="")
+function [ostr] = tidyxml (istr="", conv_fcn=[])
 
   if (iscell (istr))
     idx = find (cellfun (@ischar, istr));
     ostr = istr;
-    ostr(idx) = cellfun (@tidyxml, istr(idx), "uni", 0);
+    ostr(idx) = cellfun (@(instr) tidyxml (instr, conv_fcn), istr(idx), "uni", 0);
   elseif (! ischar (istr))
     print_usage ();
   else
-    istr = uint8 (istr);
-    ibyte = 1;
-    ustr = uint8 ([]);
-    while (true)
-      if (isequal (bitget (istr(ibyte), 8), 0))
-        ## Single byte character
-        ustr(end+1) = istr(ibyte);
-        ibyte += 1;
-      elseif (isequal (bitget (istr(ibyte), 6:8), [0 1 1]))
-        ## Start of double-byte char
-        if (isequal (bitget (istr(ibyte+1), 7:8), [0 1]))
-          % Decode byte if it is valid UTF-8
-          ustr(end+1) = bitand(31, istr(ibyte))*64 + bitand(63, istr(ibyte+1));
-        endif
-        ibyte += 2;
-      elseif (isequal (bitget (istr(ibyte), 6:8), [1 1 1]))
-        ## Drop this character (Octave does not support chars > 255).
-        ## Detect how many bytes to drop
-        ibyte += find (bitget(istr(ibyte), 8:-1:1) == 0, 1, 'first') - 1;
-      elseif (isequal (bitget (istr(ibyte), 7:8), [0 1]))
-        ## Drop this character (must follow a start byte)
-        ibyte += 1;
-      else
-        ## Should not reach here but maybe for safety?
-        ibyte += 1;
+    if (isempty (conv_fcn))
+      ustr = uint8 (istr);
+    else
+      [ustr, error_flag] = conv_fcn (istr);
+      if (error_flag)
+        warning ("conversion failed some characters might be lost");
       endif
-      if (ibyte > numel (istr))
-        break
-      endif
-    endwhile
+    endif
     ostr = char (ustr(ustr > 31 & ustr < 256));
   endif
 
diff -r cddd085dbf8c -r 5dba058baa25 inst/unicode2utf8.m
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/inst/unicode2utf8.m	Thu Oct 13 09:24:25 2016 +0200
@@ -0,0 +1,46 @@
+## Copyright (C) 2016 Markus Mützel
+## 
+## This program is free software; you can redistribute it and/or modify it
+## under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+## 
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+## 
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+## -*- texinfo -*- 
+## @deftypefn {} {[@var{ostr}, @var{error_flag}] =} unicode2utf8 (@var{ustr})
+## Encode Unicode string @var{ustr} in UTF-8 @var{ostr}.
+## If an error occured @var{error_flag} is set to true.
+##
+## UTF-8 characters with more than 2 bytes are dropped since Octave does not
+## support characters >255.
+
+## Author: Markus Mützel <markus.muetzel@gmx.de>
+## Created: 2016-10-12
+
+function [ostr, error_flag] = unicode2utf8 (ustr="")
+
+ustr = uint8 (ustr); # convert char to uint8
+ichar = 1;
+ostr = uint8 ([]);
+for (ichar = 1:numel (ustr))
+  if (ustr(ichar) < 128)
+    ## single byte character
+    ostr(end+1) = ustr(ichar);
+  elseif (ustr(ichar) < 2048)
+    ## double byte character
+    ostr(end+1) = 192 + bitshift (bitand (192, ustr(ichar)), -6);
+    ostr(end+1) = 128 + bitand (63, ustr(ichar));
+  else
+    ## Should never reach here because ustr is cast to uint8
+    error_flag = true;
+  endif
+endfor
+
+endfunction
diff -r cddd085dbf8c -r 5dba058baa25 inst/utf82unicode.m
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/inst/utf82unicode.m	Thu Oct 13 09:24:25 2016 +0200
@@ -0,0 +1,66 @@
+## Copyright (C) 2016 Markus Mützel
+## 
+## This program is free software; you can redistribute it and/or modify it
+## under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+## 
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+## 
+## You should have received a copy of the GNU General Public License
+## along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+## -*- texinfo -*- 
+## @deftypefn {} {[@var{ustr}, @var{error_flag}] =} utf82unicode (@var{istr})
+## Convert UTF-8 encoded strings @var{istr} to Unicode @var{ustr}.
+## If an error occured @var{error_flag} is set to true.
+##
+## UTF-8 characters with more than 2 bytes are dropped since Octave does not
+## support characters >255.
+
+## Author: Markus Mützel <markus.muetzel@gmx.de>
+## Created: 2016-10-12
+
+function [ustr, error_flag] = utf82unicode (istr="")
+
+error_flag = false;
+istr = uint8 (istr);
+ibyte = 1;
+ustr = uint8 ([]);
+while (true)
+  if (isequal (bitget (istr(ibyte), 8), 0))
+    ## Single byte character
+    ustr(end+1) = istr(ibyte);
+    ibyte += 1;
+  elseif (isequal (bitget (istr(ibyte), 6:8), [0 1 1]))
+    ## Start of double-byte char
+    if (isequal (bitget (istr(ibyte+1), 7:8), [0 1]))
+      ## Decode byte if it is valid UTF-8
+      ustr(end+1) = bitand (31, istr(ibyte))*64 + bitand (63, istr(ibyte+1));
+    else
+      error_flag = true;
+    endif
+    ibyte += 2;
+  elseif (isequal (bitget (istr(ibyte), 6:8), [1 1 1]))
+    ## Drop this character (Octave does not support chars > 255).
+    error_flag = true;
+    ## Detect how many bytes to drop
+    ibyte += find (bitget (istr(ibyte), 8:-1:1) == 0, 1, 'first') - 1;
+  elseif (isequal (bitget (istr(ibyte), 7:8), [0 1]))
+    ## Drop this character (must follow a start byte)
+    error_flag = true;
+    ibyte += 1;
+  else
+    ## Should not reach here but maybe for safety?
+    error_flag = true;
+    ibyte += 1;
+  endif
+  if (ibyte > numel (istr))
+    break
+  endif
+endwhile
+
+endfunction
diff -r cddd085dbf8c -r 5dba058baa25 inst/xls2oct.m
--- a/inst/xls2oct.m	Wed Oct 12 21:31:15 2016 +0200
+++ b/inst/xls2oct.m	Thu Oct 13 09:24:25 2016 +0200
@@ -195,7 +195,7 @@
       error ("xls2oct.m: file format not supported for OCT interface\n");
     endif
     ## Clean up rawarr from double-byte characters introduced by Excel/LibreOffice
-    rawarr = tidyxml (rawarr);
+    rawarr = tidyxml (rawarr, @utf82unicode);
     ## Replace XML escape sequences by regular characters
     idx = cellfun (@ischar, rawarr);
     rawarr(idx) = strrep (rawarr(idx), "&amp;",  "&");
