# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1481737655 -3600
#      Wed Dec 14 18:47:35 2016 +0100
# Node ID 62400a91c8146f7942661773eb9d818e85c48699
# Parent  953232868883f25765ce76143b021d1c3f9b9a45
Add functions native2unicode and unicode2native (bug #49842).

* native2unicode.m, unicode2native.m: Add new functions that wrap
__native2unicode__ and __unicode2native__ with input conversions and
checks.

* strfns.cc: Add DEFUNs __native2unicode__ and __unicode2native__.

* __unimplemented.m: Delete names from list of missing functions.

* strings.txi: Add doc string to manual.

diff -r 953232868883 -r 62400a91c814 doc/interpreter/strings.txi
--- a/doc/interpreter/strings.txi	Tue Dec 13 15:04:15 2016 -0500
+++ b/doc/interpreter/strings.txi	Wed Dec 14 18:47:35 2016 +0100
@@ -506,6 +506,10 @@
 
 @DOCSTRING(toupper)
 
+@DOCSTRING(unicode2native)
+
+@DOCSTRING(native2unicode)
+
 @DOCSTRING(do_string_escapes)
 
 @DOCSTRING(undo_string_escapes)
diff -r 953232868883 -r 62400a91c814 libinterp/corefcn/strfns.cc
--- a/libinterp/corefcn/strfns.cc	Tue Dec 13 15:04:15 2016 -0500
+++ b/libinterp/corefcn/strfns.cc	Wed Dec 14 18:47:35 2016 +0100
@@ -29,6 +29,9 @@
 #include <queue>
 #include <sstream>
 
+#include <unicode/unistr.h>
+#include <unicode/ucnv.h>
+
 #include "dMatrix.h"
 
 #include "Cell.h"
@@ -810,3 +813,96 @@
 %!error <list_in_columns: WIDTH must be an integer> list_in_columns (["abc", "def"], "a")
 */
 
+DEFUN (__native2unicode__, args, ,
+       doc: /* -*- texinfo -*-
+@deftypefn {} {@var{utf8_str} =} __native2unicode__ (@var{native_bytes}, @var{codepage})
+Convert byte stream @var{native_bytes} to UTF-8 using @var{codepage}.
+
+
+@seealso{native2unicode, __unicode2native__}
+@end deftypefn */)
+{
+  int nargin = args.length ();
+
+  if (nargin < 1 || nargin > 2)
+    print_usage ();
+
+  if (args(0).is_string ())
+    return ovl(args(0));
+
+  // first step: convert to UTF-16 (internal representation of UCI strings)
+  const char *codepage = "\0";
+  string_vector tmp; 
+  if (! args(1).is_numeric_type ())
+    {
+      tmp = args(1).string_vector_value ();
+      codepage = tmp(0).c_str ();
+    }
+  charNDArray t = args(0).char_array_value ();
+  t(t.numel ()) = 0; // 0-terminate
+  const char *char_array = t.fortran_vec ();
+
+  icu::UnicodeString unicode_str = icu::UnicodeString (char_array, codepage);
+  if (unicode_str.isBogus ())
+    error("%s: Error converting bytes from codepage '%s' to Unicode",
+          "native2unicode", codepage);
+
+  // second step: convert to UTF-8 (Octave's internal representation)
+  std::string utf8_string;
+  unicode_str.toUTF8String (utf8_string);
+  if (unicode_str.isBogus ())
+    error("%s: Error converting bytes from codepage '%s' to UTF-8",
+          "native2unicode", codepage);
+  utf8_string.append ("\0");
+
+  return ovl(utf8_string);
+}
+
+DEFUN (__unicode2native__, args, ,
+       doc: /* -*- texinfo -*-
+@deftypefn {} {@var{native_bytes} =} __unicode2native__ (@var{utf8_str}, @var{codepage})
+Convert UTF-8 string @var{utf8_str} to byte stream @var{native_bytes} using
+@var{codepage}.
+
+
+@seealso{unicode2native, __native2unicode__}
+@end deftypefn */)
+{
+  int nargin = args.length ();
+
+  if (nargin != 2)
+    print_usage ();
+
+  const char *codepage_str = NULL;
+  string_vector tmp; 
+  if (! args(1).is_numeric_type ())
+    {
+      tmp = args(1).string_vector_value ();
+      codepage_str = tmp(0).c_str ();
+    }
+
+  // Check whether codepage is valid identifier
+  UErrorCode u_err = U_ZERO_ERROR;
+  UConverter *conv = ucnv_open (codepage_str, &u_err);
+  if (u_err > U_ZERO_ERROR)
+    error("unicode2native: Codepage '%s' not available (error %d).",
+          codepage_str, u_err);
+
+
+  // first step: convert to UTF-16 (internal representation of ICU strings)
+  string_vector utf8_string = args(0).string_vector_value ();
+  icu::UnicodeString unicode_str =
+                        icu::UnicodeString::fromUTF8 (utf8_string[0]);
+  if (unicode_str.isBogus ())
+    error("unicode2native: Error converting from UTF-8 to Unicode");
+
+  // second step: convert to selected codepage
+  int l = unicode_str.extract (0, unicode_str.length (), NULL, 0,
+                               codepage_str);
+  charNDArray retval(dim_vector (1, l));
+  char *char_array = retval.fortran_vec ();
+  unicode_str.extract (0, unicode_str.length (), char_array, l, codepage_str);
+
+  ucnv_close(conv);
+  return ovl (NDArray (retval));
+}
diff -r 953232868883 -r 62400a91c814 scripts/help/__unimplemented__.m
--- a/scripts/help/__unimplemented__.m	Tue Dec 13 15:04:15 2016 -0500
+++ b/scripts/help/__unimplemented__.m	Wed Dec 14 18:47:35 2016 +0100
@@ -735,7 +735,6 @@
   "multibandread",
   "multibandwrite",
   "NaT",
-  "native2unicode",
   "nccreate",
   "ncdisp",
   "ncinfo",
@@ -849,7 +848,6 @@
   "uitoolbar",
   "uiwait",
   "undocheckout",
-  "unicode2native",
   "unloadlibrary",
   "unmesh",
   "unstack",
diff -r 953232868883 -r 62400a91c814 scripts/strings/module.mk
--- a/scripts/strings/module.mk	Tue Dec 13 15:04:15 2016 -0500
+++ b/scripts/strings/module.mk	Wed Dec 14 18:47:35 2016 +0100
@@ -15,6 +15,7 @@
   scripts/strings/isletter.m \
   scripts/strings/isstrprop.m \
   scripts/strings/mat2str.m \
+  scripts/strings/native2unicode.m \
   scripts/strings/ostrsplit.m \
   scripts/strings/regexptranslate.m \
   scripts/strings/rindex.m \
@@ -29,6 +30,7 @@
   scripts/strings/strtrim.m \
   scripts/strings/strtrunc.m \
   scripts/strings/substr.m \
+  scripts/strings/unicode2native.m \
   scripts/strings/untabify.m \
   scripts/strings/validatestring.m
 
diff -r 953232868883 -r 62400a91c814 scripts/strings/native2unicode.m
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scripts/strings/native2unicode.m	Wed Dec 14 18:47:35 2016 +0100
@@ -0,0 +1,106 @@
+## Copyright (C) 2016 Markus Mützel
+##
+## This file is part of Octave.
+##
+## Octave is free software; you can redistribute it and/or modify it
+## under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## Octave is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Octave; see the file COPYING.  If not, see
+## <http://www.gnu.org/licenses/>.
+
+## -*- texinfo -*-
+## @deftypefn  {} {@var{utf8_str} =} native2unicode (@var{native_bytes}, @var{codepage})
+## @deftypefnx {} {@var{utf8_str} =} native2unicode (@var{native_bytes})
+## Convert byte stream @var{native_bytes} to UTF-8 using @var{codepage}.
+##
+## The numbers in the vector @var{native_bytes} are rounded and clipped to
+## integers between 0 and 255.  This byte stream is then mapped into the
+## codepage given by the string @var{codepage} and returned in the string
+## @var{utf8_str}.  Octave uses UTF-8 as its internal encoding.
+## The string @var{codepage} must be an identifier of a valid codepage.
+## Examples for valid codepages are "ISO 8859-1", "Latin-1" or "Shift-JIS".
+## If @var{codepage} is omitted or empty, the system default codepage is used.
+##
+## If @var{native_bytes} is a string vector, it is returned as is.
+##
+## @seealso{unicode2native}
+## @end deftypefn
+
+function utf8_str = native2unicode (native_bytes, codepage)
+
+  ## check input
+  if (nargin < 1 || nargin > 2)
+    print_usage ();
+  endif
+
+  if (ischar (native_bytes))
+    utf8_str = native_bytes;
+    return
+  endif
+
+  if (! isnumeric (native_bytes) || ! isvector (native_bytes))
+    error ("native2unicode: NATIVE_BYTES must be a numeric vector.")
+  endif
+
+  is_column = false;
+  if (! isrow (native_bytes))
+    is_column = true;
+    native_bytes = native_bytes';
+  endif
+
+  if (nargin < 2 || isempty (codepage))
+    codepage = 0;
+  endif
+
+  if (! ischar (codepage) && codepage != 0)
+    error ("native2unicode: CODEPAGE must be a string or 0.")
+  endif
+
+  native_bytes = round (native_bytes);
+  native_bytes(native_bytes < 0) = 0;
+  native_bytes(native_bytes > 255) = 255;
+
+  is_null = find (native_bytes == 0);
+  if (isempty (is_null))
+    ## pass to internal function
+    utf8_str = __native2unicode__ (native_bytes, codepage);
+  else
+    ## pass substrings to internal function
+    utf8_str = "";
+    i_start = 1;
+    for (i_null = 1:length (is_null))
+      utf8_substr = __native2unicode__ (native_bytes(i_start:(is_null(i_null)-1)), codepage);
+      utf8_str = [utf8_str utf8_substr char(0)];
+      i_start = is_null(i_null) + 1;
+    endfor
+    if (i_start != length (native_bytes)+1)
+      utf8_substr = __native2unicode__ (native_bytes(i_start:end), codepage);
+      utf8_str = [utf8_str utf8_substr];
+    endif
+  endif
+
+  if (is_column)
+    utf8_str = utf8_str';
+  endif
+
+endfunction
+
+%!assert (double (native2unicode (164:170, 'ISO 8859-5')),
+%!        [208 132 208 133 208 134 208 135 208 136 208 137 208 138]);  # "ЄЅІЇЈЉЊ"
+%!assert (native2unicode ("foobar"), "foobar");
+%!assert (double (native2unicode ([0 0 120.3 0 0 122.6 0 0])), [0 0 120 0 0 123 0 0]);
+%!error <NATIVE_BYTES must be a numeric vector> native2unicode ([1 2; 3 4]);
+%!error <NATIVE_BYTES must be a numeric vector> native2unicode ({1 2 3 4});
+%!error <CODEPAGE must be a string or 0> native2unicode (164:170, 123);
+%!error <Error converting bytes from codepage 'foo' to Unicode> native2unicode (234, "foo");
+%!error <Invalid call> native2unicode ();
+%!error <Invalid call> native2unicode (1, "Latin-1", "test");
+ 
\ No newline at end of file
diff -r 953232868883 -r 62400a91c814 scripts/strings/unicode2native.m
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scripts/strings/unicode2native.m	Wed Dec 14 18:47:35 2016 +0100
@@ -0,0 +1,95 @@
+## Copyright (C) 2016 Markus Mützel
+##
+## This file is part of Octave.
+##
+## Octave is free software; you can redistribute it and/or modify it
+## under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## Octave is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Octave; see the file COPYING.  If not, see
+## <http://www.gnu.org/licenses/>.
+
+## -*- texinfo -*-
+## @deftypefn  {} {@var{native_bytes} =} unicode2native (@var{utf8_str}, @var{codepage})
+## @deftypefnx {} {@var{native_bytes} =} unicode2native (@var{utf8_str})
+## Convert UTF-8 string @var{utf8_str} to byte stream using @var{codepage}.
+##
+## The character vector @var{utf8_str} is converted to a byte stream
+## @var{native_bytes} using the codepage given by @var{codepage}.
+## The string @var{codepage} must be an identifier of a valid codepage.
+## Examples for valid codepages are "ISO 8859-1", "Latin-1" or "Shift-JIS".
+## If @var{codepage} is omitted or empty, the system default codepage is used.
+##
+## If any of the characters cannot be mapped into the codepage @var{codepage},
+## they are replaced with the appropriate substitution sequence for that
+## codepage.
+##
+## @seealso{native2unicode}
+## @end deftypefn
+
+function native_bytes = unicode2native (utf8_str, codepage)
+
+  ## check input
+  if (nargin < 1 || nargin > 2)
+    print_usage ();
+  endif
+
+
+  if (! ischar (utf8_str) || ! isvector (utf8_str))
+    error ("unicode2native: UTF8_STR must be a character vector.")
+  endif
+
+  is_column = false;
+  if (! isrow (utf8_str))
+    is_column = true;
+    utf8_str = utf8_str';
+  endif
+
+  if (nargin < 2 || isempty (codepage))
+    codepage = 0;
+  endif
+
+  if (! ischar (codepage) && codepage != 0)
+    error ("unicode2native: CODEPAGE must be a string or 0.")
+  endif
+
+  is_null = find (utf8_str == 0);
+  if (isempty (is_null))
+    ## pass to internal function
+    native_bytes = __unicode2native__ (utf8_str, codepage);
+  else
+    ## pass substrings to internal function
+    native_bytes = [];
+    i_start = 1;
+    for (i_null = 1:length (is_null))
+      native_subbytes = __unicode2native__ (utf8_str(i_start:(is_null(i_null)-1)), codepage);
+      native_bytes = [native_bytes native_subbytes 0];
+      i_start = is_null(i_null) + 1;
+    endfor
+    if (i_start != length (native_bytes)+1)
+      native_subbytes = __unicode2native__ (utf8_str(i_start:end), codepage);
+      native_bytes = [native_bytes native_subbytes];
+    endif
+  endif
+
+  if (is_column)
+    native_bytes = native_bytes';
+  endif
+
+endfunction
+
+%!assert (unicode2native ("ЄЅІЇЈЉЊ", "ISO 8859-5"), 164:170);
+%!error <UTF8_STR must be a character vector> unicode2native (["ab"; "cd"]);
+%!error <UTF8_STR must be a character vector> unicode2native ({1 2 3 4});
+%!error <CODEPAGE must be a string or 0> unicode2native ("ЄЅІЇЈЉЊ", 123);
+%!error <Codepage 'foo' not available> unicode2native ("a", "foo");
+%!error <Invalid call> unicode2native ();
+%!error <Invalid call> unicode2native ("a", "Latin-1", "test");
+ 
\ No newline at end of file
