# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1680361108 -7200
#      Sat Apr 01 16:58:28 2023 +0200
# Node ID 152d13019189cbca9dcaca93c3a37984ffffa88f
# Parent  476c84c4f2010dce42635eb9bc85605fe7494d49
Try to gather complete UTF-8 surrogates when converting encoding (bug #63930).

* liboctave/util/oct-string.cc (octave::string::codecvt_u8::do_out): Check if
end of buffer is a complete UTF-8 surrogate.

diff -r 476c84c4f201 -r 152d13019189 liboctave/util/oct-string.cc
--- a/liboctave/util/oct-string.cc	Sun Apr 02 12:02:12 2023 +0200
+++ b/liboctave/util/oct-string.cc	Sat Apr 01 16:58:28 2023 +0200
@@ -38,6 +38,7 @@
 #include "Array.h"
 #include "lo-ieee.h"
 #include "lo-mappers.h"
+#include "oct-locbuf.h"
 #include "uniconv-wrappers.h"
 #include "unistr-wrappers.h"
 #include "unwind-prot.h"
@@ -617,29 +618,83 @@
    const InternT* from, const InternT* from_end, const InternT*& from_next,
    ExternT* to, ExternT* to_end, ExternT*& to_next) const
 {
-  if (from_end < from)
+  if (from_end <= from)
     return std::codecvt<InternT, ExternT, StateT>::noconv;
 
+  // Check if buffer ends in a complete UTF-8 surrogate
+  // FIXME: If this is the last call before a stream is closed, we should
+  //        convert trailing bytes even if they look incomplete.
+  //        How can we detect that?
+  std::size_t pop_end = 0;
+  if ((*(from_end-1) & 0b10000000) == 0b10000000)
+    {
+      // The last byte is part of a surrogate. Check if it is complete.
+
+      // number of bytes of the surrogate in the buffer
+      std::size_t num_bytes_in_buf = 1;
+      // Find initial byte of surrogate
+      while (((*(from_end-num_bytes_in_buf) & 0b11000000) != 0b11000000)
+             && (num_bytes_in_buf < 4)
+             && (from_end-num_bytes_in_buf > from))
+        num_bytes_in_buf++;
+
+      // If the start of the surrogate is not in the buffer, we need to
+      // continue with the invalid UTF-8 sequence to avoid an infinite loop.
+      // Check if we found an initial byte and if there are enough bytes in the
+      // buffer to complete the surrogate.
+      if ((((*(from_end-num_bytes_in_buf) & 0b11100000) == 0b11000000)
+           && (num_bytes_in_buf < 2))  // incomplete 2-byte surrogate
+          || (((*(from_end-num_bytes_in_buf) & 0b11110000) == 0b11100000)
+              && (num_bytes_in_buf < 3))  // incomplete 3-byte surrogate
+          || (((*(from_end-num_bytes_in_buf) & 0b11111000) == 0b11110000)
+              && (num_bytes_in_buf < 4)))  // incomplete 4-byte surrogate
+        pop_end = num_bytes_in_buf;
+    }
+
+  std::size_t srclen = (from_end-from-pop_end) * sizeof (InternT);
+  std::size_t lengthp = (to_end-to) * sizeof (ExternT);
+  if (srclen < 1 || lengthp < 1)
+    return std::codecvt<InternT, ExternT, StateT>::partial;
+
+  // FIXME: It looks like the input to octave_u8_conv_to_encoding must be at
+  //        least five bytes and zero-terminated to work correctly.  Zero-pad
+  //        input.  Should this be fixed in gnulib or iconv instead?
+  // FIXME: `unicode2native` is also affected.  Should this work-around be
+  //        moved a level lower into `uniconv-wrappers.c`?
+  std::size_t minlen = 4;
+  std::size_t padlen = std::max (srclen, minlen) + 1;
+  OCTAVE_LOCAL_BUFFER (uint8_t, u8_str, padlen);
+  std::copy_n (reinterpret_cast<const uint8_t *> (from), srclen, u8_str);
+  for (std::size_t i_pad = 0; i_pad <= padlen-srclen; i_pad++)
+    u8_str[srclen+i_pad] = 0;
+
   // Convert from UTF-8 to output encoding
-  std::size_t srclen = (from_end-from) * sizeof (InternT);
-  std::size_t lengthp = (to_end-to) * sizeof (ExternT);
-  const uint8_t *u8_str = reinterpret_cast<const uint8_t *> (from);
-  char *enc_str = octave_u8_conv_to_encoding (m_enc.c_str (), u8_str, srclen,
+  char *enc_str = octave_u8_conv_to_encoding (m_enc.c_str (), u8_str, padlen,
                                               &lengthp);
 
-  size_t max = to_end - to;
+  // FIXME: This assumes that "\0" is converted to a single byte.  This might
+  //        not be true for some exotic output encodings (like UTF-7?).
+  lengthp = lengthp - (padlen-srclen);
+  if (lengthp < 1)
+    return std::codecvt<InternT, ExternT, StateT>::partial;
+
+  size_t max = (to_end - to) * sizeof (ExternT);
+  // FIXME: If the output encoding is a multibyte or variable byte encoding,
+  //        we should ensure that we don't cut off a "partial" surrogate from
+  //        the output.
   if (lengthp < max)
     max = lengthp;
 
   // copy conversion result to output
-  // FIXME: Handle incomplete UTF-8 characters at end of buffer.
   std::copy_n (enc_str, max, to);
   ::free (enc_str);
 
   from_next = from + srclen;
   to_next = to + max;
 
-  return std::codecvt<InternT, ExternT, StateT>::ok;
+  return ((pop_end > 0 || max < lengthp)
+          ? std::codecvt<InternT, ExternT, StateT>::partial
+          : std::codecvt<InternT, ExternT, StateT>::ok);
 }
 
 typename std::codecvt<InternT, ExternT, StateT>::result
