exporting patch:
# HG changeset patch
# User Daniel J Sebald <daniel.sebald@ieee.org>
# Date 1533110954 18000
#      Wed Aug 01 03:09:14 2018 -0500
# Node ID 6a26a372b4699c7f3646c3bd9b17b69734c5a5ff
# Parent  02fbbde153082d0187373586585f619a9e320447
Modify the octave_idx_type overflow check in get_size() (bug #54405)

* oct-stream.cc: Define OCTAVE_IDX_MULT_OVERFLOW() in a way analogous to the
  GNULIB routine that checks signed multiply overflow.  If the hardware-
  oriented built-in __builtin_smull_overflow() is not present, revert to a
  slower C-code equivalent.
  (stream::get_size): Add a FIXME comment about an observation that double
  values greater than or equal to 2^53 can lose resolution and map to negative
  numbers.  Test for this condition and issue a warning if true.
  (stream::get_size): Replace the existing overflow check with more robust
  OCTAVE_IDX_MULT_OVERFLOW().

* io.tst: Add three BIST tests for a few simple functional fread calls
  involving precision and skip parameters.  Add five BIST tests for various
  overflow conditions.

diff --git a/libinterp/corefcn/oct-stream.cc b/libinterp/corefcn/oct-stream.cc
--- a/libinterp/corefcn/oct-stream.cc
+++ b/libinterp/corefcn/oct-stream.cc
@@ -60,6 +60,32 @@ along with Octave; see the file COPYING.
 #include "pager.h"
 #include "utils.h"
 
+#if 1
+  #include "gnulib/lib/intprops.h"
+  #ifndef _GL_HAS_BUILTIN_OVERFLOW
+    #define _GL_HAS_BUILTIN_OVERFLOW 0
+  #endif
+#else
+ #define _GL_HAS_BUILTIN_OVERFLOW 0
+#endif
+
+inline bool OCTAVE_IDX_MULT_OVERFLOW (octave_idx_type a, octave_idx_type b, octave_idx_type *res)
+{
+#if _GL_HAS_BUILTIN_OVERFLOW
+  #if OCTAVE_ENABLE_64
+    return __builtin_smull_overflow (a, b, res);
+  #else
+    return __builtin_smul_overflow (a, b, res);
+  #endif
+#else
+  *res = a * b;
+  // abs(INTMIN) = 0, i.e., non-symmetric 2's complement, so must check that as well.
+  return (a != 0 &&
+          ((b == std::numeric_limits<octave_idx_type>::min () && a != 1) ||
+           (std::abs (b) > std::abs (std::numeric_limits<octave_idx_type>::max () / a))));
+#endif
+}
+
 namespace octave
 {
   // Programming Note: There are two very different error functions used
@@ -135,6 +161,17 @@ namespace octave
         if (d > std::numeric_limits<octave_idx_type>::max ())
           ::error ("%s: dimension too large for Octave's index type",
                    who.c_str ());
+#if 1
+        // FIXME: The size specification involves the use of double
+        //        representation, so roughly any typed-in index values from
+        //        2^53 (mantissa size of double) to 2^63 can create aliased
+        //        nr and nc, i.e., they turn out negative from wrap-around
+        //        effect.  (See Bug #54405.)  Is this necessary given huge
+        //        size?  Should this be done in interpretter?
+        if (d >= 9007199254740992)
+          ::warning ("%s: possible loss of resolution converting from float to octave_idx_type",
+                   who.c_str ());
+#endif
 
         retval = math::nint_big (d);
       }
@@ -185,8 +222,8 @@ namespace octave
         nc = get_size (dnc, who);
 
         // Check for overflow.
-        if (nr > 0 && nc > 0
-            && nc > std::numeric_limits<octave_idx_type>::max () / nr)
+        octave_idx_type elts_to_read;
+        if (OCTAVE_IDX_MULT_OVERFLOW (nr, nc, &elts_to_read))
           ::error ("%s: size too large for Octave's index type", who.c_str ());
       }
   }
diff --git a/test/io.tst b/test/io.tst
--- a/test/io.tst
+++ b/test/io.tst
@@ -844,6 +844,48 @@
 %!   end_unwind_protect
 %! endif
 
+%!test <54100>
+%! [id, msg] = tmpfile ();
+%! if (id < 0)
+%!   __printf_assert__ ("tmpfile failed: %s\n", msg);
+%! else
+%!   unwind_protect
+%!     fwrite (id, ones(10,1)*[1:11], "single");
+%!     frewind (id);
+%!     x = fread (id, [10, 11], "single");
+%!     frewind (id);
+%!     x2 = fread (id, [5, 11], "single", 4);
+%!     frewind (id);
+%!     xinf = fread (id, [8, Inf], "8*single", 2*4);
+%!     assert (all (all (x == ones(10,1)*[1:11])));
+%!     assert (all (all (x2 == ones(5,1)*[1:11])));
+%!     assert (all (all (xinf == ones(8,1)*[1:11])));
+%!   unwind_protect_cleanup
+%!     fclose (id);
+%!   end_unwind_protect
+%! endif
+
+%!test <54405>
+%! [id, msg] = tmpfile ();
+%! if (id < 0)
+%!   __printf_assert__ ("tmpfile failed: %s\n", msg);
+%! else
+%!   unwind_protect
+%!     frewind (id);
+%!     fail ("x = fread (id, [2^53-1, 2^53-1], 'single');", "dimension too large");
+%!     frewind (id);
+%!     fail ("x = fread (id, [2^53, 2^53], 'single');", "dimension too large");
+%!     frewind (id);
+%!     fail ("x = fread (id, [-9223372036854775808, -9223372036854775808], 'single');", "negative");
+%!     frewind (id);
+%!     fail ("x = fread (id, [9223372036854775807, 9223372036854775807], 'single');", "dimension too large");
+%!     frewind (id);
+%!     fail ("x = fread (id, [9223372036854775807/3, 9223372036854775807/3], 'single');", "dimension too large");
+%!   unwind_protect_cleanup
+%!     fclose (id);
+%!   end_unwind_protect
+%! endif
+
 %!assert (sprintf ("%1s", "foo"), "foo")
 %!assert (sprintf ("%.s", "foo"), char (zeros (1, 0)))
 %!assert (sprintf ("%1.s", "foo"), " ")
