# HG changeset patch
# User Andrew Janke <andrew@apjanke.net>
# Date 1547973029 18000
#      Sun Jan 20 03:30:29 2019 -0500
# Branch test-suite-summary-output-OOP
# Node ID 517418c7ea144f5737c41fa36026ba2032d7afa7
# Parent  2ae2dcf0f4d5b6b2395c6d37b445b235b2f018f4
__run_test_suite__: add failed file list to results summary, and OO refactor

diff --git a/scripts/testfun/+octave/+test/+internal/TestSuiteResult.m b/scripts/testfun/+octave/+test/+internal/TestSuiteResult.m
new file mode 100644
--- /dev/null
+++ b/scripts/testfun/+octave/+test/+internal/TestSuiteResult.m
@@ -0,0 +1,78 @@
+classdef TestSuiteResult
+  #TESTSUITERESULT The result of running tests on one or more files.
+  
+  properties
+    nTest = 0
+    nPass = 0
+    nFail = 0
+    # Count of failures that weren't xfails, skips, or regressions
+    nReallyFail = 0
+    nRegression = 0
+    nXFailBug = 0
+    nXFail = 0
+    nSkipFeature = 0
+    nSkipRuntime = 0
+    filesWithTests = {}
+    filesWithNoTests = {}
+    # List of files with failed tests, as cellstr row vector
+    failedFiles = {}
+  endproperties
+  
+  methods
+    function this = TestSuiteResult(varargin)
+      switch nargin
+        case 0
+          return
+        case 7
+          [np, n, nxf, nxb, nsk, nrtsk, nrgrs] = varargin{:};
+          failed_files = {};
+        case 8
+          [np, n, nxf, nxb, nsk, nrtsk, nrgrs, failed_files] = varargin{:};
+      endswitch
+      if ! iscellstr (failed_files)
+        error ('failed_files input must be a cellstr');
+      endif
+      this.nTest = n;
+      this.nPass = np;
+      this.nXFail = nxf;
+      this.nXFailBug = nxb;
+      this.nSkipFeature = nsk;
+      this.nSkipRuntime = nrtsk;
+      this.nRegression = nrgrs;
+      this.failedFiles = failed_files(:)';
+      this.nFail = n - np;
+      this.nReallyFail = this.nFail - nxf - nxb - nrgrs;
+    endfunction
+    
+    function out = plus(A, B)
+      klass = "octave.test.internal.TestSuiteResult";
+      if ! isa (A, klass) || ! isa (B, klass)
+        error ("Both inputs must be a %s", klass);
+      endif
+      out = A;
+      out.nTest = A.nTest + B.nTest;
+      out.nPass = A.nPass + B.nPass;
+      out.nFail = A.nFail + B.nFail;
+      out.nReallyFail = A.nReallyFail + B.nReallyFail;
+      out.nRegression = A.nRegression + B.nRegression;
+      out.nXFailBug = A.nXFailBug + B.nXFailBug;
+      out.nXFail = A.nXFail + B.nXFail;
+      out.nSkipFeature = A.nSkipFeature + B.nSkipFeature;
+      out.nSkipRuntime = A.nSkipRuntime + B.nSkipRuntime;
+      out.filesWithTests = [A.filesWithTests B.filesWithTests];
+      out.filesWithNoTests = [A.filesWithTests B.filesWithTests];
+      out.failedFiles = [A.failedFiles B.failedFiles];
+    endfunction
+    
+    function disp (this)
+      origWarn = warning;
+      warning off Octave:classdef-to-struct
+      s = builtin ('struct', this);
+      fprintf ("%s:\n", class (this));
+      disp (s);
+      warning (origWarn);
+      [txt,msg] = lastwarn;
+    endfunction
+  endmethods
+  
+endclassdef
diff --git a/scripts/testfun/__run_test_suite__.m b/scripts/testfun/__run_test_suite__.m
--- a/scripts/testfun/__run_test_suite__.m
+++ b/scripts/testfun/__run_test_suite__.m
@@ -17,29 +17,36 @@
 ## <https://www.gnu.org/licenses/>.
 
 ## -*- texinfo -*-
 ## @deftypefn  {} {} __run_test_suite__ (@var{fcndirs}, @var{fixedtestdirs})
 ## @deftypefnx {} {} __run_test_suite__ (@var{fcndirs}, @var{fixedtestdirs}, @var{topsrcdir}, @var{topbuilddir})
 ## Undocumented internal function.
 ## @end deftypefn
 
-function [pass, fail, xfail, xbug, skip, rtskip, regress] = __run_test_suite__ (fcndirs, fixedtestdirs, topsrcdir = [], topbuilddir = [])
+## varargout signature:
+## rslts = __run_test_suite__ (...)
+## [pass, fail, xfail, xbug, skip, rtskip, regress, failed_files] = __run_test_suite__ (...)
+function varargout = __run_test_suite__ (fcndirs, fixedtestdirs, topsrcdir = [], topbuilddir = [])
 
   testsdir = __octave_config_info__ ("octtestsdir");
   libinterptestdir = fullfile (testsdir, "libinterp");
   liboctavetestdir = fullfile (testsdir, "liboctave");
   fixedtestdir = fullfile (testsdir, "fixed");
   fcnfiledir = __octave_config_info__ ("fcnfiledir");
-  if (nargin == 0)
+  if (nargin < 1)
     fcndirs = { liboctavetestdir, libinterptestdir, fcnfiledir };
+  else
+    fcndirs = cellstr (fcndirs);
+  endif
+  if (nargin < 2)
     fixedtestdirs = { fixedtestdir };
+  else
+    fixedtestdirs = cellstr (fixedtestdirs);
   endif
-  files_with_no_tests = {};
-  files_with_tests = {};
   ## FIXME: These names don't really make sense if we are running
   ##        tests for an installed copy of Octave.
   if (isempty (topsrcdir))
     topsrcdir = fcnfiledir;
   endif
   if (isempty (topbuilddir))
     topbuilddir = testsdir;
   endif
@@ -47,209 +54,211 @@ function [pass, fail, xfail, xbug, skip,
   pso = page_screen_output ();
   orig_wstate = warning ();
   logfile = make_absolute_filename ("fntests.log");
   unwind_protect
     page_screen_output (false);
     warning ("on", "quiet");
     warning ("off", "Octave:deprecated-function");
     warning ("off", "Octave:legacy-function");
-    nfail = dp = dn = dxf = dxb = dsk = drtsk = drgrs = 0;
+    rslts = octave.test.internal.TestSuiteResult;
     try
       fid = fopen (logfile, "wt");
       if (fid < 0)
         error ("__run_test_suite__: could not open %s for writing", logfile);
       endif
       test ("", "explain", fid);
       puts ("\nIntegrated test scripts:\n\n");
       for i = 1:length (fcndirs)
-        [p, n, xf, xb, sk, rtsk, rgrs] = run_test_dir (fid, fcndirs{i}, false);
-        dp += p;
-        dn += n;
-        dxf += xf;
-        dxb += xb;
-        dsk += sk;
-        drtsk += rtsk;
-        drgrs += rgrs;
+        rslt = run_test_dir (fid, fcndirs{i}, false, topbuilddir, topsrcdir);
+        rslts = rslts + rslt;
       endfor
       puts ("\nFixed test scripts:\n\n");
       for i = 1:length (fixedtestdirs)
-        [p, n, xf, xb, sk, rtsk, rgrs] = run_test_dir (fid, fixedtestdirs{i}, true);
-        dp += p;
-        dn += n;
-        dxf += xf;
-        dxb += xb;
-        dsk += sk;
-        drtsk += rtsk;
-        drgrs += rgrs;
+        rslt = run_test_dir (fid, fixedtestdirs{i}, true, topbuilddir, topsrcdir);
+        rslts = rslts + rslt;
       endfor
       puts ("\nSummary:\n\n");
-      nfail = dn - dp - dxf - dxb - drgrs;
-      printf ("  %-30s %6d\n", "PASS", dp);
-      printf ("  %-30s %6d\n", "FAIL", nfail);
-      if (drgrs > 0)
-        printf ("  %-30s %6d\n", "REGRESSION", drgrs);
+      printf ("  %-30s %6d\n", "PASS", rslts.nPass);
+      printf ("  %-30s %6d\n", "FAIL", rslts.nReallyFail);
+      if (rslts.nRegression > 0)
+        printf ("  %-30s %6d\n", "REGRESSION", rslts.nRegression);
       endif
-      if (dxb > 0)
-        printf ("  %-30s %6d\n", "XFAIL (reported bug)", dxb);
+      if (rslts.nXFailBug > 0)
+        printf ("  %-30s %6d\n", "XFAIL (reported bug)", rslts.nXFailBug);
+      endif
+      if (rslts.nXFail > 0)
+        printf ("  %-30s %6d\n", "XFAIL (expected failure)", rslts.nXFail);
       endif
-      if (dxf > 0)
-        printf ("  %-30s %6d\n", "XFAIL (expected failure)", dxf);
+      if (rslts.nSkipFeature > 0)
+        printf ("  %-30s %6d\n", "SKIP (missing feature)", rslts.nSkipFeature);
+      endif
+      if (rslts.nSkipRuntime > 0)
+        printf ("  %-30s %6d\n", "SKIP (run-time condition)", rslts.nSkipRuntime);
       endif
-      if (dsk > 0)
-        printf ("  %-30s %6d\n", "SKIP (missing feature)", dsk);
-      endif
-      if (drtsk > 0)
-        printf ("  %-30s %6d\n", "SKIP (run-time condition)", drtsk);
+      if ! isempty (rslts.failedFiles)
+        printf ("\n");
+        printf ("  Failed tests:\n");
+        for i = 1:numel (rslts.failedFiles)
+          printf ("     %s\n", reduce_test_file_name (rslts.failedFiles{i}, ...
+            topbuilddir, topsrcdir));
+        endfor
       endif
       puts ("\n");
       printf ("See the file %s for additional details.\n", logfile);
-      if (dxf > 0 || dxb > 0)
+      if (rslts.nXFail > 0 || rslts.nXFailBug > 0)
         puts ("\n");
         puts ("XFAIL items are known bugs or expected failures.\n");
         puts ("Bug report numbers may be found in the log file:\n");
         puts (logfile);
         puts ("\nPlease help improve Octave by contributing fixes for them.\n");
       endif
-      if (dsk > 0 || drtsk > 0)
+      if (rslts.nSkipFeature > 0 || rslts.nSkipRuntime > 0)
         puts ("\n");
         puts ("Tests are often skipped because required features were\n");
         puts ("disabled or were not present when Octave was built.\n");
         puts ("The configure script should have printed a summary\n");
         puts ("indicating which dependencies were not found.\n");
       endif
 
       ## Weed out deprecated, legacy, and private functions
+      files_with_tests = rslts.filesWithTests;
       weed_idx = cellfun (@isempty, regexp (files_with_tests, '\<deprecated\>|\<legacy\>|\<private\>', 'once'));
       files_with_tests = files_with_tests(weed_idx);
+      files_with_no_tests = rslts.filesWithNoTests;
       weed_idx = cellfun (@isempty, regexp (files_with_no_tests, '\<deprecated\>|\<legacy\>|\<private\>', 'once'));
       files_with_no_tests = files_with_no_tests(weed_idx);
 
       report_files_with_no_tests (files_with_tests, files_with_no_tests, ".m");
 
       puts ("\nPlease help improve Octave by contributing tests for these files\n");
       printf ("(see the list in the file %s).\n\n", logfile);
 
       fprintf (fid, "\nFiles with no tests:\n\n%s",
                     list_in_columns (files_with_no_tests, 80));
       fclose (fid);
-    catch
+    catch err
       disp (lasterr ());
     end_try_catch
   unwind_protect_cleanup
     warning ("off", "all");
     warning (orig_wstate);
     page_screen_output (pso);
   end_unwind_protect
 
-  if (nargout > 0)
-    pass = dp;
-    fail = nfail;
-    xfail = dxf;
-    xbug = dxb;
-    skip = dsk;
-    rtskip = drtsk;
-    regress = drgrs;
+  if nargout == 1
+    varargout = { rslts };
+  elseif nargout > 1
+    varargout = {
+      rslts.nPass;
+      rslts.nFail;
+      rslts.nXFail;
+      rslts.nXFailBug;
+      rslts.nSkip;
+      rslts.nSkipRuntime;
+      rslts.nRegression;
+      rslts.failedFiles;
+    };
   endif
 
-  function [dp, dn, dxf, dxb, dsk, drtsk, drgrs] = run_test_dir (fid, d, is_fixed = false)
+endfunction
+
+function rslts = run_test_dir (fid, d, is_fixed, topbuilddir, topsrcdir)
 
-    lst = dir (d);
-    dp = dn = dxf = dxb = dsk = drtsk = drgrs = 0;
+  lst = dir (d);
+  rslts = octave.test.internal.TestSuiteResult;
+  for i = 1:length (lst)
+    nm = lst(i).name;
+    if (lst(i).isdir && nm(1) != "." && ! strcmp (nm, "private"))
+      rslt = run_test_dir (fid, [d, filesep, nm], is_fixed, topbuilddir, topsrcdir);
+      rslts = rslts + rslt;
+    endif
+  endfor
+
+  saved_dir = pwd ();
+  unwind_protect
+    [dnm, fnm] = fileparts (d);
+    if (fnm(1) != "@")
+      cd (d);
+    endif
     for i = 1:length (lst)
       nm = lst(i).name;
-      if (lst(i).isdir && nm(1) != "." && ! strcmp (nm, "private"))
-        [p, n, xf, xb, sk, rtsk, rgrs] = run_test_dir (fid, [d, filesep, nm], is_fixed);
-        dp += p;
-        dn += n;
-        dxf += xf;
-        dxb += xb;
-        dsk += sk;
-        drtsk += rtsk;
-        drgrs += rgrs;
+      ## Ignore hidden files
+      if (nm(1) == '.')
+        continue
+      endif
+      if ((! is_fixed && length (nm) > 2 && strcmpi (nm((end-1):end), ".m"))
+          || (! is_fixed && length (nm) > 4 && strcmpi (nm((end-3):end), "-tst"))
+          || (is_fixed && length (nm) > 4 && strcmpi (nm((end-3):end), ".tst")))
+        p = n = xf = xb = sk = rtsk = rgrs = 0;
+        ffnm = fullfile (d, nm);
+        ## Only run if contains %!test, %!assert, %!error, %!fail, or %!warning
+        if (has_tests (ffnm))
+          tmp = reduce_test_file_name (ffnm, topbuilddir, topsrcdir);
+          print_test_file_name (tmp);
+          rslt = my_test (ffnm, "quiet", fid);
+          rslt.filesWithTests{end+1} = ffnm;
+          print_pass_fail (rslt);
+          rslts = rslts + rslt;
+        else
+          rslts.filesWithNoTests{end+1} = ffnm;
+        endif
       endif
     endfor
+  unwind_protect_cleanup
+    cd (saved_dir);
+  end_unwind_protect
 
-    saved_dir = pwd ();
-    unwind_protect
-      [dnm, fnm] = fileparts (d);
-      if (fnm(1) != "@")
-        cd (d);
-      endif
-      for i = 1:length (lst)
-        nm = lst(i).name;
-        ## Ignore hidden files
-        if (nm(1) == '.')
-          continue
-        endif
-        if ((! is_fixed && length (nm) > 2 && strcmpi (nm((end-1):end), ".m"))
-            || (! is_fixed && length (nm) > 4 && strcmpi (nm((end-3):end), "-tst"))
-            || (is_fixed && length (nm) > 4 && strcmpi (nm((end-3):end), ".tst")))
-          p = n = xf = xb = sk = rtsk = rgrs = 0;
-          ffnm = fullfile (d, nm);
-          ## Only run if contains %!test, %!assert, %!error, %!fail, or %!warning
-          if (has_tests (ffnm))
-            tmp = reduce_test_file_name (ffnm, topbuilddir, topsrcdir);
-            print_test_file_name (tmp);
-            [p, n, xf, xb, sk, rtsk, rgrs] = test (ffnm, "quiet", fid);
-            print_pass_fail (p, n, xf, xb, sk, rtsk, rgrs);
-            dp += p;
-            dn += n;
-            dxf += xf;
-            dxb += xb;
-            dsk += sk;
-            drtsk += rtsk;
-            drgrs += rgrs;
-            files_with_tests(end+1) = ffnm;
-          else
-            ## To reduce the list length, only mark .cc files that contain
-            ## DEFUN definitions.
-            files_with_no_tests(end+1) = ffnm;
-          endif
-        endif
-      endfor
-    unwind_protect_cleanup
-      cd (saved_dir);
-    end_unwind_protect
+endfunction
+
 
-  endfunction
-
+function rslt = my_test (file, options, fid)
+  [p, n, xf, xb, sk, rtsk, rgrs] = test (file, options, fid);
+  rslt = octave.test.internal.TestSuiteResult(p, n, xf, xb, sk, rtsk, rgrs);
+  if rslt.nFail < 0 || rslt.nReallyFail < 0
+    keyboard
+  endif
+  if rslt.nReallyFail > 0
+    rslt.failedFiles{end+1} = file;
+  endif
+  if rslt.nTest > 0
+    rslt.filesWithTests{end+1} = file;
+  endif
 endfunction
 
 function print_test_file_name (nm)
   nmlen = numel (nm);
   filler = repmat (".", 1, 63-nmlen);
   if (nmlen > 63)
     nm = ["..", nm(nmlen-60:end)];
   endif
   printf ("  %s %s", nm, filler);
 endfunction
 
-function print_pass_fail (p, n, xf, xb, sk, rtsk, rgrs)
+function print_pass_fail (x)
 
-  if ((n + sk + rtsk + rgrs) > 0)
-    printf (" PASS %4d/%-4d", p, n);
-    nfail = n - p - xf - xb - rgrs;
-    if (nfail > 0)
-      printf ("\n%72s %3d", "FAIL ", nfail);
+  if ((x.nFail + x.nSkipFeature + x.nSkipRuntime + x.nRegression) > 0)
+    printf (" PASS %4d/%-4d", x.nFail, x.nTest);
+    if (x.nFail  > 0)
+      printf ("\n%72s %3d", "FAIL ", x.nFail);
     endif
-    if (rgrs > 0)
-      printf ("\n%72s %3d", "REGRESSION", rgrs);
+    if (x.nRegression > 0)
+      printf ("\n%72s %3d", "REGRESSION", x.nRegression);
     endif
-    if (sk > 0)
-      printf ("\n%72s %3d", "(missing feature) SKIP ", sk);
+    if (x.nSkipFeature > 0)
+      printf ("\n%72s %3d", "(missing feature) SKIP ", x.nSkipFeature);
     endif
-    if (rtsk > 0)
-      printf ("\n%72s %3d", "(run-time condition) SKIP ", rtsk);
+    if (x.nSkipRuntime > 0)
+      printf ("\n%72s %3d", "(run-time condition) SKIP ", x.nSkipRuntime);
     endif
-    if (xb > 0)
-      printf ("\n%72s %3d", "(reported bug) XFAIL", xb);
+    if (x.nXFailBug > 0)
+      printf ("\n%72s %3d", "(reported bug) XFAIL", x.nXFailBug);
     endif
-    if (xf > 0)
-      printf ("\n%72s %3d", "(expected failure) XFAIL", xf);
+    if (x.nXFail > 0)
+      printf ("\n%72s %3d", "(expected failure) XFAIL", x.nXFail);
     endif
   endif
   puts ("\n");
 
 endfunction
 
 function retval = reduce_test_file_name (nm, builddir, srcdir)
 
