# HG changeset patch # User Petter T. # Date 1692442649 -7200 # Sat Aug 19 12:57:29 2023 +0200 # Node ID 1cda2e27d8ca55f9694fe130780eda7882be3a03 # Parent f38a5d7fad7f3b9a45447ab9793ac52879d06e7c VM Updating VM bench to support scripts * test/compile-bench/bench-octave/bench.m: Support scripts * test/compile-bench/bench-octave/bench_valgrind.m: Added. Run bench in Valgrind * test/compile-bench/bench-octave/cdef_ctor.m: Added * test/compile-bench/bench-octave/cdef_foo.m: Classdef for bench * test/compile-bench/bench-octave/cdef_method1.m: Added * test/compile-bench/bench-octave/dummy_script.m: Added * test/compile-bench/bench-octave/for_loop_binop_2_script.m: Added * test/compile-bench/bench-octave/for_loop_call_script_1.m: Added * test/compile-bench/bench-octave/get_bench_conf.m: Getter for test specs * test/compile-bench/module.mk: Include new files diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/bench.m --- a/test/compile-bench/bench-octave/bench.m Tue Sep 05 09:55:38 2023 +0200 +++ b/test/compile-bench/bench-octave/bench.m Sat Aug 19 12:57:29 2023 +0200 @@ -3,31 +3,13 @@ % The tests to run % % {name, {arg_type, n}, ...} - tests = { - {"for_loop_empty", {"n", 206824596}, 1, {}}, - {"for_loop_silly", {"n", 34894840}, 1, {}}, - {"for_loop_binop_1", {"n", 20300088}, 1, {}}, - {"for_loop_sinpi", {"n", 12991066}, 1, {}}, - {"for_loop_ifs", {"n", 5874007}, 1, {}}, - {"while_loop_empty", {"n", 24237997}, 1, {}}, - {"do_until_loop_empty", {"n", 27109647}, 1, {}}, - {"for_loop_subfun_1", {"n", 11930390}, 1, {}}, - {"for_loop_matselfmul", {"rand sq",150}, 3, {}}, - {"for_sum_1", {"rand rowvec", 19267692}, 1, {}}, - {"for_sum_2", {"rand rowvec", 8742659}, 1, {}}, - {"qsort_recursive", {"rand rowvec", 107851}, 1, {}}, % Mostly copies vectors around - {"qsort_iterative", {"rand rowvec", 344418}, 1, {}}, - {"for_loop_fncall", {"n", 2164885}, 1, {}}, - {"bench_median", {"rand rowvec", 1927}, 1, {}}, - {"bench_cov", {"rand rowvec", 15261}, 1, {}}, - {"str_mod", {"n", 2335290}, 1, {}}, - {"fib", {"n", 31}, 1, {}}, - }; + tests = get_bench_conf (); reg = ''; calibrate = 0; do_both = 1; n_factor = 1; + no_compile = 0; filter = ""; i = 1; while i <= nargin @@ -40,6 +22,8 @@ elseif strcmp (arg, "n_factor") assert (i <= nargin) n_factor = varargin{i++}; + elseif strcmp (arg, "nocompile") + no_compile = 1; end end @@ -62,6 +46,7 @@ name = test{1}; complexity = test{3}; also_compile = test{4}; + is_script = test{5}; j = 1; % Skip on not matching regex, if there is one @@ -83,11 +68,15 @@ n = round (conf{2} * n_factor); arg = n; elseif strcmp (conf_type, "rand sq") - rng (0); % Reset rng + if exist ("rng") % rng is a kinda new function. Keep backwards compatibility + rng (0); % Reset rng + end n = round (conf{2} * n_factor); arg = randn (n); elseif strcmp (conf_type, "rand rowvec") - rng (0); % Reset rng + if exist ("rng") + rng (0); % Reset rng + end n = round (conf{2} * n_factor); arg = randn (n, 1); end @@ -105,11 +94,15 @@ n = round (n_norm * n_factor); arg = n; elseif strcmp (conf_type, "rand sq") - rng (0); % Reset rng + if exist ("rng") + rng (0); % Reset rng + end n = round (n_norm * n_factor); arg = randn (n); elseif strcmp (conf_type, "rand rowvec") - rng (0); % Reset rng + if exist ("rng") + rng (0); % Reset rng + end n = round (n_norm * n_factor); arg = randn (n, 1); end @@ -118,17 +111,32 @@ tic; [ccttot0, cctuser0, cctsys0] = cputime; - assert (__compile__ (name)); + if !no_compile + if ! __compile__ (name) + warning ("Could not compile %s, skipping ...", name) + continue; + end + end [ccttot1, cctuser1, cctsys1] = cputime; cctwall = toc; - [cttot0, ctuser0, ctsys0] = cputime; - tic; - __dummy_mark_1__ (); - fn (arg); - __dummy_mark_2__ (); - [cttot1, ctuser1, ctsys1] = cputime; - ctwall = toc; + try + [cttot0, ctuser0, ctsys0] = cputime; + tic; + __dummy_mark_1__ (); + if is_script + fn (); + else + fn (arg); + end + __dummy_mark_1__ (); + __dummy_mark_2__ (); + [cttot1, ctuser1, ctsys1] = cputime; + ctwall = toc; + catch + warning ("Could not run %s due to '%s', skipping ...", e.msg, name) + continue; + end printf (" %-16s %-16s %-16s %-16s %-16s\n", "t tic","t cpu", "t usr" , "t sys", "n"); printf (" Runtime: %-16g %-16g %-16g %-16g %-16g\n", ctwall, cttot1 - cttot0, ctuser1 - ctuser0, ctsys1 - ctsys0, n); diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/bench_valgrind.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/bench_valgrind.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,77 @@ +function bench_valgrind (octbin_path, factor, benchname_filter, octbin_path_ref, logdir) + + s = unix ("valgrind --version"); + if s + error ("Valgrind probably not installed"); + end + + orig_dir = pwd; + + path_to_bench_folder = which ("bench_valgrind"); + path_to_bench_folder = strrep (path_to_bench_folder, "bench_valgrind.m", ""); + logsubfolder_name = ["run_" datestr(now, "yyyy_mm_dd_HH_MM_ss")]; + + tests = get_bench_conf (); + + result = {}; + + unwind_protect + mkdir (logdir); + cd (logdir); + mkdir (logsubfolder_name) + cd (logsubfolder_name) + + for i = 1:length(tests) + + test = tests{i}; + name = test{1}; + complexity = test{3}; + also_compile = test{4}; + is_script = test{5}; + + % Skip on not matching regex, if there is one + if length (benchname_filter) && isempty (regexp (name, benchname_filter)) + continue; + end + + % logfilename, octbin, bench folder, benchname filter + cmd_template = ["valgrind --tool=callgrind --callgrind-out-file=%s --separate-recs=10 " ... + "--dump-instr=yes --collect-jumps=yes \"--dump-after=dummy_mark_1\" " ... + " %s -W --eval \"__enable_vm_eval__ (1); cd %s; bench('reg','%s','n_factor', %d);exit(0)\""]; + + logfilename1 = ["callgrind.out.log_mark_", name, "_", num2str(i), "_", name, ".log"]; + logfilename2 = ["callgrind.out.log_ref1_", name, "_", num2str(i), "_", name, ".log"]; + + cmd_1 = sprintf (cmd_template, logfilename1, octbin_path, path_to_bench_folder, name, factor); + cmd_2 = sprintf (cmd_template, logfilename2, octbin_path_ref, path_to_bench_folder, name, factor); + + [status1, stdout1] = unix (cmd_1); + [status2, stdout2] = unix (cmd_2); + + log1 = fileread ([logfilename1 ".2"]); + log2 = fileread ([logfilename2 ".2"]); + + [~,~,~,~, cost1] = regexp (log1, "summary:\\s+(\\d+)", 'once'); + cost1 = str2num (cost1{1}); + [~,~,~,~, cost2] = regexp (log2, "summary:\\s+(\\d+)", 'once'); + cost2 = str2num (cost2{1}); + + result{end + 1} = {name, cost1, cost2, cost2/cost1*100}; + endfor + + summary = sprintf ("%30s %15s %15s %15s\n", "name", "mark", "ref", "ref/mark%"); + for i = 1:length (result) + run_result = result{i}; + summary = [summary, sprintf("%30s %15d %15d %15.7f\n", run_result{:})]; + endfor + + printf ("\n\n\n%s", summary); + % Save summary to file + f = fopen ("summary.log", "w"); + fprintf (f, "%s", summary); + fclose (f); + + unwind_protect_cleanup + cd (orig_dir); + end +end diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/cdef_ctor.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/cdef_ctor.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,12 @@ +function cdef_ctor (n) + for i = 1:n + obj = cdef_foo ("asd", i); + end +end + +% bugg +function cdef_ctor1 (n) + for i = 1:n + obj = cdef_ctor1 ("asd", i); + end +end \ No newline at end of file diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/cdef_foo.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/cdef_foo.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,25 @@ +classdef cdef_foo + properties + s = ""; + a = 0; + end + + methods + function obj = cdef_foo (s, a) + obj.s = s; + obj.a = a; + end + + function c = method1 (obj, b) + c = b + obj.a; + end + + function a = getter1 (obj) + a = obj.a; + end + + function obj = setter1 (obj, a) + obj.a = a; + end + end +end \ No newline at end of file diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/cdef_method1.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/cdef_method1.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,6 @@ +function cdef_method1 (n) + obj = cdef_foo ("qwe", 123); + for i = 1:n + obj.method1 (2); + end +end \ No newline at end of file diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/dummy_script.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/dummy_script.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,1 @@ +% empty \ No newline at end of file diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/for_loop_binop_2_script.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/for_loop_binop_2_script.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,5 @@ +for i = 1:n + j = 1*2*3*4 * i; + g = j / 2 + 1; + f = g + j - 3; +end \ No newline at end of file diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/for_loop_call_script_1.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/for_loop_call_script_1.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,3 @@ + for i = 1:n + dummy_script (); + end \ No newline at end of file diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/for_loop_subfun_1.m --- a/test/compile-bench/bench-octave/for_loop_subfun_1.m Tue Sep 05 09:55:38 2023 +0200 +++ b/test/compile-bench/bench-octave/for_loop_subfun_1.m Sat Aug 19 12:57:29 2023 +0200 @@ -2,8 +2,13 @@ for i = 1:n suby (); + suby2 (1, 2); end end function suby () end + +function c = suby2 (a, b) +end + diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/bench-octave/get_bench_conf.m --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compile-bench/bench-octave/get_bench_conf.m Sat Aug 19 12:57:29 2023 +0200 @@ -0,0 +1,28 @@ +function conf = get_bench_conf () + conf = { + % name argument O(n^x) dunno is_script + {"for_loop_empty", {"n", 206824596}, 1, {}, 0}, + {"for_loop_silly", {"n", 34894840}, 1, {}, 0}, + {"for_loop_binop_1", {"n", 20300088}, 1, {}, 0}, + {"for_loop_binop_2", {"n", 10300088}, 1, {}, 0}, + {"for_loop_binop_2_script", {"n", 10300088}, 1, {}, 1}, + {"for_loop_sinpi", {"n", 12991066}, 1, {}, 0}, + {"for_loop_ifs", {"n", 5874007}, 1, {}, 0}, + {"while_loop_empty", {"n", 24237997}, 1, {}, 0}, + {"do_until_loop_empty", {"n", 27109647}, 1, {}, 0}, + {"for_loop_subfun_1", {"n", 11930390}, 1, {}, 0}, + {"for_loop_call_script_1", {"n", 11930390}, 1, {}, 1}, + {"for_loop_matselfmul", {"rand sq",150}, 3, {}, 0}, + {"for_sum_1", {"rand rowvec", 19267692}, 1, {}, 0}, + {"for_sum_2", {"rand rowvec", 8742659}, 1, {}, 0}, + {"qsort_recursive", {"rand rowvec", 107851}, 1, {}, 0}, % Mostly copies vectors around + {"qsort_iterative", {"rand rowvec", 344418}, 1, {}, 0}, + {"for_loop_fncall", {"n", 2164885}, 1, {}, 0}, + {"bench_median", {"rand rowvec", 1927}, 1, {}, 0}, + {"bench_cov", {"rand rowvec", 15261}, 1, {}, 0}, + {"str_mod", {"n", 2335290}, 1, {}, 0}, + {"fib", {"n", 31}, 1, {}, 0}, + {"cdef_ctor", {"n", 94964}, 1, {}, 0}, + {"cdef_method1", {"n", 164837}, 1, {}, 0}, + }; +end \ No newline at end of file diff -r f38a5d7fad7f -r 1cda2e27d8ca test/compile-bench/module.mk --- a/test/compile-bench/module.mk Tue Sep 05 09:55:38 2023 +0200 +++ b/test/compile-bench/module.mk Sat Aug 19 12:57:29 2023 +0200 @@ -2,9 +2,17 @@ %reldir%/bench-octave/bench.m \ %reldir%/bench-octave/bench_cov.m \ %reldir%/bench-octave/bench_median.m \ + %reldir%/bench-octave/bench_valgrind.m \ + %reldir%/bench-octave/cdef_foo.m \ + %reldir%/bench-octave/cdef_ctor.m \ + %reldir%/bench-octave/cdef_method1.m \ + %reldir%/bench-octave/dummy_script.m \ %reldir%/bench-octave/do_until_loop_empty.m \ %reldir%/bench-octave/fib.m \ %reldir%/bench-octave/for_loop_binop_1.m \ + %reldir%/bench-octave/for_loop_binop_2.m \ + %reldir%/bench-octave/for_loop_binop_2_script.m \ + %reldir%/bench-octave/for_loop_call_script_1.m \ %reldir%/bench-octave/for_loop_empty.m \ %reldir%/bench-octave/for_loop_fncall.m \ %reldir%/bench-octave/for_loop_ifs.m \ @@ -14,6 +22,7 @@ %reldir%/bench-octave/for_loop_subfun_1.m \ %reldir%/bench-octave/for_sum_1.m \ %reldir%/bench-octave/for_sum_2.m \ + %reldir%/bench-octave/get_bench_conf.m \ %reldir%/bench-octave/qsort_iterative.m \ %reldir%/bench-octave/qsort_recursive.m \ %reldir%/bench-octave/str_mod.m \