From 78bc2fd92b1f4c1cb462c4097516d4e3f35fce25 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Mon, 16 Jan 2023 15:52:07 +0100 Subject: [PATCH] chore: more benchmarking setup --- tests/bench/Makefile | 12 +++----- tests/bench/binarytrees5_multicore.ml | 2 +- tests/bench/ocaml-gc.py | 25 ++++++++++------ tests/bench/report.py | 41 ++++++++++++--------------- 4 files changed, 39 insertions(+), 41 deletions(-) diff --git a/tests/bench/Makefile b/tests/bench/Makefile index d5435363e8..649bbd798a 100644 --- a/tests/bench/Makefile +++ b/tests/bench/Makefile @@ -74,8 +74,6 @@ binarytrees%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -threaded -r %.ml.out: %.ml $(OCAML) $(OCAML_FLAGS) $< -o $@ -%.gc.ml.out: OCAML_FLAGS += -runtime-variant i -%.gc.ml: %.ml; ln -f $< $@ %.flambda.ml.out: OCAML = $(OCAML_FLAMBDA) %.flambda.ml: %.ml; ln -f $< $@ @@ -106,6 +104,7 @@ bench/parser.%.bench: BENCH_PARAMS = $(PARSER_TEST_FILE) 100 bench/binarytrees.%.bench: BENCH_PARAMS = 21 bench/binarytrees.ml.bench: BENCH_PARAMS = $$(nproc) 21 +bench/binarytrees.gc.ml.bench: BENCH_PARAMS = $$(nproc) 21 bench/deriv.%.bench: BENCH_PARAMS = 10 @@ -125,7 +124,7 @@ bench/unionfind.%.bench: BENCH_PARAMS = 3000000 bench/%gc.lean.bench: %lean.out | bench ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\ - -d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio -S 'lean_del,lean_dealloc_small,lean_free_small' | ./lean-gc.py"\ + -d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio -S 'lean_dec_ref_cold,lean_del_core,lean_free_small' | ./lean-gc.py"\ --runner output --out $@ bench/%gc.hs.bench: %hs.out | bench @@ -133,9 +132,9 @@ bench/%gc.hs.bench: %hs.out | bench -d $< "./$< +RTS -t --machine-readable -RTS $(BENCH_PARAMS) 2>&1 >/dev/null | ./ghc-gc.py"\ --runner output --out $@ -bench/%gc.ml.bench: %gc.ml.out | bench +bench/%gc.ml.bench: %ml.out | bench ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\ - -d $< "echo > $@.tmp && OCAML_INSTR_FILE=$@.tmp time -ao $@.tmp -f '%e' ./$< $(BENCH_PARAMS) >/dev/null && ./ocaml-gc.py < $@.tmp"\ + -d $< "time -ao $@.tmp -f '%U' olly trace $@.tmp './$< $(BENCH_PARAMS)' > /dev/null && ./ocaml-gc.py < $@.tmp"\ --runner output --out $@ bench/%gc.mlton.bench: %gc.mlton.out | bench @@ -158,9 +157,6 @@ bench/%.perf.bench: %.out | bench -d $< "echo > $@.tmp && time -af '%e' perf stat -e cache-misses -x ';' ./$< $(BENCH_PARAMS) 2>&1 >/dev/null | ./perf.py"\ --runner output --out $@ -# fork() breaks instrumentation -bench/binarytrees.gc.ml.bench: ; touch $@ - # no benchmarksgame versions bench/binarytrees%mlton.bench: ; touch $@ bench/binarytrees%mlton.perf.bench: ; touch $@ diff --git a/tests/bench/binarytrees5_multicore.ml b/tests/bench/binarytrees5_multicore.ml index 820a64589d..c54fc6323e 100644 --- a/tests/bench/binarytrees5_multicore.ml +++ b/tests/bench/binarytrees5_multicore.ml @@ -1,4 +1,4 @@ -// https://github.com/ocaml-bench/sandmark/blob/c668eb05aebb1dc9f57f322399bea7182640518c/benchmarks/multicore-numerical/binarytrees5_multicore.ml +(* https://github.com/ocaml-bench/sandmark/blob/c668eb05aebb1dc9f57f322399bea7182640518c/benchmarks/multicore-numerical/binarytrees5_multicore.ml *) module T = Domainslib.Task let num_domains = try int_of_string Sys.argv.(1) with _ -> 1 diff --git a/tests/bench/ocaml-gc.py b/tests/bench/ocaml-gc.py index a4b578fc44..ff574c76d9 100755 --- a/tests/bench/ocaml-gc.py +++ b/tests/bench/ocaml-gc.py @@ -1,13 +1,20 @@ #!/usr/bin/env python3 import sys +import json -events = sys.stdin.readlines() -wall_secs = float(events[-1]) -gc_nanos = 0 -for event in events[::-1]: - data = event.split() - # see also https://gitlab.com/gasche/gc-latency-experiment/blob/master/parse_ocaml_log.ml - if len(data) == 4 and data[0] == '@@' and data[3] == 'dispatch': - gc_nanos += int(data[2]) - int(data[1]) -print(f"gc: {gc_nanos*1e-9/wall_secs*100}") +lines = sys.stdin.readlines() +user_secs = float(lines[-1]) +events = json.loads("\n".join(lines[:-1])[:-2] + "]") +starts = {} +gc_msecs = 0 +for event in events: + if event["name"] in ["minor", "major"]: + tid = event["tid"] + if event["ph"] == "B": + if tid not in starts: + starts[tid] = (event["name"], int(event["ts"])) + elif tid in starts and starts[tid][0] == event["name"]: + gc_msecs += int(event["ts"]) - starts[tid][1] + del starts[tid] +print(f"gc: {gc_msecs*1e-6/user_secs*100}") diff --git a/tests/bench/report.py b/tests/bench/report.py index 0f8937cfaf..068a47a338 100755 --- a/tests/bench/report.py +++ b/tests/bench/report.py @@ -28,55 +28,50 @@ stddev_by_cat = collections.defaultdict(list) def pp(bench, cat, prop, norm): f = f"bench/{bench}{cat}.bench" if not open(f).read(): - return "" + return ";" s = single(bench, cat, prop) norm = norm if prop == 'etime' else 1 mean_by_cat[cat].append(s.mean() / norm) stddev_by_cat[cat].append(s.std_dev() / norm) - num = number.FNumber(s.mean() / norm, abs_deviation=s.std_dev() / norm) - num.settings["min_decimal_places"] = 2 if prop == 'etime' else 0 - num.settings["max_decimal_places"] = 2 if prop == 'etime' else 0 - return num.format() + ('%' if prop == 'gc' else '') + num = s.mean() / norm + return f'{num:.3f};{s.std_dev() / norm:.3f}' def mean(cat, prop): mean = st.gmean(mean_by_cat[cat]) stddev = st.gmean(stddev_by_cat[cat]) - num = number.FNumber(mean, abs_deviation=stddev) - num.settings["min_decimal_places"] = 2 if prop == 'etime' else 0 - num.settings["max_decimal_places"] = 2 if prop == 'etime' else 0 - return num.format() + ('%' if prop == 'gc' else '') #if prop == 'etime' else '-' + return f'{mean:.3f};{stddev:.3f}' CATBAG = { - '.lean': ("Lean [s]", "etime"), + '.lean': ("Lean", "etime"), '.no_reuse.lean': ("-reuse", "etime"), '.no_borrow.lean': ("-borrow", "etime"), '.no_st.lean': ("-ST", "etime"), '.gcc.lean': ("Lean+GCC9", "etime"), - '.gc.lean': ("del [%]", "gc"), - '.lean.perf': ("cache misses (CM) [1M/s]", "cache-misses"), + '.gc.lean': ("Lean del", "gc"), + '.lean.perf': ("CM", "cache-misses"), '.hs': ("GHC", "etime"), - '.gc.hs': ("GC [%]", "gc"), - '.hs.perf': ("CM", "cache-misses"), + '.gc.hs': ("GHC GC", "gc"), + '.hs.perf': ("GHC CM", "cache-misses"), '.llvm.hs': ("GHC -fllvm", "etime"), '.strict.hs': ("GHC -XStrict", "etime"), '.ml': ("OCaml", "etime"), - '.gc.ml': ("GC", "gc"), - '.ml.perf': ("CM", "cache-misses"), + '.gc.ml': ("OCaml GC", "gc"), + '.ml.perf': ("OCaml CM", "cache-misses"), '.flambda.ml': ("OCaml+Flambda", "etime"), '.mlton': ("MLton", "etime"), - '.gc.mlton': ("GC", "gc"), - '.mlton.perf': ("CM", "cache-misses"), + '.gc.mlton': ("MLton GC", "gc"), + '.mlton.perf': ("MLton CM", "cache-misses"), '.mlkit': ("MLKit", "etime"), - '.gc.mlkit': ("GC", "gc"), - '.mlkit.perf': ("CM", "cache-misses"), + '.gc.mlkit': ("MLKit GC", "gc"), + '.mlkit.perf': ("MLKit CM", "cache-misses"), '.swift': ("Swift", "etime"), - '.gc.swift': ("GC", "gc"), - '.swift.perf': ("CM", "cache-misses"), + '.gc.swift': ("Swift GC", "gc"), + '.swift.perf': ("Swift CM", "cache-misses"), } benches = os.environ['BENCHES'].split(':') cats = os.environ['CATS'].split(':') -print(";".join(["Benchmark"] + [CATBAG[cat][0] for cat in cats])) +print(";".join(["Benchmark"] + [f'{CATBAG[cat][0]};{CATBAG[cat][0]} std' for cat in cats])) for bench in benches: norm = single('rbmap' if bench.startswith('rbmap') else bench, '.lean', 'etime').mean()