chore: more benchmarking setup
This commit is contained in:
Sebastian Ullrich
parent
223f1073d1
commit
78bc2fd92b
4 changed files with 39 additions and 41 deletions
|
|
@ -74,8 +74,6 @@ binarytrees%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -threaded -r
|
|||
|
||||
%.ml.out: %.ml
|
||||
$(OCAML) $(OCAML_FLAGS) $< -o $@
|
||||
%.gc.ml.out: OCAML_FLAGS += -runtime-variant i
|
||||
%.gc.ml: %.ml; ln -f $< $@
|
||||
%.flambda.ml.out: OCAML = $(OCAML_FLAMBDA)
|
||||
%.flambda.ml: %.ml; ln -f $< $@
|
||||
|
||||
|
|
@ -106,6 +104,7 @@ bench/parser.%.bench: BENCH_PARAMS = $(PARSER_TEST_FILE) 100
|
|||
|
||||
bench/binarytrees.%.bench: BENCH_PARAMS = 21
|
||||
bench/binarytrees.ml.bench: BENCH_PARAMS = $$(nproc) 21
|
||||
bench/binarytrees.gc.ml.bench: BENCH_PARAMS = $$(nproc) 21
|
||||
|
||||
bench/deriv.%.bench: BENCH_PARAMS = 10
|
||||
|
||||
|
|
@ -125,7 +124,7 @@ bench/unionfind.%.bench: BENCH_PARAMS = 3000000
|
|||
|
||||
bench/%gc.lean.bench: %lean.out | bench
|
||||
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
||||
-d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio -S 'lean_del,lean_dealloc_small,lean_free_small' | ./lean-gc.py"\
|
||||
-d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio -S 'lean_dec_ref_cold,lean_del_core,lean_free_small' | ./lean-gc.py"\
|
||||
--runner output --out $@
|
||||
|
||||
bench/%gc.hs.bench: %hs.out | bench
|
||||
|
|
@ -133,9 +132,9 @@ bench/%gc.hs.bench: %hs.out | bench
|
|||
-d $< "./$< +RTS -t --machine-readable -RTS $(BENCH_PARAMS) 2>&1 >/dev/null | ./ghc-gc.py"\
|
||||
--runner output --out $@
|
||||
|
||||
bench/%gc.ml.bench: %gc.ml.out | bench
|
||||
bench/%gc.ml.bench: %ml.out | bench
|
||||
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
||||
-d $< "echo > $@.tmp && OCAML_INSTR_FILE=$@.tmp time -ao $@.tmp -f '%e' ./$< $(BENCH_PARAMS) >/dev/null && ./ocaml-gc.py < $@.tmp"\
|
||||
-d $< "time -ao $@.tmp -f '%U' olly trace $@.tmp './$< $(BENCH_PARAMS)' > /dev/null && ./ocaml-gc.py < $@.tmp"\
|
||||
--runner output --out $@
|
||||
|
||||
bench/%gc.mlton.bench: %gc.mlton.out | bench
|
||||
|
|
@ -158,9 +157,6 @@ bench/%.perf.bench: %.out | bench
|
|||
-d $< "echo > $@.tmp && time -af '%e' perf stat -e cache-misses -x ';' ./$< $(BENCH_PARAMS) 2>&1 >/dev/null | ./perf.py"\
|
||||
--runner output --out $@
|
||||
|
||||
# fork() breaks instrumentation
|
||||
bench/binarytrees.gc.ml.bench: ; touch $@
|
||||
|
||||
# no benchmarksgame versions
|
||||
bench/binarytrees%mlton.bench: ; touch $@
|
||||
bench/binarytrees%mlton.perf.bench: ; touch $@
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// https://github.com/ocaml-bench/sandmark/blob/c668eb05aebb1dc9f57f322399bea7182640518c/benchmarks/multicore-numerical/binarytrees5_multicore.ml
|
||||
(* https://github.com/ocaml-bench/sandmark/blob/c668eb05aebb1dc9f57f322399bea7182640518c/benchmarks/multicore-numerical/binarytrees5_multicore.ml *)
|
||||
module T = Domainslib.Task
|
||||
|
||||
let num_domains = try int_of_string Sys.argv.(1) with _ -> 1
|
||||
|
|
|
|||
|
|
@ -1,13 +1,20 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import json
|
||||
|
||||
events = sys.stdin.readlines()
|
||||
wall_secs = float(events[-1])
|
||||
gc_nanos = 0
|
||||
for event in events[::-1]:
|
||||
data = event.split()
|
||||
# see also https://gitlab.com/gasche/gc-latency-experiment/blob/master/parse_ocaml_log.ml
|
||||
if len(data) == 4 and data[0] == '@@' and data[3] == 'dispatch':
|
||||
gc_nanos += int(data[2]) - int(data[1])
|
||||
print(f"gc: {gc_nanos*1e-9/wall_secs*100}")
|
||||
lines = sys.stdin.readlines()
|
||||
user_secs = float(lines[-1])
|
||||
events = json.loads("\n".join(lines[:-1])[:-2] + "]")
|
||||
starts = {}
|
||||
gc_msecs = 0
|
||||
for event in events:
|
||||
if event["name"] in ["minor", "major"]:
|
||||
tid = event["tid"]
|
||||
if event["ph"] == "B":
|
||||
if tid not in starts:
|
||||
starts[tid] = (event["name"], int(event["ts"]))
|
||||
elif tid in starts and starts[tid][0] == event["name"]:
|
||||
gc_msecs += int(event["ts"]) - starts[tid][1]
|
||||
del starts[tid]
|
||||
print(f"gc: {gc_msecs*1e-6/user_secs*100}")
|
||||
|
|
|
|||
|
|
@ -28,55 +28,50 @@ stddev_by_cat = collections.defaultdict(list)
|
|||
def pp(bench, cat, prop, norm):
|
||||
f = f"bench/{bench}{cat}.bench"
|
||||
if not open(f).read():
|
||||
return ""
|
||||
return ";"
|
||||
s = single(bench, cat, prop)
|
||||
norm = norm if prop == 'etime' else 1
|
||||
mean_by_cat[cat].append(s.mean() / norm)
|
||||
stddev_by_cat[cat].append(s.std_dev() / norm)
|
||||
num = number.FNumber(s.mean() / norm, abs_deviation=s.std_dev() / norm)
|
||||
num.settings["min_decimal_places"] = 2 if prop == 'etime' else 0
|
||||
num.settings["max_decimal_places"] = 2 if prop == 'etime' else 0
|
||||
return num.format() + ('%' if prop == 'gc' else '')
|
||||
num = s.mean() / norm
|
||||
return f'{num:.3f};{s.std_dev() / norm:.3f}'
|
||||
|
||||
def mean(cat, prop):
|
||||
mean = st.gmean(mean_by_cat[cat])
|
||||
stddev = st.gmean(stddev_by_cat[cat])
|
||||
num = number.FNumber(mean, abs_deviation=stddev)
|
||||
num.settings["min_decimal_places"] = 2 if prop == 'etime' else 0
|
||||
num.settings["max_decimal_places"] = 2 if prop == 'etime' else 0
|
||||
return num.format() + ('%' if prop == 'gc' else '') #if prop == 'etime' else '-'
|
||||
return f'{mean:.3f};{stddev:.3f}'
|
||||
|
||||
CATBAG = {
|
||||
'.lean': ("Lean [s]", "etime"),
|
||||
'.lean': ("Lean", "etime"),
|
||||
'.no_reuse.lean': ("-reuse", "etime"),
|
||||
'.no_borrow.lean': ("-borrow", "etime"),
|
||||
'.no_st.lean': ("-ST", "etime"),
|
||||
'.gcc.lean': ("Lean+GCC9", "etime"),
|
||||
'.gc.lean': ("del [%]", "gc"),
|
||||
'.lean.perf': ("cache misses (CM) [1M/s]", "cache-misses"),
|
||||
'.gc.lean': ("Lean del", "gc"),
|
||||
'.lean.perf': ("CM", "cache-misses"),
|
||||
'.hs': ("GHC", "etime"),
|
||||
'.gc.hs': ("GC [%]", "gc"),
|
||||
'.hs.perf': ("CM", "cache-misses"),
|
||||
'.gc.hs': ("GHC GC", "gc"),
|
||||
'.hs.perf': ("GHC CM", "cache-misses"),
|
||||
'.llvm.hs': ("GHC -fllvm", "etime"),
|
||||
'.strict.hs': ("GHC -XStrict", "etime"),
|
||||
'.ml': ("OCaml", "etime"),
|
||||
'.gc.ml': ("GC", "gc"),
|
||||
'.ml.perf': ("CM", "cache-misses"),
|
||||
'.gc.ml': ("OCaml GC", "gc"),
|
||||
'.ml.perf': ("OCaml CM", "cache-misses"),
|
||||
'.flambda.ml': ("OCaml+Flambda", "etime"),
|
||||
'.mlton': ("MLton", "etime"),
|
||||
'.gc.mlton': ("GC", "gc"),
|
||||
'.mlton.perf': ("CM", "cache-misses"),
|
||||
'.gc.mlton': ("MLton GC", "gc"),
|
||||
'.mlton.perf': ("MLton CM", "cache-misses"),
|
||||
'.mlkit': ("MLKit", "etime"),
|
||||
'.gc.mlkit': ("GC", "gc"),
|
||||
'.mlkit.perf': ("CM", "cache-misses"),
|
||||
'.gc.mlkit': ("MLKit GC", "gc"),
|
||||
'.mlkit.perf': ("MLKit CM", "cache-misses"),
|
||||
'.swift': ("Swift", "etime"),
|
||||
'.gc.swift': ("GC", "gc"),
|
||||
'.swift.perf': ("CM", "cache-misses"),
|
||||
'.gc.swift': ("Swift GC", "gc"),
|
||||
'.swift.perf': ("Swift CM", "cache-misses"),
|
||||
}
|
||||
|
||||
benches = os.environ['BENCHES'].split(':')
|
||||
cats = os.environ['CATS'].split(':')
|
||||
print(";".join(["Benchmark"] + [CATBAG[cat][0] for cat in cats]))
|
||||
print(";".join(["Benchmark"] + [f'{CATBAG[cat][0]};{CATBAG[cat][0]} std' for cat in cats]))
|
||||
|
||||
for bench in benches:
|
||||
norm = single('rbmap' if bench.startswith('rbmap') else bench, '.lean', 'etime').mean()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue