219 lines
8.3 KiB
Makefile
219 lines
8.3 KiB
Makefile
## CONFIG
|
|
|
|
LEAN_BENCHES = binarytrees binarytrees.st deriv const_fold parser qsort rbmap rbmap_10 rbmap_2 rbmap_1 unionfind
|
|
CROSS_BENCHES = binarytrees binarytrees.st deriv const_fold rbmap rbmap_10 rbmap_2 rbmap_1
|
|
|
|
LEAN_CATS = .lean .no_reuse.lean .no_borrow.lean .no_st.lean
|
|
CROSS_CATS = .lean .gc.lean .hs .gc.hs .ml .gc.ml .mlton .gc.mlton .mlkit .gc.mlkit .swift .gc.swift
|
|
CROSS_TIME_CATS = .lean .hs .ml .mlton .mlkit .swift
|
|
RETIRED_CATS = .gcc.lean .llvm.hs .strict.hs .flambda.ml .lean.perf .hs.perf .ml.perf .mlton.perf .mlkit.perf .swift.perf
|
|
|
|
LEAN_FLAGS =
|
|
LEANC_FLAGS = -O3 -DNDEBUG
|
|
GHC_FLAGS = -O3
|
|
OCAML_FLAGS = -O3
|
|
MLTON_FLAGS =
|
|
MLKIT_FLAGS =
|
|
SWIFTC_FLAGS = -O -whole-module-optimization
|
|
|
|
TEMCI_FLAGS = --settings cross.yaml
|
|
|
|
TEMCI ?= temci
|
|
LEAN_BIN ?= ../../build/release/stage1/bin
|
|
GHC ?= ghc
|
|
OCAML ?= ocamlopt.opt
|
|
MLTON_BIN ?= /usr/bin
|
|
MLKIT ?= mlkit
|
|
SWIFTC ?= swiftc
|
|
|
|
PARSER_TEST_FILE ?= ../../src/Init/Prelude.lean
|
|
|
|
## IMPLEMENTATION
|
|
|
|
LEAN_INPUTS = $(foreach bench,$(LEAN_BENCHES), $(foreach cat, $(LEAN_CATS), $(bench)$(cat)))
|
|
CROSS_INPUTS = $(foreach bench,$(CROSS_BENCHES), $(foreach cat, $(CROSS_CATS), $(bench)$(cat)))
|
|
|
|
.SECONDARY: $(LEAN_INPUTS:%=%.out) $(LEAN_INPUTS:%=bench/%.bench) $(CROSS_INPUTS:%=%.out) $(CROSS_INPUTS:%=bench/%.bench)
|
|
.DELETE_ON_ERROR:
|
|
|
|
all: report_lean.tex report_cross.tex report
|
|
|
|
# disable some built-in rules
|
|
%.lean:
|
|
%.out: %
|
|
|
|
%.lean.c: %.lean
|
|
$(LEAN_BIN)/lean --c=$@ $(LEAN_FLAGS) $<
|
|
%.lean.out: %.lean.c
|
|
$(LEAN_BIN)/leanc $(LEANC_FLAGS) -o $@ $<
|
|
# Binaries x.lean.out and x.gcc.lean.out etc. are produced by the
|
|
# same rules and x.lean source file by copying the latter to
|
|
# x.gcc.lean. This also avoids conflicts between intermediate
|
|
# files of the two binaries.
|
|
%.gcc.lean.out: LEAN_BIN = $(LEAN_GCC_BIN)
|
|
%.gcc.lean: %.lean; ln -f $< $@
|
|
%.no_reuse.lean.c: LEAN_BIN = $(LEAN_NO_REUSE_BIN)
|
|
%.no_reuse.lean.out: LEAN_BIN = $(LEAN_NO_REUSE_BIN)
|
|
%.no_reuse.lean: %.lean; ln -f $< $@
|
|
%.no_borrow.lean.c: LEAN_BIN = $(LEAN_NO_BORROW_BIN)
|
|
%.no_borrow.lean.out: LEAN_BIN = $(LEAN_NO_BORROW_BIN)
|
|
%.no_borrow.lean: %.lean; ln -f $< $@
|
|
%.no_st.lean.out: LEAN_BIN = $(LEAN_NO_ST_BIN)
|
|
%.no_st.lean: %.lean; ln -f $< $@
|
|
|
|
%.hs.out: %.hs
|
|
$(GHC) $(GHC_FLAGS) -rtsopts $< -o $@
|
|
%.llvm.hs.out: GHC_FLAGS += -fllvm
|
|
%.llvm.hs: %.hs; ln -f $< $@
|
|
%.strict.hs.out: GHC_FLAGS += -XStrict
|
|
%.strict.hs: %.hs; ln -f $< $@
|
|
|
|
binarytrees.hs: binarytrees.ghc-6.hs; ln -f $< $@
|
|
# NOTE: changed `-N4` rtsopt to `-N` to be less system-dependent
|
|
binarytrees%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -threaded -rtsopts -with-rtsopts='-N -K128M -H'
|
|
binarytrees.st%hs.out: GHC_FLAGS += --make -O2 -XBangPatterns -dynamic -rtsopts -with-rtsopts='-K128M -H'
|
|
|
|
%.ml.out: %.ml
|
|
$(OCAML) $(OCAML_FLAGS) $< -o $@
|
|
%.flambda.ml.out: OCAML = $(OCAML_FLAMBDA)
|
|
%.flambda.ml: %.ml; ln -f $< $@
|
|
|
|
binarytrees.ml: binarytrees5_multicore.ml; ln -f $< $@
|
|
binarytrees.st.ml: binarytrees5.ml; ln -f $< $@
|
|
binarytrees%ml.out: OCAML_FLAGS += -noassert -fPIC -nodynlink -inline 100 -O3 -package domainslib -linkpkg
|
|
binarytrees%ml.out: OCAML = ocamlfind ocamlopt
|
|
|
|
binarytrees.st.sml: binarytrees.st.mlton-2.sml; ln -f $< $@
|
|
%.mlton.out: %.sml
|
|
$(MLTON_BIN)/mlton $(MLTON_FLAGS) -output $@ $<
|
|
%.gc.mlton.out: MLTON_FLAGS = -profile time
|
|
%.gc.sml: %.sml; ln -f $< $@
|
|
|
|
%.mlkit.out: %.sml
|
|
$(MLKIT) $(MLKIT_FLAGS) -o $@ $<
|
|
|
|
%.swift.out: %.swift
|
|
$(SWIFTC) $(SWIFTC_FLAGS) -o $@ $<
|
|
|
|
binarytrees%swift.out: SWIFTC_FLAGS = -Ounchecked -I Include/swift/apr
|
|
|
|
bench:
|
|
-@mkdir bench
|
|
|
|
bench/%.bench: %.out | bench
|
|
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS) -d $< "./$< $(BENCH_PARAMS)" --out $@
|
|
|
|
bench/parser.%.bench: BENCH_PARAMS = $(PARSER_TEST_FILE) 50
|
|
|
|
bench/binarytrees.%.bench: BENCH_PARAMS = 21
|
|
bench/binarytrees.ml.bench: BENCH_PARAMS = $$(nproc) 21
|
|
bench/binarytrees.gc.ml.bench: BENCH_PARAMS = $$(nproc) 21
|
|
|
|
bench/deriv.%.bench: BENCH_PARAMS = 10
|
|
|
|
bench/const_fold.%.bench: BENCH_PARAMS = 23
|
|
|
|
bench/qsort.%.bench: BENCH_PARAMS = 400
|
|
|
|
bench/rbmap.%.bench: BENCH_PARAMS = 2000000
|
|
|
|
rbmap_1.%.out: rbmap_checkpoint.%.out; ln -f $< $@
|
|
bench/rbmap_1.%.bench: BENCH_PARAMS = 2000000 1
|
|
|
|
rbmap_2.%.out: rbmap_checkpoint.%.out; ln -f $< $@
|
|
bench/rbmap_2.%.bench: BENCH_PARAMS = 2000000 2
|
|
|
|
rbmap_10.%.out: rbmap_checkpoint.%.out; ln -f $< $@
|
|
bench/rbmap_10.%.bench: BENCH_PARAMS = 2000000 10
|
|
|
|
bench/unionfind.%.bench: BENCH_PARAMS = 3000000
|
|
|
|
bench/%gc.lean.bench: %lean.out | bench
|
|
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
|
-d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio -S 'lean_inc_ref_cold,lean_inc_ref_n_cold,lean_dec_ref_cold,lean_del_core,lean_free_small' | ./lean-gc.py"\
|
|
--runner output --out $@
|
|
|
|
bench/%gc.hs.bench: %hs.out | bench
|
|
$(TEMCI) short exec $(TEMCI_FLAGS)\
|
|
-d $< "./$< +RTS -t --machine-readable -RTS $(BENCH_PARAMS) 2>&1 >/dev/null | ./ghc-gc.py"\
|
|
--runner output --out $@
|
|
|
|
bench/%gc.ml.bench: %ml.out | bench
|
|
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
|
-d $< "time -ao $@.tmp -f '%U' olly trace $@.tmp './$< $(BENCH_PARAMS)' > /dev/null && ./ocaml-gc.py < $@.tmp"\
|
|
--runner output --out $@
|
|
|
|
bench/%gc.mlton.bench: %gc.mlton.out | bench
|
|
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
|
-d $< './$< $(BENCH_PARAMS) >/dev/null && $(MLTON_BIN)/mlprof ./$< mlmon.out | awk '\''$$1 == "<gc>" { print "gc: " substr($$2, 0, length($$2)-1); found = 1 } END { if (found != 1) { print "gc: 0.000001" } }'\'\
|
|
--runner output --out $@
|
|
|
|
bench/%gc.mlkit.bench: %gc.mlkit.out | bench
|
|
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
|
-d $< "time -f '%e' ./$< -report_gc $(BENCH_PARAMS) 2>&1 >/dev/null | ./mlkit-gc.py"\
|
|
--runner output --out $@
|
|
|
|
bench/%gc.swift.bench: %swift.out | bench
|
|
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
|
-d $< "perf record -o $@.tmp ./$< $(BENCH_PARAMS) >/dev/null && perf report -i $@.tmp -t ';' --stdio | grep -E 'retain|release|dealloc|free' | ./lean-gc.py"\
|
|
--runner output --out $@
|
|
|
|
bench/%.perf.bench: %.out | bench
|
|
ulimit -s unlimited && $(TEMCI) short exec $(TEMCI_FLAGS)\
|
|
-d $< "echo > $@.tmp && time -af '%e' perf stat -e cache-misses -x ';' ./$< $(BENCH_PARAMS) 2>&1 >/dev/null | ./perf.py"\
|
|
--runner output --out $@
|
|
|
|
# no benchmarksgame versions
|
|
bench/binarytrees.mlton.bench: ; touch $@
|
|
bench/binarytrees.gc.mlton.bench: ; touch $@
|
|
bench/binarytrees.mlton.perf.bench: ; touch $@
|
|
bench/binarytrees.mlkit.bench: ; touch $@
|
|
bench/binarytrees.gc.mlkit.bench: ; touch $@
|
|
bench/binarytrees.mlkit.perf.bench: ; touch $@
|
|
|
|
bench_lean: $(LEAN_INPUTS:%=bench/%.bench)
|
|
bench_cross: $(CROSS_INPUTS:%=bench/%.bench)
|
|
|
|
report/lean: ; -@mkdir -p $@
|
|
report/cross: ; -@mkdir -p $@
|
|
|
|
report/lean/%: $(foreach cat, $(LEAN_CATS), bench/%$(cat).bench) | report/lean
|
|
cat $^ > $@.tmp
|
|
$(TEMCI) report $@.tmp --settings cross.yaml --reporter html2 --html2_out $@ --html2_force_override --properties etime
|
|
|
|
report/cross/%: $(foreach cat, $(CROSS_TIME_CATS), bench/%$(cat).bench) | report/cross
|
|
cat $^ > $@.tmp
|
|
$(TEMCI) report $@.tmp --settings cross.yaml --reporter html2 --html2_out $@ --html2_force_override --properties etime
|
|
|
|
report/index.html: $(LEAN_BENCHES:%=report/lean/%) $(CROSS_BENCHES:%=report/cross/%)
|
|
echo "<html><h1>Lean variant benchmarks</h1><ul>"> $@
|
|
for bench in $(LEAN_BENCHES); do \
|
|
echo "<li><a href='lean/$$bench/report.html'>$$bench</a><br><img src='lean/$$bench/fig___SinglesProperty______etime___25.0.svg'></img></li>" >> $@; \
|
|
done
|
|
echo "</ul><h1>Cross-language benchmarks</h1><ul>">> $@
|
|
for bench in $(CROSS_BENCHES); do \
|
|
echo "<li><a href='cross/$$bench/report.html'>$$bench</a><br><img src='cross/$$bench/fig___SinglesProperty______etime___25.0.svg'></img></li>" >> $@; \
|
|
done
|
|
echo "</ul></html>">> $@
|
|
|
|
report: report/index.html
|
|
|
|
# yes.
|
|
space = $() $()
|
|
|
|
report_lean.csv report_lean_rbmap.csv: bench_lean
|
|
BENCHES=$(subst $(space),:,$(LEAN_BENCHES)) CATS=$(subst $(space),:,$(LEAN_CATS)) ./report.py report_lean
|
|
column -s';' -t < $@
|
|
|
|
report_cross.csv report_cross_rbmap.csv: bench_cross
|
|
BENCHES=$(subst $(space),:,$(CROSS_BENCHES)) CATS=$(subst $(space),:,$(CROSS_CATS)) ./report.py report_cross
|
|
column -s';' -t < $@
|
|
|
|
TO_TEX = 's/-/---/g;s/%/\\%/g;s/\.$$//g;s/(\([0-9]\)\([0-9]\))/\\ensuremath{\\tilde{\1}\\tilde{\2}}/g;s/(\([0-9]\))/\\ensuremath{\\tilde{\1}}/g;s/;/ \& /g;s/$$/\\\\/'
|
|
report_%.tex: report_%.csv
|
|
tail -n +2 $< | head -n -1 | sed 's/^\([^;]\+\)/\\verb!\1!/;'$(TO_TEX) > $@
|
|
echo -n '\midrule ' >> $@
|
|
tail -n 1 $< | sed 's/. /.\\ /;'$(TO_TEX) >> $@
|
|
|
|
clean:
|
|
-rm *.out bench/*
|