08eb78a5b2
This PR sets up the new integrated test/bench suite. It then migrates all benchmarks and some related tests to the new suite. There's also some documentation and some linting. For now, a lot of the old tests are left alone so this PR doesn't become even larger than it already is. Eventually, all tests should be migrated to the new suite though so there isn't a confusing mix of two systems.
72 lines
2 KiB
Bash
Executable file
72 lines
2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
source ../../env_bench.sh
|
|
source "$TEST_DIR/util.sh"
|
|
|
|
STAGE_THIS="stage$STAGE"
|
|
STAGE_NEXT="stage$((STAGE + 1))"
|
|
|
|
BUILD_ROOT="$(realpath "$BUILD_DIR/..")"
|
|
BUILD_THIS="$(realpath "$BUILD_ROOT/$STAGE_THIS")"
|
|
BUILD_NEXT="$(realpath "$BUILD_ROOT/$STAGE_NEXT")"
|
|
|
|
OUT="$(realpath measurements.jsonl)"
|
|
|
|
|
|
|
|
echo
|
|
echo ">"
|
|
echo "> Configuring $STAGE_NEXT..."
|
|
echo ">"
|
|
|
|
# Building a stage mostly affects files in that stage's build directory.
|
|
# However, the bench suite runs inside the source directory for developer UX
|
|
# reasons, so some stage-specific bench suite files are generated in the source
|
|
# directory (namely the env_*.sh files).
|
|
#
|
|
# To avoid messing up the rest of the bench suite, we restore those files to
|
|
# STAGE_THIS's versions immediately after we configure STAGE_NEXT. Yes, this is
|
|
# a big hack, but it allows running the build benchmark as part of the bench
|
|
# suite instead of completely separately.
|
|
#
|
|
# Configuring STAGE_NEXT also builds all stages up to and including STAGE_THIS.
|
|
make -C "$BUILD_ROOT" -j"$(nproc)" "$STAGE_NEXT-configure"
|
|
make -C "$BUILD_ROOT" -j"$(nproc)" "$STAGE_THIS-configure"
|
|
|
|
|
|
|
|
echo
|
|
echo ">"
|
|
echo "> Warming up $STAGE_NEXT..."
|
|
echo ">"
|
|
|
|
make -C "$BUILD_NEXT" -j"$(nproc)"
|
|
find "$BUILD_NEXT/lib" -name "*.olean" -delete
|
|
rm -f measurements.jsonl
|
|
|
|
|
|
|
|
echo
|
|
echo ">"
|
|
echo "> Building $STAGE_NEXT..."
|
|
echo ">"
|
|
|
|
LAKE_OVERRIDE_LEAN=true LEAN="$(realpath fake_root/bin/lean)" \
|
|
WRAPPER_PREFIX="$(realpath fake_root)" WRAPPER_OUT="$OUT" \
|
|
lakeprof record -- \
|
|
"$TEST_DIR/measure.py" -t build -d -a -- \
|
|
make -C "$BUILD_NEXT" -j"$(nproc)"
|
|
|
|
|
|
|
|
echo
|
|
echo ">"
|
|
echo "> Analyzing lakeprof data..."
|
|
echo ">"
|
|
|
|
# Lakeprof must be executed in the src dir because it obtains some metadata by
|
|
# calling lake in its current working directory.
|
|
mv lakeprof.log "$SRC_DIR"
|
|
pushd "$SRC_DIR"
|
|
lakeprof report -pj | jq '{metric: "build/lakeprof/longest build path//wall-clock", value: .[-1][2], unit: "s"}' -c >> "$OUT"
|
|
lakeprof report -rj | jq '{metric: "build/lakeprof/longest rebuild path//wall-clock", value: .[-1][2], unit: "s"}' -c >> "$OUT"
|
|
popd
|