129 lines
5.5 KiB
Bash
Executable File
129 lines
5.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# What does the hand-written backend cost, against LLVM, on the same programs?
|
|
#
|
|
# survey.sh answers "does it agree". This answers "what does agreeing cost",
|
|
# which is item 7 of HANDOFF-x86-rt.md and the one thing about this backend
|
|
# nobody had a number for. It builds each program the same two ways the
|
|
# survey does, and for each records three sizes and a time:
|
|
#
|
|
# file the whole executable on disk. Mostly runtime and libc glue, and
|
|
# the least interesting of the three -- it is here because it is
|
|
# the number anybody looks at first, and it should be visible how
|
|
# much of it is noise.
|
|
# text the .text section, from `size -A`. Still contains flan_rt.o,
|
|
# which is the same object on both sides.
|
|
# own the sum of the sizes of the defined symbols named `flan.<name>`
|
|
# -- the program's *own* code and nothing else. The runtime's C is
|
|
# `flan_<name>` with an underscore, so the two do not collide, and
|
|
# spot-checking a runtime symbol on both sides (flan_map_clone,
|
|
# 0x4b3 either way) says the runtime really is byte-identical and
|
|
# the difference in `own` is all codegen.
|
|
#
|
|
# The `own` column is the measurement; the other two are context.
|
|
#
|
|
# A fourth build, LLVM with --debug, is the reference that makes the number
|
|
# readable. --debug forces -O0, so it is LLVM's codegen with its optimiser
|
|
# switched off -- the closest thing available to what this backend is doing,
|
|
# which has no optimiser at all. Without it every ratio silently blames the
|
|
# backend for the whole of mem2reg and inlining. (--x86 --debug is refused,
|
|
# so the column exists on one side only, and that is the point of it.)
|
|
#
|
|
# Time is best-of-N, not a mean: a mean measures the other tenants of the
|
|
# machine. Even so, a corpus program is mostly process startup -- these are
|
|
# milliseconds -- so read the time column only where it is tens of
|
|
# milliseconds or more, and read the rest as size.
|
|
#
|
|
# Usage: spike/x86/cost.sh [name-substring ...] -> a TSV on stdout
|
|
# COST_FLAGS=--dev extra flags, given to both sides, as SURVEY_FLAGS is
|
|
# COST_REPS=5 timing repetitions
|
|
# COST_O0=0 skip the LLVM -O0 reference column
|
|
set -u
|
|
orig=$(pwd)
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
root=$(cd "$here/../.." && pwd)
|
|
cd "$root" || exit 1
|
|
|
|
if [ -n "${FLAN:-}" ]; then
|
|
case $FLAN in /*) flan=$FLAN;; *) flan=$orig/$FLAN;; esac
|
|
else
|
|
dune build --root . bin/main.exe 2>&1 | head -30
|
|
flan=$root/_build/default/bin/main.exe
|
|
fi
|
|
test -x "$flan" || { echo "build failed" >&2; exit 1; }
|
|
|
|
# Not mktemp under /tmp by default: this writes a few hundred executables of
|
|
# a megabyte or two, and a full /tmp on this machine has already frozen one
|
|
# session. The guard is cheap and a wedged run is not.
|
|
out=${COST_OUT:-${TMPDIR:-/tmp}/flan-cost.$$}
|
|
mkdir -p "$out" || exit 1
|
|
trap 'rm -rf "$out"' EXIT
|
|
free=$(df -Pk "$out" | awk 'NR==2 {print $4}')
|
|
[ "$free" -gt 2000000 ] || { echo "less than 2GB free at $out" >&2; exit 1; }
|
|
|
|
forever="dev-loop dev-watch"
|
|
REPS=${COST_REPS:-5}
|
|
read -r -a extra <<<"${COST_FLAGS:-}"
|
|
o0=${COST_O0:-1}
|
|
[ -z "${COST_FLAGS:-}" ] || o0=0
|
|
|
|
# The sum of the defined text symbols the compiler itself named. `nm -S`
|
|
# prints value, size, type, name; a symbol with no size is not printed with
|
|
# four fields at all, which is why the guard is on NF.
|
|
own () {
|
|
nm --defined-only -S "$1" 2>/dev/null \
|
|
| awk 'NF==4 && ($3=="T"||$3=="t") && $4 ~ /^flan\./ {n+=strtonum("0x"$2)} END{print n+0}'
|
|
}
|
|
text () { size -A "$1" 2>/dev/null | awk '$1==".text" {print $2}'; }
|
|
|
|
# Best of REPS, in whole microseconds. A program that fails on one run and
|
|
# not another would make this meaningless, so the exit status of the first
|
|
# run is remembered and a run that disagrees with it poisons the row as -1.
|
|
# A program that hits the timeout is not timed at all: several of the corpus
|
|
# programs are agents or daemons that sit waiting for something that is not
|
|
# there, and five repetitions of a twenty-second wait, twice, is most of an
|
|
# afternoon spent measuring `timeout`.
|
|
best () {
|
|
exe=$1; min=; rc0=
|
|
for i in $(seq "$REPS"); do
|
|
t0=$(date +%s%N)
|
|
( cd "$out" && timeout 20 "$exe" >/dev/null 2>&1 </dev/null )
|
|
rc=$?
|
|
t1=$(date +%s%N)
|
|
[ -n "$rc0" ] || rc0=$rc
|
|
[ "$rc" = "$rc0" ] || { echo "-1"; return; }
|
|
d=$(( (t1 - t0) / 1000 ))
|
|
[ "$d" -lt 15000000 ] || { echo "-1"; return; }
|
|
if [ -z "$min" ] || [ "$d" -lt "$min" ]; then min=$d; fi
|
|
done
|
|
echo "$min"
|
|
}
|
|
|
|
printf 'name\tllvm_file\tllvm_text\tllvm_own\to0_own\tx86_file\tx86_text\tx86_own\tllvm_us\tx86_us\n'
|
|
|
|
for src in "$root"/test/programs/*.flan "$root"/spike/x86/*.flan; do
|
|
name=$(basename "$src" .flan)
|
|
if [ $# -gt 0 ]; then
|
|
want=0
|
|
for pat in "$@"; do case "$name" in *"$pat"*) want=1;; esac; done
|
|
[ $want = 1 ] || continue
|
|
fi
|
|
case " $forever " in *" $name "*) continue;; esac
|
|
|
|
rm -f "$out/l" "$out/x" "$out/d"
|
|
"$flan" build "$src" "${extra[@]}" -o "$out/l" >/dev/null 2>&1 || continue
|
|
# No main is a link failure, and it leaves nothing behind to measure.
|
|
test -x "$out/l" || continue
|
|
"$flan" build "$src" --x86 "${extra[@]}" -o "$out/x" >/dev/null 2>&1 || continue
|
|
test -x "$out/x" || continue
|
|
|
|
d0=0
|
|
if [ "$o0" = 1 ] && "$flan" build "$src" --debug -o "$out/d" >/dev/null 2>&1; then
|
|
d0=$(own "$out/d")
|
|
fi
|
|
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$name" \
|
|
"$(stat -c %s "$out/l")" "$(text "$out/l")" "$(own "$out/l")" "$d0" \
|
|
"$(stat -c %s "$out/x")" "$(text "$out/x")" "$(own "$out/x")" \
|
|
"$(best "$out/l")" "$(best "$out/x")"
|
|
done
|