#!/usr/bin/env bash
# What does the hand-written backend cost, against LLVM, on the same programs?
#
# survey.sh answers "does it agree". This answers "what does agreeing cost",
# which is item 7 of docs/handoffs/HANDOFF-x86-rt.md and the one thing about this backend
# nobody had a number for. It builds each program the same two ways the
# survey does, and for each records three sizes and a time:
#
#   file    the whole executable on disk. Mostly runtime and libc glue, and
#           the least interesting of the three -- it is here because it is
#           the number anybody looks at first, and it should be visible how
#           much of it is noise.
#   text    the .text section, from `size -A`. Still contains flan_rt.o,
#           which is the same object on both sides.
#   own     the sum of the sizes of the defined symbols named `flan.<name>`
#           -- the program's *own* code and nothing else. The runtime's C is
#           `flan_<name>` with an underscore, so the two do not collide, and
#           spot-checking a runtime symbol on both sides (flan_map_clone,
#           0x4b3 either way) says the runtime really is byte-identical and
#           the difference in `own` is all codegen.
#
# The `own` column is the measurement; the other two are context.
#
# A fourth build, LLVM with --debug, is the reference that makes the number
# readable. --debug forces -O0, so it is LLVM's codegen with its optimiser
# switched off -- the closest thing available to what this backend is doing,
# which has no optimiser at all. Without it every ratio silently blames the
# backend for the whole of mem2reg and inlining. (--x86 --debug is refused,
# so the column exists on one side only, and that is the point of it.)
#
# Time is best-of-N, not a mean: a mean measures the other tenants of the
# machine. Even so, a corpus program is mostly process startup -- these are
# milliseconds -- so read the time column only where it is tens of
# milliseconds or more, and read the rest as size.
#
# Usage: spike/x86/cost.sh [name-substring ...]   -> a TSV on stdout
#   COST_FLAGS=--dev   extra flags, given to both sides, as SURVEY_FLAGS is
#   COST_REPS=5        timing repetitions
#   COST_O0=0          skip the LLVM -O0 reference column
set -u
orig=$(pwd)
here=$(cd "$(dirname "$0")" && pwd)
root=$(cd "$here/../.." && pwd)
cd "$root" || exit 1

if [ -n "${FLAN:-}" ]; then
  case $FLAN in /*) flan=$FLAN;; *) flan=$orig/$FLAN;; esac
else
  dune build --root . bin/main.exe 2>&1 | head -30
  flan=$root/_build/default/bin/main.exe
fi
test -x "$flan" || { echo "build failed" >&2; exit 1; }

# Not mktemp under /tmp by default: this writes a few hundred executables of
# a megabyte or two, and a full /tmp on this machine has already frozen one
# session. The guard is cheap and a wedged run is not.
# A directory of this run's own, made with a plain mkdir so that a second
# copy of this script cannot land in the first one's: two runs sharing a
# scratch directory overwrite each other's `l` and `x` between the build and
# the timing, and the result is a row of numbers that belong to two different
# programs. That happened once here and the numbers looked entirely ordinary.
work=${COST_OUT:-${TMPDIR:-/tmp}/flan-cost}
mkdir -p "$work" || exit 1
out=$work/run.$$
mkdir "$out" || exit 1
trap 'rm -rf "$out"' EXIT
free=$(df -Pk "$out" | awk 'NR==2 {print $4}')
[ "$free" -gt 2000000 ] || { echo "less than 2GB free at $out" >&2; exit 1; }

forever="dev-loop dev-watch"
REPS=${COST_REPS:-5}
read -r -a extra <<<"${COST_FLAGS:-}"
o0=${COST_O0:-1}
[ -z "${COST_FLAGS:-}" ] || o0=0

# The sum of the defined text symbols the compiler itself named. `nm -S`
# prints value, size, type, name; a symbol with no size is not printed with
# four fields at all, which is why the guard is on NF.
own () {
  nm --defined-only -S "$1" 2>/dev/null \
    | awk 'NF==4 && ($3=="T"||$3=="t") && $4 ~ /^flan\./ {n+=strtonum("0x"$2)} END{print n+0}'
}
text () { size -A "$1" 2>/dev/null | awk '$1==".text" {print $2}'; }

# Best of REPS, in whole microseconds. A program that fails on one run and
# not another would make this meaningless, so the exit status of the first
# run is remembered and a run that disagrees with it poisons the row as -1.
# A program that hits the timeout is not timed at all: several of the corpus
# programs are agents or daemons that sit waiting for something that is not
# there, and five repetitions of a twenty-second wait, twice, is most of an
# afternoon spent measuring `timeout`.
best () {
  exe=$1; min=; rc0=
  for i in $(seq "$REPS"); do
    t0=$(date +%s%N)
    ( cd "$out" && timeout 20 "$exe" >/dev/null 2>&1 </dev/null )
    rc=$?
    t1=$(date +%s%N)
    [ -n "$rc0" ] || rc0=$rc
    [ "$rc" = "$rc0" ] || { echo "-1"; return; }
    d=$(( (t1 - t0) / 1000 ))
    [ "$d" -lt 15000000 ] || { echo "-1"; return; }
    if [ -z "$min" ] || [ "$d" -lt "$min" ]; then min=$d; fi
  done
  echo "$min"
}

printf 'name\tllvm_file\tllvm_text\tllvm_own\to0_own\tx86_file\tx86_text\tx86_own\tllvm_us\tx86_us\n'

for src in "$root"/test/programs/*.flan "$root"/spike/x86/*.flan; do
  name=$(basename "$src" .flan)
  if [ $# -gt 0 ]; then
    want=0
    for pat in "$@"; do case "$name" in *"$pat"*) want=1;; esac; done
    [ $want = 1 ] || continue
  fi
  case " $forever " in *" $name "*) continue;; esac

  rm -f "$out/l" "$out/x" "$out/d"
  "$flan" build "$src" "${extra[@]}" -o "$out/l" >/dev/null 2>&1 || continue
  # No main is a link failure, and it leaves nothing behind to measure.
  test -x "$out/l" || continue
  "$flan" build "$src" --x86 "${extra[@]}" -o "$out/x" >/dev/null 2>&1 || continue
  test -x "$out/x" || continue

  d0=0
  if [ "$o0" = 1 ] && "$flan" build "$src" --debug -o "$out/d" >/dev/null 2>&1; then
    d0=$(own "$out/d")
  fi

  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$name" \
    "$(stat -c %s "$out/l")" "$(text "$out/l")" "$(own "$out/l")" "$d0" \
    "$(stat -c %s "$out/x")" "$(text "$out/x")" "$(own "$out/x")" \
    "$(best "$out/l")" "$(best "$out/x")"
done