flan/spike/x86/survey.sh

#!/usr/bin/env bash
# Does the hand-written backend agree with LLVM?
#
# The only honest test of a hand-encoded backend is what the program prints and
# what it exits with -- DISCUSS.md item 15 and item 16 both say so, and both
# say it after a disassembly that read perfectly beside a wrong answer. So this
# builds every program in test/programs twice, runs both, and diffs stdout,
# stderr and the exit status. objdump is for after a program already has the
# wrong answer.
#
# stderr is not an afterthought: every message the condition machinery produces
# goes there -- the bounds and slice errors, the three restart refusals, the
# transfer failure -- and each carries a location string this backend emits by
# hand as a .rodata label and a length in a register. An exit status of 134
# with the wrong text beside it is exactly the failure that looks like a
# match.
#
# Both sides get the same bounds-check setting (the default: on). A sweep that
# compared a checked build against an unchecked one would say nothing about
# bounds.flan, which is the one program the two backends disagreed about.
#
# Five outcomes, and the third is the progress meter:
#
#   MATCH     built both ways, same stdout, same stderr, same exit status
#   DIFFER    built both ways, and disagreed
#   REFUSED   X86.Unsupported -- a node this backend does not lower (exit 3)
#   NOX86     failed to build through --x86 for some other reason
#   SKIP      no main, does not compile at all, or does not terminate
#
# Over test/programs, and over spike/x86's own probes, which are here for the
# paths the corpus does not walk.
#
# Usage: spike/x86/survey.sh [name-substring ...]
set -u
orig=$(pwd)
here=$(cd "$(dirname "$0")" && pwd)
root=$(cd "$here/../.." && pwd)
cd "$root" || exit 1

# FLAN is how the dune @x86 alias hands this script a compiler that dune has
# already built. Building it here instead would be a second dune inside the
# first one's lock, which does not run at all; and the alias has bin/main.exe
# in its deps precisely so it does not have to. Standalone -- the way the
# baseline in every handoff was measured -- nothing sets it and the build
# below is what it always was.
if [ -n "${FLAN:-}" ]; then
  # Resolved against the directory this was invoked from, not against $root:
  # dune spells its deps relative to the dune file, and the cd above has
  # already happened by the time this is read.
  case $FLAN in /*) flan=$FLAN;; *) flan=$orig/$FLAN;; esac
else
  dune build --root . bin/main.exe 2>&1 | head -30
  flan=$root/_build/default/bin/main.exe
fi
test -x "$flan" || { echo "build failed"; exit 1; }

# Where the corpus is read from. Under dune the script runs from the build
# tree, where test/programs is present but spike/x86 is not, so the alias
# points this at the source tree and gets both.
corpus=${SURVEY_CORPUS:-$root}

out=$(mktemp -d); trap 'rm -rf "$out"' EXIT

# The two that run until something stops them. Not a failure and not a match;
# they are excluded by name because a timeout cannot tell them apart from a
# backend that hung.
forever="dev-loop dev-watch"

TIMEOUT=${TIMEOUT:-20}

# Extra flags, given to *both* sides. SURVEY_FLAGS=--dev is the one that has a
# use: a dev build with nothing yet redefined must behave exactly like a
# release one -- the indirection cell is the only difference -- so the whole
# corpus is a test of the cells, and of nothing else changing beside them.
# Off by default, so the counts above the line stay the same measurement.
read -r -a extra <<<"${SURVEY_FLAGS:-}"

declare -a match=() differ=() refused=() nox86=() skip=()

for src in "$corpus"/test/programs/*.flan "$corpus"/spike/x86/*.flan; do
  name=$(basename "$src" .flan)
  if [ $# -gt 0 ]; then
    want=0
    for pat in "$@"; do case "$name" in *"$pat"*) want=1;; esac; done
    [ $want = 1 ] || continue
  fi
  case " $forever " in *" $name "*) skip+=("$name:runs-forever"); continue;; esac

  # LLVM first. A program that does not compile at all, or has no main, is not
  # this backend's business -- the frontend refused it either way.
  if ! "$flan" build "$src" "${extra[@]}" -o "$out/$name.llvm" \
       >"$out/$name.llvm.err" 2>&1; then
    if grep -q "in function \`_start\|undefined reference to \`main\|crt1.o" "$out/$name.llvm.err"; then
      skip+=("$name:no-main")
    else
      skip+=("$name:does-not-compile")
    fi
    continue
  fi

  "$flan" build "$src" --x86 "${extra[@]}" -o "$out/$name.x86" \
    >"$out/$name.x86.err" 2>&1
  rc=$?
  if [ $rc = 3 ]; then
    why=$(head -1 "$out/$name.x86.err" | sed 's/^x86: //')
    refused+=("$name:$why")
    continue
  fi
  if [ $rc != 0 ]; then
    nox86+=("$name:$(head -1 "$out/$name.x86.err")")
    continue
  fi

  ( cd "$out" && timeout "$TIMEOUT" "$out/$name.llvm" \
      >"$out/$name.llvm.out" 2>"$out/$name.llvm.diag" )
  a=$?
  ( cd "$out" && timeout "$TIMEOUT" "$out/$name.x86" \
      >"$out/$name.x86.out" 2>"$out/$name.x86.diag" )
  b=$?
  if [ "$a" = "$b" ] && cmp -s "$out/$name.llvm.out" "$out/$name.x86.out" \
     && cmp -s "$out/$name.llvm.diag" "$out/$name.x86.diag"; then
    match+=("$name")
  else
    differ+=("$name:llvm=$a/x86=$b")
    if [ "${SURVEY_SHOW:-}" = 1 ]; then
      echo "--- $name: llvm exit $a, x86 exit $b"
      diff "$out/$name.llvm.out" "$out/$name.x86.out" | head -20
      diff "$out/$name.llvm.diag" "$out/$name.x86.diag" | head -20
    fi
  fi
done

echo
echo "MATCH   ${#match[@]}"
echo "DIFFER  ${#differ[@]}"
[ "${#differ[@]}" = 0 ] || printf '          %s\n' "${differ[@]}"
echo "REFUSED ${#refused[@]}"
if [ "${#refused[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then
  printf '%s\n' "${refused[@]}" | sed 's/^[^:]*://' | sort | uniq -c | sort -rn \
    | sed 's/^/          /'
fi
echo "NOX86   ${#nox86[@]}"
[ "${#nox86[@]}" = 0 ] || printf '          %s\n' "${nox86[@]}"
echo "SKIP    ${#skip[@]}"
if [ "${#skip[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then
  printf '%s\n' "${skip[@]}" | sed 's/^[^:]*://' | sort | uniq -c \
    | sed 's/^/          /'
fi

# Strict mode, for the @x86 alias: the counts above are a report, and a report
# nobody reads is how two refusals from another lane's new primitive sat in
# the tree for a month. A DIFFER is a wrong answer and a refusal by name is a
# node this backend has stopped lowering; either is a failure. NOX86 and SKIP
# are not: the first is usually a toolchain that is not installed here, and
# the second is the frontend refusing the program on both sides.
if [ "${SURVEY_STRICT:-}" = 1 ]; then
  if [ "${#differ[@]}" != 0 ] || [ "${#refused[@]}" != 0 ]; then
    echo
    echo "x86 survey FAILED: ${#differ[@]} differ, ${#refused[@]} refused"
    exit 1
  fi
  echo
  echo "x86 survey ok: ${#match[@]} match"
fi