Joseph Ferano
8e5fcf54cc
dev-chatty.flan outlives the surveys' twenty seconds by design, and unlike dev-repl it prints while it does -- so the two backends stop at different lines and the diff reports on scheduling rather than on lowering. It joins dev-loop and dev-watch in the excluded-by-name list in both sweeps, with the distinction written down. rt_flush_out is guarded on __wasm__: the pipe it is careful about belongs to a merged flan dev, which is only ever a native host, and wasm32 need not answer for a descriptor mode its runtime may model differently. And three comments that went false with the _exit: the atexit registration in the merged entry point is no longer there for rt_die, which unlinks the socket for itself now, so both places that said so say what it is actually left covering.
174 lines
7.1 KiB
Bash
Executable File
174 lines
7.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Does the hand-written backend agree with LLVM?
|
|
#
|
|
# The only honest test of a hand-encoded backend is what the program prints and
|
|
# what it exits with -- docs/DISCUSS.md item 15 and item 16 both say so, and both
|
|
# say it after a disassembly that read perfectly beside a wrong answer. So this
|
|
# builds every program in test/programs twice, runs both, and diffs stdout,
|
|
# stderr and the exit status. objdump is for after a program already has the
|
|
# wrong answer.
|
|
#
|
|
# stderr is not an afterthought: every message the condition machinery produces
|
|
# goes there -- the bounds and slice errors, the three restart refusals, the
|
|
# transfer failure -- and each carries a location string this backend emits by
|
|
# hand as a .rodata label and a length in a register. An exit status of 134
|
|
# with the wrong text beside it is exactly the failure that looks like a
|
|
# match.
|
|
#
|
|
# Both sides get the same bounds-check setting (the default: on). A sweep that
|
|
# compared a checked build against an unchecked one would say nothing about
|
|
# bounds.flan, which is the one program the two backends disagreed about.
|
|
#
|
|
# Five outcomes, and the third is the progress meter:
|
|
#
|
|
# MATCH built both ways, same stdout, same stderr, same exit status
|
|
# DIFFER built both ways, and disagreed
|
|
# REFUSED X86.Unsupported -- a node this backend does not lower (exit 3)
|
|
# NOX86 failed to build through --x86 for some other reason
|
|
# SKIP no main, does not compile at all, or does not terminate
|
|
#
|
|
# Over test/programs, and over spike/x86's own probes, which are here for the
|
|
# paths the corpus does not walk.
|
|
#
|
|
# Usage: spike/x86/survey.sh [name-substring ...]
|
|
set -u
|
|
orig=$(pwd)
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
root=$(cd "$here/../.." && pwd)
|
|
cd "$root" || exit 1
|
|
|
|
# FLAN is how the dune @x86 alias hands this script a compiler that dune has
|
|
# already built. Building it here instead would be a second dune inside the
|
|
# first one's lock, which does not run at all; and the alias has bin/main.exe
|
|
# in its deps precisely so it does not have to. Standalone -- the way the
|
|
# baseline in every handoff was measured -- nothing sets it and the build
|
|
# below is what it always was.
|
|
if [ -n "${FLAN:-}" ]; then
|
|
# Resolved against the directory this was invoked from, not against $root:
|
|
# dune spells its deps relative to the dune file, and the cd above has
|
|
# already happened by the time this is read.
|
|
case $FLAN in /*) flan=$FLAN;; *) flan=$orig/$FLAN;; esac
|
|
else
|
|
dune build --root . bin/main.exe 2>&1 | head -30
|
|
flan=$root/_build/default/bin/main.exe
|
|
fi
|
|
test -x "$flan" || { echo "build failed"; exit 1; }
|
|
|
|
# Where the corpus is read from. Under dune the script runs from the build
|
|
# tree, where test/programs is present but spike/x86 is not, so the alias
|
|
# points this at the source tree and gets both.
|
|
corpus=${SURVEY_CORPUS:-$root}
|
|
|
|
out=$(mktemp -d); trap 'rm -rf "$out"' EXIT
|
|
|
|
# The ones that run until something stops them. Not a failure and not a match;
|
|
# they are excluded by name because a timeout cannot tell them apart from a
|
|
# backend that hung.
|
|
#
|
|
# dev-chatty is the third and is here for a sharper reason than the other two.
|
|
# It also outlives the timeout -- it is sized to outlast test_dev's checks and
|
|
# is killed with the connection -- but what makes it unusable here is that it
|
|
# *prints* while it does, 4K a frame. Two backends stopped by a clock stop at
|
|
# different lines, so the diff is a report about scheduling rather than about
|
|
# lowering, and it fails the alias every run. dev-repl outlives the timeout in
|
|
# the same way and is not listed, because it prints nothing and the two
|
|
# truncations are both empty.
|
|
forever="dev-loop dev-watch dev-chatty"
|
|
|
|
TIMEOUT=${TIMEOUT:-20}
|
|
|
|
# Extra flags, given to *both* sides. SURVEY_FLAGS=--dev is the one that has a
|
|
# use: a dev build with nothing yet redefined must behave exactly like a
|
|
# release one -- the indirection cell is the only difference -- so the whole
|
|
# corpus is a test of the cells, and of nothing else changing beside them.
|
|
# Off by default, so the counts above the line stay the same measurement.
|
|
read -r -a extra <<<"${SURVEY_FLAGS:-}"
|
|
|
|
declare -a match=() differ=() refused=() nox86=() skip=()
|
|
|
|
for src in "$corpus"/test/programs/*.flan "$corpus"/spike/x86/*.flan; do
|
|
name=$(basename "$src" .flan)
|
|
if [ $# -gt 0 ]; then
|
|
want=0
|
|
for pat in "$@"; do case "$name" in *"$pat"*) want=1;; esac; done
|
|
[ $want = 1 ] || continue
|
|
fi
|
|
case " $forever " in *" $name "*) skip+=("$name:runs-forever"); continue;; esac
|
|
|
|
# LLVM first. A program that does not compile at all, or has no main, is not
|
|
# this backend's business -- the frontend refused it either way.
|
|
if ! "$flan" build "$src" "${extra[@]}" -o "$out/$name.llvm" \
|
|
>"$out/$name.llvm.err" 2>&1; then
|
|
if grep -q "in function \`_start\|undefined reference to \`main\|crt1.o" "$out/$name.llvm.err"; then
|
|
skip+=("$name:no-main")
|
|
else
|
|
skip+=("$name:does-not-compile")
|
|
fi
|
|
continue
|
|
fi
|
|
|
|
"$flan" build "$src" --x86 "${extra[@]}" -o "$out/$name.x86" \
|
|
>"$out/$name.x86.err" 2>&1
|
|
rc=$?
|
|
if [ $rc = 3 ]; then
|
|
why=$(head -1 "$out/$name.x86.err" | sed 's/^x86: //')
|
|
refused+=("$name:$why")
|
|
continue
|
|
fi
|
|
if [ $rc != 0 ]; then
|
|
nox86+=("$name:$(head -1 "$out/$name.x86.err")")
|
|
continue
|
|
fi
|
|
|
|
( cd "$out" && timeout "$TIMEOUT" "$out/$name.llvm" \
|
|
>"$out/$name.llvm.out" 2>"$out/$name.llvm.diag" )
|
|
a=$?
|
|
( cd "$out" && timeout "$TIMEOUT" "$out/$name.x86" \
|
|
>"$out/$name.x86.out" 2>"$out/$name.x86.diag" )
|
|
b=$?
|
|
if [ "$a" = "$b" ] && cmp -s "$out/$name.llvm.out" "$out/$name.x86.out" \
|
|
&& cmp -s "$out/$name.llvm.diag" "$out/$name.x86.diag"; then
|
|
match+=("$name")
|
|
else
|
|
differ+=("$name:llvm=$a/x86=$b")
|
|
if [ "${SURVEY_SHOW:-}" = 1 ]; then
|
|
echo "--- $name: llvm exit $a, x86 exit $b"
|
|
diff "$out/$name.llvm.out" "$out/$name.x86.out" | head -20
|
|
diff "$out/$name.llvm.diag" "$out/$name.x86.diag" | head -20
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "MATCH ${#match[@]}"
|
|
echo "DIFFER ${#differ[@]}"
|
|
[ "${#differ[@]}" = 0 ] || printf ' %s\n' "${differ[@]}"
|
|
echo "REFUSED ${#refused[@]}"
|
|
if [ "${#refused[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then
|
|
printf '%s\n' "${refused[@]}" | sed 's/^[^:]*://' | sort | uniq -c | sort -rn \
|
|
| sed 's/^/ /'
|
|
fi
|
|
echo "NOX86 ${#nox86[@]}"
|
|
[ "${#nox86[@]}" = 0 ] || printf ' %s\n' "${nox86[@]}"
|
|
echo "SKIP ${#skip[@]}"
|
|
if [ "${#skip[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then
|
|
printf '%s\n' "${skip[@]}" | sed 's/^[^:]*://' | sort | uniq -c \
|
|
| sed 's/^/ /'
|
|
fi
|
|
|
|
# Strict mode, for the @x86 alias: the counts above are a report, and a report
|
|
# nobody reads is how two refusals from another lane's new primitive sat in
|
|
# the tree for a month. A DIFFER is a wrong answer and a refusal by name is a
|
|
# node this backend has stopped lowering; either is a failure. NOX86 and SKIP
|
|
# are not: the first is usually a toolchain that is not installed here, and
|
|
# the second is the frontend refusing the program on both sides.
|
|
if [ "${SURVEY_STRICT:-}" = 1 ]; then
|
|
if [ "${#differ[@]}" != 0 ] || [ "${#refused[@]}" != 0 ]; then
|
|
echo
|
|
echo "x86 survey FAILED: ${#differ[@]} differ, ${#refused[@]} refused"
|
|
exit 1
|
|
fi
|
|
echo
|
|
echo "x86 survey ok: ${#match[@]} match"
|
|
fi
|