From 37ccca56a355ce24eb997c6b1bd7c497c4637d07 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Fri, 25 Sep 2026 07:23:12 +0700 Subject: [PATCH] Every sweep reads the corpus through one alias, and a build that raises is a FAIL line in each of them The sanitizer sweep also lists the five macro programs that run. --- TODO.org | 21 ++-- spike/js/survey.sh | 20 +++- spike/x86/survey.sh | 20 +++- test/dune | 215 ++++++++-------------------------------- test/test_acceptance.ml | 11 +- test/test_sanitize.ml | 20 +++- test/test_support.ml | 8 ++ test/test_valgrind.ml | 2 +- 8 files changed, 121 insertions(+), 196 deletions(-) diff --git a/TODO.org b/TODO.org index fcf9257b..d3b237b5 100644 --- a/TODO.org +++ b/TODO.org @@ -1983,14 +1983,21 @@ Deliberate, because a path-based load is the one shape the browser cannot have. Said so it is not later read as an accident. It holds for the shipped programs rather than for the repository — a test fixture still loads an image from a path. -** TODO A package under test/programs needs a glob line in four places -Four sweeps each walk the programs directory and the glob does not descend. -Without the line the corpus row fails with "no package at ..." and prints no -failure line, so a grep for failures reads green over it. +** DONE A package under test/programs needs a glob line in four places +CLOSED: [2026-09-25] +One =corpus= alias in =test/dune= holds =(source_tree programs)= and the +workspace files the corpus imports; the tests, =test_web=, =@sanitize=, +=@valgrind=, =@x86= and =@js= depend on it, and =@page= takes the same +=source_tree=. A package added under =programs/= needs no line anywhere. A build +that raises is a =FAIL= line in every sweep: the acceptance pool, the two +sanitizer binaries, and a =MISSING= count that fails both survey scripts. -** TODO The macro programs are not in the sanitizer sweep -That sweep runs an explicit list, not a glob, so landing the macro programs did -not add them. A one-line edit. +** DONE The macro programs are not in the sanitizer sweep +CLOSED: [2026-09-25] +The five that run — =macros=, =macro-params=, =macro-unless=, =pkg-macro=, +=prelude-macros= — are rows in =test_sanitize.ml='s list; the refused ones stay +out with the other negative cases. The list stays explicit rather than a glob. +Not yet run under the sweep: that waits for the batched =@sanitize=. ** TODO The mutation pass has not been re-run Sixty mutations, nineteen of which left the whole suite green; all nineteen are diff --git a/spike/js/survey.sh b/spike/js/survey.sh index b76ebca9..3a85fb5a 100755 --- a/spike/js/survey.sh +++ b/spike/js/survey.sh @@ -17,12 +17,13 @@ # DIFFER, which is a wrong answer, and a CRASH, which is a program the backend # emitted and node would not run. # -# Five outcomes: +# Six outcomes: # # MATCH built both ways, same stdout, same stderr, same exit status # DIFFER built both ways, and disagreed # REFUSED Js.Unsupported -- named by the backend, with a location (exit 3) # CRASH emitted JS that node refused to run, or that threw +# MISSING imports a package the build tree does not have; a failure # SKIP no main, does not compile at all, or does not terminate # # Over test/programs, and over spike/js's own probes, which are here for the @@ -78,7 +79,7 @@ TIMEOUT=${TIMEOUT:-20} # and neither answer is defined. Nothing else changes. read -r -a extra <<<"${SURVEY_FLAGS:-}" -declare -a match=() differ=() refused=() crash=() skip=() +declare -a match=() differ=() refused=() crash=() skip=() missing=() for src in "$corpus"/test/programs/*.flan "$corpus"/spike/js/*.flan; do name=$(basename "$src" .flan) @@ -93,7 +94,12 @@ for src in "$corpus"/test/programs/*.flan "$corpus"/spike/js/*.flan; do # this backend's business -- the frontend refused it either way. if ! "$flan" build "$src" "${extra[@]}" -o "$out/$name.llvm" \ >"$out/$name.llvm.err" 2>&1; then - if grep -q "in function \`_start\|undefined reference to \`main\|crt1.o" "$out/$name.llvm.err"; then + # A package the build tree does not have is not the frontend refusing + # the program: it is a sweep that did not look at it, and it is counted + # as that rather than folded into the skips. + if grep -q "no package at" "$out/$name.llvm.err"; then + missing+=("$name:$(grep -m1 -o 'no package at [^ ]*' "$out/$name.llvm.err")") + elif grep -q "in function \`_start\|undefined reference to \`main\|crt1.o" "$out/$name.llvm.err"; then skip+=("$name:no-main") else skip+=("$name:does-not-compile") @@ -152,6 +158,8 @@ if [ "${#refused[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then fi echo "CRASH ${#crash[@]}" [ "${#crash[@]}" = 0 ] || printf ' %s\n' "${crash[@]}" +echo "MISSING ${#missing[@]}" +[ "${#missing[@]}" = 0 ] || printf ' %s\n' "${missing[@]}" echo "SKIP ${#skip[@]}" if [ "${#skip[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then printf '%s\n' "${skip[@]}" | sed 's/^[^:]*://' | sort | uniq -c \ @@ -161,9 +169,11 @@ fi # Strict mode, for the @js alias. A refusal is *not* a failure here -- see the # header -- so only a wrong answer and a program node could not run are. if [ "${SURVEY_STRICT:-}" = 1 ]; then - if [ "${#differ[@]}" != 0 ] || [ "${#crash[@]}" != 0 ]; then + if [ "${#differ[@]}" != 0 ] || [ "${#crash[@]}" != 0 ] \ + || [ "${#missing[@]}" != 0 ]; then echo - echo "js survey FAILED: ${#differ[@]} differ, ${#crash[@]} crash" + echo "js survey FAILED: ${#differ[@]} differ, ${#crash[@]} crash, \ +${#missing[@]} missing a package" exit 1 fi echo diff --git a/spike/x86/survey.sh b/spike/x86/survey.sh index d142626e..da96bf72 100755 --- a/spike/x86/survey.sh +++ b/spike/x86/survey.sh @@ -20,12 +20,13 @@ # compared a checked build against an unchecked one would say nothing about # bounds.flan, which is the one program the two backends disagreed about. # -# Five outcomes, and the third is the progress meter: +# Six outcomes, and the third is the progress meter: # # MATCH built both ways, same stdout, same stderr, same exit status # DIFFER built both ways, and disagreed # REFUSED X86.Unsupported -- a node this backend does not lower (exit 3) # NOX86 failed to build through --x86 for some other reason +# MISSING imports a package the build tree does not have; a failure # SKIP no main, does not compile at all, or does not terminate # # Over test/programs, over spike/x86's own probes, which are here for the paths @@ -127,7 +128,7 @@ TIMEOUT=${TIMEOUT:-20} # Off by default, so the counts above the line stay the same measurement. read -r -a extra <<<"${SURVEY_FLAGS:-}" -declare -a match=() differ=() refused=() nox86=() skip=() +declare -a match=() differ=() refused=() nox86=() skip=() missing=() for src in "$corpus"/test/programs/*.flan "$corpus"/spike/x86/*.flan \ "$corpus"/spike/js/*.flan; do @@ -147,7 +148,12 @@ for src in "$corpus"/test/programs/*.flan "$corpus"/spike/x86/*.flan \ # this backend's business -- the frontend refused it either way. if ! "$flan" build "$src" "${extra[@]}" -o "$out/$name.llvm" \ >"$out/$name.llvm.err" 2>&1; then - if grep -q "in function \`_start\|undefined reference to \`main\|crt1.o" "$out/$name.llvm.err"; then + # A package the build tree does not have is not the frontend refusing + # the program: it is a sweep that did not look at it, and it is counted + # as that rather than folded into the skips. + if grep -q "no package at" "$out/$name.llvm.err"; then + missing+=("$name:$(grep -m1 -o 'no package at [^ ]*' "$out/$name.llvm.err")") + elif grep -q "in function \`_start\|undefined reference to \`main\|crt1.o" "$out/$name.llvm.err"; then skip+=("$name:no-main") else skip+=("$name:does-not-compile") @@ -198,6 +204,8 @@ if [ "${#refused[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then fi echo "NOX86 ${#nox86[@]}" [ "${#nox86[@]}" = 0 ] || printf ' %s\n' "${nox86[@]}" +echo "MISSING ${#missing[@]}" +[ "${#missing[@]}" = 0 ] || printf ' %s\n' "${missing[@]}" echo "SKIP ${#skip[@]}" if [ "${#skip[@]}" != 0 ] && [ "${SURVEY_QUIET:-}" != 1 ]; then printf '%s\n' "${skip[@]}" | sed 's/^[^:]*://' | sort | uniq -c \ @@ -211,9 +219,11 @@ fi # are not: the first is usually a toolchain that is not installed here, and # the second is the frontend refusing the program on both sides. if [ "${SURVEY_STRICT:-}" = 1 ]; then - if [ "${#differ[@]}" != 0 ] || [ "${#refused[@]}" != 0 ]; then + if [ "${#differ[@]}" != 0 ] || [ "${#refused[@]}" != 0 ] \ + || [ "${#missing[@]}" != 0 ]; then echo - echo "x86 survey FAILED: ${#differ[@]} differ, ${#refused[@]} refused" + echo "x86 survey FAILED: ${#differ[@]} differ, ${#refused[@]} refused, \ +${#missing[@]} missing a package" exit 1 fi echo diff --git a/test/dune b/test/dune index f00bff7b..58110461 100644 --- a/test/dune +++ b/test/dune @@ -1,19 +1,16 @@ -(tests - (names test_flan test_acceptance test_reload test_agent test_session test_dev test_emacs test_repl test_cider test_dyn) - ; Explicit because test_sanitize lives in this directory and is not one of - ; these: two stanzas in one directory have to say which modules are whose. - ; watchdog is every binary's clock: a hanging test reports nothing, so each - ; of these arms an alarm that turns "for ever" into a failing run. test_support - ; is the other shared module: the failure counter, the report tail, the poll, - ; the daemon wait and the front half of a compile, which the binaries that - ; wanted them had each been carrying their own copy of. - (modules test_flan test_acceptance test_reload test_agent test_session - test_dev test_emacs test_repl test_cider test_dyn watchdog - test_support) - (libraries flan unix) - ; The acceptance programs are part of the test corpus: if the reader, the - ; parser or the checker regresses on them we want to know here, not at the CLI. +; The corpus: every file a sweep over test/programs can reach, in one place. +; Each stanza below that walks the programs depends on this alias rather than +; listing the files, so a package or an asset directory added under programs/ +; is in every sweep without a line anywhere else. source_tree rather than a +; glob per directory, because glob_files does not descend and every package is +; a directory. +(alias + (name corpus) (deps + (source_tree programs) + ; The acceptance programs are part of the test corpus: if the reader, the + ; parser or the checker regresses on them we want to know here, not at the + ; CLI. (file %{workspace_root}/calc-me.flan) (file %{workspace_root}/sand.flan) ; The brush sheet. sand.flan no longer embeds it — the front-end was cut back @@ -36,50 +33,28 @@ ; The ported raylib examples. Only one of them has a headless acceptance ; case, but it imports its example as a package and that example imports ; examples/digits.flan, so the directory has to be here whole. - (glob_files %{workspace_root}/examples/*) - (glob_files programs/*.flan) - ; The package tree the multi-level cases import: pkg-diamond reaches shape - ; through area and draw, and pkg-cycle reaches a ring. Each directory is a - ; package, so each comes whole — a glob per directory rather than one over - ; programs/pkgs/*, because dune's glob does not descend. - (glob_files programs/pkgs/shape/*) - (glob_files programs/pkgs/area/*) - (glob_files programs/pkgs/draw/*) - (glob_files programs/pkgs/ring-a/*) - (glob_files programs/pkgs/ring-b/*) - (glob_files programs/pkgs/ring-c/*) - ; The package that exports a data type, which pkg-data.flan imports. - (glob_files programs/pkgs/tree/*) - ; The packages that declare macros: one whose macros a program calls - ; qualified, and the two whose macros do not terminate — a ring, and one - ; that never settles. Each is its own directory, so each needs its own glob. - (glob_files programs/pkgs/mac/*) - (glob_files programs/pkgs/macring/*) - (glob_files programs/pkgs/macspin/*) - ; The package whose body calls the builtin get while the program importing - ; it defines a get of its own — shadow-builtin.flan, which is the pin that - ; a shadow stops at the file that declared it. - (glob_files programs/pkgs/shadowed/*) - ; The package whose exports are generic, which pkg-generic.flan and - ; pkg-generic-reject.flan import: the body has to be present where the copy - ; is made, so the directory comes whole like every other package. - (glob_files programs/pkgs/gen/*) + (glob_files %{workspace_root}/examples/*))) + +(tests + (names test_flan test_acceptance test_reload test_agent test_session test_dev test_emacs test_repl test_cider test_dyn) + ; Explicit because test_sanitize lives in this directory and is not one of + ; these: two stanzas in one directory have to say which modules are whose. + ; watchdog is every binary's clock: a hanging test reports nothing, so each + ; of these arms an alarm that turns "for ever" into a failing run. test_support + ; is the other shared module: the failure counter, the report tail, the poll, + ; the daemon wait and the front half of a compile, which the binaries that + ; wanted them had each been carrying their own copy of. + (modules test_flan test_acceptance test_reload test_agent test_session + test_dev test_emacs test_repl test_cider test_dyn watchdog + test_support) + (libraries flan unix) + (deps + (alias corpus) ; The synthetic C header the importer's table reads. Committed rather than ; reached for on the machine: the raylib case needs raylib installed, at the ; right version, with a variable set, so it skips everywhere and covers ; nothing. This one does not move. (glob_files headers/*.h) - ; The files programs/embed.flan bakes in. An embed reads them at *compile* - ; time, so they are a dependency of the checker run and not of the program. - (glob_files programs/assets/*) - ; And the tileset programs/edn-read.flan bakes in, which is under assets/ and - ; not in it: embed.flan holds (embed-dir "assets") in a [3 EmbedFile], so a - ; fourth file beside those three is a type error in an unrelated program. - ; embed-dir does not descend and neither does a glob, so this is its own line. - (glob_files programs/assets/edn/*) - ; And the config programs/json-provide.flan derives a struct from, under - ; assets/ for the same reason and needing its own line for the same one. - (glob_files programs/assets/json/*) ; The reload primitive's host: a C main that dlopens what Build.shared made. (file reload_host.c) ; A shared object that is not a redefinition module, for the agent's refusal @@ -89,8 +64,8 @@ ; reaches, driven directly. (file dev_limits.c) ; And the third: the dynamic-value runtime, which has no Flan spelling yet - ; at all. Its host program is under programs/ and is picked up by the glob - ; above; the header dyn_ops.c includes is the compiler's, dropped into the + ; at all. Its host program is under programs/ and comes in with the + ; corpus; the header dyn_ops.c includes is the compiler's, dropped into the ; build directory beside each translation unit, so it is not a dependency ; here. (file dyn_ops.c) @@ -115,20 +90,7 @@ (modules test_web test_support) (libraries flan unix) (deps - (glob_files programs/*.flan) - (glob_files programs/assets/*) - (glob_files programs/assets/edn/*) - (glob_files programs/assets/json/*) - ; The raylib bindings and the ported example the raylib case builds. The - ; example imports examples/digits.flan, so the directory comes whole. - (glob_files %{workspace_root}/vendor/raylib/*) - (glob_files %{workspace_root}/examples/*) - ; sand.flan for the browser, with the sheet it embeds and the dev agent it - ; imports — the agent's directory has to be whole, because the file that - ; makes a web build possible is the one Build selects out of it. - (file %{workspace_root}/sand.flan) - (file %{workspace_root}/brush.png) - (glob_files %{workspace_root}/vendor/agent/*) + (alias corpus) ; flan run --target=web is refused by the CLI, so the CLI has to be here. (file %{workspace_root}/bin/main.exe))) @@ -149,18 +111,8 @@ (rule (alias sanitize) (deps + (alias corpus) test_sanitize.exe - (file %{workspace_root}/calc-me.flan) - (file %{workspace_root}/sand.flan) - (file %{workspace_root}/brush.png) - (glob_files %{workspace_root}/vendor/raylib/*) - (glob_files %{workspace_root}/vendor/agent/*) - (glob_files %{workspace_root}/vendor/edn/*) - (glob_files %{workspace_root}/vendor/json/*) - (glob_files %{workspace_root}/examples/*) - (glob_files programs/*.flan) - (glob_files programs/assets/*) - (glob_files programs/assets/edn/*) ; p13-dyn-collect.flan, which lives with the x86 probes because that is the ; lane that wrote it, and is in this sweep because of what it does rather ; than where it is: it is the only program anywhere that allocates past @@ -172,16 +124,7 @@ ; a Flan program: flan_dyn.c has no Flan spelling yet. It is also the one ; translation unit here that frees the most, which is what makes it worth a ; sanitized run at all. See [dyn_sweep]. - (file dyn_ops.c) - (glob_files programs/assets/json/*) - ; The package tree pkg-diamond.flan imports — it reaches shape through area - ; and draw — a glob per directory because dune's glob does not descend. - ; Missing until now, and the alias only looked green because `dune test` had - ; run first and left the directories in _build: from a clean tree the sweep - ; died on pkg-diamond before it reached a single sanitized run. - (glob_files programs/pkgs/shape/*) - (glob_files programs/pkgs/area/*) - (glob_files programs/pkgs/draw/*)) + (file dyn_ops.c)) (action (run ./test_sanitize.exe))) ; The corpus a third time, under Valgrind's memcheck. Its own alias for the @@ -204,38 +147,11 @@ (rule (alias valgrind) (deps + (alias corpus) test_valgrind.exe ; The suppression file, which is all reasons and no suppressions; its own ; header says why that is the finding rather than an oversight. - (file valgrind.supp) - (file %{workspace_root}/calc-me.flan) - (file %{workspace_root}/sand.flan) - (file %{workspace_root}/brush.png) - (glob_files %{workspace_root}/vendor/raylib/*) - (glob_files %{workspace_root}/vendor/agent/*) - (glob_files %{workspace_root}/vendor/edn/*) - (glob_files %{workspace_root}/vendor/json/*) - (glob_files %{workspace_root}/examples/*) - (glob_files programs/*.flan) - (glob_files programs/assets/*) - (glob_files programs/assets/edn/*) - (glob_files programs/assets/json/*) - ; The package tree the multi-level cases import, as in the test stanza - ; above: a glob per directory, because dune's glob does not descend. - (glob_files programs/pkgs/shape/*) - (glob_files programs/pkgs/area/*) - (glob_files programs/pkgs/draw/*) - (glob_files programs/pkgs/ring-a/*) - (glob_files programs/pkgs/ring-b/*) - (glob_files programs/pkgs/ring-c/*) - (glob_files programs/pkgs/tree/*) - ; And the macro-declaring packages, for the same reason. - (glob_files programs/pkgs/mac/*) - (glob_files programs/pkgs/macring/*) - (glob_files programs/pkgs/macspin/*) - ; And the package shadow-builtin.flan imports. - (glob_files programs/pkgs/shadowed/*) - (glob_files programs/pkgs/gen/*)) + (file valgrind.supp)) (action (run ./test_valgrind.exe))) ; The corpus a fourth time, through the hand-written x86-64 backend, compared @@ -262,6 +178,7 @@ (rule (alias x86) (deps + (alias corpus) (file %{workspace_root}/spike/x86/survey.sh) (glob_files %{workspace_root}/spike/x86/*.flan) ; The js spike's programs are plain flan programs and the sweep reads them @@ -269,33 +186,7 @@ ; the glob matches nothing, and nothing is exactly what it was reporting ; while p1-int-semantics.flan sat there with a wrong shift in it. (glob_files %{workspace_root}/spike/js/*.flan) - (file %{workspace_root}/bin/main.exe) - (file %{workspace_root}/calc-me.flan) - (file %{workspace_root}/sand.flan) - (file %{workspace_root}/brush.png) - (glob_files %{workspace_root}/vendor/raylib/*) - (glob_files %{workspace_root}/vendor/agent/*) - (glob_files %{workspace_root}/vendor/edn/*) - (glob_files %{workspace_root}/vendor/json/*) - (glob_files %{workspace_root}/examples/*) - (glob_files programs/*.flan) - (glob_files programs/assets/*) - (glob_files programs/assets/edn/*) - (glob_files programs/assets/json/*) - ; A glob per package directory, because dune's glob does not descend. - (glob_files programs/pkgs/shape/*) - (glob_files programs/pkgs/area/*) - (glob_files programs/pkgs/draw/*) - (glob_files programs/pkgs/ring-a/*) - (glob_files programs/pkgs/ring-b/*) - (glob_files programs/pkgs/ring-c/*) - (glob_files programs/pkgs/tree/*) - (glob_files programs/pkgs/mac/*) - (glob_files programs/pkgs/macring/*) - (glob_files programs/pkgs/macspin/*) - ; And the package shadow-builtin.flan imports. - (glob_files programs/pkgs/shadowed/*) - (glob_files programs/pkgs/gen/*)) + (file %{workspace_root}/bin/main.exe)) (action (setenv SURVEY_STRICT 1 (setenv SURVEY_QUIET 1 @@ -344,7 +235,7 @@ (file %{workspace_root}/calc-me.flan) (file %{workspace_root}/conditions.org) (glob_files %{workspace_root}/emacs/*.el) - (glob_files programs/*.flan) + (source_tree programs) ; And sand.flan itself, which the hash comes from: quotes.sh runs ; test/programs/sand-headless.flan and that imports ../../sand.flan as a ; single-file package. Missing until now, the same hole @sanitize had — the @@ -445,34 +336,10 @@ (rule (alias js) (deps + (alias corpus) (file %{workspace_root}/spike/js/survey.sh) (glob_files %{workspace_root}/spike/js/*.flan) - (file %{workspace_root}/bin/main.exe) - (file %{workspace_root}/calc-me.flan) - (file %{workspace_root}/sand.flan) - (file %{workspace_root}/brush.png) - (glob_files %{workspace_root}/vendor/raylib/*) - (glob_files %{workspace_root}/vendor/agent/*) - (glob_files %{workspace_root}/vendor/edn/*) - (glob_files %{workspace_root}/vendor/json/*) - (glob_files %{workspace_root}/examples/*) - (glob_files programs/*.flan) - (glob_files programs/assets/*) - (glob_files programs/assets/edn/*) - (glob_files programs/assets/json/*) - (glob_files programs/pkgs/shape/*) - (glob_files programs/pkgs/area/*) - (glob_files programs/pkgs/draw/*) - (glob_files programs/pkgs/ring-a/*) - (glob_files programs/pkgs/ring-b/*) - (glob_files programs/pkgs/ring-c/*) - (glob_files programs/pkgs/tree/*) - (glob_files programs/pkgs/mac/*) - (glob_files programs/pkgs/macring/*) - (glob_files programs/pkgs/macspin/*) - ; And the package shadow-builtin.flan imports. - (glob_files programs/pkgs/shadowed/*) - (glob_files programs/pkgs/gen/*)) + (file %{workspace_root}/bin/main.exe)) (action (setenv SURVEY_STRICT 1 (setenv SURVEY_QUIET 1 diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index 58a8cf4e..cddec395 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -248,7 +248,16 @@ module Pool = struct true by construction rather than by relying on the timer-clearing behaviour alone. *) Sys.set_signal Sys.sigalrm Sys.Signal_default; - let result = try f () with e -> Some (Printexc.to_string e) in + (* An exception is a failed row like any other, and says so in the same + shape: a diagnostic on its own — "no package at ..." from a package + missing from the build tree — reads as noise between passing rows. *) + let result = + try f () + with e -> + Some + (Printf.sprintf "FAIL %s\n raised: %s\n" name + (Printexc.to_string e)) + in if result = None then cleanup_workdir (); let oc = Unix.out_channel_of_descr w in Marshal.to_channel oc result []; diff --git a/test/test_sanitize.ml b/test/test_sanitize.ml index d760a961..196c69d9 100644 --- a/test/test_sanitize.ml +++ b/test/test_sanitize.ml @@ -97,7 +97,9 @@ let reported text = List.exists (contains text) markers redefinition module is built by llc and ld, not by clang, so nothing instruments it — one more thing this sweep does not prove. - nth-gone, pkg-hidden-main, pkg-two-aliases, pkg-two-mains, pkg-cycle, - which are negative cases and are expected not to compile. + and the macro programs that are refused (macro-arity, macro-cycle, + macro-spin, macro-destructure, macro-loc-* and the pkg-macro-* + refusals), which are negative cases and are expected not to compile. calc-me is here and is not in test/programs: it is the one string parser in the corpus, which makes it the likeliest to push [scratch] or [escaped] @@ -168,6 +170,15 @@ let corpus = block after the growth that moved it. *) "programs/json.flan", []; "programs/machine.flan", []; + (* The macro programs that run: a macro is expanded by a module dlopened + into the compiler, but what reaches this sweep is the program the + expansion produced, which is ordinary code and can be wrong in the + ordinary ways. The ones that refuse are negative cases, below. *) + "programs/macros.flan", []; + "programs/macro-params.flan", []; + "programs/macro-unless.flan", []; + "programs/pkg-macro.flan", []; + "programs/prelude-macros.flan", []; "programs/math.flan", []; "programs/math3.flan", []; "programs/pkg-diamond.flan", []; @@ -255,10 +266,13 @@ let sweep ~checks label = List.iter (fun (path, args) -> match compile ~sanitize:false ~checks path with - | exception Failure m -> fail "%s %s: unsanitized build: %s" label path m + | exception e -> + fail "%s %s: unsanitized build: %s" label path (Test_support.raised e) | plain -> (match compile ~sanitize:true ~checks path with - | exception Failure m -> fail "%s %s: sanitized build: %s" label path m + | exception e -> + fail "%s %s: sanitized build: %s" label path + (Test_support.raised e) | san -> let c1, t1 = run plain args in let c2, t2 = run san args in diff --git a/test/test_support.ml b/test/test_support.ml index 41228ca6..28feea6f 100644 --- a/test/test_support.ml +++ b/test/test_support.ml @@ -32,6 +32,14 @@ let failures = ref 0 let fail fmt = Printf.ksprintf (fun s -> incr failures; print_endline ("FAIL " ^ s)) fmt +(* What a build that raised says, for a [fail] line. [Failure] carries its + text bare; anything else, a [Loc.Error] most of all, goes through the + printer [Loc] registers, which is the diagnostic itself. A sweep that caught + only [Failure] died on the first source error it met — "no package at ...", + from a package the build tree was missing — with no FAIL line and every row + after it unrun. *) +let raised = function Failure m -> m | e -> Printexc.to_string e + (* The tail every suite ends on: a single line when nothing failed, and a count plus a nonzero exit when something did. The label names the suite, because these binaries run under dune's parallelism and their output is diff --git a/test/test_valgrind.ml b/test/test_valgrind.ml index 6e834a87..e881581a 100644 --- a/test/test_valgrind.ml +++ b/test/test_valgrind.ml @@ -145,7 +145,7 @@ let clean () = (* One program, both ways. *) let check label path args ~checks = match compile ~checks path with - | exception Failure m -> fail "%s %s: build: %s" label path m + | exception e -> fail "%s %s: build: %s" label path (Test_support.raised e) | exe -> clean (); let c1, t1, _ = run exe args in