One binary that is both a Flan program and the compiler that built it

The first six probes each proved a piece. merged.sh puts them together: the
program's @main is renamed out of the way, a C main takes the main thread and
runs it there, caml_startup happens on a thread beside it, and clang links the
lot -- the emitted program object, flan_rt.c, flan_dev.c, flan_agent.c and the
whole compiler as one -output-complete-obj. It runs, and the compiler inside it
compiles the very source the program was built from.

Nothing is wired up. The two halves share an address space and do not speak.
That is the point: the question was whether they can, not what they would say.

sig.sh and symbols.sh answer the two questions the first pass got wrong or
skipped. The SIGSEGV reading in harness5.c was taken at the wrong moment --
OCaml 5 starts domains after caml_startup returns, so the disposition had to be
read from inside the runtime, and against a plain ocamlopt executable as a
control. symbols.sh is the hazard nobody looks for until the link fails: four
.c files that are compiled into two different processes today, and the OCaml
runtime, all landing in one link.

spike/embed/.gitignore

@@ -9,3 +9,6 @@ spike3
 spike4
 spike5
 spike6
+spike5b
+spike5b_std
+stubs5b.o

spike/embed/harness5b.c

@@ -0,0 +1,110 @@
+/* Step 5b: the SIGSEGV question, asked properly.
+ *
+ * 5a read the disposition either side of caml_startup and found SIG_DFL both
+ * times, which would mean no collision at all. That is too good, and it is
+ * because OCaml 5 installs the handler per *domain*, on the domain's own
+ * thread, not once during startup. So this asks at four moments, and then asks
+ * the only question that decides anything: with the break loop holding SIGSEGV,
+ * does an OCaml stack overflow still raise Stack_overflow, or does it become a
+ * hard crash?
+ *
+ * Two ways of taking it are compared:
+ *   take_segv  -- install ours and discard OCaml's, the naive thing;
+ *   chain_segv -- install ours, keep OCaml's, and forward to it.
+ */
+#include <caml/callback.h>
+#include <caml/mlvalues.h>
+#include <caml/memory.h>
+#include <signal.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+static struct sigaction ocaml_segv;
+static int have_ocaml_segv = 0;
+
+CAMLprim value spike_show_segv(value when) {
+  struct sigaction cur;
+  memset(&cur, 0, sizeof cur);
+  sigaction(SIGSEGV, NULL, &cur);
+  printf("  SIGSEGV %-46s handler=%p flags=%#x%s\n", String_val(when),
+         (cur.sa_flags & SA_SIGINFO) ? (void *)cur.sa_sigaction
+                                     : (void *)cur.sa_handler,
+         (unsigned)cur.sa_flags,
+         cur.sa_handler == SIG_DFL ? "  (SIG_DFL)" : "");
+  fflush(stdout);
+  return Val_unit;
+}
+
+/* The break loop's handler. It does not long-jump here -- the point is only to
+ * see whether it is reached and whether OCaml still works around it. */
+static void break_segv(int sig, siginfo_t *info, void *ctx) {
+  (void)sig;
+  if (have_ocaml_segv && ocaml_segv.sa_sigaction &&
+      ocaml_segv.sa_handler != SIG_DFL && ocaml_segv.sa_handler != SIG_IGN) {
+    /* Chained: hand the fault to OCaml, which turns a guard-page hit into
+     * Stack_overflow and re-raises anything else. */
+    ocaml_segv.sa_sigaction(sig, info, ctx);
+    return;
+  }
+  /* Taken outright: nothing below us. A real break loop would stop and serve;
+   * here we can only abort, which is the honest cost of discarding OCaml's. */
+  printf("  break loop caught SIGSEGV at %p with nothing to chain to\n",
+         info->si_addr);
+  fflush(stdout);
+  _exit(9);
+}
+
+static void install(int keep_old) {
+  struct sigaction sa;
+  memset(&sa, 0, sizeof sa);
+  memset(&ocaml_segv, 0, sizeof ocaml_segv);
+  sigaction(SIGSEGV, NULL, &ocaml_segv);
+  have_ocaml_segv = keep_old;
+  sa.sa_sigaction = break_segv;
+  sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_NODEFER;
+  sigemptyset(&sa.sa_mask);
+  sigaction(SIGSEGV, &sa, NULL);
+}
+
+/* A sweep, so "the OCaml runtime installs handlers" can be stated as a list
+ * rather than a worry. Called from OCaml with the runtime and a domain up. */
+CAMLprim value spike_sweep(value u) {
+  static const int sigs[] = { SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGINT, SIGTERM,
+                              SIGPIPE, SIGCHLD, SIGUSR1, SIGUSR2, SIGABRT,
+                              SIGALRM, SIGPROF, SIGVTALRM, SIGWINCH };
+  static const char *names[] = { "SEGV", "BUS", "FPE", "ILL", "INT", "TERM",
+                                 "PIPE", "CHLD", "USR1", "USR2", "ABRT",
+                                 "ALRM", "PROF", "VTALRM", "WINCH" };
+  struct sigaction c;
+  unsigned i;
+  (void)u;
+  for (i = 0; i < sizeof sigs / sizeof *sigs; i++) {
+    memset(&c, 0, sizeof c);
+    sigaction(sigs[i], NULL, &c);
+    if (c.sa_handler != SIG_DFL)
+      printf("  SIG%-8s %s\n", names[i],
+             c.sa_handler == SIG_IGN ? "SIG_IGN" : "custom handler");
+  }
+  printf("  (every signal not named above is SIG_DFL)\n");
+  fflush(stdout);
+  return Val_unit;
+}
+
+CAMLprim value spike_take_segv(value u) { (void)u; install(0); return Val_unit; }
+CAMLprim value spike_chain_segv(value u) { (void)u; install(1); return Val_unit; }
+
+#ifndef SPIKE_NO_MAIN
+int main(int argc, char **argv) {
+  struct sigaction cur;
+  (void)argc;
+  memset(&cur, 0, sizeof cur);
+  sigaction(SIGSEGV, NULL, &cur);
+  printf("  SIGSEGV %-46s handler=%p%s\n", "before caml_startup",
+         (void *)cur.sa_handler, cur.sa_handler == SIG_DFL ? "  (SIG_DFL)" : "");
+  /* Everything else runs from sig_ml.ml's module initialiser, so the readings
+   * happen on the runtime's own thread at the moments that matter. */
+  caml_startup(argv);
+  return 0;
+}
+#endif

spike/embed/merged.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Step 8: the thing the whole spike is really asking about -- ONE binary that
+# is both a compiled Flan program and the OCaml compiler, with clang doing the
+# final link.
+#
+# Everything before this proved a piece. This proves the shape: the Flan
+# program's own main() is renamed out of the way, a C main() takes the main
+# thread and runs the program there, and caml_startup happens on a side thread
+# beside it. That is exactly item 11's inversion, built for real.
+#
+# It is NOT the merged architecture -- nothing is wired up, the compiler and the
+# program do not talk. It is a link and a size and a startup number.
+set -u
+here=$(cd "$(dirname "$0")" && pwd)
+root=$(cd "$here/../.." && pwd)
+src=${1:-test/programs/edn.flan}
+cd "$root" || exit 1
+out=$(mktemp -d); trap 'rm -rf "$out"' EXIT
+
+FLAN=_build/default/bin/main.exe
+OCAMLLIB=$(ocamlopt -where)
+SYSLIBS="-lm -lpthread -ldl -lzstd"
+
+echo "program: $src"
+
+# 1. The Flan program, as it is built today, for the baseline sizes.
+"$FLAN" build "$src" -o "$out/rel" || exit 1
+"$FLAN" build "$src" --dev -o "$out/dev" || exit 1
+
+# 2. The same program as an object, with its main renamed so a C main can own
+#    the process. Emit writes @main literally; sed is enough to move it.
+"$FLAN" emit "$src" --dev > "$out/prog.ll" || exit 1
+sed -i 's/define i32 @main(/define i32 @flan_program_main(/' "$out/prog.ll"
+grep -q 'define i32 @flan_program_main(' "$out/prog.ll" || {
+  echo "could not find @main in the emitted IR -- adjust the rename"; exit 1; }
+clang -c -x ir "$out/prog.ll" -o "$out/prog.o" || exit 1
+
+# 3. The runtime the program needs, and the agent beside it.
+clang -c -O2 runtime/flan_rt.c  -o "$out/rt.o"  || exit 1
+clang -c -O2 runtime/flan_dev.c -o "$out/dev.o" || exit 1
+clang -c -O2 vendor/agent/flan_agent.c -o "$out/ag.o" || exit 1
+
+# 4. The whole OCaml compiler as one object.
+ocamlfind ocamlopt -thread -package unix,threads.posix -linkpkg \
+  -output-complete-obj \
+  -I "$root/_build/default/lib/.flan.objs/byte" \
+  -I "$root/_build/default/lib/.flan.objs/native" \
+  -o "$out/compiler.o" "$root/_build/default/lib/flan.cmxa" \
+  "$here/thread_ml.ml" || exit 1
+
+# 5. One link. clang, as the project already does it.
+clang -I"$OCAMLLIB" "$here/merged_main.c" "$out/prog.o" "$out/rt.o" "$out/dev.o" \
+  "$out/ag.o" "$out/compiler.o" -o "$out/merged" $SYSLIBS || exit 1
+
+echo
+echo "sizes:"
+for f in rel dev merged; do
+  printf '  %-30s %9d bytes\n' "$f" "$(stat -c%s "$out/$f")"
+done
+printf '  %-30s %9d bytes\n' "what the compiler adds to a dev build" \
+  "$(( $(stat -c%s "$out/merged") - $(stat -c%s "$out/dev") ))"
+
+echo
+echo "running the merged binary:"
+"$out/merged" "$src"
+echo "exit: $?"

spike/embed/merged_main.c

@@ -0,0 +1,77 @@
+/* One process: the Flan program on the main thread, the OCaml compiler beside
+ * it on a domain of its own.
+ *
+ * This is the shape item 11 settles on, and the reason it is written this way
+ * round rather than the other: on macOS the window has to be on the main
+ * thread, so the game keeps main() and the compiler moves to the side --
+ * beside the listener vendor/agent/flan_agent.c already starts there.
+ *
+ * The program and the compiler do not talk to each other here. Wiring them up
+ * is the real work; this only shows they can share an address space, a link,
+ * and a process, with clang doing the final link. */
+#include <caml/callback.h>
+#include <caml/alloc.h>
+#include <caml/mlvalues.h>
+#include <caml/threads.h>
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdio.h>
+#include <time.h>
+
+/* The Flan program's entry point, renamed out of main's way by merged.sh. */
+extern int flan_program_main(int argc, char **argv);
+
+static char **g_argv;
+static const char *g_src;
+/* The Flan program's main calls exit(), so the compiler has to be up before it
+ * starts -- which is the honest ordering anyway: the image comes up and serves,
+ * then the program runs, the way starting an SBCL image does. */
+static atomic_int compiler_ready = 0;
+
+static double ms_since(struct timespec a) {
+  struct timespec b;
+  clock_gettime(CLOCK_MONOTONIC, &b);
+  return (b.tv_sec - a.tv_sec) * 1e3 + (b.tv_nsec - a.tv_nsec) / 1e6;
+}
+
+static void *compiler_side(void *unused) {
+  struct timespec t0;
+  const value *f;
+  (void)unused;
+  clock_gettime(CLOCK_MONOTONIC, &t0);
+  caml_startup(g_argv);
+  printf("[compiler] up on a side thread in %.3f ms\n", ms_since(t0));
+  f = caml_named_value("spike_thread_compile");
+  if (f) {
+    clock_gettime(CLOCK_MONOTONIC, &t0);
+    printf("[compiler] %s\n", String_val(caml_callback(*f, caml_copy_string(g_src))));
+    printf("[compiler] compiled the running program from inside it, in %.3f ms\n",
+           ms_since(t0));
+  }
+  caml_release_runtime_system();
+  atomic_store(&compiler_ready, 1);
+  return NULL;
+}
+
+int main(int argc, char **argv) {
+  pthread_t comp;
+  int rc;
+  g_argv = argv;
+  g_src = argc > 1 ? argv[1] : "test/programs/edn.flan";
+
+  if (pthread_create(&comp, NULL, compiler_side, NULL) != 0) return 1;
+
+  while (!atomic_load(&compiler_ready)) {
+    struct timespec t = { 0, 2000000L };
+    nanosleep(&t, NULL);
+  }
+
+  /* The main thread is the program's, and it never enters OCaml. */
+  printf("[program] running on the main thread\n");
+  rc = flan_program_main(argc, argv);
+  printf("[program] returned %d\n", rc);
+
+  pthread_join(comp, NULL);
+  printf("one process: a Flan program and the OCaml compiler, same binary\n");
+  return 0;
+}

spike/embed/sig.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Step 5b on its own: the SIGSEGV question, embedded and standalone side by
+# side. The standalone build is the control -- if OCaml behaves the same in a
+# plain ocamlopt executable, then embedding changed nothing about signals.
+set -u
+here=$(cd "$(dirname "$0")" && pwd)
+cd "$here" || exit 1
+OCAMLLIB=$(ocamlopt -where)
+SYSLIBS="-lm -lpthread -ldl -lzstd"
+
+echo "=== 5b-embedded: caml_startup called from a C main() ==="
+ocamlfind ocamlopt -package unix -linkpkg -output-complete-obj \
+  -o embed5b.o sig_ml.ml || exit 1
+clang -I"$OCAMLLIB" harness5b.c embed5b.o -o spike5b $SYSLIBS || exit 1
+./spike5b; echo "exit: $?"
+
+echo
+echo "=== 5b-standalone: the same OCaml, as a plain ocamlopt executable ==="
+# The control. Same stubs, but OCaml owns main().
+clang -c -I"$OCAMLLIB" -DSPIKE_NO_MAIN harness5b.c -o stubs5b.o || exit 1
+ocamlfind ocamlopt -package unix -linkpkg -o spike5b_std sig_ml.ml stubs5b.o \
+  -cclib -lzstd || exit 1
+./spike5b_std; echo "exit: $?"

spike/embed/sig_ml.ml

@@ -0,0 +1,34 @@
+(* Step 5b: OCaml 5.2 installs its SIGSEGV handler per-domain, not once at
+   startup, so "read the disposition after caml_startup" is not the whole
+   question. This asks it at four moments, and then asks the thing that
+   actually matters: does Stack_overflow still get raised once the break loop
+   has taken SIGSEGV? *)
+
+external show : string -> unit = "spike_show_segv"
+external take_segv : unit -> unit = "spike_take_segv"
+external chain_segv : unit -> unit = "spike_chain_segv"
+external sweep : unit -> unit = "spike_sweep"
+
+let rec deep n = if n <= 0 then 0 else 1 + deep (n - 1) + (if n < 0 then deep n else 0)
+
+let overflow_result () =
+  try
+    let n = deep 100_000_000 in
+    Printf.sprintf "returned %d (no overflow)" n
+  with Stack_overflow -> "Stack_overflow raised"
+
+let () =
+  show "at module init (main domain up)";
+  let d = Domain.spawn (fun () -> show "inside a spawned domain") in
+  Domain.join d;
+  show "after Domain.join";
+  print_endline "every signal the OCaml runtime is holding:";
+  sweep ();
+  Printf.sprintf "before touching SIGSEGV: %s" (overflow_result ()) |> print_endline;
+  take_segv ();
+  show "after the break loop takes SIGSEGV outright";
+  Printf.sprintf "with SIGSEGV taken outright: %s" (overflow_result ())
+  |> print_endline;
+  chain_segv ();
+  show "after the break loop chains to OCaml's handler";
+  Printf.sprintf "with SIGSEGV chained: %s" (overflow_result ()) |> print_endline

spike/embed/symbols.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Step 7: the integration hazard nobody asks about until the link fails.
+#
+# Today runtime/flan_rt.c, runtime/flan_dev.c and vendor/agent/flan_agent.c are
+# compiled into the *program*, and lib/dynload_stubs.c into the *compiler*.
+# Merging the processes puts all four and the OCaml runtime in one link. This
+# checks, symbol by symbol, whether anything collides.
+set -u
+here=$(cd "$(dirname "$0")" && pwd)
+root=$(cd "$here/../.." && pwd)
+out=$(mktemp -d)
+trap 'rm -rf "$out"' EXIT
+cd "$root" || exit 1
+
+defs() { nm --defined-only "$@" 2>/dev/null | awk 'NF==3 {print $3}' | sort -u; }
+
+clang -c runtime/flan_rt.c            -o "$out/rt.o"    || exit 1
+clang -c runtime/flan_dev.c           -o "$out/dev.o"   || exit 1
+clang -c vendor/agent/flan_agent.c    -o "$out/ag.o"    || exit 1
+clang -c -I"$(ocamlopt -where)" "$here/dynload_stubs.c" -o "$out/dl.o" || exit 1
+
+defs "$out/rt.o" "$out/dev.o" "$out/ag.o" "$out/dl.o" > "$out/flan.syms"
+defs /home/joe/.opam/default/lib/ocaml/libasmrun.a > "$out/ml.syms" 2>/dev/null
+[ -s "$out/ml.syms" ] || defs "$(ocamlopt -where)/libasmrun.a" > "$out/ml.syms"
+
+echo "Flan's own C defines $(wc -l < "$out/flan.syms") symbols;" \
+     "libasmrun defines $(wc -l < "$out/ml.syms")."
+echo "collisions between Flan's C and the OCaml runtime:"
+if comm -12 "$out/flan.syms" "$out/ml.syms" | grep . ; then
+  echo "  ^^ those would have to be renamed"
+else
+  echo "  none"
+fi
+
+echo "collisions among Flan's own four .c files:"
+for a in rt dev ag dl; do defs "$out/$a.o" > "$out/$a.syms"; done
+found=0
+for a in rt dev ag dl; do
+  for b in rt dev ag dl; do
+    [ "$a" \< "$b" ] || continue
+    c=$(comm -12 "$out/$a.syms" "$out/$b.syms")
+    [ -n "$c" ] && { echo "  $a vs $b:"; echo "$c" | sed 's/^/    /'; found=1; }
+  done
+done
+[ $found -eq 0 ] && echo "  none"
+
+echo "what Flan's C needs that the OCaml runtime also exports (shared libc etc):"
+nm --undefined-only "$out/rt.o" "$out/ag.o" 2>/dev/null | awk 'NF==2{print $2}' \
+  | sort -u > "$out/need.syms"
+comm -12 "$out/need.syms" "$out/ml.syms" | sed 's/^/  /' | head -20