A backend mismatch is refused by the loader, and says so

lib/emit.ml

@@ -78,6 +78,17 @@ let struct_name_of (t : Types.t) =
 let cellptr n = "@" ^ quoted ("flan.cellp." ^ n)
 let globalptr n = "@" ^ quoted ("flan.gp." ^ n)
 
+(* Which backend built this image. A dev build defines its own marker and a
+   redefinition module emits a data relocation against the one it was built
+   for, so a crossed pair — an LLVM module in an [--x86] host, or the reverse —
+   is refused by the loader at [dlopen] instead of running until the first call
+   into a redefined function that takes or returns a struct, which is where the
+   two conventions disagree and where the crossed pair was measured dying with
+   SIGSEGV. See [X86.abi_marker], which is the same mechanism spelled for the
+   other backend, and HANDOFF-x86-abi-marker.md. *)
+let abi_marker = "flan.abi.llvm"
+let abi_marker_sym = "@" ^ quoted abi_marker
+
 (* ── Types ─────────────────────────────────────────────────────────── *)
 
 let rec ll (t : Types.t) =
@@ -2914,6 +2925,13 @@ let program ?(checks = true) ?(dev = false) ?(debug = false) ?(pnames = [])
      Nothing has been redefined yet, so a dev build starts out behaving exactly
      like a release one — the indirection is the only difference. *)
   if dev then begin
+    (* The ABI marker, defined here so a redefinition module can bind against
+       it, and only in a dev build: a release build has no cells and nothing to
+       load into one, so it keeps exactly the module text it had before this
+       existed. [-rdynamic] is what puts it in the executable's dynamic symbol
+       table, and a dev build is the only build that gets that either. *)
+    Buffer.add_string m.out
+      (Printf.sprintf "%s = global i64 0\n" abi_marker_sym);
     List.iter
       (fun (fn : Tast.fn) ->
          Buffer.add_string m.out
@@ -3029,6 +3047,17 @@ let redefinition ?(checks = true) ?(dev = false) ?(debug = false)
               (globalptr g.Tast.gname)))
     p.Tast.globals;
   if dev then begin
+    (* The host's ABI marker, and a pointer-sized datum holding its address.
+       That datum is a relocation the loader has to resolve while it maps the
+       object, so a host built by the other backend — which defines
+       [flan.abi.x86] and not this — fails the [dlopen] outright, rather than
+       loading and then dying at the first call into a redefined function that
+       takes or returns a struct. Hidden, so this module's own copy can never
+       be interposed by another loaded module's; the relocation against the
+       host's marker is the only part that matters. *)
+    Buffer.add_string m.out
+      (Printf.sprintf "%s = external global i64\n%s = hidden global ptr %s\n\n"
+         abi_marker_sym ("@" ^ quoted "flan.abi.require") abi_marker_sym);
     (* The cells are the host's, like the globals. Referencing one is how a
        redefined function reaches its siblings, and storing into one is how it
        replaces itself. A name the host lacks gets a slot instead, filled by

lib/x86.ml

@@ -408,6 +408,28 @@ let gsym n = asm_sym ("flan." ^ n)
    Byte-for-byte or the link fails and the piece served nothing. *)
 let csym n = asm_sym ("flan.cell." ^ n)
 
+(* The marker that says which backend built an image, and it is the whole of
+   the answer to the one way these two backends can be mixed and be wrong.
+   [emit.ml] and this file agree on every scalar and disagree on every
+   aggregate — this file passes a struct by pointer with a hidden [sret] and
+   LLVM classifies per SysV — so a redefinition module from one backend
+   dlopened into a host from the other links, loads, and then dies at the first
+   call into a redefined function that takes or returns a struct. That was
+   measured as SIGSEGV; see HANDOFF-x86-aggregates.md.
+
+   A dev build defines its own marker and a redefinition module emits a data
+   relocation against the marker it was itself built for. A matched pair binds
+   it and notices nothing. A crossed pair has no such symbol to bind, and the
+   loader refuses the module at [dlopen] — before a single instruction of the
+   new body runs, and with the missing symbol naming the backend in the
+   message. That is the property: the mismatch is caught by the loader rather
+   than by the processor, at load rather than at a call.
+
+   [Emit.abi_marker] is the same string for the LLVM half. The two must stay
+   distinct and neither may ever be defined by both backends, or the refusal
+   quietly stops refusing. *)
+let abi_marker = "flan.abi.x86"
+
 (* ── Debug information ───────────────────────────────────────────────── *)
 
 (* DWARF, written out as bytes, for the same reason the instructions are — and
@@ -3266,6 +3288,19 @@ let program ~checks ?(dev = false) ?(debug = false) (p : Tast.program) : string
     (Printf.sprintf "\t.section\t.init_array,\"aw\",@init_array\n\t.align\t8\n%s\
                      \t.quad\t%s\n\n"
        (if dev then "\t.quad\tflan_dev_reg_enable\n" else "") init_sym);
+  (* The ABI marker, and only in a dev build: it exists for redefinition
+     modules to bind against, a release build has no cells to load one into,
+     and gating it here is what keeps a release build's assembly byte-for-byte
+     what it was. [.globl] and default visibility, for the reason the cells
+     have them — a dlopened object has to be able to see it, which is also why
+     [Build.executable] passes [-rdynamic] for a dev build and nothing else. *)
+  if dev then
+    Buffer.add_string out
+      (Printf.sprintf
+         "\t.data\n\t.globl\t%s\n\t.align\t8\n\t.type\t%s, @object\n\
+          \t.size\t%s, 8\n%s:\n\t.quad\t0\n\n"
+         (asm_sym abi_marker) (asm_sym abi_marker) (asm_sym abi_marker)
+         (asm_sym abi_marker));
   if dev then Buffer.add_string out (emit_cells p);
   Buffer.add_string out (emit_globals_data md p.Tast.globals);
   Buffer.add_string out "\n\t.section\t.rodata\n";
@@ -3423,6 +3458,18 @@ let redefinition ~checks ?(dev = true) ?(known = fun _ -> true)
     "\t.size\tflan_reload_install, . - flan_reload_install\n\n";
   let out = Buffer.create 8192 in
   Buffer.add_buffer out text;
+  (* The ABI marker this module requires of its host. A pointer-sized datum
+     holding the host's marker is a relocation the loader has to resolve while
+     it maps the object, whatever it does about lazy binding of calls, so a
+     host that does not define [flan.abi.x86] fails the [dlopen] outright. A
+     call would do as well under [RTLD_NOW], which is what both loaders here
+     pass, but a datum does not depend on that and costs eight bytes.
+
+     The label is local: nothing outside this module names it, and only the
+     relocation against the marker matters. *)
+  Buffer.add_string out
+    (Printf.sprintf "\n\t.data\n\t.align\t8\n%s:\n\t.quad\t%s\n"
+       (asm_sym "flan.abi.require") (asm_sym abi_marker));
   Buffer.add_string out "\n\t.section\t.rodata\n";
   Buffer.add_buffer out rodata;
   Buffer.add_string out "\n\t.section\t.note.GNU-stack,\"\",@progbits\n";

test/reload_host.c

@@ -34,6 +34,7 @@
 #include <dlfcn.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <string.h>
 #include <time.h>
 
 /* The Flan symbols the executable itself defines. Flan names contain
@@ -62,11 +63,43 @@ static double now_ms(void) {
   return (double)t.tv_sec * 1e3 + (double)t.tv_nsec / 1e6;
 }
 
+/* The same sentence [vendor/agent/flan_agent.c] says, for the same failure.
+ * Duplicated rather than shared: the agent is vendored to be dropped into a
+ * user's game and carries no header of its own, and this host is a test
+ * fixture that links against neither it nor the runtime's dev half.
+ *
+ * A dev build defines a marker naming the backend that built it, and a
+ * redefinition module holds a pointer to the marker it was itself built for.
+ * The two backends agree on scalars and disagree on every aggregate, so a
+ * crossed pair would run until the first call into a redefined function that
+ * takes or returns a struct and then die with SIGSEGV. The marker turns that
+ * into a relocation the loader cannot resolve. What it says then is
+ * "undefined symbol: flan.abi.x86", so the marker's name is matched — not the
+ * loader's phrasing, which is libc's to change — and the reason is stated. */
+static const char *abi_mismatch(const char *err) {
+  if (err == NULL) return NULL;
+  if (strstr(err, "flan.abi.x86") != NULL)
+    return "the module and this host were built by different backends: the "
+           "module came from the x86 dev backend and needs flan.abi.x86, "
+           "which this host does not define. The two backends pass every "
+           "struct differently. Rebuild the host with --x86.";
+  if (strstr(err, "flan.abi.llvm") != NULL)
+    return "the module and this host were built by different backends: the "
+           "module came from LLVM and needs flan.abi.llvm, which an --x86 "
+           "host does not define. The two backends pass every struct "
+           "differently. Rebuild the host without --x86.";
+  return NULL;
+}
+
 static int install(const char *path) {
   double t0 = now_ms();
   void *h = dlopen(path, RTLD_NOW | RTLD_LOCAL);
   if (h == NULL) {
-    fprintf(stderr, "dlopen %s: %s\n", path, dlerror());
+    /* [dlerror] is one-shot, so the pointer is taken once and used twice. */
+    const char *err = dlerror();
+    const char *why = abi_mismatch(err);
+    if (why != NULL) fprintf(stderr, "flan: %s\n", why);
+    else fprintf(stderr, "dlopen %s: %s\n", path, err);
     return 0;
   }
   install_fn f = (install_fn)(uintptr_t)dlsym(h, "flan_reload_install");

test/test_reload.ml

@@ -335,37 +335,88 @@ let () =
       List.iter (fun p -> try Sys.remove p with Sys_error _ -> ())
         [ h; m1; m2; o; e ]
     in
-    agg_run "llvm" dev (fun q name ->
+    let agg_llvm_mod q name =
       let o = tmp name in
       let ir = Emit.redefinition ~dev:true ~known:agg_known q ~fns:agg_fns in
       ignore (Build.shared ~opts:dev ~ir ~out:o ());
-      o);
+      o
-    agg_run "x86" x86 (fun q name ->
+    in
+    let agg_x86_mod q name =
       let o = tmp name in
       let asm =
         X86.redefinition ~checks:true ~dev:true ~known:agg_known q ~fns:agg_fns
       in
       ignore (Build.shared_x86 ~opts:x86 ~asm ~out:o ());
-      o);
+      o
-    (* The mismatch, which is the same measurement run crossed. An --x86 host
+    in
-       given LLVM-built modules dies with SIGSEGV on the first call into a
+    agg_run "llvm" dev agg_llvm_mod;
-       redefined aggregate body — measured, not argued:
+    agg_run "x86" x86 agg_x86_mod;
+
+    (* The same measurement run crossed, which is what the marker symbol is
+       for. Before it, an --x86 host given LLVM-built modules loaded them and
+       then died with SIGSEGV on the first call into a redefined aggregate
+       body:
 
          got: "a1\nhost 54063108\na1\n" (exit 139)
 
-       That is not asserted here. It is undefined behaviour and what it prints
+       That could not be asserted. It was undefined behaviour and what it
-       is a property of whichever LLVM is installed; a test that pins it would
+       printed was a property of whichever LLVM happened to be installed; a
-       be pinning the shape of a crash.
+       test pinning it would have been pinning the shape of a crash.
 
-       What is asserted is narrower, and the gap between the two is the finding
+       It is deterministic now, which is why it is here. A dev build defines a
-       rather than a caveat on it: [Build.opts] is where the backend choice
+       marker naming the backend that built it — [flan.abi.x86] or
-       lives, so a builder handed the *other* backend's option record refuses.
+       [flan.abi.llvm] — and a redefinition module holds a pointer to the one
-       The crossed run above passes both refusals — it hands [Build.shared] an
+       it was itself built for. That pointer is a relocation the loader has to
-       LLVM record and never calls [Build.shared_x86] at all — and it still
+       resolve while it maps the object, so a crossed pair fails the [dlopen]
-       segfaults with this guard in place, re-measured after it landed. The
+       outright, before a single instruction of the new body runs. Both
-       guard catches a caller holding one option record; it cannot catch a
+       directions, because a marker only one of the two backends emitted would
-       caller holding two, and [flan reload] is exactly that caller. See
+       refuse in one direction and say nothing in the other.
-       HANDOFF-x86-aggregates.md. *)
+
+       Asserted on the message as well as the exit status, the way the
+       retyped-global and registry-overflow cases below are: a nonzero exit is
+       not by itself this refusal, and the point of the exercise is that what
+       reaches a user names the reason rather than repeating the loader's
+       "undefined symbol". *)
+    let agg_cross label opts mkmod wants =
+      let h = tmp ("agg-xhost-" ^ label) in
+      ignore
+        (Build.executable ~opts ~csrcs:[ "reload_host.c" ] ~lflags:[ "-ldl" ]
+           a1 ~out:h);
+      let m1 = mkmod a1 ("agg-cross-" ^ label ^ "-1.so") in
+      let o = tmp ("agg-xout-" ^ label) and e = tmp ("agg-xerr-" ^ label) in
+      let code =
+        Sys.command
+          (Printf.sprintf "%s %s > %s 2> %s" (Filename.quote h)
+             (Filename.quote m1) (Filename.quote o) (Filename.quote e))
+      in
+      let said = In_channel.with_open_bin e In_channel.input_all in
+      if code = 0 then
+        fail "%s: a crossed pair loaded and ran (exit 0)" label;
+      if not (has said "built by different backends") then
+        fail "%s: a crossed pair was refused without naming the reason: %S"
+          label said;
+      (* Which marker is missing is which backend built the module, so this is
+         also what says the refusal fired for the right direction rather than
+         for the other one. *)
+      if not (has said wants) then
+        fail "%s: the refusal named the wrong marker (wanted %s): %S" label
+          wants said;
+      List.iter (fun p -> try Sys.remove p with Sys_error _ -> ())
+        [ h; m1; o; e ]
+    in
+    (* An --x86 host handed an LLVM module: the pair the CLI can build today,
+       since [flan reload] has no --x86 spelling. *)
+    agg_cross "x86-host-llvm-module" x86 agg_llvm_mod "flan.abi.llvm";
+    (* And the reverse, which no command spells but [X86.redefinition] does. *)
+    agg_cross "llvm-host-x86-module" dev agg_x86_mod "flan.abi.x86";
+
+    (* The option-record guard, which is the older and narrower half of the
+       same answer: [Build.opts] is where the backend choice lives, so a
+       builder handed the *other* backend's option record refuses by name. It
+       catches a caller holding one option record and reaching for the wrong
+       builder. It cannot catch a caller holding two — the crossed runs above
+       pass both of these refusals — which is what the marker is for. See
+       HANDOFF-x86-aggregates.md and HANDOFF-x86-abi-marker.md. *)
     (match Build.shared ~opts:x86 ~ir:"" ~out:(tmp "never.so") () with
      | _ -> fail "Build.shared accepted an --x86 option record"
      | exception Failure m when has m "--x86" -> ()

vendor/agent/flan_agent.c

@@ -674,6 +674,49 @@ static void emit(sink *o, const void *p, size_t n) {
 
 static void reply(sink *o, const char *s) { emit(o, s, strlen(s)); }
 
+/* A dlopen failure the compiler's two backends are responsible for, turned
+ * into a sentence that says so.
+ *
+ * Flan has two native backends. They agree about every scalar and disagree
+ * about every aggregate — the x86 dev backend passes a struct by pointer with
+ * a hidden sret, LLVM classifies it per the SysV psABI — so a redefinition
+ * module built by one and loaded into a host built by the other links, loads,
+ * and then dies with SIGSEGV at the first call into a redefined function that
+ * takes or returns a struct. A dev build therefore defines a marker naming its
+ * backend and a module holds a pointer to the marker it was built for, which
+ * is a relocation the loader must resolve while it maps the object. A crossed
+ * pair has no such symbol and is refused here, before any of the new code
+ * runs.
+ *
+ * What the loader says at that point is "undefined symbol: flan.abi.x86",
+ * which is true and tells nobody anything. So the marker's name is matched —
+ * the name, not the loader's phrasing, which is libc's to change — and the
+ * reason is stated instead. Which marker is missing says which backend built
+ * the module, and the host is necessarily the other one.
+ *
+ * Returns NULL for a failure that is about something else, which is then
+ * passed through as the loader wrote it. */
+static const char *abi_mismatch(const char *err) {
+  if (err == NULL) return NULL;
+  if (strstr(err, "flan.abi.x86") != NULL)
+    return "the module and the running program were built by different "
+           "backends: the module came from the x86 dev backend and needs "
+           "flan.abi.x86, which this program does not define. The two "
+           "backends pass every struct differently, so the pair would die at "
+           "the first call into a redefined function that takes or returns "
+           "one. Rebuild the program with --x86 so that both halves agree.";
+  if (strstr(err, "flan.abi.llvm") != NULL)
+    return "the module and the running program were built by different "
+           "backends: the module came from LLVM and needs flan.abi.llvm, "
+           "which an --x86 program does not define. The two backends pass "
+           "every struct differently, so the pair would die at the first call "
+           "into a redefined function that takes or returns one. Rebuild the "
+           "program without --x86: there is no --x86 spelling for building a "
+           "redefinition module yet, so the program is the half that has to "
+           "move.";
+  return NULL;
+}
+
 /* One line, one answer, one module. This is the whole of what the agent is
  * asked, and it is reached two ways: from the socket below, and — in a build
  * where the compiler is a thread in this same process — by being called. The
@@ -1120,8 +1163,12 @@ static void handle_line(char *line, sink *o) {
   }
   void *h = dlopen(line, RTLD_NOW | RTLD_LOCAL);
   if (h == NULL) {
+    /* [dlerror] is one-shot and the next dl call may clobber what it returned,
+     * so the pointer is taken once and used for both the test and the reply. */
+    const char *err = dlerror();
+    const char *why = abi_mismatch(err);
     reply(o, "err ");
-    reply(o, dlerror());
+    reply(o, why != NULL ? why : err);
     reply(o, "\n");
     return;
   }