From a9d93c7b64498537ac51c53c46232f6d3d96e4fc Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Thu, 10 Sep 2026 19:08:48 +0700 Subject: [PATCH] Updates to building --- .gitignore | 4 +++ NEXT.md | 98 +++++++++++++++++++++++++++++++++++++++++++++------- lib/build.ml | 83 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 167 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 4c0d4bc..299b109 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ _opam/ # The pre-rewrite menhir/ocamllex frontend: reference only, excluded from the # build by the root dune file. Its contents are in git history at 2c232dd. old-ocaml/ + +# Built executables from `flan build` +/calc-me +/sand diff --git a/NEXT.md b/NEXT.md index 3192c70..0474117 100644 --- a/NEXT.md +++ b/NEXT.md @@ -112,6 +112,14 @@ little-endian reading of the packed integer, so an identity would have passed a weaker test. That case is in the acceptance table, skipped if `libraylib` is not installed. +The bindings are 18 calls: window (`init-window`, `close-window`, +`window-should-close?`, `set-target-fps`, `set-trace-log-level`), keyboard +(`key-pressed?`/`down?`/`released?`), mouse (`mouse-button-pressed?`/`down?`/ +`released?`, `get-mouse-position`), `get-color`, and drawing (`begin-drawing`, +`end-drawing`, `draw-fps`, `clear-background`, `draw-rectangle`), plus the +`Key`, `MouseButton` and `TraceLogLevel` enums. Adding one is three lines: a +`declare`, an `extern` prototype, and a one-line wrapper. + No raylib headers are needed: `shim.c` declares the prototypes it uses, so the build depends on the shared library being linkable and not on `raylib-devel`. `vendor/raylib/link` carries `-l:libraylib.so.550` because Fedora ships the @@ -166,6 +174,10 @@ decisions rather than fixes: `(defn main [])` is unchanged — the short form, as plan.org says. +Painting is on **hold left mouse button** rather than on space, since the mouse +bindings exist now. Space is still what cycles the colour, on release, which is +a leftover and probably wants to move to the right button or to a key press. + ## Bounds checks — done at milestone 3 `at` and `slice` emit `icmp` → `br` → cold block → `call` → `unreachable`; a @@ -250,26 +262,86 @@ proposal. ## Where build time goes -`flan build calc-me.flan` is ~140ms, and ~95% of it is clang: +`flan build calc-me.flan` was ~160ms, and ~95% of it was clang. **The object +cache is in**, and it is now ~110ms: | Step | Cost | |---|---| | frontend: read → parse → load → check → emit | <10ms, below the timer | | `clang` on the `.ll` | 60ms — `llc` does the same codegen in **20ms** | -| `clang` on `flan_rt.c` | 40ms — recompiled every build, never changes | +| `clang` on `flan_rt.c` | 40ms — **now cached, paid once** | | link | 20ms | -sand.flan additionally recompiles `shim.c` every build. Two cheap wins take the -base to ~60ms: cache `flan_rt.o` (and the packages' `.o`), and skip the clang -driver for the `.ll` (`llc` + link directly). Both are a subset of the dev -path's machinery, so doing them now is not wasted work. +Every C translation unit a build needs — the host shim and each package's shim +— goes through `Build.compile_c`, which compiles to a `.o` under +`$TMPDIR/flan-objcache` and reuses it. The key is a digest of the source text, +the compiler (its path, size and mtime, so an upgrade invalidates without +paying a `clang --version` subprocess per build), `opts.opt` and `opts.target`. +The opt level has to be in there: the acceptance table builds the same programs +at `-O0` and `-O2`, and an `-O2` object must not serve an `-O0` build. The +object is written to a temporary name and `rename`d into place, so two +concurrent builds cannot see a half-written one. + +Measured: calc-me 160ms → 110ms; sand ~720ms → ~700ms, since sand's time is +mostly linking libraylib and its `shim.c` was never the cost. The cache is +keyed by content, so it never needs invalidating by hand — `rm -rf` on the +directory is only ever a disk-space decision. + +The other cheap win is still open: skip the clang driver for the `.ll` (`llc` + +link directly), worth another ~40ms. It is a subset of the dev path's +machinery. Check `llc`'s major version against clang's before relying on it — +the emitted IR text is currently absorbed by the driver behind +`-Wno-override-module`, and a version mismatch surfaces as IR parse errors. **There is still no REPL.** Nothing does redefinition, `dlopen`, or nREPL. `build` is the only way to run code. -## Next +## Next — the REPL is the priority -1. **wasm32.** The user installed `wasi-libc-devel` and `wasi-libc-static`; the +Decided in conversation: wasm32 can wait (it is believed to be a solved problem +once the builtins archive is in place), and **the dev loop is the thesis of the +project**, so it comes first. Staged so each step is runnable on its own — +the failure mode is building a daemon and a protocol before knowing the reload +primitive works. + +1. **The reload primitive, measured.** `llc` + `ld -shared` → `.so` → + `dlopen` → call. No sockets, no protocol. A test that compiles one function, + loads it, calls it, recompiles it changed, and calls it again. plan.org's + 16ms was measured with `clang` in isolation and never in this codebase. + It forces the first real change: `emit.ml` needs a mode that compiles one + redefinable function into its own module *against the existing globals*, + rather than as a whole program. +2. **Indirection cells.** Every cross-function call in a dev build goes through + a pointer; redefinition is one atomic store. A fork in `emit.ml` between dev + and release codegen, and the first time `Build.opts` means something + semantic rather than an optimisation level. `test/programs/` gets a case + where a running loop's callee is swapped mid-run. +3. **The agent, in C.** A socket listener in the game process, `dlopen` off the + game thread with `RTLD_NOW`, and the staged cell publish at a frame + boundary. It lives next to `flan_rt.c` — no OCaml runtime in the game + binary. sand.flan is the test: redefine `settle` while grains are falling + and see the behaviour change with no stutter and no dropped frame. +4. **The daemon and nREPL** (bencode over a socket; `eval`, `load-file`, + `describe`, `interrupt`), then **5. the Emacs client** — a focused ~3–5k + line client, not a CIDER fork. Deliberately last and deliberately separate: + the protocol is mechanical once 1–3 exist, and the editor client is where + the taste is. + +**Two decisions to settle before step 2**, because both change codegen and are +painful to retrofit: + +- **Do cells cover globals, or only functions?** plan.org says redefining a + `defvar` is not covered (open decision #6, milestone 7). But sand's `grid` is + a global, and "edit the code, keep the sand" is exactly the demo — which + works only if globals *survive* a reload, meaning the new `.so` must not + re-emit them. +- **What is a redefinition unit — one function, or a file?** A file is much + easier to make correct and is what `load-file` wants anyway; one function is + what `C-c C-c` wants and is where the 16ms number comes from. + +Deferred until after the dev loop: + +6. **wasm32.** The user installed `wasi-libc-devel` and `wasi-libc-static`; the sysroot is `/usr/wasm32-wasi` and `wasm-ld` is present. `clang --target=wasm32-wasi --sysroot=/usr/wasm32-wasi` gets past the headers and then **fails to link**: it wants @@ -282,11 +354,7 @@ path's machinery, so doing them now is not wasted work. Note plan.org has the *web* build linking raylib via emscripten, which brings its own sysroot: wasi-sdk is right for the headless table, not necessarily for the eventual game build. -2. **The dev path / REPL.** `llc` + `ld -shared` + `dlopen` ≈ 16ms, a compiler - daemon plus an in-game reload agent (plan.org, Dev architecture). There is - now a frame loop for it to not stutter, which was the reason to do it after - milestone 4. -3. **Loose ends from milestone 4**, none of them blocking: block-scoped +7. **Loose ends from milestone 4**, none of them blocking: block-scoped `defer`; package visibility, so `rl/get-color-raw` is not callable; a package importing a package; imported unions. @@ -302,6 +370,10 @@ The tests assert on the reason, not just on the failure. ## Untracked on purpose +`calc-me` and `sand`, the executables `flan build` drops beside their sources, +are now in `.gitignore` — anchored (`/calc-me`, `/sand`) so the patterns cannot +also match `sand-sim/` or anything nested. + `old-ocaml/` — the pre-rewrite menhir/ocamllex frontend, kept as reference and excluded from the build by the root `dune` file. Its contents are also in git history at `2c232dd`. diff --git a/lib/build.ml b/lib/build.ml index 368b8ff..6ffeeef 100644 --- a/lib/build.ml +++ b/lib/build.ml @@ -25,6 +25,15 @@ let workdir () = (try Unix.mkdir d 0o700 with Unix.Unix_error (Unix.EEXIST, _, _) -> ()); d +(* The object cache, which unlike [workdir] is stable across builds. The C that + goes into a build — the host shim and the packages' shims — is the same on + every build and never the thing being edited, yet it was being recompiled + each time: 40ms of a 140ms build for [flan_rt.c] alone. *) +let cachedir () = + let d = Filename.concat (Filename.get_temp_dir_name ()) "flan-objcache" in + (try Unix.mkdir d 0o700 with Unix.Unix_error (Unix.EEXIST, _, _) -> ()); + d + type opts = { target : string option; (* None is the host; "wasm32-wasi" is the other *) opt : string; @@ -38,6 +47,65 @@ type opts = { checks. Dropping them is a release decision, not an optimisation one. *) let default = { target = None; opt = "-O2"; keep = false; checks = true } +(* What the compiler itself is, cheaply: its path, size and mtime. A clang + upgrade changes one of those, so the key changes with it — without paying a + [clang --version] subprocess on every build, which would cost most of what + the cache buys. *) +let clang_stamp = + lazy + (let path = + if Filename.is_relative clang then + let dirs = String.split_on_char ':' (try Sys.getenv "PATH" with Not_found -> "") in + (try List.find (fun d -> Sys.file_exists (Filename.concat d clang)) + dirs |> fun d -> Filename.concat d clang + with Not_found -> clang) + else clang + in + match Unix.stat path with + | st -> Printf.sprintf "%s:%d:%f" path st.Unix.st_size st.Unix.st_mtime + | exception Unix.Unix_error _ -> path) + +(* Compile one C translation unit to an object file, reusing a cached one when + the source text, the compiler and the flags are all unchanged. The key has + to carry [opt] and [target]: the acceptance table builds the same programs + at -O0 and -O2, and an -O2 object must not serve an -O0 build. *) +let compile_c ~opts ~src ~name = + let key = + Digest.to_hex + (Digest.string + (String.concat "\000" + [ name; src; Lazy.force clang_stamp; opts.opt; + (match opts.target with None -> "" | Some t -> t) ])) + in + let obj = Filename.concat (cachedir ()) (key ^ ".o") in + if not (Sys.file_exists obj) then begin + let dir = workdir () in + let c = Filename.concat dir name in + write c src; + (* A distinct temporary target, renamed into place, so two builds running + at once cannot see a half-written object. *) + let tmp = Printf.sprintf "%s.%d.tmp" obj (Unix.getpid ()) in + let cmd = + String.concat " " + ([ Filename.quote clang; opts.opt; "-c" ] + @ (match opts.target with None -> [] | Some t -> [ "--target=" ^ t ]) + @ [ Filename.quote c; "-o"; Filename.quote tmp ]) + in + let code = Sys.command cmd in + if code <> 0 then + failwith (Printf.sprintf "%s failed (exit %d) on %s" clang code name); + (try Unix.rename tmp obj with Unix.Unix_error _ -> ()); + (try Sys.remove c with Sys_error _ -> ()) + end; + obj + +let read_file path = + let ch = open_in_bin path in + let n = in_channel_length ch in + let s = really_input_string ch n in + close_in ch; + s + (* [csrcs] and [lflags] come from the imported packages (see [Load]): the C shim a package binds through, and the arguments needed to link the library it binds to. *) @@ -45,20 +113,25 @@ let executable ?(opts = default) ?(csrcs = []) ?(lflags = []) (p : Tast.program) ~out = let dir = workdir () in let ll = Filename.concat dir (Filename.basename out ^ ".ll") in - let rt = Filename.concat dir "flan_rt.c" in write ll (Emit.program ~checks:opts.checks p); - write rt Runtime_src.source; + let objs = + compile_c ~opts ~src:Runtime_src.source ~name:"flan_rt.c" + :: List.map + (fun c -> + compile_c ~opts ~src:(read_file c) ~name:(Filename.basename c)) + csrcs + in let cmd = String.concat " " ([ Filename.quote clang; opts.opt; "-Wno-override-module" ] @ (match opts.target with None -> [] | Some t -> [ "--target=" ^ t ]) - @ [ Filename.quote ll; Filename.quote rt ] - @ List.map Filename.quote csrcs + @ [ Filename.quote ll ] + @ List.map Filename.quote objs @ lflags @ [ "-o"; Filename.quote out ]) in let code = Sys.command cmd in if code <> 0 then failwith (Printf.sprintf "%s failed (exit %d); the IR is at %s" clang code ll); - if not opts.keep then (try Sys.remove ll; Sys.remove rt with Sys_error _ -> ()); + if not opts.keep then (try Sys.remove ll with Sys_error _ -> ()); out