From 889fdc30d9d8d9b68b14877dc8f85e3ad87baf9c Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 19 Sep 2026 06:59:22 +0700 Subject: [PATCH] A file that does not parse says so, and the provider is written down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The louder failure had the quieter answer. A generated reader accumulates errors on the cursor — which is what lets it be a straight line of assignments — and the cursor is made and dropped inside the entry point, so a stray brace in a file read at run time handed the program a zeroed struct and said nothing at all. That is the one thing the rest of vendor:edn refuses to do: read-file answers an Option precisely so a malformed document is distinguishable from one that is literally nil, and the hand-written reader in test/programs/edn.flan tests ok? and prints the reason. ReadFailed is the derived reader being as honest, in both packages, and it sits beside SchemaDrift because both are "the file is not what this program was built for". Then the writing-down. docs/BUILT.md gets the section: the four things the macro system did not have and now does, each general and none of them mentioning EDN — a macro reading a file at the call site's path, a package macro calling its package, one call answering several declarations, and compile-error, which is the one piece that had to go in the compiler and the reason it had to. The set rule the real game file decided is there too, and what defjson shares, which is the design and not the code. NEXT.md item 9 and PORTING.md §3.9 both close. Not as (read-edn T bytes): the struct comes from the *file* rather than from a type declared by hand, so the ~80 lines PORTING prices for two schemas are not written at all. The competing answer PORTING names — compile-time embedding — turned out to be the other half rather than a competitor: the shape comes from the file at compile time and the bytes may come from an embed beside it, which is exactly what test/programs/edn-provide.flan does. --- NEXT.md | 11 ++- docs/BUILT.md | 175 +++++++++++++++++++++++++++++++++ docs/PORTING.md | 22 +++-- test/programs/edn-provide.flan | 28 +++++- test/test_acceptance.ml | 15 ++- vendor/edn/provide.flan | 35 ++++++- vendor/json/provide.flan | 23 ++++- 7 files changed, 292 insertions(+), 17 deletions(-) diff --git a/NEXT.md b/NEXT.md index 5e88e35..379f3c8 100644 --- a/NEXT.md +++ b/NEXT.md @@ -1073,8 +1073,15 @@ rather than cumulative, which is a deliberate divergence from `watch.clj` argued five numbers, and the window is the editor's". **Item 6 landed the same day**: `test/programs/frame-rollback.flan` is the worked example — `snapshot` at the top of the frame, `restore` in the `continue` clause — and `bounds-condition.flan`'s half-written abandoned frame is the thought it finishes. That was -the last Tier 1 item anyone was going to move. Items 7 (`drop`), 8 (generics) and 9 (`(read-edn T bytes)`) are still -on that list and still deferred with reasons written beside each; none is a blocker for this game. +the last Tier 1 item anyone was going to move. **Item 9 (`(read-edn T bytes)`) landed 2026-09-19**, and not as +`read-edn`: `(edn/defedn Tileset "assets/tileset.edn")` derives the struct from the *file* at compile time and +emits a reader with it, so the ~80 lines `docs/PORTING.md` prices for two schemas are not written by hand and not +written by the compiler either. It is a macro in `vendor:edn`, and `defjson` is the same over `vendor:json`. The +competing answer PORTING names — compile-time embedding — is not a competitor after all but the other half: the +shape comes from the file at compile time and the bytes may come from an `embed` beside it. See +[`docs/BUILT.md`](docs/BUILT.md), "A type provider: the struct a data file implies". Items 7 (`drop`) and 8 +(generics) are still on that list and still deferred with reasons written beside each; neither is a blocker for +this game. **What `docs/PORTING.md` says NOT to build, with evidence:** escaping closures (one capture site, fixed by one parameter), `Handle`/pools, `Result`/`try`, `handler-case`, `loop`/`recur` and tail calls, user allocators, structural typing — diff --git a/docs/BUILT.md b/docs/BUILT.md index ceb9d46..4b15117 100644 --- a/docs/BUILT.md +++ b/docs/BUILT.md @@ -5804,3 +5804,178 @@ the same comparison the prelude uses. An infinity still prints signed: there the the value, and the backends were always agreed about it. Pinned in `test/programs/format.flan`, which is in the x86 survey corpus, so one program holds both the printed form under `dune test` and the agreement under the survey. + +## A type provider: the struct a data file implies, derived while the program is compiled + +`(edn/defedn Tileset "assets/tileset.edn")` reads the file at expansion time, works out +what shape it is, and answers the struct that shape implies together with a reader over +the tokenizer next door. `(.texture-path data)` is then a field load off a struct nobody +declared: no `Value`, no `match`, no runtime tag, nothing looked up by name. F#'s type +providers with the useful half and none of the plugin protocol, and `vendor/json`'s +`defjson` is the same thing over a different grammar. + +Both are macros in their packages, not compiler builtins. That was the requirement and it +held, but only after four things the macro system did not have. Each is general and none +of them mentions EDN. + +### A macro may read a file, at the path the call site is written at + +`(embed "assets/x.edn")` resolves against the directory of the *source file the form is +written in* — check.ml's `embed_path`, and Odin's rule before it — because anything else +makes a package's assets depend on where `flan` happened to be invoked from. A macro had +no way to honour that rule: it runs inside the compiler and could always have called +`slurp`, but a `Form` carries no location, deliberately, so a macro handed +`(defedn T "assets/x.edn")` knows the path and not what it is relative to. + +So the compiler tells it. `lib/macro.ml`'s `dir_of` pokes the call site's directory into +two C symbols in the macro module's own `flan_rt.c` before every expansion, and the +prelude's `(macro-slurp "...")` joins the two. C data and not a Flan global because +`Build.macro_module` emits with hidden visibility and only the `flan.macro.*` thunks stay +exported — the other half of that same comment is that the C goes on resolving the way it +always did, which is what makes these findable. + +It answers `None` rather than signalling, and that is why it is not `slurp`. A condition +raised inside an expansion is raised in the compiler, through the module's own copy of the +runtime, which is the failure `Build.macro_module`'s hidden-visibility note measured: it +takes the process down instead of parking it. Absence arriving as an answer is what lets +a provider refuse *about* a missing file, with a location, which is the sentence its +author wanted anyway. + +Re-expansion re-reads, and nothing had to be built for that: the cached macro module holds +the macro's code, not the data, and `expand_form` calls the macro fresh. Editing the +`.edn` and building again produces the struct that file now implies — measured by adding +a key and finding it in the emitted IR. + +### A package's macro may call the package's functions + +`Load.qualify_macro` already renamed a package macro's body so a call to the package's own +`next` reads `edn/next`; the intent was written down. The module was then compiled from +the prelude and the `defmacro`s alone, so the call arrived at the checker as *the call +edn/next into an imported package*. That was the machinery missing a piece, not a rule — +and it is why a derivation this size can be ordinary Flan over the tokenizer next door +instead of a second scanner inlined into a macro body. + +`Parse.imported_decls` carries the package's declarations beside its macros, already +qualified, and `Macro.compile` trims them to what the macro bodies reach. raylib's five +`with-*` are pure quasiquote, so nothing of raylib is reachable and its module is the one +it always was — which matters, because raylib's declarations are `declare`s against a +library a macro module has no linker argument for. The trim is over names and happens +before `Check`, since a surviving `Declare` would be emitted whether or not the checker +was asked about it. + +One thing this exposed. A quasiquote inside a package's *ordinary function* was never +qualified — only a `defmacro` body goes through `qualify_macro` — so a +`(defn ... [c (Ptr Cursor)] ...)` emitted from a derivation helper reached the importer +naming a type it had never heard of. The name survives into a *string*, which is exactly +the property the expander's walk depends on and exactly what puts it out of a rename's +reach. `rename_expr` now qualifies a literal `(Form.Sym {.s "..."})` naming something the +package owns. Until a package's macros could call its functions there was no helper that +built code, so this could not have shown before. + +### One call, several declarations + +Expansion is form-for-form, and every macro written until now expanded to an *expression*. +A provider produces the struct *and* the reader over it, and a struct per nesting level in +the data: three declarations and more from one form, which no arrangement of one-for-one +reaches. A top-level `(do ...)` is now its items, spliced in place, after expansion and +before the declaration walk. Nobody writes one in a file, and the single-declaration entry +point says so by name for anyone who tries. + +### `compile-error`, because a name carries a name and not a sentence + +This is the one piece that had to go in the compiler, and the prelude's `unless` had +already written down why: *a macro has no error facility, so a malformed call answers a +name nothing defines and the report is the right place with the wrong sentence.* A +provider's refusals are all sentence — *the third element of this vector is a string where +the first two were integers*, at line 3 column 9 of a file the compiler is not reading — +and no symbol an expansion could invent holds that. + +So a macro that has to refuse expands to `(compile-error "...")`, one arm in check.ml's +builtin match. A builtin because it has to fail *while checking*: a declared function +would compile, link and run, and the compile it was meant to stop would have succeeded. +`Loc.from_macro` already stamps the call site onto every node of an expansion, so the +location is the form the author wrote and the sentence is the macro's — the two halves the +prelude's note says are never both right at once. It is wrapped in a `defn` with a +gensym'd name, because a top-level position takes a declaration and the body is where an +expression the checker walks can live. + +### The rules it derives, and the one the game file decided + +A map with keyword keys is a struct, one field per key. An integer is `i64`, a float +`f64`, a boolean `bool`, a string `string` — *copied*, which is `read.flan`'s contract and +not the tokenizer's: a `Token`'s text points into the buffer and a struct that outlives +the buffer cannot hold one. A vector of one repeated shape is `(Vec T)`. A nested map is a +struct named for the path that reaches it, `Tileset-selected-cells`, with a hyphen because +`/` is package qualification and because every name in this language is hyphenated +already, so no case conversion has to be written at macro time. + +The set rule is the one the real file decided. A set is this repo's `(Map T bool)` — +check.ml says exactly that where it refuses a `()` value — so its elements are map *keys*, +and a vector inside a set is therefore a fixed array `[n T]` and not a `(Vec T)`: a Vec is +not a map key and `[2 i64]` is. `game-data.edn` is a set of `[x y]` pairs, so that is the +case rather than a corner of it. Every element must then be the same *length* as well as +the same shape, which falls out of the type comparison already being made, since the +length is in the type. + +Everything else is refused while expanding, with the line and column in the **data** file: +a heterogeneous collection names both positions, an empty one has no element to derive +from, a `nil` has no type, and a map with a key that is not a keyword is not a struct. A +file the compiler could not make sense of is one the program would have read wrongly. + +`(vec-new)` and `(map-new)` have to be *told* what they build by naming a type, and +`(Vec i64)` and `[2 i64]` have no name to be. Both fall back to what the context wants and +a signature is a type position where anything can be written, so each collection gets a +one-line constructor stating its type. The reader reads better for it: it says +`(cells-new a)` where it would otherwise carry a type nobody wrote. + +### Two conditions at read time, for the two ways a file stops matching + +The struct was derived from the file as it was when the program was compiled. +`SchemaDrift` names a key that has arrived or a field that has gone, once each, because a +missing key otherwise leaves a field at zero — a texture path of `""` and a count of `0` — +and the program draws nothing for a reason nothing reports. `ReadFailed` is the louder +one: a generated reader accumulates errors on the cursor, and the cursor is made and +dropped inside the entry point, so a file that does not parse would have handed back a +zeroed struct with nothing said. `read-file` answers an `Option` precisely so a malformed +document is distinguishable from one that is literally nil, and a derived reader is held +to the same honesty. Neither is fatal: signalling a condition no handler takes carries on, +so a program that would rather not care writes nothing. + +### What `defjson` shares, which is the design and not the code + +`vendor/json` imports `vendor/edn` for nothing, and borrowing a shape walk across that +line would be a dependency for the sake of a resemblance. What carries over is the shape +of the answer — one walk giving a type, the declarations it needs and the expression that +reads one; a refusal carried in a field rather than raised; the typed constructor per +collection; the gensym'd `compile-error` wrapper. + +Four things are genuinely different. Strings go through `string-of` and never through +`.text`, because `.text` is the raw interior with escapes undecoded and a field read off +it would hold a backslash and an `n` where the file meant a newline. Commas and colons are +tokens rather than whitespace. An object's keys are strings, so a key is refused when it +is not a name a program could write, and refused again when it carries an escape — the +generated reader compares against the bytes as written, which costs no allocation per key +and is only the same question when the name is written plainly. And there are no sets, so +there is no map-key path and no fixed array: every collection is a `(Vec T)`, and +`defjson` is the smaller of the two by half. JSON has no integer type either; the +tokenizer draws the line at whether a number has a fraction or an exponent, which is the +only line there is, so `1` derives `i64` and `1.0` derives `f64`. + +### What is checked + +`test/programs/edn-provide.flan` reads the real `assets/edn/tileset.edn` through a derived +struct, and its first five lines are `edn-read.flan`'s first five character for character. +Two readers over one file agreeing is what says the derived one is right; either alone +could be self-consistently wrong. The pair memberships are the derivation deciding in +public — `[3 4]` is a key and `[9 9]` is not, where a version that made the set 108 loose +integers would have compiled and answered differently on all four. + +The refusals write their own data file, because the data file *is* the test, and each is +asserted on the position it names rather than on the fact of failing. One checks a line +and column into a file the compiler is not reading, which is the whole of what +`compile-error` was added for. `test_session.ml` expands a `defedn` through a session at +an origin the editor would have sent, which is the `C-c C-m` path and the live-tuning +loop: it asserts both that the data file resolved against the buffer's directory and that +what comes back is code a person can read. The generated readers survey under `@x86` — the +fixed-array map key is a hash and equality pair nothing generated had asked the backend +for before — and `@sanitize` is clean over both programs. diff --git a/docs/PORTING.md b/docs/PORTING.md index 4349110..f4f643a 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -214,10 +214,11 @@ tileset file itself. So this is writable today, and the tileset needs no hand-written reader at all: it reads as a `Value.Table` whose `:selected-cells` is a `Value.Set` of pairs, against an arena, -released by one `free-all`. A hand-written reader is still what a *struct* costs, because -`(read-edn T bytes)` is not built — call it ~80 lines for the bitmask table if it wants -to land in a struct rather than a `Value`. See §3 for whether it should be written at -all. +released by one `free-all`. A hand-written reader is no longer what a *struct* costs: +`(edn/defedn Tileset "assets/tileset.edn")` derives the struct from the file while the +program is compiled and emits the reader with it, so the ~80 lines this used to price are +not written at all. See §3 item 9, and `docs/BUILT.md`, "A type provider: the struct a +data file implies". --- @@ -629,10 +630,15 @@ not compete for the same slot. game rather than in the engine. **Do not sequence it ahead of items 4–6 on this game's account.** -9. **`(read-edn T bytes)`.** `vendor:edn` already makes the asset readers writable; this - removes ~80 lines of hand-written cursor walking for two schemas. Convenience, and it - competes with compile-time embedding, which may be the better answer for both files - anyway. +9. ~~**`(read-edn T bytes)`.**~~ **Built**, and not under that name: `(edn/defedn Tileset + "assets/tileset.edn")` reads the *file* while the program is compiled, derives the + struct its shape implies, and emits a reader with it — so the ~80 lines are neither + hand-written nor written by the compiler from a type that was declared by hand. It is a + macro in `vendor:edn`, not a builtin, and `defjson` is the same over `vendor:json`. + The competition with compile-time embedding was the wrong reading of it: they are the + two halves of one answer, the shape from the file at compile time and the bytes from an + `embed` beside it. See `docs/BUILT.md`, "A type provider: the struct a data file + implies". **Not ranked, because this game does not need them:** escaping closures and capture (one site, fixed by one parameter), `Handle` and pools (nothing to pool), `Result`/`try` diff --git a/test/programs/edn-provide.flan b/test/programs/edn-provide.flan index 8c49ae0..318eaab 100644 --- a/test/programs/edn-provide.flan +++ b/test/programs/edn-provide.flan @@ -84,11 +84,37 @@ (println (.hp t)) (println (.speed t))))) +;; ── A file that does not parse ────────────────────────────────────── +;; +;; The louder failure, and the one that had the quieter answer until ReadFailed +;; existed: a generated reader accumulates errors on the cursor, and the cursor +;; is made and dropped inside the entry point, so a stray brace gave back a +;; zeroed struct with nothing said. The rest of this package refuses to do +;; that — read-file answers an Option so that a malformed document is +;; distinguishable from one that is literally nil — and a derived reader has to +;; be at least as honest. +(defconst broken string "{:name \"orc\" :hp }") + +(defn show-broken [a Allocator] () + (handler-bind + [(edn/ReadFailed [e] + (do (print "unreadable ") + (print (.struct e)) + (print ": ") + (println (edn/error-message (.code e)))))] + (let [t (Tuning-of-bytes (bytes broken) a)] + ;; Read anyway, and zeroed, which is the half a handler that carries on + ;; is choosing. Printed so that "it signalled" and "it gave back nothing + ;; usable" are two claims rather than one. + (println (.hp t))))) + (defn main [] i32 (let [a (heap-allocator)] (show-tileset a) (println "") (show-tuning a) (println "") - (show-drift a)) + (show-drift a) + (println "") + (show-broken a)) 0) diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index 0d8ac70..ecdb5f3 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -664,13 +664,24 @@ let () = :level, and the bytes read carry the opposite. Both conditions are named, in the order the reader meets them — the unknown key as it arrives, the missing field when the map closes — and the read carries - on, which is the other half of the contract. :speed reads 0. *) + on, which is the other half of the contract. :speed reads 0. + + Then the louder failure, which had the quieter answer until ReadFailed + existed: a generated reader accumulates errors on the cursor, and the + cursor is made and dropped inside the entry point, so a file that does + not parse handed back a zeroed struct with nothing said. The rest of the + package refuses to do that — read-file answers an Option so a malformed + document is distinguishable from one that is literally nil — and this is + the derived reader being as honest. Two lines and not one: it signalled, + and what it gave back is nothing usable. *) let edn_provide_out = "./source-assets/Sprout Lands Premium/Objects/Mushrooms, Flowers, \ Stones.png\n\ 54\ntrue\ntrue\ntrue\nfalse\n\n\ goblin\n12\n1.5\nno\n5\n14\n16\n24\n-2\n0\n\n\ - extra level in Tuning\nmissing speed in Tuning\nimp\n3\n0\n" + extra level in Tuning\nmissing speed in Tuning\nimp\n3\n0\n\n\ + unreadable Tuning: unexpected token: not the kind the caller was \ + reading\n0\n" in outputs "defedn over the tileset and a tuning file" "programs/edn-provide.flan" edn_provide_out; diff --git a/vendor/edn/provide.flan b/vendor/edn/provide.flan index 19e75d7..c4b9589 100644 --- a/vendor/edn/provide.flan +++ b/vendor/edn/provide.flan @@ -183,6 +183,30 @@ extra? bool pos i32]) +;; ── And the one a file that does not parse signals ────────────────── +;; +;; The louder failure had the quieter answer until this existed. A generated +;; reader accumulates errors on the cursor rather than returning them — which +;; is what lets it be a straight line of assignments — and the cursor is made +;; and dropped inside the entry point, so a stray brace in a file read at run +;; time gave the program a zeroed struct and said nothing at all. +;; +;; That is the one thing the rest of this package refuses to do. `read-file` +;; answers an Option precisely so that a malformed document is distinguishable +;; from a document that is literally nil, and the hand-written reader in +;; test/programs/edn.flan tests ok? and prints the reason. A derived reader has +;; to be at least as honest. +;; +;; A condition and not an Option, to match SchemaDrift beside it: both are "the +;; file is not what this program was built for", and a handler that wants to +;; carry on with a half-read struct may, while one that wants to stop has +;; something to stop on. `code` is an err-* constant, which `error-message` +;; turns into a sentence. +(defstruct ReadFailed + [struct string + code i32 + pos i32]) + ;; ── Deriving ──────────────────────────────────────────────────────── ;; ;; One value, from the cursor's current position, consumed. `name` is what a @@ -537,8 +561,15 @@ ;; field is an allocation — spec-memory's rule, and the reason the ;; destination is never implicit. (defn ~bname [b [u8] a Allocator] ~sname - (let [c (cursor b)] - (~rname (addr c) a))) + (let [c (cursor b) + out (~rname (addr c) a)] + ;; The cursor is made and dropped here, so this is the only + ;; place that can ask whether the read worked. See ReadFailed. + (when (not (ok? (addr c))) + (signal (ReadFailed {.struct ~(Form.Str {.s name}) + .code (.err c) + .pos (.err-pos c)}))) + out)) (defn ~fname [p string a Allocator] ~sname (let [b (slurp p a)] (~bname (as-slice b) a))))))))) diff --git a/vendor/json/provide.flan b/vendor/json/provide.flan index 842dd5c..3da211f 100644 --- a/vendor/json/provide.flan +++ b/vendor/json/provide.flan @@ -159,6 +159,18 @@ extra? bool pos i32]) +;; And the one a file that does not parse signals. A generated reader +;; accumulates errors on the cursor rather than returning them — which is what +;; lets it be a straight line of assignments — and the cursor is made and +;; dropped inside the entry point, so without this a stray brace in a file read +;; at run time would give the program a zeroed struct and say nothing. The +;; louder failure would have had the quieter answer. `code` is an err-* +;; constant, which `error-message` turns into a sentence. +(defstruct ReadFailed + [struct string + code i32 + pos i32]) + ;; ── Deriving ──────────────────────────────────────────────────────── (defn derive [c (Ptr Cursor) name string src [u8]] Derived @@ -430,8 +442,15 @@ ;; allocation, and spec-memory's rule is that the destination is ;; never implicit. (defn ~bname [b [u8] a Allocator] ~sname - (let [c (cursor b)] - (~rname (addr c) a))) + (let [c (cursor b) + out (~rname (addr c) a)] + ;; The cursor is made and dropped here, so this is the only + ;; place that can ask whether the read worked. See ReadFailed. + (when (not (ok? (addr c))) + (signal (ReadFailed {.struct ~(Form.Str {.s name}) + .code (.err c) + .pos (.err-pos c)}))) + out)) (defn ~fname [p string a Allocator] ~sname (let [b (slurp p a)] (~bname (as-slice b) a)))))))))