From b0bc40ca05c2f7bd7de22cce9dcf3ddd0c826b17 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 21:48:37 +0700 Subject: [PATCH 1/6] atan2 and pow go out to libm, and clamp is a macro rather than four functions The two declares inherit the sin/cos caveat in full and not the sqrt one: IEEE-754 requires nothing of atan2f or powf either, so they are the third and fourth places in the prelude where native and wasm32 may differ in the last bit. Every case in math2.flan is therefore a value that is exact in binary -- a quadrant boundary, a power of two, a perfect square -- rather than one that would pin a particular libm and then fail on wasi. clamp is the interesting one. The prelude already argued against wrapping (min hi (max lo x)) in a function, and that argument gets stronger rather than weaker: min and max are builtins at every numeric type and there are no generics, so a clamp *function* is one copy per type. A macro is type-agnostic for free and emits nothing at all. The test calls the same three words at i32, i64, u8 and f32 to show it, and counts evaluations to show that each argument appears once -- the shape that names x twice reads identically and calls it twice. lo above hi answers hi and is not checked. A macro has no error facility, so the only diagnostic available would be a run-time one, in the construct whose whole point is that it costs nothing at run time. --- lib/prelude.ml | 53 +++++++++++++++++++++++++-- test/programs/math2.flan | 78 ++++++++++++++++++++++++++++++++++++++++ test/test_acceptance.ml | 21 +++++++++++ 3 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 test/programs/math2.flan diff --git a/lib/prelude.ml b/lib/prelude.ml index b88f4cd..059d186 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -225,9 +225,9 @@ let source = {flan| ;; ── Numbers ─────────────────────────────────────────────────────────── ;; -;; Only the ones that encode a decision. clamp is (min hi (max lo x)) over two -;; builtins and abs is (max x (- 0 x)); a wrapper over those is a function -;; emitted into every program to save a caller nothing. The one honest caveat +;; Only the ones that encode a decision. abs is (max x (- 0 x)); a wrapper over +;; that is a function emitted into every program to save a caller nothing. The +;; one honest caveat ;; on that abs: at the least representable integer it answers itself, because ;; the negation wraps. That is what every two's-complement abs does, a ;; function here would do it too, and the only fix is not to hand it that @@ -242,6 +242,28 @@ let source = {flan| ;; NaN (every comparison fails, so the same max picks the subtracted side, ;; which is still a NaN). There is nothing left for a function to fix. +;; clamp is a macro and not a function, and the reason is the objection above +;; turned around rather than dropped. min and max are builtins that work at +;; every numeric type; a clamp *function* cannot, because there are no +;; generics, so it would be one copy per type — a clamp-i32, a clamp-f32, a +;; clamp-i64 — each emitted into every program to save a caller eleven +;; characters. A macro is type-agnostic for free and emits nothing at all: what +;; the program contains after expansion is the (min hi (max lo x)) the caller +;; would have written. +;; +;; Each of x, lo and hi appears exactly once in the expansion, so nothing here +;; is evaluated twice and an argument with a side effect behaves as it reads. +;; +;; lo above hi is not checked, and the answer there is hi — the outer min wins. +;; That is the same rule Odin's clamp follows and there is nowhere better to +;; put a complaint: a macro has no error facility (see `unless` at the foot of +;; this file), so a diagnostic would have to be a run-time one, in the one +;; construct whose whole point is that it costs nothing at run time. +(defmacro clamp [args] + (if (!= (len args) 3) + `(clamp-takes-a-value-a-low-and-a-high) + `(min ~(at args 2) (max ~(at args 1) ~(at args 0))))) + ;; Zero for zero, and zero for NaN — neither is positive nor negative, so ;; neither comparison fires. A caller that needs to know which it got should ;; be testing for NaN, not reading a sign. @@ -375,6 +397,31 @@ let source = {flan| (declare sin-f32 [x f32] f32 "sinf") (declare cos-f32 [x f32] f32 "cosf") +;; atan2 and pow inherit the paragraph above in full, and not the sqrt one. +;; IEEE-754 requires nothing of atan2f or powf either, so these are the third +;; and fourth places in this file where native and wasm32 may disagree in the +;; last bit, and the sand-grid rule stands unchanged: a hash compared across +;; targets must not be routed through any of the four. +;; +;; They are here for the reason the trig pair is. Without them a program that +;; wants a heading or a falloff curve writes the identical two declare lines at +;; the top of its own file, which is the same libm call with the same caveat +;; and nobody's name on it. +;; +;; atan2's y comes first, as it does in C, and the order is the answer rather +;; than a convention: knowing the quadrant of (y, x) is the whole of what it +;; has over (atan (/ y x)), and it is recovered from the two signs. It is +;; defined at x = 0, where the division is not. +;; +;; One caveat of pow-f32's own, because it is the one that gets reported as a +;; bug: it is not exact at integer exponents. powf goes through a logarithm, +;; so (pow-f32 10.0 2.0) is 100.0 or the float next to it depending on the +;; libm, and an index computed by casting that to i32 is off by one on the +;; wrong side. A small integer power is a multiplication, and should be +;; written as one. +(declare atan2-f32 [y f32 x f32] f32 "atan2f") +(declare pow-f32 [x f32 y f32] f32 "powf") + ;; ── Byte classes ────────────────────────────────────────────────────── ;; ;; ASCII only, and deliberately: a byte is a byte here, there is no code point diff --git a/test/programs/math2.flan b/test/programs/math2.flan new file mode 100644 index 0000000..145cbb0 --- /dev/null +++ b/test/programs/math2.flan @@ -0,0 +1,78 @@ +;;;; atan2, pow, and clamp — the three gaps NEXT.md's second-tier list names +;;;; under "maths gaps". +;;;; +;;;; Two of them are declares over libm and the third is a macro, and that +;;;; split is the whole content of this file. atan2f and powf are not correctly +;;;; rounded under IEEE-754, exactly as sinf and cosf are not, so every case +;;;; below is a value whose answer is exact in binary — a quadrant boundary, a +;;;; power of two, a perfect square — rather than one that would pin a +;;;; particular libm's last bit and then fail on wasi-libc. +;;;; +;;;; clamp is a macro because a function could not be: min and max are builtins +;;;; at every numeric type and there are no generics, so a clamp function is +;;;; one copy per type. The proof that the macro is not one copy per type is +;;;; that the same three-word call below is made at i32, i64, u8 and f32. + +(defn show [x f32] + (print x) + (print " ")) + +;;; clamp must evaluate each of its three arguments exactly once. A macro that +;;; repeated one — say (if (< x lo) lo (if (> x hi) hi x)), which names x twice +;;; — would read identically and call this twice. +(defvar calls i32 0) + +(defn tick [x i32] i32 + (set calls (+ calls 1)) + x) + +(defn main [] i32 + ;; atan2 across all four quadrants and on both axes, which is the whole + ;; reason for it over a division: the quadrant is recovered from two signs, + ;; and (/ y x) has thrown it away before atan sees it. The two on the x = 0 + ;; axis are where the division does not exist at all. + (show (atan2-f32 1.0 1.0)) ; pi/4 + (show (atan2-f32 1.0 -1.0)) ; 3pi/4 + (show (atan2-f32 -1.0 -1.0)) ; -3pi/4 + (show (atan2-f32 -1.0 1.0)) ; -pi/4 + (show (atan2-f32 0.0 1.0)) ; 0 + (show (atan2-f32 1.0 0.0)) ; pi/2 + (show (atan2-f32 -1.0 0.0)) ; -pi/2 + (println "") + + ;; pow. A power of two, a half-power that is a perfect square, a negative + ;; exponent, and the two edge exponents every implementation special-cases. + (show (pow-f32 2.0 10.0)) ; 1024 + (show (pow-f32 9.0 0.5)) ; 3 + (show (pow-f32 2.0 -2.0)) ; 0.25 + (show (pow-f32 5.0 0.0)) ; 1 + (show (pow-f32 5.0 1.0)) ; 5 + (println "") + + ;; clamp: below the range, above it, and inside it untouched. + (print (clamp 0 1 3)) (print " ") ; 1 + (print (clamp 5 1 3)) (print " ") ; 3 + (print (clamp 2 1 3)) (print " ") ; 2 + ;; Both ends are inclusive, which is the off-by-one a hand-written clamp + ;; gets wrong with a < where it wanted <=. + (print (clamp 1 1 3)) (print " ") ; 1 + (print (clamp 3 1 3)) ; 3 + (println "") + + ;; The same three words at three more types, none of which a function could + ;; have served without a copy of its own. + (print (clamp (i64 900) (i64 0) (i64 255))) (print " ") ; 255 + (print (clamp (u8 200) (u8 0) (u8 255))) (print " ") ; 200 + (show (clamp 2.5 0.0 1.0)) ; 1 + (println "") + + ;; lo above hi is not an error and answers hi: the outer min wins. Written + ;; down here rather than left for someone to discover. + (print (clamp 7 5 1)) + (println "") + + ;; Three arguments, three evaluations. + (print (clamp (tick 5) (tick 1) (tick 3))) (print " ") + (print calls) + (println "") + 0) diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index 37db25f..098c3df 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -205,6 +205,27 @@ let () = in outputs "rounding and sqrt" "programs/math.flan" math_out; outputs ~opt:"-O0" "rounding and sqrt, -O0" "programs/math.flan" math_out; + (* atan2, pow and clamp. Every float here is exact in binary — quadrant + boundaries, powers of two, a perfect square — because atan2f and powf + are no more correctly rounded than sinf is, and a case pinning one + libm's last bit would pass native and fail wasi. The -O0 run is the one + that proves the two symbols resolve: at -O2 LLVM constant-folds a powf + of two literals and nothing is left to link, which is the same trap the + sqrt note describes. clamp is a macro, so the interesting lines are the + four types it is called at (a function would be four copies) and the + call counter, which is 3 and would be 4 or 5 for a macro that named an + argument twice. *) + let math2_out = + "0.785398 2.35619 -2.35619 -0.785398 0 1.5708 -1.5708 \n\ + 1024 3 0.25 1 5 \n\ + 1 3 2 1 3\n\ + 255 200 1 \n\ + 1\n\ + 3 3\n" + in + outputs "atan2, pow and clamp" "programs/math2.flan" math2_out; + outputs ~opt:"-O0" "atan2, pow and clamp, -O0" "programs/math2.flan" + math2_out; (* index-of-bytes, trim, the byte classes and parse-f64. The search cases are the ones that separate a correct loop from a lucky one: a match only at the end, "aab" in "aaab" (where the first byte matches twice From 02850d728221f8a4220266a00376a9992942b301 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 21:51:42 +0700 Subject: [PATCH 2/6] The prelude returns new bytes now: a builder, join, split and the case pair Eight functions that the file used to refuse by name, and the refusal was always one sentence -- there is no allocator -- which stopped being true when Vec landed. Three rules hold across all of them and are written at the head of the section: the result is owned and the caller frees it, the allocator is the context's, and no signature carries a Result because no allocating operation returns an error. The builder is not a type. Odin's strings.Builder wraps a [dynamic]u8; here the (Vec u8) already is that and already has push, so a wrapper would be a move-only struct whose only method is the one it wraps. What was missing is appending a run of bytes, and append! is that -- taking a (Ptr (Vec u8)), because a Vec parameter moves and a by-value builder would be consumed by its first append. append-i64! and append-f64! are the argument for the whole shape. The runtime renders numbers into one shared static buffer, so two of its results cannot be held at once; these copy out before returning, so a builder holds as many numbers as it likes. strings.flan puts two integers and a float on one line to show it. split returns a (Vec [u8]) and not a (Vec (Vec u8)): the fields borrow the input, and the owning shape is refused outright because a Vec copies and releases its elements bytewise. Constructing it needed a one-line slices-new, because (vec-new) takes its element type as a bare symbol and [u8] is not one -- a compiler gap, noted rather than worked around in silence. replace-bytes guards its empty needle with an if and not an early return: a returned Vec is a move, the dead set spans the function, and a return on one branch would kill the binding on the other. --- lib/prelude.ml | 160 +++++++++++++++++++++++++++++++++++++ test/programs/strings.flan | 152 +++++++++++++++++++++++++++++++++++ test/test_acceptance.ml | 26 ++++++ 3 files changed, 338 insertions(+) create mode 100644 test/programs/strings.flan diff --git a/lib/prelude.ml b/lib/prelude.ml index 059d186..9de2d1e 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -791,6 +791,166 @@ let source = {flan| (return false))) true))) +;; ── Building bytes, which is the tier that needed an allocator ──────── +;; +;; Everything above this line is slice-based and allocation-free, because when +;; it was written there was nothing to allocate from. Everything below it +;; *returns new storage*, which is the whole difference, and there are three +;; rules that hold for all of it. +;; +;; **The result is owned and the caller frees it.** Each of these hands back a +;; (Vec u8) or a (Vec [u8]), which is move-only: it goes with the call that +;; takes it, and nothing is released at scope exit — not at the end of a let, +;; not at the end of a function (spec-memory.md, "When storage is released"). +;; A caller writes (free v) or lets a (free-all a) take the whole region. +;; +;; **The allocator is the context's, and `with-allocator` is the override.** +;; spec-memory.md makes allocation use the current implicit allocator and +;; forbids falling back to a hidden global one. (vec-new) and (map-new) take an +;; optional trailing allocator because the checker builds them; a Flan defn has +;; fixed arity and cannot, so the choice here was an allocator parameter on +;; every one of these signatures or none. None: a caller wanting a frame arena +;; writes (with-allocator a (join parts sep)) and the Vec records the arena, so +;; the free and the clone never need it named again. +;; +;; **No Result, anywhere.** Running out of storage signals StorageExhausted +;; under a `retry` restart and no allocating operation returns an error +;; (spec-memory.md, "Allocation failure"), so these signatures say what they +;; produce and nothing about how they might fail. + +;; The builder. It is not a type: strings.Builder in Odin is a struct wrapping +;; a [dynamic]u8, and here the (Vec u8) *is* that, with push already on it — a +;; wrapper would be a move-only struct owning a Vec whose only method is the +;; one the Vec already has. What was actually missing is appending a run of +;; bytes rather than one, and that is this. +;; +;; It takes a (Ptr (Vec u8)) and not a (Vec u8), and the difference is not +;; style: a Vec parameter *moves*, so (append! b s) taking one by value would +;; consume the caller's builder on the first call and refuse the second. +(defn append! [b (Ptr (Vec u8)) s [u8]] + (dotimes [i (len s)] + (push (deref b) (at s i)))) + +;; The two number appends, and they are the reason this shape is worth having +;; rather than a formatter that answers a slice. i64->bytes and f64->bytes +;; render into one shared static buffer in the runtime, so two of their results +;; cannot be held at once and (concat [(i64->bytes a) (i64->bytes b)]) is two +;; views of the same bytes — the second call overwrote the first. These copy +;; out of that buffer before returning, so the hazard ends at the call: a +;; builder can hold as many numbers as it likes. +(defn append-i64! [b (Ptr (Vec u8)) n i64] + (append! b (i64->bytes n))) + +(defn append-f64! [b (Ptr (Vec u8)) x f64] + (append! b (f64->bytes x))) + +;; concat and join. Both take a slice of slices, which is the shape a caller +;; already has: an array literal of them, [(bytes "a") (bytes b)], slices to a +;; [[u8]] and copies nothing. +;; +;; join with an empty separator is concat, and concat is here anyway because +;; the empty (bytes "") a caller would have to write is the kind of argument +;; that reads like a mistake at the call site. +(defn concat [parts [[u8]]] (Vec u8) + (let [b (vec-new u8)] + (dotimes [i (len parts)] + (append! (addr b) (at parts i))) + b)) + +;; n parts yield n-1 separators, and the empty slice of parts yields the empty +;; result rather than a leading separator — which is the off-by-one a join +;; written as "append part then separator, then chop the tail" gets wrong on +;; exactly that input, because there is no tail to chop. +(defn join [parts [[u8]] sep [u8]] (Vec u8) + (let [b (vec-new u8)] + (dotimes [i (len parts)] + (when (> i 0) + (append! (addr b) sep)) + (append! (addr b) (at parts i))) + b)) + +(defn repeat-bytes [s [u8] n i32] (Vec u8) + (let [b (vec-new u8)] + (dotimes [i n] + (append! (addr b) s)) + b)) + +;; The allocating halves of the ASCII case pair. The note above lower-ascii +;; explains why lowering a [u8] *in place* is a trap — a string literal is +;; emitted into .rodata, so the store either segfaults at -O0 or is deleted at +;; -O2 — and this is the shape that has no such hole: the bytes it writes are +;; its own. +(defn to-lower [s [u8]] (Vec u8) + (let [b (vec-new u8)] + (dotimes [i (len s)] + (push b (lower-ascii (at s i)))) + b)) + +(defn to-upper [s [u8]] (Vec u8) + (let [b (vec-new u8)] + (dotimes [i (len s)] + (push b (upper-ascii (at s i)))) + b)) + +;; Every non-overlapping occurrence, left to right, which is the rule that +;; makes (replace-bytes (bytes "aaa") (bytes "aa") (bytes "b")) answer "ba" and +;; not "bb" or "b". +;; +;; An empty `from` matches nothing and the result is a copy of the input. The +;; alternative reading — that it matches at every position — is what turns this +;; into an infinite loop, and Odin's replace guards the same case for the same +;; reason. +;; +;; The guard is an `if` and not an early `(return b)`, which is not a style +;; choice: returning a Vec *moves* it, and the move analysis is a dead set over +;; the whole function, so a `return b` on one branch kills the binding for the +;; `b` at the foot of the other. One exit, one move. +(defn replace-bytes [s [u8] from [u8] to [u8]] (Vec u8) + (let [b (vec-new u8) + i 0] + (if (= (len from) 0) + (append! (addr b) s) + (while (< i (len s)) + (match (index-of-bytes (slice s i (len s)) from) + (Some k) + (do + (append! (addr b) (slice s i (+ i k))) + (append! (addr b) to) + (set i (+ i k (len from)))) + None + (do + (append! (addr b) (slice s i (len s))) + (set i (len s)))))) + b)) + +;; A (Vec [u8]) cannot be written at a let, and this one-line function is where +;; the type is said instead. (vec-new) takes its element type as a *bare +;; symbol* — check.ml's vec_new_elem resolves one name and nothing else — so +;; (vec-new [u8]) is not accepted, and a let has no type annotation to say it +;; the other way. A return type does say it. That is a compiler gap rather than +;; a language decision, and it is written down in NEXT.md. +(defn slices-new [] (Vec [u8]) (vec-new)) + +;; split, which the file used to refuse by name. The fields are slices *of the +;; input* and not copies, so nothing here owns bytes and the result dies with +;; whatever `s` pointed at — a (Vec (Vec u8)) is the shape that would own them +;; and it is refused outright, because a Vec's elements are copied and released +;; bytewise and an owner cannot survive that. +;; +;; The rule is split-on-byte's, unchanged and worth restating: n separators +;; always yield n+1 fields, so the empty input yields one empty field and a +;; trailing separator yields a trailing empty one. That is Odin's allocating +;; strings.split and not Odin's iterator, which disagree with each other. +(defn split [s [u8] sep u8] (Vec [u8]) + (let [v (slices-new) + it (split-on-byte s sep) + going true] + (while going + (match (split-next! (addr it)) + (Some f) (push v f) + None (set going false))) + v)) + ;; ── Refused, by name ────────────────────────────────────────────────── ;; ;; Every one of these needs to produce bytes that did not exist in its input, diff --git a/test/programs/strings.flan b/test/programs/strings.flan new file mode 100644 index 0000000..d1e441a --- /dev/null +++ b/test/programs/strings.flan @@ -0,0 +1,152 @@ +;;;; The prelude's second tier: the functions that return new storage. +;;;; +;;;; Every one of these was refused by name in prelude.ml until there was an +;;;; allocator to return a Vec from, and this file is the corpus that says the +;;;; refusals are lifted. The cases are chosen the way the slice-algorithm +;;;; tests were: each is an input a plausible wrong version gets wrong. +;;;; +;;;; Everything allocated here is freed, even though leaking is defined +;;;; behaviour (spec-memory.md), because this file is the example people copy. + +;;; A (Vec u8) printed as text, without the caller writing the two-step every +;;; time. as-slice borrows -- it copies ptr+len and never the elements -- so v +;;; is still the owner afterwards and is still free-able. +(defn show [v (Ptr (Vec u8))] + (println (string (as-slice (deref v))))) + +(defn main [] i32 + ;; The builder. Three appends and two numbers into one Vec, which is the + ;; case the shared static scratch buffer in the runtime makes impossible for + ;; i64->bytes on its own: two of its results cannot be held at once, and + ;; these two numbers are both in the answer. + (let [b (vec-new u8)] + (append! (addr b) (bytes "x=")) + (append-i64! (addr b) 42) + (append! (addr b) (bytes " y=")) + (append-i64! (addr b) -7) + (append! (addr b) (bytes " r=")) + (append-f64! (addr b) 1.5) + (show (addr b)) ; x=42 y=-7 r=1.5 + (free b)) + + ;; concat over three parts, and over none -- the empty result rather than a + ;; trap. + (let [parts [(bytes "one") (bytes "") (bytes "two")]] + (let [c (concat (slice parts 0 3))] + (show (addr c)) ; onetwo + (free c))) + (let [parts [(bytes "unused")]] + (let [c (concat (slice parts 0 0))] + (println (len c)) ; 0 + (free c))) + + ;; join: n parts, n-1 separators. The one-part case is the one that must not + ;; emit a separator at all, and the zero-part case is the one a "append then + ;; chop the tail" join gets wrong because there is no tail. + (let [parts [(bytes "a") (bytes "b") (bytes "c")]] + (let [j (join (slice parts 0 3) (bytes ", "))] + (show (addr j)) ; a, b, c + (free j)) + (let [j (join (slice parts 0 1) (bytes ", "))] + (show (addr j)) ; a + (free j)) + (let [j (join (slice parts 0 0) (bytes ", "))] + (println (len j)) ; 0 + (free j)) + ;; An empty separator is concat. + (let [j (join (slice parts 0 3) (bytes ""))] + (show (addr j)) ; abc + (free j))) + + ;; repeat, including zero times. + (let [r (repeat-bytes (bytes "ab") 3)] + (show (addr r)) ; ababab + (free r)) + (let [r (repeat-bytes (bytes "ab") 0)] + (println (len r)) ; 0 + (free r)) + + ;; The allocating case pair. The input is a string literal, which lives in + ;; .rodata -- an in-place lower would either segfault at -O0 or be deleted at + ;; -O2, and that is exactly why these exist. Digits and punctuation pass + ;; through untouched, which is the range check a table-free version gets + ;; wrong by shifting every byte. + (let [l (to-lower (bytes "Hello, World 42!"))] + (show (addr l)) ; hello, world 42! + (free l)) + (let [u (to-upper (bytes "Hello, World 42!"))] + (show (addr u)) ; HELLO, WORLD 42! + (free u)) + + ;; replace. "aaa" with "aa" -> "b" is the non-overlapping rule: the answer is + ;; "ba", because the match consumes both a's and the scan resumes after them. + (let [r (replace-bytes (bytes "aaa") (bytes "aa") (bytes "b"))] + (show (addr r)) ; ba + (free r)) + ;; A replacement longer than what it replaces, and one that is empty. + (let [r (replace-bytes (bytes "a,b,c") (bytes ",") (bytes " -- "))] + (show (addr r)) ; a -- b -- c + (free r)) + (let [r (replace-bytes (bytes "a,b,c") (bytes ",") (bytes ""))] + (show (addr r)) ; abc + (free r)) + ;; No occurrence is a copy, and an empty `from` is a copy -- the reading + ;; where it matches everywhere is an infinite loop. + (let [r (replace-bytes (bytes "abc") (bytes "z") (bytes "!"))] + (show (addr r)) ; abc + (free r)) + (let [r (replace-bytes (bytes "abc") (bytes "") (bytes "!"))] + (show (addr r)) ; abc + (free r)) + + ;; split. n separators, n+1 fields, always -- so the trailing empty field is + ;; present, which is where Odin's own iterator and its allocating split + ;; disagree with each other. + (let [f (split (bytes "a,b,c") \,)] + (println (len f)) ; 3 + (println (string (at f 0))) ; a + (println (string (at f 2))) ; c + (free f)) + (let [f (split (bytes "a,b,") \,)] + (println (len f)) ; 3 + (println (len (at f 2))) ; 0 + (free f)) + (let [f (split (bytes ",a") \,)] + (println (len f)) ; 2 + (println (len (at f 0))) ; 0 + (free f)) + ;; No separator at all is one field, and the empty input is one empty field. + (let [f (split (bytes "abc") \,)] + (println (len f)) ; 1 + (println (string (at f 0))) ; abc + (free f)) + (let [f (split (bytes "") \,)] + (println (len f)) ; 1 + (println (len (at f 0))) ; 0 + (free f)) + + ;; The fields are slices of the input and nothing was copied: this one + ;; round-trips through join, and the separator it rebuilds with is a + ;; different one, so an implementation that handed back the original slice + ;; would print the original string. + (let [f (split (bytes "a,b,c") \,)] + (let [j (join (as-slice f) (bytes "/"))] + (show (addr j)) ; a/b/c + (free j)) + (free f)) + + ;; The allocator is the context's, so with-allocator moves the whole tier + ;; into an arena -- which is the answer to the fixed arity of a defn, and the + ;; reason none of these takes an allocator argument. free-all releases every + ;; one of them at once, including the Vec still bound below it. + (let [a (arena-new 4096)] + (with-allocator a + (let [parts [(bytes "in") (bytes "arena")]] + (let [j (join (slice parts 0 2) (bytes "-"))] + (show (addr j)) ; in-arena + ;; Not freed: an arena cannot release one block, and free-all is + ;; what releases this. + (free j)))) + (free-all a) + (arena-destroy a)) + 0) diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index 098c3df..cf62871 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -452,6 +452,32 @@ let () = outputs "vec" "programs/vec.flan" vec_out; outputs ~opt:"-O0" "vec, -O0" "programs/vec.flan" vec_out; outputs ~dev:true "vec, dev" "programs/vec.flan" vec_out; + (* The prelude's second tier: the functions that return new storage, every + one of which the prelude used to refuse by name for want of an + allocator. The cases are the ones that separate a correct version from a + lucky one — join over zero and one parts, where the separator count is + -1 and 0; "aaa" with "aa" -> "b", which is "ba" only if the match is + non-overlapping; an empty `from` to replace, which is an infinite loop + under the other reading; and split's trailing and leading empty fields, + which is where Odin's iterator and Odin's own allocating split disagree. + The builder line holds two rendered integers and a float at once, which + is precisely what the runtime's shared static scratch makes impossible + for i64->bytes on its own. + + The dev run earns its place here more than most: it is the one that + checks a container's recorded allocator epoch, so it is what would catch + one of these Vecs being used after the arena under it was released. *) + let strings_out = + "x=42 y=-7 r=1.5\nonetwo\n0\na, b, c\na\n0\nabc\nababab\n0\n\ + hello, world 42!\nHELLO, WORLD 42!\n\ + ba\na -- b -- c\nabc\nabc\nabc\n\ + 3\na\nc\n3\n0\n2\n0\n1\nabc\n1\n0\na/b/c\nin-arena\n" + in + outputs "string building" "programs/strings.flan" strings_out; + outputs ~opt:"-O0" "string building, -O0" "programs/strings.flan" + strings_out; + outputs ~dev:true "string building, dev" "programs/strings.flan" + strings_out; (* A debug build, because [dty] is a separate path from everything above: [outputs ~dev:true] goes through the cells, not through DWARF, and a type with no arm there dies at emit rather than being merely undebugged. From b5e7351c7e0fe1c98b154fa2e6c6ae64129470ce Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 21:55:17 +0700 Subject: [PATCH 3/6] A number with a precision, which %g cannot be asked for f64->bytes is snprintf "%g": six significant digits, exponent notation of its own accord, and no precision to pass it. A frame time of 1/60 comes back as 0.0166667 and a score past a million as 1.23457e+06. format-f64 returns a Vec instead, so it inherits neither that nor the shared static scratch buffer -- and it is the reason append-i64! exists, because it renders the integer part and the fraction through that one buffer in strict sequence. Half away from zero at the last digit kept, which is round-f32's rule and not printf's. 0.125 at two places is 0.13 here and 0.12 there; matching printf would mean pinning a particular libc's nearest-even on the binary value, and that answer is not the same on every target anyway. The three cases that ship broken are each one line and each tested: the carry, where the rounded fraction equals the scale and is the next integer (0.999995 at five places prints "0.100000" without it); the zero padding, without which 1.005 at three places prints "1.5"; and the sign, which belongs to the number rather than to its integer part, since -0.5 has an integer part of 0 and 0 carries no sign. The clamp on the precision is spelled (min 9 (max 0 prec)) and not with the clamp macro, and the reason is a finding: the prelude is never macro-expanded. macro.ml's pass runs over the file being compiled, and the prelude arrives at the checker through Check.program's own prepend, so a prelude function calling a prelude macro resolves the macro's underlying defn -- the one that takes a [Form] -- and reports an arity error. --- lib/prelude.ml | 101 ++++++++++++++++++++++++++++++++++++-- test/programs/format.flan | 86 ++++++++++++++++++++++++++++++++ test/test_acceptance.ml | 30 +++++++++++ 3 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 test/programs/format.flan diff --git a/lib/prelude.ml b/lib/prelude.ml index 9de2d1e..5e84334 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -227,9 +227,8 @@ let source = {flan| ;; ;; Only the ones that encode a decision. abs is (max x (- 0 x)); a wrapper over ;; that is a function emitted into every program to save a caller nothing. The -;; one honest caveat -;; on that abs: at the least representable integer it answers itself, because -;; the negation wraps. That is what every two's-complement abs does, a +;; one honest caveat on that abs: at the least representable integer it +;; answers itself, because the negation wraps. That is what every two's-complement abs does, a ;; function here would do it too, and the only fix is not to hand it that ;; value — so it is written down rather than wrapped. ;; @@ -951,6 +950,102 @@ let source = {flan| None (set going false))) v)) +;; ── A number with a precision ───────────────────────────────────────── +;; +;; The one formatting job the runtime cannot do. f64->bytes is snprintf "%g", +;; which is six significant digits and switches to exponent notation on its +;; own: a frame time of 0.0166667 is what a caller wanted two decimals of, and +;; 1.23457e+06 is what a score looks like once it passes a million. There is no +;; precision to pass it, and there cannot be — it renders into one shared +;; static buffer in the runtime, which is the same reason two of its results +;; cannot be held at once. +;; +;; This returns a Vec, so neither problem is inherited. It uses i64->bytes +;; twice and the two calls are strictly sequential — the integer part is copied +;; into the Vec before the fraction is rendered — which is the discipline the +;; shared buffer requires and the one append-i64! exists to make automatic. +;; +;; Half away from zero, the same rule round-f32 follows, applied at the last +;; digit kept. That is not bit-for-bit printf: printf rounds the *binary* value +;; to nearest-even at the decimal digit, and this rounds the decimal expansion +;; half-up, so a value sitting exactly on a half — 0.999995 at five places — +;; comes out 1.00000 here and may come out 0.99999 there. Choosing the rule the +;; rest of this file already uses beats matching a libc whose answer is not the +;; same on every target anyway. +;; +;; Precision is clamped to 0..9 rather than refused. 10^9 is the largest power +;; of ten that leaves room in the f64 product below, and a precision argument +;; is almost always a literal, so a refusal would be a run-time condition for a +;; mistake visible in the source. +;; +;; The clamp is written out as (min 9 (max 0 prec)) and not as the `clamp` +;; macro two hundred lines up, and that is a limit rather than a preference: +;; **the prelude is not macro-expanded**. macro.ml's pass runs over the file +;; being compiled, and the prelude reaches the checker through Check.program's +;; own prepend, having never been through the expander — so a prelude function +;; calling a prelude macro resolves the macro's underlying defn, which takes +;; one [Form] argument, and the report is an arity error at the call. It is +;; written down in NEXT.md beside the other macro gaps. +;; +;; Three inputs do not have decimal expansions and are named before the cast +;; that would be undefined on them: NaN, which fails every comparison and is +;; therefore tested with (not (= x x)) and nothing else, and the two +;; infinities, which are the values satisfying (= x (* x 2.0)) away from zero. +;; A magnitude past 9e18 has no fractional bits left at all and would not fit +;; in the i64 the integer part is carried in, so it falls back to f64->bytes — +;; which is the honest answer there rather than an approximation of one. +;; +;; -0.0 prints as "0.00": the sign test is (< x 0.0), which -0.0 fails. A +;; caller that needs the sign of a zero should not be reading it out of text. +(defn format-f64 [x f64 prec i32] (Vec u8) + (let [b (vec-new u8) + p (min 9 (max 0 prec))] + (cond + (not (= x x)) + (append! (addr b) (bytes "nan")) + + (and (= x (* x 2.0)) (!= x 0.0)) + (append! (addr b) (bytes (if (< x 0.0) "-inf" "inf"))) + + :else + (let [neg (< x 0.0) + m (if neg (- 0.0 x) x)] + (if (>= m 9.0e18) + (append! (addr b) (f64->bytes x)) + (let [scale (i64 1)] + (dotimes [i p] + (set scale (* scale 10))) + ;; The split is exact: (i64 m) truncates toward zero and m is + ;; non-negative here, and the subtraction of an integer from the + ;; float it came from is exact at every magnitude an f64 can hold. + ;; Only the scaling below rounds, and it rounds a value already + ;; under 1. + (let [ip (i64 m) + fr (i64 (+ (* (- m (f64 ip)) (f64 scale)) 0.5))] + ;; The carry, which is the bug this shape is otherwise written + ;; with: 0.999995 at five places scales to exactly 100000, which + ;; is not a fraction at all — it is the next integer, and without + ;; this line it prints as "0.100000". + (when (>= fr scale) + (set fr 0) + (set ip (+ ip 1))) + ;; The sign goes on separately, because the integer part is a + ;; magnitude: -0.5 at one place has an integer part of 0, and + ;; i64->bytes of 0 has no sign to carry. + (when neg + (push b \-)) + (append-i64! (addr b) ip) + (when (> p 0) + (push b \.) + ;; Left-padded with zeros to exactly p digits. fr is under + ;; scale by the carry above, so it never needs more, and + ;; without the padding 1.005 at three places prints "1.5". + (let [d (i64->bytes fr)] + (dotimes [i (- p (len d))] + (push b \0)) + (append! (addr b) d)))))))) + b)) + ;; ── Refused, by name ────────────────────────────────────────────────── ;; ;; Every one of these needs to produce bytes that did not exist in its input, diff --git a/test/programs/format.flan b/test/programs/format.flan new file mode 100644 index 0000000..d6931b0 --- /dev/null +++ b/test/programs/format.flan @@ -0,0 +1,86 @@ +;;;; format-f64: a number rendered to a fixed number of decimal places. +;;;; +;;;; The runtime's f64->bytes is snprintf "%g" and there is no precision to +;;;; pass it, so this is the first number formatter in the language that a +;;;; caller can steer. Every case below is one a plausible wrong version gets +;;;; wrong, and three of them are the ones that actually ship broken: the +;;;; carry, where the rounded fraction equals the scale and is not a fraction +;;;; at all; the zero padding, without which 1.005 prints as "1.5"; and the +;;;; sign, which belongs to the number and not to its integer part, because +;;;; -0.5 has an integer part of 0 and 0 carries no sign. + +(defn show [x f64 p i32] + (let [v (format-f64 x p)] + (println (string (as-slice v))) + (free v))) + +(defn main [] i32 + ;; The ordinary cases, and the one %g cannot do at all: 1/60 wanted to two + ;; places is a frame time, and "%g" answers 0.0166667. + (show 3.14159 2) ; 3.14 + (show 0.0166667 2) ; 0.02 + (show 1234.5 1) ; 1234.5 + (show 2.0 0) ; 2 + (show 2.0 3) ; 2.000 + + ;; Rounding is half away from zero at the last digit kept, on both signs. + (show 0.125 2) ; 0.13 + (show -0.125 2) ; -0.13 + (show 2.5 0) ; 3 + (show -2.5 0) ; -3 + + ;; The carry. 0.999995 scaled by 10^5 rounds to exactly 100000, which is the + ;; next integer; without the carry this prints "0.100000". + (show 0.999995 5) ; 1.00000 + (show 9.99 1) ; 10.0 + (show -9.99 1) ; -10.0 + (show 0.99 0) ; 1 + + ;; Zero padding. The fraction of 1.005 at three places is 5, and five digits + ;; is not the same number as 005. + (show 1.005 3) ; 1.005 + (show 1.0001 4) ; 1.0001 + (show 7.0 6) ; 7.000000 + + ;; The sign lives on the number, not on the integer part: both of these have + ;; an integer part of 0, which i64->bytes renders without a sign. + (show -0.5 2) ; -0.50 + (show -0.004 2) ; -0.00 + + ;; -0.0 prints as a plain zero. The sign test is (< x 0.0), which -0.0 fails, + ;; and text is not where the sign of a zero should be read from. + (show 0.0 2) ; 0.00 + (show -0.0 2) ; 0.00 + + ;; Precision is clamped rather than refused, at both ends. + (show 1.5 -3) ; 2 + (show 1.5 40) ; 1.500000000 + + ;; The three inputs with no decimal expansion. + (show (/ 0.0 0.0) 2) ; nan + (show (/ 1.0 0.0) 2) ; inf + (show (/ -1.0 0.0) 2) ; -inf + + ;; Past 9e18 an f64 has no fractional bits and the integer part does not fit + ;; in an i64, so this falls back to %g rather than approximating. + (show 1e20 2) ; 1e+20 + + ;; A large magnitude that does fit, where the fraction is genuinely gone: an + ;; f64 has no bits below 1 up there, so the padding produces the zeros. + (show 1234567890123.0 2) ; 1234567890123.00 + + ;; And the thing it is for: a formatted number inside a built string, which + ;; needs the integer part copied out before the fraction is rendered, because + ;; both come through the runtime's one shared scratch buffer. + (let [b (vec-new u8)] + (append! (addr b) (bytes "fps ")) + (let [f (format-f64 59.94 1)] + (append! (addr b) (as-slice f)) + (free f)) + (append! (addr b) (bytes " / frame ")) + (let [f (format-f64 0.0166667 4)] + (append! (addr b) (as-slice f)) + (free f)) + (println (string (as-slice b))) ; fps 59.9 / frame 0.0167 + (free b)) + 0) diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index cf62871..10fdad1 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -478,6 +478,36 @@ let () = strings_out; outputs ~dev:true "string building, dev" "programs/strings.flan" strings_out; + (* format-f64, the first number formatter a caller can steer. The three + lines that would ship wrong are pinned deliberately: 0.999995 at five + places, where the rounded fraction equals the scale and is the next + integer rather than a fraction; 1.005 at three, where dropping the zero + padding prints "1.5"; and -0.5, where the sign belongs to the number and + the integer part it would otherwise ride on is 0, which i64->bytes + renders unsigned. + + 0.125 at two places answers 0.13 and printf's "%.2f" answers 0.12. That + is not a defect: this rounds the decimal expansion half away from zero, + which is round-f32's rule and the rest of this file's, where printf + rounds the binary value to nearest-even. Pinning 0.12 here would be + pinning a libc. + + The last line is the one the whole shape is for — two numbers in one + built string, which the runtime's single shared scratch buffer makes + impossible for a formatter that answers a slice. *) + let format_out = + "3.14\n0.02\n1234.5\n2\n2.000\n\ + 0.13\n-0.13\n3\n-3\n\ + 1.00000\n10.0\n-10.0\n1\n\ + 1.005\n1.0001\n7.000000\n\ + -0.50\n-0.00\n0.00\n0.00\n\ + 2\n1.500000000\n\ + nan\ninf\n-inf\n1e+20\n1234567890123.00\n\ + fps 59.9 / frame 0.0167\n" + in + outputs "a number with a precision" "programs/format.flan" format_out; + outputs ~opt:"-O0" "a number with a precision, -O0" "programs/format.flan" + format_out; (* A debug build, because [dty] is a separate path from everything above: [outputs ~dev:true] goes through the cells, not through DWARF, and a type with no arm there dies at emit rather than being merely undebugged. From 7ce6043c473b2c020bc2c8df3e862a5b59f47e21 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 21:58:12 +0700 Subject: [PATCH 4/6] A second and third sort, and why there is not a generic one sort-i32! was the only sort in the language. sort-f32! and sort-bytes! are the other two, and they are copies rather than an abstraction for a reason worth naming precisely: map, filter, reduce and a sort taking a comparator are not blocked on generics, they are blocked on *function values*. Types.Fn exists and check.ml refuses it with "a function type is not implemented yet -- milestone 5", and there is nothing else in the language to pass. Generics on top of that is what would make them one copy instead of one per element type. The f32 family carries one caveat the i32 family cannot have: a NaN makes the order undefined, because every comparison against one is false, so the insertion loop never moves it and never moves anything past it. sum-f32 accumulates in f64 for a stronger version of sum-i32's argument -- an f32 total does not wrap, it absorbs, and the answer comes out silently short. The test prints the difference rather than the total, because %g hides it. sort-bytes! is the one a caller of split actually wants, and its ordering is memcmp's: bytewise, unsigned, prefix first. Not alphabetical -- "Zebra" sorts before "apple" -- and the note says so, for the same reason the ASCII-case note refuses a locale. The slices move and the bytes never do, so it sorts fields borrowed out of a string literal, which an in-place byte sort could not. --- lib/prelude.ml | 138 ++++++++++++++++++++++++++++++++-- test/programs/algorithms.flan | 120 +++++++++++++++++++++++++++++ test/test_acceptance.ml | 24 ++++++ 3 files changed, 276 insertions(+), 6 deletions(-) create mode 100644 test/programs/algorithms.flan diff --git a/lib/prelude.ml b/lib/prelude.ml index 5e84334..48e6b51 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -90,15 +90,27 @@ let source = {flan| ;; ── Slice algorithms, all in place ──────────────────────────────────── ;; -;; Over [i32] and nothing else. There are no generics, so one of these per -;; element type is one *copy* per element type, emitted into every program; -;; i32 is the type indices, ids and tile values already have, and f32 copies -;; wait until a program actually wants them. +;; One family per element type, because there are no generics: each of these +;; is a *copy* per element type, and the set below is i32 (what indices, ids +;; and tile values are), f32 (what positions, velocities and weights are) and +;; [u8] (what a field coming out of `split` is). ;; ;; A slice is ptr+len and non-owning, so these mutate the storage they were ;; handed: sorting (slice grid 4 9) sorts those five elements of grid and -;; leaves the rest alone. That is the whole reason the shape is in-place — -;; there is no allocator to return a new sequence from. +;; leaves the rest alone. That was originally forced — there was no allocator +;; to return a new sequence from — and it stays the right shape now that there +;; is one, because sorting a thing you already own should not allocate. The +;; allocating tier is further down, and a caller sorts a Vec by sorting +;; (as-slice v). +;; +;; **map, filter, reduce and a sort taking a comparator are not here, and they +;; are not blocked on generics.** They are blocked on *function values*: each +;; of them takes a callable as an argument, Types.Fn exists but check.ml +;; refuses it with "a function type is not implemented yet — milestone 5", and +;; there is nothing else in the language to pass. Generics on top of that is +;; what would make them one copy instead of one per element type; without +;; either, the honest form is the concrete fold, which is what sum-i32 and +;; sum-f32 below already are — (reduce + 0) with the + written in. (defn swap-i32! [s [i32] i i32 j i32] (let [t (at s i)] @@ -164,6 +176,74 @@ let source = {flan| (set t (+ t (i64 (at s i))))) t)) +;; ── The same family over f32 ────────────────────────────────────────── +;; +;; sort-i32! was the only sort in the language, which is what NEXT.md's second +;; tier means by "a sort that is not integers-only". This is the second, and it +;; is a copy and not an abstraction — see the note above on why. +;; +;; One caveat that has no counterpart in the i32 family, because it cannot +;; arise there: **a NaN in the input makes the order undefined.** Every +;; comparison against a NaN is false, so the insertion loop never moves one and +;; never moves anything past one; what comes out is sorted within each run +;; between NaNs and not sorted across them. That is what C's qsort with a naive +;; comparator does too. The fix is not to have NaNs in the array — which is +;; also the only fix, since there is no ordering of the reals that a NaN sits +;; anywhere in. + +(defn swap-f32! [s [f32] i i32 j i32] + (let [t (at s i)] + (set (at s i) (at s j)) + (set (at s j) t))) + +(defn reverse-f32! [s [f32]] + (let [i 0 + j (- (len s) 1)] + (while (< i j) + (swap-f32! s i j) + (set i (+ i 1)) + (set j (- j 1))))) + +(defn sort-f32! [s [f32]] + (let [i 1] + (while (< i (len s)) + (let [j i] + (while (and (> j 0) (> (at s (- j 1)) (at s j))) + (swap-f32! s (- j 1) j) + (set j (- j 1)))) + (set i (+ i 1))))) + +;; None for an empty slice, exactly as min-i32 does. A NaN in the input is not +;; special-cased and propagates the same way it does through the builtins: the +;; comparison fails, so the running value simply does not change. +(defn min-f32 [s [f32]] (Option f32) + (if (= (len s) 0) + None + (let [m (at s 0)] + (dotimes [i (len s)] + (set m (min m (at s i)))) + (Some m)))) + +(defn max-f32 [s [f32]] (Option f32) + (if (= (len s) 0) + None + (let [m (at s 0)] + (dotimes [i (len s)] + (set m (max m (at s i)))) + (Some m)))) + +;; Accumulates in f64 and widens each element explicitly, which is sum-i32's +;; argument in its floating form and a stronger one: summing a screenful of f32 +;; in f32 does not wrap, it *absorbs* — once the running total is large enough, +;; adding a small element rounds to no change at all, and the answer is silently +;; short rather than obviously wrong. An f64 accumulator has 29 more bits of +;; mantissa and pushes that failure out of reach of any array a game holds. +(defn sum-f32 [s [f32]] f64 + (let [t 0.0] + (dotimes [i (len s)] + (set t (+ t (f64 (at s i))))) + t)) + ;; ── Bytes ───────────────────────────────────────────────────────────── ;; ;; Over [u8] and not over string, so (bytes s) is what a caller writes and one @@ -790,6 +870,52 @@ let source = {flan| (return false))) true))) +;; ── Ordering byte slices, and sorting them ──────────────────────────── +;; +;; The third element type the slice family covers, and the one a caller of +;; `split` actually has: a [[u8]] of fields, wanting to come out in order. +;; +;; The order is bytewise-lexicographic — memcmp's, and the one every sane +;; sorted format uses. It is explicitly *not* alphabetical and not a collation: +;; "Zebra" sorts before "apple" because 'Z' is 90 and 'a' is 97, and a +;; non-ASCII byte sorts by its UTF-8 encoding, which for code points happens to +;; agree with code-point order and for anything a human would call alphabetical +;; does not. A locale-aware comparison is not a byte operation at all, for the +;; same reasons the ASCII-case note above gives. +;; +;; The comparison is over u8 and therefore unsigned, which is the bug a version +;; written over a signed byte type has: 0x80 would compare *below* 0x00 and +;; every multi-byte character would sort before every ASCII one. +;; +;; A prefix sorts before what extends it — "ab" before "abc" — which falls out +;; of running to the shorter length and then comparing lengths, and is the case +;; a loop written to (len a) alone reads off the end for. +(defn bytes j 0) (bytes Date: Sat, 12 Sep 2026 22:01:30 +0700 Subject: [PATCH 5/6] The refusal block loses half its entries, and the four that stay say why The list at the foot of prelude.ml was one sentence -- every entry needed to produce bytes that did not exist in its input, and there was no allocator -- and that sentence has been false since Vec landed. Seven entries move up into the code, and string-from-bytes turns out to have been the `string` builtin all along: (string (as-slice v)) is the round trip, free precisely because the layouts are identical. What is left is refused for four different reasons and is written that way now: pad and center for nothing at all except that no caller has asked; format and sprintf for variadics of mixed type; map, filter, reduce and sort-by for function values; map-keys and map-values for a map iterator that does not exist in the runtime. NEXT.md's queued section is struck and carries the four findings, each with the change it wants named -- flan_map_next plus one builtin for the iterator; milestone 5's function values for the higher-order three; vec_new_elem taking a type expression rather than a bare name, which is what forces slices-new to exist; and an array literal with no way to say it is [f32], which is what forces every float in algorithms.flan to be cast. BUILT.md gets the section. --- BUILT.md | 127 +++++++++++++++++++++++++++++++++++++++++++++++++ NEXT.md | 81 ++++++++++++++++++++++++------- lib/prelude.ml | 59 +++++++++++++++-------- 3 files changed, 230 insertions(+), 37 deletions(-) diff --git a/BUILT.md b/BUILT.md index b29fc07..24acc60 100644 --- a/BUILT.md +++ b/BUILT.md @@ -2435,6 +2435,133 @@ reason `-linkall` is not optional. Say plainly what that coverage is not: nothin before this landed, so `macro-unless.flan` is a test written after the feature. The corpus written before it is `sand.flan` and `web/examples/control.flan`, and both compile unchanged. +## The prelude's second tier: the functions that return new storage + +Everything in the prelude before this was slice-based and allocation-free, and NEXT.md's diagnosis of why was exact: +there was nothing to allocate from when it was written. `Vec`, `Map`, an arena and `StorageExhausted` changed that, +and this is the tier that follows — 24 additions, of which the twelve that matter most **return new things** +instead of writing into a buffer the caller supplies. The rest fill in the slice family at the element types that +were missing, and one of them is a macro. + +Three rules hold across all of it, and they are stated once at the head of the section rather than repeated: + +1. **The result is owned and the caller frees it.** Nothing is released at scope exit — not at the end of a `let`, + not at the end of a function (`spec-memory.md`). A caller writes `(free v)`, or lets a `(free-all a)` take the + whole region. +2. **The allocator is the context's, and `with-allocator` is the override.** This is the one design decision the + spec did not settle by itself. `(vec-new)` and `(map-new)` take an optional trailing allocator because the + *checker* builds them and can vary their arity; a Flan `defn` cannot, so the choice was an allocator parameter on + every signature or none. None — `(with-allocator a (join parts sep))` is the override, the `Vec` records the + arena, and `free` and `clone` never need it named again. +3. **No `Result` anywhere.** Allocation failure signals `StorageExhausted` under `retry`, and no allocating + operation returns an error, so every signature says what it produces and nothing about how it might fail. + +### The builder is not a type + +Odin's `strings.Builder` wraps a `[dynamic]u8`. Here the `(Vec u8)` already **is** that and already has `push`, so +the struct would be a move-only wrapper whose only method is the one it wraps. What was actually missing is appending +a *run* of bytes, and `append!` is that. + +It takes a `(Ptr (Vec u8))` and not a `(Vec u8)`, and that is not style: a `Vec` parameter **moves**, so a by-value +builder would be consumed by its first append and refused on the second. + +`append-i64!` and `append-f64!` are the argument for the whole shape. NEXT.md's "Sharp edges" records that +`flan_i64_to_bytes` and its neighbours render into one `static char scratch[64]`, so two formatted numbers cannot be +held at once; these copy out of that buffer before returning, so the hazard ends at the call and a builder holds as +many numbers as it likes. `strings.flan` puts two integers and a float on one line, which is the case that could not +be written before. + +### `split` answers a `(Vec [u8])`, and the owning shape is unrepresentable + +The fields are slices *of the input*. That is not a performance choice — `(Vec (Vec u8))` is **refused outright** +(`programs/vec-of-vec.flan`, "copies and releases elements bytewise"), so there is no owning shape to have chosen +instead. It follows that the result dies with whatever the input pointed at, which is the same contract `trim` and +`split-next!` already have. + +The rule is `split-on-byte`'s, unchanged: n separators always yield n+1 fields, so an empty input yields one empty +field and a trailing separator yields a trailing empty one. That is Odin's allocating `strings.split` and not Odin's +`split_by_byte_iterator`, which disagree with each other on exactly that input. + +Constructing it needed a one-line `(defn slices-new [] (Vec [u8]) (vec-new))`, because `check.ml`'s `vec_new_elem` +takes the element type as a single bare symbol and `[u8]` is not one — so a `(Vec [u8])` can only be made where the +*context* names the type, and a return type is a context while a `let` is not. Written down in NEXT.md as a compiler +gap rather than worked around silently. + +### `format-f64`, and the rounding rule it does not share with printf + +`f64->bytes` is `snprintf "%g"`: six significant digits, exponent notation of its own accord, no precision to pass +it. A frame time of 1/60 comes back `0.0166667` and a score past a million `1.23457e+06`. + +`format-f64` returns a `Vec`, so it inherits neither that nor the shared scratch buffer, and it renders the integer +part and the fraction through that buffer in strict sequence — the discipline `append-i64!` exists to make automatic. + +It rounds **half away from zero at the last digit kept**, which is `round-f32`'s rule and the rest of the prelude's. +printf rounds the *binary* value to nearest-even at the decimal digit, so `0.125` at two places is `0.13` here and +`0.12` there. Matching printf would mean pinning a particular libc's answer, and that answer is not the same on every +target anyway. + +Three lines in it are the ones a plausible version ships without, and each is a separate test case: + +- **The carry.** `0.999995` at five places scales to exactly `100000`, which is not a fraction — it is the next + integer. Without the carry it prints `0.100000`. +- **The zero padding.** The fraction of `1.005` at three places is `5`, and `5` is not `005`; without the pad it + prints `1.5`. +- **The sign.** It belongs to the number, not to its integer part: `-0.5` has an integer part of `0`, and + `i64->bytes` of `0` carries no sign. + +`-0.0` prints as `0.00`, because the sign test is `(< x 0.0)`, which `-0.0` fails. Past `9e18` the integer part does +not fit in an `i64` and there are no fractional bits left anyway, so it falls back to `%g` rather than approximating. + +### `clamp` is a macro, and `atan2`/`pow` are declares + +`clamp` is the second prelude `defmacro` after `unless`, and the reason is the prelude's own objection to wrapping +`(min hi (max lo x))` turned around rather than dropped. `min` and `max` are builtins at *every* numeric type and +there are no generics, so a clamp **function** is one copy per type — `clamp-i32`, `clamp-f32`, `clamp-i64`. A macro +is type-agnostic for free and emits nothing at all. `math2.flan` makes the same three-word call at `i32`, `i64`, `u8` +and `f32` to show it, and counts evaluations to show each argument appears once. + +`atan2-f32` and `pow-f32` inherit `sin-f32`/`cos-f32`'s caveat in full and not `sqrt-f32`'s: IEEE-754 requires +nothing of `atan2f` or `powf` either, so they are the third and fourth places in the prelude where native and wasm32 +may differ in the last bit. Every case in `math2.flan` is therefore a value exact in binary — a quadrant boundary, a +power of two, a perfect square — and the `-O0` run is the one that proves the symbols resolve, since at `-O2` LLVM +constant-folds a `powf` of two literals and leaves nothing to link. + +### What could not be built, and why it is not "no generics" + +Four things on NEXT.md's list did not land, and the interesting part is that the reason differs in each case. + +- **`Map` keys and values** need a **map iterator**, and there is none. `flan_map_len`, `_get`, `_put`, `_has`, + `_clone`, `_reserve`, `_free` is the runtime's entire map surface; nothing walks the open-addressed block. One + runtime function taking a cursor and one builtin in `check.ml` to emit the key and value sizes is the whole job, + and none of it is a generics question. +- **`map`, `filter`, `reduce` and a comparator sort** are blocked on **function values**, which is sharper than "no + generics" and matters because generics alone would not fix it. `Types.Fn` exists; `check.ml` refuses it with "a + function type is not implemented yet — milestone 5"; there is nothing in the language to pass. The concrete answer + is the one that shipped: `sort-f32!` and `sort-bytes!` are the second and third sorts in the language, and + `sum-i32`/`sum-f32` already are `reduce` with the `+` written in. +- **The prelude is never macro-expanded**, so a prelude function may not call a prelude macro. `Macro.program` runs + over the file being compiled; the prelude reaches the checker through `Check.program`'s prepend. The call resolves + to the macro's underlying `defn` and reports an arity error, which is why `format-f64` writes + `(min 9 (max 0 prec))`. +- **A returned `Vec` is a move and the dead set spans the function**, so an early `(return v)` on one branch kills + the binding at the foot of another. `replace-bytes` guards its empty needle with an `if` rather than a + `when`/`return` for that reason. + +### The refusal block is down from eight reasons to four + +The list at the foot of `prelude.ml` used to be one sentence — every entry needed to produce bytes that did not exist +in its input, and there was no allocator. `join`, `concat`, `split`, `to-lower`, `to-upper`, `repeat` and `replace` +have moved up into the code; `string-from-bytes` turned out to be the `string` builtin all along, and +`(string (as-slice v))` is the round trip, free precisely because the layouts are identical. + +What remains is refused for four different reasons, and is now written that way: `pad`/`center` for *nothing at all* +except that no caller has asked; `format`/`sprintf` for variadics of mixed type; `map`/`filter`/`reduce`/`sort-by` +for function values; `map-keys`/`map-values` for the missing iterator. + +Tests: `programs/strings.flan`, `programs/format.flan`, `programs/algorithms.flan`, `programs/math2.flan`, each at +`-O2` and `-O0`, and `strings.flan` also in a dev build — the one that checks a container's recorded allocator epoch, +so it is what would catch one of these `Vec`s being used after the arena under it was released. + ## `defer` may be written in a `let` The whole of this project's resource-cleanup answer, and NEXT.md records `drop` and a `with-cleanup` form as both diff --git a/NEXT.md b/NEXT.md index ca0bea9..b52474b 100644 --- a/NEXT.md +++ b/NEXT.md @@ -518,26 +518,60 @@ hand-written backend is needed at all. It is research first, not building. The batch below stays valid and none of it is blocked by the question. -## Queued: a second tier of the standard library, after macros +## ~~Queued: a second tier of the standard library, after macros~~ — **landed** -Blocked only on `lib/prelude.ml`, which the macro lane holds. Start it when that merges. +See [`BUILT.md`](BUILT.md), "The prelude's second tier". The diagnosis here was right and the prelude had 44 +allocation-free functions because there was nothing to allocate from; there are 24 more now, and the "Refused, by +name" block at the foot of `prelude.ml` is down from eight entries to four, each with a *different* reason rather +than the one shared sentence. -**The gap, stated plainly: the whole prelude predates the allocator.** All 44 functions are slice-based and -allocation-free, because when they were written there was nothing to allocate from. `Vec` and `Map` now exist, so a -second tier is possible — functions that *return new things* rather than writing into a buffer the caller supplies. +What landed: `append!`/`append-i64!`/`append-f64!` (the builder), `concat`, `join`, `split` returning a +`(Vec [u8])`, `repeat-bytes`, `replace-bytes`, `to-lower`, `to-upper`, `slices-new`; `format-f64` with a precision; +`atan2-f32` and `pow-f32`; `clamp` as a `defmacro`; and the slice family at two more element types — +`sort-f32!`, `reverse-f32!`, `swap-f32!`, `min-f32`, `max-f32`, `sum-f32`, `bytesbytes` directly — nothing was taken away — but a caller assembling a line of + text has a way not to meet it. + - **Writing through a string literal is undefined, and the two build modes disagree about how.** `(let [s (bytes "Hi")] (set (at s 0) \h))` stores into a `private unnamed_addr constant`. At `-O0` that is a store to read-only @@ -1417,6 +1457,15 @@ What follows is only the part that is still missing. round it could be compiled in after something else. It would fail with an unknown name rather than with a reason, which is worth fixing the day the prelude wants one. +- **A prelude *function* may not call a prelude macro either**, which is the neighbouring gap and was found by + walking into it. `Macro.program` runs over the file being compiled; the prelude arrives at the checker through + `Check.program`'s own prepend and is never handed to the expander at all. A `defmacro` is an ordinary `defn` taking + one `[Form]` by the time the checker sees it, so the call resolves to that and the report is "clamp takes 1 + argument, given 3" — pointing at the prelude, about a call the author wrote as a macro use. `format-f64` writes + `(min 9 (max 0 prec))` in place of `(clamp prec 0 9)` because of it. The fix is not obviously cheap: expanding the + prelude means building a macro module to compile the prelude that the macro module is built from, which is the same + bootstrap the `when`/`dotimes` item above describes. + - **A quasiquote inside a quasiquote is refused.** Nothing counts nesting levels — not the reader, deliberately, and not the desugaring. Only a macro that writes a macro wants one. diff --git a/lib/prelude.ml b/lib/prelude.ml index 48e6b51..760586f 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -1172,29 +1172,46 @@ let source = {flan| (append! (addr b) d)))))))) b)) -;; ── Refused, by name ────────────────────────────────────────────────── +;; ── Still refused, and what the reason is now ───────────────────────── ;; -;; Every one of these needs to produce bytes that did not exist in its input, -;; and there is no allocator, so each is absent rather than approximated. -;; None of them is hard to write once `(Vec u8)` and an allocator exist; all -;; of them are impossible to write honestly today. +;; This list used to be one sentence long — every entry needed to produce bytes +;; that did not exist in its input, and there was no allocator. That sentence +;; stopped being true when `Vec` landed, and most of the list has moved up into +;; the building section above: join, concat, split, to-lower, to-upper, repeat +;; and replace are all written now, and `string-from-bytes` turned out to be +;; the `string` builtin all along — (string (as-slice v)) is the round trip, +;; and the layouts being identical is exactly why it is free. ;; -;; join, concat build one buffer out of several inputs. -;; to-lower, to-upper a new string, per Odin's conversion.odin. The -;; byte-wise and folding-comparison forms above are -;; what is available without one. -;; split the *sequence* of fields is itself an allocation. -;; split-on-byte / split-next! above is the same -;; information with no sequence to own. -;; replace, repeat, pad same reason as join. -;; string-from-bytes a [u8] cannot become a `string` here even though -;; the layouts are identical; see the report. -;; format, sprintf Odin's fmt.aprintf family, all allocating. -;; Builder strings.Builder is (defstruct Builder [buf -;; (Vec u8)]), which spec-memory.md already makes -;; move-only by the rule that a struct containing a -;; Vec is move-only. It needs the Vec, not a spec -;; change. +;; What is left is refused for four *different* reasons, which is why they are +;; named separately rather than under one heading. +;; +;; pad, center Nothing. These are three lines each over repeat-bytes +;; and concat, and they are absent only because no +;; caller has asked. Write them when one does. +;; format, sprintf A format *string* — Odin's fmt.aprintf family. It +;; needs variadic arguments of mixed type, which is a +;; function-value and generics question, not an +;; allocation one. format-f64 above is the piece of it +;; that was actually wanted, and `print`/`println` are +;; already the structural walk over any one value. +;; map, filter, reduce Function values. See the head of the slice-algorithm +;; sort-by section: check.ml refuses a function type outright, +;; and there is nothing in the language to pass. +;; map-keys, map-values A Map iterator. `len` reaches a Map and `get`, +;; `put` and `has-key?` address one entry, but there is +;; no entry point in the runtime that walks the block — +;; flan_map_len, _get, _put, _has, _clone, _reserve and +;; _free is the whole surface. This is the one item on +;; NEXT.md's second-tier list that could not be built +;; here at all, and it wants one runtime function and +;; one builtin rather than anything from the language. +;; +;; Builder Not refused — declined. strings.Builder in Odin +;; wraps a [dynamic]u8; here the (Vec u8) *is* that and +;; already has push, so the struct would be a move-only +;; wrapper whose only method is the one it wraps. What +;; was missing was appending a run of bytes, and +;; `append!` above is that. ;; ── Files: embedding, slurp and barf ────────────────────────────────── ;; ;; One entry per file in an (embed-dir "...") — Odin's Load_Directory_File From 242f8c2047322385f06d9c398d20b7aed080a0b4 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 22:03:34 +0700 Subject: [PATCH 6/6] Three section headers still said there was no allocator The refusal list was rewritten and the prologues that pointed at it were not, so prelude.ml claimed in three places that what it now contains is impossible: the splitting header said `split` is refused at the foot of the file, forty lines above `split`; the ASCII-case header said Odin's allocating to_lower is not available here, next to the one that was written; and the UTF-8 header said the rest of core/strings is refused rather than ported. Each keeps its point rather than losing it. The iterator is still the shape that owns nothing and still the right call when there is no result to own; lower-ascii and bytes-ci=? are still the right calls when a copy is not wanted, since folding a comparison over two inputs beats lowering both. What changed is the reason, which used to be the absence of an allocator and is now a choice between two shapes that both exist. And strings.flan told the reader the opposite of what it did -- "not freed", on the line above the free. vec.flan already had the right framing: the free is written, it keeps the block because an arena cannot release one, and that is the difference the capability set exists to state. --- lib/prelude.ml | 29 ++++++++++++++++++----------- test/programs/strings.flan | 10 ++++++---- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/lib/prelude.ml b/lib/prelude.ml index 760586f..d2b50bf 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -607,8 +607,11 @@ let source = {flan| ;; Ported from Odin's core/unicode/utf8/utf8.odin, which is the one corner of ;; a string library that is allocation-free by construction: decoding is ;; classification, and every answer it gives is a number. Everything else in -;; Odin's core/strings and all of core/fmt takes `allocator := -;; context.allocator`, and is therefore refused below rather than ported. +;; Odin's core/strings takes `allocator := context.allocator`, which is why +;; this corner came first and the rest waited; most of that rest is ported now +;; and lives in the building section below. core/fmt is still absent, and the +;; reason it stays absent is not allocation — see the refusal list at the foot +;; of this file. ;; ;; Odin's 256-entry accept_sizes table becomes a cond over the lead byte here. ;; The table is the cache-friendly form and the cond is the one you can check @@ -791,12 +794,13 @@ let source = {flan| ;; ── Splitting ───────────────────────────────────────────────────────── ;; -;; `split` returning a sequence of fields must allocate the sequence, and -;; there is no allocator — so it is refused by name at the bottom of this -;; file, and this is the shape that survives. It is Odin's -;; split_by_byte_iterator (strings.odin): a cursor holding the rest of the -;; input, handing back one field at a time. Every field is a slice *of the -;; caller's bytes*; nothing is copied and nothing is owned. +;; The iterator, which owns nothing. `split` returning a sequence of fields has +;; to allocate that sequence, and it does — it is in the building section below +;; — but this stays the right call whenever you do not want to own the result: +;; it is Odin's split_by_byte_iterator (strings.odin), a cursor holding the +;; rest of the input and handing back one field at a time. Every field is a +;; slice *of the caller's bytes*; nothing is copied, nothing is owned, and +;; there is no free to remember. `split` is built on exactly this. ;; ;; One divergence, and it is a wart of Odin's rather than a decision. Odin's ;; iterator stops on an empty final field, so "a,b," iterates a and b and the @@ -828,9 +832,12 @@ let source = {flan| ;; ── ASCII case ──────────────────────────────────────────────────────── ;; ;; Byte in, byte out, and *not* a function over a slice. Odin's to_lower and -;; to_upper both allocate a new string (core/strings/conversion.odin), which -;; is not available here; the obvious substitute — lowering a [u8] in place — -;; is a trap, and it is worth saying why rather than shipping it. A string +;; to_upper both allocate a new string (core/strings/conversion.odin) and so do +;; the ones in the building section below; these are the forms that allocate +;; nothing, and they stay the right call when a copy is not wanted — folding a +;; comparison over two inputs beats lowering both and comparing. What is *not* +;; on offer is the third shape, lowering a [u8] in place, and it is worth +;; saying why rather than shipping it. A string ;; literal is emitted `private unnamed_addr constant` (emit.ml), so (bytes ;; "Hello") is a [u8] pointing straight into read-only memory. An in-place ;; lower-ascii! type checks against that slice, and what happens next depends diff --git a/test/programs/strings.flan b/test/programs/strings.flan index d1e441a..17d8929 100644 --- a/test/programs/strings.flan +++ b/test/programs/strings.flan @@ -137,15 +137,17 @@ ;; The allocator is the context's, so with-allocator moves the whole tier ;; into an arena -- which is the answer to the fixed arity of a defn, and the - ;; reason none of these takes an allocator argument. free-all releases every - ;; one of them at once, including the Vec still bound below it. + ;; reason none of these takes an allocator argument. free-all is what + ;; releases the region, and arena-destroy hands it back. (let [a (arena-new 4096)] (with-allocator a (let [parts [(bytes "in") (bytes "arena")]] (let [j (join (slice parts 0 2) (bytes "-"))] (show (addr j)) ; in-arena - ;; Not freed: an arena cannot release one block, and free-all is - ;; what releases this. + ;; The free is written because the binding is dead after it either + ;; way, and it keeps the block: an arena cannot release one, which + ;; is the difference the capability set exists to state. free-all + ;; below is what actually releases this. (free j)))) (free-all a) (arena-destroy a))