From 2acb70b8db0128954c255aaa5378f17fe876647f Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 03:49:52 +0700 Subject: [PATCH 1/4] Decoding is the only part of a string library that needs no allocator Odin's core/strings and all of core/fmt take an allocator; core/unicode/utf8 does not, because decoding is classification and every answer is a number. That line is where the port stops, and the refusals at the foot of the file say so by name rather than leaving a caller to find out. The accept_sizes table becomes a cond over the lead byte. Its four awkward rows are the ones a hand-written decoder gets wrong one at a time, so they are written out: 0xc0/0xc1 lead nothing, 0xe0 and 0xf0 have a raised second-byte floor against overlongs, 0xed has a lowered ceiling against the surrogates. Two divergences from Odin, both the parse-i64 argument again. A malformed sequence carries ok:false instead of decoding to U+FFFD, which is a real code point a caller cannot tell from a failure; and encode-rune! answers None rather than silently substituting U+FFFD for a rune it was not given. Width stays 1 on a bad byte, which is Odin's rule and load-bearing: every loop here advances by it, and a 0 would hang rather than answer wrong. split cannot return a sequence it would have to own, so the cursor is what survives. It follows the allocating strings.split rather than Odin's own iterator, which drops a trailing empty field and disagrees with it. Case conversion is byte-wise and not in place: a literal is emitted into read-only memory, so lowering (bytes "Hi") would type check and segfault. --- lib/prelude.ml | 281 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) diff --git a/lib/prelude.ml b/lib/prelude.ml index 7955089..1e4e72a 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -407,6 +407,287 @@ let source = {flan| ;; Trailing junk is the case strtod is silent about, so the position has ;; to land exactly on the end. (if (= i (len s)) (Some (bytes->f64 s)) None))) + +;; ── UTF-8 ───────────────────────────────────────────────────────────── +;; +;; Ported from Odin's core/unicode/utf8/utf8.odin, which is the one corner of +;; a string library that is allocation-free by construction: decoding is +;; classification, and every answer it gives is a number. Everything else in +;; Odin's core/strings and all of core/fmt takes `allocator := +;; context.allocator`, and is therefore refused below rather than ported. +;; +;; Odin's 256-entry accept_sizes table becomes a cond over the lead byte here. +;; The table is the cache-friendly form and the cond is the one you can check +;; by reading, and nothing in a game decodes UTF-8 in a hot loop — DrawText +;; hands the bytes straight to raylib. +;; +;; The four rules that table encodes, and which a hand-written decoder gets +;; wrong one at a time: +;; +;; 0x80..0xc1 never a lead byte. 0x80..0xbf are continuation bytes, and +;; 0xc0 and 0xc1 could only ever begin an *overlong* two-byte +;; spelling of an ASCII character — the encoding that lets +;; "\xc0\xaf" smuggle a "/" past a check for one. +;; 0xe0 second byte 0xa0..0xbf and not 0x80..0xbf; the low half is +;; the overlong three-byte range. +;; 0xed second byte 0x80..0x9f. The high half is U+D800..U+DFFF, +;; the UTF-16 surrogates, which are not scalar values. +;; 0xf0, 0xf4 second byte 0x90..0xbf and 0x80..0x8f: overlong below, +;; and past U+10FFFF above. 0xf5..0xff lead nothing at all. +;; +;; A rune is an i32 and not a type of its own. That is Odin's answer too — +;; its `rune` is a four-byte integer distinguished only by a flag on the +;; basic-type row (src/types.cpp, the Basic_rune entry) — so nothing in the +;; checker has to learn a new type for any of this. + +;; One deliberate divergence from Odin, and it is the parse-i64 argument over +;; again. Odin's decode_rune answers RUNE_ERROR — U+FFFD — for malformed +;; bytes, and U+FFFD is a perfectly real code point that a well-formed string +;; may contain, so a caller cannot tell a decoded replacement character from a +;; failure to decode. This carries `ok` instead, and leaves `code` 0 when it +;; is false. +;; +;; `width` is 1 on a malformed byte and 0 only for an empty input. That is +;; Odin's rule and it is load-bearing rather than cosmetic: every loop below +;; advances by `width`, so a 0 there on a bad byte is an infinite loop, not a +;; wrong number. +(defstruct Rune [code i32 width i32 ok bool]) + +(defn rune-start? [b u8] bool + (!= (bit-and b 0xc0) 0x80)) + +(defn decode-rune [s [u8]] Rune + (when (= (len s) 0) + (return (Rune {:code 0 :width 0 :ok false}))) + (let [b0 (at s 0)] + (when (< b0 0x80) + (return (Rune {:code (i32 b0) :width 1 :ok true}))) + ;; size 0 means "this byte cannot lead"; lo/hi are the *second* byte's + ;; accepted range, which is the only place the overlong and surrogate + ;; rules live. Bytes three and four are always 0x80..0xbf. + (let [size 0 + lo (u8 0x80) + hi (u8 0xbf)] + (cond + (< b0 0xc2) (set size 0) + (<= b0 0xdf) (set size 2) + (= b0 0xe0) (do (set size 3) (set lo (u8 0xa0))) + (<= b0 0xec) (set size 3) + (= b0 0xed) (do (set size 3) (set hi (u8 0x9f))) + (<= b0 0xef) (set size 3) + (= b0 0xf0) (do (set size 4) (set lo (u8 0x90))) + (<= b0 0xf3) (set size 4) + (= b0 0xf4) (do (set size 4) (set hi (u8 0x8f))) + :else (set size 0)) + (when (= size 0) + (return (Rune {:code 0 :width 1 :ok false}))) + ;; A sequence cut off by the end of the slice. Width 1, so a caller + ;; scanning a buffer boundary makes progress instead of stalling. + (when (> size (len s)) + (return (Rune {:code 0 :width 1 :ok false}))) + (let [b1 (at s 1)] + (when (or (< b1 lo) (> b1 hi)) + (return (Rune {:code 0 :width 1 :ok false}))) + (when (= size 2) + (return (Rune {:code (bit-or (<< (i32 (bit-and b0 0x1f)) 6) + (i32 (bit-and b1 0x3f))) + :width 2 :ok true}))) + (let [b2 (at s 2)] + (when (or (< b2 0x80) (> b2 0xbf)) + (return (Rune {:code 0 :width 1 :ok false}))) + (when (= size 3) + (return (Rune {:code (bit-or (bit-or (<< (i32 (bit-and b0 0x0f)) 12) + (<< (i32 (bit-and b1 0x3f)) 6)) + (i32 (bit-and b2 0x3f))) + :width 3 :ok true}))) + (let [b3 (at s 3)] + (when (or (< b3 0x80) (> b3 0xbf)) + (return (Rune {:code 0 :width 1 :ok false}))) + (Rune {:code (bit-or (bit-or (<< (i32 (bit-and b0 0x07)) 18) + (bit-or (<< (i32 (bit-and b1 0x3f)) 12) + (<< (i32 (bit-and b2 0x3f)) 6))) + (i32 (bit-and b3 0x3f))) + :width 4 :ok true}))))))) + +;; Decode at a byte offset. None when the offset is not on a rune boundary or +;; the bytes there are malformed, which is stricter than Odin's rune_at — that +;; one hands back RUNE_ERROR and the caller carries on with a wrong character. +(defn rune-at [s [u8] i i32] (Option i32) + (if (or (< i 0) (>= i (len s))) + None + (let [r (decode-rune (slice s i (len s)))] + (if (.ok r) (Some (.code r)) None)))) + +;; Counted through decode-rune rather than through a second walk of its own. +;; Odin keeps a separate rune_count_in_bytes that re-implements the size +;; table; two copies of that classification is two places for the surrogate +;; rule to be right in only one of them. +;; +;; A malformed byte counts as one, which is what a replacement-character +;; renderer would draw, so this agrees with what the screen shows. +(defn rune-count [s [u8]] i32 + (let [i 0 + n 0] + (while (< i (len s)) + (let [r (decode-rune (slice s i (len s)))] + (set i (+ i (.width r))) + (set n (+ n 1)))) + n)) + +(defn valid-utf8? [s [u8]] bool + (let [i 0] + (while (< i (len s)) + (let [r (decode-rune (slice s i (len s)))] + (when (not (.ok r)) + (return false)) + (set i (+ i (.width r))))) + true)) + +;; How many bytes this code point encodes to, or None if it is not a scalar +;; value. Odin's rune_size answers -1 for the refusals; a sentinel index is +;; exactly what index-of-i32 avoids above, so this is an Option like the rest +;; of the file. +(defn rune-size [code i32] (Option i32) + (cond + (< code 0) None + (<= code 0x7f) (Some 1) + (<= code 0x7ff) (Some 2) + (and (>= code 0xd800) (<= code 0xdfff)) None + (<= code 0xffff) (Some 3) + (<= code 0x10ffff) (Some 4) + :else None)) + +;; Encoding is the one operation here whose result is not a slice of its +;; input, because the bytes it makes existed nowhere before. With no allocator +;; the only shape left is Odin's own allocation-free one — strings.Builder +;; built by builder_from_bytes over a caller's backing array (builder.odin, +;; builder_from_bytes: "Uses Nil Allocator - Does NOT allocate") — reduced to +;; its essential case: write into a buffer the caller owns, and say how much +;; was written. +;; +;; None rather than a partial write when the buffer is short, and None rather +;; than Odin's silent substitution of U+FFFD for an invalid rune. Odin's +;; encode_rune rewrites a surrogate or an out-of-range value to the +;; replacement character and reports success; the caller then finds three +;; bytes of U+FFFD in its buffer and no indication that it asked for something +;; else. Nothing is written at all when this answers None. +(defn encode-rune! [dst [u8] code i32] (Option i32) + (match (rune-size code) + None None + (Some w) + (if (> w (len dst)) + None + (do + (cond + (= w 1) + (set (at dst 0) (u8 code)) + (= w 2) + (do (set (at dst 0) (u8 (bit-or 0xc0 (>> code 6)))) + (set (at dst 1) (u8 (bit-or 0x80 (bit-and code 0x3f))))) + (= w 3) + (do (set (at dst 0) (u8 (bit-or 0xe0 (>> code 12)))) + (set (at dst 1) (u8 (bit-or 0x80 (bit-and (>> code 6) 0x3f)))) + (set (at dst 2) (u8 (bit-or 0x80 (bit-and code 0x3f))))) + :else + (do (set (at dst 0) (u8 (bit-or 0xf0 (>> code 18)))) + (set (at dst 1) (u8 (bit-or 0x80 (bit-and (>> code 12) 0x3f)))) + (set (at dst 2) (u8 (bit-or 0x80 (bit-and (>> code 6) 0x3f)))) + (set (at dst 3) (u8 (bit-or 0x80 (bit-and code 0x3f)))))) + (Some w))))) + +;; ── Splitting ───────────────────────────────────────────────────────── +;; +;; `split` returning a sequence of fields must allocate the sequence, and +;; there is no allocator — so it is refused by name at the bottom of this +;; file, and this is the shape that survives. It is Odin's +;; split_by_byte_iterator (strings.odin): a cursor holding the rest of the +;; input, handing back one field at a time. Every field is a slice *of the +;; caller's bytes*; nothing is copied and nothing is owned. +;; +;; One divergence, and it is a wart of Odin's rather than a decision. Odin's +;; iterator stops on an empty final field, so "a,b," iterates a and b and the +;; trailing empty field is lost — while Odin's own allocating strings.split +;; returns ["a", "b", ""] for the same input. The two disagree. This follows +;; split: n separators always yield n+1 fields, an empty input yields one +;; empty field, and `rest` is exhausted only after the last one is taken. That +;; is the rule you can state without exceptions, and the one a caller counting +;; comma-separated columns needs. +(defstruct Split [rest [u8] sep u8 more bool]) + +(defn split-on-byte [s [u8] sep u8] Split + (Split {:rest s :sep sep :more true})) + +(defn split-next! [it (Ptr Split)] (Option [u8]) + (when (not (.more it)) + (return None)) + (match (index-of-byte (.rest it) (.sep it)) + (Some i) + (let [field (slice (.rest it) 0 i)] + (set (.rest it) (slice (.rest it) (+ i 1) (len (.rest it)))) + (Some field)) + None + (let [field (.rest it)] + (set (.more it) false) + (set (.rest it) (slice (.rest it) (len (.rest it)) (len (.rest it)))) + (Some field)))) + +;; ── ASCII case ──────────────────────────────────────────────────────── +;; +;; Byte in, byte out, and *not* a function over a slice. Odin's to_lower and +;; to_upper both allocate a new string (core/strings/conversion.odin), which +;; is not available here; the obvious substitute — lowering a [u8] in place — +;; is a trap, and it is worth saying why rather than shipping it. A string +;; literal is emitted `private unnamed_addr constant` (emit.ml), so it lives +;; in read-only memory, and (bytes "Hello") is a [u8] pointing straight at it. +;; An in-place lower-ascii! would type check against that slice and segfault +;; on the store. Given a byte function, a caller that really does own its +;; buffer writes the two-line loop itself and can see what it is writing to. +;; +;; ASCII only, and only the 26 letters: case outside ASCII is not a byte +;; operation at all — it is per-code-point, it is not length-preserving (ß +;; upcases to SS), and it is locale-dependent (Turkish dotless ı). A byte +;; table that pretended otherwise would be wrong in the quiet way. +(defn lower-ascii [b u8] u8 + (if (and (>= b \A) (<= b \Z)) (+ b 32) b)) + +(defn upper-ascii [b u8] u8 + (if (and (>= b \a) (<= b \z)) (- b 32) b)) + +;; Case-insensitive comparison as a fold over both inputs, which is the useful +;; half of to_lower and needs no storage at all: comparing two lowered copies +;; is what a caller wanted, and this is that answer without either copy. +(defn bytes-ci=? [a [u8] b [u8]] bool + (if (!= (len a) (len b)) + false + (do + (dotimes [i (len a)] + (when (!= (lower-ascii (at a i)) (lower-ascii (at b i))) + (return false))) + true))) + +;; ── Refused, by name ────────────────────────────────────────────────── +;; +;; Every one of these needs to produce bytes that did not exist in its input, +;; and there is no allocator, so each is absent rather than approximated. +;; None of them is hard to write once `(Vec u8)` and an allocator exist; all +;; of them are impossible to write honestly today. +;; +;; join, concat build one buffer out of several inputs. +;; to-lower, to-upper a new string, per Odin's conversion.odin. The +;; byte-wise and folding-comparison forms above are +;; what is available without one. +;; split the *sequence* of fields is itself an allocation. +;; split-on-byte / split-next! above is the same +;; information with no sequence to own. +;; replace, repeat, pad same reason as join. +;; string-from-bytes a [u8] cannot become a `string` here even though +;; the layouts are identical; see the report. +;; format, sprintf Odin's fmt.aprintf family, all allocating. +;; Builder strings.Builder is (defstruct Builder [buf +;; (Vec u8)]), which spec-memory.md already makes +;; move-only by the rule that a struct containing a +;; Vec is move-only. It needs the Vec, not a spec +;; change. |flan} let file = "" From 273dd18eb0133864c2f45e2afbd480944c38670d Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 03:53:18 +0700 Subject: [PATCH 2/4] The valid decodes prove almost nothing; the malformed ones are the test A decoder that only masks and shifts gets every well-formed character right, so a corpus of real text passes it. What separates it from a correct one is the second group here: an overlong two- and three-byte "/", a surrogate, a code point past U+10FFFF, a lead byte that leads nothing, a lone continuation byte, and a character truncated by the end of its slice. Each isolates one row of the accept_sizes table, and each must answer width 1 so a scan advances. The invalid sequences are byte arrays because no valid string contains them and the reader has no \xNN escape to spell them with. Encoding is checked by round trip. An encoder and a decoder wrong in the same direction agree with each other, and expected bytes would not catch that. The emoji line caught a use-after-return while this was being written: a (defn whole [a [4 u8]] [u8] (slice a 0 4)) helper returns a slice into the copy a [n T] parameter makes in the callee's frame. The compiler accepts it in silence. The comment stays where the helper was. --- test/programs/utf8.flan | 248 ++++++++++++++++++++++++++++++++++++++++ test/test_acceptance.ml | 42 +++++++ 2 files changed, 290 insertions(+) create mode 100644 test/programs/utf8.flan diff --git a/test/programs/utf8.flan b/test/programs/utf8.flan new file mode 100644 index 0000000..7e662a8 --- /dev/null +++ b/test/programs/utf8.flan @@ -0,0 +1,248 @@ +;;;; UTF-8 decoding and encoding, the split cursor, and ASCII case. +;;;; +;;;; Every case here is one a plausible wrong decoder passes. A decoder that +;;;; only masks and shifts — takes the top bits of the lead byte for a length +;;;; and the low six of everything after — gets all of the *valid* input right +;;;; and every line of the second group wrong, so the valid decodes prove +;;;; almost nothing on their own and the malformed ones are the test. +;;;; +;;;; The four that matter, each isolating one row of Odin's accept_sizes +;;;; table: +;;;; +;;;; c0 af an overlong two-byte "/". Accepted, it is the encoding +;;;; that smuggles a slash past a check for one. +;;;; e0 80 af an overlong three-byte "/", which the raised second-byte +;;;; floor on 0xe0 is the only thing rejecting. +;;;; ed a0 80 U+D800, a UTF-16 surrogate, which is not a scalar value. +;;;; Only the lowered ceiling on 0xed rejects it. +;;;; f4 90 80 80 U+110000, one past the last code point. +;;;; +;;;; And three more shapes: a lone continuation byte, a lead byte that leads +;;;; nothing (0xf5), and a sequence truncated by the end of the slice — the +;;;; last taken as a slice of a *valid* literal, which is what reading a +;;;; buffer boundary actually hands you. +;;;; +;;;; The invalid sequences are byte arrays because the reader has no \xNN +;;;; escape in either a string or a character literal, and none of them can be +;;;; derived from a valid string. + +(defconst overlong2 [2 u8] [0xc0 0xaf]) +(defconst overlong3 [3 u8] [0xe0 0x80 0xaf]) +(defconst surrogate [3 u8] [0xed 0xa0 0x80]) +(defconst above-max [4 u8] [0xf4 0x90 0x80 0x80]) +(defconst lead-f5 [4 u8] [0xf5 0x80 0x80 0x80]) +(defconst lone-cont [1 u8] [0x80]) +(defconst emoji [4 u8] [0xf0 0x9f 0x98 0x80]) ; U+1F600 +(defconst bad-tail [3 u8] [0x61 0xff 0x62]) ; "a", junk, "b" + +(defvar scratch [4 u8]) + +;; Each array is sliced at the point of use rather than through a +;; (defn whole [a [4 u8]] [u8] (slice a 0 4)) helper. That helper is a +;; use-after-return and the compiler accepts it in silence: a [n T] parameter +;; is a *value* and copies into the callee's frame, so the slice it hands back +;; points at a frame that has already gone. It was written here first, and the +;; emoji line is what caught it — it decoded as malformed because it was +;; reading whatever the next call left on the stack. This is the escaping +;; borrow spec-memory.md leaves to the programmer under "Borrowing", and it is +;; the case a future provenance pass would reject. + +;; code/width/ok, so a wrong answer names which of the three it got wrong +;; rather than just failing. +(defn show-dec [s [u8]] + (let [r (decode-rune s)] + (print-i64 (i64 (.code r))) (print-str "/") + (print-i64 (i64 (.width r))) (print-str "/") + (print-str (if (.ok r) "t" "f")) + (print-str " "))) + +(defn show-bool [b bool] + (print-str (if b "t" "f"))) + +(defn show-opt [o (Option i32)] + (print-i64 (i64 (match o (Some v) v None -1))) + (print-str " ")) + +;; Encode into the scratch buffer and decode straight back out of it. A round +;; trip is the only check that catches an encoder and a decoder that are +;; wrong in the same direction — printing the bytes would not. +(defn round-trip [code i32] i32 + (match (encode-rune! (slice scratch 0 4) code) + None -1 + (Some w) + (let [r (decode-rune (slice scratch 0 w))] + (if (and (.ok r) (= (.width r) w)) (.code r) -1)))) + +(defn show-i32 [x i32] + (print-i64 (i64 x)) + (print-str " ")) + +(defn show-split [s [u8] sep u8] + (let [it (split-on-byte s sep) + going true] + (while going + (match (split-next! (addr it)) + (Some f) (do (print-str "[") (print-bytes f) (print-str "]")) + None (set going false))) + (print-str " "))) + +(defn main [] i32 + ;; Valid, one of each width. The empty slice is width 0 — the only input + ;; that gets a 0, because every loop below advances by width and a 0 on a + ;; malformed byte would hang instead of answering. + (show-dec (bytes "")) ; 0/0/f + (show-dec (bytes "A")) ; 65/1/t + (show-dec (bytes "é")) ; 233/2/t + (show-dec (bytes "日")) ; 26085/3/t + (show-dec (slice emoji 0 4)) ; 128512/4/t + (newline) + + ;; Malformed. Every one is 0/1/f: width 1 so a scan makes progress. + (show-dec (slice lone-cont 0 1)) ; a continuation byte leading + (show-dec (slice overlong2 0 2)) ; overlong "/" + (show-dec (slice overlong3 0 3)) ; overlong "/" again, three bytes + (show-dec (slice surrogate 0 3)) ; U+D800 + (show-dec (slice above-max 0 4)) ; U+110000 + (show-dec (slice lead-f5 0 4)) ; 0xf5 leads nothing + (newline) + + ;; Truncated: a valid character cut short by the end of the slice, at both + ;; possible cut points, and the interior of one taken on its own. + (show-dec (slice (bytes "日") 0 1)) ; lead byte alone + (show-dec (slice (bytes "日") 0 2)) ; lead plus one continuation + (show-dec (slice (bytes "日") 1 3)) ; starts mid-character + (show-dec (slice (bytes "é") 1 2)) ; a lone continuation from a literal + (newline) + + ;; rune-start? is what a caller scans backwards with. + (show-bool (rune-start? (at (bytes "日") 0))) + (show-bool (rune-start? (at (bytes "日") 1))) + (show-bool (rune-start? \A)) + (newline) + + ;; Counting. The empty string is 0 and not 1; the mixed string is 8 runes + ;; in 13 bytes, which is the whole distinction; and a malformed byte counts + ;; as one, so a count never disagrees with what a renderer would draw. + (print-i64 (i64 (rune-count (bytes "")))) (print-str " ") + (print-i64 (i64 (rune-count (bytes "abc")))) (print-str " ") + (print-i64 (i64 (rune-count (bytes "héllo 日本")))) (print-str " ") + (print-i64 (i64 (len (bytes "héllo 日本")))) (print-str " ") + (print-i64 (i64 (rune-count (slice bad-tail 0 3)))) + (newline) + + (show-bool (valid-utf8? (bytes ""))) + (show-bool (valid-utf8? (bytes "héllo 日本"))) + (show-bool (valid-utf8? (slice surrogate 0 3))) + (show-bool (valid-utf8? (slice overlong2 0 2))) + (show-bool (valid-utf8? (slice bad-tail 0 3))) + (show-bool (valid-utf8? (slice emoji 0 4))) + (newline) + + ;; rune-at: on a boundary, off a boundary, and out of range. Off a boundary + ;; is None rather than a replacement character, which is where this is + ;; stricter than Odin's rune_at. + (show-opt (rune-at (bytes "日本") 0)) ; 26085 + (show-opt (rune-at (bytes "日本") 3)) ; 26412 + (show-opt (rune-at (bytes "日本") 1)) ; -1, mid-character + (show-opt (rune-at (bytes "日本") 6)) ; -1, past the end + (show-opt (rune-at (bytes "") 0)) ; -1 + (newline) + + ;; rune-size, at every boundary and on both sides of it. + (show-opt (rune-size -1)) + (show-opt (rune-size 0)) + (show-opt (rune-size 0x7f)) + (show-opt (rune-size 0x80)) + (show-opt (rune-size 0x7ff)) + (show-opt (rune-size 0x800)) + (show-opt (rune-size 0xd7ff)) + (show-opt (rune-size 0xd800)) + (show-opt (rune-size 0xdfff)) + (show-opt (rune-size 0xe000)) + (show-opt (rune-size 0xffff)) + (show-opt (rune-size 0x10000)) + (show-opt (rune-size 0x10ffff)) + (show-opt (rune-size 0x110000)) + (newline) + + ;; Round trips, one per width and at the boundaries. + (show-i32 (round-trip 0)) + (show-i32 (round-trip 0x41)) + (show-i32 (round-trip 0x7f)) + (show-i32 (round-trip 0x80)) + (show-i32 (round-trip 0x7ff)) + (show-i32 (round-trip 0x800)) + (show-i32 (round-trip 0xffff)) + (show-i32 (round-trip 0x10000)) + (show-i32 (round-trip 0x10ffff)) + (newline) + + ;; Refused by encode-rune!, and nothing is written when it refuses. + (show-opt (encode-rune! (slice scratch 0 4) 0xd800)) ; -1, surrogate + (show-opt (encode-rune! (slice scratch 0 4) 0x110000)) ; -1, past the end + (show-opt (encode-rune! (slice scratch 0 4) -1)) ; -1, negative + (show-opt (encode-rune! (slice scratch 0 2) 0x65e5)) ; -1, buffer short + (show-opt (encode-rune! (slice scratch 0 0) 0x41)) ; -1, no room at all + (show-opt (encode-rune! (slice scratch 0 1) 0x41)) ; 1, exactly enough + (newline) + + ;; Splitting. n separators give n+1 fields, always: an interior empty field + ;; survives, a leading and a trailing one do too, and an input with no + ;; separator at all is one field rather than none. The empty input is the + ;; case Odin's own iterator disagrees with its allocating split on — it is + ;; one empty field here. + (show-split (bytes "a,b,c") \,) ; [a][b][c] + (show-split (bytes "a,,b") \,) ; [a][][b] + (show-split (bytes "abc") \,) ; [abc] + (show-split (bytes "") \,) ; [] + (show-split (bytes ",") \,) ; [][] + (show-split (bytes ",a") \,) ; [][a] + (show-split (bytes "a,") \,) ; [a][] + (newline) + + ;; A field is a slice of the input, so trim and parse-i64 work straight off + ;; one with nothing copied in between — which is the entire reason the + ;; cursor shape exists. + (let [it (split-on-byte (bytes " 10 , 20 ,30") \,) + total (i64 0) + going true] + (while going + (match (split-next! (addr it)) + (Some f) (set total (+ total (match (parse-i64 (trim f)) (Some v) v None 0))) + None (set going false))) + (print-i64 total) + (newline)) + + ;; ASCII case. The boundary bytes on both sides of each range are what a + ;; wrong mask gets wrong: '@' and '[' sit either side of A-Z, and '`' and + ;; '{' either side of a-z, so a conversion written as (bit-xor b 32) — + ;; which works for every letter — turns '@' into '`' and is caught here. + (show-i32 (i32 (lower-ascii \A))) + (show-i32 (i32 (lower-ascii \Z))) + (show-i32 (i32 (lower-ascii \a))) + (show-i32 (i32 (lower-ascii \@))) ; 64, just below 'A' + (show-i32 (i32 (lower-ascii \[))) ; 91, just above 'Z' + (show-i32 (i32 (upper-ascii \a))) + (show-i32 (i32 (upper-ascii \z))) + (show-i32 (i32 (upper-ascii \A))) + (show-i32 (i32 (upper-ascii \`))) ; 96, just below 'a' + (show-i32 (i32 (upper-ascii \{))) ; 123, just above 'z' + (show-i32 (i32 (lower-ascii \5))) ; digits are untouched + (newline) + + ;; A non-ASCII byte must pass through both untouched, which is the claim + ;; that "ASCII only" is a rule and not an oversight. + (show-i32 (i32 (lower-ascii (at (bytes "é") 0)))) + (show-i32 (i32 (upper-ascii (at (bytes "é") 0)))) + (newline) + + (show-bool (bytes-ci=? (bytes "Hello") (bytes "hELLO"))) ; t + (show-bool (bytes-ci=? (bytes "Hello") (bytes "hello!"))) ; f length first + (show-bool (bytes-ci=? (bytes "") (bytes ""))) ; t + (show-bool (bytes-ci=? (bytes "a") (bytes "b"))) ; f + ;; '@' is 'A'+32 apart from '`' the way a letter is from its own case, so a + ;; fold written as a bit-xor would call these two equal. They are not. + (show-bool (bytes-ci=? (bytes "@") (bytes "`"))) ; f + (show-bool (bytes-ci=? (bytes "é") (bytes "é"))) ; t bytes match + (newline) + 0) diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index 9d138ff..1fb74f2 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -881,6 +881,48 @@ ERR@7 unexpected token: not the kind the caller was reading outputs "signedness" "programs/signedness.flan" signed_out; outputs ~opt:"-O0" "signedness, -O0" "programs/signedness.flan" signed_out; + (* ── Strings: UTF-8, the split cursor, and ASCII case ────────────── + + The prelude's port of Odin's core/unicode/utf8, which is the only part + of a string library that needs no allocator. The valid decodes prove + almost nothing on their own — a decoder that just masks and shifts gets + every one of them right — so the test is the malformed group: an + overlong two- and three-byte "/", a UTF-16 surrogate, a code point past + U+10FFFF, a lead byte that leads nothing, a lone continuation byte, and + a valid character truncated by the end of its slice. Each isolates one + row of the accept_sizes table, and each answers width 1 so that a scan + makes progress rather than hanging. + + Encoding is checked by round trip rather than against expected bytes, + because an encoder and a decoder that are wrong in the same direction + agree with each other and disagree with nothing else. + + At -O0 as well, for the reason the slice algorithms run there: a slice + is a two-word struct through an alloca, decode-rune returns a struct by + value, and the split cursor is mutated through a (Ptr Split) — mem2reg + is exactly what would hide any of those being copied when it should be + shared. *) + let utf8_out = + "0/0/f 65/1/t 233/2/t 26085/3/t 128512/4/t \n\ + 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f \n\ + 0/1/f 0/1/f 0/1/f 0/1/f \n\ + tft\n\ + 0 3 8 13 3\n\ + ttffft\n\ + 26085 26412 -1 -1 -1 \n\ + -1 1 1 2 2 3 3 -1 -1 3 3 4 4 -1 \n\ + 0 65 127 128 2047 2048 65535 65536 1114111 \n\ + -1 -1 -1 -1 -1 1 \n\ + [a][b][c] [a][][b] [abc] [] [][] [][a] [a][] \n\ + 60\n\ + 97 122 97 64 91 65 90 65 96 123 53 \n\ + 195 195 \n\ + tftfft\n" + in + outputs "utf-8, splitting and ascii case" "programs/utf8.flan" utf8_out; + outputs ~opt:"-O0" "utf-8, splitting and ascii case, -O0" + "programs/utf8.flan" utf8_out; + if !failures = 0 then print_endline "acceptance: all tests passed" else begin Printf.printf "\n%d failure(s)\n" !failures; From 4957497d022ace56f54d8c3b49f289be5a09a6d4 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 03:56:40 +0700 Subject: [PATCH 3/4] A mutation run found the four-byte overlong nobody was testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relaxing 0xf0's second-byte floor from 0x90 to 0x80 left the whole suite green: every other row of the table had a case pinning it and that one did not, so f0 80 80 af decoded happily as "/". The same smuggled slash the two- and three-byte cases exist to catch, missed in the fourth width. Seven mutations verified red after this: the lead-byte floor at 0xc2 and the second-byte bounds on 0xe0, 0xed, 0xf0 and 0xf4, the truncated-sequence width, and the split cursor dropping its trailing empty field. An eighth, lower-ascii written as a bit-xor, is red on the bytes either side of the letters — which is why those are in the table and the letters alone are not. --- test/programs/utf8.flan | 4 ++++ test/test_acceptance.ml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/programs/utf8.flan b/test/programs/utf8.flan index 7e662a8..db5b269 100644 --- a/test/programs/utf8.flan +++ b/test/programs/utf8.flan @@ -28,6 +28,7 @@ (defconst overlong2 [2 u8] [0xc0 0xaf]) (defconst overlong3 [3 u8] [0xe0 0x80 0xaf]) +(defconst overlong4 [4 u8] [0xf0 0x80 0x80 0xaf]) (defconst surrogate [3 u8] [0xed 0xa0 0x80]) (defconst above-max [4 u8] [0xf4 0x90 0x80 0x80]) (defconst lead-f5 [4 u8] [0xf5 0x80 0x80 0x80]) @@ -101,6 +102,9 @@ (show-dec (slice lone-cont 0 1)) ; a continuation byte leading (show-dec (slice overlong2 0 2)) ; overlong "/" (show-dec (slice overlong3 0 3)) ; overlong "/" again, three bytes + (show-dec (slice overlong4 0 4)) ; and four. Added after a mutation run: + ; relaxing 0xf0's floor to 0x80 left the + ; whole suite green without this line. (show-dec (slice surrogate 0 3)) ; U+D800 (show-dec (slice above-max 0 4)) ; U+110000 (show-dec (slice lead-f5 0 4)) ; 0xf5 leads nothing diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index 1fb74f2..63e5ea8 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -904,7 +904,7 @@ ERR@7 unexpected token: not the kind the caller was reading shared. *) let utf8_out = "0/0/f 65/1/t 233/2/t 26085/3/t 128512/4/t \n\ - 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f \n\ + 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f \n\ 0/1/f 0/1/f 0/1/f 0/1/f \n\ tft\n\ 0 3 8 13 3\n\ From ecf3882fa28177892192b62b29a095b5d28160e8 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Sat, 12 Sep 2026 04:01:25 +0700 Subject: [PATCH 4/4] Refusing to encode is a claim about the buffer, and nothing was checking it encode-rune! says nothing is written when it answers None, and every None case in the table passed that claim without testing it: an encoder that lays the lead byte down and only then notices the buffer is short returns None exactly as a correct one does. So a known byte goes into scratch, a refused encoding is asked for, and the byte is read back. Storing before the length test turns the line from 65 -1 65 -1 65 into 65 -1 0 -1 0. The read-only claim beside lower-ascii was reasoned from the emitted linkage rather than observed, and observing it was worse than the guess. With (set (at (bytes "Hi") 0) \h): at -O0 the store is emitted against the constant and the program takes SIGSEGV; at -O2 LLVM deletes it as undefined behaviour and the program prints "Hi" and exits 0. The same source either dies or silently does nothing depending on a flag. The comment now says that instead of predicting a segfault. --- lib/prelude.ml | 21 ++++++++++++++++----- test/programs/utf8.flan | 13 +++++++++++++ test/test_acceptance.ml | 1 + 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/lib/prelude.ml b/lib/prelude.ml index 1e4e72a..988914f 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -637,11 +637,22 @@ let source = {flan| ;; to_upper both allocate a new string (core/strings/conversion.odin), which ;; is not available here; the obvious substitute — lowering a [u8] in place — ;; is a trap, and it is worth saying why rather than shipping it. A string -;; literal is emitted `private unnamed_addr constant` (emit.ml), so it lives -;; in read-only memory, and (bytes "Hello") is a [u8] pointing straight at it. -;; An in-place lower-ascii! would type check against that slice and segfault -;; on the store. Given a byte function, a caller that really does own its -;; buffer writes the two-line loop itself and can see what it is writing to. +;; literal is emitted `private unnamed_addr constant` (emit.ml), so (bytes +;; "Hello") is a [u8] pointing straight into read-only memory. An in-place +;; lower-ascii! type checks against that slice, and what happens next depends +;; on the optimiser — which is the worst of the available answers. Measured, +;; with (set (at (bytes "Hi") 0) \h): +;; +;; -O0 the store is emitted against the constant and the program takes +;; SIGSEGV. +;; -O2 LLVM deletes the store as undefined behaviour and the program +;; carries on and prints "Hi". +;; +;; So the same source either dies or silently does nothing depending on a +;; flag, and the -O2 half is the quiet-wrongness class this file keeps +;; refusing elsewhere. Given a byte function instead, a caller that really +;; does own its buffer writes the two-line loop itself over storage it can +;; see the declaration of. ;; ;; ASCII only, and only the 26 letters: case outside ASCII is not a byte ;; operation at all — it is per-code-point, it is not length-preserving (ß diff --git a/test/programs/utf8.flan b/test/programs/utf8.flan index db5b269..a45d2d2 100644 --- a/test/programs/utf8.flan +++ b/test/programs/utf8.flan @@ -190,6 +190,19 @@ (show-opt (encode-rune! (slice scratch 0 1) 0x41)) ; 1, exactly enough (newline) + ;; "Nothing is written when it refuses" is a claim about the buffer, not + ;; about the return value, and the None cases above do not test it: an + ;; encoder that lays down the lead byte and only then notices the buffer is + ;; short returns None exactly as this one does, and every line above still + ;; passes. So put a known byte in scratch, ask for an encoding that must be + ;; refused, and read the byte back. + (show-i32 (round-trip 0x41)) ; 65, scratch[0] = A + (show-opt (encode-rune! (slice scratch 0 2) 0x65e5)) ; -1, needs 3 bytes + (show-i32 (i32 (at scratch 0))) ; 65 still + (show-opt (encode-rune! (slice scratch 0 4) 0xd800)) ; -1, surrogate + (show-i32 (i32 (at scratch 0))) ; 65 still + (newline) + ;; Splitting. n separators give n+1 fields, always: an interior empty field ;; survives, a leading and a trailing one do too, and an input with no ;; separator at all is one field rather than none. The empty input is the diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index 63e5ea8..e47dc10 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -913,6 +913,7 @@ ERR@7 unexpected token: not the kind the caller was reading -1 1 1 2 2 3 3 -1 -1 3 3 4 4 -1 \n\ 0 65 127 128 2047 2048 65535 65536 1114111 \n\ -1 -1 -1 -1 -1 1 \n\ + 65 -1 65 -1 65 \n\ [a][b][c] [a][][b] [abc] [] [][] [][a] [a][] \n\ 60\n\ 97 122 97 64 91 65 90 65 96 123 53 \n\