diff --git a/lib/prelude.ml b/lib/prelude.ml index 7955089..988914f 100644 --- a/lib/prelude.ml +++ b/lib/prelude.ml @@ -407,6 +407,298 @@ let source = {flan| ;; Trailing junk is the case strtod is silent about, so the position has ;; to land exactly on the end. (if (= i (len s)) (Some (bytes->f64 s)) None))) + +;; ── UTF-8 ───────────────────────────────────────────────────────────── +;; +;; Ported from Odin's core/unicode/utf8/utf8.odin, which is the one corner of +;; a string library that is allocation-free by construction: decoding is +;; classification, and every answer it gives is a number. Everything else in +;; Odin's core/strings and all of core/fmt takes `allocator := +;; context.allocator`, and is therefore refused below rather than ported. +;; +;; Odin's 256-entry accept_sizes table becomes a cond over the lead byte here. +;; The table is the cache-friendly form and the cond is the one you can check +;; by reading, and nothing in a game decodes UTF-8 in a hot loop — DrawText +;; hands the bytes straight to raylib. +;; +;; The four rules that table encodes, and which a hand-written decoder gets +;; wrong one at a time: +;; +;; 0x80..0xc1 never a lead byte. 0x80..0xbf are continuation bytes, and +;; 0xc0 and 0xc1 could only ever begin an *overlong* two-byte +;; spelling of an ASCII character — the encoding that lets +;; "\xc0\xaf" smuggle a "/" past a check for one. +;; 0xe0 second byte 0xa0..0xbf and not 0x80..0xbf; the low half is +;; the overlong three-byte range. +;; 0xed second byte 0x80..0x9f. The high half is U+D800..U+DFFF, +;; the UTF-16 surrogates, which are not scalar values. +;; 0xf0, 0xf4 second byte 0x90..0xbf and 0x80..0x8f: overlong below, +;; and past U+10FFFF above. 0xf5..0xff lead nothing at all. +;; +;; A rune is an i32 and not a type of its own. That is Odin's answer too — +;; its `rune` is a four-byte integer distinguished only by a flag on the +;; basic-type row (src/types.cpp, the Basic_rune entry) — so nothing in the +;; checker has to learn a new type for any of this. + +;; One deliberate divergence from Odin, and it is the parse-i64 argument over +;; again. Odin's decode_rune answers RUNE_ERROR — U+FFFD — for malformed +;; bytes, and U+FFFD is a perfectly real code point that a well-formed string +;; may contain, so a caller cannot tell a decoded replacement character from a +;; failure to decode. This carries `ok` instead, and leaves `code` 0 when it +;; is false. +;; +;; `width` is 1 on a malformed byte and 0 only for an empty input. That is +;; Odin's rule and it is load-bearing rather than cosmetic: every loop below +;; advances by `width`, so a 0 there on a bad byte is an infinite loop, not a +;; wrong number. +(defstruct Rune [code i32 width i32 ok bool]) + +(defn rune-start? [b u8] bool + (!= (bit-and b 0xc0) 0x80)) + +(defn decode-rune [s [u8]] Rune + (when (= (len s) 0) + (return (Rune {:code 0 :width 0 :ok false}))) + (let [b0 (at s 0)] + (when (< b0 0x80) + (return (Rune {:code (i32 b0) :width 1 :ok true}))) + ;; size 0 means "this byte cannot lead"; lo/hi are the *second* byte's + ;; accepted range, which is the only place the overlong and surrogate + ;; rules live. Bytes three and four are always 0x80..0xbf. + (let [size 0 + lo (u8 0x80) + hi (u8 0xbf)] + (cond + (< b0 0xc2) (set size 0) + (<= b0 0xdf) (set size 2) + (= b0 0xe0) (do (set size 3) (set lo (u8 0xa0))) + (<= b0 0xec) (set size 3) + (= b0 0xed) (do (set size 3) (set hi (u8 0x9f))) + (<= b0 0xef) (set size 3) + (= b0 0xf0) (do (set size 4) (set lo (u8 0x90))) + (<= b0 0xf3) (set size 4) + (= b0 0xf4) (do (set size 4) (set hi (u8 0x8f))) + :else (set size 0)) + (when (= size 0) + (return (Rune {:code 0 :width 1 :ok false}))) + ;; A sequence cut off by the end of the slice. Width 1, so a caller + ;; scanning a buffer boundary makes progress instead of stalling. + (when (> size (len s)) + (return (Rune {:code 0 :width 1 :ok false}))) + (let [b1 (at s 1)] + (when (or (< b1 lo) (> b1 hi)) + (return (Rune {:code 0 :width 1 :ok false}))) + (when (= size 2) + (return (Rune {:code (bit-or (<< (i32 (bit-and b0 0x1f)) 6) + (i32 (bit-and b1 0x3f))) + :width 2 :ok true}))) + (let [b2 (at s 2)] + (when (or (< b2 0x80) (> b2 0xbf)) + (return (Rune {:code 0 :width 1 :ok false}))) + (when (= size 3) + (return (Rune {:code (bit-or (bit-or (<< (i32 (bit-and b0 0x0f)) 12) + (<< (i32 (bit-and b1 0x3f)) 6)) + (i32 (bit-and b2 0x3f))) + :width 3 :ok true}))) + (let [b3 (at s 3)] + (when (or (< b3 0x80) (> b3 0xbf)) + (return (Rune {:code 0 :width 1 :ok false}))) + (Rune {:code (bit-or (bit-or (<< (i32 (bit-and b0 0x07)) 18) + (bit-or (<< (i32 (bit-and b1 0x3f)) 12) + (<< (i32 (bit-and b2 0x3f)) 6))) + (i32 (bit-and b3 0x3f))) + :width 4 :ok true}))))))) + +;; Decode at a byte offset. None when the offset is not on a rune boundary or +;; the bytes there are malformed, which is stricter than Odin's rune_at — that +;; one hands back RUNE_ERROR and the caller carries on with a wrong character. +(defn rune-at [s [u8] i i32] (Option i32) + (if (or (< i 0) (>= i (len s))) + None + (let [r (decode-rune (slice s i (len s)))] + (if (.ok r) (Some (.code r)) None)))) + +;; Counted through decode-rune rather than through a second walk of its own. +;; Odin keeps a separate rune_count_in_bytes that re-implements the size +;; table; two copies of that classification is two places for the surrogate +;; rule to be right in only one of them. +;; +;; A malformed byte counts as one, which is what a replacement-character +;; renderer would draw, so this agrees with what the screen shows. +(defn rune-count [s [u8]] i32 + (let [i 0 + n 0] + (while (< i (len s)) + (let [r (decode-rune (slice s i (len s)))] + (set i (+ i (.width r))) + (set n (+ n 1)))) + n)) + +(defn valid-utf8? [s [u8]] bool + (let [i 0] + (while (< i (len s)) + (let [r (decode-rune (slice s i (len s)))] + (when (not (.ok r)) + (return false)) + (set i (+ i (.width r))))) + true)) + +;; How many bytes this code point encodes to, or None if it is not a scalar +;; value. Odin's rune_size answers -1 for the refusals; a sentinel index is +;; exactly what index-of-i32 avoids above, so this is an Option like the rest +;; of the file. +(defn rune-size [code i32] (Option i32) + (cond + (< code 0) None + (<= code 0x7f) (Some 1) + (<= code 0x7ff) (Some 2) + (and (>= code 0xd800) (<= code 0xdfff)) None + (<= code 0xffff) (Some 3) + (<= code 0x10ffff) (Some 4) + :else None)) + +;; Encoding is the one operation here whose result is not a slice of its +;; input, because the bytes it makes existed nowhere before. With no allocator +;; the only shape left is Odin's own allocation-free one — strings.Builder +;; built by builder_from_bytes over a caller's backing array (builder.odin, +;; builder_from_bytes: "Uses Nil Allocator - Does NOT allocate") — reduced to +;; its essential case: write into a buffer the caller owns, and say how much +;; was written. +;; +;; None rather than a partial write when the buffer is short, and None rather +;; than Odin's silent substitution of U+FFFD for an invalid rune. Odin's +;; encode_rune rewrites a surrogate or an out-of-range value to the +;; replacement character and reports success; the caller then finds three +;; bytes of U+FFFD in its buffer and no indication that it asked for something +;; else. Nothing is written at all when this answers None. +(defn encode-rune! [dst [u8] code i32] (Option i32) + (match (rune-size code) + None None + (Some w) + (if (> w (len dst)) + None + (do + (cond + (= w 1) + (set (at dst 0) (u8 code)) + (= w 2) + (do (set (at dst 0) (u8 (bit-or 0xc0 (>> code 6)))) + (set (at dst 1) (u8 (bit-or 0x80 (bit-and code 0x3f))))) + (= w 3) + (do (set (at dst 0) (u8 (bit-or 0xe0 (>> code 12)))) + (set (at dst 1) (u8 (bit-or 0x80 (bit-and (>> code 6) 0x3f)))) + (set (at dst 2) (u8 (bit-or 0x80 (bit-and code 0x3f))))) + :else + (do (set (at dst 0) (u8 (bit-or 0xf0 (>> code 18)))) + (set (at dst 1) (u8 (bit-or 0x80 (bit-and (>> code 12) 0x3f)))) + (set (at dst 2) (u8 (bit-or 0x80 (bit-and (>> code 6) 0x3f)))) + (set (at dst 3) (u8 (bit-or 0x80 (bit-and code 0x3f)))))) + (Some w))))) + +;; ── Splitting ───────────────────────────────────────────────────────── +;; +;; `split` returning a sequence of fields must allocate the sequence, and +;; there is no allocator — so it is refused by name at the bottom of this +;; file, and this is the shape that survives. It is Odin's +;; split_by_byte_iterator (strings.odin): a cursor holding the rest of the +;; input, handing back one field at a time. Every field is a slice *of the +;; caller's bytes*; nothing is copied and nothing is owned. +;; +;; One divergence, and it is a wart of Odin's rather than a decision. Odin's +;; iterator stops on an empty final field, so "a,b," iterates a and b and the +;; trailing empty field is lost — while Odin's own allocating strings.split +;; returns ["a", "b", ""] for the same input. The two disagree. This follows +;; split: n separators always yield n+1 fields, an empty input yields one +;; empty field, and `rest` is exhausted only after the last one is taken. That +;; is the rule you can state without exceptions, and the one a caller counting +;; comma-separated columns needs. +(defstruct Split [rest [u8] sep u8 more bool]) + +(defn split-on-byte [s [u8] sep u8] Split + (Split {:rest s :sep sep :more true})) + +(defn split-next! [it (Ptr Split)] (Option [u8]) + (when (not (.more it)) + (return None)) + (match (index-of-byte (.rest it) (.sep it)) + (Some i) + (let [field (slice (.rest it) 0 i)] + (set (.rest it) (slice (.rest it) (+ i 1) (len (.rest it)))) + (Some field)) + None + (let [field (.rest it)] + (set (.more it) false) + (set (.rest it) (slice (.rest it) (len (.rest it)) (len (.rest it)))) + (Some field)))) + +;; ── ASCII case ──────────────────────────────────────────────────────── +;; +;; Byte in, byte out, and *not* a function over a slice. Odin's to_lower and +;; to_upper both allocate a new string (core/strings/conversion.odin), which +;; is not available here; the obvious substitute — lowering a [u8] in place — +;; is a trap, and it is worth saying why rather than shipping it. A string +;; literal is emitted `private unnamed_addr constant` (emit.ml), so (bytes +;; "Hello") is a [u8] pointing straight into read-only memory. An in-place +;; lower-ascii! type checks against that slice, and what happens next depends +;; on the optimiser — which is the worst of the available answers. Measured, +;; with (set (at (bytes "Hi") 0) \h): +;; +;; -O0 the store is emitted against the constant and the program takes +;; SIGSEGV. +;; -O2 LLVM deletes the store as undefined behaviour and the program +;; carries on and prints "Hi". +;; +;; So the same source either dies or silently does nothing depending on a +;; flag, and the -O2 half is the quiet-wrongness class this file keeps +;; refusing elsewhere. Given a byte function instead, a caller that really +;; does own its buffer writes the two-line loop itself over storage it can +;; see the declaration of. +;; +;; ASCII only, and only the 26 letters: case outside ASCII is not a byte +;; operation at all — it is per-code-point, it is not length-preserving (ß +;; upcases to SS), and it is locale-dependent (Turkish dotless ı). A byte +;; table that pretended otherwise would be wrong in the quiet way. +(defn lower-ascii [b u8] u8 + (if (and (>= b \A) (<= b \Z)) (+ b 32) b)) + +(defn upper-ascii [b u8] u8 + (if (and (>= b \a) (<= b \z)) (- b 32) b)) + +;; Case-insensitive comparison as a fold over both inputs, which is the useful +;; half of to_lower and needs no storage at all: comparing two lowered copies +;; is what a caller wanted, and this is that answer without either copy. +(defn bytes-ci=? [a [u8] b [u8]] bool + (if (!= (len a) (len b)) + false + (do + (dotimes [i (len a)] + (when (!= (lower-ascii (at a i)) (lower-ascii (at b i))) + (return false))) + true))) + +;; ── Refused, by name ────────────────────────────────────────────────── +;; +;; Every one of these needs to produce bytes that did not exist in its input, +;; and there is no allocator, so each is absent rather than approximated. +;; None of them is hard to write once `(Vec u8)` and an allocator exist; all +;; of them are impossible to write honestly today. +;; +;; join, concat build one buffer out of several inputs. +;; to-lower, to-upper a new string, per Odin's conversion.odin. The +;; byte-wise and folding-comparison forms above are +;; what is available without one. +;; split the *sequence* of fields is itself an allocation. +;; split-on-byte / split-next! above is the same +;; information with no sequence to own. +;; replace, repeat, pad same reason as join. +;; string-from-bytes a [u8] cannot become a `string` here even though +;; the layouts are identical; see the report. +;; format, sprintf Odin's fmt.aprintf family, all allocating. +;; Builder strings.Builder is (defstruct Builder [buf +;; (Vec u8)]), which spec-memory.md already makes +;; move-only by the rule that a struct containing a +;; Vec is move-only. It needs the Vec, not a spec +;; change. |flan} let file = "" diff --git a/test/programs/utf8.flan b/test/programs/utf8.flan new file mode 100644 index 0000000..a45d2d2 --- /dev/null +++ b/test/programs/utf8.flan @@ -0,0 +1,265 @@ +;;;; UTF-8 decoding and encoding, the split cursor, and ASCII case. +;;;; +;;;; Every case here is one a plausible wrong decoder passes. A decoder that +;;;; only masks and shifts — takes the top bits of the lead byte for a length +;;;; and the low six of everything after — gets all of the *valid* input right +;;;; and every line of the second group wrong, so the valid decodes prove +;;;; almost nothing on their own and the malformed ones are the test. +;;;; +;;;; The four that matter, each isolating one row of Odin's accept_sizes +;;;; table: +;;;; +;;;; c0 af an overlong two-byte "/". Accepted, it is the encoding +;;;; that smuggles a slash past a check for one. +;;;; e0 80 af an overlong three-byte "/", which the raised second-byte +;;;; floor on 0xe0 is the only thing rejecting. +;;;; ed a0 80 U+D800, a UTF-16 surrogate, which is not a scalar value. +;;;; Only the lowered ceiling on 0xed rejects it. +;;;; f4 90 80 80 U+110000, one past the last code point. +;;;; +;;;; And three more shapes: a lone continuation byte, a lead byte that leads +;;;; nothing (0xf5), and a sequence truncated by the end of the slice — the +;;;; last taken as a slice of a *valid* literal, which is what reading a +;;;; buffer boundary actually hands you. +;;;; +;;;; The invalid sequences are byte arrays because the reader has no \xNN +;;;; escape in either a string or a character literal, and none of them can be +;;;; derived from a valid string. + +(defconst overlong2 [2 u8] [0xc0 0xaf]) +(defconst overlong3 [3 u8] [0xe0 0x80 0xaf]) +(defconst overlong4 [4 u8] [0xf0 0x80 0x80 0xaf]) +(defconst surrogate [3 u8] [0xed 0xa0 0x80]) +(defconst above-max [4 u8] [0xf4 0x90 0x80 0x80]) +(defconst lead-f5 [4 u8] [0xf5 0x80 0x80 0x80]) +(defconst lone-cont [1 u8] [0x80]) +(defconst emoji [4 u8] [0xf0 0x9f 0x98 0x80]) ; U+1F600 +(defconst bad-tail [3 u8] [0x61 0xff 0x62]) ; "a", junk, "b" + +(defvar scratch [4 u8]) + +;; Each array is sliced at the point of use rather than through a +;; (defn whole [a [4 u8]] [u8] (slice a 0 4)) helper. That helper is a +;; use-after-return and the compiler accepts it in silence: a [n T] parameter +;; is a *value* and copies into the callee's frame, so the slice it hands back +;; points at a frame that has already gone. It was written here first, and the +;; emoji line is what caught it — it decoded as malformed because it was +;; reading whatever the next call left on the stack. This is the escaping +;; borrow spec-memory.md leaves to the programmer under "Borrowing", and it is +;; the case a future provenance pass would reject. + +;; code/width/ok, so a wrong answer names which of the three it got wrong +;; rather than just failing. +(defn show-dec [s [u8]] + (let [r (decode-rune s)] + (print-i64 (i64 (.code r))) (print-str "/") + (print-i64 (i64 (.width r))) (print-str "/") + (print-str (if (.ok r) "t" "f")) + (print-str " "))) + +(defn show-bool [b bool] + (print-str (if b "t" "f"))) + +(defn show-opt [o (Option i32)] + (print-i64 (i64 (match o (Some v) v None -1))) + (print-str " ")) + +;; Encode into the scratch buffer and decode straight back out of it. A round +;; trip is the only check that catches an encoder and a decoder that are +;; wrong in the same direction — printing the bytes would not. +(defn round-trip [code i32] i32 + (match (encode-rune! (slice scratch 0 4) code) + None -1 + (Some w) + (let [r (decode-rune (slice scratch 0 w))] + (if (and (.ok r) (= (.width r) w)) (.code r) -1)))) + +(defn show-i32 [x i32] + (print-i64 (i64 x)) + (print-str " ")) + +(defn show-split [s [u8] sep u8] + (let [it (split-on-byte s sep) + going true] + (while going + (match (split-next! (addr it)) + (Some f) (do (print-str "[") (print-bytes f) (print-str "]")) + None (set going false))) + (print-str " "))) + +(defn main [] i32 + ;; Valid, one of each width. The empty slice is width 0 — the only input + ;; that gets a 0, because every loop below advances by width and a 0 on a + ;; malformed byte would hang instead of answering. + (show-dec (bytes "")) ; 0/0/f + (show-dec (bytes "A")) ; 65/1/t + (show-dec (bytes "é")) ; 233/2/t + (show-dec (bytes "日")) ; 26085/3/t + (show-dec (slice emoji 0 4)) ; 128512/4/t + (newline) + + ;; Malformed. Every one is 0/1/f: width 1 so a scan makes progress. + (show-dec (slice lone-cont 0 1)) ; a continuation byte leading + (show-dec (slice overlong2 0 2)) ; overlong "/" + (show-dec (slice overlong3 0 3)) ; overlong "/" again, three bytes + (show-dec (slice overlong4 0 4)) ; and four. Added after a mutation run: + ; relaxing 0xf0's floor to 0x80 left the + ; whole suite green without this line. + (show-dec (slice surrogate 0 3)) ; U+D800 + (show-dec (slice above-max 0 4)) ; U+110000 + (show-dec (slice lead-f5 0 4)) ; 0xf5 leads nothing + (newline) + + ;; Truncated: a valid character cut short by the end of the slice, at both + ;; possible cut points, and the interior of one taken on its own. + (show-dec (slice (bytes "日") 0 1)) ; lead byte alone + (show-dec (slice (bytes "日") 0 2)) ; lead plus one continuation + (show-dec (slice (bytes "日") 1 3)) ; starts mid-character + (show-dec (slice (bytes "é") 1 2)) ; a lone continuation from a literal + (newline) + + ;; rune-start? is what a caller scans backwards with. + (show-bool (rune-start? (at (bytes "日") 0))) + (show-bool (rune-start? (at (bytes "日") 1))) + (show-bool (rune-start? \A)) + (newline) + + ;; Counting. The empty string is 0 and not 1; the mixed string is 8 runes + ;; in 13 bytes, which is the whole distinction; and a malformed byte counts + ;; as one, so a count never disagrees with what a renderer would draw. + (print-i64 (i64 (rune-count (bytes "")))) (print-str " ") + (print-i64 (i64 (rune-count (bytes "abc")))) (print-str " ") + (print-i64 (i64 (rune-count (bytes "héllo 日本")))) (print-str " ") + (print-i64 (i64 (len (bytes "héllo 日本")))) (print-str " ") + (print-i64 (i64 (rune-count (slice bad-tail 0 3)))) + (newline) + + (show-bool (valid-utf8? (bytes ""))) + (show-bool (valid-utf8? (bytes "héllo 日本"))) + (show-bool (valid-utf8? (slice surrogate 0 3))) + (show-bool (valid-utf8? (slice overlong2 0 2))) + (show-bool (valid-utf8? (slice bad-tail 0 3))) + (show-bool (valid-utf8? (slice emoji 0 4))) + (newline) + + ;; rune-at: on a boundary, off a boundary, and out of range. Off a boundary + ;; is None rather than a replacement character, which is where this is + ;; stricter than Odin's rune_at. + (show-opt (rune-at (bytes "日本") 0)) ; 26085 + (show-opt (rune-at (bytes "日本") 3)) ; 26412 + (show-opt (rune-at (bytes "日本") 1)) ; -1, mid-character + (show-opt (rune-at (bytes "日本") 6)) ; -1, past the end + (show-opt (rune-at (bytes "") 0)) ; -1 + (newline) + + ;; rune-size, at every boundary and on both sides of it. + (show-opt (rune-size -1)) + (show-opt (rune-size 0)) + (show-opt (rune-size 0x7f)) + (show-opt (rune-size 0x80)) + (show-opt (rune-size 0x7ff)) + (show-opt (rune-size 0x800)) + (show-opt (rune-size 0xd7ff)) + (show-opt (rune-size 0xd800)) + (show-opt (rune-size 0xdfff)) + (show-opt (rune-size 0xe000)) + (show-opt (rune-size 0xffff)) + (show-opt (rune-size 0x10000)) + (show-opt (rune-size 0x10ffff)) + (show-opt (rune-size 0x110000)) + (newline) + + ;; Round trips, one per width and at the boundaries. + (show-i32 (round-trip 0)) + (show-i32 (round-trip 0x41)) + (show-i32 (round-trip 0x7f)) + (show-i32 (round-trip 0x80)) + (show-i32 (round-trip 0x7ff)) + (show-i32 (round-trip 0x800)) + (show-i32 (round-trip 0xffff)) + (show-i32 (round-trip 0x10000)) + (show-i32 (round-trip 0x10ffff)) + (newline) + + ;; Refused by encode-rune!, and nothing is written when it refuses. + (show-opt (encode-rune! (slice scratch 0 4) 0xd800)) ; -1, surrogate + (show-opt (encode-rune! (slice scratch 0 4) 0x110000)) ; -1, past the end + (show-opt (encode-rune! (slice scratch 0 4) -1)) ; -1, negative + (show-opt (encode-rune! (slice scratch 0 2) 0x65e5)) ; -1, buffer short + (show-opt (encode-rune! (slice scratch 0 0) 0x41)) ; -1, no room at all + (show-opt (encode-rune! (slice scratch 0 1) 0x41)) ; 1, exactly enough + (newline) + + ;; "Nothing is written when it refuses" is a claim about the buffer, not + ;; about the return value, and the None cases above do not test it: an + ;; encoder that lays down the lead byte and only then notices the buffer is + ;; short returns None exactly as this one does, and every line above still + ;; passes. So put a known byte in scratch, ask for an encoding that must be + ;; refused, and read the byte back. + (show-i32 (round-trip 0x41)) ; 65, scratch[0] = A + (show-opt (encode-rune! (slice scratch 0 2) 0x65e5)) ; -1, needs 3 bytes + (show-i32 (i32 (at scratch 0))) ; 65 still + (show-opt (encode-rune! (slice scratch 0 4) 0xd800)) ; -1, surrogate + (show-i32 (i32 (at scratch 0))) ; 65 still + (newline) + + ;; Splitting. n separators give n+1 fields, always: an interior empty field + ;; survives, a leading and a trailing one do too, and an input with no + ;; separator at all is one field rather than none. The empty input is the + ;; case Odin's own iterator disagrees with its allocating split on — it is + ;; one empty field here. + (show-split (bytes "a,b,c") \,) ; [a][b][c] + (show-split (bytes "a,,b") \,) ; [a][][b] + (show-split (bytes "abc") \,) ; [abc] + (show-split (bytes "") \,) ; [] + (show-split (bytes ",") \,) ; [][] + (show-split (bytes ",a") \,) ; [][a] + (show-split (bytes "a,") \,) ; [a][] + (newline) + + ;; A field is a slice of the input, so trim and parse-i64 work straight off + ;; one with nothing copied in between — which is the entire reason the + ;; cursor shape exists. + (let [it (split-on-byte (bytes " 10 , 20 ,30") \,) + total (i64 0) + going true] + (while going + (match (split-next! (addr it)) + (Some f) (set total (+ total (match (parse-i64 (trim f)) (Some v) v None 0))) + None (set going false))) + (print-i64 total) + (newline)) + + ;; ASCII case. The boundary bytes on both sides of each range are what a + ;; wrong mask gets wrong: '@' and '[' sit either side of A-Z, and '`' and + ;; '{' either side of a-z, so a conversion written as (bit-xor b 32) — + ;; which works for every letter — turns '@' into '`' and is caught here. + (show-i32 (i32 (lower-ascii \A))) + (show-i32 (i32 (lower-ascii \Z))) + (show-i32 (i32 (lower-ascii \a))) + (show-i32 (i32 (lower-ascii \@))) ; 64, just below 'A' + (show-i32 (i32 (lower-ascii \[))) ; 91, just above 'Z' + (show-i32 (i32 (upper-ascii \a))) + (show-i32 (i32 (upper-ascii \z))) + (show-i32 (i32 (upper-ascii \A))) + (show-i32 (i32 (upper-ascii \`))) ; 96, just below 'a' + (show-i32 (i32 (upper-ascii \{))) ; 123, just above 'z' + (show-i32 (i32 (lower-ascii \5))) ; digits are untouched + (newline) + + ;; A non-ASCII byte must pass through both untouched, which is the claim + ;; that "ASCII only" is a rule and not an oversight. + (show-i32 (i32 (lower-ascii (at (bytes "é") 0)))) + (show-i32 (i32 (upper-ascii (at (bytes "é") 0)))) + (newline) + + (show-bool (bytes-ci=? (bytes "Hello") (bytes "hELLO"))) ; t + (show-bool (bytes-ci=? (bytes "Hello") (bytes "hello!"))) ; f length first + (show-bool (bytes-ci=? (bytes "") (bytes ""))) ; t + (show-bool (bytes-ci=? (bytes "a") (bytes "b"))) ; f + ;; '@' is 'A'+32 apart from '`' the way a letter is from its own case, so a + ;; fold written as a bit-xor would call these two equal. They are not. + (show-bool (bytes-ci=? (bytes "@") (bytes "`"))) ; f + (show-bool (bytes-ci=? (bytes "é") (bytes "é"))) ; t bytes match + (newline) + 0) diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index f5f8b1d..21ac026 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -1544,6 +1544,49 @@ ERR@7 unexpected token: not the kind the caller was reading end else print_endline "acceptance: lldb cases skipped (no lldb on PATH)"; + (* ── Strings: UTF-8, the split cursor, and ASCII case ────────────── + + The prelude's port of Odin's core/unicode/utf8, which is the only part + of a string library that needs no allocator. The valid decodes prove + almost nothing on their own — a decoder that just masks and shifts gets + every one of them right — so the test is the malformed group: an + overlong two- and three-byte "/", a UTF-16 surrogate, a code point past + U+10FFFF, a lead byte that leads nothing, a lone continuation byte, and + a valid character truncated by the end of its slice. Each isolates one + row of the accept_sizes table, and each answers width 1 so that a scan + makes progress rather than hanging. + + Encoding is checked by round trip rather than against expected bytes, + because an encoder and a decoder that are wrong in the same direction + agree with each other and disagree with nothing else. + + At -O0 as well, for the reason the slice algorithms run there: a slice + is a two-word struct through an alloca, decode-rune returns a struct by + value, and the split cursor is mutated through a (Ptr Split) — mem2reg + is exactly what would hide any of those being copied when it should be + shared. *) + let utf8_out = + "0/0/f 65/1/t 233/2/t 26085/3/t 128512/4/t \n\ + 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f \n\ + 0/1/f 0/1/f 0/1/f 0/1/f \n\ + tft\n\ + 0 3 8 13 3\n\ + ttffft\n\ + 26085 26412 -1 -1 -1 \n\ + -1 1 1 2 2 3 3 -1 -1 3 3 4 4 -1 \n\ + 0 65 127 128 2047 2048 65535 65536 1114111 \n\ + -1 -1 -1 -1 -1 1 \n\ + 65 -1 65 -1 65 \n\ + [a][b][c] [a][][b] [abc] [] [][] [][a] [a][] \n\ + 60\n\ + 97 122 97 64 91 65 90 65 96 123 53 \n\ + 195 195 \n\ + tftfft\n" + in + outputs "utf-8, splitting and ascii case" "programs/utf8.flan" utf8_out; + outputs ~opt:"-O0" "utf-8, splitting and ascii case, -O0" + "programs/utf8.flan" utf8_out; + if !failures = 0 then print_endline "acceptance: all tests passed" else begin Printf.printf "\n%d failure(s)\n" !failures;