Merge branch 'strings-odin' into dev-loop
# Conflicts: # test/test_acceptance.ml
This commit is contained in:
commit
ee5abd40fc
292
lib/prelude.ml
292
lib/prelude.ml
@ -407,6 +407,298 @@ let source = {flan|
|
||||
;; Trailing junk is the case strtod is silent about, so the position has
|
||||
;; to land exactly on the end.
|
||||
(if (= i (len s)) (Some (bytes->f64 s)) None)))
|
||||
|
||||
;; ── UTF-8 ─────────────────────────────────────────────────────────────
|
||||
;;
|
||||
;; Ported from Odin's core/unicode/utf8/utf8.odin, which is the one corner of
|
||||
;; a string library that is allocation-free by construction: decoding is
|
||||
;; classification, and every answer it gives is a number. Everything else in
|
||||
;; Odin's core/strings and all of core/fmt takes `allocator :=
|
||||
;; context.allocator`, and is therefore refused below rather than ported.
|
||||
;;
|
||||
;; Odin's 256-entry accept_sizes table becomes a cond over the lead byte here.
|
||||
;; The table is the cache-friendly form and the cond is the one you can check
|
||||
;; by reading, and nothing in a game decodes UTF-8 in a hot loop — DrawText
|
||||
;; hands the bytes straight to raylib.
|
||||
;;
|
||||
;; The four rules that table encodes, and which a hand-written decoder gets
|
||||
;; wrong one at a time:
|
||||
;;
|
||||
;; 0x80..0xc1 never a lead byte. 0x80..0xbf are continuation bytes, and
|
||||
;; 0xc0 and 0xc1 could only ever begin an *overlong* two-byte
|
||||
;; spelling of an ASCII character — the encoding that lets
|
||||
;; "\xc0\xaf" smuggle a "/" past a check for one.
|
||||
;; 0xe0 second byte 0xa0..0xbf and not 0x80..0xbf; the low half is
|
||||
;; the overlong three-byte range.
|
||||
;; 0xed second byte 0x80..0x9f. The high half is U+D800..U+DFFF,
|
||||
;; the UTF-16 surrogates, which are not scalar values.
|
||||
;; 0xf0, 0xf4 second byte 0x90..0xbf and 0x80..0x8f: overlong below,
|
||||
;; and past U+10FFFF above. 0xf5..0xff lead nothing at all.
|
||||
;;
|
||||
;; A rune is an i32 and not a type of its own. That is Odin's answer too —
|
||||
;; its `rune` is a four-byte integer distinguished only by a flag on the
|
||||
;; basic-type row (src/types.cpp, the Basic_rune entry) — so nothing in the
|
||||
;; checker has to learn a new type for any of this.
|
||||
|
||||
;; One deliberate divergence from Odin, and it is the parse-i64 argument over
|
||||
;; again. Odin's decode_rune answers RUNE_ERROR — U+FFFD — for malformed
|
||||
;; bytes, and U+FFFD is a perfectly real code point that a well-formed string
|
||||
;; may contain, so a caller cannot tell a decoded replacement character from a
|
||||
;; failure to decode. This carries `ok` instead, and leaves `code` 0 when it
|
||||
;; is false.
|
||||
;;
|
||||
;; `width` is 1 on a malformed byte and 0 only for an empty input. That is
|
||||
;; Odin's rule and it is load-bearing rather than cosmetic: every loop below
|
||||
;; advances by `width`, so a 0 there on a bad byte is an infinite loop, not a
|
||||
;; wrong number.
|
||||
(defstruct Rune [code i32 width i32 ok bool])
|
||||
|
||||
(defn rune-start? [b u8] bool
|
||||
(!= (bit-and b 0xc0) 0x80))
|
||||
|
||||
(defn decode-rune [s [u8]] Rune
|
||||
(when (= (len s) 0)
|
||||
(return (Rune {:code 0 :width 0 :ok false})))
|
||||
(let [b0 (at s 0)]
|
||||
(when (< b0 0x80)
|
||||
(return (Rune {:code (i32 b0) :width 1 :ok true})))
|
||||
;; size 0 means "this byte cannot lead"; lo/hi are the *second* byte's
|
||||
;; accepted range, which is the only place the overlong and surrogate
|
||||
;; rules live. Bytes three and four are always 0x80..0xbf.
|
||||
(let [size 0
|
||||
lo (u8 0x80)
|
||||
hi (u8 0xbf)]
|
||||
(cond
|
||||
(< b0 0xc2) (set size 0)
|
||||
(<= b0 0xdf) (set size 2)
|
||||
(= b0 0xe0) (do (set size 3) (set lo (u8 0xa0)))
|
||||
(<= b0 0xec) (set size 3)
|
||||
(= b0 0xed) (do (set size 3) (set hi (u8 0x9f)))
|
||||
(<= b0 0xef) (set size 3)
|
||||
(= b0 0xf0) (do (set size 4) (set lo (u8 0x90)))
|
||||
(<= b0 0xf3) (set size 4)
|
||||
(= b0 0xf4) (do (set size 4) (set hi (u8 0x8f)))
|
||||
:else (set size 0))
|
||||
(when (= size 0)
|
||||
(return (Rune {:code 0 :width 1 :ok false})))
|
||||
;; A sequence cut off by the end of the slice. Width 1, so a caller
|
||||
;; scanning a buffer boundary makes progress instead of stalling.
|
||||
(when (> size (len s))
|
||||
(return (Rune {:code 0 :width 1 :ok false})))
|
||||
(let [b1 (at s 1)]
|
||||
(when (or (< b1 lo) (> b1 hi))
|
||||
(return (Rune {:code 0 :width 1 :ok false})))
|
||||
(when (= size 2)
|
||||
(return (Rune {:code (bit-or (<< (i32 (bit-and b0 0x1f)) 6)
|
||||
(i32 (bit-and b1 0x3f)))
|
||||
:width 2 :ok true})))
|
||||
(let [b2 (at s 2)]
|
||||
(when (or (< b2 0x80) (> b2 0xbf))
|
||||
(return (Rune {:code 0 :width 1 :ok false})))
|
||||
(when (= size 3)
|
||||
(return (Rune {:code (bit-or (bit-or (<< (i32 (bit-and b0 0x0f)) 12)
|
||||
(<< (i32 (bit-and b1 0x3f)) 6))
|
||||
(i32 (bit-and b2 0x3f)))
|
||||
:width 3 :ok true})))
|
||||
(let [b3 (at s 3)]
|
||||
(when (or (< b3 0x80) (> b3 0xbf))
|
||||
(return (Rune {:code 0 :width 1 :ok false})))
|
||||
(Rune {:code (bit-or (bit-or (<< (i32 (bit-and b0 0x07)) 18)
|
||||
(bit-or (<< (i32 (bit-and b1 0x3f)) 12)
|
||||
(<< (i32 (bit-and b2 0x3f)) 6)))
|
||||
(i32 (bit-and b3 0x3f)))
|
||||
:width 4 :ok true})))))))
|
||||
|
||||
;; Decode at a byte offset. None when the offset is not on a rune boundary or
|
||||
;; the bytes there are malformed, which is stricter than Odin's rune_at — that
|
||||
;; one hands back RUNE_ERROR and the caller carries on with a wrong character.
|
||||
(defn rune-at [s [u8] i i32] (Option i32)
|
||||
(if (or (< i 0) (>= i (len s)))
|
||||
None
|
||||
(let [r (decode-rune (slice s i (len s)))]
|
||||
(if (.ok r) (Some (.code r)) None))))
|
||||
|
||||
;; Counted through decode-rune rather than through a second walk of its own.
|
||||
;; Odin keeps a separate rune_count_in_bytes that re-implements the size
|
||||
;; table; two copies of that classification is two places for the surrogate
|
||||
;; rule to be right in only one of them.
|
||||
;;
|
||||
;; A malformed byte counts as one, which is what a replacement-character
|
||||
;; renderer would draw, so this agrees with what the screen shows.
|
||||
(defn rune-count [s [u8]] i32
|
||||
(let [i 0
|
||||
n 0]
|
||||
(while (< i (len s))
|
||||
(let [r (decode-rune (slice s i (len s)))]
|
||||
(set i (+ i (.width r)))
|
||||
(set n (+ n 1))))
|
||||
n))
|
||||
|
||||
(defn valid-utf8? [s [u8]] bool
|
||||
(let [i 0]
|
||||
(while (< i (len s))
|
||||
(let [r (decode-rune (slice s i (len s)))]
|
||||
(when (not (.ok r))
|
||||
(return false))
|
||||
(set i (+ i (.width r)))))
|
||||
true))
|
||||
|
||||
;; How many bytes this code point encodes to, or None if it is not a scalar
|
||||
;; value. Odin's rune_size answers -1 for the refusals; a sentinel index is
|
||||
;; exactly what index-of-i32 avoids above, so this is an Option like the rest
|
||||
;; of the file.
|
||||
(defn rune-size [code i32] (Option i32)
|
||||
(cond
|
||||
(< code 0) None
|
||||
(<= code 0x7f) (Some 1)
|
||||
(<= code 0x7ff) (Some 2)
|
||||
(and (>= code 0xd800) (<= code 0xdfff)) None
|
||||
(<= code 0xffff) (Some 3)
|
||||
(<= code 0x10ffff) (Some 4)
|
||||
:else None))
|
||||
|
||||
;; Encoding is the one operation here whose result is not a slice of its
|
||||
;; input, because the bytes it makes existed nowhere before. With no allocator
|
||||
;; the only shape left is Odin's own allocation-free one — strings.Builder
|
||||
;; built by builder_from_bytes over a caller's backing array (builder.odin,
|
||||
;; builder_from_bytes: "Uses Nil Allocator - Does NOT allocate") — reduced to
|
||||
;; its essential case: write into a buffer the caller owns, and say how much
|
||||
;; was written.
|
||||
;;
|
||||
;; None rather than a partial write when the buffer is short, and None rather
|
||||
;; than Odin's silent substitution of U+FFFD for an invalid rune. Odin's
|
||||
;; encode_rune rewrites a surrogate or an out-of-range value to the
|
||||
;; replacement character and reports success; the caller then finds three
|
||||
;; bytes of U+FFFD in its buffer and no indication that it asked for something
|
||||
;; else. Nothing is written at all when this answers None.
|
||||
(defn encode-rune! [dst [u8] code i32] (Option i32)
|
||||
(match (rune-size code)
|
||||
None None
|
||||
(Some w)
|
||||
(if (> w (len dst))
|
||||
None
|
||||
(do
|
||||
(cond
|
||||
(= w 1)
|
||||
(set (at dst 0) (u8 code))
|
||||
(= w 2)
|
||||
(do (set (at dst 0) (u8 (bit-or 0xc0 (>> code 6))))
|
||||
(set (at dst 1) (u8 (bit-or 0x80 (bit-and code 0x3f)))))
|
||||
(= w 3)
|
||||
(do (set (at dst 0) (u8 (bit-or 0xe0 (>> code 12))))
|
||||
(set (at dst 1) (u8 (bit-or 0x80 (bit-and (>> code 6) 0x3f))))
|
||||
(set (at dst 2) (u8 (bit-or 0x80 (bit-and code 0x3f)))))
|
||||
:else
|
||||
(do (set (at dst 0) (u8 (bit-or 0xf0 (>> code 18))))
|
||||
(set (at dst 1) (u8 (bit-or 0x80 (bit-and (>> code 12) 0x3f))))
|
||||
(set (at dst 2) (u8 (bit-or 0x80 (bit-and (>> code 6) 0x3f))))
|
||||
(set (at dst 3) (u8 (bit-or 0x80 (bit-and code 0x3f))))))
|
||||
(Some w)))))
|
||||
|
||||
;; ── Splitting ─────────────────────────────────────────────────────────
|
||||
;;
|
||||
;; `split` returning a sequence of fields must allocate the sequence, and
|
||||
;; there is no allocator — so it is refused by name at the bottom of this
|
||||
;; file, and this is the shape that survives. It is Odin's
|
||||
;; split_by_byte_iterator (strings.odin): a cursor holding the rest of the
|
||||
;; input, handing back one field at a time. Every field is a slice *of the
|
||||
;; caller's bytes*; nothing is copied and nothing is owned.
|
||||
;;
|
||||
;; One divergence, and it is a wart of Odin's rather than a decision. Odin's
|
||||
;; iterator stops on an empty final field, so "a,b," iterates a and b and the
|
||||
;; trailing empty field is lost — while Odin's own allocating strings.split
|
||||
;; returns ["a", "b", ""] for the same input. The two disagree. This follows
|
||||
;; split: n separators always yield n+1 fields, an empty input yields one
|
||||
;; empty field, and `rest` is exhausted only after the last one is taken. That
|
||||
;; is the rule you can state without exceptions, and the one a caller counting
|
||||
;; comma-separated columns needs.
|
||||
(defstruct Split [rest [u8] sep u8 more bool])
|
||||
|
||||
(defn split-on-byte [s [u8] sep u8] Split
|
||||
(Split {:rest s :sep sep :more true}))
|
||||
|
||||
(defn split-next! [it (Ptr Split)] (Option [u8])
|
||||
(when (not (.more it))
|
||||
(return None))
|
||||
(match (index-of-byte (.rest it) (.sep it))
|
||||
(Some i)
|
||||
(let [field (slice (.rest it) 0 i)]
|
||||
(set (.rest it) (slice (.rest it) (+ i 1) (len (.rest it))))
|
||||
(Some field))
|
||||
None
|
||||
(let [field (.rest it)]
|
||||
(set (.more it) false)
|
||||
(set (.rest it) (slice (.rest it) (len (.rest it)) (len (.rest it))))
|
||||
(Some field))))
|
||||
|
||||
;; ── ASCII case ────────────────────────────────────────────────────────
|
||||
;;
|
||||
;; Byte in, byte out, and *not* a function over a slice. Odin's to_lower and
|
||||
;; to_upper both allocate a new string (core/strings/conversion.odin), which
|
||||
;; is not available here; the obvious substitute — lowering a [u8] in place —
|
||||
;; is a trap, and it is worth saying why rather than shipping it. A string
|
||||
;; literal is emitted `private unnamed_addr constant` (emit.ml), so (bytes
|
||||
;; "Hello") is a [u8] pointing straight into read-only memory. An in-place
|
||||
;; lower-ascii! type checks against that slice, and what happens next depends
|
||||
;; on the optimiser — which is the worst of the available answers. Measured,
|
||||
;; with (set (at (bytes "Hi") 0) \h):
|
||||
;;
|
||||
;; -O0 the store is emitted against the constant and the program takes
|
||||
;; SIGSEGV.
|
||||
;; -O2 LLVM deletes the store as undefined behaviour and the program
|
||||
;; carries on and prints "Hi".
|
||||
;;
|
||||
;; So the same source either dies or silently does nothing depending on a
|
||||
;; flag, and the -O2 half is the quiet-wrongness class this file keeps
|
||||
;; refusing elsewhere. Given a byte function instead, a caller that really
|
||||
;; does own its buffer writes the two-line loop itself over storage it can
|
||||
;; see the declaration of.
|
||||
;;
|
||||
;; ASCII only, and only the 26 letters: case outside ASCII is not a byte
|
||||
;; operation at all — it is per-code-point, it is not length-preserving (ß
|
||||
;; upcases to SS), and it is locale-dependent (Turkish dotless ı). A byte
|
||||
;; table that pretended otherwise would be wrong in the quiet way.
|
||||
(defn lower-ascii [b u8] u8
|
||||
(if (and (>= b \A) (<= b \Z)) (+ b 32) b))
|
||||
|
||||
(defn upper-ascii [b u8] u8
|
||||
(if (and (>= b \a) (<= b \z)) (- b 32) b))
|
||||
|
||||
;; Case-insensitive comparison as a fold over both inputs, which is the useful
|
||||
;; half of to_lower and needs no storage at all: comparing two lowered copies
|
||||
;; is what a caller wanted, and this is that answer without either copy.
|
||||
(defn bytes-ci=? [a [u8] b [u8]] bool
|
||||
(if (!= (len a) (len b))
|
||||
false
|
||||
(do
|
||||
(dotimes [i (len a)]
|
||||
(when (!= (lower-ascii (at a i)) (lower-ascii (at b i)))
|
||||
(return false)))
|
||||
true)))
|
||||
|
||||
;; ── Refused, by name ──────────────────────────────────────────────────
|
||||
;;
|
||||
;; Every one of these needs to produce bytes that did not exist in its input,
|
||||
;; and there is no allocator, so each is absent rather than approximated.
|
||||
;; None of them is hard to write once `(Vec u8)` and an allocator exist; all
|
||||
;; of them are impossible to write honestly today.
|
||||
;;
|
||||
;; join, concat build one buffer out of several inputs.
|
||||
;; to-lower, to-upper a new string, per Odin's conversion.odin. The
|
||||
;; byte-wise and folding-comparison forms above are
|
||||
;; what is available without one.
|
||||
;; split the *sequence* of fields is itself an allocation.
|
||||
;; split-on-byte / split-next! above is the same
|
||||
;; information with no sequence to own.
|
||||
;; replace, repeat, pad same reason as join.
|
||||
;; string-from-bytes a [u8] cannot become a `string` here even though
|
||||
;; the layouts are identical; see the report.
|
||||
;; format, sprintf Odin's fmt.aprintf family, all allocating.
|
||||
;; Builder strings.Builder is (defstruct Builder [buf
|
||||
;; (Vec u8)]), which spec-memory.md already makes
|
||||
;; move-only by the rule that a struct containing a
|
||||
;; Vec is move-only. It needs the Vec, not a spec
|
||||
;; change.
|
||||
|flan}
|
||||
|
||||
let file = "<prelude>"
|
||||
|
||||
265
test/programs/utf8.flan
Normal file
265
test/programs/utf8.flan
Normal file
@ -0,0 +1,265 @@
|
||||
;;;; UTF-8 decoding and encoding, the split cursor, and ASCII case.
|
||||
;;;;
|
||||
;;;; Every case here is one a plausible wrong decoder passes. A decoder that
|
||||
;;;; only masks and shifts — takes the top bits of the lead byte for a length
|
||||
;;;; and the low six of everything after — gets all of the *valid* input right
|
||||
;;;; and every line of the second group wrong, so the valid decodes prove
|
||||
;;;; almost nothing on their own and the malformed ones are the test.
|
||||
;;;;
|
||||
;;;; The four that matter, each isolating one row of Odin's accept_sizes
|
||||
;;;; table:
|
||||
;;;;
|
||||
;;;; c0 af an overlong two-byte "/". Accepted, it is the encoding
|
||||
;;;; that smuggles a slash past a check for one.
|
||||
;;;; e0 80 af an overlong three-byte "/", which the raised second-byte
|
||||
;;;; floor on 0xe0 is the only thing rejecting.
|
||||
;;;; ed a0 80 U+D800, a UTF-16 surrogate, which is not a scalar value.
|
||||
;;;; Only the lowered ceiling on 0xed rejects it.
|
||||
;;;; f4 90 80 80 U+110000, one past the last code point.
|
||||
;;;;
|
||||
;;;; And three more shapes: a lone continuation byte, a lead byte that leads
|
||||
;;;; nothing (0xf5), and a sequence truncated by the end of the slice — the
|
||||
;;;; last taken as a slice of a *valid* literal, which is what reading a
|
||||
;;;; buffer boundary actually hands you.
|
||||
;;;;
|
||||
;;;; The invalid sequences are byte arrays because the reader has no \xNN
|
||||
;;;; escape in either a string or a character literal, and none of them can be
|
||||
;;;; derived from a valid string.
|
||||
|
||||
(defconst overlong2 [2 u8] [0xc0 0xaf])
|
||||
(defconst overlong3 [3 u8] [0xe0 0x80 0xaf])
|
||||
(defconst overlong4 [4 u8] [0xf0 0x80 0x80 0xaf])
|
||||
(defconst surrogate [3 u8] [0xed 0xa0 0x80])
|
||||
(defconst above-max [4 u8] [0xf4 0x90 0x80 0x80])
|
||||
(defconst lead-f5 [4 u8] [0xf5 0x80 0x80 0x80])
|
||||
(defconst lone-cont [1 u8] [0x80])
|
||||
(defconst emoji [4 u8] [0xf0 0x9f 0x98 0x80]) ; U+1F600
|
||||
(defconst bad-tail [3 u8] [0x61 0xff 0x62]) ; "a", junk, "b"
|
||||
|
||||
(defvar scratch [4 u8])
|
||||
|
||||
;; Each array is sliced at the point of use rather than through a
|
||||
;; (defn whole [a [4 u8]] [u8] (slice a 0 4)) helper. That helper is a
|
||||
;; use-after-return and the compiler accepts it in silence: a [n T] parameter
|
||||
;; is a *value* and copies into the callee's frame, so the slice it hands back
|
||||
;; points at a frame that has already gone. It was written here first, and the
|
||||
;; emoji line is what caught it — it decoded as malformed because it was
|
||||
;; reading whatever the next call left on the stack. This is the escaping
|
||||
;; borrow spec-memory.md leaves to the programmer under "Borrowing", and it is
|
||||
;; the case a future provenance pass would reject.
|
||||
|
||||
;; code/width/ok, so a wrong answer names which of the three it got wrong
|
||||
;; rather than just failing.
|
||||
(defn show-dec [s [u8]]
|
||||
(let [r (decode-rune s)]
|
||||
(print-i64 (i64 (.code r))) (print-str "/")
|
||||
(print-i64 (i64 (.width r))) (print-str "/")
|
||||
(print-str (if (.ok r) "t" "f"))
|
||||
(print-str " ")))
|
||||
|
||||
(defn show-bool [b bool]
|
||||
(print-str (if b "t" "f")))
|
||||
|
||||
(defn show-opt [o (Option i32)]
|
||||
(print-i64 (i64 (match o (Some v) v None -1)))
|
||||
(print-str " "))
|
||||
|
||||
;; Encode into the scratch buffer and decode straight back out of it. A round
|
||||
;; trip is the only check that catches an encoder and a decoder that are
|
||||
;; wrong in the same direction — printing the bytes would not.
|
||||
(defn round-trip [code i32] i32
|
||||
(match (encode-rune! (slice scratch 0 4) code)
|
||||
None -1
|
||||
(Some w)
|
||||
(let [r (decode-rune (slice scratch 0 w))]
|
||||
(if (and (.ok r) (= (.width r) w)) (.code r) -1))))
|
||||
|
||||
(defn show-i32 [x i32]
|
||||
(print-i64 (i64 x))
|
||||
(print-str " "))
|
||||
|
||||
(defn show-split [s [u8] sep u8]
|
||||
(let [it (split-on-byte s sep)
|
||||
going true]
|
||||
(while going
|
||||
(match (split-next! (addr it))
|
||||
(Some f) (do (print-str "[") (print-bytes f) (print-str "]"))
|
||||
None (set going false)))
|
||||
(print-str " ")))
|
||||
|
||||
(defn main [] i32
|
||||
;; Valid, one of each width. The empty slice is width 0 — the only input
|
||||
;; that gets a 0, because every loop below advances by width and a 0 on a
|
||||
;; malformed byte would hang instead of answering.
|
||||
(show-dec (bytes "")) ; 0/0/f
|
||||
(show-dec (bytes "A")) ; 65/1/t
|
||||
(show-dec (bytes "é")) ; 233/2/t
|
||||
(show-dec (bytes "日")) ; 26085/3/t
|
||||
(show-dec (slice emoji 0 4)) ; 128512/4/t
|
||||
(newline)
|
||||
|
||||
;; Malformed. Every one is 0/1/f: width 1 so a scan makes progress.
|
||||
(show-dec (slice lone-cont 0 1)) ; a continuation byte leading
|
||||
(show-dec (slice overlong2 0 2)) ; overlong "/"
|
||||
(show-dec (slice overlong3 0 3)) ; overlong "/" again, three bytes
|
||||
(show-dec (slice overlong4 0 4)) ; and four. Added after a mutation run:
|
||||
; relaxing 0xf0's floor to 0x80 left the
|
||||
; whole suite green without this line.
|
||||
(show-dec (slice surrogate 0 3)) ; U+D800
|
||||
(show-dec (slice above-max 0 4)) ; U+110000
|
||||
(show-dec (slice lead-f5 0 4)) ; 0xf5 leads nothing
|
||||
(newline)
|
||||
|
||||
;; Truncated: a valid character cut short by the end of the slice, at both
|
||||
;; possible cut points, and the interior of one taken on its own.
|
||||
(show-dec (slice (bytes "日") 0 1)) ; lead byte alone
|
||||
(show-dec (slice (bytes "日") 0 2)) ; lead plus one continuation
|
||||
(show-dec (slice (bytes "日") 1 3)) ; starts mid-character
|
||||
(show-dec (slice (bytes "é") 1 2)) ; a lone continuation from a literal
|
||||
(newline)
|
||||
|
||||
;; rune-start? is what a caller scans backwards with.
|
||||
(show-bool (rune-start? (at (bytes "日") 0)))
|
||||
(show-bool (rune-start? (at (bytes "日") 1)))
|
||||
(show-bool (rune-start? \A))
|
||||
(newline)
|
||||
|
||||
;; Counting. The empty string is 0 and not 1; the mixed string is 8 runes
|
||||
;; in 13 bytes, which is the whole distinction; and a malformed byte counts
|
||||
;; as one, so a count never disagrees with what a renderer would draw.
|
||||
(print-i64 (i64 (rune-count (bytes "")))) (print-str " ")
|
||||
(print-i64 (i64 (rune-count (bytes "abc")))) (print-str " ")
|
||||
(print-i64 (i64 (rune-count (bytes "héllo 日本")))) (print-str " ")
|
||||
(print-i64 (i64 (len (bytes "héllo 日本")))) (print-str " ")
|
||||
(print-i64 (i64 (rune-count (slice bad-tail 0 3))))
|
||||
(newline)
|
||||
|
||||
(show-bool (valid-utf8? (bytes "")))
|
||||
(show-bool (valid-utf8? (bytes "héllo 日本")))
|
||||
(show-bool (valid-utf8? (slice surrogate 0 3)))
|
||||
(show-bool (valid-utf8? (slice overlong2 0 2)))
|
||||
(show-bool (valid-utf8? (slice bad-tail 0 3)))
|
||||
(show-bool (valid-utf8? (slice emoji 0 4)))
|
||||
(newline)
|
||||
|
||||
;; rune-at: on a boundary, off a boundary, and out of range. Off a boundary
|
||||
;; is None rather than a replacement character, which is where this is
|
||||
;; stricter than Odin's rune_at.
|
||||
(show-opt (rune-at (bytes "日本") 0)) ; 26085
|
||||
(show-opt (rune-at (bytes "日本") 3)) ; 26412
|
||||
(show-opt (rune-at (bytes "日本") 1)) ; -1, mid-character
|
||||
(show-opt (rune-at (bytes "日本") 6)) ; -1, past the end
|
||||
(show-opt (rune-at (bytes "") 0)) ; -1
|
||||
(newline)
|
||||
|
||||
;; rune-size, at every boundary and on both sides of it.
|
||||
(show-opt (rune-size -1))
|
||||
(show-opt (rune-size 0))
|
||||
(show-opt (rune-size 0x7f))
|
||||
(show-opt (rune-size 0x80))
|
||||
(show-opt (rune-size 0x7ff))
|
||||
(show-opt (rune-size 0x800))
|
||||
(show-opt (rune-size 0xd7ff))
|
||||
(show-opt (rune-size 0xd800))
|
||||
(show-opt (rune-size 0xdfff))
|
||||
(show-opt (rune-size 0xe000))
|
||||
(show-opt (rune-size 0xffff))
|
||||
(show-opt (rune-size 0x10000))
|
||||
(show-opt (rune-size 0x10ffff))
|
||||
(show-opt (rune-size 0x110000))
|
||||
(newline)
|
||||
|
||||
;; Round trips, one per width and at the boundaries.
|
||||
(show-i32 (round-trip 0))
|
||||
(show-i32 (round-trip 0x41))
|
||||
(show-i32 (round-trip 0x7f))
|
||||
(show-i32 (round-trip 0x80))
|
||||
(show-i32 (round-trip 0x7ff))
|
||||
(show-i32 (round-trip 0x800))
|
||||
(show-i32 (round-trip 0xffff))
|
||||
(show-i32 (round-trip 0x10000))
|
||||
(show-i32 (round-trip 0x10ffff))
|
||||
(newline)
|
||||
|
||||
;; Refused by encode-rune!, and nothing is written when it refuses.
|
||||
(show-opt (encode-rune! (slice scratch 0 4) 0xd800)) ; -1, surrogate
|
||||
(show-opt (encode-rune! (slice scratch 0 4) 0x110000)) ; -1, past the end
|
||||
(show-opt (encode-rune! (slice scratch 0 4) -1)) ; -1, negative
|
||||
(show-opt (encode-rune! (slice scratch 0 2) 0x65e5)) ; -1, buffer short
|
||||
(show-opt (encode-rune! (slice scratch 0 0) 0x41)) ; -1, no room at all
|
||||
(show-opt (encode-rune! (slice scratch 0 1) 0x41)) ; 1, exactly enough
|
||||
(newline)
|
||||
|
||||
;; "Nothing is written when it refuses" is a claim about the buffer, not
|
||||
;; about the return value, and the None cases above do not test it: an
|
||||
;; encoder that lays down the lead byte and only then notices the buffer is
|
||||
;; short returns None exactly as this one does, and every line above still
|
||||
;; passes. So put a known byte in scratch, ask for an encoding that must be
|
||||
;; refused, and read the byte back.
|
||||
(show-i32 (round-trip 0x41)) ; 65, scratch[0] = A
|
||||
(show-opt (encode-rune! (slice scratch 0 2) 0x65e5)) ; -1, needs 3 bytes
|
||||
(show-i32 (i32 (at scratch 0))) ; 65 still
|
||||
(show-opt (encode-rune! (slice scratch 0 4) 0xd800)) ; -1, surrogate
|
||||
(show-i32 (i32 (at scratch 0))) ; 65 still
|
||||
(newline)
|
||||
|
||||
;; Splitting. n separators give n+1 fields, always: an interior empty field
|
||||
;; survives, a leading and a trailing one do too, and an input with no
|
||||
;; separator at all is one field rather than none. The empty input is the
|
||||
;; case Odin's own iterator disagrees with its allocating split on — it is
|
||||
;; one empty field here.
|
||||
(show-split (bytes "a,b,c") \,) ; [a][b][c]
|
||||
(show-split (bytes "a,,b") \,) ; [a][][b]
|
||||
(show-split (bytes "abc") \,) ; [abc]
|
||||
(show-split (bytes "") \,) ; []
|
||||
(show-split (bytes ",") \,) ; [][]
|
||||
(show-split (bytes ",a") \,) ; [][a]
|
||||
(show-split (bytes "a,") \,) ; [a][]
|
||||
(newline)
|
||||
|
||||
;; A field is a slice of the input, so trim and parse-i64 work straight off
|
||||
;; one with nothing copied in between — which is the entire reason the
|
||||
;; cursor shape exists.
|
||||
(let [it (split-on-byte (bytes " 10 , 20 ,30") \,)
|
||||
total (i64 0)
|
||||
going true]
|
||||
(while going
|
||||
(match (split-next! (addr it))
|
||||
(Some f) (set total (+ total (match (parse-i64 (trim f)) (Some v) v None 0)))
|
||||
None (set going false)))
|
||||
(print-i64 total)
|
||||
(newline))
|
||||
|
||||
;; ASCII case. The boundary bytes on both sides of each range are what a
|
||||
;; wrong mask gets wrong: '@' and '[' sit either side of A-Z, and '`' and
|
||||
;; '{' either side of a-z, so a conversion written as (bit-xor b 32) —
|
||||
;; which works for every letter — turns '@' into '`' and is caught here.
|
||||
(show-i32 (i32 (lower-ascii \A)))
|
||||
(show-i32 (i32 (lower-ascii \Z)))
|
||||
(show-i32 (i32 (lower-ascii \a)))
|
||||
(show-i32 (i32 (lower-ascii \@))) ; 64, just below 'A'
|
||||
(show-i32 (i32 (lower-ascii \[))) ; 91, just above 'Z'
|
||||
(show-i32 (i32 (upper-ascii \a)))
|
||||
(show-i32 (i32 (upper-ascii \z)))
|
||||
(show-i32 (i32 (upper-ascii \A)))
|
||||
(show-i32 (i32 (upper-ascii \`))) ; 96, just below 'a'
|
||||
(show-i32 (i32 (upper-ascii \{))) ; 123, just above 'z'
|
||||
(show-i32 (i32 (lower-ascii \5))) ; digits are untouched
|
||||
(newline)
|
||||
|
||||
;; A non-ASCII byte must pass through both untouched, which is the claim
|
||||
;; that "ASCII only" is a rule and not an oversight.
|
||||
(show-i32 (i32 (lower-ascii (at (bytes "é") 0))))
|
||||
(show-i32 (i32 (upper-ascii (at (bytes "é") 0))))
|
||||
(newline)
|
||||
|
||||
(show-bool (bytes-ci=? (bytes "Hello") (bytes "hELLO"))) ; t
|
||||
(show-bool (bytes-ci=? (bytes "Hello") (bytes "hello!"))) ; f length first
|
||||
(show-bool (bytes-ci=? (bytes "") (bytes ""))) ; t
|
||||
(show-bool (bytes-ci=? (bytes "a") (bytes "b"))) ; f
|
||||
;; '@' is 'A'+32 apart from '`' the way a letter is from its own case, so a
|
||||
;; fold written as a bit-xor would call these two equal. They are not.
|
||||
(show-bool (bytes-ci=? (bytes "@") (bytes "`"))) ; f
|
||||
(show-bool (bytes-ci=? (bytes "é") (bytes "é"))) ; t bytes match
|
||||
(newline)
|
||||
0)
|
||||
@ -1544,6 +1544,49 @@ ERR@7 unexpected token: not the kind the caller was reading
|
||||
end
|
||||
else print_endline "acceptance: lldb cases skipped (no lldb on PATH)";
|
||||
|
||||
(* ── Strings: UTF-8, the split cursor, and ASCII case ──────────────
|
||||
|
||||
The prelude's port of Odin's core/unicode/utf8, which is the only part
|
||||
of a string library that needs no allocator. The valid decodes prove
|
||||
almost nothing on their own — a decoder that just masks and shifts gets
|
||||
every one of them right — so the test is the malformed group: an
|
||||
overlong two- and three-byte "/", a UTF-16 surrogate, a code point past
|
||||
U+10FFFF, a lead byte that leads nothing, a lone continuation byte, and
|
||||
a valid character truncated by the end of its slice. Each isolates one
|
||||
row of the accept_sizes table, and each answers width 1 so that a scan
|
||||
makes progress rather than hanging.
|
||||
|
||||
Encoding is checked by round trip rather than against expected bytes,
|
||||
because an encoder and a decoder that are wrong in the same direction
|
||||
agree with each other and disagree with nothing else.
|
||||
|
||||
At -O0 as well, for the reason the slice algorithms run there: a slice
|
||||
is a two-word struct through an alloca, decode-rune returns a struct by
|
||||
value, and the split cursor is mutated through a (Ptr Split) — mem2reg
|
||||
is exactly what would hide any of those being copied when it should be
|
||||
shared. *)
|
||||
let utf8_out =
|
||||
"0/0/f 65/1/t 233/2/t 26085/3/t 128512/4/t \n\
|
||||
0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f 0/1/f \n\
|
||||
0/1/f 0/1/f 0/1/f 0/1/f \n\
|
||||
tft\n\
|
||||
0 3 8 13 3\n\
|
||||
ttffft\n\
|
||||
26085 26412 -1 -1 -1 \n\
|
||||
-1 1 1 2 2 3 3 -1 -1 3 3 4 4 -1 \n\
|
||||
0 65 127 128 2047 2048 65535 65536 1114111 \n\
|
||||
-1 -1 -1 -1 -1 1 \n\
|
||||
65 -1 65 -1 65 \n\
|
||||
[a][b][c] [a][][b] [abc] [] [][] [][a] [a][] \n\
|
||||
60\n\
|
||||
97 122 97 64 91 65 90 65 96 123 53 \n\
|
||||
195 195 \n\
|
||||
tftfft\n"
|
||||
in
|
||||
outputs "utf-8, splitting and ascii case" "programs/utf8.flan" utf8_out;
|
||||
outputs ~opt:"-O0" "utf-8, splitting and ascii case, -O0"
|
||||
"programs/utf8.flan" utf8_out;
|
||||
|
||||
if !failures = 0 then print_endline "acceptance: all tests passed"
|
||||
else begin
|
||||
Printf.printf "\n%d failure(s)\n" !failures;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user