Joseph Ferano
2e203f64b8
The INSERTIONSORT crash, all three rulings (FIX.org 2026-09-20): - (bytes s) allocates a writable copy through the allocator surface — context or (bytes s a), StorageExhausted with retry, a registry note in dev builds (flan_bytes_dup, lowered like vec-new). (bytes-view s) is the old zero-cost reinterpret, renamed, read-only by convention; every in-repo reader swept over to it. (string b) unchanged. - String constants were already read-only on both backends at -O0; now pinned — bytes-copy.flan rows on LLVM/-O0/--x86, and dies_segv rows asserting the write-through-view trap on both backends. - A dev build installs a SIGSEGV/SIGBUS handler by the same dev-only constructor slot that arms the registry: one line naming the address and the innermost frame, then the trap-hook park — stopped, not dead, the daemon serving. No agent: message and re-raise. Release builds untouched. Pinned by trap_park over dev-segv.flan.
261 lines
11 KiB
Plaintext
261 lines
11 KiB
Plaintext
;;;; The EDN tokenizer, and a struct reader written by hand against it.
|
|
;;;;
|
|
;;;; The second half is the point. `(read-edn Enemy bytes)` — the compiler
|
|
;;;; emitting a parser from a walk over Enemy's fields — is not built yet, so
|
|
;;;; what this file proves is that the cursor is usable *without* it: read-enemy
|
|
;;;; below is what that emitted code will look like, written out by hand. An API
|
|
;;;; that only a compiler could call would be present rather than usable.
|
|
;;;;
|
|
;;;; Every case here is one a plausible wrong version fails. Named where it is
|
|
;;;; not obvious.
|
|
|
|
(import edn "vendor:edn")
|
|
|
|
;; ── A dump of the token stream ──────────────────────────────────────
|
|
;;
|
|
;; One letter per kind, then the text in brackets, so both halves of every
|
|
;; token are asserted. A tokenizer that got the kinds right and the slices
|
|
;; wrong — off by the quote, off by the colon — would pass on the letters
|
|
;; alone.
|
|
|
|
(defn kind-letter [k i32] string
|
|
(cond
|
|
(= k edn/tok-eof) "."
|
|
(= k edn/tok-error) "!"
|
|
(= k edn/tok-nil) "n"
|
|
(= k edn/tok-bool) "b"
|
|
(= k edn/tok-int) "i"
|
|
(= k edn/tok-float) "f"
|
|
(= k edn/tok-string) "s"
|
|
(= k edn/tok-keyword) "k"
|
|
(= k edn/tok-symbol) "y"
|
|
(= k edn/tok-vec-open) "["
|
|
(= k edn/tok-vec-close) "]"
|
|
(= k edn/tok-map-open) "{"
|
|
(= k edn/tok-map-close) "}"
|
|
(= k edn/tok-list-open) "("
|
|
(= k edn/tok-list-close) ")"
|
|
;; A set closes on `}`, so the dump shows `#` … `}` and there is no letter
|
|
;; for a set's closer to have.
|
|
(= k edn/tok-set-open) "#"
|
|
:else "?"))
|
|
|
|
(defn dump [src string] ()
|
|
(let [b (bytes-view src)
|
|
c (edn/cursor b)
|
|
t (edn/next (addr c))]
|
|
(while (and (edn/ok? (addr c)) (!= (.kind t) edn/tok-eof))
|
|
(print (kind-letter (.kind t)))
|
|
(print "<")
|
|
(print (.text t))
|
|
(print ">")
|
|
(set t (edn/next (addr c))))
|
|
(when (not (edn/ok? (addr c)))
|
|
(print "ERR@")
|
|
(print (edn/error-pos (addr c))))
|
|
(println "")))
|
|
|
|
;; The refusals. Asserted on the *reason*, not on the fact of failing: a
|
|
;; tokenizer that answered err-unexpected-byte for every one of these would
|
|
;; pass a test that only checked that it failed.
|
|
(defn refusal [src string] ()
|
|
(let [b (bytes-view src)
|
|
c (edn/cursor b)]
|
|
(while (and (edn/ok? (addr c))
|
|
(!= (.kind (edn/next (addr c))) edn/tok-eof)))
|
|
(print (edn/error-pos (addr c)))
|
|
(print " ")
|
|
(print (edn/error-message (edn/error (addr c))))
|
|
(println "")))
|
|
|
|
;; ── The worked example: a struct read by hand ───────────────────────
|
|
|
|
;; `name` is a [u8] and not a copy of one, so an Enemy is only valid while the
|
|
;; buffer it was read out of is. That is the lifetime contract from the package
|
|
;; header, and it is what a struct reader inherits by using slices.
|
|
(defstruct Enemy
|
|
[name [u8]
|
|
hp i32
|
|
speed f32
|
|
boss? bool])
|
|
|
|
;; The shape the compiler-emitted version will have: open the map, loop on the
|
|
;; keys, dispatch each known one onto its field, and skip whatever is left over
|
|
;; so an extra key in a data file is not fatal. Errors accumulate on the cursor
|
|
;; rather than being returned, which is why this can be a straight line of
|
|
;; assignments with one test at the end.
|
|
(defn read-enemy [c (Ptr edn/Cursor)] Enemy
|
|
(let [e (Enemy {.hp 0 .speed 0.0 .boss? false})]
|
|
(edn/expect c edn/tok-map-open)
|
|
(while (edn/ok? c)
|
|
(let [k (edn/next c)]
|
|
(when (or (not (edn/ok? c)) (= (.kind k) edn/tok-map-close))
|
|
(return e))
|
|
(when (!= (.kind k) edn/tok-keyword)
|
|
(edn/fail c edn/err-unexpected-token (.pos k))
|
|
(return e))
|
|
(cond
|
|
(edn/keyword=? k "name")
|
|
(set (.name e) (.text (edn/expect c edn/tok-string)))
|
|
|
|
(edn/keyword=? k "hp")
|
|
(set (.hp e) (i32 (match (edn/int-of (edn/expect c edn/tok-int))
|
|
(Some v) v None 0)))
|
|
|
|
;; The one field read with `next` rather than `expect`, because two
|
|
;; kinds are acceptable for it. The None arm is what keeps that from
|
|
;; being a hole: a string here fails rather than defaulting to 0.0.
|
|
(edn/keyword=? k "speed")
|
|
(let [v (edn/next c)]
|
|
(match (edn/float-of v)
|
|
(Some x) (set (.speed e) (f32 x))
|
|
None (edn/fail c edn/err-unexpected-token (.pos v))))
|
|
|
|
(edn/keyword=? k "boss?")
|
|
(set (.boss? e) (match (edn/bool-of (edn/expect c edn/tok-bool))
|
|
(Some v) v None false))
|
|
|
|
;; An unknown key: read past its value, however big it is.
|
|
:else
|
|
(when (not (edn/skip-value c))
|
|
(return e)))))
|
|
e))
|
|
|
|
(defn show-enemy [src string] ()
|
|
(let [b (bytes-view src)
|
|
c (edn/cursor b)
|
|
e (read-enemy (addr c))]
|
|
(if (edn/ok? (addr c))
|
|
(do
|
|
(print "[")
|
|
(print (.name e))
|
|
(print "] hp=")
|
|
(print (.hp e))
|
|
(print " speed=")
|
|
(print (.speed e))
|
|
(print " boss=")
|
|
(print (if (.boss? e) "yes" "no")))
|
|
(do
|
|
(print "ERR@")
|
|
(print (edn/error-pos (addr c)))
|
|
(print " ")
|
|
(print (edn/error-message (edn/error (addr c))))))
|
|
(println "")))
|
|
|
|
(defn main [] i32
|
|
;; ── Scalars, and the boundaries between them ──────────────────────
|
|
(dump "1") ; i<1>
|
|
(dump "-1 +2 0") ; the signs are part of the number
|
|
(dump "1.5 -2.5e3 .5") ; f, and a leading dot is a float
|
|
(dump "true false nil") ; b b n — and not three symbols
|
|
;; `-` alone is a symbol, `foo/bar` is NOT a ratio, and the uppercase half
|
|
;; of the alphabet test is only exercised by a name that has one in it.
|
|
(dump "foo Enemy/Goblin -")
|
|
(dump ":a :foo/bar") ; k, text without the colon
|
|
(println "")
|
|
|
|
;; A number followed immediately by a delimiter, with no space. A scanner
|
|
;; that only stopped on whitespace reads "1]" or "1;x" as one atom and then
|
|
;; fails to parse it.
|
|
(dump "[1]")
|
|
(dump "[1 2][3]") ; two tokens with no space between them
|
|
(dump "{:a 1}")
|
|
(dump "1;c") ; a comment starting against the number
|
|
(dump ":a;c") ; a keyword ending at a comment
|
|
(println "")
|
|
|
|
;; Empty collections, and nesting. An empty map is the case a reader that
|
|
;; assumes at least one key-value pair gets wrong.
|
|
(dump "{}")
|
|
(dump "[]")
|
|
(dump "()")
|
|
(dump "[[1] [2 [3]]]")
|
|
(dump "{:a {:b []}}")
|
|
(println "")
|
|
|
|
;; Sets. `#{` is one token and two bytes, and the `}` that ends it is the
|
|
;; same token a map's is — which is the whole of what the balance stack was
|
|
;; told. The last two are the cases a `#` arm that forgot to push get wrong:
|
|
;; a set inside a map has to close the set before the map, and an empty set
|
|
;; is the one where the opener and the closer are adjacent.
|
|
(dump "#{1 2}")
|
|
(dump "#{}")
|
|
(dump "{:a #{1 2} :b 3}")
|
|
(dump "#{[4 3] [2 2]}")
|
|
(println "")
|
|
|
|
;; A keyword at the very end of input — the loop has to test the length
|
|
;; before reading the byte, or this walks off the end.
|
|
(dump ":a")
|
|
(dump "1")
|
|
(dump "\"x\"")
|
|
(println "")
|
|
|
|
;; Comments. The last one has no trailing newline, which is the case that
|
|
;; separates a scan-to-newline from a scan-to-newline-or-end.
|
|
(dump "; only a comment\n1")
|
|
(dump "1 ; trailing\n2")
|
|
(dump "1 ; no newline at the end")
|
|
(dump ";") ; a bare comment marker, nothing after it
|
|
(println "")
|
|
|
|
;; Commas are whitespace in EDN, and are not tokens.
|
|
(dump "[1, 2 ,3]")
|
|
(println "")
|
|
|
|
;; Strings. The second is the one that matters: a `[` and a `;` inside a
|
|
;; string must not open a vector or start a comment.
|
|
(dump "\"hi\"")
|
|
(dump "\"a[b;c\" 1")
|
|
(dump "\"\" 1") ; the empty string is a token with empty text
|
|
(dump "\"a b\"")
|
|
(println "")
|
|
|
|
;; ── The refusals, each asserted on its own reason ─────────────────
|
|
(refusal "\"a\\nb\"") ; an escape inside a string
|
|
(refusal "\"a\\\"b\"") ; an escaped quote — the case where a wrong
|
|
; version returns `a\` and leaves `b"` behind
|
|
(refusal "\"unterminated") ; not a refusal, but the other string failure
|
|
;; A set is read now, so what is left to refuse about one is its balance. A
|
|
;; `#{` that pushed nothing would answer "no error" for both of these.
|
|
(refusal "#{1 2)") ; the wrong closer for a set
|
|
(refusal "#{1 2") ; end of input with the set still open
|
|
(refusal "#foo {}") ; a tagged literal
|
|
(refusal "#inst \"2024\"") ; named separately
|
|
(refusal "#uuid \"x\"")
|
|
(refusal "^{:a 1} [1]") ; metadata
|
|
(refusal "22/7") ; a ratio
|
|
(refusal "\\a") ; a character literal
|
|
(refusal "12x") ; starts like a number, is not one
|
|
(refusal "[1 :]") ; a colon with no name
|
|
(refusal "@") ; not the start of any value — and the case a
|
|
; scan-to-delimiter reads as a one-byte symbol
|
|
(refusal "`x") ; a Clojure reader macro, not EDN
|
|
(refusal "[1 2}") ; the wrong closer
|
|
(refusal "]") ; a closer with nothing open
|
|
(refusal "[1 2") ; end of input with something still open
|
|
;; 33 opening brackets against a 32-deep stack. The error message is the
|
|
;; least of what this checks: a `>` where the guard needs `>=` writes one
|
|
;; past the end of a fixed array, and the answer is a bounds trap rather
|
|
;; than a wrong message. The offset is the 33rd bracket.
|
|
(refusal "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[")
|
|
(println "")
|
|
|
|
;; ── The struct reader ─────────────────────────────────────────────
|
|
(show-enemy "{:name \"goblin\" :hp 12 :speed 1.5 :boss? false}")
|
|
;; Fields in a different order, one missing (zeroed), one unknown key whose
|
|
;; value is a whole nested collection that skip-value has to walk past.
|
|
(show-enemy "{:boss? true :loot [:gold {:n 3} [[]]] :hp 40 :name \"dragon\"}")
|
|
;; An unknown key whose value is a set, which skip-value has to walk past on
|
|
;; the balance stack like any other collection — and a set nested in it, so
|
|
;; that a `#{` pushing nothing would leave the map open and swallow :hp.
|
|
(show-enemy "{:name \"wisp\" :tags #{:a #{:b} [1]} :hp 5}")
|
|
;; :speed given as an integer — 2 and 2.0 are the same number.
|
|
(show-enemy "{:name \"imp\" :hp 1 :speed 2}")
|
|
(show-enemy "{}")
|
|
;; Wrong type for a field: the reader stops and names the position.
|
|
(show-enemy "{:name 7}")
|
|
;; A comment inside the map, and commas.
|
|
(show-enemy "{:name \"orc\", ; a note\n :hp 9}")
|
|
0)
|