Joseph Ferano
96ab4c9cf0
print-str, print-i64, print-f64, print-bytes, print-line and newline leave the prelude. print and println are the whole printing surface now, and print is the better call at every one of the sites that used them: it is the same structural walk without the newline, so the no-newline case the family was kept for is covered, and it takes the value as it is. The old print-i64 forced an explicit (i64 x) at every call site, because this language widens nothing implicitly; that cast is gone from 127 places. Dropping it moves one answer. hash-grid returns u64, and the cast through the signed printer showed sand-headless's hash as -2851001042534928384. print routes a u64 through flan_u64_to_bytes, so it now prints 15595743031174623232 — the same 64 bits, read as the unsigned number they are. The pinned expectation follows the correction. test-flan-dev.el and test_session.ml both reached for print-line as "a name the prelude has"; they reach for rand-seed instead.
240 lines
9.6 KiB
Plaintext
240 lines
9.6 KiB
Plaintext
;;;; The EDN tokenizer, and a struct reader written by hand against it.
|
|
;;;;
|
|
;;;; The second half is the point. `(read-edn Enemy bytes)` — the compiler
|
|
;;;; emitting a parser from a walk over Enemy's fields — is not built yet, so
|
|
;;;; what this file proves is that the cursor is usable *without* it: read-enemy
|
|
;;;; below is what that emitted code will look like, written out by hand. An API
|
|
;;;; that only a compiler could call would be present rather than usable.
|
|
;;;;
|
|
;;;; Every case here is one a plausible wrong version fails. Named where it is
|
|
;;;; not obvious.
|
|
|
|
(import edn "vendor:edn")
|
|
|
|
;; ── A dump of the token stream ──────────────────────────────────────
|
|
;;
|
|
;; One letter per kind, then the text in brackets, so both halves of every
|
|
;; token are asserted. A tokenizer that got the kinds right and the slices
|
|
;; wrong — off by the quote, off by the colon — would pass on the letters
|
|
;; alone.
|
|
|
|
(defn kind-letter [k i32] string
|
|
(cond
|
|
(= k edn/tok-eof) "."
|
|
(= k edn/tok-error) "!"
|
|
(= k edn/tok-nil) "n"
|
|
(= k edn/tok-bool) "b"
|
|
(= k edn/tok-int) "i"
|
|
(= k edn/tok-float) "f"
|
|
(= k edn/tok-string) "s"
|
|
(= k edn/tok-keyword) "k"
|
|
(= k edn/tok-symbol) "y"
|
|
(= k edn/tok-vec-open) "["
|
|
(= k edn/tok-vec-close) "]"
|
|
(= k edn/tok-map-open) "{"
|
|
(= k edn/tok-map-close) "}"
|
|
(= k edn/tok-list-open) "("
|
|
(= k edn/tok-list-close) ")"
|
|
:else "?"))
|
|
|
|
(defn dump [src string]
|
|
(let [b (bytes src)
|
|
c (edn/cursor b)
|
|
t (edn/next (addr c))]
|
|
(while (and (edn/ok? (addr c)) (!= (.kind t) edn/tok-eof))
|
|
(print (kind-letter (.kind t)))
|
|
(print "<")
|
|
(print (.text t))
|
|
(print ">")
|
|
(set t (edn/next (addr c))))
|
|
(when (not (edn/ok? (addr c)))
|
|
(print "ERR@")
|
|
(print (edn/error-pos (addr c))))
|
|
(println "")))
|
|
|
|
;; The refusals. Asserted on the *reason*, not on the fact of failing: a
|
|
;; tokenizer that answered err-unexpected-byte for every one of these would
|
|
;; pass a test that only checked that it failed.
|
|
(defn refusal [src string]
|
|
(let [b (bytes src)
|
|
c (edn/cursor b)]
|
|
(while (and (edn/ok? (addr c))
|
|
(!= (.kind (edn/next (addr c))) edn/tok-eof)))
|
|
(print (edn/error-pos (addr c)))
|
|
(print " ")
|
|
(print (edn/error-message (edn/error (addr c))))
|
|
(println "")))
|
|
|
|
;; ── The worked example: a struct read by hand ───────────────────────
|
|
|
|
;; `name` is a [u8] and not a copy of one, so an Enemy is only valid while the
|
|
;; buffer it was read out of is. That is the lifetime contract from the package
|
|
;; header, and it is what a struct reader inherits by using slices.
|
|
(defstruct Enemy
|
|
[name [u8]
|
|
hp i32
|
|
speed f32
|
|
boss? bool])
|
|
|
|
;; The shape the compiler-emitted version will have: open the map, loop on the
|
|
;; keys, dispatch each known one onto its field, and skip whatever is left over
|
|
;; so an extra key in a data file is not fatal. Errors accumulate on the cursor
|
|
;; rather than being returned, which is why this can be a straight line of
|
|
;; assignments with one test at the end.
|
|
(defn read-enemy [c (Ptr edn/Cursor)] Enemy
|
|
(let [e (Enemy {:hp 0 :speed 0.0 :boss? false})]
|
|
(edn/expect c edn/tok-map-open)
|
|
(while (edn/ok? c)
|
|
(let [k (edn/next c)]
|
|
(when (or (not (edn/ok? c)) (= (.kind k) edn/tok-map-close))
|
|
(return e))
|
|
(when (!= (.kind k) edn/tok-keyword)
|
|
(edn/fail c edn/err-unexpected-token (.pos k))
|
|
(return e))
|
|
(cond
|
|
(edn/keyword=? k "name")
|
|
(set (.name e) (.text (edn/expect c edn/tok-string)))
|
|
|
|
(edn/keyword=? k "hp")
|
|
(set (.hp e) (i32 (match (edn/int-of (edn/expect c edn/tok-int))
|
|
(Some v) v None 0)))
|
|
|
|
;; The one field read with `next` rather than `expect`, because two
|
|
;; kinds are acceptable for it. The None arm is what keeps that from
|
|
;; being a hole: a string here fails rather than defaulting to 0.0.
|
|
(edn/keyword=? k "speed")
|
|
(let [v (edn/next c)]
|
|
(match (edn/float-of v)
|
|
(Some x) (set (.speed e) (f32 x))
|
|
None (edn/fail c edn/err-unexpected-token (.pos v))))
|
|
|
|
(edn/keyword=? k "boss?")
|
|
(set (.boss? e) (match (edn/bool-of (edn/expect c edn/tok-bool))
|
|
(Some v) v None false))
|
|
|
|
;; An unknown key: read past its value, however big it is.
|
|
:else
|
|
(when (not (edn/skip-value c))
|
|
(return e)))))
|
|
e))
|
|
|
|
(defn show-enemy [src string]
|
|
(let [b (bytes src)
|
|
c (edn/cursor b)
|
|
e (read-enemy (addr c))]
|
|
(if (edn/ok? (addr c))
|
|
(do
|
|
(print "[")
|
|
(print (.name e))
|
|
(print "] hp=")
|
|
(print (.hp e))
|
|
(print " speed=")
|
|
(print (.speed e))
|
|
(print " boss=")
|
|
(print (if (.boss? e) "yes" "no")))
|
|
(do
|
|
(print "ERR@")
|
|
(print (edn/error-pos (addr c)))
|
|
(print " ")
|
|
(print (edn/error-message (edn/error (addr c))))))
|
|
(println "")))
|
|
|
|
(defn main [] i32
|
|
;; ── Scalars, and the boundaries between them ──────────────────────
|
|
(dump "1") ; i<1>
|
|
(dump "-1 +2 0") ; the signs are part of the number
|
|
(dump "1.5 -2.5e3 .5") ; f, and a leading dot is a float
|
|
(dump "true false nil") ; b b n — and not three symbols
|
|
;; `-` alone is a symbol, `foo/bar` is NOT a ratio, and the uppercase half
|
|
;; of the alphabet test is only exercised by a name that has one in it.
|
|
(dump "foo Enemy/Goblin -")
|
|
(dump ":a :foo/bar") ; k, text without the colon
|
|
(println "")
|
|
|
|
;; A number followed immediately by a delimiter, with no space. A scanner
|
|
;; that only stopped on whitespace reads "1]" or "1;x" as one atom and then
|
|
;; fails to parse it.
|
|
(dump "[1]")
|
|
(dump "[1 2][3]") ; two tokens with no space between them
|
|
(dump "{:a 1}")
|
|
(dump "1;c") ; a comment starting against the number
|
|
(dump ":a;c") ; a keyword ending at a comment
|
|
(println "")
|
|
|
|
;; Empty collections, and nesting. An empty map is the case a reader that
|
|
;; assumes at least one key-value pair gets wrong.
|
|
(dump "{}")
|
|
(dump "[]")
|
|
(dump "()")
|
|
(dump "[[1] [2 [3]]]")
|
|
(dump "{:a {:b []}}")
|
|
(println "")
|
|
|
|
;; A keyword at the very end of input — the loop has to test the length
|
|
;; before reading the byte, or this walks off the end.
|
|
(dump ":a")
|
|
(dump "1")
|
|
(dump "\"x\"")
|
|
(println "")
|
|
|
|
;; Comments. The last one has no trailing newline, which is the case that
|
|
;; separates a scan-to-newline from a scan-to-newline-or-end.
|
|
(dump "; only a comment\n1")
|
|
(dump "1 ; trailing\n2")
|
|
(dump "1 ; no newline at the end")
|
|
(dump ";") ; a bare comment marker, nothing after it
|
|
(println "")
|
|
|
|
;; Commas are whitespace in EDN, and are not tokens.
|
|
(dump "[1, 2 ,3]")
|
|
(println "")
|
|
|
|
;; Strings. The second is the one that matters: a `[` and a `;` inside a
|
|
;; string must not open a vector or start a comment.
|
|
(dump "\"hi\"")
|
|
(dump "\"a[b;c\" 1")
|
|
(dump "\"\" 1") ; the empty string is a token with empty text
|
|
(dump "\"a b\"")
|
|
(println "")
|
|
|
|
;; ── The refusals, each asserted on its own reason ─────────────────
|
|
(refusal "\"a\\nb\"") ; an escape inside a string
|
|
(refusal "\"a\\\"b\"") ; an escaped quote — the case where a wrong
|
|
; version returns `a\` and leaves `b"` behind
|
|
(refusal "\"unterminated") ; not a refusal, but the other string failure
|
|
(refusal "#{1 2}") ; a set
|
|
(refusal "#foo {}") ; a tagged literal
|
|
(refusal "#inst \"2024\"") ; named separately
|
|
(refusal "#uuid \"x\"")
|
|
(refusal "^{:a 1} [1]") ; metadata
|
|
(refusal "22/7") ; a ratio
|
|
(refusal "\\a") ; a character literal
|
|
(refusal "12x") ; starts like a number, is not one
|
|
(refusal "[1 :]") ; a colon with no name
|
|
(refusal "@") ; not the start of any value — and the case a
|
|
; scan-to-delimiter reads as a one-byte symbol
|
|
(refusal "`x") ; a Clojure reader macro, not EDN
|
|
(refusal "[1 2}") ; the wrong closer
|
|
(refusal "]") ; a closer with nothing open
|
|
(refusal "[1 2") ; end of input with something still open
|
|
;; 33 opening brackets against a 32-deep stack. The error message is the
|
|
;; least of what this checks: a `>` where the guard needs `>=` writes one
|
|
;; past the end of a fixed array, and the answer is a bounds trap rather
|
|
;; than a wrong message. The offset is the 33rd bracket.
|
|
(refusal "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[")
|
|
(println "")
|
|
|
|
;; ── The struct reader ─────────────────────────────────────────────
|
|
(show-enemy "{:name \"goblin\" :hp 12 :speed 1.5 :boss? false}")
|
|
;; Fields in a different order, one missing (zeroed), one unknown key whose
|
|
;; value is a whole nested collection that skip-value has to walk past.
|
|
(show-enemy "{:boss? true :loot [:gold {:n 3} [[]]] :hp 40 :name \"dragon\"}")
|
|
;; :speed given as an integer — 2 and 2.0 are the same number.
|
|
(show-enemy "{:name \"imp\" :hp 1 :speed 2}")
|
|
(show-enemy "{}")
|
|
;; Wrong type for a field: the reader stops and names the position.
|
|
(show-enemy "{:name 7}")
|
|
;; A comment inside the map, and commas.
|
|
(show-enemy "{:name \"orc\", ; a note\n :hp 9}")
|
|
0)
|