From e9cdbb321b0d5462b7a4d6b12e6571d77f040db8 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Thu, 10 Sep 2026 14:40:34 +0700 Subject: [PATCH] Lisp based flan --- README.md | 1 - bin/dune | 1 - bin/main.ml | 23 +- calc-me.flan | 123 ++++++++ docs/overview.md | 39 --- dune-project | 28 +- flan.opam | 32 --- lib/dune | 5 - lib/examples.ml | 4 - lib/form.ml | 40 +++ lib/loc.ml | 19 ++ lib/oflan.mly | 65 ----- lib/olexer.mll | 58 ---- lib/omniflan.ml | 29 -- lib/parse.ml | 31 -- lib/reader.ml | 197 +++++++++++++ overview.md | 54 ++++ plan.org | 687 +++++++++++++++++++++++++++++++++++++++++++++ sand.flan | 151 ++++++++++ spec-conditions.md | 115 ++++++++ spec-memory.md | 154 ++++++++++ syntax-sketch.flan | 198 +++++++++++++ test/dune | 3 +- test/test_flan.ml | 121 ++++++++ 24 files changed, 1881 insertions(+), 297 deletions(-) delete mode 100644 README.md create mode 100644 calc-me.flan delete mode 100644 docs/overview.md delete mode 100644 flan.opam delete mode 100644 lib/examples.ml create mode 100644 lib/form.ml create mode 100644 lib/loc.ml delete mode 100644 lib/oflan.mly delete mode 100644 lib/olexer.mll delete mode 100644 lib/omniflan.ml delete mode 100644 lib/parse.ml create mode 100644 lib/reader.ml create mode 100644 overview.md create mode 100644 plan.org create mode 100644 sand.flan create mode 100644 spec-conditions.md create mode 100644 spec-memory.md create mode 100644 syntax-sketch.flan diff --git a/README.md b/README.md deleted file mode 100644 index 74ab06f..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -# Flan \ No newline at end of file diff --git a/bin/dune b/bin/dune index 52f5287..4bc9a0a 100644 --- a/bin/dune +++ b/bin/dune @@ -1,4 +1,3 @@ (executable - (public_name flan) (name main) (libraries flan)) diff --git a/bin/main.ml b/bin/main.ml index 247911b..94d735b 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -1,9 +1,18 @@ -open Flan.Parse +(* flan — milestone 2 driver. Right now: read a file and print the forms back, + which is the first thing worth having and the first thing worth testing. *) let () = - let source_str = Flan.Examples.let_bind_int in - let lexbuf = Lexing.from_string source_str in - Printf.printf "Convert source \"%s\" ->\n" source_str; - match parse_program lexbuf with - | Ok ast -> print_ast ast - | Error msg -> print_endline ("ERROR: \n" ^ msg) + match Array.to_list Sys.argv with + | _ :: "read" :: files when files <> [] -> + List.iter + (fun path -> + try + Flan.Reader.read_file path + |> List.iter (fun f -> print_endline (Flan.Form.to_string f)) + with Flan.Loc.Error (loc, msg) -> + Printf.eprintf "%s: %s\n" (Flan.Loc.to_string loc) msg; + exit 1) + files + | _ -> + prerr_endline "usage: flan read ..."; + exit 2 diff --git a/calc-me.flan b/calc-me.flan new file mode 100644 index 0000000..058dbec --- /dev/null +++ b/calc-me.flan @@ -0,0 +1,123 @@ +;;;; calc-me — THE FIRST ACCEPTANCE PROGRAM (build sequence milestone 2). +;;;; +;;;; $ calc-me "1 + 2 * (3 - 0.5) / 2" +;;;; 3.5 +;;;; +;;;; Chosen to be the smallest program that is still a real one. What it needs: +;;;; functions, recursion, structs, (Ptr T) and `addr`, byte slices, `at`/`len`, +;;;; `while`, `set` on locals and on fields, `cond`, `match`, Option, i32/u8/f64, +;;;; and argv. What it deliberately does NOT need: an allocator, Vec, Map, any +;;;; generic function, any macro the user wrote, FFI beyond argv and stdout, +;;;; a window, or a frame loop. It runs headless, so it is the same test on +;;;; native and on wasm32 — which is how the second target gets proven early. +;;;; +;;;; No `ns` form and no package declaration: the package name is inferred from +;;;; the directory. `(package calc)` is written only when the name must differ +;;;; from the directory name. See plan.org "Modules". + +;; ── Cursor over the input. A plain value struct; recursive descent shares +;; ── one by pointer. `addr` takes the address of a local; the pointer never +;; ── outlives the frame, so no allocator is involved. +(defstruct Cursor + [src [u8] ; non-owning slice into argv — calc-me never owns a byte + pos i32]) ; no initialiser means zeroed + +(defn peek [c (Ptr Cursor)] u8 + (if (< (.pos c) (len (.src c))) + (at (.src c) (.pos c)) + 0)) ; 0 doubles as end-of-input + +(defn advance [c (Ptr Cursor)] + (set (.pos c) (+ (.pos c) 1))) ; field access auto-derefs one level + +(defn skip-spaces [c (Ptr Cursor)] + (while (= (peek c) \space) + (advance c))) + +(defn digit? [b u8] bool + (and (>= b \0) (<= b \9))) + +;; ── number := digit+ ("." digit+)? ──────────────────────────────────── +(defn parse-number [c (Ptr Cursor)] (Option f64) + (skip-spaces c) + (let [start (.pos c)] + (while (digit? (peek c)) + (advance c)) + (when (= (peek c) \.) + (advance c) + (while (digit? (peek c)) + (advance c))) + (if (= start (.pos c)) + None + (Some (bytes->f64 (slice (.src c) start (.pos c))))))) + +;; ── primary := number | "(" expr ")" | "-" primary ──────────────────── +;; `some` unwraps Some and early-returns None from THIS function. It and +;; Option are the only error handling here; Result, try and errdefer wait. +(defn parse-primary [c (Ptr Cursor)] (Option f64) + (skip-spaces c) + (cond + (= (peek c) \-) + (do (advance c) + (Some (- 0.0 (some (parse-primary c))))) + + (= (peek c) \() + (do (advance c) + (let [v (some (parse-expr c 1))] + (skip-spaces c) + (if (= (peek c) \)) + (do (advance c) (Some v)) + None))) ; unbalanced paren + + :else + (parse-number c))) + +(defn precedence [op u8] i32 + (cond + (or (= op \+) (= op \-)) 1 + (or (= op \*) (= op \/)) 2 + :else 0)) ; 0 means "not an operator" + +(defn apply-op [op u8 l f64 r f64] f64 + (cond + (= op \+) (+ l r) + (= op \-) (- l r) + (= op \*) (* l r) + :else (/ l r))) + +;; ── expr := primary (op primary)*, precedence climbing ──────────────── +;; Left-associative: the right operand is parsed at prec+1, so 1-2-3 is +;; (1-2)-3 and not 1-(2-3). Mutually recursive with parse-primary; top-level +;; names in a package are order-independent, so no forward declaration. +(defn parse-expr [c (Ptr Cursor) min-prec i32] (Option f64) + (let [lhs (some (parse-primary c))] + (skip-spaces c) + (let [prec (precedence (peek c))] + (while (and (> prec 0) (>= prec min-prec)) + (let [op (peek c)] + (advance c) + (set lhs (apply-op op lhs (some (parse-expr c (+ prec 1)))))) + (skip-spaces c) + (set prec (precedence (peek c))))) + (Some lhs))) + +;; ── Whole input, or nothing. Trailing junk is an error, not ignored. ── +(defn evaluate [src [u8]] (Option f64) + (let [c (Cursor {:src src})] ; pos omitted: zeroed + (let [v (some (parse-expr (addr c) 1))] + (skip-spaces (addr c)) + (if (= (peek (addr c)) 0) + (Some v) + None)))) + +;; Entry point: (defn main [args [string]] i32). Both the parameter and the +;; return type are optional — sand.flan uses the bare (defn main []) form. +;; print-str/print-f64/print-line are Flan functions over the write-stdout +;; primitive, NOT an overloaded println: compile-time overloading waits for +;; milestone 5, so until then the acceptance programs name the type. +(defn main [args [string]] i32 + (if (< (len args) 2) + (do (print-line "usage: calc-me \"1 + 2 * 3\"") 1) + (match (evaluate (bytes (nth args 1))) + (Some v) (do (print-f64 v) (print-line "") 0) + None (do (print-line "calc-me: cannot parse") 1)))) diff --git a/docs/overview.md b/docs/overview.md deleted file mode 100644 index 049e29b..0000000 --- a/docs/overview.md +++ /dev/null @@ -1,39 +0,0 @@ -# Overview - -Concepts: - -- Primitives - - u8, i32, f32, bool, char - - lists, slice, fixed-length array builtin, matrices -- Control flow - - for, while, break -- Structs & tuples -- Let bindings -- `Ptr a` -- Pattern matching -- ADTs -- Mutability - - const by default? -- Functions -- Array syntax - - ranges `arr[1..]`, `arr[..3]` - - index `arr[4]` -- Stdlib - - string - - vec/dynarray - - hashtable - - option/result - -C-like with Roc syntax. -Start with interpreter. Output C later down the track - -```rust -let slice = &[u32]; - -[1,23,45,4,1] -let new_vec: Vec = iterator.iter_mut() - -> map(|x| *x = 100 ) -- allocs? - -> filter -- allocs? - -> filter -- allocs? - .collect() -``` \ No newline at end of file diff --git a/dune-project b/dune-project index eacd961..a931209 100644 --- a/dune-project +++ b/dune-project @@ -1,27 +1,7 @@ (lang dune 3.15) -(using menhir 3.0) - (name flan) -(generate_opam_files true) - -(source - (github username/reponame)) - -(authors "Author Name") - -(maintainers "Maintainer Name") - -(license LICENSE) - -(documentation https://url/to/documentation) - -(package - (name flan) - (synopsis "A short synopsis") - (description "A longer description") - (depends ocaml dune menhir) - (tags - (topics "to describe" your project))) - -; See the complete stanza docs at https://dune.readthedocs.io/en/stable/dune-files.html#dune-project +; No menhir, no ocamllex: the reader is hand-written. S-expressions do not need +; a parser generator, hand-written gives better source locations, and every +; dependency here is something that would have to be reimplemented in Flan if +; the compiler is ever self-hosted. diff --git a/flan.opam b/flan.opam deleted file mode 100644 index 8507c27..0000000 --- a/flan.opam +++ /dev/null @@ -1,32 +0,0 @@ -# This file is generated by dune, edit dune-project instead -opam-version: "2.0" -synopsis: "A short synopsis" -description: "A longer description" -maintainer: ["Maintainer Name"] -authors: ["Author Name"] -license: "LICENSE" -tags: ["topics" "to describe" "your" "project"] -homepage: "https://github.com/username/reponame" -doc: "https://url/to/documentation" -bug-reports: "https://github.com/username/reponame/issues" -depends: [ - "ocaml" - "dune" {>= "3.15"} - "menhir" - "odoc" {with-doc} -] -build: [ - ["dune" "subst"] {dev} - [ - "dune" - "build" - "-p" - name - "-j" - jobs - "@install" - "@runtest" {with-test} - "@doc" {with-doc} - ] -] -dev-repo: "git+https://github.com/username/reponame.git" diff --git a/lib/dune b/lib/dune index 082be58..ad39be8 100644 --- a/lib/dune +++ b/lib/dune @@ -1,7 +1,2 @@ (library (name flan)) - -(menhir - (modules oflan)) - -(ocamllex olexer) diff --git a/lib/examples.ml b/lib/examples.ml deleted file mode 100644 index 65c3a8c..0000000 --- a/lib/examples.ml +++ /dev/null @@ -1,4 +0,0 @@ -(** Examples of syntax / programs as strings that can be imported and tested *) - -let let_bind_int = "let x = 10" -let let_bind_str = "let s = \"hello\" " diff --git a/lib/form.ml b/lib/form.ml new file mode 100644 index 0000000..6c81f11 --- /dev/null +++ b/lib/form.ml @@ -0,0 +1,40 @@ +(** The reader's output: syntax, before any typing or macro expansion. + + Deliberately dumb. [true], [false] and [nil] are ordinary symbols here and + are resolved later; the reader knows nothing about special forms. *) + +type t = { + v : value; + loc : Loc.t; +} + +and value = + | Sym of string (* foo rl/draw-fps .pos + *) + | Kw of string (* :space :else (leading : dropped) *) + | Int of int64 (* 42 -1 0xE6B800FF *) + | Float of float (* 0.05 *) + | Str of string (* "SAND" *) + | Byte of int (* \space \0 \( (0..255) *) + | List of t list (* (f x) *) + | Vec of t list (* [1 2 3] and every binding/type bracket *) + | Map of t list (* {:key v} in value position, {K V} in type position *) + +let make v loc = { v; loc } + +let rec to_string f = + let seq l = String.concat " " (List.map to_string l) in + match f.v with + | Sym s -> s + | Kw s -> ":" ^ s + | Int i -> Int64.to_string i + | Float x -> Printf.sprintf "%g" x + | Str s -> Printf.sprintf "%S" s + | Byte b -> + (match Char.chr b with + | ' ' -> "\\space" + | '\t' -> "\\tab" + | '\n' -> "\\newline" + | c -> Printf.sprintf "\\%c" c) + | List l -> "(" ^ seq l ^ ")" + | Vec l -> "[" ^ seq l ^ "]" + | Map l -> "{" ^ seq l ^ "}" diff --git a/lib/loc.ml b/lib/loc.ml new file mode 100644 index 0000000..5589e73 --- /dev/null +++ b/lib/loc.ml @@ -0,0 +1,19 @@ +(** Source locations. Every form carries one: error messages, the step debugger + and nREPL's find-definition all need them, and retrofitting locations onto a + reader is far worse than carrying them from the start. *) + +type t = { + file : string; + line : int; (* 1-based *) + col : int; (* 1-based *) +} + +let make file line col = { file; line; col } +let unknown = { file = ""; line = 0; col = 0 } + +let to_string t = Printf.sprintf "%s:%d:%d" t.file t.line t.col + +(** Raised by every stage of the frontend. *) +exception Error of t * string + +let fail loc fmt = Printf.ksprintf (fun msg -> raise (Error (loc, msg))) fmt diff --git a/lib/oflan.mly b/lib/oflan.mly deleted file mode 100644 index 84aefdc..0000000 --- a/lib/oflan.mly +++ /dev/null @@ -1,65 +0,0 @@ -/* Declarations */ - -%{ - open Omniflan.Ast -%} - -%token Eof -%token Newline -%token Let - -%token False -%token True -%token If -%token Then -%token Else -%token Print -%token Ident -%token Int -%token F32 - -%token LParen -%token RParen -%token LBrace -%token RBrace -%token LBracket -%token RBracket -%token Dot -%token Comma -%token Colon -%token Semicolon -%token Plus -%token Minus -%token Star -%token Slash -%token Bang -%token Equal -%token EqualEqual -%token BangEqual -%token LT -%token GT -%token LTE -%token GTE - -%start prog - -%% -/* Grammar */ - -expr: - | i = Int; { Int i } - -stmt: - | Let; var_name = Ident; Equal; bound_expr = expr - { Let { - loc = $startpos; - var_name = var_name; - bindee = bound_expr - } - } - -toplevel_item: - | stmt = stmt { Stmt stmt } - -prog: - | prog = separated_list(Newline, toplevel_item); Eof { prog } diff --git a/lib/olexer.mll b/lib/olexer.mll deleted file mode 100644 index 3c0f301..0000000 --- a/lib/olexer.mll +++ /dev/null @@ -1,58 +0,0 @@ -{ - open Lexing - open Oflan - - exception SyntaxError of string - - let next_line lexbuf = - let pos = lexbuf.lex_curr_p in - lexbuf.lex_curr_p <- - { pos with pos_bol = lexbuf.lex_curr_pos; - pos_lnum = pos.pos_lnum + 1 - } -} - -let digit = ['0'-'9'] -let digits = digit* -let alpha = ['a'-'z' 'A'-'Z'] -let ident = (alpha) (alpha|digit|'_')* (* regex for identifier *) -let whitespace = [' ' '\t']+ -let newline = '\r' | '\n' | "\r\n" - -let int = digits - -rule read = - parse - | whitespace { read lexbuf } - | newline { next_line lexbuf; read lexbuf } - | int { Int (int_of_string (Lexing.lexeme lexbuf))} - | "let" { Let } - | "if" { If } - | "then" { Then } - | "else" { Else } - | "print" { Print } - | ident { Ident (Lexing.lexeme lexbuf) } - | '(' { LParen } - | ')' { RParen } - | '[' { LBracket } - | ']' { RBracket } - | '{' { LBrace } - | '}' { RBrace } - | '.' { Dot } - | ',' { Comma } - | ':' { Colon } - | ';' { Semicolon } - | '+' { Plus } - | '-' { Minus } - | '*' { Star } - | '/' { Slash } - | '!' { Bang } - | '=' { Equal } - | "==" { EqualEqual } - | "!=" { BangEqual } - | '<' { LT } - | '>' { GT } - | "<=" { LTE } - | ">=" { GTE } - | eof { Eof } - | _ { raise (SyntaxError ("Unexpected char: " ^ Lexing.lexeme lexbuf)) } diff --git a/lib/omniflan.ml b/lib/omniflan.ml deleted file mode 100644 index 13047dd..0000000 --- a/lib/omniflan.ml +++ /dev/null @@ -1,29 +0,0 @@ -(* -Notes - -For now everything will be split into modules inside this one big file while prototyping. -*) - -module Ast = struct - type loc = Lexing.position - type unary_op = Negate - type binary_op = Add | Subtract | Multiply | Divide - type literal = Int of int - - type expr = Int of int - (* | Literal of literal *) - (* | BinaryOp of { lhs: expr; rhs: expr; operator: binary_op } *) - (* | IfElse of { condition: expr; if_expr: expr; else_expr: expr } *) - - and stmt = - | Let of { loc : loc; var_name : string; bindee : expr } (* Let binding "let x = 5" *) - | FuncDecl (* TODO: arguments *) - - and toplevel_item = Stmt of stmt - - type builtin_type = I32 | F32 | Bool | Char - type program = toplevel_item list -end - -module Typer = struct end -(** This module helps take an untyped AST and produce a typed AST *) diff --git a/lib/parse.ml b/lib/parse.ml deleted file mode 100644 index 4487362..0000000 --- a/lib/parse.ml +++ /dev/null @@ -1,31 +0,0 @@ -open Lexing - -exception SyntaxError of string - -(* Prints the line number and character number where the error occurred.*) -let print_error_position lexbuf = - let pos = lexbuf.lex_curr_p in - Printf.sprintf "Line:%d Position:%d" pos.pos_lnum (pos.pos_cnum - pos.pos_bol + 1) - -let parse_program lexbuf = - try Ok (Oflan.prog Olexer.read lexbuf) with - | SyntaxError msg -> - let error_msg = Printf.sprintf "%s: %s\n" (print_error_position lexbuf) msg in - Error error_msg - | Oflan.Error -> - let error_msg = Printf.sprintf "%s: syntax error\n" (print_error_position lexbuf) in - Error error_msg - -open Omniflan.Ast - -let string_of_expr expr = match expr with Int i -> "Int " ^ string_of_int i - -let string_of_stmt stmt = - match stmt with - | Let s -> Printf.sprintf "(%d) Let %s = %s" s.loc.pos_lnum s.var_name (string_of_expr s.bindee) - | FuncDecl -> failwith "TODO" - -let print_ast prog = - List.iter - (fun toplevel -> match toplevel with Stmt stmt -> print_endline (string_of_stmt stmt)) - prog diff --git a/lib/reader.ml b/lib/reader.ml new file mode 100644 index 0000000..6ab0f74 --- /dev/null +++ b/lib/reader.ml @@ -0,0 +1,197 @@ +(** S-expression reader. + + Hand-written rather than ocamllex/menhir: a Lisp needs no parser generator, + locations come out cleaner, and it keeps the compiler dependency-free. + + ['x] reads as [(quote x)]. That is not macro support — it is here because + restart names are quoted symbols ([(invoke-restart 'skip-form)]) and without + it the apostrophe would silently become part of the symbol's name. + + Not handled yet: quasiquote/unquote (milestone 5, with macros) and metadata + ([^:async]). Metadata is rejected rather than read as a symbol, so it cannot + rot into a silently-wrong name the way quote would have. *) + +type state = { + src : string; + file : string; + mutable pos : int; + mutable line : int; + mutable col : int; +} + +let of_string ~file src = { src; file; pos = 0; line = 1; col = 1 } + +let here st = Loc.make st.file st.line st.col +let at_end st = st.pos >= String.length st.src +let peek st = if at_end st then '\000' else st.src.[st.pos] +let peek2 st = + if st.pos + 1 >= String.length st.src then '\000' else st.src.[st.pos + 1] + +let advance st = + if not (at_end st) then begin + if st.src.[st.pos] = '\n' then (st.line <- st.line + 1; st.col <- 1) + else st.col <- st.col + 1; + st.pos <- st.pos + 1 + end + +(* Symbol constituents. Note '-' and '?' and '!' and '/' and '.' are all + ordinary: `empty-at?`, `rl/draw-fps`, `.pos`, `->>` are single symbols. *) +let is_delimiter = function + | '(' | ')' | '[' | ']' | '{' | '}' | '"' | ';' | '\000' -> true + | c -> c = ' ' || c = '\t' || c = '\n' || c = '\r' || c = ',' + +let is_digit c = c >= '0' && c <= '9' + +let rec skip_trivia st = + match peek st with + | ' ' | '\t' | '\n' | '\r' | ',' -> advance st; skip_trivia st + | ';' -> + while (not (at_end st)) && peek st <> '\n' do advance st done; + skip_trivia st + | _ -> () + +let take_while st pred = + let start = st.pos in + while (not (at_end st)) && pred (peek st) do advance st done; + String.sub st.src start (st.pos - start) + +(* ── Atoms ─────────────────────────────────────────────────────────── *) + +let read_string st = + let loc = here st in + advance st; (* opening quote *) + let buf = Buffer.create 16 in + let rec go () = + if at_end st then Loc.fail loc "unterminated string" + else match peek st with + | '"' -> advance st + | '\\' -> + advance st; + let c = peek st in + advance st; + Buffer.add_char buf + (match c with + | 'n' -> '\n' | 't' -> '\t' | 'r' -> '\r' + | '\\' -> '\\' | '"' -> '"' | '0' -> '\000' + | c -> Loc.fail loc "unknown string escape \\%c" c); + go () + | c -> advance st; Buffer.add_char buf c; go () + in + go (); + Form.make (Form.Str (Buffer.contents buf)) loc + +(* \space \tab \newline \return \nul, or \ *) +let read_byte st = + let loc = here st in + advance st; (* backslash *) + if at_end st then Loc.fail loc "expected a character after \\"; + let first = peek st in + advance st; + let rest = take_while st (fun c -> not (is_delimiter c)) in + let name = String.make 1 first ^ rest in + let code = match name with + | "space" -> 32 + | "tab" -> 9 + | "newline" -> 10 + | "return" -> 13 + | "nul" -> 0 + | n when String.length n = 1 -> Char.code n.[0] + | n -> Loc.fail loc "unknown character literal \\%s" n + in + Form.make (Form.Byte code) loc + +(* A token that started with a digit, or with '-'/'+' followed by a digit. *) +let read_number st = + let loc = here st in + let text = take_while st (fun c -> not (is_delimiter c)) in + let is_hex = + String.length text > 2 + && text.[0] = '0' + && (text.[1] = 'x' || text.[1] = 'X') + in + if is_hex then + match Int64.of_string_opt text with + | Some i -> Form.make (Form.Int i) loc + | None -> Loc.fail loc "malformed hex literal %s" text + else if String.contains text '.' || String.contains text 'e' then + match float_of_string_opt text with + | Some f -> Form.make (Form.Float f) loc + | None -> Loc.fail loc "malformed float literal %s" text + else + match Int64.of_string_opt text with + | Some i -> Form.make (Form.Int i) loc + | None -> Loc.fail loc "malformed integer literal %s" text + +let read_symbol_or_keyword st = + let loc = here st in + let text = take_while st (fun c -> not (is_delimiter c)) in + if text = "" then Loc.fail loc "unexpected character %C" (peek st); + if text.[0] = ':' then begin + if String.length text = 1 then Loc.fail loc "empty keyword"; + Form.make (Form.Kw (String.sub text 1 (String.length text - 1))) loc + end else + Form.make (Form.Sym text) loc + +(* ── Forms ─────────────────────────────────────────────────────────── *) + +let closer = function + | '(' -> ')' | '[' -> ']' | '{' -> '}' + | _ -> assert false + +let wrap open_c items = + match open_c with + | '(' -> Form.List items + | '[' -> Form.Vec items + | '{' -> Form.Map items + | _ -> assert false + +let rec read_form st = + skip_trivia st; + let loc = here st in + match peek st with + | '\000' -> Loc.fail loc "unexpected end of input" + | '(' | '[' | '{' as open_c -> read_seq st open_c loc + | ')' | ']' | '}' as c -> Loc.fail loc "unbalanced %C" c + | '"' -> read_string st + | '\\' -> read_byte st + | '\'' -> + advance st; + let quoted = read_form st in + Form.make (Form.List [ Form.make (Form.Sym "quote") loc; quoted ]) loc + | '^' -> + Loc.fail loc "metadata (^) is not supported yet" + + | c when is_digit c -> read_number st + | ('-' | '+') when is_digit (peek2 st) -> read_number st + | _ -> read_symbol_or_keyword st + +and read_seq st open_c loc = + advance st; + let want = closer open_c in + let rec go acc = + skip_trivia st; + if at_end st then + Loc.fail loc "unclosed %C, expected %C" open_c want + else + let c = peek st in + if c = want then (advance st; List.rev acc) + else if c = ')' || c = ']' || c = '}' then + Loc.fail (here st) "expected %C to close %C, found %C" want open_c c + else go (read_form st :: acc) + in + Form.make (wrap open_c (go [])) loc + +(** All top-level forms in a source string. *) +let read_all ~file src = + let st = of_string ~file src in + let rec go acc = + skip_trivia st; + if at_end st then List.rev acc else go (read_form st :: acc) + in + go [] + +let read_file path = + let ic = open_in_bin path in + Fun.protect ~finally:(fun () -> close_in ic) (fun () -> + let n = in_channel_length ic in + read_all ~file:path (really_input_string ic n)) diff --git a/overview.md b/overview.md new file mode 100644 index 0000000..8bb5089 --- /dev/null +++ b/overview.md @@ -0,0 +1,54 @@ +# Overview — superseded + +This was the first brainstorm. It is kept for history and is **no longer +accurate**. Read instead: + +- `plan.org` — the design, the build sequence, and the open decisions +- `spec-memory.md` — ownership, containers, places, generics, function values +- `spec-conditions.md` — conditions and restarts, operational semantics +- `sand.flan` — the first acceptance program +- `syntax-sketch.flan` — the syntax, annotated with the decisions above + +## What changed since + +| This document said | Now | +|---|---| +| "C-like with Roc syntax" | S-expressions, Clojure's brackets; C's value model | +| "Start with interpreter, output C later" | Frontend → typed IR → LLVM IR as text → `clang`. An interpreter is acceptable for milestone 2 only | +| "lists, slice, fixed-length array, matrices" | Four container types, distinct ownership: `[n T]`, `[T]`, `(Vec T)`, `(Map K V)` — see spec-memory.md | +| `arr[4]`, `arr[1..]` index syntax | `(at a i)`, `(as-slice a lo hi)` — no infix, no bracket indexing | +| `const by default?` | Locals are assignable places; parameters are not; `const` qualifies slices and pointers | +| Rust-style iterator chains ending in `.collect()` | `->>` threading over slices; every collecting operation allocates from an explicit allocator, usually the frame arena | +| `Ptr a` | Kept, as `(Ptr a)`. Cross-references use `(Handle a)` instead | +| option/result in the stdlib | Kept, and layered with conditions — see plan.org "Error handling, layered" | + +## Original text + +``` +Concepts: + +- Primitives + - u8, i32, f32, bool, char + - lists, slice, fixed-length array builtin, matrices +- Control flow + - for, while, break +- Structs & tuples +- Let bindings +- `Ptr a` +- Pattern matching +- ADTs +- Mutability + - const by default? +- Functions +- Array syntax + - ranges `arr[1..]`, `arr[..3]` + - index `arr[4]` +- Stdlib + - string + - vec/dynarray + - hashtable + - option/result + +C-like with Roc syntax. +Start with interpreter. Output C later down the track +``` diff --git a/plan.org b/plan.org new file mode 100644 index 0000000..51d0994 --- /dev/null +++ b/plan.org @@ -0,0 +1,687 @@ +#+TITLE: Flan — Design Plan +#+DATE: 2026-09-10 + +* Specs +Two documents are normative and are settled ahead of implementation. Anything in +this plan that contradicts them is out of date. +- [[file:spec-memory.md][spec-memory.md]] — ownership, the four container types, + copies and moves, assignable places, generics without type classes, function + values. +- [[file:spec-conditions.md][spec-conditions.md]] — the six hard cases of + conditions/restarts: what ~signal~ returns, no-handler behaviour, restart + signatures, name shadowing, cleanup during a transfer, and crossing + compiler-generated and foreign frames. + +* What Flan is +A minimal Lisp for game development. Clojure's brackets and a small slice of its +API, C's memory and value model. No GC. + +Not a Common Lisp, not a Clojure. In one line: *Odin with a Lisp frontend and a +live REPL.* + +** References +- *Odin* — closest existing language. LLVM backend, manual memory, no GC, ships + amd64 + arm64 + wasm32 (~js_wasm32~, ~wasi_wasm32~, ~freestanding_wasm32~). + Proves this exact pipeline. Take directly: + - ~context.allocator~ / ~context.temp_allocator~ (already this plan's design; + ~free_all(context.temp_allocator)~ per frame /is/ the frame arena) + - ~$T~ compile-time parametric polymorphism — monomorphisation without a heavy + type system + - ~#soa~ struct-of-arrays syntax + - fixed arrays with component-wise ops and swizzles (~[4]f32~, ~v.xyzw~), + built-in ~matrix~ type + - ~defer~, tagged unions, ~distinct~ types, bit sets + - ~vendor:raylib~ + Flan diverges by adding what Odin deliberately lacks: s-expressions, macros, + conditions/restarts, hot reload, interactive development. +- *SBCL* — indirection cells for redefinition, conditions/restarts, the break loop. +- *Janet* — reference only; its immutable/mutable split is rejected (see Data model). +- *Carp* — statically typed Lisp with inference, no GC (ownership-based), compiles + to C, aimed at games. Closest precedent for the /language/ shape, as Odin is for + the /implementation/ shape. +- *jank* — cautionary; see Hot reload. + +* Non-goals +- Numeric tower (no bignums, rationals, complex) +- CLOS/MOP, ~format~, pathnames, streams, sequences-over-anything, CL reader +- Clojure's lazy seqs, JVM interop, ~core.async~ +- Persistent collections, and immutable collection types generally (see Data model) +- Live-image development at SBCL's level +- Consoles + +* Memory — no GC +Allocators all the way down; ~malloc~ hidden behind them. + +| Tier | Strategy | Cost | +|-----------+---------------------------------+----------------| +| Frame | arena, bulk reset each frame | free | +| Entities | pool + generational handles | free | +| Subsystem | region, freed wholesale | free | +| Dev/REPL | leaks by design, reset on reload| dev only | + +- Allocator is part of the calling convention, so a refcounted allocator can be + added later without a language change. +- Generational handles instead of pointers for cross-references: a stale + reference is detectable, not undefined behaviour. +- Symbols and code live in a permanent arena that only grows. + +** Why no persistent collections +Structure sharing destroys clear ownership, which is the only thing that forces a +collector. Replaced by value structs that copy on assignment — see Data model. + +** The consequence that drives everything +No GC means *no object headers*. Every struct is exactly its C layout, arrays are +C arrays, so there is no marshalling layer and no wrapper allocation. + +FFI is still a boundary — ownership, who allocates, string and slice +representation, struct padding, callbacks into Flan, error handling, and the +platform ABI all remain real work. What C layout buys is that the *data* crosses +for free. See spec-conditions.md §6 for the one hard rule: a restart transfer +cannot cross a foreign frame. + +* Data model +No immutable collection types. Value semantics plus ~const~, as in C/Zig/Odin — +not Janet's tuple/struct vs array/table split, which was an answer to a GC'd +world. + +- Four container types, distinct in both type and ownership — see spec-memory.md, + which is normative: + | Notation | Layout | Assignment | Owns | + |-------------+-----------------+------------------+------| + | ~[n T]~ | inline, n items | copies | no | + | ~[T]~ | ptr+len | copies the view | no | + | ~(Vec T)~ | ptr+len+cap | *moves* | yes | + | ~(Map K V)~ | open addressing | *moves* | yes | + ~Vec~ and ~Map~ are monomorphic on element type and record their allocator. Not + a Lua-style array/hash hybrid — that is what makes Lua's layout and performance + unpredictable. +- Operations: ~get~, ~put~, ~remove~, ~push~, ~pop~, ~nth~, ~len~, ~update~. + Copying is explicit: ~(clone m)~, and owning containers move rather than copy on + assignment. No ~!~ convention — nothing is immutable, so it + would carry no information. No ~assoc~; it only existed as the copy-returning form. +- ~const~ qualifier on references and slices: compile-time contract that a + callee will not mutate. Zero runtime cost. +- Value structs copy on assignment — but only *value* structs. Ownership is + structural: a struct is a value type iff every field is, so one ~Vec~ field + makes it move-only. This is what keeps "copies on assignment" from meaning a + shallow copy that aliases owned storage. Deep copies are always explicit: + ~(clone x)~. Value structs are the snapshot / undo / replay story; they need no + separate type. +- Literals live in read-only memory. +- Struct literals name fields: ~(Cursor {:src src :pos 0})~. *Omitted fields are + zeroed*, as in Odin — the same rule as a declaration with no initialiser, so + ~(Cursor {:src src})~ is complete and means ~pos~ is 0. +- *Zero is initialisation (ZII), with an opt-out.* No initialiser means + all-bytes-zero. ~(defvar buf [65536 u8] uninit)~ skips it, exactly as Odin's + ~---~ does, for a large buffer that is about to be overwritten. ~uninit~ is + greppable and rare by design; reading an ~uninit~ value before writing it is + undefined, and dev builds poison the memory so the bug is loud. +- Slices as ptr+len, non-owning; raw pointers ~(Ptr T)~ with explicit ~deref~; + visible casts. ~resolve~ yields ~(Ptr a)~ where ~deref~ yields a value — that is + how a matched struct is mutated in place rather than as a copy. +- Flat and SoA arrays. +- No implicit allocation anywhere in the core. + +Immutability also serves the optimiser: a value known never to be mutated can be +copied into registers and stack-allocated freely. Mutability is what forces heap +identity. + +* Types +*Statically typed.* Not forced — tagging and boxing give dynamic typing without a +collector, as Forth-lineage and refcounted dynamic languages show. It is chosen, +and the reason is that dynamic typing would require paying a tag word on every +value, which is exactly the header cost dropping the GC was meant to avoid. Under +static typing the tag is paid only where it is asked for, in ~any~. + +- Types are mandatory; *inference* makes them feel optional. Annotate function + signatures, infer locals — Odin/Zig/Rust ergonomics. +- Signatures are annotated as inline name/type pairs, as in ~let~ and + ~defstruct~: ~(defn area [s Shape] f32 ...)~. No separate ~declare~ form — + ~declare~ is kept only where there is no body (forward declarations, FFI). +- Annotations at function boundaries are unavoidable, because compile-time + overloading is incompatible with full inference. Locals are inferred. +- An omitted return type means ~Unit~ — a real zero-sized type with one value, + not C's ~void~. Generic code over it works, so there is no ~Action~/~Func~ split. +- Every type notation reads as exactly one data item: ~[f32]~, ~[4 f32]~, + ~(Vec f32)~, ~{string i32}~, ~(Ptr World)~, ~(Fn [f32] bool)~, ~(Option a)~, + ~(Handle a)~. +- ~i8..i64~, ~u8..u64~, ~f32~, ~f64~ as real machine types; wrapping arithmetic. +- Vector width 128-bit. Fixed arrays with component-wise ops and swizzles. +- Parametric polymorphism by monomorphisation (Odin's model, no type classes, no + HKTs). Lowercase type names are variables, Capitalized are concrete — no sigil. + This is what makes ~map~/~filter~/~reduce~ and the monomorphic containers work. + The price, made explicit in spec-memory.md: with no constraints, a type variable + supports only what every type supports. ~=~, ~<~, ~+~, ~hash~ and ~print~ over an + unconstrained ~a~ are rejected, not silently instantiated — they are passed in as + function values. Compile-time interfaces, if they are ever wanted, come after the + base checker is stable. +- Function values split three ways (spec-memory.md): ~(Fn [T1 T2] R)~ is a plain + pointer with no environment — the only kind that crosses FFI or sits in a reload + cell; a *non-escaping* ~fn~ captures enclosing locals by value into a stack + environment, which is what ~reduce~ callbacks and ~handler-bind~ handlers use; + an *escaping* closure needs a heap environment and is still an open decision. +- No monads, no HKTs, no type classes. Effects are direct; error handling is + conditions plus ~Option~ and ~or-else~. Monadic sequencing, if ever wanted, is a + macro. +- ~any~ is an explicit opt-in tagged union, for heterogeneous containers and debug + printing. It is the only place a tag word is paid. +- Tagged unions, ~distinct~ types, bit sets. + +** Consequences +- Multimethods are *compile-time overload resolution*, not runtime dispatch. + Genuine runtime dispatch is on an explicit enum or tagged union. +- Conditions still work: a condition is a struct, the signal channel is a tagged + union, ~handler-case~ type matching resolves at compile time. +- The REPL works — the compiler knows each site's type and emits the right printer, + as in the OCaml and Haskell REPLs. +- Macros are unaffected; they run on syntax before typing. + +* Semantics kept +** Conditions and restarts +The headline feature. Four operators: ~handler-bind~, ~handler-case~, +~restart-case~, ~invoke-restart~. No condition class hierarchy — struct types plus +predicate matching. *Operational semantics: spec-conditions.md, which is +normative.* The ~500-line estimate below was for the operators alone and does not +include the explicit transfer lowering, which is compiler work. + +~signal~ does not mean "fail". It means: here is something notable, here is the +data, and here are the ways I know how to continue. Restarts are a menu; a handler +installed by an outer caller reads the data and picks one — or picks nothing and +returns, in which case the signaller simply carries on. + +| Handler does | Behaviour you get | +|---------------------+-------------------------| +| returns normally | accumulate and continue | +| invokes a restart | recover / retry / substitute | +| unwinds (~handler-case~) | try/catch | + +*Why this replaces most error types.* A condition signalled deep in a call stack +never appears in intermediate signatures. Nothing to thread, no ~From~ conversions, +no ~anyhow~ equivalent. Compare Rust, where every layer must name every error type +it passes through. + +*Accumulation.* A handler that records and invokes a ~continue~-style restart gives +error collection with no applicative or monad. This is how CL compilers report every +error in one pass. + +*Costs nothing when unused.* The handler stack is a linked list of stack-allocated +frames: ~handler-bind~ is a couple of stores, ~signal~ with no handler is a null +check. No allocation, safe inside a frame loop. + +*Condition objects live on the signalling frame's stack*, since nothing unwinds +before the handler runs. Because conditions are value structs, accumulating one into +an outliving array copies it. A pointer-to-condition would dangle. + +*Under static typing.* Restarts are dynamically scoped and named, so +~(invoke-restart 'skip-form)~ cannot be fully checked at compile time. Accept a +runtime error initially; a statically tracked restart set (as Zig tracks error sets) +is a nice-to-have, not a blocker. ~signal~ has type ~Unit~, ~invoke-restart~ and +~error~ have type ~Never~, and every restart clause shares one type with the +~restart-case~ body — so a ~restart-case~ in value position needs a fall-through +that produces the type or diverges. + +*Unwinding* is only the /transfer/ — invoking an outer restart. Lowered explicitly, +see Compilation. + +** Error handling, layered +1. ~Option~ for expected absence: lookup miss, empty collection, end of stream. +2. Conditions for exceptional failure where a caller may have a recovery policy. +3. ~Result~ where failure should be visible in the signature (parsers, fallible + pure functions). Error sets *inferred* from the body (Zig's model) so ~try~ + widens callers automatically; explicit sets required on exported functions. + +Rule of thumb: if you can name the one correct recovery at the point of failure, +return a ~Result~. If the answer is "depends who is calling", signal a condition. + +Mechanisms — two unwrap operators, because they are two different things (Zig's +split): +- ~try~ — unwrap ~Ok~, else early-return ~Err~, widening the enclosing error set +- ~some~ — unwrap ~Some~, else early-return ~None~ +- ~(ok-or opt err)~ / ~(ok res)~ — conversions, always *explicit*. This is the one + thing Rust got right and ~From~/~anyhow~ got wrong: the noise is not ~?~, it is + the implicit conversion machinery ~?~ demands. ~try~ does *not* accept an + ~Option~ in a ~Result~-returning function. +- ~some->~ (short-circuiting thread), ~or-else~, ~if-let~ +- ~errdefer~ — cleanup on the failure path only, pairs with arenas + +Async composes by nesting, no new mechanism: ~(try (await (http-get url)))~ — +~await~ unwraps the task, ~try~ unwraps the ~Result~ inside it. + +~try~ and ~some~ are macros expanding to early returns, and early return is the +explicit non-local-exit lowering, so all of this compiles identically on native and +wasm32. + +*Restarts go at the resync point, once* — the loop over top-level forms in a +parser, not inside every function below it. Intermediate frames stay silent about +restarts for the same reason they stay silent about conditions. + +No monads, no HKTs. Chaining that would want do-notation is a macro. + +*Async coupling:* with a state-machine transform the handler stack must live in the +/task/ state, not thread-local, or a handler established before an ~await~ is out of +scope after resumption. Cheap if designed in, painful later. + +** Multimethods +Compile-time overload resolution on argument types. No precedence lists, no method +combination, no MOP, no runtime dispatch table — see Types. + +** Macros +- Needs a ~&env~ equivalent: macros must see lexical environment (names of + locals in scope). Required by the step debugger. Decide now, painful to retrofit. +- Hygiene model: open decision. + +* Host language +*OCaml.* The compiler only; the runtime and stdlib are Flan with a few C +primitives, and are never bootstrapped away. + +The LLVM question does not bear on this: the release backend emits LLVM IR *as +text* and shells out to ~clang~, so no language needs LLVM bindings, and C++ or +Rust buy nothing here. What the choice actually turns on is that milestones 2–5 +are a reader, a typed IR, a checker and a tree-walking interpreter — variants and +exhaustive pattern matching, which is the one domain where OCaml is not a +preference but a clear win. There is also a menhir lexer/parser already started +in ~old-ocaml/~. + +The honest alternative is Rust, and it wins on exactly one axis: if the compiler +is a language you will not enjoy maintaining in three months, that outweighs +being 30% shorter. Nothing technical breaks either way. + +*Self-hosting is not a goal* and must not drive this. It appears nowhere in the +build sequence. For a game language it buys dogfooding at the price of a second +compiler to maintain forever. Choose as if the host language is permanent. + +* Milestone-2 primitives +The interpreter provides these; everything else is written in Flan. Keeping the +list short is the whole strategy — it is what makes the LLVM backend and the +wasm32 target cheap, because a primitive is the only thing implemented twice. + +| Primitive | Notes | +|------------------------+-------| +| ~argv~ | ~[string]~, borrowed, never freed | +| ~write-stdout~ | takes ~[u8]~; the ONE output primitive | +| ~exit~ | ~i32~ status | +| ~len~ ~at~ ~slice~ | on fixed arrays and slices | +| ~bytes~ | ~string~ → ~[u8]~, a view, no copy | +| ~bytes->f64~ ~bytes->i64~ | and the inverses, for printing | +| ~addr~ | address of a place | +| arithmetic, comparison, casts | per machine type | + +Printing is *not* a primitive. ~print-str~, ~print-f64~ and friends are Flan +functions over ~write-stdout~. A single overloaded ~println~ waits for milestone +5 — until then the acceptance programs name the type, because compile-time +overloading before the checker is stable is how a small language stops being one. + +*Entry point.* ~(defn main [args [string]] i32)~. Both the parameter and the +return type are optional: omitting ~args~ means the program ignores argv, +omitting the return type means ~Unit~ and an exit status of 0. sand.flan uses +the short form, calc-me the long one. + +*RNG is ours, not libc's.* ~rand-f32~ is a seeded PRNG implemented in Flan +(xoshiro or PCG), because a grid hash is only a regression test if the sequence +is byte-identical on native and wasm32. Decided here rather than at milestone 4, +since a headless deterministic sand run is the cross-target test. + +* Modules +There *are* modules — Odin calls them packages, and so does Flan. What is removed +is Clojure's ~ns~ form: no path that must mirror the directory, no +~:require~/~:refer~/~:as~/~:import~ vocabulary, no per-file namespace object. + +- *The directory is the package.* Every file in a directory shares one top-level + scope. Files in a package do not import each other, and top-level names are + order-independent, so mutually recursive functions need no forward declaration. +- *The package declaration is optional*, which is the one place Flan diverges + from Odin — Odin requires ~package foo~ as the first line of every file and + requires it to agree across the directory. Flan infers the package name from + the directory name, and ~(package parser)~ is written only when the name must + differ from the directory (a directory named ~flan-parser~, a scratch directory + with a name that is not an identifier). When present it must agree across the + directory, as in Odin. +- *A loose file in ~~/scratch/~ is a package of one.* No project file, no + manifest, no declaration. Open it, connect the REPL, start working. The + ceremony budget for "new file, running REPL" is zero — this is the requirement + the whole scheme is designed around, and it is why the declaration is optional + rather than mandatory. +- *Cross-package:* ~(import rl "vendor:raylib")~, and everything from it is + qualified ~rl/foo~. One form, one meaning, no unqualified-import mode. +- Collections in the path (~vendor:~, ~core:~) are Odin's, and are just + root-directory aliases. + +* Compilation +*Two backends and three paths.* The split is not dev-vs-release; it is +/does this code have a frame budget/. + +#+begin_src +expression eval: flan → typed IR → interpreter ~1ms +dev redefinition: flan → typed IR → .ll → llc → ld -shared → dlopen → cell store + ~16ms (MEASURED) +release build: flan → typed IR → .ll → clang --target={native,wasm32} +#+end_src + +*Hard requirement: eval is immediate.* Not "fast enough for a build" — immediate, +because the whole point of the live loop is that you see the result. 16ms is one +frame at 60fps and under the ~50ms threshold where a response stops feeling +instantaneous. The rule that buys it: *never invoke the ~clang~ driver on the dev +path.* + +*Expression eval* — ~C-c C-e~, calling a function, inspecting a var, running a +test — goes to the tree-walking interpreter. Sub-millisecond, no subprocess. This +is the permanent REPL backend, not a milestone-2 scaffold. + +*Dev redefinition* — ~C-c C-c~ on a function inside a running game — cannot use +the interpreter, because that code has an 8ms frame budget. It recompiles the one +function, links it, and does the atomic indirection-cell store. This is what the +Hot reload section has always described; the interpreter does not replace it. + +*Release* is whole-program AOT with direct calls and no cells. + +** Measured redefinition latency +Single function, x86-64, clang 20.1.8, 20 iterations each: + +| Step | Per call | Dev path? | +|------------------------------------------+----------+-----------| +| ~clang -shared~ (driver: compile + link) | 52.0ms | *no* | +| ~clang -c~ (driver: compile only) | 22.5ms | no | +| ~llc -filetype=obj~ | 13.9ms | yes | +| ~ld -shared~ from the ~.o~ | 2.4ms | yes | + +~llc~ + ~ld~ + ~dlopen~ ≈ *16ms*. The clang driver is the cost, not codegen — +it forks a second process and re-does argument and target resolution. Codegen +itself barely scales with function size: 721 lines of IR took 16.7ms against +13.9ms for 8 lines, because ~10ms is ~llc~ startup loading libLLVM. A realistic +redefined function lands in the same 15–17ms. + +This is why an in-process ORC JIT is not needed. It would take 16ms to ~3ms; the +difference is below perception, and the price is a version-pinned libLLVM and C++ +linkage from the host language, forever. + +** Redefinition must not stutter the running game +The 16ms is /not/ paid by the game thread. ~llc~ and ~ld~ are already separate +processes running on other cores. What the game process does is smaller: + +| Step | Cost | Game thread? | +|----------------------------------+-----------+--------------| +| ~llc~, ~ld~ | 16.3ms | no, separate processes | +| ~dlopen~ the new ~.so~ | ~0.1–1ms | *must not be* | +| atomic store into the cell | ns | yes, and free | + +Two design choices are load-bearing, and neither is automatic: + +1. *~dlopen~ happens on the reload thread.* It mmaps, relocates and takes the + loader lock; off-thread it blocks nobody, because the game thread is not doing + dynamic linking. Load with ~RTLD_NOW~ so lazy PLT resolution cannot ambush the + game thread on a later first call. +2. *Publish at a frame boundary, in a batch.* This matters more than the + threading. Storing each cell the moment it is ready lets the game observe a + half-applied redefinition — two functions that changed together applied one + frame apart, or a function swapped mid-frame with half the entities already + updated by the old code. Instead the reload thread stages the complete set of + new pointers and sets a flag; the game loop tests the flag once at the top of + the frame and does N stores. One relaxed atomic load per frame when nothing + changed. + +Residual cost: the first call into new code page-faults and misses i-cache. Tens +of microseconds, not visible. + +*** Dev architecture: daemon plus agent +- *Compiler daemon*, a separate process: the OCaml frontend, the nREPL server, + and the ~llc~/~ld~ invocations. Editors talk to this. +- *Reload agent*, linked into the game binary: a socket listener, ~dlopen~, and + the frame-boundary cell publisher. A few hundred lines, and no OCaml runtime + in the game. + +This is why *the dev runtime is multithreaded* — it needs the reload thread. That +is settled, and is independent of whether the /language/ exposes threads, which +is still open decision #4. + +It also bears on whether the interpreter survives: if the agent can ~dlopen~ and +call anything in 16ms, then even "eval this expression against live game state" +can be a compiled ~.so~, and no interpreter is needed inside the game process. + +** Why LLVM IR as text +| | text ~.ll~ → ~clang~ | libLLVM bindings | emit C | +|---+---+---+---| +| Build dependency | a ~clang~ on PATH | matching libLLVM, version-pinned, C++ linkage | any C compiler | +| Breaks on LLVM upgrade | no | routinely | no | +| Debuggable | ~.ll~ is readable | print-from-memory | readable, but lies about origin | +| In-process JIT | *no* | yes (ORC) | no | +| Control of layout / ABI / tail calls | full | full | poor | + +The only column text loses is the JIT one, and the measurement above shows the +loss is ~13ms — below perception. ORC remains addable later behind the same typed +IR without touching the language, but nothing currently argues for it. + +** The interpreter cannot run sand +Do not plan around it. 200 × 280 = 56,000 cells, scanned by ~game-update~ and +again by ~game-draw~ — ~112,000 interpreted cell-visits per frame against an +8.3ms budget at 120fps. At an optimistic 100ns per visit (environment +allocation, argument binding, two index computations, a compare) that is 11ms +before ~settle~, ~paint~, or a single raylib call. Expect 20–30fps. + +This is an estimate, not a measurement, which is why *milestone 2 exits with a +measured interpreter throughput number* — before milestone 4 depends on it. +Milestone 4's interactive acceptance test runs on the compiled dev build; the +interpreter is not in that loop. + +*** Open: does the interpreter survive milestone 3? +Now that compiled redefinition is measured at 16ms, the case for a /permanent/ +interpreter is weaker than it looked. 16ms is perceptually instant for expression +eval too, and one backend removes a standing obligation — two backends must agree +on observable behaviour forever, and every divergence is a bug that reproduces in +only one of them. + +Against dropping it: the interpreter is clearly right for milestone 2 (far less +work than an LLVM backend, better error messages, no linking), and the +instrumentation-based step debugger wants it. Decide at milestone 3 exit on +measured numbers, not now. + +All three paths share the frontend and the typed IR and must agree on observable +behaviour. That agreement is what the acceptance programs test. + +- Non-local exit lowered *explicitly* (result propagation + branch targets), not + via platform unwinding. Same on both targets, no dependency on the WASM + exception-handling proposal. Escape analysis narrows which functions pay for it. +- Stdlib written *in Flan*, not the host language. A few hundred primitives per + backend, everything else on top. This is what keeps a second backend cheap. + +* Targets +- *Desktop*: AOT to native, x86-64 and arm64. Primary development target. +- *WASM*: AOT build artifact only, *not interactive-first* — no REPL, no hot + reload, no debugger there. That is not the same as untested: every runtime or + ABI feature ships with automated wasm32 tests in CI from the first one, because + a divergence found at ship time is a rewrite. +- Host binary links raylib natively; web links raylib via emscripten. + +** One narrow host ABI, implemented twice +The portability risk is the host interface, not the language. Divergence points: +- Filesystem — pack assets, one abstraction, never touch paths +- Threads — decide now; retrofitting is worse than the reverse +- Blocking — browser main thread cannot block +- Audio/input/window — constrain to the raylib subset identical on both + +* Dev vs release builds +Deliberately different. + +| | Dev | Release | +|---------+---------------------------+------------| +| Backend | interpreter /and/ LLVM | LLVM/clang | +| Calls | indirection cells | direct | +| Code | never freed | static | +| Frames | shadow stack | none | +| Structs | version word | none | +| Reload | yes | no | + +Build and run the release config regularly, not just at ship time. + +* Hot reload +Every cross-function call goes through an *indirection cell*; redefinition is one +atomic pointer store. *Old code is never unloaded*, so a thread mid-execution +finishes safely in the old version. + +This is the fix for jank issue #947 (segfault redefining a running loop's +function plus its callee — their JIT relinks and unloads under a running thread). + +** What redefinition cannot do +Patch a mid-execution frame and continue at the same PC — its register +allocation belongs to the old compilation. No implementation does this. "Resume" +means re-entering from an established restart point. + +* Tooling +Server speaks *nREPL* (bencode over socket) — the transport and the core ops +(~eval~, ~load-file~, ~describe~, ~interrupt~) are genuinely reusable, and that is +what the ~500–1000 lines buys. It does *not* buy CIDER/Conjure/Calva +compatibility: their useful operations assume Clojure-shaped vars, namespaces, +nses-of-symbols and middleware. Treat "speaks nREPL" as milestone 7a and "an +editor client that is pleasant" as a separate milestone 7b. In the dev runtime: +- eval string in package; compile form/file with source locations +- completion, arglist, describe, find-definition +- backtrace + restarts; interrupt + +** Emacs client +Focused client, ~3–5k lines. Do *not* fork CIDER (~30k lines elisp, deeply +Clojure-coupled) — reference it. ~clojure-mode~-derived major mode, overlay +rendering, hydra for stepping bindings (~transient~ is the maintained +alternative). + +** Step debugger +Instrumentation-based, like CIDER's — macroexpansion wraps subforms with a +breakpoint that messages the editor and blocks. No native debug info needed. +Limit: only instrumented code. + +** Pause on exception +Better than CIDER's, because ~handler-bind~ has not unwound the stack. Dev-mode +global handler messages the editor and blocks with the full live stack and all +restarts available. Fix the function, resume via restart. + +* Build sequence +Deliberately ordered so each step is runnable and the next one cannot start until +the previous checker is stable. The failure mode this exists to prevent is +building the whole live environment at once. + +1. *Freeze the model.* spec-memory.md and spec-conditions.md — done before any + code. Fixed arrays, non-owning slices, move-only ~Vec~/~Map~, allocators, + ~Ptr~, explicit ~clone~; the six restart cases. /Done./ +2. *Run calc-me.flan on the interpreter.* Reader, typed IR, checker, + tree-walking backend. /Exit criterion includes a measured throughput number/ + — interpreted calls per second on a tight loop — because milestone 4's frame + budget depends on it (see Compilation). Packages, structs, ~(Ptr T)~ + ~addr~, byte slices, + ~at~/~len~, ~while~, ~set~ on the fixed place list, ~cond~, ~match~, ~Option~ + + ~some~, ~i32~/~u8~/~f64~, recursion, argv, stdout. No allocator, no ~Vec~, + no generics, no user macros, no FFI, no window. Headless, so the acceptance + test is a table of expression/result pairs. +3. *Emit LLVM IR and pass the same calc-me test AOT*, on native and wasm32 in CI. + Both backends, one test table, one narrow host ABI (argv, stdout, exit). This + is where the second target gets proven — while there is almost nothing to port. +4. *Run sand.flan.* Fixed 2-D arrays, ~dotimes~, ~defer~, and typed FFI to + raylib including keyword→enum coercion. Acceptance test twice: headless (N + frames, hash the grid — runnable in CI on both targets) and interactive at + 120 fps. +5. *Generics and macro expansion*, once the base checker is stable. ~defmacro~, + ~&env~, hygiene. Until here, ~when~/~unless~/~until~/~cond~/~dotimes~ are + special forms in the compiler. +6. *Allocators, ~Vec~/~Map~, ~Result~/~try~/~errdefer~, then conditions and + restarts* against spec-conditions.md, with dedicated tests per numbered case. +7. *Hot reload* — free in the interpreter, indirection cells for compiled dev + builds, with the compatibility limits written down and enforced: signature + changes, struct layout changes, live callbacks held by C, captured environments. +8. *Debugger, nREPL, async* — last, and 8 splits into transport (8a) and editor + client (8b). + +Milestones 1–4 are the project. Everything from 5 on is optional in the sense +that a language that stops there is still usable; nothing before 5 is. + +Ordering note: sand cannot be first even though it is the better demo, because +it needs raylib FFI, keyword→enum coercion and a window before a single line of +it runs. calc-me needs argv and stdout. + +* Runtime budget +Scoped to *milestones 1–3 only*. The earlier version of this table put the whole +plan — conditions, explicit transfer lowering, macros, hot reload, debugger +support — at 15–25k, which is not credible: each of those is architecture work +that touches the frontend, the IR and the backend at once. + +| Piece | Lines | Milestone | +|-----------------------------------+--------+-----------| +| Allocators | 2–3k | 2 | +| Core data (fixed/slice/Vec/Map) | 2–3k | 2 | +| Reader + frontend + checker | 5–8k | 2 | +| LLVM IR lowering | 3–5k | 2 | +| raylib FFI + host ABI ×2 | 1–2k | 3 | +| *Subtotal, a language that runs sand* | 13–21k | | + +Beyond that, estimated but not budgeted, because these are the parts that are +architecture rather than volume: + +| Piece | Note | +|----------------------------------------+------| +| Macroexpander + ~&env~ | touches the reader and the checker | +| Generics / monomorphisation | touches the whole checker | +| Conditions + explicit transfer lowering | frontend *and* IR *and* backend | +| Hot reload cells + compatibility rules | changes the calling convention | +| Debugger instrumentation + nREPL | needs ~&env~ and source locations | + +Janet is 36k including a bytecode VM, and Janet has no static types, no +monomorphisation, no restarts and no reload. + +* Open decisions +None of these block milestone 2. The milestone each one must be answered by is +marked. + +1. *Host language: OCaml or Rust* — the only thing blocking the scaffold. See + Host language. /Milestone 2./ +2. Macro hygiene model. /Milestone 5./ +3. Escape analysis: automatic promotion of escaping frame-arena values, or + explicit. /Milestone 6./ +4. Threads *in the language*: yes or no, decided before the host ABI. (The dev + /runtime/ is multithreaded regardless — it needs a reload thread. Settled; see + Compilation.) /Milestone 3, before the host ABI./ +5. /Escaping/ closures. /Milestone 6./ Settled in spec-memory.md: + ~(Fn ...)~ is a bare pointer with no environment (callbacks, reload cells, FFI), + and a *non-escaping* ~fn~ captures by value into a stack environment — which is + what makes ~handler-bind~ handlers able to see enclosing locals, without which + conditions are not worth building. Still open: a closure that is stored, + returned, or pushed into a container. Which allocator owns its environment, and + what happens when the frame arena resets? Decided together with #3, because + the same escape analysis classifies both. +6. Hot-reload compatibility rules. /Milestone 7./ Cells cover a function + body changing. Not covered: a changed signature, a changed struct layout with + live instances, a function pointer already handed to C, a captured environment, + and redefining a ~defvar~. Each needs an answer of the form "rejected", + "accepted with a migration", or "accepted and the old code keeps running". +7. Does the interpreter survive milestone 3, or is the compiled path the only + backend? /Milestone 3, on measured numbers./ See Compilation. +8. ~(Option a)~ /settled:/ an ordinary stdlib union with ~Some~/~None~; the + compiler niche-optimises ~(Option (Ptr T))~ to a nullable pointer. The + CL-vs-Clojure truthiness question is moot under static typing. + +*Settled since the first draft* (see the specs): +- Module system → the directory is the package, Odin's model. No ~ns~ form. See + Modules. +- Zero values → ZII by default, with an opt-out. A declaration with no + initialiser is all-bytes-zero; ~(zeroed)~ re-zeroes something later; ~uninit~ + opts out for large buffers about to be overwritten. It is a ~memset~, not a + ~memcpy~; zeroed globals live in BSS and cost nothing. +- Package declaration → optional, inferred from the directory name. See Modules. +- Keywords at typed call sites → yes. ~:space~ resolves at compile time against + the parameter's enum type, ~rl/key-space~ names the same value, and a typo is + a compile error checked against the enum's members. No runtime cost. Needs the + FFI enum declared, so it lands with milestone 4. +- Dev redefinition latency → ~16ms, measured: ~llc~ + ~ld -shared~ + ~dlopen~, + never the ~clang~ driver, ~dlopen~ off the game thread, cells published in a + batch at a frame boundary. See Compilation. +- Dev backend → interpreter for milestone 2 certainly. Whether it /survives/ + milestone 3 is open, not settled — see Compilation. +- ~set~ on places → a fixed list of assignable forms, not ~setf~. +- Loop story → imperative ~while~/~for~ with ~break~/~continue~ and ~return~; + ~loop~/~recur~ only if it later earns its place. sand.flan is ported. +- Generic parameters → inferred at call sites, no explicit instantiation; and no + type classes, so unconstrained operators over a type variable are rejected. + +* Unverified claims in this plan +- *Scope is the biggest risk, not any single feature.* A language, inference, + monomorphisation, macros, an explicit-memory runtime, conditions/restarts, hot + reload, a debugger, nREPL tooling, native and WASM — each is reasonable, the + set is not one project. The build sequence above exists because of this; if + something has to give, it gives from milestone 4 upward. +- Line-count estimates are extrapolations. +- LLVM → wasm32 with manual memory is validated by Odin shipping it; not yet + validated for s-expression macros + conditions/restarts on top. diff --git a/sand.flan b/sand.flan new file mode 100644 index 0000000..4b81efd --- /dev/null +++ b/sand.flan @@ -0,0 +1,151 @@ +;;;; Falling sand — Flan port of the Odin/Janet/Lisp/jank versions in ~/Development/fnm. +;;;; +;;;; THE SECOND ACCEPTANCE PROGRAM — build sequence milestone 4. calc-me.flan +;;;; comes first: sand cannot run at all until raylib FFI, keyword->enum +;;;; coercion and a window exist, and none of those should be on the critical +;;;; path to "the language runs something". +;;;; +;;;; It is tested twice: headless (N frames, hash the grid — the version CI runs +;;;; on native and wasm32) and interactive at 120 fps. +;;;; +;;;; Note what it still deliberately does not use: no Vec, no Map, no generics, +;;;; no user-written macros, no conditions, no allocator other than the stack +;;;; and static storage. +;;;; +;;;; Notation reminders (see plan.org and spec-memory.md): +;;;; [n T] fixed array, length n, element T — a VALUE, copies +;;;; [T] slice, ptr+len, non-owning (Vec T) owning, move-only +;;;; (Ptr T) pointer (Handle T) generational handle +;;;; types are inline name/type pairs, as in `let` and `defstruct` +;;;; an omitted return type means Unit +;;;; lowercase in a TYPE position is a type variable; in a LENGTH position +;;;; it is an ordinary compile-time value, so [rows [cols u32]] is unambiguous + +(import rl "vendor:raylib") ; directory = package; declaration optional + +(defconst screen-width 1400) +(defconst screen-height 1000) +(defconst cell-size 5) +(defconst gravity 0.05) +(defconst rows (/ screen-height cell-size)) +(defconst cols (/ screen-width cell-size)) +(defconst brush-size 10) + +;; Packed 0xRRGGBBAA. A cell of 0 means empty, so no Option and no tag word. +(defconst colors [4 u32] [0xE6B800FF 0x3B6E8CFF 0xA83232FF 0xCC6B1FFF]) + +;; Flat, unboxed, statically sized. No headers, so these are exactly +;; rows*cols*4 bytes each — the same memory the Odin port has. Fixed arrays are +;; values, so `(set grid (zeroed))` overwrites in place rather than reallocating. +;; No initialiser means all-bytes-zero (plan.org, zero values), so these are +;; BSS and cost nothing to start. `(zeroed)` below is the explicit spelling for +;; re-zeroing later — a memset, not an allocation. +(defvar grid [rows [cols u32]]) +(defvar velocity [rows [cols f32]]) +(defvar current-color u32) + +(defn clear-grid [] + (set grid (zeroed)) + (set velocity (zeroed))) + +(defn empty-at? [row i32 col i32] bool + (= 0 (at grid row col))) + +;; Locals are assignable places (spec-memory.md); parameters are not. +(defn paint [] + (let [m (rl/get-mouse-position) + row (/ (i32 (.y m)) cell-size) + col (/ (i32 (.x m)) cell-size) + half (/ brush-size 2)] + (dotimes [x brush-size] + (dotimes [y brush-size] + (let [r (+ y (- row half)) + c (+ x (- col half))] + (when (and (>= r 0) (< r (- rows 1)) + (>= c 0) (< c (- cols 1)) + (empty-at? r c) + (< (rand-f32) 0.5)) + (set (at grid r c) (nth colors current-color)) + (set (at velocity r c) 1.0))))))) + +(defn move-grain [from-row i32 from-col i32 + to-row i32 to-col i32 + vel f32] + (set (at grid to-row to-col) (at grid from-row from-col)) + (set (at grid from-row from-col) 0) + (set (at velocity to-row to-col) vel) + (set (at velocity from-row from-col) 0.0)) + +;; Move the grain at [row col] as far down as it can, sliding to a free +;; diagonal neighbour when the cell below is taken. +;; +;; Imperative `while` with early `return`, not loop/recur — see plan.org +;; "Loop story". The recur version read as a tail call but was a countdown +;; over a mutable scan position, which is what a while loop is. +(defn settle [row i32 col i32] + (let [vel (+ gravity (at velocity row col)) + y (min (- rows 1) (+ row (i32 vel)))] + (while (> y row) + (when (empty-at? y col) + (move-grain row col y col vel) + (return)) + (let [left? (and (> col 0) (empty-at? y (- col 1))) + right? (and (< col (- cols 1)) (empty-at? y (+ col 1)))] + (when (or left? right?) + (let [side (cond + (not left?) 1 + (not right?) -1 + :else (if (< (rand-f32) 0.5) 1 -1))] + (move-grain row col y (+ col side) vel) + (return)))) + (set y (- y 1))) + ;; Nowhere to fall: reset the accumulated velocity and stay put. + (set (at velocity row col) 0.0))) + +;; Every cross-function call in a dev build routes through an indirection cell, +;; so redefining this from the REPL reaches the running loop on the next frame. +;; No `varfn` (Janet), no `let update = ref` (OCaml), no var-routing (jank). +;; Release builds compile the same source to direct calls. +;; +;; A cell holds an (Fn ...) — a plain function pointer, no captured environment; +;; this one is (Fn [] Unit), `settle`'s is (Fn [i32 i32] Unit). Redefining +;; `settle` while `game-update` is mid-frame is safe +;; because old code is never unloaded; changing its SIGNATURE is not, and the +;; reload rejects it. See plan.org "What redefinition cannot do". +(defn game-update [] + (when (rl/key-pressed? :r) (clear-grid)) + (when (rl/key-down? :space) (paint)) + (when (rl/key-released? :space) + (set current-color (% (+ current-color 1) (len colors)))) + ;; Bottom-up, so a grain settles at most once per frame. + (let [row (- rows 2)] + (while (>= row 0) + (dotimes [col cols] + (unless (empty-at? row col) + (settle row col))) + (set row (- row 1))))) + +(defn game-draw [] + (rl/clear-background rl/black) + (dotimes [row rows] + (dotimes [col cols] + (let [c (at grid row col)] + (unless (= 0 c) + (rl/draw-rectangle (i32 (* col cell-size)) + (i32 (* row cell-size)) + cell-size cell-size + (rl/get-color c)))))) + (rl/draw-fps 20 20)) + +(defn main [] + (rl/set-trace-log-level :warning) + (rl/init-window screen-width screen-height "SAND") + (defer (rl/close-window)) + (rl/set-target-fps 120) + ;; Bare (defn main []) — argv and the i32 status are both optional. + ;; Nothing in this loop allocates, so context/temp is never even touched. + (until (rl/window-should-close?) + (game-update) + (rl/begin-drawing) + (game-draw) + (rl/end-drawing))) diff --git a/spec-conditions.md b/spec-conditions.md new file mode 100644 index 0000000..9c6902e --- /dev/null +++ b/spec-conditions.md @@ -0,0 +1,115 @@ +# Spec 2 — Conditions and restarts, operational semantics + +Status: **frozen** for the six hard cases below. Everything not listed here is +still open, but nothing in the implementation may depend on the unlisted parts. + +Four operators: `handler-bind`, `handler-case`, `restart-case`, `invoke-restart`. +No condition class hierarchy — condition types are structs, matching is by type +plus an optional predicate. + +## 1. `signal` returns `Unit` + +`(signal c)` has type `Unit`, always. When every applicable handler returns +normally without transferring, `signal` returns `Unit` and the signalling +function simply carries on. This is the accumulation case. + +The alternative — `signal` producing a value supplied by the handler — was +rejected: it forces every signal site to declare a default value and a result +type, which is a much heavier language for one convenience. + +The consequence is visible in the syntax. A `restart-case` in value position +must produce its type on the *fall-through* path too: + +``` +(defn load-texture [path string] (Handle Texture) + (if (file-exists? path) + (rl/load-texture path) + (restart-case + (do (signal (AssetMissing {:path path})) + (abort "unhandled AssetMissing")) ; fall-through must not return + (use-placeholder [] placeholder-texture) + (retry [] (load-texture path))))) +``` + +`abort` has type `Never`, which unifies with anything. Any expression of type +`Never` (a `return`, a call to a diverging function) is equally acceptable there. + +## 2. No handler + +`signal` with no matching handler on the handler stack is a **no-op** that +returns `Unit`. It does not abort, does not print, does not enter a break loop. +`(error c)` is the diverging variant: same lookup, but with type `Never` and, if +nothing handles it, it enters the dev-build break loop or aborts in release. + +The cost when unused is the intended one: `handler-bind` is a couple of stores +onto a stack-allocated linked-list frame, and `signal` with an empty stack is a +null check. + +## 3. Restart signatures + +``` +(restart-case BODY + (name [p1 T1 p2 T2] BODY-1) + ...) +``` + +- Parameters are annotated inline, like any other binding form. +- **Every clause body and the `restart-case` body must have the same type**, and + that is the type of the whole form. +- `(invoke-restart 'name arg ...)` has type `Never` — it never returns to the + invoking handler. Control resumes at the `restart-case`, which yields the + clause's value to *its* continuation. +- Argument count and types are checked at **runtime** in the first + implementation, because restarts are dynamically scoped and named. A statically + tracked restart set (Zig's error-set model) remains a nice-to-have. + +## 4. Name shadowing + +Restart lookup walks the dynamic restart stack from innermost outward and takes +the **first** frame offering the name. An inner `restart-case` therefore shadows +an outer one with the same name for the duration of its body. This is what makes +"restarts go at the resync point" composable: an inner parser's `skip-form` is +found before an outer one's. + +`(find-restart 'name)` returns `(Option Restart)` so a handler can test before +committing; `(compute-restarts)` lists the visible frames for the debugger. + +## 5. Cleanup during a transfer + +Invoking a restart transfers control outward past zero or more frames. + +- `defer` forms in every frame between the `invoke-restart` and the target + `restart-case` **do run**, innermost first, before the clause body starts. +- `errdefer` forms **do not run**. `errdefer` is bound to the `Result` failure + path (`try` returning `Err`) only. A restart transfer is not a failure — it is + a chosen recovery, and the recovery may well want the resource. +- The condition object lives on the *signalling* frame's stack. Nothing has + unwound when a handler runs, so it is valid there; but once a transfer starts, + the signalling frame dies. Anything a handler keeps must be copied out + (conditions are value structs, so `(push errors c)` copies). + +## 6. Crossing compiler-generated frames + +Transfer is lowered **explicitly** — result propagation plus branch targets — not +via platform unwinding, so that native and wasm32 behave identically. That means +every function on the path between the invoke and the target must be +transfer-aware: it returns a discriminated "normal value / transferring to frame +N" result, checks it after each call, and forwards. + +- The compiler marks a function transfer-transparent if it can call, directly or + indirectly, anything that may invoke a restart. Escape analysis narrows this + set; functions outside it pay nothing. +- **Foreign frames cannot be crossed.** A restart transfer whose path passes + through a C frame (a raylib callback, an `extern` function calling back into + Flan) is a runtime error, not undefined behaviour. Handlers installed across an + FFI boundary must therefore either return normally or use `handler-case` + installed inside the callback. +- With the async state-machine transform, the handler and restart stacks live in + the **task** state, not thread-local, so a handler established before an + `await` is still in scope after resumption. + +## What this does not settle + +Condition inheritance/predicate-matching details, the break-loop UI, restart +interaction with threads, and whether `handler-case` should be a macro over +`handler-bind` + a transfer. None of these block milestone 5. diff --git a/spec-memory.md b/spec-memory.md new file mode 100644 index 0000000..17722bb --- /dev/null +++ b/spec-memory.md @@ -0,0 +1,154 @@ +# Spec 1 — Ownership, containers, and copies + +Status: **frozen**. Closes plan.org open decisions #6 and #10, and resolves the +contradiction between "value structs copy on assignment" and owning containers. +Everything else in the design references this vocabulary. + +## The four container types + +| Notation | Layout | Assignment | Owns storage | Allocator | +|-----------|-------------------|------------|--------------|-----------| +| `[n T]` | n contiguous `T` | copies | no (inline) | — | +| `[T]` | ptr + len | copies the *view* | no | — | +| `(Vec T)` | ptr + len + cap | **moves** | yes | stored | +| `(Map K V)` | open-addressed, flat key/value arrays | **moves** | yes | stored | + +- `[n T]` is a value. It lives wherever it is declared, copies on assignment and + on pass-by-value, and is what `defconst colors [4 u32] ...` and + `(defvar grid [rows [cols u32]] ...)` are. +- `[T]` is a **non-owning slice**: a borrowed window into a `[n T]`, a `(Vec T)`, + or a literal in read-only memory. Copying a slice copies ptr+len, never the + elements. A slice may be `const`-qualified; freeing through one is not possible + because a slice has no allocator and no `cap`. +- `(Vec T)` and `(Map K V)` are **move-only**. Binding, passing, or returning one + transfers ownership; the source binding is dead afterwards and using it is a + compile error. There is no shallow copy, so there is no double free. + +## Copying is always explicit + +`(clone x)` produces an independent deep copy of a `Vec`/`Map` using the current +allocator; `(clone x alloc)` names one. Value types (`[n T]`, structs of value +types, primitives) need no `clone` — assignment already copies them. + +A struct containing a `Vec` field is itself move-only. Ownership is structural, +not declared: a type is a value type iff all of its fields are. + +## Borrowing + +- `(as-slice v)` / `(as-slice v lo hi)` view a `Vec` or fixed array as `[T]`. +- A slice is invalidated by any operation that may reallocate the owner (`push`, + `put`, `reserve`). This is **not checked** in the first implementation; dev + builds carry a generation word on `Vec` and trap on use of a stale slice. +- Cross-referencing long-lived objects uses `(Handle a)` into a pool, never a + raw pointer or slice. A stale handle is detectable. + +## Taking an address + +`(addr x)` yields `(Ptr T)` for any assignable place `x` — a local, a global, a +field, an element. The pointer is non-owning and does not extend anything's +lifetime, so `addr` of a local is only valid while that frame lives. This is the +same escape question as case 3 below and is checked by the same analysis; until +that analysis exists, `addr` of a local may not be stored or returned. + +`addr` is how a value struct is shared mutably without an allocator — recursive +descent over a cursor, an entity passed down a call chain — and it is why +milestone 2 needs no heap at all. + +## Places — what `set` accepts + +A fixed set of assignable forms, not a `setf`-style extensible place mechanism: + +``` +(set x v) ; a local or a defvar +(set (.field x) v) ; struct field; x may be a struct, (Ptr S) or (Handle S) +(set (at a i ...) v) ; fixed array, slice, or Vec element +(set (get m k) v) ; map entry +(set (deref p) v) ; whole-object store through a pointer +``` + +`.field` and `at` auto-deref exactly one pointer or handle level, which is what +makes `(set (.hp e) ...)` legal when `e : (Ptr Enemy)` and illegal when +`e : Enemy` bound by value. + +**Mutating something you matched.** Pattern bindings bind *values*, so a matched +struct is a copy. To mutate in place, obtain a pointer first — the pointer is +visible in the type: + +``` +(match (resolve w h) ; (Option (Ptr Enemy)) + (Some e) (set (.hp e) ...) ; e : (Ptr Enemy), field access derefs + None ...) +``` + +`deref` yields a value; `resolve` yields a pointer. Both are overloaded on +`(Ptr a)` and `(Handle a)` and resolve at compile time. + +## Generics + +Parametric polymorphism is monomorphisation, with **no type classes and no +constraints**. The consequence is a hard rule: + +> A type variable `a` supports only what every type supports: move, `clone`, +> field-free storage. It does **not** support `=`, `<`, `+`, `hash`, or `print`. + +Anything else is passed in explicitly as a function value: + +``` +(defn largest [xs [a] gt (Fn [a a] bool)] (Option a) ...) +``` + +Ordered/arithmetic operators over `a` are therefore rejected, not silently +instantiated. The alternatives — compile-time interfaces, or intrinsics +restricted to primitives — are deliberately deferred until the base checker is +stable (build sequence milestone 4). + +Type arguments are **inferred at call sites** from the argument types; there is +no explicit instantiation syntax in the first implementation. A type variable +that appears only in the return type is therefore an error. + +## Function values + +Three cases, split by whether the value escapes the frame that made it. + +**1. `(Fn [T1 T2] R)` — a plain function pointer.** No captured environment, no +allocation, C calling convention plus the implicit allocator argument. This is +what raylib callbacks, hot-reload indirection cells, and function *parameters* +use. A top-level `defn` is one, so `(largest hps >)` passes `>` at `i32` +directly. This is the only function type that may cross an FFI boundary or sit +in a reload cell. + +**2. Non-escaping `fn` — captures by value into a stack environment.** A `fn` +whose value provably does not outlive the frame that created it gets an +environment allocated in that frame and captures the named locals **by value** +at the point of creation. No heap, no allocator, no lifetime question. This +covers essentially every lambda in practice: + +- callbacks to `reduce` / `filter` / `each` / `map`, which consume them and return +- comparators passed to a function that does not store them +- `handler-bind` handler bodies + +That last one is not a convenience. A handler must be able to see the enclosing +locals — `(fn [c] (push errors c) (invoke-restart 'skip-form))` capturing a local +`(Vec ParseError)` *is* the accumulation pattern, and conditions are not worth +building without it. Handlers are strictly non-escaping: the `handler-bind` frame +outlives every call to them. + +Captured `Vec`/`Map` are captured **by pointer**, not moved, since the capture +does not outlive the owner. A non-escaping `fn` is therefore not itself an owner. + +**3. Escaping closures — still open.** A `fn` stored in a struct, pushed into a +container, or returned needs a heap environment and an answer to "which allocator +owns it, and what happens when the frame arena resets". Not settled; see +plan.org open decisions. Escape analysis (open decision #4) is the same analysis +that classifies cases 2 and 3, so they are decided together. + +**Early exit inside a `fn`.** `try`, `some`, and `return` in a `fn` body exit the +`fn`, not the enclosing function — a `fn` is a function. Code that wants to +propagate out of a loop uses an imperative loop form, not a callback. + +## Allocators + +The allocator is part of the calling convention (`context/allocator`, +`context/temp`). `Vec` and `Map` record the allocator they were created with, so +`free` and `clone` never need it named again. No core operation allocates +implicitly. diff --git a/syntax-sketch.flan b/syntax-sketch.flan new file mode 100644 index 0000000..23f1d03 --- /dev/null +++ b/syntax-sketch.flan @@ -0,0 +1,198 @@ +;; Syntax sketch. Not final — illustrates the decisions in plan.org. +;; +;; Rules held here: +;; - every type notation reads as exactly ONE data item +;; - types are inline name/type pairs, as in `let` and `defstruct` +;; - an omitted return type means Unit (a real zero-sized type, not C's void) +;; - lowercase type names are variables, Capitalized are concrete +;; - no `!` convention (nothing is immutable), no `->`, no sigils +;; +;; Normative references: spec-memory.md (ownership, containers, places, +;; generics, function values) and spec-conditions.md (restart semantics). + +(import rl "vendor:raylib") ; directory = package, declaration optional; + ; imports are always qualified rl/foo + +;; ── Type notation ───────────────────────────────────────────────────── +;; [4 f32] fixed array — a value, copies on assignment +;; [f32] slice, ptr+len — a NON-OWNING view, copies the view only +;; (Vec f32) owning growable, ptr+len+cap — MOVE-ONLY, carries allocator +;; {string i32} owning hashmap — move-only, shorthand for (Map string i32) +;; +;; Braces are read by position: in a TYPE position {K V} is a map type; in a +;; VALUE position {:field v ...} is a struct or condition literal. There is no +;; map literal — a map is built with make-map and an allocator. +;; (Ptr World) pointer +;; (Fn [f32] bool) function pointer, no captured environment +;; (Option a) union from the stdlib +;; (Handle a) generational handle into a pool +;; +;; A struct is a value type iff all its fields are. One Vec field makes it +;; move-only. Copying an owning container is always explicit: (clone v). + +(defalias Vec2 [2 f32]) +(defalias Vec4 [4 f32]) + +;; ── Structs are value types with C layout, no header word ───────────── +(defstruct Enemy + [pos Vec2 + vel Vec2 + hp i32 + spr (Handle Texture)]) + +(defunion Shape + [(Circle [r f32]) + (Rect [w f32 h f32])]) + +;; ── Locals inferred; only signatures are annotated ─────────────────── +(defn area [s Shape] f32 + (match s + (Circle r) (* PI r r) + (Rect w h) (* w h))) + +;; ── Lowercase = type variable. Monomorphised at each call site ──────── +;; There are no type classes, so `a` supports only what EVERY type supports. +;; Ordering is not that — it is passed in as a function value. Type arguments +;; are inferred from the argument types; there is no explicit instantiation. +;; The inner `fn` captures `gt`, a parameter: legal because it does not outlive +;; this frame (spec-memory.md, non-escaping fn). +(defn largest [xs [a] gt (Fn [a a] bool)] (Option a) + (if (> (len xs) 0) + (Some (reduce (fn [x y] (if (gt x y) x y)) (nth xs 0) xs)) + None)) + +;; (largest hps >) — `>` at i32 is an ordinary function value +;; (largest es (fn [x y] (> (.hp x) (.hp y)))) + +;; Parameters are immutable values; pass a pointer to mutate. `[Enemy]` is a +;; borrowed slice — centroid neither owns nor frees the storage. +(defn centroid [es [Enemy]] Vec2 + (/ (reduce (fn [acc e] (+ acc (.pos e))) [0 0] es) + (f32 (len es)))) + +;; ── Handles, not pointers, for anything cross-referenced ────────────── +;; Pattern bindings bind VALUES, so matching a struct out of a pool would give +;; a copy and `set` would mutate the copy. `resolve` yields (Option (Ptr a)) +;; instead, and the pointer is visible in the binding's type. `deref` is the +;; by-value counterpart. Both are overloaded on (Ptr a)/(Handle a). +(defn damage [w (Ptr World) h (Handle Enemy) amount i32] + (match (resolve w h) + (Some e) (set (.hp e) (- (.hp e) amount)) ; e : (Ptr Enemy), field derefs + None (log "stale enemy handle"))) + +;; ── Error handling is layered ───────────────────────────────────────── +;; Option expected absence: lookup miss, empty collection, end of stream +;; Result failure that belongs in the signature; error set inferred +;; Condition failure where the CALLER owns the recovery policy +;; +;; Rule: if you can name the one correct recovery at the point of failure, +;; return a Result. If the answer is "depends who's calling", signal. + +;; `some` unwraps Some, else early-returns None. +(defn player-weapon [w (Ptr World)] (Option Weapon) + (let [p (some (find-player w)) + s (some (slot (.inventory p) 3))] + (Some (.weapon s)))) + +;; `try` unwraps Ok, else early-returns Err, widening this function's error +;; set. Option→Result conversion is explicit — no implicit From, no anyhow. +(defn load-config [path string] (Result Config) + (let [text (try (read-file path)) + table (try (parse-toml text)) + port (try (ok-or (get table "port") + (MissingKey {:key "port"})))] + (Ok (Config {:port port})))) + +;; errdefer runs only on the Result failure path — NOT on a restart transfer +;; (spec-conditions.md §5). Pairs with explicit allocation. +(defn load-atlas [path string] (Result Atlas) + (let [buf (alloc-image context/allocator)] + (errdefer (free buf)) + (try (decode-png path buf)) + (Ok (Atlas {:image buf})))) + +;; ── Conditions: handlers run on the signalling frame, nothing unwinds ─ +;; load-texture cannot know the right recovery — an editor wants a placeholder, +;; a release build wants to abort, a hot-reload session wants to retry after the +;; file is fixed on disk. So it offers a menu and the caller chooses. +(defcondition AssetMissing [path string]) + +;; `signal` has type Unit and RETURNS if every handler returns normally, so the +;; fall-through path of a restart-case in value position must still produce the +;; type. `abort` has type Never, which unifies with (Handle Texture). +;; Every clause body and the restart-case body share one type. +(defn load-texture [path string] (Handle Texture) + (if (file-exists? path) + (rl/load-texture path) + (restart-case + (do (signal (AssetMissing {:path path})) + (abort "unhandled AssetMissing")) + (use-placeholder [] placeholder-texture) + (retry [] (load-texture path))))) + +;; Intermediate frames say nothing about AssetMissing. Nothing to thread. +;; invoke-restart has type Never: it does not return to the handler. +(defn load-level [path string] Level + (handler-bind [AssetMissing (fn [c] + (log "missing asset:" (.path c)) + (invoke-restart 'use-placeholder))] + (parse-level (slurp path)))) + +;; A handler that returns normally does not unwind, so the signaller carries on. +;; That is error accumulation with no monad or applicative. Restarts go at the +;; resync point — once — not in every function below it. +;; The result is an owning (Vec Form): it is pushed to, and it is returned by +;; move, so the caller owns it. +(defn parse-all [p (Ptr Parser)] (Vec Form) + (let [forms (make-vec Form)] + (until (at-end? p) + (restart-case + (push forms (parse-form p)) + (skip-form [] (skip-to-next-delimiter p)))) + forms)) + +(defn collect-parse-errors [src string] (Result Ast) + (let [errors (make-vec ParseError)] + (handler-bind [ParseError (fn [c] + (push errors c) ; value struct: copies out of + ; the signalling frame + (invoke-restart 'skip-form))] + (let [ast (parse-all (parser src))] + (if (zero? (len errors)) + (Ok ast) + (Err (Errors errors))))))) ; errors moves into the Err + +;; ── Allocators. context/temp resets each frame; nothing freed by hand ─ +;; `filter` allocates a (Vec Enemy) from the current allocator, which is why +;; this is wrapped: the frame arena is bulk-reset, so the Vec is never freed +;; individually. `each` borrows it as a slice. +(defn draw-frame [w (Ptr World) dt f32] + (with-allocator context/temp + (->> (as-slice (.enemies w)) + (filter (fn [e] (on-screen? (.pos e)))) + (each (fn [e] (rl/draw-texture (.spr e) (.pos e)))))) + (free-all context/temp)) + +;; ── defer for explicit resources ────────────────────────────────────── +;; defer DOES run when a restart transfer passes through this frame. +(defn save-world [w (Ptr World) path string] + (let [f (open path :write)] + (defer (close f)) + (write-bytes f (serialize w)))) + +;; ── Fixed arrays: component-wise ops and swizzles, no library ───────── +(defn reflect [v Vec4 n Vec4] Vec4 + (- v (* 2.0 (dot v n) n))) + +(defn to-2d [v Vec4] Vec2 (.xy v)) + +;; ── Later: async as a state-machine transform, not fibers ───────────── +;; The handler and restart stacks live in the task state, not thread-local. +;; +;; An imperative loop, not (each (fn [p] (try ...))): `try` and `return` inside a +;; `fn` exit the FN, so a callback would swallow the Err instead of propagating +;; it out of preload. +(defn ^:async preload [paths [string]] (Result Unit) + (for [p paths] + (try (await (load-texture-async p)))) + (Ok unit)) diff --git a/test/dune b/test/dune index fc92eeb..5d4e2d7 100644 --- a/test/dune +++ b/test/dune @@ -1,2 +1,3 @@ (test - (name test_flan)) + (name test_flan) + (libraries flan)) diff --git a/test/test_flan.ml b/test/test_flan.ml index e69de29..d2992d7 100644 --- a/test/test_flan.ml +++ b/test/test_flan.ml @@ -0,0 +1,121 @@ +(* Reader tests. Plain assertions, no test framework — another dependency that + would have to be reimplemented if the compiler is ever self-hosted. *) + +open Flan + +let failures = ref 0 + +let check name cond = + if not cond then begin + incr failures; + Printf.printf "FAIL %s\n" name + end + +let reads name src expected = + match Reader.read_all ~file:"" src with + | forms -> + let got = String.concat " " (List.map Form.to_string forms) in + if got <> expected then begin + incr failures; + Printf.printf "FAIL %s\n src: %s\n got: %s\n wanted: %s\n" + name src got expected + end + | exception Loc.Error (loc, msg) -> + incr failures; + Printf.printf "FAIL %s\n src: %s\n error: %s: %s\n" + name src (Loc.to_string loc) msg + +let rejects name src = + match Reader.read_all ~file:"" src with + | _ -> incr failures; Printf.printf "FAIL %s: expected a read error\n" name + | exception Loc.Error _ -> () + +let () = + (* ── Atoms ─────────────────────────────────────────────────────── *) + reads "integer" "42" "42"; + reads "negative" "-1" "-1"; + reads "float" "0.05" "0.05"; + reads "hex" "0xE6B800FF" "3870818559"; + reads "string" "\"SAND\"" "\"SAND\""; + reads "symbol" "empty-at?" "empty-at?"; + reads "qualified" "rl/draw-fps" "rl/draw-fps"; + reads "field access" ".pos" ".pos"; + reads "operator" "->>" "->>"; + reads "bare minus" "-" "-"; + reads "keyword" ":space" ":space"; + reads "else keyword" ":else" ":else"; + + (* Byte literals, as used by calc-me's tokenizer. *) + reads "byte named" "\\space" "\\space"; + reads "byte digit" "\\0" "\\0"; + reads "byte paren" "\\(" "\\("; + reads "byte dot" "\\." "\\."; + + (* ── Sequences ─────────────────────────────────────────────────── *) + reads "list" "(+ 1 2)" "(+ 1 2)"; + reads "vector" "[1 2 3]" "[1 2 3]"; + reads "map literal" "{:src src :pos 0}" "{:src src :pos 0}"; + reads "type notation" "[4 f32]" "[4 f32]"; + reads "nested type" "[rows [cols u32]]" "[rows [cols u32]]"; + reads "commas as space" "[1, 2, 3]" "[1 2 3]"; + reads "nested" "(a (b [c {:d e}]))" "(a (b [c {:d e}]))"; + + (* ── Trivia ────────────────────────────────────────────────────── *) + reads "line comment" "; nope\n42" "42"; + reads "trailing comment" "42 ; nope" "42"; + reads "banner comment" ";;;; header\n(f)" "(f)"; + reads "multiple forms" "(a) (b)" "(a) (b)"; + reads "empty source" "" ""; + reads "only comments" "; nothing here" ""; + + (* ── Quote ─────────────────────────────────────────────────────── *) + (* Restart names are quoted symbols. Before this existed, 'skip-form read as + a symbol *named* "'skip-form", which is silently a different symbol from + skip-form and nothing would ever have reported it. *) + reads "quote symbol" "'skip-form" "(quote skip-form)"; + reads "quote in call" "(invoke-restart 'use-placeholder)" + "(invoke-restart (quote use-placeholder))"; + reads "quote list" "'(a b)" "(quote (a b))"; + + (* The whole class: no reader-significant character may end up inside a name. *) + let rec bad_names f = + let open Form in + match f.v with + | Sym s | Kw s -> + if String.exists (fun c -> c = '\'' || c = '^') s then [ s ] else [] + | List l | Vec l | Map l -> List.concat_map bad_names l + | _ -> [] + in + let corpus = "(invoke-restart 'skip-form) (a 'b [c 'd] {:e 'f}) '(g 'h)" in + check "no sigils leak into names" + (bad_names (Form.make (Form.List (Reader.read_all ~file:"" corpus)) + Loc.unknown) = []); + + (* ── Errors ────────────────────────────────────────────────────── *) + rejects "unclosed list" "(f x"; + rejects "unbalanced close" ")"; + rejects "mismatched" "(f x]"; + rejects "unterminated str" "\"abc"; + rejects "empty keyword" ":"; + rejects "unknown char" "\\bogus"; + rejects "metadata" "^:async"; + rejects "dangling quote" "'"; + + (* ── Locations ─────────────────────────────────────────────────── *) + (match Reader.read_all ~file:"f.flan" "(a)\n (b)" with + | [ a; b ] -> + check "loc line 1" (a.loc.line = 1 && a.loc.col = 1); + check "loc line 2" (b.loc.line = 2 && b.loc.col = 3); + check "loc file" (a.loc.file = "f.flan") + | _ -> check "loc: two forms" false); + + (match Reader.read_all ~file:"f.flan" "(f\n bad" with + | _ -> check "unclosed reports opening loc" false + | exception Loc.Error (loc, _) -> + check "unclosed reports opening loc" (loc.line = 1 && loc.col = 1)); + + if !failures = 0 then print_endline "reader: all tests passed" + else begin + Printf.printf "\n%d failure(s)\n" !failures; + exit 1 + end