Quasiquote is a desugaring, and a defmacro is a defn

Two things that look like plumbing and are the frontend half of expansion.

A quasiquote becomes calls to the prelude's three form-building functions and
nothing else: form-nil, form-cons for an item, form-append for a splice. It is
pure, it needs nothing loaded, and it runs over every form on the way into
Parse.program and Parse.decl, which is what lets the prelude's own macros parse
in a process that has not built a macro module yet.

Running it *before* the expander's walk is not an ordering preference. A cond
macro's body contains a quasiquoted (cond ...) for its own tail; with the
quasiquote still standing, the walk would see that head and expand it then and
there, against the wrong arguments. Desugared first, that subform is a
(Form.Sym {.s "cond"}) and there is no head left to mistake. So the walk needs
no idea that quoting exists, which is the whole reason this runs first.

Nesting levels are not counted -- not by the reader, which was written that way
deliberately, and not here. A quasiquote inside a quasiquote is refused by
name. Only a macro that writes a macro wants one, nothing in the corpus does,
and CL's level arithmetic costs more than the use case is worth so far.

A defmacro is now an Ast.Defn: (defmacro m [args] body) is
(defn m [args [Form]] Form body). There is no Ast.Defmacro and there is not
going to be one -- a macro is [Form] -> Form, compiled by the same backend as
everything else, and the only thing that makes it a macro is that the expander
calls it at compile time. One parameter, the slice of forms at the call site,
so variadics come free in a language with no &rest; two parameters is a
misunderstanding rather than an arity error and says so.

Parse.expander is the hook the walk arrives through, because expanding a macro
means compiling and dlopening it, so the expander sits above Check and Build
and Parse sits below them. Nothing fills it in yet.

The quasiquote refusal stays as a backstop: it now means a form reached the
parser without coming through program or decl. gensym's refusal is gone -- it
is an ordinary prelude function returning a Form, and a macro body calls it
like any other.

lib/expand.ml

@@ -144,3 +144,90 @@ let call ~loc (fn : Dynload.addr) (args : Form.t list) : Form.t =
   let out = Dynload.take form_size in
   Dynload.call fn a (Int64.of_int n) out;
   unmarshal ~loc out
+
+(* ── Quasiquote ────────────────────────────────────────────────────
+   A desugaring over [Form], and nothing more: a quasiquoted (if ~t ~b) becomes
+   calls to the prelude's form-building surface, which the checker then sees as
+   ordinary code. There is no quasiquote left in the language after this runs,
+   which is why the expander's own walk needs no idea that quoting exists: by
+   the time it looks for macro calls, a [cond] written inside a quasiquote is a
+   (Form.Sym {.s "cond"}) and there is no head there to mistake for a call the
+   compiler should make now.
+
+   The reader stays dumb and produces (quasiquote x), (unquote x) and
+   (unquote-splicing x) with no idea whether one is inside another. Counting
+   levels is this file's job, and it does not: a quasiquote inside a quasiquote
+   is refused by name. A macro that writes a macro is the only thing that wants
+   one, nothing in the corpus does, and CL's level arithmetic has a real cost
+   that no use case has asked for. *)
+
+let sym loc s = Form.make (Form.Sym s) loc
+let lst loc xs = Form.make (Form.List xs) loc
+
+(* (Form.Case {.field value}) — a node of the image, written as the Flan
+   constructor the prelude declares. *)
+let node loc case field v =
+  lst loc [ sym loc ("Form." ^ case);
+            Form.make (Form.Map [ sym loc ("." ^ field); Form.make v loc ]) loc ]
+
+let unquote_of (f : Form.t) =
+  match f.Form.v with
+  | Form.List [ { Form.v = Form.Sym "unquote"; _ }; x ] -> Some x
+  | _ -> None
+
+let splice_of (f : Form.t) =
+  match f.Form.v with
+  | Form.List [ { Form.v = Form.Sym "unquote-splicing"; _ }; x ] -> Some x
+  | _ -> None
+
+let rec quote (f : Form.t) : Form.t =
+  let loc = f.Form.loc in
+  match unquote_of f with
+  (* The escape: whatever the program wrote, evaluated. It is already a Form,
+     because a Form is what a macro body deals in. *)
+  | Some x -> x
+  | None ->
+    match splice_of f with
+    | Some _ ->
+      Loc.fail loc
+        "~@x splices into a list or a vector, and there is nothing here for it \
+         to splice into"
+    | None ->
+      match f.Form.v with
+      | Form.List ({ Form.v = Form.Sym "quasiquote"; _ } :: _) ->
+        Loc.fail loc
+          "a quasiquote inside a quasiquote is not implemented: the reader does \
+           not count nesting levels and neither does this, so the inner one has \
+           no meaning to give. Build the inner form with form-cons"
+      | Form.Sym s -> node loc "Sym" "s" (Form.Str s)
+      | Form.Kw s -> node loc "Kw" "s" (Form.Str s)
+      | Form.Int i -> node loc "Int" "i" (Form.Int i)
+      | Form.Float x -> node loc "Float" "x" (Form.Float x)
+      | Form.Str s -> node loc "Str" "s" (Form.Str s)
+      | Form.Byte b -> node loc "Byte" "b" (Form.Int (Int64.of_int b))
+      | Form.List xs -> node loc "List" "xs" (seq loc xs).Form.v
+      | Form.Vec xs -> node loc "Vec" "xs" (seq loc xs).Form.v
+      | Form.Map xs -> node loc "Map" "xs" (seq loc xs).Form.v
+
+(* The [Form] slice one bracket's worth of items comes to. Built right to left,
+   so each item is consed onto what follows it and a splice is an append — the
+   three prelude functions and no fourth. *)
+and seq loc items =
+  List.fold_left
+    (fun acc (item : Form.t) ->
+       match splice_of item with
+       | Some x -> lst item.Form.loc [ sym item.Form.loc "form-append"; x; acc ]
+       | None -> lst item.Form.loc [ sym item.Form.loc "form-cons"; quote item; acc ])
+    (lst loc [ sym loc "form-nil" ])
+    (List.rev items)
+
+(* Every quasiquote in a form, outermost first. Pure, total, and dependent on
+   nothing but Form, which is what lets [Parse] run it on the way in rather
+   than needing the whole expander wired up first. *)
+let rec quasiquote (f : Form.t) : Form.t =
+  match f.Form.v with
+  | Form.List [ { Form.v = Form.Sym "quasiquote"; _ }; x ] -> quote x
+  | Form.List xs -> Form.make (Form.List (List.map quasiquote xs)) f.Form.loc
+  | Form.Vec xs -> Form.make (Form.Vec (List.map quasiquote xs)) f.Form.loc
+  | Form.Map xs -> Form.make (Form.Map (List.map quasiquote xs)) f.Form.loc
+  | _ -> f

lib/parse.ml

@@ -296,9 +296,14 @@ and form f mk (head : Form.t) (args : Form.t list) : Ast.expr =
   (* The reader now produces these three, so they arrive here as ordinary heads
      and would fall through to Call — coming back from the checker as "unknown
      name quasiquote", which says nothing about what is actually missing. *)
+  (* [Expand.quasiquote] runs over every form on the way into [program] and
+     [decl], so a quasiquote is gone before this file looks at it and this arm
+     cannot be reached by anything that came through either. It is kept as the
+     backstop for the path that did not: a form built by hand and handed
+     straight to [expr]. *)
   | Sym "quasiquote" ->
-    fail f "`x is read, but not expanded: macro expansion is not wired up yet \
+    fail f "a quasiquote reached the parser undesugared, which means this form \
-            (NEXT.md says what it needs)"
+            did not come through Parse.program or Parse.decl"
 
   (* Not a milestone, a mistake: these two mean nothing anywhere else, and the
      reader cannot tell, because it does not track where it is. *)
@@ -311,13 +316,6 @@ and form f mk (head : Form.t) (args : Form.t list) : Ast.expr =
   | Sym "defmacro" ->
     fail f "defmacro is a top-level declaration, not an expression"
 
-  (* Neither a reader token nor a special form: an ordinary function that a
-     macro body calls while the macro runs. There is nowhere for it to run
-     yet, so it says that rather than arriving as an unknown name. *)
-  | Sym "gensym" ->
-    fail f "gensym is only meaningful inside a macro body, and macro expansion \
-            is not wired up yet (NEXT.md says what it needs)"
-
   (* Recognised, deliberately unimplemented. Rejected rather than left to fall
      through to Call, where they would parse and mean nothing. *)
   | Sym ("handler-case"
@@ -760,25 +758,38 @@ let rec decl types (f : Form.t) : Ast.decl =
      | [ n; t; v ] -> mk (Ast.Defconst (sym n, Some (texpr t), expr v))
      | _ -> fail f "defconst is (defconst name Type? value)")
 
-  (* Checked for shape and then refused, which is deliberate. Getting the shape
+  (* A macro is an ordinary function, and this is where it becomes one:
-     wrong and getting the whole feature are two different mistakes, and a
+     [(defmacro m [args] body)] is [(defn m [args [Form]] Form body)]. There is
-     "defmacro is (defmacro ...)" that only ever fired after expansion landed
+     no [Ast.Defmacro] and there is not going to be one -- a macro has the type
-     would be a rule nothing enforced in the meantime.
+     [[Form] -> Form], it is compiled by the same backend as everything else,
+     and the only thing that makes it a macro is that [Expand] calls it at
+     compile time instead of the program calling it at run time.
 
-     The refusal is not about parsing. Expanding a macro means running it, and
+     One parameter, the slice of the argument forms, rather than one declared
-     there is no interpreter — the compiled path is the only backend. So it
+     parameter per argument. It needs no reader or parser change and it gives
-     means compiling the macro and dlopening it into the compiler, which is
+     variadics for free, which is what [unless] and [when] need in a language
-     what Emit.redefinition and Build.shared already do for the dev loop.
+     with no &rest.
-     NEXT.md writes down how that goes together. *)
+
+     The shape rules stay exactly as they were, because they were enforced
+     before the feature existed on purpose: getting the shape wrong and getting
+     the whole feature are different mistakes. *)
   | List ({ v = Sym "defmacro"; _ } :: args) ->
     (match args with
-     | n :: { v = Form.Vec ps; _ } :: body when body <> [] ->
+     | n :: { v = Form.Vec [ p ]; _ } :: body when body <> [] ->
-       let name = sym n in
+       let form_t = { Ast.t = Ast.Tname "Form"; tloc = f.loc } in
+       mk (Ast.Defn
+             { Ast.name = sym n;
+               params = [ { Ast.fname = sym p;
+                            fty = { Ast.t = Ast.Tslice form_t; tloc = p.loc };
+                            floc = p.loc } ];
+               ret = Some form_t; fbody = body_of body; nloc = n.loc })
+     | _ :: { v = Form.Vec ps; _ } :: body when body <> [] ->
        List.iter (fun (p : Form.t) -> ignore (sym p)) ps;
        fail f
-         "defmacro %s parses, but is not expanded: running a macro means \
+         "a macro takes one parameter, the forms at its call site, and this \
-          compiling it and loading it into the compiler, which is not wired \
+          one names %d. There is no &rest and no arity: (defmacro m [args] \
-          up yet (NEXT.md says what it needs)" name
+          ...) and (len args) is how many were written"
+         (List.length ps)
      | _ ->
        fail f "defmacro is (defmacro name [param ...] body ...)")
 
@@ -902,11 +913,37 @@ let prelude_types =
 let declared_types (forms : Form.t list) : Names.t =
   types_in (Lazy.force prelude_types) forms
 
+(* Macro expansion, which runs over [Form] and therefore before anything in
+   this file. It cannot be called directly: expanding a macro means compiling
+   it and dlopening it, so the expander sits above [Check] and [Build] and this
+   module sits below them. [Macro] fills this in, and lib/dune passes -linkall
+   so that it always has -- an executable that links the library gets the
+   installation whether or not it names the module.
+
+   The default is the identity because [Macro] is what knows which names are
+   macros; with nothing installed, a call to one arrives at the checker as an
+   unknown name, which is wrong but not silent. *)
+let expander : (Form.t list -> Form.t list) ref = ref (fun fs -> fs)
+
 let program (forms : Form.t list) : Ast.decl list =
+  (* Quasiquote first and always, because it is pure and needs nothing loaded:
+     it is what turns a macro body into ordinary code, and the prelude's own
+     macros have to parse in a process that has not built a macro module yet.
+     Then expansion, which may need one. *)
+  let forms = !expander (List.map Expand.quasiquote forms) in
   let types = declared_types forms in
   temps := 0;
   List.map (decl types) forms
 
 (* Single-declaration entry point, for tests and the REPL. Sees the builtin and
    prelude types plus whatever this one form declares. *)
-let decl (f : Form.t) : Ast.decl = temps := 0; decl (declared_types [ f ]) f
+let decl (f : Form.t) : Ast.decl =
+  temps := 0;
+  match !expander [ Expand.quasiquote f ] with
+  | [ f ] -> decl (declared_types [ f ]) f
+  | fs ->
+    (* One declaration in, one out. A macro at the top level would break that,
+       and there is no top-level macro call: [decl] dispatches on the head and
+       a macro name is not one of the heads it knows. *)
+    Loc.fail f.loc "expanding this declaration produced %d of them"
+      (List.length fs)

test/test_flan.ml

@@ -389,12 +389,23 @@ let () =
   parse_rejects "restart-case" "(restart-case body (r [] 1))";
   parse_rejects "loop/recur"   "(loop [x 1] (recur x))";
 
-  (* ── Macros: the front half is here, the expander is not ───────── *)
+  (* ── Macros ─────────────────────────────────────────────────────── *)
-  (* Was "unknown top-level form (defmacro ...)" — refused, but not by name and
+  (* A defmacro is a defn. There is no Ast.Defmacro and there is not going to
-     with no reason, which is the hole the house rule had at the top level. *)
+     be one: a macro is [Form] -> Form, compiled by the same backend as
-  parse_rejects "defmacro declaration" "(defmacro m [x] x)"
+     everything else, and what makes it a macro is that the expander calls it
-    ~needle:"not expanded";
+     at compile time rather than the program calling it at run time. *)
-  (* Shape and feature are separate mistakes and get separate reasons. *)
+  (match (parse_decl "(defmacro m [args] (at args 0))").d with
+   | Defn { name = "m"; params = [ p ]; ret = Some r; _ } ->
+     (match p.fty.t, r.t with
+      | Tslice { t = Tname "Form"; _ }, Tname "Form" -> ()
+      | _ -> check "defmacro is [Form] -> Form" false)
+   | _ -> check "defmacro parses as a defn" false);
+
+  (* One parameter, the forms at the call site. Two is not an arity mistake, it
+     is a misunderstanding of what a macro takes, and it gets its own reason. *)
+  parse_rejects "defmacro with two parameters" "(defmacro m [a b] a)"
+    ~needle:"a macro takes one parameter";
+  (* Shape and feature were separate mistakes and stay separate reasons. *)
   parse_rejects "defmacro with no body" "(defmacro m [x])"
     ~needle:"defmacro is (defmacro name [param ...] body ...)";
   parse_rejects "defmacro with no params" "(defmacro m x)"
@@ -404,18 +415,45 @@ let () =
   parse_rejects "defmacro in expression position" "(defn f [] (defmacro m [] 1))"
     ~needle:"top-level declaration";
 
-  (* The reader now hands these three to the parser, so each says what is
+  (* Quasiquote is a desugaring over Form, and it has already run by the time
-     actually wrong rather than arriving at the checker as an unknown name. *)
+     the parser sees anything, so what is written here is what a macro body
-  parse_rejects "quasiquote in a function" "(defn f [] `(a b))"
+     actually compiles to: the prelude's three form-building functions and
-    ~needle:"not expanded";
+     nothing else. Spelled out rather than described, because the desugaring
+     *is* the contract with the prelude. *)
+  let desugars name src want =
+    match read src with
+    | [ f ] ->
+      let got = Form.to_string (Expand.quasiquote f) in
+      if got <> want then begin
+        incr failures;
+        Printf.printf "FAIL  %s\n        got:    %s\n        wanted: %s\n"
+          name got want
+      end
+    | _ -> check (name ^ ": one form") false
+  in
+  desugars "a quasiquoted list is form-cons over Form nodes" "`(a ~b)"
+    "(Form.List {.xs (form-cons (Form.Sym {.s \"a\"}) (form-cons b (form-nil)))})";
+  desugars "a splice is form-append" "`(a ~@bs)"
+    "(Form.List {.xs (form-cons (Form.Sym {.s \"a\"}) (form-append bs (form-nil)))})";
+  (* A vector keeps its bracket through the desugaring: a binding vector is the
+     commonest thing a macro builds and Form.Vec is not Form.List. *)
+  desugars "a quasiquoted vector stays a vector" "`[~x 1]"
+    "(Form.Vec {.xs (form-cons x (form-cons (Form.Int {.i 1}) (form-nil)))})";
+  (* Levels are not counted -- not by the reader, deliberately, and not here,
+     which is why the inner one is refused by name rather than given a meaning
+     nobody chose. *)
+  parse_rejects "a quasiquote inside a quasiquote" "(defn f [] Form `(a `(b)))"
+    ~needle:"quasiquote inside a quasiquote";
   (* Not a missing feature — an unquote outside a quasiquote is a mistake, and
      the reader cannot catch it because it does not track where it is. *)
   parse_rejects "unquote outside a quasiquote" "(defn f [] ~x)"
     ~needle:"means nothing outside a quasiquote";
   parse_rejects "splice where a splice makes no sense" "(defn f [] (+ 1 ~@xs))"
     ~needle:"splices only into a list or a vector";
-  parse_rejects "gensym outside a macro" "(defn f [] (gensym))"
+  (* A splice with no bracket around it. The quasiquote is real here, so this
-    ~needle:"only meaningful inside a macro body";
+     one is the desugaring's refusal and not the parser's. *)
+  parse_rejects "splice not inside a bracket" "(defn f [] Form `~@xs)"
+    ~needle:"nothing here for it to splice into";
 
   (* ── Malformed syntax is caught with a location ────────────────── *)
   parse_rejects "odd let bindings"  "(let [a])";