flan/lib/dev.ml
Joseph Ferano d9ce1baa53 Take the abort, and refuse a name that is two requests
Every check of `abort' so far was of it being refused while the program runs.
The accepted path -- the one that ends a program -- was code that had never
run and answered `ok'. So the break block breaks its program once more, by
installing a `step' that errors into the loop that calls it, and takes the
exit: the daemon owns the program's lifetime, so no `close' is sent and the
daemon coming down on its own is the assertion.

And `restart' refuses a name with a control character in it. The agent's
contract is one line per request; a newline in a name is a second request
smuggled into the first. `completing-read' with require-match cannot produce
one, but the guarantee belongs to the end holding the socket, and an editor
is not the only thing that can speak to it.
2026-09-11 19:42:23 +07:00

576 lines
24 KiB
OCaml

(** [flan dev]: one long-lived session, the program it belongs to running
beside it, and a socket an editor talks to.
This is the piece between an editor and everything else. What it adds over
[flan reload] is that the session *persists*: a [defvar] added by one
evaluation is part of the program the next one is checked against, and the
set of names the running process was built with is the one from the build
this daemon actually made. A CLI that rebuilds its session from source each
time cannot have either.
It owns the build, which is what makes its layout rules mean anything: a
session's struct layouts and global types describe the memory of a process
only if it is the session that compiled it. So the daemon launches the
program rather than attaching to one. *)
type t = {
session : Session.t;
child : int; (* the running program *)
agent : string; (* where it listens for modules *)
dir : string; (* modules are built here, one per eval *)
stdout : Unix.file_descr; (* the program's output, on its way to here *)
out : Buffer.t; (* ...buffered until an editor asks for it *)
mutable n : int; (* dlopen caches by path: never reuse one *)
}
(* The program's stdout is a pipe into this process, so that an editor can see
it. That makes draining it a *liveness* requirement and not a nicety: a pipe
nobody reads fills at 64K and the next write blocks the program forever. So
it is read from the accept loop's select, not only when someone asks. *)
let capacity = 256 * 1024
let drain t =
let b = Bytes.create 8192 in
let rec go () =
match Unix.select [ t.stdout ] [] [] 0. with
| [], _, _ -> ()
| _ ->
(match Unix.read t.stdout b 0 8192 with
| 0 -> ()
| n ->
Buffer.add_subbytes t.out b 0 n;
(* Bounded: a program that prints every frame must not grow this
process without limit. The newest text is the useful end. *)
if Buffer.length t.out > capacity then begin
let keep = Buffer.sub t.out (Buffer.length t.out - capacity) capacity in
Buffer.clear t.out;
Buffer.add_string t.out keep
end;
go ()
| exception Unix.Unix_error (Unix.EAGAIN, _, _) -> ()
| exception Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> ()
| exception Unix.Unix_error _ -> ())
in
go ()
let take t =
drain t;
let s = Buffer.contents t.out in
Buffer.clear t.out;
s
let await ?(ms = 5000) f =
let rec go ms =
if f () then true
else if ms <= 0 then false
else begin ignore (Unix.select [] [] [] 0.005); go (ms - 5) end
in
go ms
(* ── Delivery ──────────────────────────────────────────────────────── *)
(* The agent answers "ok" when it has queued a module, and anything else is a
refusal with a reason. Reporting that back rather than swallowing it is what
keeps a failed delivery from looking like a successful evaluation — the
whole class of bug this socket makes possible. *)
let deliver t path =
let s = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Fun.protect
~finally:(fun () -> try Unix.close s with Unix.Unix_error _ -> ())
(fun () ->
Unix.connect s (Unix.ADDR_UNIX t.agent);
let msg = path ^ "\n" in
ignore (Unix.write_substring s msg 0 (String.length msg));
let b = Bytes.create 1024 in
let buf = Buffer.create 64 in
let rec drain () =
match Unix.read s b 0 1024 with
| 0 -> ()
| n -> Buffer.add_subbytes buf b 0 n; drain ()
| exception Unix.Unix_error _ -> ()
in
drain ();
String.trim (Buffer.contents buf))
(* Read back the value of the last expression evaluated, with the counter that
says whether it is a new one. The thunk runs on the game thread whenever the
program next reaches a frame boundary, which is not a moment the daemon gets
to know about, so this waits for the counter to move rather than assuming it
has. *)
let result t =
let s = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Fun.protect
~finally:(fun () -> try Unix.close s with Unix.Unix_error _ -> ())
(fun () ->
Unix.connect s (Unix.ADDR_UNIX t.agent);
ignore (Unix.write_substring s "result\n" 0 7);
let b = Bytes.create 4096 in
let buf = Buffer.create 256 in
let rec drain () =
match Unix.read s b 0 4096 with
| 0 -> ()
| n -> Buffer.add_subbytes buf b 0 n; drain ()
| exception Unix.Unix_error _ -> ()
in
drain ();
let text = Buffer.contents buf in
match String.index_opt text '\n' with
| None -> None
| Some i ->
let header = String.sub text 0 i in
let body = String.sub text (i + 1) (String.length text - i - 1) in
(match String.split_on_char ' ' header with
| [ g; _ ] ->
(match Int64.of_string_opt g with
| Some g -> Some (g, body)
| None -> None)
| _ -> None))
(* ── The break state ───────────────────────────────────────────────── *)
(* Everything above is about changing a *running* program. This is the other
half: an unhandled [error] does not kill a dev build, it stops the game
thread on the frame that erred and waits. The agent's socket is where that
shows, and the daemon is the only thing holding that socket — so an editor
asks here or not at all.
One line out, one line back, exactly like [result]: the agent is not a
protocol and must not become one. *)
let ask t verb =
let s = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Fun.protect
~finally:(fun () -> try Unix.close s with Unix.Unix_error _ -> ())
(fun () ->
Unix.connect s (Unix.ADDR_UNIX t.agent);
let msg = verb ^ "\n" in
ignore (Unix.write_substring s msg 0 (String.length msg));
let b = Bytes.create 4096 in
let buf = Buffer.create 128 in
let rec drain () =
match Unix.read s b 0 4096 with
| 0 -> ()
| n -> Buffer.add_subbytes buf b 0 n; drain ()
| exception Unix.Unix_error _ -> ()
in
drain ();
Buffer.contents buf)
type state =
| Running
| Stopped of string (* the condition's class name *)
| Unreachable of string (* no answer: exited, or never listened *)
(* [status] is answered whether or not the program is stopped — "running" is an
answer, not a refusal. Everything else the break loop offers is refused
while running, and rightly: there is no restart stack to walk. But the
question an editor asks *without already knowing* is this one, so it had to
have an answer in both states or there would be nothing to poll. *)
let state t =
match ask t "status" with
| "" -> Unreachable "the program is not answering on its socket"
| text ->
let line = String.trim (List.hd (String.split_on_char '\n' text)) in
if line = "running" then Running
else if String.length line > 8 && String.sub line 0 8 = "stopped " then
Stopped (String.sub line 8 (String.length line - 8))
else Unreachable ("the program answered " ^ line)
| exception Unix.Unix_error (e, _, _) -> Unreachable (Unix.error_message e)
(* Innermost first, terminated by a line that is a single dot — the agent's
framing, not this one's. A refusal comes back as a line starting "err ", and
is passed on rather than turned into an empty list: no restarts and cannot
say are different answers. *)
let restarts t =
match ask t "restarts" with
| text ->
let lines = String.split_on_char '\n' text in
if List.exists (fun l -> String.length l >= 3 && String.sub l 0 3 = "err") lines
then Error (String.trim text)
else
Ok
(List.filter
(fun l -> l <> "" && l <> ".")
(List.map String.trim lines))
| exception Unix.Unix_error (e, _, _) -> Error (Unix.error_message e)
let alive t =
match Unix.waitpid [ Unix.WNOHANG ] t.child with
| 0, _ -> true
| _ -> false
| exception Unix.Unix_error _ -> false
(* ── Ops ───────────────────────────────────────────────────────────── *)
(* Every reply is a plist with a :status, so an editor can dispatch on one key
and never has to guess whether a missing field means failure. *)
let ok fields =
"(:status \"ok\"" ^ String.concat "" (List.map (fun f -> " " ^ f) fields) ^ ")"
(* Anything the program printed since the last reply rides along with this one.
An editor that had to ask separately would miss the output an evaluation
itself caused, which is the output anyone actually wants to see. *)
let with_output t reply =
match take t with
| "" -> reply
| text ->
let i = String.length reply - 1 in
String.sub reply 0 i ^ " :output " ^ Wire.quote text ^ ")"
(* The break state rides along with every reply, exactly as the program's own
output does, and for the same reason: a program can stop at any moment and
nothing in a request/response protocol will mention it unless every response
does. An editor that had to *ask* would find out about a stop only when it
happened to wonder — and the most common moment for a program to stop is the
instant after an evaluation, which is a reply it is already reading.
It is the annotation, not the ops, that decides these two fields, so that
there is one place in the daemon that says whether the program is stopped
and the break ops cannot disagree with the poll. *)
let with_break t reply =
let fields =
match state t with
| Stopped c -> " :stopped t :condition " ^ Wire.quote c
| Running -> " :stopped nil"
(* Unreachable is not "running": the honest shape of "it exited" is
[:alive nil] from [describe], and claiming a state we could not read
would be the [ok]-means-probably failure in miniature. *)
| Unreachable _ -> " :stopped nil"
in
String.sub reply 0 (String.length reply - 1) ^ fields ^ ")"
let error ?loc msg =
"(:status \"error\" :message " ^ Wire.quote msg
^ (match loc with None -> "" | Some l -> " :loc " ^ Wire.quote l)
^ ")"
let eval t ~code ~origin =
if not (alive t) then error "the program exited; restart flan dev"
else
match Session.eval ~origin t.session code with
| c when not c.Session.installs ->
(* Accepted into the session and nothing to send: a declaration the
program already has, with no body and no new storage. Saying "ok" and
shipping an empty module would report success for a change that cannot
have taken effect. *)
ok
[ ":names " ^ Wire.strings c.Session.names; ":fns ()";
":note " ^ Wire.quote "nothing to install" ]
| c ->
t.n <- t.n + 1;
let out = Filename.concat t.dir (Printf.sprintf "m%d.so" t.n) in
(match Build.shared ~opts:{ Build.default with Build.dev = true }
~ir:c.Session.ir ~out () with
| timing ->
(match deliver t out with
| "ok" ->
ok
[ ":names " ^ Wire.strings c.Session.names;
":fns " ^ Wire.strings c.Session.fns;
Printf.sprintf ":ms %.1f"
(timing.Build.llc_ms +. timing.Build.link_ms) ]
| reply -> error ("the program refused the module: " ^ reply)
| exception Unix.Unix_error (e, _, _) ->
error
("cannot reach the program on " ^ t.agent ^ ": "
^ Unix.error_message e))
| exception Failure m -> error m)
| exception Loc.Error (l, msg) -> error ~loc:(Loc.to_string l) msg
(* Redefining a name installs a body; evaluating an expression has no name to
install into, so the module carries a thunk the agent runs once. The value
comes back through the runtime rather than through this reply, because the
frame boundary it runs at is the program's to choose. *)
let eval_expr t ~code ~origin =
if not (alive t) then error "the program exited; restart flan dev"
else
match Session.eval_expr ~origin t.session code with
| c ->
let before = match result t with Some (g, _) -> g | None -> 0L in
t.n <- t.n + 1;
let out = Filename.concat t.dir (Printf.sprintf "e%d.so" t.n) in
(match Build.shared ~opts:{ Build.default with Build.dev = true }
~ir:c.Session.ir ~out () with
| _ ->
(match deliver t out with
| "ok" ->
let rec wait ms =
match result t with
| Some (g, v) when Int64.compare g before > 0 -> Some v
| _ when ms <= 0 -> None
| _ ->
ignore (Unix.select [] [] [] 0.005);
if alive t then wait (ms - 5) else None
in
(match wait 5000 with
| Some v -> ok [ ":value " ^ Wire.quote v ]
| None ->
error
"the program did not reach a frame boundary; is it calling \
(agent/poll)?")
| reply -> error ("the program refused the module: " ^ reply)
| exception Unix.Unix_error (e, _, _) ->
error ("cannot reach the program: " ^ Unix.error_message e))
| exception Failure m -> error m)
| exception Loc.Error (l, msg) -> error ~loc:(Loc.to_string l) msg
let describe t =
ok
[ ":fns "
^ Wire.strings
(List.map (fun (f : Tast.fn) -> f.Tast.name)
t.session.Session.program.Tast.fns);
":globals "
^ Wire.strings
(List.map (fun (g : Tast.global) -> g.Tast.gname)
t.session.Session.program.Tast.globals);
":alive " ^ (if alive t then "t" else "nil") ]
(* [describe] answers what exists; this answers what each one *is*. Its own op
rather than more fields on [describe], because [describe] is polled — an
editor uses it to drain the program's output — and this is asked once on
connect and again after each install. Putting signatures on the poll would
pay for them every time anyone looked at the output buffer.
One entry per name: (name kind signature loc). Four strings, so the editor
reads it with [read] and nothing here needs a new wire type. [loc] is empty
where there is none to give — only [Tast.fn] carries one — and an editor
that finds it empty must say so rather than guess a file.
Parameter *names* are not in the Tast, so a signature shows types only. *)
let signature_of_fn (f : Tast.fn) =
Printf.sprintf "%s [%s] %s" f.Tast.name
(String.concat " " (List.map Types.to_string f.Tast.params))
(Types.to_string f.Tast.ret)
let entry ~name ~kind ~sign ~loc =
Wire.list [ Wire.quote name; Wire.quote kind; Wire.quote sign; Wire.quote loc ]
let defs t =
let p = t.session.Session.program in
let fns =
List.filter_map
(fun (f : Tast.fn) ->
match f.Tast.fparent with
(* A handler-bind clause the checker lifted out. Nobody wrote this
name, so completing it is noise and jumping to it is meaningless. *)
| Some _ -> None
| None ->
Some
(entry ~name:f.Tast.name ~kind:"fn" ~sign:(signature_of_fn f)
~loc:(Loc.to_string f.Tast.floc)))
p.Tast.fns
in
let globals =
List.map
(fun (g : Tast.global) ->
entry ~name:g.Tast.gname
~kind:(if g.Tast.gconst then "const" else "var")
~sign:
(Printf.sprintf "%s %s" g.Tast.gname (Types.to_string g.Tast.gty))
~loc:"")
p.Tast.globals
in
let externs =
List.map
(fun (e : Tast.extern) ->
entry ~name:e.Tast.ename ~kind:"extern"
~sign:
(Printf.sprintf "%s [%s] %s" e.Tast.ename
(String.concat " " (List.map Types.to_string e.Tast.eparams))
(Types.to_string e.Tast.eret))
~loc:"")
p.Tast.externs
in
ok [ ":defs " ^ Wire.list (fns @ globals @ externs) ]
(* What is on offer where the program stopped. [:stopped] and [:condition] are
not here: the annotation puts them on this reply as it puts them on every
other, so an editor reads the same two keys whatever it asked. What this op
adds is the restart names, which cost a second round trip to the program and
are wanted only when someone is about to choose one. *)
let break t =
if not (alive t) then error "the program exited; restart flan dev"
else
match state t with
| Running -> ok []
| Unreachable m -> error ("cannot ask the program whether it stopped: " ^ m)
| Stopped _ ->
(match restarts t with
| Ok names -> ok [ ":restarts " ^ Wire.strings names ]
| Error m -> error ("the program refused to list its restarts: " ^ m))
(* A choice is validated by the *program*, on its listener thread, against a
stack the stopped game thread is holding still — not here. The daemon has no
copy of that stack and anything it checked would be a guess that was true a
moment ago.
"ok" therefore means accepted, and says so: the resume happens when the
stopped thread next comes round its loop, which is microseconds away and
still not now. An editor that read [ok] as "running again" would poll once,
find it stopped, and re-open the prompt it had just answered. *)
let choose t ~name =
if not (alive t) then error "the program exited; restart flan dev"
else if String.exists (fun c -> Char.code c < 32 || Char.code c = 127) name then
(* The agent's contract is one line per request. A name carrying a newline
would be a second request smuggled into the first, and the guarantee is
this end's to keep: [completing-read] cannot produce one, but the daemon
is what holds the socket and an editor is not the only thing that can
speak to it. *)
error "a restart name cannot contain a control character"
else
match ask t ("restart " ^ name) with
| reply when String.trim reply = "ok" ->
ok
[ ":restart " ^ Wire.quote name;
":note "
^ Wire.quote "accepted; the program resumes at its next pass of the break loop" ]
| reply -> error (String.trim reply)
| exception Unix.Unix_error (e, _, _) ->
error ("cannot reach the program: " ^ Unix.error_message e)
(* The other way out. The program exits 134 where it stopped, which ends this
daemon too — it owns the program's lifetime and has nothing left to serve.
Refused while running, by the program, for the same reason a restart is. *)
let abort t =
if not (alive t) then error "the program exited; restart flan dev"
else
match ask t "abort" with
| reply when String.trim reply = "ok" ->
ok [ ":note " ^ Wire.quote "the program is exiting; flan dev ends with it" ]
| reply -> error (String.trim reply)
| exception Unix.Unix_error (e, _, _) ->
error ("cannot reach the program: " ^ Unix.error_message e)
let handle t req =
match Wire.string_field req "op" with
| Some "eval" ->
(match Wire.string_field req "code" with
| Some code ->
let origin =
match Wire.string_field req "file" with Some f -> f | None -> "<editor>"
in
eval t ~code ~origin
| None -> error "eval needs :code")
| Some "eval-expr" ->
(match Wire.string_field req "code" with
| Some code ->
let origin =
match Wire.string_field req "file" with Some f -> f | None -> "<editor>"
in
eval_expr t ~code ~origin
| None -> error "eval-expr needs :code")
| Some "describe" -> describe t
| Some "defs" -> defs t
| Some "break" -> break t
| Some "restart" ->
(match Wire.string_field req "name" with
| Some name -> choose t ~name
| None -> error "restart needs :name")
| Some "abort" -> abort t
| Some "close" -> ok []
| Some op -> error ("unknown op: " ^ op)
| None -> error "no :op"
(* ── The loop ──────────────────────────────────────────────────────── *)
(* One connection at a time. An editor is one client, evaluations are
sequential by nature — each one is checked against the program the last one
left behind — and a second concurrent evaluation would be racing for the
same session anyway. *)
(* Returns whether the client asked to end the session. One editor per daemon,
so [close] shuts the whole thing down rather than waiting for another
connection nobody is going to make. *)
let serve t fd =
let rec go () =
match Wire.recv fd with
| src ->
let op, reply =
match Wire.parse src with
| req -> (Wire.string_field req "op", handle t req)
| exception Loc.Error (_, m) -> (None, error ("bad request: " ^ m))
in
Wire.send fd (with_output t (with_break t reply));
if op = Some "close" then true else go ()
| exception Wire.Closed -> false
| exception Unix.Unix_error _ -> false
in
go ()
let start ~file ~sock =
let t0 = Unix.gettimeofday () in
(* Absolute, because every location this daemon ever reports is derived from
it and an editor is not in this process's working directory. [flan dev
src/game.flan] run from a project root would otherwise send back
"src/game.flan:12:7", which the editor can only resolve by guessing which
directory it was relative to. *)
let file = try Unix.realpath file with Unix.Unix_error _ -> file in
let session, l = Session.create ~file in
let dir =
Filename.concat (Filename.get_temp_dir_name ())
(Printf.sprintf "flan-dev-%d" (Unix.getpid ()))
in
(try Unix.mkdir dir 0o700 with Unix.Unix_error (Unix.EEXIST, _, _) -> ());
let exe = Filename.concat dir "program" in
ignore
(Build.executable ~opts:{ Build.default with Build.dev = true }
~csrcs:l.Load.csrcs ~lflags:l.Load.lflags session.Session.host ~out:exe);
let agent = Filename.concat dir "agent.sock" in
(* The program's source names some socket path; the daemon is the one that
knows where it wants to talk to it, so it overrides through the
environment. Guessing instead would fail silently — everything compiles,
the module is built, and nothing ever receives it. *)
Unix.putenv "FLAN_AGENT_SOCKET" agent;
(* Through a pipe, so the program's own output can reach an editor instead of
only the terminal the daemon was started in. *)
let rd, wr = Unix.pipe ~cloexec:false () in
let child = Unix.create_process exe [| exe |] Unix.stdin wr Unix.stderr in
Unix.close wr;
Unix.set_nonblock rd;
(* Wait for it to bind before accepting an evaluation. One that arrives first
would fail for a reason that reads like a compiler bug. *)
if not (await (fun () -> Sys.file_exists agent)) then begin
(try Unix.kill child Sys.sigterm with Unix.Unix_error _ -> ());
failwith
("the program never listened on " ^ agent
^ " — does it call (agent/start ...)?")
end;
let t =
{ session; child; agent; dir; stdout = rd; out = Buffer.create 4096; n = 0 }
in
(try Unix.unlink sock with Unix.Unix_error _ -> ());
let ls = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Unix.bind ls (Unix.ADDR_UNIX sock);
Unix.listen ls 4;
Printf.eprintf "flan dev: %s ready on %s (%.0fms)\n%!" file sock
((Unix.gettimeofday () -. t0) *. 1000.);
(* [accept] would block past the program's own exit, so it is waited on with
a timeout and the child checked each time round: a daemon whose program
has finished has nothing left to do, and an editor waiting on it would
wait forever. *)
let rec accept_loop () =
if alive t then
(* The program's pipe is in the same select as the listening socket: it
has to be drained whether or not an editor is asking for anything. *)
match Unix.select [ ls; t.stdout ] [] [] 0.2 with
| [], _, _ -> accept_loop ()
| ready, _, _ when not (List.mem ls ready) -> drain t; accept_loop ()
| _ ->
(match Unix.accept ls with
| fd, _ ->
let closed = serve t fd in
(try Unix.close fd with Unix.Unix_error _ -> ());
if not closed then accept_loop ()
| exception Unix.Unix_error (Unix.EINTR, _, _) -> accept_loop ())
| exception Unix.Unix_error (Unix.EINTR, _, _) -> accept_loop ()
in
Fun.protect
~finally:(fun () ->
(try Unix.kill child Sys.sigterm with Unix.Unix_error _ -> ());
(try Unix.close ls with Unix.Unix_error _ -> ());
(try Unix.close rd with Unix.Unix_error _ -> ());
(try Unix.unlink sock with Unix.Unix_error _ -> ()))
accept_loop