flan/lib/dev.ml
Joseph Ferano 5f4005e61d A stopped program, driven from Emacs
The break loop was reachable from a raw socket. This is the half that makes
it reachable from an editor, and it all follows from one fact: a program
stops at a moment nobody asked about.

So the state is learned twice, on purpose. It rides on every reply, beside
the program's output and for the same reason -- the likeliest instant for a
program to stop is the one just after an evaluation, which is a reply the
client is already reading, and learning it a second later from a poll would
mean learning it after the echo area had said the evaluation was fine. And a
timer asks anyway, once a second with `describe', because a program that
stops in a frame of its own game loop produces no reply at all and folding
state into replies that never come says nothing. The timer never reconnects
-- that would quietly erase the `lost' state that exists to be seen -- and
skips while a request is in flight, since accept-process-output runs timers
and a poll firing inside a read would eat that read's reply.

Three ops: `break' for the restart names, `restart' and `abort'. The
annotation owns :stopped and :condition rather than the ops, so one place in
the daemon decides whether the program is stopped and the poll and the prompt
cannot disagree. "ok" from `restart' means accepted, not resumed: the choice
is validated against the stopped stack and taken when that thread next comes
round, so it says so and the client clears its own flag rather than polling
once, finding it stopped, and re-opening the prompt it just answered.

The agent grew one verb, `status', answered in both states. Everything else
the break loop offers is refused while running, rightly; but the question an
editor asks without already knowing had to have an answer either way or there
would be nothing to poll.

And flan_agent_poll had to become re-entrant, which was a bug rather than an
addition. A C-x C-e thunk may itself error, and the break loop that catches
it polls again from inside that call. The old loop cached both indices and
stored tail at the end, rewinding over everything the nested poll consumed --
re-running the thunk that had just stopped the program, which is an unbounded
recursion of breaks. Each job is now claimed before it is run. test_dev.ml
evaluates an expression that errors and resumes it, which fails against the
old shape.
2026-09-11 19:39:29 +07:00

569 lines
24 KiB
OCaml

(** [flan dev]: one long-lived session, the program it belongs to running
beside it, and a socket an editor talks to.
This is the piece between an editor and everything else. What it adds over
[flan reload] is that the session *persists*: a [defvar] added by one
evaluation is part of the program the next one is checked against, and the
set of names the running process was built with is the one from the build
this daemon actually made. A CLI that rebuilds its session from source each
time cannot have either.
It owns the build, which is what makes its layout rules mean anything: a
session's struct layouts and global types describe the memory of a process
only if it is the session that compiled it. So the daemon launches the
program rather than attaching to one. *)
type t = {
session : Session.t;
child : int; (* the running program *)
agent : string; (* where it listens for modules *)
dir : string; (* modules are built here, one per eval *)
stdout : Unix.file_descr; (* the program's output, on its way to here *)
out : Buffer.t; (* ...buffered until an editor asks for it *)
mutable n : int; (* dlopen caches by path: never reuse one *)
}
(* The program's stdout is a pipe into this process, so that an editor can see
it. That makes draining it a *liveness* requirement and not a nicety: a pipe
nobody reads fills at 64K and the next write blocks the program forever. So
it is read from the accept loop's select, not only when someone asks. *)
let capacity = 256 * 1024
let drain t =
let b = Bytes.create 8192 in
let rec go () =
match Unix.select [ t.stdout ] [] [] 0. with
| [], _, _ -> ()
| _ ->
(match Unix.read t.stdout b 0 8192 with
| 0 -> ()
| n ->
Buffer.add_subbytes t.out b 0 n;
(* Bounded: a program that prints every frame must not grow this
process without limit. The newest text is the useful end. *)
if Buffer.length t.out > capacity then begin
let keep = Buffer.sub t.out (Buffer.length t.out - capacity) capacity in
Buffer.clear t.out;
Buffer.add_string t.out keep
end;
go ()
| exception Unix.Unix_error (Unix.EAGAIN, _, _) -> ()
| exception Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> ()
| exception Unix.Unix_error _ -> ())
in
go ()
let take t =
drain t;
let s = Buffer.contents t.out in
Buffer.clear t.out;
s
let await ?(ms = 5000) f =
let rec go ms =
if f () then true
else if ms <= 0 then false
else begin ignore (Unix.select [] [] [] 0.005); go (ms - 5) end
in
go ms
(* ── Delivery ──────────────────────────────────────────────────────── *)
(* The agent answers "ok" when it has queued a module, and anything else is a
refusal with a reason. Reporting that back rather than swallowing it is what
keeps a failed delivery from looking like a successful evaluation — the
whole class of bug this socket makes possible. *)
let deliver t path =
let s = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Fun.protect
~finally:(fun () -> try Unix.close s with Unix.Unix_error _ -> ())
(fun () ->
Unix.connect s (Unix.ADDR_UNIX t.agent);
let msg = path ^ "\n" in
ignore (Unix.write_substring s msg 0 (String.length msg));
let b = Bytes.create 1024 in
let buf = Buffer.create 64 in
let rec drain () =
match Unix.read s b 0 1024 with
| 0 -> ()
| n -> Buffer.add_subbytes buf b 0 n; drain ()
| exception Unix.Unix_error _ -> ()
in
drain ();
String.trim (Buffer.contents buf))
(* Read back the value of the last expression evaluated, with the counter that
says whether it is a new one. The thunk runs on the game thread whenever the
program next reaches a frame boundary, which is not a moment the daemon gets
to know about, so this waits for the counter to move rather than assuming it
has. *)
let result t =
let s = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Fun.protect
~finally:(fun () -> try Unix.close s with Unix.Unix_error _ -> ())
(fun () ->
Unix.connect s (Unix.ADDR_UNIX t.agent);
ignore (Unix.write_substring s "result\n" 0 7);
let b = Bytes.create 4096 in
let buf = Buffer.create 256 in
let rec drain () =
match Unix.read s b 0 4096 with
| 0 -> ()
| n -> Buffer.add_subbytes buf b 0 n; drain ()
| exception Unix.Unix_error _ -> ()
in
drain ();
let text = Buffer.contents buf in
match String.index_opt text '\n' with
| None -> None
| Some i ->
let header = String.sub text 0 i in
let body = String.sub text (i + 1) (String.length text - i - 1) in
(match String.split_on_char ' ' header with
| [ g; _ ] ->
(match Int64.of_string_opt g with
| Some g -> Some (g, body)
| None -> None)
| _ -> None))
(* ── The break state ───────────────────────────────────────────────── *)
(* Everything above is about changing a *running* program. This is the other
half: an unhandled [error] does not kill a dev build, it stops the game
thread on the frame that erred and waits. The agent's socket is where that
shows, and the daemon is the only thing holding that socket — so an editor
asks here or not at all.
One line out, one line back, exactly like [result]: the agent is not a
protocol and must not become one. *)
let ask t verb =
let s = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Fun.protect
~finally:(fun () -> try Unix.close s with Unix.Unix_error _ -> ())
(fun () ->
Unix.connect s (Unix.ADDR_UNIX t.agent);
let msg = verb ^ "\n" in
ignore (Unix.write_substring s msg 0 (String.length msg));
let b = Bytes.create 4096 in
let buf = Buffer.create 128 in
let rec drain () =
match Unix.read s b 0 4096 with
| 0 -> ()
| n -> Buffer.add_subbytes buf b 0 n; drain ()
| exception Unix.Unix_error _ -> ()
in
drain ();
Buffer.contents buf)
type state =
| Running
| Stopped of string (* the condition's class name *)
| Unreachable of string (* no answer: exited, or never listened *)
(* [status] is answered whether or not the program is stopped — "running" is an
answer, not a refusal. Everything else the break loop offers is refused
while running, and rightly: there is no restart stack to walk. But the
question an editor asks *without already knowing* is this one, so it had to
have an answer in both states or there would be nothing to poll. *)
let state t =
match ask t "status" with
| "" -> Unreachable "the program is not answering on its socket"
| text ->
let line = String.trim (List.hd (String.split_on_char '\n' text)) in
if line = "running" then Running
else if String.length line > 8 && String.sub line 0 8 = "stopped " then
Stopped (String.sub line 8 (String.length line - 8))
else Unreachable ("the program answered " ^ line)
| exception Unix.Unix_error (e, _, _) -> Unreachable (Unix.error_message e)
(* Innermost first, terminated by a line that is a single dot — the agent's
framing, not this one's. A refusal comes back as a line starting "err ", and
is passed on rather than turned into an empty list: no restarts and cannot
say are different answers. *)
let restarts t =
match ask t "restarts" with
| text ->
let lines = String.split_on_char '\n' text in
if List.exists (fun l -> String.length l >= 3 && String.sub l 0 3 = "err") lines
then Error (String.trim text)
else
Ok
(List.filter
(fun l -> l <> "" && l <> ".")
(List.map String.trim lines))
| exception Unix.Unix_error (e, _, _) -> Error (Unix.error_message e)
let alive t =
match Unix.waitpid [ Unix.WNOHANG ] t.child with
| 0, _ -> true
| _ -> false
| exception Unix.Unix_error _ -> false
(* ── Ops ───────────────────────────────────────────────────────────── *)
(* Every reply is a plist with a :status, so an editor can dispatch on one key
and never has to guess whether a missing field means failure. *)
let ok fields =
"(:status \"ok\"" ^ String.concat "" (List.map (fun f -> " " ^ f) fields) ^ ")"
(* Anything the program printed since the last reply rides along with this one.
An editor that had to ask separately would miss the output an evaluation
itself caused, which is the output anyone actually wants to see. *)
let with_output t reply =
match take t with
| "" -> reply
| text ->
let i = String.length reply - 1 in
String.sub reply 0 i ^ " :output " ^ Wire.quote text ^ ")"
(* The break state rides along with every reply, exactly as the program's own
output does, and for the same reason: a program can stop at any moment and
nothing in a request/response protocol will mention it unless every response
does. An editor that had to *ask* would find out about a stop only when it
happened to wonder — and the most common moment for a program to stop is the
instant after an evaluation, which is a reply it is already reading.
It is the annotation, not the ops, that decides these two fields, so that
there is one place in the daemon that says whether the program is stopped
and the break ops cannot disagree with the poll. *)
let with_break t reply =
let fields =
match state t with
| Stopped c -> " :stopped t :condition " ^ Wire.quote c
| Running -> " :stopped nil"
(* Unreachable is not "running": the honest shape of "it exited" is
[:alive nil] from [describe], and claiming a state we could not read
would be the [ok]-means-probably failure in miniature. *)
| Unreachable _ -> " :stopped nil"
in
String.sub reply 0 (String.length reply - 1) ^ fields ^ ")"
let error ?loc msg =
"(:status \"error\" :message " ^ Wire.quote msg
^ (match loc with None -> "" | Some l -> " :loc " ^ Wire.quote l)
^ ")"
let eval t ~code ~origin =
if not (alive t) then error "the program exited; restart flan dev"
else
match Session.eval ~origin t.session code with
| c when not c.Session.installs ->
(* Accepted into the session and nothing to send: a declaration the
program already has, with no body and no new storage. Saying "ok" and
shipping an empty module would report success for a change that cannot
have taken effect. *)
ok
[ ":names " ^ Wire.strings c.Session.names; ":fns ()";
":note " ^ Wire.quote "nothing to install" ]
| c ->
t.n <- t.n + 1;
let out = Filename.concat t.dir (Printf.sprintf "m%d.so" t.n) in
(match Build.shared ~opts:{ Build.default with Build.dev = true }
~ir:c.Session.ir ~out () with
| timing ->
(match deliver t out with
| "ok" ->
ok
[ ":names " ^ Wire.strings c.Session.names;
":fns " ^ Wire.strings c.Session.fns;
Printf.sprintf ":ms %.1f"
(timing.Build.llc_ms +. timing.Build.link_ms) ]
| reply -> error ("the program refused the module: " ^ reply)
| exception Unix.Unix_error (e, _, _) ->
error
("cannot reach the program on " ^ t.agent ^ ": "
^ Unix.error_message e))
| exception Failure m -> error m)
| exception Loc.Error (l, msg) -> error ~loc:(Loc.to_string l) msg
(* Redefining a name installs a body; evaluating an expression has no name to
install into, so the module carries a thunk the agent runs once. The value
comes back through the runtime rather than through this reply, because the
frame boundary it runs at is the program's to choose. *)
let eval_expr t ~code ~origin =
if not (alive t) then error "the program exited; restart flan dev"
else
match Session.eval_expr ~origin t.session code with
| c ->
let before = match result t with Some (g, _) -> g | None -> 0L in
t.n <- t.n + 1;
let out = Filename.concat t.dir (Printf.sprintf "e%d.so" t.n) in
(match Build.shared ~opts:{ Build.default with Build.dev = true }
~ir:c.Session.ir ~out () with
| _ ->
(match deliver t out with
| "ok" ->
let rec wait ms =
match result t with
| Some (g, v) when Int64.compare g before > 0 -> Some v
| _ when ms <= 0 -> None
| _ ->
ignore (Unix.select [] [] [] 0.005);
if alive t then wait (ms - 5) else None
in
(match wait 5000 with
| Some v -> ok [ ":value " ^ Wire.quote v ]
| None ->
error
"the program did not reach a frame boundary; is it calling \
(agent/poll)?")
| reply -> error ("the program refused the module: " ^ reply)
| exception Unix.Unix_error (e, _, _) ->
error ("cannot reach the program: " ^ Unix.error_message e))
| exception Failure m -> error m)
| exception Loc.Error (l, msg) -> error ~loc:(Loc.to_string l) msg
let describe t =
ok
[ ":fns "
^ Wire.strings
(List.map (fun (f : Tast.fn) -> f.Tast.name)
t.session.Session.program.Tast.fns);
":globals "
^ Wire.strings
(List.map (fun (g : Tast.global) -> g.Tast.gname)
t.session.Session.program.Tast.globals);
":alive " ^ (if alive t then "t" else "nil") ]
(* [describe] answers what exists; this answers what each one *is*. Its own op
rather than more fields on [describe], because [describe] is polled — an
editor uses it to drain the program's output — and this is asked once on
connect and again after each install. Putting signatures on the poll would
pay for them every time anyone looked at the output buffer.
One entry per name: (name kind signature loc). Four strings, so the editor
reads it with [read] and nothing here needs a new wire type. [loc] is empty
where there is none to give — only [Tast.fn] carries one — and an editor
that finds it empty must say so rather than guess a file.
Parameter *names* are not in the Tast, so a signature shows types only. *)
let signature_of_fn (f : Tast.fn) =
Printf.sprintf "%s [%s] %s" f.Tast.name
(String.concat " " (List.map Types.to_string f.Tast.params))
(Types.to_string f.Tast.ret)
let entry ~name ~kind ~sign ~loc =
Wire.list [ Wire.quote name; Wire.quote kind; Wire.quote sign; Wire.quote loc ]
let defs t =
let p = t.session.Session.program in
let fns =
List.filter_map
(fun (f : Tast.fn) ->
match f.Tast.fparent with
(* A handler-bind clause the checker lifted out. Nobody wrote this
name, so completing it is noise and jumping to it is meaningless. *)
| Some _ -> None
| None ->
Some
(entry ~name:f.Tast.name ~kind:"fn" ~sign:(signature_of_fn f)
~loc:(Loc.to_string f.Tast.floc)))
p.Tast.fns
in
let globals =
List.map
(fun (g : Tast.global) ->
entry ~name:g.Tast.gname
~kind:(if g.Tast.gconst then "const" else "var")
~sign:
(Printf.sprintf "%s %s" g.Tast.gname (Types.to_string g.Tast.gty))
~loc:"")
p.Tast.globals
in
let externs =
List.map
(fun (e : Tast.extern) ->
entry ~name:e.Tast.ename ~kind:"extern"
~sign:
(Printf.sprintf "%s [%s] %s" e.Tast.ename
(String.concat " " (List.map Types.to_string e.Tast.eparams))
(Types.to_string e.Tast.eret))
~loc:"")
p.Tast.externs
in
ok [ ":defs " ^ Wire.list (fns @ globals @ externs) ]
(* What is on offer where the program stopped. [:stopped] and [:condition] are
not here: the annotation puts them on this reply as it puts them on every
other, so an editor reads the same two keys whatever it asked. What this op
adds is the restart names, which cost a second round trip to the program and
are wanted only when someone is about to choose one. *)
let break t =
if not (alive t) then error "the program exited; restart flan dev"
else
match state t with
| Running -> ok []
| Unreachable m -> error ("cannot ask the program whether it stopped: " ^ m)
| Stopped _ ->
(match restarts t with
| Ok names -> ok [ ":restarts " ^ Wire.strings names ]
| Error m -> error ("the program refused to list its restarts: " ^ m))
(* A choice is validated by the *program*, on its listener thread, against a
stack the stopped game thread is holding still — not here. The daemon has no
copy of that stack and anything it checked would be a guess that was true a
moment ago.
"ok" therefore means accepted, and says so: the resume happens when the
stopped thread next comes round its loop, which is microseconds away and
still not now. An editor that read [ok] as "running again" would poll once,
find it stopped, and re-open the prompt it had just answered. *)
let choose t ~name =
if not (alive t) then error "the program exited; restart flan dev"
else
match ask t ("restart " ^ name) with
| reply when String.trim reply = "ok" ->
ok
[ ":restart " ^ Wire.quote name;
":note "
^ Wire.quote "accepted; the program resumes at its next pass of the break loop" ]
| reply -> error (String.trim reply)
| exception Unix.Unix_error (e, _, _) ->
error ("cannot reach the program: " ^ Unix.error_message e)
(* The other way out. The program exits 134 where it stopped, which ends this
daemon too — it owns the program's lifetime and has nothing left to serve.
Refused while running, by the program, for the same reason a restart is. *)
let abort t =
if not (alive t) then error "the program exited; restart flan dev"
else
match ask t "abort" with
| reply when String.trim reply = "ok" ->
ok [ ":note " ^ Wire.quote "the program is exiting; flan dev ends with it" ]
| reply -> error (String.trim reply)
| exception Unix.Unix_error (e, _, _) ->
error ("cannot reach the program: " ^ Unix.error_message e)
let handle t req =
match Wire.string_field req "op" with
| Some "eval" ->
(match Wire.string_field req "code" with
| Some code ->
let origin =
match Wire.string_field req "file" with Some f -> f | None -> "<editor>"
in
eval t ~code ~origin
| None -> error "eval needs :code")
| Some "eval-expr" ->
(match Wire.string_field req "code" with
| Some code ->
let origin =
match Wire.string_field req "file" with Some f -> f | None -> "<editor>"
in
eval_expr t ~code ~origin
| None -> error "eval-expr needs :code")
| Some "describe" -> describe t
| Some "defs" -> defs t
| Some "break" -> break t
| Some "restart" ->
(match Wire.string_field req "name" with
| Some name -> choose t ~name
| None -> error "restart needs :name")
| Some "abort" -> abort t
| Some "close" -> ok []
| Some op -> error ("unknown op: " ^ op)
| None -> error "no :op"
(* ── The loop ──────────────────────────────────────────────────────── *)
(* One connection at a time. An editor is one client, evaluations are
sequential by nature — each one is checked against the program the last one
left behind — and a second concurrent evaluation would be racing for the
same session anyway. *)
(* Returns whether the client asked to end the session. One editor per daemon,
so [close] shuts the whole thing down rather than waiting for another
connection nobody is going to make. *)
let serve t fd =
let rec go () =
match Wire.recv fd with
| src ->
let op, reply =
match Wire.parse src with
| req -> (Wire.string_field req "op", handle t req)
| exception Loc.Error (_, m) -> (None, error ("bad request: " ^ m))
in
Wire.send fd (with_output t (with_break t reply));
if op = Some "close" then true else go ()
| exception Wire.Closed -> false
| exception Unix.Unix_error _ -> false
in
go ()
let start ~file ~sock =
let t0 = Unix.gettimeofday () in
(* Absolute, because every location this daemon ever reports is derived from
it and an editor is not in this process's working directory. [flan dev
src/game.flan] run from a project root would otherwise send back
"src/game.flan:12:7", which the editor can only resolve by guessing which
directory it was relative to. *)
let file = try Unix.realpath file with Unix.Unix_error _ -> file in
let session, l = Session.create ~file in
let dir =
Filename.concat (Filename.get_temp_dir_name ())
(Printf.sprintf "flan-dev-%d" (Unix.getpid ()))
in
(try Unix.mkdir dir 0o700 with Unix.Unix_error (Unix.EEXIST, _, _) -> ());
let exe = Filename.concat dir "program" in
ignore
(Build.executable ~opts:{ Build.default with Build.dev = true }
~csrcs:l.Load.csrcs ~lflags:l.Load.lflags session.Session.host ~out:exe);
let agent = Filename.concat dir "agent.sock" in
(* The program's source names some socket path; the daemon is the one that
knows where it wants to talk to it, so it overrides through the
environment. Guessing instead would fail silently — everything compiles,
the module is built, and nothing ever receives it. *)
Unix.putenv "FLAN_AGENT_SOCKET" agent;
(* Through a pipe, so the program's own output can reach an editor instead of
only the terminal the daemon was started in. *)
let rd, wr = Unix.pipe ~cloexec:false () in
let child = Unix.create_process exe [| exe |] Unix.stdin wr Unix.stderr in
Unix.close wr;
Unix.set_nonblock rd;
(* Wait for it to bind before accepting an evaluation. One that arrives first
would fail for a reason that reads like a compiler bug. *)
if not (await (fun () -> Sys.file_exists agent)) then begin
(try Unix.kill child Sys.sigterm with Unix.Unix_error _ -> ());
failwith
("the program never listened on " ^ agent
^ " — does it call (agent/start ...)?")
end;
let t =
{ session; child; agent; dir; stdout = rd; out = Buffer.create 4096; n = 0 }
in
(try Unix.unlink sock with Unix.Unix_error _ -> ());
let ls = Unix.socket Unix.PF_UNIX Unix.SOCK_STREAM 0 in
Unix.bind ls (Unix.ADDR_UNIX sock);
Unix.listen ls 4;
Printf.eprintf "flan dev: %s ready on %s (%.0fms)\n%!" file sock
((Unix.gettimeofday () -. t0) *. 1000.);
(* [accept] would block past the program's own exit, so it is waited on with
a timeout and the child checked each time round: a daemon whose program
has finished has nothing left to do, and an editor waiting on it would
wait forever. *)
let rec accept_loop () =
if alive t then
(* The program's pipe is in the same select as the listening socket: it
has to be drained whether or not an editor is asking for anything. *)
match Unix.select [ ls; t.stdout ] [] [] 0.2 with
| [], _, _ -> accept_loop ()
| ready, _, _ when not (List.mem ls ready) -> drain t; accept_loop ()
| _ ->
(match Unix.accept ls with
| fd, _ ->
let closed = serve t fd in
(try Unix.close fd with Unix.Unix_error _ -> ());
if not closed then accept_loop ()
| exception Unix.Unix_error (Unix.EINTR, _, _) -> accept_loop ())
| exception Unix.Unix_error (Unix.EINTR, _, _) -> accept_loop ()
in
Fun.protect
~finally:(fun () ->
(try Unix.kill child Sys.sigterm with Unix.Unix_error _ -> ());
(try Unix.close ls with Unix.Unix_error _ -> ());
(try Unix.close rd with Unix.Unix_error _ -> ());
(try Unix.unlink sock with Unix.Unix_error _ -> ()))
accept_loop