From 366a8724baf7962e37fcec8e84f87b28e3f0cb27 Mon Sep 17 00:00:00 2001 From: Joseph Ferano Date: Thu, 17 Sep 2026 21:49:40 +0700 Subject: [PATCH] A byte count that cannot be represented is StorageExhausted, not a smaller block Every size the containers compute is a product of a capacity the program chose and an element size the checker did, and a product that wraps leaves a block that fits beside a capacity that does not. The next write goes past the end of an allocation a sanitizer was told to expect, which is the one corruption nothing in the suite could have found. The Vec's growth, the Pool's two blocks and their sum, the map's five runs and the budget check now go through checked arithmetic. A size with no representation reports along the path an out-of-memory already takes, with the largest number the condition's field can hold, since the true one has none. The test pins the case the guard exists for: an element of 2^33 + 1 bytes at a capacity of 2^31 wraps to 2 GiB, which a heap allocator answers. --- runtime/flan_rt.c | 116 ++++++++++++++++++++++++---- test/programs/reserve-overflow.flan | 30 +++++++ test/test_acceptance.ml | 21 +++++ 3 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 test/programs/reserve-overflow.flan diff --git a/runtime/flan_rt.c b/runtime/flan_rt.c index 2ca6ff4..ba767a3 100644 --- a/runtime/flan_rt.c +++ b/runtime/flan_rt.c @@ -780,9 +780,47 @@ struct flan_allocator { int64_t budget; }; -/* Would this request put the allocator over its budget? */ +/* ── Byte counts that cannot wrap ────────────────────────────────────── + * + * Every size this file computes is a signed 64-bit count of bytes, and every + * one of them is a product or a sum of numbers a Flan program chose: a + * capacity from (vec-reserve!), an element size from the checker. Signed + * overflow is undefined, and the defined-in-practice outcome is worse than + * the undefined one — a product that wraps to a small positive allocates a + * block that fits while the container records the unwrapped capacity, and the + * next push memcpys past the end of it. Nothing traps, nothing is reported, + * and a sanitizer sees a write inside a block it was told to expect. + * + * So the arithmetic goes through these two, which answer "did it fit" the way + * every allocating entry point in this file does. A count that does not fit + * is *not* a new condition: the request is one no allocator could satisfy, so + * it reports as StorageExhausted along the path an out-of-memory takes, and + * the caller's [flan_fail_bytes] carries FLAN_BYTES_UNREPRESENTABLE — the + * largest number the field can hold, which is honest in the only way it can + * be, since the true size has no representation to report. */ +#define FLAN_BYTES_UNREPRESENTABLE INT64_MAX + +static int flan_mul_bytes(int64_t a, int64_t b, int64_t *out) { + if (a < 0 || b < 0) return 0; + return !__builtin_mul_overflow(a, b, out); +} + +static int flan_add_bytes(int64_t a, int64_t b, int64_t *out) { + if (a < 0 || b < 0) return 0; + return !__builtin_add_overflow(a, b, out); +} + +/* Would this request put the allocator over its budget? + * + * The sum is checked for the same reason the products are: a size that makes + * [live_bytes + size] wrap negative would compare below any ceiling and pass, + * which is the one answer this function must never give. A request that + * cannot even be added to what is already live is over every budget there is. */ static int flan_over_budget(flan_allocator *a, int64_t size) { - return a->budget > 0 && a->live_bytes + size > a->budget; + int64_t total; + if (a->budget <= 0) return 0; + if (!flan_add_bytes(a->live_bytes, size, &total)) return 1; + return total > a->budget; } /* ── The allocation registry's two halves, and why they are split ────── @@ -1322,7 +1360,7 @@ void flan_vec_region_only(flan_vec *v, const uint8_t *loc, int64_t loclen) { static int8_t flan_vec_grow(flan_vec *v, int64_t want, int64_t size, int64_t align) { flan_allocator *a = flan_vec_adopt(v); - int64_t cap = v->cap; + int64_t cap = v->cap, bytes; void *p; if (want <= cap) return 1; /* Doubling, from four. Four rather than one because the three reallocations @@ -1333,13 +1371,23 @@ static int8_t flan_vec_grow(flan_vec *v, int64_t want, int64_t size, if (cap > (int64_t)1 << 40) { cap = want; break; } cap *= 2; } - flan_fail_bytes = cap * size; + /* [want] arrives from (vec-reserve!) unfiltered, and the doubling loop above + * hands a want past 1<<40 straight through as the capacity, so this product + * is the one the program picked times the one the checker did. See the note + * on flan_mul_bytes. */ + if (!flan_mul_bytes(cap, size, &bytes)) { + flan_fail_bytes = FLAN_BYTES_UNREPRESENTABLE; + flan_fail_align = align; + flan_fail_id = (int64_t)(intptr_t)a; + return 0; + } + flan_fail_bytes = bytes; flan_fail_align = align; flan_fail_id = (int64_t)(intptr_t)a; if (v->ptr) - p = a->proc(a, FLAN_ALLOC_RESIZE, v->ptr, v->cap * size, cap * size, align); + p = a->proc(a, FLAN_ALLOC_RESIZE, v->ptr, v->cap * size, bytes, align); else - p = a->proc(a, FLAN_ALLOC_ALLOC, NULL, 0, cap * size, align); + p = a->proc(a, FLAN_ALLOC_ALLOC, NULL, 0, bytes, align); if (!p) return 0; v->ptr = p; v->cap = cap; @@ -1563,6 +1611,7 @@ static int8_t flan_pool_grow(flan_pool *p, int64_t want, int64_t size, int64_t align) { flan_allocator *a = flan_pool_adopt(p); int64_t cap = p->cap, sslot = (int64_t)sizeof(flan_pool_slot); + int64_t ibytes, sbytes, total; void *ni, *ns; if (want <= cap) return 1; /* Doubling from four, exactly as the Vec grows. */ @@ -1571,19 +1620,30 @@ static int8_t flan_pool_grow(flan_pool *p, int64_t want, int64_t size, if (cap > (int64_t)1 << 40) { cap = want; break; } cap *= 2; } - flan_fail_bytes = cap * size + cap * sslot; + /* Two products and their sum, all three checked: the pool asks for the items + * and the slots as separate blocks but reports them as one number, and a + * wrap in either half is the same memcpy past the end the Vec's is. */ + if (!flan_mul_bytes(cap, size, &ibytes) + || !flan_mul_bytes(cap, sslot, &sbytes) + || !flan_add_bytes(ibytes, sbytes, &total)) { + flan_fail_bytes = FLAN_BYTES_UNREPRESENTABLE; + flan_fail_align = align; + flan_fail_id = (int64_t)(intptr_t)a; + return 0; + } + flan_fail_bytes = total; flan_fail_align = align; flan_fail_id = (int64_t)(intptr_t)a; - ni = a->proc(a, FLAN_ALLOC_ALLOC, NULL, 0, cap * size, align); + ni = a->proc(a, FLAN_ALLOC_ALLOC, NULL, 0, ibytes, align); if (!ni) return 0; - ns = a->proc(a, FLAN_ALLOC_ALLOC, NULL, 0, cap * sslot, 8); + ns = a->proc(a, FLAN_ALLOC_ALLOC, NULL, 0, sbytes, 8); if (!ns) { /* An allocator without can-free leaks the first block here. That is the * defined outcome and not a new one: the request failed because the * region is exhausted, and the region is about to be released whole or * the ceiling raised and the call re-attempted. */ if (a->caps & FLAN_CAN_FREE) - a->proc(a, FLAN_ALLOC_FREE, ni, cap * size, 0, align); + a->proc(a, FLAN_ALLOC_FREE, ni, ibytes, 0, align); return 0; } if (p->len > 0) { @@ -2036,9 +2096,15 @@ static int64_t flan_cell_size(int64_t size) { * one, far outweighing the per-slot indexing the cell shift covers. */ static int64_t flan_run_bytes(int64_t epc, int64_t cell, int64_t shift, int64_t count) { - int64_t cells = - (shift >= 0) ? ((count + epc - 1) >> shift) : ((count + epc - 1) / epc); - return cells * cell; + int64_t cells, bytes; + if (count < 0 || count > FLAN_BYTES_UNREPRESENTABLE - epc) + return FLAN_BYTES_UNREPRESENTABLE; + cells = (shift >= 0) ? ((count + epc - 1) >> shift) : ((count + epc - 1) / epc); + /* One predictable branch on a path that is otherwise two shifts. The count + * is a capacity the program asked for, so the product is only bounded by + * what the checker knows the element to be — see flan_mul_bytes. */ + if (!flan_mul_bytes(cells, cell, &bytes)) return FLAN_BYTES_UNREPRESENTABLE; + return bytes; } static int64_t flan_cells_bytes(int64_t size, int64_t count) { @@ -2077,10 +2143,24 @@ static uint8_t *flan_cell_at(uint8_t *base, int64_t size, int64_t epc, * the element in flight. Odin allocates the same two, for the same reason: the * swap is a memcpy between type-erased buffers and there is no local of the * right type to hold one. */ +/* Saturating rather than refusing, because this is called for its number by + * the geometry as well as by the allocation, and the geometry has nowhere to + * put a failure. FLAN_BYTES_UNREPRESENTABLE out of here is the only value the + * allocation is allowed to see and not attempt: flan_map_alloc turns it into + * the StorageExhausted an impossible request deserves. */ static int64_t flan_map_block_size(int64_t ksize, int64_t vsize, int64_t cap) { - return flan_cells_bytes(ksize, cap) + flan_cells_bytes(vsize, cap) - + flan_cells_bytes((int64_t)sizeof(flan_map_hash), cap) - + flan_cells_bytes(ksize, 2) + flan_cells_bytes(vsize, 2); + int64_t total = 0; + int64_t parts[5]; + int i; + parts[0] = flan_cells_bytes(ksize, cap); + parts[1] = flan_cells_bytes(vsize, cap); + parts[2] = flan_cells_bytes((int64_t)sizeof(flan_map_hash), cap); + parts[3] = flan_cells_bytes(ksize, 2); + parts[4] = flan_cells_bytes(vsize, 2); + for (i = 0; i < 5; i++) + if (!flan_add_bytes(total, parts[i], &total)) + return FLAN_BYTES_UNREPRESENTABLE; + return total; } /* Everything an operation needs to walk the block, computed once on entry. @@ -2278,6 +2358,10 @@ static int8_t flan_map_alloc(flan_map *m, flan_allocator *a, int64_t log2cap, flan_fail_bytes = bytes; flan_fail_align = FLAN_MAP_CACHE_LINE; flan_fail_id = (int64_t)(intptr_t)a; + /* A block whose size does not fit in the count is a request no allocator can + * answer, and asking anyway would hand a wrapped number to a proc that might + * take it. The failure is the allocator's own. */ + if (bytes == FLAN_BYTES_UNREPRESENTABLE) return 0; p = a->proc(a, FLAN_ALLOC_ALLOC, NULL, 0, bytes, FLAN_MAP_CACHE_LINE); if (!p) return 0; m->data = p; diff --git a/test/programs/reserve-overflow.flan b/test/programs/reserve-overflow.flan new file mode 100644 index 0000000..3591f7a --- /dev/null +++ b/test/programs/reserve-overflow.flan @@ -0,0 +1,30 @@ +;;;; A capacity whose byte count has no representation — the overflow half of +;;;; spec-memory.md's "Allocation failure". +;;;; +;;;; (reserve v n) forwards n to the growth path unfiltered and the bytes asked +;;;; for are the capacity times the element size, so a large element and a +;;;; large count multiply past what a signed 64-bit count of bytes can hold. +;;;; The element here is 2^33 + 1 bytes and the capacity the doubling settles +;;;; on is 2^31, whose product is 2^64 + 2^31: the wrap leaves 2 GiB, which a +;;;; heap allocator answers. The block then fits and the recorded capacity does +;;;; not, and the first push past 2 GiB writes outside it with nothing said — +;;;; no trap, no diagnostic, and a sanitizer sees a write inside a block it was +;;;; told to expect. That is why the guard is on the arithmetic and not on the +;;;; allocator's answer. +;;;; +;;;; A size that cannot be represented is a request no allocator can satisfy, +;;;; so it reports as StorageExhausted along the path an out-of-memory takes, +;;;; and with nothing handling it the program stops here rather than carrying +;;;; on — the same rule exhausted-unhandled.flan pins for a ceiling that was +;;;; genuinely reached. +(defstruct Big + [f0 [1073741824 u8] f1 [1073741824 u8] f2 [1073741824 u8] + f3 [1073741824 u8] f4 [1073741824 u8] f5 [1073741824 u8] + f6 [1073741824 u8] f7 [1073741824 u8] tail [1 u8]]) + +(defn main [] i32 + (let [v (vec-new Big)] + (println "before") + (reserve v 1073741825) + (println "unreachable")) + 0) diff --git a/test/test_acceptance.ml b/test/test_acceptance.ml index a99ab31..61dffa2 100644 --- a/test/test_acceptance.ml +++ b/test/test_acceptance.ml @@ -741,6 +741,27 @@ let () = end; (try Sys.remove exe with Sys_error _ -> ()); + (* The request whose *size* is the thing that does not fit. The count times + the element size wraps to a number a heap allocator answers, so the + block is real, the recorded capacity is not, and the write past it is + the failure no sanitizer can see — it is inside a block ASan was told to + expect. Guarded arithmetic makes it the same StorageExhausted an + exhausted region raises, because it is the same answer: the storage + asked for is not available. *) + let exe = compile "programs/reserve-overflow.flan" in + let code, text = run exe None in + if code <> 134 || not (contains text "before") + || not (contains text "unhandled StorageExhausted") + || contains text "unreachable" + then begin + incr failures; + Printf.printf + "FAIL a byte count that cannot be represented is StorageExhausted\n\ + \ got: %S (exit %d)\n wanted: exit 134, naming the condition\n" + text code + end; + (try Sys.remove exe with Sys_error _ -> ()); + (* -- Files, NEXT.md decisions 1, 2 and 5 -------------------------- Embedding first, because it is the one that costs nothing at run time and needs no host ABI at all: a compiler feature, so no linker