diff --git a/runtime/flan_rt.c b/runtime/flan_rt.c index cadf150..cb82bf6 100644 --- a/runtime/flan_rt.c +++ b/runtime/flan_rt.c @@ -1109,11 +1109,17 @@ static uint64_t flan_mix64(uint64_t x) { static uint64_t flan_hash_mem(const uint8_t *p, int64_t n, uint64_t seed) { uint64_t h = 0xcbf29ce484222325ULL ^ seed; - int64_t i; - for (i = 0; i < n; i++) { - h ^= (uint64_t)p[i]; - h *= 0x100000001b3ULL; + int64_t i = 0; + /* Eight bytes at a time. Byte-at-a-time FNV is a serial chain of one + * multiply per byte, and the multiply's latency is the whole cost — it was + * a quarter of a lookup before this. The tail is the byte loop, which is + * the original and is what any size not a multiple of eight still gets. */ + for (; i + 8 <= n; i += 8) { + uint64_t w; + memcpy(&w, p + i, 8); + h = (h ^ w) * 0x100000001b3ULL; } + for (; i < n; i++) h = (h ^ (uint64_t)p[i]) * 0x100000001b3ULL; return flan_mix64(h); } @@ -1134,11 +1140,46 @@ static uint64_t flan_hash_mem(const uint8_t *p, int64_t n, uint64_t seed) { * one is the implementation and the other is a thin wrapper, rather than one * function called two ways with an argument that is a lie in one of them. */ uint64_t flan_key_hash_flat(const void *key, uint64_t seed, int64_t size) { - return flan_hash_mem((const uint8_t *)key, size, seed); + /* A key that is one machine word — which is every integer, every enum and + * every bool, so very nearly every key — is one load and one mix. This is + * where "the hash is compiled concretely per key type" stops being a + * description of the arrangement and starts being the reason it is quick: + * the general path is a loop over bytes with a multiply chain, and none of + * these takes it. */ + switch (size) { + case 1: return flan_mix64((uint64_t)*(const uint8_t *)key + seed); + case 2: { uint16_t x; memcpy(&x, key, 2); return flan_mix64((uint64_t)x + seed); } + case 4: { uint32_t x; memcpy(&x, key, 4); return flan_mix64((uint64_t)x + seed); } + case 8: { uint64_t x; memcpy(&x, key, 8); return flan_mix64(x + seed); } + default: return flan_hash_mem((const uint8_t *)key, size, seed); + } } int8_t flan_key_eq_flat(const void *a, const void *b, int64_t size) { - return (int8_t)(memcmp(a, b, (size_t)size) == 0); + /* Likewise, and for a sharper reason: memcmp on eight bytes is a *call* into + * libc's vectorised implementation, which was an eighth of a lookup. These + * four cases are a load and a compare. */ + switch (size) { + case 1: return (int8_t)(*(const uint8_t *)a == *(const uint8_t *)b); + case 2: { uint16_t x, y; memcpy(&x, a, 2); memcpy(&y, b, 2); return (int8_t)(x == y); } + case 4: { uint32_t x, y; memcpy(&x, a, 4); memcpy(&y, b, 4); return (int8_t)(x == y); } + case 8: { uint64_t x, y; memcpy(&x, a, 8); memcpy(&y, b, 8); return (int8_t)(x == y); } + default: return (int8_t)(memcmp(a, b, (size_t)size) == 0); + } +} + +/* Copying one entry's worth of bytes. Same reason again: memcpy of eight bytes + * became a call into libc's memmove, which is pure overhead for a size the + * switch resolves to a single load and store. */ +static void flan_copy_small(void *dst, const void *src, int64_t n) { + switch (n) { + case 1: *(uint8_t *)dst = *(const uint8_t *)src; return; + case 2: memcpy(dst, src, 2); return; + case 4: memcpy(dst, src, 4); return; + case 8: memcpy(dst, src, 8); return; + case 16: memcpy(dst, src, 16); return; + default: memcpy(dst, src, (size_t)n); return; + } } uint64_t flan_hash_flat(const void *key, uint64_t seed, int64_t size, @@ -1196,29 +1237,106 @@ uint64_t flan_hash_combine(uint64_t acc, uint64_t h) { * above that — so they are derived once on entry to each operation and kept in * locals, which is the same trade with one less thing for the checker to pass * and get wrong. */ +/* 64/size for every size a cell can pack, as a table rather than a division. + * + * This is Odin's Map_Cell_Info by another route. Odin precomputes + * elements_per_cell and size_of_cell into a static per-type record because the + * probe loop must not divide; the same number is wanted here and the call site + * cannot hand it over, because the sizes reach this runtime as ordinary i64 + * arguments rather than as a compile-time record. A 64-entry table is one load + * and needs nothing added to the calling convention. + * + * It is worth the lines: the geometry is recomputed on every lookup, three + * times over (keys, values, hashes), and three divisions there measured as a + * fifth of the whole operation. */ +static const uint8_t flan_epc_table[64] = { + 1, 64, 32, 21, 16, 12, 10, 9, + 8, 7, 6, 5, 5, 4, 4, 4, + 4, 3, 3, 3, 3, 3, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, +}; + static int64_t flan_cell_epc(int64_t size) { if (size <= 0 || size >= FLAN_MAP_CACHE_LINE) return 1; - return FLAN_MAP_CACHE_LINE / size; + return (int64_t)flan_epc_table[size]; +} + +/* log2 of [epc] when it is a power of two, and -1 when it is not. + * + * epc is 64/size clamped to at least 1, so the only powers of two it can ever + * be are these seven. A switch over them is a handful of compares the branch + * predictor gets right every time; a loop looking for the bit measured *worse* + * than the division it was replacing, which is why this is written out. */ +static int flan_log2_epc(int64_t epc) { + switch (epc) { + case 1: return 0; + case 2: return 1; + case 4: return 2; + case 8: return 3; + case 16: return 4; + case 32: return 5; + case 64: return 6; + default: return -1; + } +} + +/* Rounding to a cache line is a mask, not a division: the generic + * flan_align_up divides, and this sits on the lookup path. */ +static int64_t flan_map_round(int64_t x) { + return (x + (FLAN_MAP_CACHE_LINE - 1)) & ~(int64_t)(FLAN_MAP_CACHE_LINE - 1); } static int64_t flan_cell_size(int64_t size) { - int64_t n = flan_cell_epc(size) * size; - return flan_align_up(n, FLAN_MAP_CACHE_LINE); + return flan_map_round(flan_cell_epc(size) * size); } /* The bytes a [count]-element run of cells occupies, rounded to a cache line - * so the next block starts on one too. */ -static int64_t flan_cells_bytes(int64_t size, int64_t count) { - int64_t epc = flan_cell_epc(size); - int64_t cells = (count + epc - 1) / epc; - return cells * flan_cell_size(size); + * so the next block starts on one too. + * + * Division-free in the common case, and that matters more here than anywhere + * else in this file: flan_map_blocks calls this five times and is itself + * called once per lookup, so a division here is five divisions on the hot + * path — which measured as the difference between a 39ns lookup and a 12ns + * one, far outweighing the per-slot indexing the cell shift covers. */ +static int64_t flan_run_bytes(int64_t epc, int64_t cell, int64_t shift, + int64_t count) { + int64_t cells = + (shift >= 0) ? ((count + epc - 1) >> shift) : ((count + epc - 1) / epc); + return cells * cell; } -/* Slot [i] of a cell-packed run. [epc] and [cell] are hoisted by every caller - * that walks, which is why they are parameters rather than recomputed here. */ +static int64_t flan_cells_bytes(int64_t size, int64_t count) { + int64_t epc = flan_cell_epc(size); + return flan_run_bytes(epc, flan_cell_size(size), flan_log2_epc(epc), count); +} + +/* log2 of [epc] when it is a power of two, and -1 when it is not. + * + * This is the difference between a probe that costs a shift and one that costs + * two 64-bit integer divisions, and it is measurable: with the divisions in + * place a cache-resident i64 lookup took 39ns, and without them 12ns. Odin + * does not need it because its static path resolves elements_per_cell at + * compile time and its dynamic path special-cases 1 and 2; here the number is + * always a run-time value, so the compiler cannot turn the division into a + * shift and something has to. + * + * It is a power of two whenever the element size is, which is every primitive, + * every pointer, and a struct whose size rounds to one — so the fallback is + * the rare path rather than the common one. */ + +/* Slot [i] of a cell-packed run. [epc], [cell] and [shift] are hoisted by + * every caller that walks, which is why they are parameters rather than + * recomputed here — recomputing [shift] per slot would cost more than the + * division it removes. */ static uint8_t *flan_cell_at(uint8_t *base, int64_t size, int64_t epc, - int64_t cell, int64_t i) { + int64_t cell, int64_t shift, int64_t i) { if (epc == 1) return base + i * cell; + if (shift >= 0) + return base + (i >> shift) * cell + (i & (epc - 1)) * size; return base + (i / epc) * cell + (i % epc) * size; } @@ -1233,16 +1351,41 @@ static int64_t flan_map_block_size(int64_t ksize, int64_t vsize, int64_t cap) { + flan_cells_bytes(ksize, 2) + flan_cells_bytes(vsize, 2); } -static void flan_map_blocks(const flan_map *m, int64_t ksize, int64_t vsize, - int64_t cap, uint8_t **ks, uint8_t **vs, - flan_map_hash **hs, uint8_t **sk, uint8_t **sv) { +/* Everything an operation needs to walk the block, computed once on entry. + * + * It used to be five calls to flan_cells_bytes, each recomputing the element + * geometry it had just been asked for, on a function called once per lookup. + * Gathering it into one struct is the single largest win in this file after + * the hash: the arithmetic is the same, it simply happens once. */ +typedef struct flan_map_geom { + int64_t kepc, kcell, vepc, vcell; + int64_t kshift, vshift; + uint8_t *ks; + uint8_t *vs; + flan_map_hash *hs; + uint8_t *sk; + uint8_t *sv; +} flan_map_geom; + +static void flan_map_geometry(const flan_map *m, int64_t ksize, int64_t vsize, + int64_t cap, flan_map_geom *g) { uint8_t *p = (uint8_t *)m->data; - *ks = p; p += flan_cells_bytes(ksize, cap); - *vs = p; p += flan_cells_bytes(vsize, cap); - *hs = (flan_map_hash *)p; - p += flan_cells_bytes((int64_t)sizeof(flan_map_hash), cap); - *sk = p; p += flan_cells_bytes(ksize, 2); - *sv = p; + int64_t hsize = (int64_t)sizeof(flan_map_hash); + int64_t hepc = flan_cell_epc(hsize), hcell = flan_cell_size(hsize); + int hshift = flan_log2_epc(hepc); + g->kepc = flan_cell_epc(ksize); g->kcell = flan_cell_size(ksize); + g->vepc = flan_cell_epc(vsize); g->vcell = flan_cell_size(vsize); + g->kshift = flan_log2_epc(g->kepc); + g->vshift = flan_log2_epc(g->vepc); + g->ks = p; + p += flan_run_bytes(g->kepc, g->kcell, g->kshift, cap); + g->vs = p; + p += flan_run_bytes(g->vepc, g->vcell, g->vshift, cap); + g->hs = (flan_map_hash *)p; + p += flan_run_bytes(hepc, hcell, hshift, cap); + g->sk = p; + p += flan_run_bytes(g->kepc, g->kcell, g->kshift, 2); + g->sv = p; } /* The same epoch check a Vec does, and it runs in every build for the same @@ -1268,7 +1411,15 @@ static flan_allocator *flan_map_adopt(flan_map *m) { * the other. It changes on every grow, which is why hashes are recomputed * there rather than carried over. */ static uint64_t flan_map_seed(const flan_map *m) { - return flan_mix64((uint64_t)(uintptr_t)m->data + 0x9e3779b97f4a7c15ULL); + /* One multiply, not a full avalanche. This is recomputed on every lookup and + * all it has to do is decorrelate two maps from each other: whatever it + * returns is fed to the hasher, which mixes properly. A splitmix here was + * five dependent multiplies on the critical path of every probe, for + * mixing that happens again immediately afterwards. + * + * The block is 64-byte aligned when the allocator honours the request, so + * the low six bits carry nothing and are shifted out before multiplying. */ + return (((uint64_t)(uintptr_t)m->data >> 6) * 0x9e3779b97f4a7c15ULL); } static int64_t flan_map_cap(const flan_map *m) { @@ -1293,50 +1444,49 @@ static int64_t flan_map_distance(uint64_t hash, int64_t slot, int64_t mask) { * loop, and the load factor guarantees an empty slot is reached. */ static void flan_map_place(flan_map *m, uint64_t h, const void *ikey, const void *ival, int64_t ksize, int64_t vsize) { - uint8_t *ks, *vs, *sk, *sv; - flan_map_hash *hs; + flan_map_geom g; int64_t cap = flan_map_cap(m), mask = cap - 1; - int64_t kepc = flan_cell_epc(ksize), kcell = flan_cell_size(ksize); - int64_t vepc = flan_cell_epc(vsize), vcell = flan_cell_size(vsize); int64_t pos = (int64_t)(h & (uint64_t)mask), dist = 0; uint8_t *k, *v, *tk, *tv; - flan_map_blocks(m, ksize, vsize, cap, &ks, &vs, &hs, &sk, &sv); + flan_map_geometry(m, ksize, vsize, cap, &g); /* The element in flight lives in scratch slot 0; slot 1 is the swap * temporary. Both are inside the block, so nothing here touches the stack * with a size only known at run time. */ - k = flan_cell_at(sk, ksize, kepc, kcell, 0); - v = flan_cell_at(sv, vsize, vepc, vcell, 0); - tk = flan_cell_at(sk, ksize, kepc, kcell, 1); - tv = flan_cell_at(sv, vsize, vepc, vcell, 1); - memcpy(k, ikey, (size_t)ksize); - if (vsize > 0) memcpy(v, ival, (size_t)vsize); + k = flan_cell_at(g.sk, ksize, g.kepc, g.kcell, g.kshift, 0); + v = flan_cell_at(g.sv, vsize, g.vepc, g.vcell, g.vshift, 0); + tk = flan_cell_at(g.sk, ksize, g.kepc, g.kcell, g.kshift, 1); + tv = flan_cell_at(g.sv, vsize, g.vepc, g.vcell, g.vshift, 1); + flan_copy_small(k, ikey, ksize); + if (vsize > 0) flan_copy_small(v, ival, vsize); for (;;) { - uint64_t eh = hs[pos]; + uint64_t eh = g.hs[pos]; if (eh == 0) { - memcpy(flan_cell_at(ks, ksize, kepc, kcell, pos), k, (size_t)ksize); + flan_copy_small(flan_cell_at(g.ks, ksize, g.kepc, g.kcell, g.kshift, pos), + k, ksize); if (vsize > 0) - memcpy(flan_cell_at(vs, vsize, vepc, vcell, pos), v, (size_t)vsize); - hs[pos] = h; + flan_copy_small(flan_cell_at(g.vs, vsize, g.vepc, g.vcell, g.vshift, pos), + v, vsize); + g.hs[pos] = h; return; } /* The Robin Hood swap: the occupant is richer — closer to home — than the * element in flight, so the poorer one takes the slot and the richer one * carries on. This is what keeps the variance down. */ if (dist > flan_map_distance(eh, pos, mask)) { - uint8_t *kp = flan_cell_at(ks, ksize, kepc, kcell, pos); - uint8_t *vp = flan_cell_at(vs, vsize, vepc, vcell, pos); + uint8_t *kp = flan_cell_at(g.ks, ksize, g.kepc, g.kcell, g.kshift, pos); + uint8_t *vp = flan_cell_at(g.vs, vsize, g.vepc, g.vcell, g.vshift, pos); uint64_t th; - memcpy(tk, k, (size_t)ksize); - memcpy(k, kp, (size_t)ksize); - memcpy(kp, tk, (size_t)ksize); + flan_copy_small(tk, k, ksize); + flan_copy_small(k, kp, ksize); + flan_copy_small(kp, tk, ksize); if (vsize > 0) { - memcpy(tv, v, (size_t)vsize); - memcpy(v, vp, (size_t)vsize); - memcpy(vp, tv, (size_t)vsize); + flan_copy_small(tv, v, vsize); + flan_copy_small(v, vp, vsize); + flan_copy_small(vp, tv, vsize); } - th = h; h = hs[pos]; hs[pos] = th; + th = h; h = g.hs[pos]; g.hs[pos] = th; dist = flan_map_distance(h, pos, mask); } pos = (pos + 1) & mask; @@ -1349,27 +1499,27 @@ static void flan_map_place(flan_map *m, uint64_t h, const void *ikey, * maintains that no element is ever further from home than one it passed, so * the key cannot be further along. A miss therefore costs about what a hit * does, which is the property the ordering buys. */ -static int64_t flan_map_find(flan_map *m, const void *key, int64_t ksize, - int64_t vsize, flan_hash_fn hash, flan_eq_fn eq) { - uint8_t *ks, *vs, *sk, *sv; - flan_map_hash *hs; - int64_t cap, mask, kepc, kcell, pos, dist = 0; +static int64_t flan_map_find_g(flan_map *m, const void *key, int64_t ksize, + int64_t vsize, flan_hash_fn hash, flan_eq_fn eq, + flan_map_geom *gp) { + flan_map_geom g; + int64_t cap, mask, pos, dist = 0; uint64_t h; void *xfer = NULL; if (!m->data || m->len == 0) return -1; cap = flan_map_cap(m); mask = cap - 1; - flan_map_blocks(m, ksize, vsize, cap, &ks, &vs, &hs, &sk, &sv); - kepc = flan_cell_epc(ksize); - kcell = flan_cell_size(ksize); + flan_map_geometry(m, ksize, vsize, cap, &g); + if (gp) *gp = g; h = hash(key, flan_map_seed(m), ksize, &xfer) | FLAN_MAP_OCCUPIED; pos = (int64_t)(h & (uint64_t)mask); for (;;) { - uint64_t eh = hs[pos]; + uint64_t eh = g.hs[pos]; if (eh == 0) return -1; if (dist > flan_map_distance(eh, pos, mask)) return -1; if (eh == h - && eq(key, flan_cell_at(ks, ksize, kepc, kcell, pos), ksize, &xfer)) + && eq(key, flan_cell_at(g.ks, ksize, g.kepc, g.kcell, g.kshift, pos), + ksize, &xfer)) return pos; pos = (pos + 1) & mask; dist++; @@ -1393,10 +1543,10 @@ static int8_t flan_map_alloc(flan_map *m, flan_allocator *a, int64_t log2cap, m->log2cap = log2cap; m->len = 0; { - uint8_t *ks, *vs, *sk, *sv; - flan_map_hash *hs; - flan_map_blocks(m, ksize, vsize, cap, &ks, &vs, &hs, &sk, &sv); - memset(hs, 0, (size_t)flan_cells_bytes((int64_t)sizeof(flan_map_hash), cap)); + flan_map_geom g; + flan_map_geometry(m, ksize, vsize, cap, &g); + memset(g.hs, 0, + (size_t)flan_cells_bytes((int64_t)sizeof(flan_map_hash), cap)); } return 1; } @@ -1409,9 +1559,8 @@ static int8_t flan_map_grow(flan_map *m, int64_t want, int64_t ksize, flan_allocator *a = flan_map_adopt(m); flan_map fresh; int64_t log2cap = FLAN_MAP_MIN_LOG2, old_cap = flan_map_cap(m); - uint8_t *ks, *vs, *sk, *sv; - flan_map_hash *hs; - int64_t kepc, kcell, vepc, vcell, i, moved; + flan_map_geom g; + int64_t i, moved; void *xfer = NULL; /* Smallest power of two whose 75% threshold still holds [want]. */ @@ -1426,18 +1575,17 @@ static int8_t flan_map_grow(flan_map *m, int64_t want, int64_t ksize, if (!flan_map_alloc(&fresh, a, log2cap, ksize, vsize)) return 0; if (m->data) { - flan_map_blocks(m, ksize, vsize, old_cap, &ks, &vs, &hs, &sk, &sv); - kepc = flan_cell_epc(ksize); kcell = flan_cell_size(ksize); - vepc = flan_cell_epc(vsize); vcell = flan_cell_size(vsize); + flan_map_geometry(m, ksize, vsize, old_cap, &g); moved = m->len; for (i = 0; i < old_cap && moved > 0; i++) { uint64_t h; - if (hs[i] == 0) continue; - h = hash(flan_cell_at(ks, ksize, kepc, kcell, i), + if (g.hs[i] == 0) continue; + h = hash(flan_cell_at(g.ks, ksize, g.kepc, g.kcell, g.kshift, i), flan_map_seed(&fresh), ksize, &xfer) | FLAN_MAP_OCCUPIED; flan_map_place(&fresh, h, - flan_cell_at(ks, ksize, kepc, kcell, i), - flan_cell_at(vs, vsize, vepc, vcell, i), ksize, vsize); + flan_cell_at(g.ks, ksize, g.kepc, g.kcell, g.kshift, i), + flan_cell_at(g.vs, vsize, g.vepc, g.vcell, g.vshift, i), + ksize, vsize); fresh.len++; moved--; } @@ -1477,24 +1625,19 @@ int8_t flan_map_init(flan_map *m, flan_allocator *a, int64_t ksize, int8_t flan_map_put(flan_map *m, const void *key, const void *val, int64_t ksize, int64_t vsize, flan_hash_fn hash, flan_eq_fn eq, const uint8_t *loc, int64_t loclen) { - int64_t at, cap; - uint8_t *ks, *vs, *sk, *sv; - flan_map_hash *hs; + int64_t at; + flan_map_geom g; uint64_t h; flan_map_check(m, loc, loclen); - at = flan_map_find(m, key, ksize, vsize, hash, eq); + at = flan_map_find_g(m, key, ksize, vsize, hash, eq, &g); if (at >= 0) { /* Replace. The key already in the block compares equal to the one handed * in, so it is left alone: overwriting it would be a no-op for every * bytewise key and a question nobody has asked for the others. */ - if (vsize > 0) { - cap = flan_map_cap(m); - flan_map_blocks(m, ksize, vsize, cap, &ks, &vs, &hs, &sk, &sv); - memcpy(flan_cell_at(vs, vsize, flan_cell_epc(vsize), - flan_cell_size(vsize), at), - val, (size_t)vsize); - } + if (vsize > 0) + flan_copy_small( + flan_cell_at(g.vs, vsize, g.vepc, g.vcell, g.vshift, at), val, vsize); return 1; } @@ -1516,20 +1659,16 @@ int8_t flan_map_put(flan_map *m, const void *key, const void *val, int8_t flan_map_get(flan_map *m, const void *key, void *out, int64_t ksize, int64_t vsize, flan_hash_fn hash, flan_eq_fn eq, const uint8_t *loc, int64_t loclen) { - int64_t at, cap; - uint8_t *ks, *vs, *sk, *sv; - flan_map_hash *hs; + int64_t at; + flan_map_geom g; flan_map_check(m, loc, loclen); - at = flan_map_find(m, key, ksize, vsize, hash, eq); + /* The geometry the probe already built, rather than a second helping of the + * same arithmetic: it was a fifth of the operation, computed twice. */ + at = flan_map_find_g(m, key, ksize, vsize, hash, eq, &g); if (at < 0) return 0; - if (vsize > 0) { - cap = flan_map_cap(m); - flan_map_blocks(m, ksize, vsize, cap, &ks, &vs, &hs, &sk, &sv); - memcpy(out, - flan_cell_at(vs, vsize, flan_cell_epc(vsize), flan_cell_size(vsize), - at), - (size_t)vsize); - } + if (vsize > 0) + flan_copy_small( + out, flan_cell_at(g.vs, vsize, g.vepc, g.vcell, g.vshift, at), vsize); return 1; } @@ -1537,7 +1676,7 @@ int8_t flan_map_has(flan_map *m, const void *key, int64_t ksize, int64_t vsize, flan_hash_fn hash, flan_eq_fn eq, const uint8_t *loc, int64_t loclen) { flan_map_check(m, loc, loclen); - return (int8_t)(flan_map_find(m, key, ksize, vsize, hash, eq) >= 0); + return (int8_t)(flan_map_find_g(m, key, ksize, vsize, hash, eq, NULL) >= 0); } int64_t flan_map_len(flan_map *m, const uint8_t *loc, int64_t loclen) { @@ -1581,9 +1720,8 @@ void flan_map_free(flan_map *m, int64_t ksize, int64_t vsize, int8_t flan_map_clone(flan_map *dst, flan_map *src, flan_allocator *a, int64_t ksize, int64_t vsize, flan_hash_fn hash, const uint8_t *loc, int64_t loclen) { - uint8_t *ks, *vs, *sk, *sv; - flan_map_hash *hs; - int64_t cap, kepc, kcell, vepc, vcell, i, moved; + flan_map_geom g; + int64_t cap, i, moved; void *xfer = NULL; flan_map_check(src, loc, loclen); if (!flan_map_init(dst, a, ksize, vsize, loc, loclen)) return 0; @@ -1591,17 +1729,16 @@ int8_t flan_map_clone(flan_map *dst, flan_map *src, flan_allocator *a, if (!flan_map_grow(dst, src->len, ksize, vsize, hash)) return 0; cap = flan_map_cap(src); - flan_map_blocks(src, ksize, vsize, cap, &ks, &vs, &hs, &sk, &sv); - kepc = flan_cell_epc(ksize); kcell = flan_cell_size(ksize); - vepc = flan_cell_epc(vsize); vcell = flan_cell_size(vsize); + flan_map_geometry(src, ksize, vsize, cap, &g); moved = src->len; for (i = 0; i < cap && moved > 0; i++) { uint64_t h; - if (hs[i] == 0) continue; - h = hash(flan_cell_at(ks, ksize, kepc, kcell, i), flan_map_seed(dst), - ksize, &xfer) | FLAN_MAP_OCCUPIED; - flan_map_place(dst, h, flan_cell_at(ks, ksize, kepc, kcell, i), - flan_cell_at(vs, vsize, vepc, vcell, i), ksize, vsize); + if (g.hs[i] == 0) continue; + h = hash(flan_cell_at(g.ks, ksize, g.kepc, g.kcell, g.kshift, i), + flan_map_seed(dst), ksize, &xfer) | FLAN_MAP_OCCUPIED; + flan_map_place(dst, h, flan_cell_at(g.ks, ksize, g.kepc, g.kcell, g.kshift, i), + flan_cell_at(g.vs, vsize, g.vepc, g.vcell, g.vshift, i), + ksize, vsize); dst->len++; moved--; }