ident
All repositories: gitoria
22.6 KB
// MessagePack codec for hl:mpackdb — byte-compatible with npm msgpackr 1.11.5// as mpackdb v1.0.7 uses it (decision D12).//// Measured msgpackr surface (probed against the real library, see// testdata/gen_corpus.mjs):// - NO record/shared-structure extensions: the standalone pack() export// serializes plain objects as ordinary MessagePack maps.// - Objects ALWAYS encode as map16 (0xde) — even empty — never fixmap/map32.// Maps with > 65535 keys throw in msgpackr; we mirror with error.MapTooLarge.// - Numbers: integer-valued f64 in [-2^31, 2^32) uses the smallest// uint/int family (fixint, cc, cd, ce / negfixint, d0, d1, d2). Everything// else (non-integers, |v| outside that range, NaN, ±Inf) is float64 (0xcb).// msgpackr never emits float32 or int64/uint64 for JS numbers.// - Strings: fixstr ≤31 bytes, str8 ≤255, str16 ≤65535, str32 beyond.// - Arrays: fixarray ≤15, array16 ≤65535, array32 beyond.// - Binary: bin8/16/32 by length.// - undefined: fixext1 type 0x00 data 0x00 (d4 00 00) — kept as a distinct// Value so re-encode is byte-exact; it surfaces as null in Hybriel.// - Date: timestamp ext -1 (d6ff / d7ff / c70cff). Kept as a raw ext Value// for byte-exact round-trips; timestampMs() interprets it.//// Decode accepts the FULL MessagePack spec (all int/str/bin/array/map families,// float32, int64/uint64, all ext forms); encode emits only the canonical// msgpackr subset above.const std = @import("std");pub const Error = error{Truncated,InvalidFormat, // 0xc1MapTooLarge,OutOfMemory,};pub const Ext = struct {type: i8,data: []const u8,};pub const Entry = struct {key: Value,value: Value,};pub const Value = union(enum) {nil,undef, // msgpackr `undefined` (fixext1 type 0)boolean: bool,number: f64,str: []const u8,bin: []const u8,array: []Value,map: []Entry,ext: Ext,/// Interpret a timestamp ext (-1) as milliseconds since epoch.pub fn timestampMs(self: Value) ?f64 {if (self != .ext or self.ext.type != -1) return null;const d = self.ext.data;switch (d.len) {4 => {const sec = std.mem.readInt(u32, d[0..4], .big);return @as(f64, @floatFromInt(sec)) * 1000.0;},8 => {const packed_v = std.mem.readInt(u64, d[0..8], .big);const ns: f64 = @floatFromInt(packed_v >> 34);const sec: f64 = @floatFromInt(packed_v & 0x3ffffffff);return sec * 1000.0 + ns / 1_000_000.0;},12 => {const ns = std.mem.readInt(u32, d[0..4], .big);const sec = std.mem.readInt(i64, d[4..12], .big);return @as(f64, @floatFromInt(sec)) * 1000.0 + @as(f64, @floatFromInt(ns)) / 1_000_000.0;},else => return null,}}/// Map field lookup by string key (returns null for missing / non-map).pub fn get(self: Value, key: []const u8) ?Value {if (self != .map) return null;for (self.map) |e| {if (e.key == .str and std.mem.eql(u8, e.key.str, key)) return e.value;}return null;}};pub const Decoded = struct {value: Value,len: usize, // bytes consumed};/// Decode one MessagePack value from `bytes`. All slices in the returned Value/// reference memory allocated from `alloc` (str/bin/ext data are duplicated,/// so the Value outlives `bytes`). Use an arena and free it wholesale.pub fn decode(alloc: std.mem.Allocator, bytes: []const u8) Error!Decoded {var d = Decoder{ .alloc = alloc, .bytes = bytes, .pos = 0 };const v = try d.readValue();return .{ .value = v, .len = d.pos };}const Decoder = struct {alloc: std.mem.Allocator,bytes: []const u8,pos: usize,fn need(self: *Decoder, n: usize) Error!void {if (self.pos + n > self.bytes.len) return Error.Truncated;}fn take(self: *Decoder, n: usize) Error![]const u8 {try self.need(n);const s = self.bytes[self.pos .. self.pos + n];self.pos += n;return s;}fn readBigInt(self: *Decoder, comptime T: type) Error!T {const n = @divExact(@typeInfo(T).int.bits, 8);const s = try self.take(n);return std.mem.readInt(T, s[0..n], .big);}fn dupSlice(self: *Decoder, n: usize) Error![]const u8 {const s = try self.take(n);return self.alloc.dupe(u8, s) catch return Error.OutOfMemory;}fn readArray(self: *Decoder, n: usize) Error!Value {const items = self.alloc.alloc(Value, n) catch return Error.OutOfMemory;for (items) |*it| it.* = try self.readValue();return .{ .array = items };}fn readMap(self: *Decoder, n: usize) Error!Value {const entries = self.alloc.alloc(Entry, n) catch return Error.OutOfMemory;for (entries) |*e| {e.key = try self.readValue();e.value = try self.readValue();}return .{ .map = entries };}fn readExt(self: *Decoder, n: usize) Error!Value {try self.need(1 + n);const t: i8 = @bitCast(self.bytes[self.pos]);self.pos += 1;const data = try self.dupSlice(n);// msgpackr undefined: fixext1 type 0, data byte 0if (t == 0 and n == 1 and data[0] == 0) return .undef;return .{ .ext = .{ .type = t, .data = data } };}fn readValue(self: *Decoder) Error!Value {try self.need(1);const b = self.bytes[self.pos];self.pos += 1;switch (b) {0x00...0x7f => return .{ .number = @floatFromInt(b) },0x80...0x8f => return self.readMap(b & 0x0f),0x90...0x9f => return self.readArray(b & 0x0f),0xa0...0xbf => return .{ .str = try self.dupSlice(b & 0x1f) },0xc0 => return .nil,0xc1 => return Error.InvalidFormat,0xc2 => return .{ .boolean = false },0xc3 => return .{ .boolean = true },0xc4 => return .{ .bin = try self.dupSlice(try self.readBigInt(u8)) },0xc5 => return .{ .bin = try self.dupSlice(try self.readBigInt(u16)) },0xc6 => return .{ .bin = try self.dupSlice(try self.readBigInt(u32)) },0xc7 => return self.readExt(try self.readBigInt(u8)),0xc8 => return self.readExt(try self.readBigInt(u16)),0xc9 => return self.readExt(try self.readBigInt(u32)),0xca => {const bits = try self.readBigInt(u32);return .{ .number = @as(f32, @bitCast(bits)) };},0xcb => {const bits = try self.readBigInt(u64);return .{ .number = @bitCast(bits) };},0xcc => return .{ .number = @floatFromInt(try self.readBigInt(u8)) },0xcd => return .{ .number = @floatFromInt(try self.readBigInt(u16)) },0xce => return .{ .number = @floatFromInt(try self.readBigInt(u32)) },0xcf => return .{ .number = @floatFromInt(try self.readBigInt(u64)) },0xd0 => return .{ .number = @floatFromInt(@as(i8, @bitCast(try self.readBigInt(u8)))) },0xd1 => return .{ .number = @floatFromInt(@as(i16, @bitCast(try self.readBigInt(u16)))) },0xd2 => return .{ .number = @floatFromInt(@as(i32, @bitCast(try self.readBigInt(u32)))) },0xd3 => return .{ .number = @floatFromInt(@as(i64, @bitCast(try self.readBigInt(u64)))) },0xd4 => return self.readExt(1),0xd5 => return self.readExt(2),0xd6 => return self.readExt(4),0xd7 => return self.readExt(8),0xd8 => return self.readExt(16),0xd9 => return .{ .str = try self.dupSlice(try self.readBigInt(u8)) },0xda => return .{ .str = try self.dupSlice(try self.readBigInt(u16)) },0xdb => return .{ .str = try self.dupSlice(try self.readBigInt(u32)) },0xdc => return self.readArray(try self.readBigInt(u16)),0xdd => return self.readArray(try self.readBigInt(u32)),0xde => return self.readMap(try self.readBigInt(u16)),0xdf => return self.readMap(try self.readBigInt(u32)),0xe0...0xff => return .{ .number = @floatFromInt(@as(i8, @bitCast(b))) },}}};/// Encode a Value in the canonical msgpackr form (see header comment).pub fn encode(alloc: std.mem.Allocator, value: Value) Error![]u8 {var buf: std.ArrayList(u8) = .empty;errdefer buf.deinit(alloc);try encodeInto(alloc, &buf, value);return buf.toOwnedSlice(alloc) catch Error.OutOfMemory;}pub fn encodeInto(alloc: std.mem.Allocator, buf: *std.ArrayList(u8), value: Value) Error!void {const w = Writer{ .alloc = alloc, .buf = buf };try w.writeValue(value);}const Writer = struct {alloc: std.mem.Allocator,buf: *std.ArrayList(u8),fn byte(self: Writer, b: u8) Error!void {self.buf.append(self.alloc, b) catch return Error.OutOfMemory;}fn raw(self: Writer, s: []const u8) Error!void {self.buf.appendSlice(self.alloc, s) catch return Error.OutOfMemory;}fn big(self: Writer, comptime T: type, v: T) Error!void {var tmp: [@divExact(@typeInfo(T).int.bits, 8)]u8 = undefined;std.mem.writeInt(T, &tmp, v, .big);try self.raw(&tmp);}fn writeNumber(self: Writer, v: f64) Error!void {// msgpackr rule (measured): integer-valued and within [-2^31, 2^32) →// smallest int family; else float64. -0 encodes as fixint 0.if (v == @floor(v) and v >= -2147483648.0 and v <= 4294967295.0) {if (v >= 0) {const u: u32 = @intFromFloat(v);if (u < 0x80) return self.byte(@intCast(u));if (u <= 0xff) {try self.byte(0xcc);return self.byte(@intCast(u));}if (u <= 0xffff) {try self.byte(0xcd);return self.big(u16, @intCast(u));}try self.byte(0xce);return self.big(u32, u);} else {const i: i32 = @intFromFloat(v);if (i >= -32) return self.byte(@bitCast(@as(i8, @intCast(i))));if (i >= -128) {try self.byte(0xd0);return self.byte(@bitCast(@as(i8, @intCast(i))));}if (i >= -32768) {try self.byte(0xd1);return self.big(u16, @bitCast(@as(i16, @intCast(i))));}try self.byte(0xd2);return self.big(u32, @bitCast(i));}}try self.byte(0xcb);return self.big(u64, @bitCast(v));}fn writeStr(self: Writer, s: []const u8) Error!void {if (s.len <= 31) {try self.byte(0xa0 | @as(u8, @intCast(s.len)));} else if (s.len <= 0xff) {try self.byte(0xd9);try self.byte(@intCast(s.len));} else if (s.len <= 0xffff) {try self.byte(0xda);try self.big(u16, @intCast(s.len));} else {try self.byte(0xdb);try self.big(u32, @intCast(s.len));}return self.raw(s);}fn writeValue(self: Writer, value: Value) Error!void {switch (value) {.nil => try self.byte(0xc0),.undef => try self.raw(&[_]u8{ 0xd4, 0x00, 0x00 }),.boolean => |b| try self.byte(if (b) 0xc3 else 0xc2),.number => |n| try self.writeNumber(n),.str => |s| try self.writeStr(s),.bin => |s| {if (s.len <= 0xff) {try self.byte(0xc4);try self.byte(@intCast(s.len));} else if (s.len <= 0xffff) {try self.byte(0xc5);try self.big(u16, @intCast(s.len));} else {try self.byte(0xc6);try self.big(u32, @intCast(s.len));}try self.raw(s);},.array => |items| {if (items.len <= 15) {try self.byte(0x90 | @as(u8, @intCast(items.len)));} else if (items.len <= 0xffff) {try self.byte(0xdc);try self.big(u16, @intCast(items.len));} else {try self.byte(0xdd);try self.big(u32, @intCast(items.len));}for (items) |it| try self.writeValue(it);},.map => |entries| {// msgpackr: objects are ALWAYS map16, >65535 keys throws.if (entries.len > 0xffff) return Error.MapTooLarge;try self.byte(0xde);try self.big(u16, @intCast(entries.len));for (entries) |e| {try self.writeValue(e.key);try self.writeValue(e.value);}},.ext => |e| {switch (e.data.len) {1 => try self.byte(0xd4),2 => try self.byte(0xd5),4 => try self.byte(0xd6),8 => try self.byte(0xd7),16 => try self.byte(0xd8),else => {if (e.data.len <= 0xff) {try self.byte(0xc7);try self.byte(@intCast(e.data.len));} else if (e.data.len <= 0xffff) {try self.byte(0xc8);try self.big(u16, @intCast(e.data.len));} else {try self.byte(0xc9);try self.big(u32, @intCast(e.data.len));}},}try self.byte(@bitCast(e.type));try self.raw(e.data);},}}};// =========================================================================// mpackdb record framing: [4-byte LE total size incl. prefix][msgpack]// =========================================================================/// Serialize a value with mpackdb's 4-byte little-endian length prefix.pub fn serialize(alloc: std.mem.Allocator, value: Value) Error![]u8 {var buf: std.ArrayList(u8) = .empty;errdefer buf.deinit(alloc);buf.appendSlice(alloc, &[_]u8{ 0, 0, 0, 0 }) catch return Error.OutOfMemory;try encodeInto(alloc, &buf, value);const total: u32 = @intCast(buf.items.len);std.mem.writeInt(u32, buf.items[0..4], total, .little);return buf.toOwnedSlice(alloc) catch Error.OutOfMemory;}/// Read the length prefix of a framed record (total size incl. the 4 bytes).pub fn readSize(bytes: []const u8) Error!u32 {if (bytes.len < 4) return Error.Truncated;return std.mem.readInt(u32, bytes[0..4], .little);}/// Deserialize a framed record ([4-byte LE size][msgpack]).pub fn deserializeFramed(alloc: std.mem.Allocator, bytes: []const u8) Error!Value {const total = try readSize(bytes);if (total < 5 or total > bytes.len) return Error.Truncated;const d = try decode(alloc, bytes[4..total]);return d.value;}// =========================================================================// Tests// =========================================================================const testing = std.testing;/// Linux-syscall file read (plugins avoid std.Io plumbing; same pattern as/// plugins/fs/fs.zig). Returns null when the file cannot be opened.fn readFileLinux(alloc: std.mem.Allocator, path: []const u8) ?[]u8 {const linux = std.os.linux;const path_z = alloc.dupeZ(u8, path) catch return null;defer alloc.free(path_z);const rc = linux.open(path_z, .{}, 0);const fd: i32 = @bitCast(@as(u32, @truncate(rc)));if (fd < 0) return null;defer _ = linux.close(fd);var content: std.ArrayList(u8) = .empty;var buf: [65536]u8 = undefined;while (true) {const n = linux.read(fd, &buf, buf.len);if (@as(isize, @bitCast(n)) <= 0) break;content.appendSlice(alloc, buf[0..n]) catch return null;}return content.toOwnedSlice(alloc) catch null;}test "corpus: decode + canonical re-encode is byte-identical (msgpackr 1.11.5)" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();const alloc = arena.allocator();// Corpus lives next to this file; zig build test runs with cwd native/.const candidates = [_][]const u8{"../plugins/mpackdb/testdata/corpus.json","plugins/mpackdb/testdata/corpus.json",};var content: ?[]u8 = null;for (candidates) |p| {content = readFileLinux(alloc, p) orelse continue;break;}const json = content orelse return error.CorpusNotFound;const parsed = try std.json.parseFromSlice(std.json.Value, alloc, json, .{});const arr = parsed.value.array;try testing.expect(arr.items.len >= 70);var checked: usize = 0;for (arr.items) |item| {const obj = item.object;const hex = obj.get("hex").?.string;const framed = if (obj.get("framed")) |f| f.bool else false;const bytes = try alloc.alloc(u8, hex.len / 2);_ = try std.fmt.hexToBytes(bytes, hex);if (framed) {// Frame round-trip: parse prefix, decode, re-serialize, compare all.const total = try readSize(bytes);try testing.expectEqual(bytes.len, @as(usize, total));const v = try deserializeFramed(alloc, bytes);const re = try serialize(alloc, v);try testing.expectEqualSlices(u8, bytes, re);} else {const d = try decode(alloc, bytes);try testing.expectEqual(bytes.len, d.len);const re = try encode(alloc, d.value);try testing.expectEqualSlices(u8, bytes, re);}checked += 1;}try testing.expect(checked == arr.items.len);}test "decode spot checks" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();const alloc = arena.allocator();// {name:'Alice', age:30} as msgpackr encodes it (map16)var bytes: [19]u8 = undefined;_ = try std.fmt.hexToBytes(&bytes, "de0002a46e616d65a5416c696365a36167651e");const d = try decode(alloc, &bytes);const name = d.value.get("name").?;try testing.expectEqualStrings("Alice", name.str);try testing.expectEqual(@as(f64, 30), d.value.get("age").?.number);}test "decode non-canonical families (full spec)" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();const alloc = arena.allocator();// fixmap {a:1}var b1: [4]u8 = undefined;_ = try std.fmt.hexToBytes(&b1, "81a16101");const v1 = try decode(alloc, &b1);try testing.expectEqual(@as(f64, 1), v1.value.get("a").?.number);// map32 {a:1}var b2: [8]u8 = undefined;_ = try std.fmt.hexToBytes(&b2, "df00000001a16101");const v2 = try decode(alloc, &b2);try testing.expectEqual(@as(f64, 1), v2.value.get("a").?.number);// int64 -1 (d3 ffffffffffffffff)var b3: [9]u8 = undefined;_ = try std.fmt.hexToBytes(&b3, "d3ffffffffffffffff");const v3 = try decode(alloc, &b3);try testing.expectEqual(@as(f64, -1), v3.value.number);// uint64 2^32 (cf 0000000100000000)var b4: [9]u8 = undefined;_ = try std.fmt.hexToBytes(&b4, "cf0000000100000000");const v4 = try decode(alloc, &b4);try testing.expectEqual(@as(f64, 4294967296), v4.value.number);// float32 0.5 (ca 3f000000)var b5: [5]u8 = undefined;_ = try std.fmt.hexToBytes(&b5, "ca3f000000");const v5 = try decode(alloc, &b5);try testing.expectEqual(@as(f64, 0.5), v5.value.number);// 0xc1 is invalidconst bad = [_]u8{0xc1};try testing.expectError(Error.InvalidFormat, decode(alloc, &bad));}test "undefined is distinct and byte-exact" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();const alloc = arena.allocator();const bytes = [_]u8{ 0xd4, 0x00, 0x00 };const d = try decode(alloc, &bytes);try testing.expect(d.value == .undef);const re = try encode(alloc, d.value);try testing.expectEqualSlices(u8, &bytes, re);}test "timestamp ext interpretation" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();const alloc = arena.allocator();// date whole sec: d6 ff 66a39486 → 1721996422000 msvar b: [6]u8 = undefined;_ = try std.fmt.hexToBytes(&b, "d6ff66a39486");const d = try decode(alloc, &b);try testing.expectEqual(@as(f64, 1721996422000), d.value.timestampMs().?);// date ms: d7 ff 1d53530066a39486 → 1721996422123 msvar b2: [10]u8 = undefined;_ = try std.fmt.hexToBytes(&b2, "d7ff1d53530066a39486");const d2 = try decode(alloc, &b2);try testing.expectApproxEqAbs(@as(f64, 1721996422123), d2.value.timestampMs().?, 0.001);}test "framing" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();const alloc = arena.allocator();var entries = [_]Entry{.{ .key = .{ .str = "id" }, .value = .{ .number = 1 } },};const v = Value{ .map = &entries };const framed = try serialize(alloc, v);try testing.expectEqual(@as(u32, @intCast(framed.len)), try readSize(framed));const back = try deserializeFramed(alloc, framed);try testing.expectEqual(@as(f64, 1), back.get("id").?.number);}test "encode number boundaries match msgpackr" {var arena = std.heap.ArenaAllocator.init(testing.allocator);defer arena.deinit();const alloc = arena.allocator();const cases = [_]struct { v: f64, hex: []const u8 }{.{ .v = 0, .hex = "00" },.{ .v = 127, .hex = "7f" },.{ .v = 128, .hex = "cc80" },.{ .v = 256, .hex = "cd0100" },.{ .v = 65536, .hex = "ce00010000" },.{ .v = 2147483648, .hex = "ce80000000" },.{ .v = 4294967296, .hex = "cb41f0000000000000" },.{ .v = -1, .hex = "ff" },.{ .v = -32, .hex = "e0" },.{ .v = -33, .hex = "d0df" },.{ .v = -129, .hex = "d1ff7f" },.{ .v = -32769, .hex = "d2ffff7fff" },.{ .v = -2147483648, .hex = "d280000000" },.{ .v = -2147483649, .hex = "cbc1e0000000200000" },.{ .v = 0.5, .hex = "cb3fe0000000000000" },};for (cases) |c| {const enc = try encode(alloc, Value{ .number = c.v });const expect_bytes = try alloc.alloc(u8, c.hex.len / 2);_ = try std.fmt.hexToBytes(expect_bytes, c.hex);try testing.expectEqualSlices(u8, expect_bytes, enc);}}
Branches
- mainmain branch
Latest commits
- f8bdcbc2ident: Hybriel master 06617221 (plugin allocators 3a781359 + 413f60e4, mpackdb 2cb7ae5e, http1 773de63e); all gates greenmre
- ff78726cident: Hybriel master 190aa11d (fc838894 GC correctness, #127, #126 closure scopes); gates all greenmre
- a3a7d21aident: Hybriel master 8efba065 (#126 GC by bytes, #48 lambda params copy); session-writing lambdas take &sessionmre
- 98226b41antcolony#40: mission references point to the moved missionsmre
- ff805b9aantcolony#40: history (LOG.md), worker briefs (missions/) and reports moved here from antcolony, numbered per project; old numbers in antcolony docs/mission-map.mdmre
- 51a7bcdfident: Hybriel master 73267707 (#122); /code uses the new page() signature; pending address passed as parameter; once-checksmre
- 836f644fident#24: installable app (manifest, service worker, data-free offline /start), own iconmre
- 8bebbbf2deploy.sh: back up live storage/.sessions/.env before every deploy (newest 5 kept)mre
- cc063ea2deploy.sh: never send .git or .gitignore to Byrodinmre
- 81b15b7bState of 2026-09-27, before the move to gitoriamre