micro-packed
Version:
Define complex binary structures using composable primitives
1,263 lines (1,262 loc) • 120 kB
JavaScript
import { hex as baseHex, utf8 } from '@scure/base';
/**
* Define complex binary structures using composable primitives.
* Main ideas:
* - Encode / decode can be chained, same as in `scure-base`
* - A complex structure can be created from an array and struct of primitive types
* - Strings / bytes are arrays with specific optimizations: we can just read bytes directly
* without creating plain array first and reading each byte separately.
* - Types are inferred from definition
* @module
* @example
* Define a struct with numbers, strings, bytes, and nested arrays.
* ```ts
* import * as P from 'micro-packed';
* const s = P.struct({
* field1: P.U32BE, // 32-bit unsigned big-endian integer
* field2: P.string(P.U8), // String with U8 length prefix
* field3: P.bytes(32), // 32 bytes
* field4: P.array(P.U16BE, P.struct({ // Array of structs with U16BE length
* subField1: P.U64BE, // 64-bit unsigned big-endian integer
* subField2: P.string(10) // 10-byte string
* }))
* });
* ```
*/
// TODO: remove dependency on scure-base & inline?
/*
Exports can be groupped like this:
- Primitive types: P.bytes, P.string, P.hex, P.constant, P.pointer
- Complex types: P.array, P.struct, P.tuple, P.map, P.tag, P.mappedTag
- Padding, prefix, magic: P.padLeft, P.padRight, P.prefix, P.magic, P.magicBytes
- Flags: P.flag, P.flagged, P.optional
- Wrappers: P.apply, P.wrap, P.lazy
- Bit fiddling: P.bits, P.bitset
- utils: P.validate, coders.decimal
- Debugger
*/
/**
* Shortcut to zero-length (empty) byte array.
* Keep public Bytes typing, not TRet<Bytes>, so variables inferred from this
* constant can later accept caller-owned Bytes backed by any ArrayBufferLike.
*/
export const EMPTY = /* @__PURE__ */ Uint8Array.of();
/**
* Shortcut to one-element (element is 0) byte array.
* Keep the same public Bytes typing rationale as EMPTY.
*/
export const NULL = /* @__PURE__ */ Uint8Array.of(0);
/** Prototype-sensitive names cannot roundtrip as normal fields on plain decoded objects. */
const restrictedKeys = /* @__PURE__ */ new Set(['__proto__', 'constructor', 'prototype']);
const validateFieldName = (name, label) => {
if (typeof name !== 'string')
throw new Error(`${label} should be string, got ${typeof name}`);
if (name.includes('..'))
throw new TypeError(`${label} ${name} cannot contain path parent ..`);
if (name.includes('/'))
throw new TypeError(`${label} ${name} cannot contain path separator /`);
if (restrictedKeys.has(name))
throw new Error(`${label} ${name} is reserved`);
};
/** Checks if two Uint8Arrays are equal. Not constant-time. */
function equalBytes(a, b) {
if (a.length !== b.length)
return false;
for (let i = 0; i < a.length; i++)
if (a[i] !== b[i])
return false;
return true;
}
function createFindBytes(needle) {
if (needle.length === 1) {
const byte = needle[0];
return (data, pos = 0) => {
const idx = data.indexOf(byte, pos);
return idx === -1 ? undefined : idx;
};
}
// KMP avoids quadratic scans on repeated-prefix terminators.
const back = new Uint32Array(needle.length);
for (let i = 1, j = 0; i < needle.length; i++) {
while (j && needle[i] !== needle[j])
j = back[j - 1];
if (needle[i] === needle[j])
back[i] = ++j;
}
return (data, pos = 0) => {
for (let i = pos, j = 0; i < data.length; i++) {
while (j && data[i] !== needle[j])
j = back[j - 1];
if (data[i] !== needle[j])
continue;
if (++j === needle.length)
return i - needle.length + 1;
}
return undefined;
};
}
const findBytes = (needle, data, pos = 0) => createFindBytes(needle)(data, pos);
/** Compares values used as encoded-domain constants; byte arrays compare by contents. */
function equal(a, b) {
const aBytes = isBytes(a);
const bBytes = isBytes(b);
if (aBytes || bBytes)
return aBytes && bBytes && equalBytes(a, b);
return a === b;
}
/** Checks if the given value is a Uint8Array. */
function isBytes(a) {
// Plain `instanceof Uint8Array` is too strict for some Buffer / proxy / cross-realm cases. The
// fallback still requires a real ArrayBuffer view, so plain JSON-deserialized
// `{ constructor: ... }` spoofing is rejected. `BYTES_PER_ELEMENT === 1` keeps
// the fallback on byte-oriented views.
return (a instanceof Uint8Array ||
(ArrayBuffer.isView(a) &&
a.constructor.name === 'Uint8Array' &&
'BYTES_PER_ELEMENT' in a &&
a.BYTES_PER_ELEMENT === 1));
}
/**
* Concatenates multiple Uint8Arrays.
* Engines limit functions to 65K+ arguments.
* @param arrays Array of Uint8Array elements
* @returns Concatenated Uint8Array
*/
function concatBytes(...arrays) {
let sum = 0;
for (let i = 0; i < arrays.length; i++) {
const a = arrays[i];
if (!isBytes(a))
throw new Error('Uint8Array expected');
sum += a.length;
}
const res = new Uint8Array(sum);
for (let i = 0, pad = 0; i < arrays.length; i++) {
const a = arrays[i];
res.set(a, pad);
pad += a.length;
}
return res;
}
/**
* Creates DataView from Uint8Array
* @param arr - bytes
* @returns DataView
*/
const createView = (arr) => new DataView(arr.buffer, arr.byteOffset, arr.byteLength);
const _0n = /* @__PURE__ */ BigInt(0);
const _1n = /* @__PURE__ */ BigInt(1);
const _2n = /* @__PURE__ */ BigInt(2);
const _8n = /* @__PURE__ */ BigInt(8);
const _10n = /* @__PURE__ */ BigInt(10);
const _255n = /* @__PURE__ */ BigInt(255);
/**
* Checks if the provided value is object-like for option/schema bags.
* This intentionally matches noble-curves and noble-hashes by using the
* `[object Object]` tag instead of rejecting class/proxy/env objects by prototype;
* stricter checks caused compatibility reports in proxied environments.
* Array, Uint8Array and others are not plain objects.
* @param obj - The value to be checked.
*/
function isPlainObject(obj) {
return Object.prototype.toString.call(obj) === '[object Object]';
}
function isNum(num) {
return Number.isSafeInteger(num);
}
const hasOwn = (obj, key) => Object.prototype.hasOwnProperty.call(obj, key);
/**
* Miscellaneous helpers reused by the coder internals and tests.
* @example
* Reuse a couple of byte helpers without pulling in the full namespace.
* ```ts
* import { utils } from 'micro-packed';
* const left = Uint8Array.of(1);
* const right = Uint8Array.of(2);
* utils.equalBytes(utils.concatBytes(left, right), Uint8Array.of(1, 2));
* ```
*/
export const utils = /* @__PURE__ */ Object.freeze({
equalBytes,
isBytes,
isCoder,
checkBounds,
concatBytes,
createView,
isPlainObject,
});
// NOTE: we can't have terminator separate function, since it won't know about boundaries
// E.g. array of U16LE ([1,2,3]) would be [1, 0, 2, 0, 3, 0]
// But terminator will find array at index '1', which happens to be inside of an element itself
/**
* Can be:
* - Dynamic (CoderType)
* - Fixed (number)
* - Terminated (usually zero): Uint8Array with terminator
* - Field path to field with length (string)
* - Infinity (null) - decodes until end of buffer
* Used in:
* - bytes (string, prefix is implementation of bytes)
* - array
*/
const lengthCoder = (len) => {
if (len !== null && typeof len !== 'string' && !isCoder(len) && !isBytes(len) && !isNum(len)) {
// Constructor argument validation uses TypeError.
// Stream/data failures keep contextual Error paths.
throw new TypeError(`lengthCoder: expected null | number | Uint8Array | CoderType, got ${len} (${typeof len})`);
}
if (typeof len === 'number' && len < 0)
throw new Error(`lengthCoder: wrong length=${len}`);
if (isBytes(len) && !len.length)
throw new Error('lengthCoder: empty terminator');
return {
encodeStream(w, value) {
if (len === null)
return;
if (isCoder(len))
return len.encodeStream(w, value);
let byteLen;
if (typeof len === 'number')
byteLen = len;
else if (typeof len === 'string')
byteLen = Path.resolve(w.stack, len);
if (typeof byteLen === 'bigint')
byteLen = Number(byteLen);
if (!isNum(byteLen) || byteLen < 0 || byteLen !== value)
throw w.err(`Wrong length: ${byteLen} len=${len} exp=${value} (${typeof value})`);
},
decodeStream(r) {
let byteLen;
if (isCoder(len))
byteLen = len.decodeStream(r);
else if (typeof len === 'number')
byteLen = len;
else if (typeof len === 'string')
byteLen = Path.resolve(r.stack, len);
if (typeof byteLen === 'bigint')
byteLen = Number(byteLen);
else if (typeof byteLen !== 'number')
throw r.err(`Wrong length: ${byteLen}`);
if (!isNum(byteLen) || byteLen < 0)
throw r.err(`Wrong length: ${byteLen}`);
return byteLen;
},
};
};
/**
* Small bitset structure to store position of ranges that have been read.
* Can be more efficient when internal trees are utilized at the cost of complexity.
* Needs `O(N/8)` memory for parsing.
* Purpose: if there are pointers in parsed structure,
* they can cause read of two distinct ranges:
* [0-32, 64-128], which means 'pos' is not enough to handle them
*/
const Bitset = /* @__PURE__ */ Object.freeze({
BITS: 32,
FULL_MASK: -1 >>> 0, // 1<<32 will overflow
len: (len) => {
if (!isNum(len) || len < 0)
throw new Error(`wrong len=${len}`);
return Math.ceil(len / 32);
},
create: (len) => new Uint32Array(Bitset.len(len)),
clean: (bs) => bs.fill(0),
debug: (bs) => Array.from(bs).map((i) => (i >>> 0).toString(2).padStart(32, '0')),
checkLen: (bs, len) => {
if (Bitset.len(len) === bs.length)
return;
throw new Error(`wrong length=${bs.length}. Expected: ${Bitset.len(len)}`);
},
chunkLen: (bsLen, pos, len) => {
if (!isNum(bsLen) || bsLen < 0)
throw new Error(`wrong bsLen=${bsLen}`);
if (!isNum(pos) || pos < 0)
throw new Error(`wrong pos=${pos}`);
if (!isNum(len) || len < 0)
throw new Error(`wrong len=${len}`);
if (pos > bsLen - len)
throw new Error(`wrong range=${pos}/${len} of ${bsLen}`);
},
set: (bs, chunk, value, allowRewrite = true) => {
if (!isNum(chunk) || chunk < 0 || chunk >= bs.length)
return false;
if (!allowRewrite && (bs[chunk] & value) !== 0)
return false;
bs[chunk] |= value;
return true;
},
pos: (pos, i) => ({
chunk: Math.floor((pos + i) / 32),
mask: 1 << (32 - ((pos + i) % 32) - 1),
}),
indices: (bs, len, invert = false) => {
Bitset.checkLen(bs, len);
const { FULL_MASK, BITS } = Bitset;
const left = BITS - (len % BITS);
const lastMask = left ? (FULL_MASK >>> left) << left : FULL_MASK;
const res = [];
for (let i = 0; i < bs.length; i++) {
let c = bs[i];
if (invert)
c = ~c; // allows to gen unset elements
// apply mask to last element, so we won't iterate non-existent items
if (i === bs.length - 1)
c &= lastMask;
if (c === 0)
continue; // fast-path
for (let j = 0; j < BITS; j++) {
const m = 1 << (BITS - j - 1);
if (c & m)
res.push(i * BITS + j);
}
}
return res;
},
range: (arr) => {
// Bitset.indices() returns sorted unique positions; this helper only merges adjacent runs.
const res = [];
let cur;
for (const i of arr) {
if (cur === undefined || i !== cur.pos + cur.length)
res.push((cur = { pos: i, length: 1 }));
else
cur.length += 1;
}
return res;
},
rangeDebug: (bs, len, invert = false) => `[${Bitset.range(Bitset.indices(bs, len, invert))
.map((i) => `(${i.pos}/${i.length})`)
.join(', ')}]`,
setRange: (bs, bsLen, pos, len, allowRewrite = true) => {
Bitset.chunkLen(bsLen, pos, len);
// Empty ranges are valid reader-bookkeeping no-ops; mask math below assumes at least one bit.
if (len === 0)
return true;
const { FULL_MASK, BITS } = Bitset;
// Try to set range with maximum efficiency:
// - first chunk is always '0000[1111]' (only right ones)
// - middle chunks are set to '[1111 1111]' (all ones)
// - last chunk is always '[1111]0000' (only left ones)
// - max operations: (N/32) + 2 (first and last)
const first = pos % BITS ? Math.floor(pos / BITS) : undefined;
const lastPos = pos + len;
const last = lastPos % BITS ? Math.floor(lastPos / BITS) : undefined;
const canSet = (chunk, value) => chunk >= 0 && chunk < bs.length && (bs[chunk] & value) === 0;
if (!allowRewrite) {
// Check the whole range before writing so a late overlap cannot leave earlier chunks mutated.
if (first !== undefined && first === last) {
if (!canSet(first, (FULL_MASK >>> (BITS - len)) << (BITS - len - pos)))
return false;
}
else {
if (first !== undefined && !canSet(first, FULL_MASK >>> pos % BITS))
return false;
const start = first !== undefined ? first + 1 : pos / BITS;
const end = last !== undefined ? last : lastPos / BITS;
for (let i = start; i < end; i++)
if (!canSet(i, FULL_MASK))
return false;
if (last !== undefined && first !== last)
if (!canSet(last, FULL_MASK << (BITS - (lastPos % BITS))))
return false;
}
}
// special case, whole range inside single chunk
if (first !== undefined && first === last)
return Bitset.set(bs, first, (FULL_MASK >>> (BITS - len)) << (BITS - len - pos), allowRewrite);
if (first !== undefined) {
// first chunk
if (!Bitset.set(bs, first, FULL_MASK >>> pos % BITS, allowRewrite))
return false;
}
// middle chunks
const start = first !== undefined ? first + 1 : pos / BITS;
const end = last !== undefined ? last : lastPos / BITS;
for (let i = start; i < end; i++)
if (!Bitset.set(bs, i, FULL_MASK, allowRewrite))
return false;
if (last !== undefined && first !== last)
if (!Bitset.set(bs, last, FULL_MASK << (BITS - (lastPos % BITS)), allowRewrite))
// last chunk
return false;
return true;
},
});
const Path = /* @__PURE__ */ Object.freeze({
/**
* Internal method for handling stack of paths (debug, errors, dynamic fields via path)
* `.pop()` always happens after the wrapped function.
* Fields inside the object are tracked via Reader/Writer enterField()/exitField(),
* which the debugger overrides to observe per-field byte ranges.
* NOTE: we don't want to do '.pop' on error!
*/
pushObj: (stack, obj, objFn) => {
stack.push({ obj });
objFn();
stack.pop();
},
path: (stack) => {
const res = [];
for (const i of stack)
if (i.field !== undefined)
res.push(i.field === '' ? '""' : `${i.field}`);
// Path.err() uses this string for user-visible context. Empty keys need explicit rendering so
// field("") is distinguishable from the root path; slash-containing keys are still raw.
return res.join('/');
},
err: (name, stack, msg) => {
const text = `${name}(${Path.path(stack)}): ${typeof msg === 'string' ? msg : msg.message}`;
// Path context is the primary diagnostic. Do not attach `cause`: Node inspection expands nested
// cause stacks and makes the original, path-prefixed failure harder to scan.
// Keep specific validation classes after adding the path prefix. Otherwise public coder
// APIs flatten inner TypeError / RangeError guards back to plain Error.
const err = msg instanceof TypeError
? new TypeError(text)
: msg instanceof RangeError
? new RangeError(text)
: new Error(text);
if (msg instanceof Error && msg.stack) {
const from = `${msg.name}: ${msg.message}`;
const to = `${err.name}: ${err.message}`;
err.stack = msg.stack.startsWith(from) ? `${to}${msg.stack.slice(from.length)}` : msg.stack;
}
return err;
},
resolve: (stack, path) => {
const parts = path.split('/');
// Leading '..' segments mean parent traversal and '/' separates nested fields, so literal
// keys using those tokens are not addressable through this helper.
const objPath = stack.map((i) => i.obj);
let i = 0;
for (; i < parts.length; i++) {
if (parts[i] === '..')
objPath.pop();
else
break;
}
let cur = objPath.pop();
for (; i < parts.length; i++) {
if (!cur || cur[parts[i]] === undefined)
return undefined;
cur = cur[parts[i]];
}
return cur;
},
});
/**
* Internal structure. Reader class for reading from a byte array.
* `stack` is internal: for debugger and logging
* @class Reader
*/
class _Reader {
pos = 0;
data;
opts;
stack;
parent;
parentOffset;
bitBuf = 0;
bitPos = 0;
bs; // bitset
view;
constructor(data, opts = {}, stack = [], parent = undefined, parentOffset = 0) {
if (!isBytes(data))
throw new TypeError(`Reader: expected Uint8Array, got ${typeof data}`);
if (!isPlainObject(opts))
throw new TypeError(`ReaderOpts: expected plain object, got ${opts}`);
if (opts.allowUnreadBytes !== undefined && typeof opts.allowUnreadBytes !== 'boolean')
throw new TypeError(`ReaderOpts.allowUnreadBytes: expected boolean, got ${typeof opts.allowUnreadBytes}`);
if (opts.allowMultipleReads !== undefined && typeof opts.allowMultipleReads !== 'boolean')
throw new TypeError(`ReaderOpts.allowMultipleReads: expected boolean, got ${typeof opts.allowMultipleReads}`);
this.data = data;
this.opts = opts;
this.stack = stack;
this.parent = parent;
this.parentOffset = parentOffset;
this.view = createView(data);
}
/** Internal method for pointers. */
_enablePointers() {
// Pointer decoding enables tracking before the pointed child reader starts consuming bytes, so
// only the root reader owns the bitset and seeds it from the already-consumed prefix.
if (this.parent)
return this.parent._enablePointers();
if (this.bs)
return;
this.bs = Bitset.create(this.data.length);
Bitset.setRange(this.bs, this.data.length, 0, this.pos, this.opts.allowMultipleReads);
}
markBytesBS(pos, len) {
if (this.parent)
return this.parent.markBytesBS(this.parentOffset + pos, len);
if (!len)
return true;
// Before pointers are enabled there is no bitset yet, so linear cursor checks remain the only
// guard; overlap tracking starts only after _enablePointers() allocates the root bitset.
if (!this.bs)
return true;
return Bitset.setRange(this.bs, this.data.length, pos, len, false);
}
markBytes(len) {
const pos = this.pos;
const res = this.markBytesBS(pos, len);
// Keep failed overlap reads at their start so diagnostics point at the repeated span.
if (!this.opts.allowMultipleReads && !res)
throw this.err(`multiple read pos=${pos} len=${len}`);
this.pos += len;
return res;
}
pushObj(obj, objFn) {
return Path.pushObj(this.stack, obj, objFn);
}
enterField(field) {
const last = this.stack[this.stack.length - 1];
// A field outside any pushObj() scope or nested inside another field is a coder bug.
if (last === undefined || last.field !== undefined)
throw this.err(`enterField: invalid stack state, field=${field}`);
last.field = field;
}
// Intentionally not called on throw, so Path.err() can report the failing leaf.
exitField() {
this.stack[this.stack.length - 1].field = undefined;
}
readView(n, fn) {
if (!isNum(n) || n < 0)
throw this.err(`readView: wrong length=${n}`);
if (this.pos + n > this.data.length)
throw this.err('readView: Unexpected end of buffer');
const res = fn(this.view, this.pos);
this.markBytes(n);
return res;
}
// read bytes by absolute offset
absBytes(n) {
if (!isNum(n) || n < 0 || n > this.data.length)
throw new Error('Unexpected end of buffer');
return this.data.subarray(n);
}
finish() {
// ReaderOpts documents allowUnreadBytes as "Allow decoding to finish with unread trailing
// bytes." Prefix decoders may intentionally parse only a value from a larger byte array, so
// this skips all final unread-input checks, including residual bits and pointer-read gaps.
if (this.opts.allowUnreadBytes)
return;
if (this.bitPos) {
throw this.err(`${this.bitPos} bits left after unpack: ${baseHex.encode(this.data.subarray(this.pos))}`);
}
if (this.bs && !this.parent) {
const notRead = Bitset.indices(this.bs, this.data.length, true);
if (notRead.length) {
const formatted = Bitset.range(notRead)
.map(({ pos, length }) => `(${pos}/${length})[${baseHex.encode(this.data.subarray(pos, pos + length))}]`)
.join(', ');
throw this.err(`unread byte ranges: ${formatted} (total=${this.data.length})`);
}
else
return; // all bytes read, everything is ok
}
// Default: no pointers enabled
if (!this.isEnd()) {
throw this.err(`${this.leftBytes} bytes ${this.bitPos} bits left after unpack: ${baseHex.encode(this.data.subarray(this.pos))}`);
}
}
// User methods
err(msg) {
return Path.err('Reader', this.stack, msg);
}
offsetReader(n) {
if (!isNum(n) || n < 0 || n > this.data.length)
throw this.err('offsetReader: Unexpected end of buffer');
return new _Reader(this.absBytes(n), this.opts, this.stack, this, n);
}
bytes(n, peek = false) {
if (this.bitPos)
throw this.err('readBytes: bitPos not empty');
if (!isNum(n) || n < 0)
throw this.err(`readBytes: wrong length=${n}`);
if (this.pos + n > this.data.length)
throw this.err('readBytes: Unexpected end of buffer');
const slice = this.data.subarray(this.pos, this.pos + n);
if (!peek)
this.markBytes(n);
return slice;
}
byte(peek = false) {
if (this.bitPos)
throw this.err('readByte: bitPos not empty');
if (this.pos + 1 > this.data.length)
throw this.err('readByte: Unexpected end of buffer');
const data = this.data[this.pos];
if (!peek)
this.markBytes(1);
return data;
}
get leftBytes() {
return this.data.length - this.pos;
}
get totalBytes() {
return this.data.length;
}
isEnd() {
return this.pos >= this.data.length && !this.bitPos;
}
progress() {
return this.pos * 8 - this.bitPos;
}
// bits are read in BE mode (left to right): (0b1000_0000).readBits(1) == 1
bits(bits) {
// Reject before bitwise shifts: JS coerces negative/fractional shift counts into other widths.
if (!isNum(bits) || bits < 0)
throw this.err(`BitReader: wrong length=${bits}`);
if (bits > 32)
throw this.err('BitReader: cannot read more than 32 bits in single call');
let out = 0;
while (bits) {
if (!this.bitPos) {
this.bitBuf = this.byte();
this.bitPos = 8;
}
const take = Math.min(bits, this.bitPos);
this.bitPos -= take;
out = (out << take) | ((this.bitBuf >> this.bitPos) & (2 ** take - 1));
this.bitBuf &= 2 ** this.bitPos - 1;
bits -= take;
}
// Fix signed integers
return out >>> 0;
}
find(needle, pos = this.pos) {
if (!isBytes(needle))
throw this.err(`find: needle is not bytes! ${needle}`);
if (this.bitPos)
throw this.err('find: bitPos not empty');
if (!needle.length)
throw this.err(`find: needle is empty`);
if (!isNum(pos) || pos < 0)
throw this.err(`find: wrong pos=${pos}`);
return findBytes(needle, this.data, pos);
}
}
class _Writer {
pos = 0;
stack;
// Small writes are carved out of writer-owned chunks; caller-provided bytes() buffers are kept
// by reference between them. A single realloc'd buffer was measured slower for tiny encodes
// (basic/encode bench: 395ns -> 560ns), copy-on-grow is what chunking avoids.
buffers = [];
// Every chunk ever allocated by this writer, so finish(clean) can zeroize them whole.
chunks = [];
chunk;
chunkView; // lazy: only writeView() needs it
chunkPos = 0;
run;
nextChunkSize = 0; // 0 = size the first chunk to the first write
ptrs = [];
bitBuf = 0;
bitPos = 0;
finished = false;
constructor(stack = []) {
this.stack = stack;
}
// Reserves `len` contiguous writer-owned bytes and returns their offset inside `this.chunk`.
carve(len) {
if (this.chunk === undefined || this.chunk.length - this.chunkPos < len) {
// Chunks grow geometrically to amortize allocations, capped to bound over-allocation and
// zeroization work. The 64-byte floor keeps small encodes in one chunk (one lazy DataView).
const size = Math.max(len, this.nextChunkSize, 64);
this.nextChunkSize = Math.min(size * 8, 4096);
this.chunk = new Uint8Array(size);
this.chunkView = undefined;
this.chunkPos = 0;
this.run = undefined;
this.chunks.push(this.chunk);
}
const pos = this.chunkPos;
if (this.run === undefined) {
this.run = { chunk: this.chunk, start: pos, end: pos + len };
this.buffers.push(this.run);
}
else
this.run.end += len;
this.chunkPos = pos + len;
this.pos += len;
return pos;
}
pushObj(obj, objFn) {
return Path.pushObj(this.stack, obj, objFn);
}
enterField(field) {
const last = this.stack[this.stack.length - 1];
// A field outside any pushObj() scope or nested inside another field is a coder bug.
if (last === undefined || last.field !== undefined)
throw this.err(`enterField: invalid stack state, field=${field}`);
last.field = field;
}
// Intentionally not called on throw, so Path.err() can report the failing leaf.
exitField() {
this.stack[this.stack.length - 1].field = undefined;
}
writeView(len, fn) {
if (this.finished)
throw this.err('buffer: finished');
if (!isNum(len) || len < 0 || len > 8)
throw new Error(`wrong writeView length=${len}`);
if (this.bitPos)
throw this.err('writeBytes: ends with non-empty bit buffer');
const pos = this.carve(len);
if (this.chunkView === undefined)
this.chunkView = createView(this.chunk);
fn(this.chunkView, pos);
}
// User methods
err(msg) {
// Finished-state guards call err('buffer: finished'), so err itself must not recurse there.
return Path.err('Writer', this.stack, msg);
}
bytes(b) {
if (this.finished)
throw this.err('buffer: finished');
if (this.bitPos)
throw this.err('writeBytes: ends with non-empty bit buffer');
if (!isBytes(b))
throw this.err(`writeBytes: expected Uint8Array, got ${typeof b}`);
// Keep caller-provided buffers by reference until finish(); mutating them afterwards changes
// the encoded output.
this.buffers.push(b);
// Later carved writes must not merge into a run that would be emitted before `b`.
this.run = undefined;
this.pos += b.length;
}
byte(b) {
if (this.finished)
throw this.err('buffer: finished');
if (this.bitPos)
throw this.err('writeByte: ends with non-empty bit buffer');
if (!isNum(b) || b < 0 || b > 255)
throw this.err(`writeByte: wrong value=${b}`);
// carve() first: it may allocate/replace this.chunk.
const pos = this.carve(1);
this.chunk[pos] = b;
}
finish(clean = true) {
if (this.finished)
throw this.err('buffer: finished');
if (this.bitPos)
throw this.err('buffer: ends with non-empty bit buffer');
// Can't use concatBytes, because it limits amount of arguments (65K).
const buffers = this.buffers;
let sum = 0;
for (let i = 0; i < buffers.length; i++) {
const b = buffers[i];
sum += isBytes(b) ? b.length : b.end - b.start;
}
for (let i = 0; i < this.ptrs.length; i++)
sum += this.ptrs[i].buffer.length;
const buf = new Uint8Array(sum);
let pad = 0;
for (let i = 0; i < buffers.length; i++) {
const b = buffers[i];
if (isBytes(b)) {
buf.set(b, pad);
pad += b.length;
}
else {
buf.set(b.chunk.subarray(b.start, b.end), pad);
pad += b.end - b.start;
}
}
for (let i = 0; i < this.ptrs.length; i++) {
buf.set(this.ptrs[i].buffer, pad);
pad += this.ptrs[i].buffer.length;
}
for (let pos = this.pos, i = 0; i < this.ptrs.length; i++) {
const ptr = this.ptrs[i];
buf.set(ptr.ptr.encode(pos), ptr.pos);
pos += ptr.buffer.length;
}
// Cleanup
if (clean) {
// bytes() keeps caller-provided buffers by reference, so only writer-owned chunks are
// zeroized — whole, including unused tails: same hygiene as per-buffer cleanup.
for (const c of this.chunks)
c.fill(0);
this.chunks = [];
this.chunk = undefined;
this.chunkView = undefined;
this.chunkPos = 0;
this.run = undefined;
this.buffers = [];
for (const p of this.ptrs)
p.buffer.fill(0);
this.ptrs = [];
this.finished = true;
this.bitBuf = 0;
}
return buf;
}
bits(value, bits) {
if (this.finished)
throw this.err('buffer: finished');
// Reject before bitwise shifts: JS coerces negative/fractional values and widths.
if (!isNum(bits) || bits < 0)
throw this.err(`writeBits: wrong length=${bits}`);
if (bits > 32)
throw this.err('writeBits: cannot write more than 32 bits in single call');
if (!isNum(value) || value < 0)
throw this.err(`writeBits: wrong value=${value}`);
if (value >= 2 ** bits)
throw this.err(`writeBits: value (${value}) >= 2**bits (${bits})`);
while (bits) {
const take = Math.min(bits, 8 - this.bitPos);
this.bitBuf = (this.bitBuf << take) | (value >> (bits - take));
this.bitPos += take;
bits -= take;
value &= 2 ** bits - 1;
if (this.bitPos === 8) {
this.bitPos = 0;
// carve() first: it may allocate/replace this.chunk. The Uint8Array store wraps mod 256,
// matching the previous `new Uint8Array([bitBuf])` semantics for int32-coerced values.
const pos = this.carve(1);
this.chunk[pos] = this.bitBuf;
}
}
}
}
// Immutable LE<->BE
const swapEndianness = (b) => Uint8Array.from(b).reverse();
function checkMinimalBigintBytes(value, le, signed, err) {
if (!value.length)
return;
const b = le ? swapEndianness(value) : value;
const fail = () => {
throw err('bigint: non-minimal encoding');
};
if (!signed) {
if (b[0] === 0)
fail();
return;
}
// Encoders emit the shortest signed two's-complement byte string that preserves the sign bit.
if (b[0] === 0 && (b.length === 1 || (b[1] & 128) === 0))
fail();
if (b.length > 1 && b[0] === 255 && (b[1] & 128) !== 0)
fail();
}
/** Internal function for checking bit bounds of bigint in signed/unsinged form */
function checkBounds(value, bits, signed) {
if (signed) {
if (bits <= _0n)
throw new Error(`checkBounds: signed bits must be positive, got ${bits}`);
// [-(2**(32-1)), 2**(32-1)-1]
const signBit = _2n ** (bits - _1n);
if (value < -signBit || value >= signBit)
throw new Error(`value out of signed bounds. Expected ${-signBit} <= ${value} < ${signBit}`);
}
else {
// [0, 2**32-1]
const max = _2n ** bits;
if (_0n > value || value >= max)
throw new Error(`value out of unsigned bounds. Expected 0 <= ${value} < ${max}`);
}
}
function _wrap(inner) {
const _inner = inner;
return {
// NOTE: we cannot export validate here, since it is likely mistake.
// Raw inner throws propagate unchanged; path-aware errors must use w.err/r.err or validate().
encodeStream: _inner.encodeStream,
decodeStream: _inner.decodeStream,
size: _inner.size,
encode: (value) => {
const w = new _Writer();
_inner.encodeStream(w, value);
return w.finish();
},
decode: (data, opts = {}) => {
if (!isBytes(data))
throw new TypeError(`decode: expected Uint8Array, got ${typeof data}`);
const r = new _Reader(data, opts);
const res = _inner.decodeStream(r);
r.finish();
return res;
},
};
}
/**
* Validates a value before encoding and after decoding using a provided function.
* @param inner - The inner CoderType.
* @param fn - The validation function.
* @returns CoderType which check value with validation function.
* @throws On wrong inner coder or validator argument types. {@link TypeError}
* @example
* Reject values outside the accepted range during both encode and decode.
* ```ts
* import * as P from 'micro-packed';
* const val = (n: number) => {
* if (n > 10) throw new Error(`${n} > 10`);
* return n;
* };
*
* // Checks that values are <= 10 during encoding and decoding.
* const RangedInt = P.validate(P.U32LE, val);
* ```
*/
export function validate(inner, fn) {
if (!isCoder(inner))
throw new TypeError(`validate: invalid inner value ${inner}`);
if (typeof fn !== 'function')
throw new TypeError('validate: fn should be function');
return _wrap({
size: inner.size,
encodeStream: (w, value) => {
let res;
try {
res = fn(value);
}
catch (e) {
// Validator callbacks are caller code: if they throw non-Error garbage, diagnostics are on
// them. Review policy: "if they throw garbage, then it is on them".
throw w.err(e);
}
inner.encodeStream(w, res);
},
decodeStream: (r) => {
const res = inner.decodeStream(r);
try {
return fn(res);
}
catch (e) {
throw r.err(e);
}
},
});
}
/**
* Wraps a stream encoder into a generic encoder and optionally validation function
* @param inner - Stream coder with optional validation hook.
* @returns The wrapped CoderType.
* @throws On wrong wrapped stream-coder shapes. {@link TypeError}
* @example
* Start from stream methods, then add validation if needed.
* ```ts
* import * as P from 'micro-packed';
* const U8 = P.wrap({
* encodeStream: (w, value) => w.byte(value),
* decodeStream: (r) => r.byte(),
* });
* const checkedU8 = P.wrap({
* encodeStream: (w, value) => w.byte(value),
* decodeStream: (r) => r.byte(),
* validate: (n: number) => {
* if (n > 10) throw new Error(`${n} > 10`);
* return n;
* }
* });
* ```
*/
// Keep this as a plain contextual object type instead of TArg<>/TRet<> helpers:
// recursive object mapping breaks unannotated encodeStream/decodeStream parameters,
// and _wrap() already returns the CoderType<T> shape without byte-array normalization.
export const wrap = (inner) => {
const _inner = inner;
// Public wrap() is the boundary for raw stream coders; reject malformed shapes before a
// half-constructed coder fails later during encode/decode.
if (!isPlainObject(_inner))
throw new TypeError(`wrap: invalid inner value ${_inner}`);
if (typeof _inner.encodeStream !== 'function')
throw new TypeError('wrap: encodeStream should be function');
if (typeof _inner.decodeStream !== 'function')
throw new TypeError('wrap: decodeStream should be function');
if (_inner.size !== undefined && (!isNum(_inner.size) || _inner.size < 0))
throw new TypeError(`wrap: invalid size ${_inner.size}`);
if (_inner.validate !== undefined && typeof _inner.validate !== 'function')
throw new TypeError('wrap: validate should be function');
const res = _wrap(_inner);
return _inner.validate !== undefined ? validate(res, _inner.validate) : res;
};
const isBaseCoder = (elm) => isPlainObject(elm) && typeof elm.decode === 'function' && typeof elm.encode === 'function';
/**
* Checks if the given value is a CoderType.
* @param elm - The value to check.
* @returns True if the value is a CoderType, false otherwise.
* @example
* Guard unknown values before calling encode/decode helpers on them.
* ```ts
* import { isCoder, U8 } from 'micro-packed';
* isCoder(U8);
* ```
*/
export function isCoder(elm) {
return (isPlainObject(elm) &&
isBaseCoder(elm) &&
typeof elm.encodeStream === 'function' &&
typeof elm.decodeStream === 'function' &&
(elm.size === undefined || (isNum(elm.size) && elm.size >= 0)));
}
// Coders (like in @scure/base) for common operations
/**
* Base coder for working with dictionaries (records, objects, key-value map)
* Dictionary is dynamic type like: `[key: string, value: any][]`
* @returns base coder that encodes/decodes between arrays of key-value tuples and dictionaries.
* @example
* Convert between tuple entries and a plain object record.
* ```ts
* import * as P from 'micro-packed';
* const dict: P.CoderType<Record<string, number>> = P.apply(
* P.array(P.U16BE, P.tuple([P.cstring, P.U32LE] as const)),
* P.coders.dict()
* );
* ```
*/
function dict() {
return {
encode: (from) => {
if (!Array.isArray(from))
throw new Error('array expected');
const to = {};
const seen = new Set();
for (const item of from) {
if (!Array.isArray(item) || item.length !== 2)
throw new Error(`array of two elements expected`);
const name = item[0];
const value = item[1];
validateFieldName(name, 'dict: key');
// Stored undefined is still a present key, so duplicate detection cannot inspect values.
if (seen.has(name))
throw new Error(`key(${name}) appears twice in struct`);
seen.add(name);
to[name] = value;
}
return to;
},
decode: (to) => {
if (!isPlainObject(to))
throw new Error(`expected plain object, got ${to}`);
for (const name in to)
validateFieldName(name, 'dict: key');
return Object.entries(to);
},
};
}
/**
* Safely converts bigint to number.
* Sometimes pointers / tags use u64 or other big numbers which cannot be represented by number,
* but we still can use them since real value will be smaller than u32
*/
const numberBigint = /* @__PURE__ */ Object.freeze({
encode: (from) => {
if (typeof from !== 'bigint')
throw new Error(`expected bigint, got ${typeof from}`);
if (from > BigInt(Number.MAX_SAFE_INTEGER))
throw new Error(`element bigger than MAX_SAFE_INTEGER=${from}`);
// Number() silently rounds bigint values outside the safe integer range on either side.
if (from < BigInt(Number.MIN_SAFE_INTEGER))
throw new Error(`element smaller than MIN_SAFE_INTEGER=${from}`);
return Number(from);
},
decode: (to) => {
if (!isNum(to))
throw new Error('element is not a safe integer');
return BigInt(to);
},
});
/**
* Base coder for working with TypeScript enums.
* @param e - TypeScript enum.
* @returns base coder that encodes/decodes between numbers and enum keys.
* @example
* Map enum numbers to their string keys and back.
* ```ts
* import * as P from 'micro-packed';
* enum Color { Red, Green, Blue }
* const colorCoder = P.coders.tsEnum(Color);
* colorCoder.encode(Color.Red); // 'Red'
* colorCoder.decode('Green'); // 1
* ```
*/
function tsEnum(e) {
if (!isPlainObject(e))
throw new Error('plain object expected');
return {
encode: (from) => {
if (!isNum(from) || !(from in e))
throw new Error(`wrong value ${from}`);
return e[from];
},
decode: (to) => {
if (typeof to !== 'string')
throw new Error(`wrong value ${typeof to}`);
const value = e[to];
// TypeScript numeric enums include reverse-map keys like "0"; decode accepts names only.
if (!hasOwn(e, to) || !isNum(value))
throw new Error(`wrong value ${to}`);
return value;
},
};
}
/**
* Base coder for working with decimal numbers.
* @param precision - Number of decimal places.
* @param round - Round fraction part if bigger than precision (throws error by default)
* @returns base coder that encodes/decodes between bigints and decimal strings.
* @example
* Convert bigint amounts into fixed-precision decimal strings.
* ```ts
* import * as P from 'micro-packed';
* const decimal8 = P.coders.decimal(8);
* decimal8.encode(630880845n); // '6.30880845'
* decimal8.decode('6.30880845'); // 630880845n
* ```
*/
function decimal(precision, round = false) {
if (!isNum(precision) || precision < 0)
throw new Error(`decimal/precision: wrong value ${precision}`);
if (typeof round !== 'boolean')
throw new Error(`decimal/round: expected boolean, got ${typeof round}`);
const decimalMask = _10n ** BigInt(precision);
return {
encode: (from) => {
if (typeof from !== 'bigint')
throw new Error(`expected bigint, got ${typeof from}`);
let s = (from < _0n ? -from : from).toString(10);
let sep = s.length - precision;
if (sep < 0) {
s = s.padStart(s.length - sep, '0');
sep = 0;
}
let i = s.length - 1;
for (; i >= sep && s[i] === '0'; i--)
;
let int = s.slice(0, sep);
let frac = s.slice(sep, i + 1);
if (!int)
int = '0';
if (from < _0n)
int = '-' + int;
if (!frac)
return int;
return `${int}.${frac}`;
},
decode: (to) => {
if (typeof to !== 'string')
throw new Error(`expected string, got ${typeof to}`);
let neg = false;
if (to.startsWith('-')) {
neg = true;
to = to.slice(1);
}
if (!/^(0|[1-9]\d*)(\.\d+)?$/.test(to))
throw new Error(`wrong string value=${to}`);
let sep = to.indexOf('.');
sep = sep === -1 ? to.length : sep;
// Split by separator and strip trailing zeros from fraction.
// Always returns [string, string]; .split() doesn't.
const intS = to.slice(0, sep);
const fracS = to.slice(sep + 1).replace(/0+$/, '');
const int = BigInt(intS) * decimalMask;
if (!round && fracS.length > precision) {
throw new Error(`fractional part cannot be represented with this precision (num=${to}, prec=${precision})`);
}
const fracLen = Math.min(fracS.length, precision);
const frac = BigInt(fracS.slice(0, fracLen)) * _10n ** BigInt(precision - fracLen);
const value = int + frac;
// All negative zero spellings collapse to 0n, so reject after parsing.
if (neg && value === _0n)
throw new Error(`negative zero is not allowed`);
return neg ? -value : value;
},
};
}
/**
* Combines multiple coders into a single coder, allowing conditional
* encoding/decoding based on input.
* Acts as a parser combinator, splitting complex conditional coders into smaller parts.
*
* `encode = [Ae, Be]; decode = [Ad, Bd]`
* ->
* `match([{encode: Ae, decode: Ad}, {encode: Be; decode: Bd}])`
*
* @param lst - Array of coders to match.
* @returns Combined coder for conditional encoding/decoding.
*/
function match(lst) {
if (!Array.isArray(lst))
throw new Error(`expected array, got ${typeof lst}`);
for (const i of lst)
if (!isBaseCoder(i))
throw new Error(`wrong base coder ${i}`);
return {
encode: (from) => {
for (const c of lst) {
let elm;
try {
elm = c.encode(from);
}
catch {
// match() is a branch selector: coders may signal "not this branch" by throwing.
continue;
}
if (elm !== undefined)
return elm;
}
throw new Error(`match/encode: cannot find match in ${from}`);
},
decode: (to) => {
for (const c of lst) {
let elm;
try {
elm = c.decode(to);
}
catch {
// match() is a branch selector: coders may signal "not this branch" by throwing.
continue;
}
if (elm !== undefined)
return elm;
}
throw new Error(`match/decode: cannot find match in ${to}`);
},
};
}
/** Reverses direction of coder */
const reverse = (coder) => {
if (!isBaseCoder(coder))
throw new Error('BaseCoder expected');
// Call through the source coder so method-style encode/decode implementations keep their receiver.
return { encode: (to) => coder.decode(to), decode: (from) => coder.encode(from) };
};
/**
* Collection of reusable base coders and helpers.
* @example
* Build a reusable decimal-string adapter.
* ```ts
* import { coders } from 'micro-packed';
* const decimal2 = coders.decimal(2);
* decimal2.encode(123n); // '1.23'
* ```
*/
export const coders = /* @__PURE__ */ Object.freeze({ dict, numberBigint, tsEnum, decimal, match, reverse });
/**
* CoderType for parsing individual bits.
* NOTE: Structure should parse whole amount of bytes before it can start parsing byte-level elements.
* @param len - Number of bits to parse.
* @returns CoderType representing the parsed bits.
* @throws On invalid bit-length configuration or bit values. {@link Error}
* @throws On wrong argument types forwarded into wrapped numeric validators. {@link TypeError}
* @example
* Pack several bit fields into a single byte.
* ```ts
* import * as P from 'micro-packed';
* const s = P.struct({ magic: P.bits(1), version: P.bits(1), tag: P.bits(4), len: P.bits(2) });
* ```
*/
export const bits = (len) => {
// Reader/Writer bit helpers operate on one 0..32-bit chunk; reject impossible coders up front.
if (!isNum(len) || len < 0 || len > 32)
throw new Error(`bits: wrong length ${len} (${typeof len})`);
return wrap({
encodeStream: (w, value) => w.bits(value, len),
decodeStream: (r) => r.bits(len),
validate: (value) => {
if (!isNum(value))
throw new Error(`bits: wrong va