micro-packed
Version:
Define complex binary structures using composable primitives
1,253 lines (1,220 loc) • 129 kB
text/typescript
import type { TArg, TRet } from '@scure/base';
import { hex as baseHex, utf8, type Coder as BaseCoder } from '@scure/base';
export type { TArg, TRet } from '@scure/base';
/**
* Define complex binary structures using composable primitives.
* Main ideas:
* - Encode / decode can be chained, same as in `scure-base`
* - A complex structure can be created from an array and struct of primitive types
* - Strings / bytes are arrays with specific optimizations: we can just read bytes directly
* without creating plain array first and reading each byte separately.
* - Types are inferred from definition
* @module
* @example
* Define a struct with numbers, strings, bytes, and nested arrays.
* ```ts
* import * as P from 'micro-packed';
* const s = P.struct({
* field1: P.U32BE, // 32-bit unsigned big-endian integer
* field2: P.string(P.U8), // String with U8 length prefix
* field3: P.bytes(32), // 32 bytes
* field4: P.array(P.U16BE, P.struct({ // Array of structs with U16BE length
* subField1: P.U64BE, // 64-bit unsigned big-endian integer
* subField2: P.string(10) // 10-byte string
* }))
* });
* ```
*/
// TODO: remove dependency on scure-base & inline?
/*
Exports can be groupped like this:
- Primitive types: P.bytes, P.string, P.hex, P.constant, P.pointer
- Complex types: P.array, P.struct, P.tuple, P.map, P.tag, P.mappedTag
- Padding, prefix, magic: P.padLeft, P.padRight, P.prefix, P.magic, P.magicBytes
- Flags: P.flag, P.flagged, P.optional
- Wrappers: P.apply, P.wrap, P.lazy
- Bit fiddling: P.bits, P.bitset
- utils: P.validate, coders.decimal
- Debugger
*/
/**
* Shortcut to zero-length (empty) byte array.
* Keep public Bytes typing, not TRet<Bytes>, so variables inferred from this
* constant can later accept caller-owned Bytes backed by any ArrayBufferLike.
*/
export const EMPTY: Bytes = /* @__PURE__ */ Uint8Array.of();
/**
* Shortcut to one-element (element is 0) byte array.
* Keep the same public Bytes typing rationale as EMPTY.
*/
export const NULL: Bytes = /* @__PURE__ */ Uint8Array.of(0);
/** Prototype-sensitive names cannot roundtrip as normal fields on plain decoded objects. */
const restrictedKeys = /* @__PURE__ */ new Set(['__proto__', 'constructor', 'prototype']);
const validateFieldName = (name: unknown, label: string): void => {
if (typeof name !== 'string') throw new Error(`${label} should be string, got ${typeof name}`);
if (name.includes('..')) throw new TypeError(`${label} ${name} cannot contain path parent ..`);
if (name.includes('/')) throw new TypeError(`${label} ${name} cannot contain path separator /`);
if (restrictedKeys.has(name)) throw new Error(`${label} ${name} is reserved`);
};
/** Checks if two Uint8Arrays are equal. Not constant-time. */
function equalBytes(a: TArg<Uint8Array>, b: TArg<Uint8Array>): boolean {
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
return true;
}
type BytesFinder = (data: TArg<Uint8Array>, pos?: number) => number | undefined;
function createFindBytes(needle: TArg<Uint8Array>): TRet<BytesFinder> {
if (needle.length === 1) {
const byte = needle[0];
return (data, pos = 0) => {
const idx = data.indexOf(byte, pos);
return idx === -1 ? undefined : idx;
};
}
// KMP avoids quadratic scans on repeated-prefix terminators.
const back = new Uint32Array(needle.length);
for (let i = 1, j = 0; i < needle.length; i++) {
while (j && needle[i] !== needle[j]) j = back[j - 1];
if (needle[i] === needle[j]) back[i] = ++j;
}
return (data, pos = 0) => {
for (let i = pos, j = 0; i < data.length; i++) {
while (j && data[i] !== needle[j]) j = back[j - 1];
if (data[i] !== needle[j]) continue;
if (++j === needle.length) return i - needle.length + 1;
}
return undefined;
};
}
const findBytes = (needle: TArg<Uint8Array>, data: TArg<Uint8Array>, pos = 0): number | undefined =>
createFindBytes(needle)(data, pos);
/** Compares values used as encoded-domain constants; byte arrays compare by contents. */
function equal(a: unknown, b: unknown): boolean {
const aBytes = isBytes(a);
const bBytes = isBytes(b);
if (aBytes || bBytes) return aBytes && bBytes && equalBytes(a, b);
return a === b;
}
/** Checks if the given value is a Uint8Array. */
function isBytes(a: unknown): a is Bytes {
// Plain `instanceof Uint8Array` is too strict for some Buffer / proxy / cross-realm cases. The
// fallback still requires a real ArrayBuffer view, so plain JSON-deserialized
// `{ constructor: ... }` spoofing is rejected. `BYTES_PER_ELEMENT === 1` keeps
// the fallback on byte-oriented views.
return (
a instanceof Uint8Array ||
(ArrayBuffer.isView(a) &&
a.constructor.name === 'Uint8Array' &&
'BYTES_PER_ELEMENT' in a &&
a.BYTES_PER_ELEMENT === 1)
);
}
/**
* Concatenates multiple Uint8Arrays.
* Engines limit functions to 65K+ arguments.
* @param arrays Array of Uint8Array elements
* @returns Concatenated Uint8Array
*/
function concatBytes(...arrays: TArg<Uint8Array[]>): TRet<Uint8Array> {
let sum = 0;
for (let i = 0; i < arrays.length; i++) {
const a = arrays[i];
if (!isBytes(a)) throw new Error('Uint8Array expected');
sum += a.length;
}
const res = new Uint8Array(sum);
for (let i = 0, pad = 0; i < arrays.length; i++) {
const a = arrays[i];
res.set(a, pad);
pad += a.length;
}
return res;
}
/**
* Creates DataView from Uint8Array
* @param arr - bytes
* @returns DataView
*/
const createView = (arr: TArg<Uint8Array>) =>
new DataView(arr.buffer, arr.byteOffset, arr.byteLength);
const _0n = /* @__PURE__ */ BigInt(0);
const _1n = /* @__PURE__ */ BigInt(1);
const _2n = /* @__PURE__ */ BigInt(2);
const _8n = /* @__PURE__ */ BigInt(8);
const _10n = /* @__PURE__ */ BigInt(10);
const _255n = /* @__PURE__ */ BigInt(255);
/**
* Checks if the provided value is object-like for option/schema bags.
* This intentionally matches noble-curves and noble-hashes by using the
* `[object Object]` tag instead of rejecting class/proxy/env objects by prototype;
* stricter checks caused compatibility reports in proxied environments.
* Array, Uint8Array and others are not plain objects.
* @param obj - The value to be checked.
*/
function isPlainObject(obj: any): boolean {
return Object.prototype.toString.call(obj) === '[object Object]';
}
function isNum(num: unknown): num is number {
return Number.isSafeInteger(num);
}
const hasOwn = (obj: object, key: PropertyKey): boolean =>
Object.prototype.hasOwnProperty.call(obj, key);
/**
* Miscellaneous helpers reused by the coder internals and tests.
* @example
* Reuse a couple of byte helpers without pulling in the full namespace.
* ```ts
* import { utils } from 'micro-packed';
* const left = Uint8Array.of(1);
* const right = Uint8Array.of(2);
* utils.equalBytes(utils.concatBytes(left, right), Uint8Array.of(1, 2));
* ```
*/
export const utils: TRet<{
equalBytes: typeof equalBytes;
isBytes: typeof isBytes;
isCoder: typeof isCoder;
checkBounds: typeof checkBounds;
concatBytes: typeof concatBytes;
createView: (arr: TArg<Uint8Array>) => DataView;
isPlainObject: typeof isPlainObject;
}> = /* @__PURE__ */ Object.freeze({
equalBytes,
isBytes,
isCoder,
checkBounds,
concatBytes,
createView,
isPlainObject,
});
// Types
/** Byte-array alias used throughout the public API. */
export type Bytes = Uint8Array;
/** Optional value helper used by conditional coders. */
export type Option<T> = T | undefined;
/** Coder encodes and decodes between two types. */
export interface Coder<F, T> {
/**
* Encodes (converts) a decoded value into its serialized representation.
* @param from - Value to encode.
* @returns Encoded representation.
*/
encode(from: F): T;
/**
* Decodes (converts) a serialized value back into its decoded representation.
* @param to - Encoded representation to decode.
* @returns Decoded value.
*/
decode(to: T): F;
}
/** BytesCoder converts value between a type and a byte array. */
export interface BytesCoder<T> extends Coder<T, Bytes> {
/** Fixed-size hint in bytes, when known. */
size?: number;
/**
* Encodes a value into a byte array.
* @param data - Value to encode.
* @returns Encoded bytes.
*/
encode: (data: T) => Bytes;
/**
* Decodes a byte array into a value.
* @param data - Bytes to decode.
* @param opts - Reader options used while decoding. See {@link ReaderOpts}.
* @returns Decoded value.
*/
decode: (data: Bytes, opts?: ReaderOpts) => T;
}
/** BytesCoderStream converts value between a type and a byte array, using streams. */
export interface BytesCoderStream<T> {
/** Fixed-size hint in bytes, when known. */
size?: number;
/**
* Encodes a value into a Writer stream.
* @param w - Writer stream.
* @param value - Value to encode.
*/
encodeStream: (w: Writer, value: T) => void;
/**
* Decodes a value from a Reader stream.
* @param r - Reader stream.
* @returns Decoded value.
*/
decodeStream: (r: Reader) => T;
}
/** Full coder interface with both stream and byte-array helpers. */
export type CoderType<T> = BytesCoderStream<T> & BytesCoder<T>;
/** CoderType with a known fixed byte size. */
export type Sized<T> = CoderType<T> & { size: number };
/** Extract the decoded value type from a coder. */
export type UnwrapCoder<T> = T extends CoderType<infer U> ? U : T;
/**
* Validation function. Should return value after validation.
* Can be used to narrow types
*/
export type Validate<T> = (elm: T) => T;
/** Length descriptor accepted by variable-size coders. */
export type Length = CoderType<number> | CoderType<bigint> | number | Bytes | string | null;
// NOTE: we can't have terminator separate function, since it won't know about boundaries
// E.g. array of U16LE ([1,2,3]) would be [1, 0, 2, 0, 3, 0]
// But terminator will find array at index '1', which happens to be inside of an element itself
/**
* Can be:
* - Dynamic (CoderType)
* - Fixed (number)
* - Terminated (usually zero): Uint8Array with terminator
* - Field path to field with length (string)
* - Infinity (null) - decodes until end of buffer
* Used in:
* - bytes (string, prefix is implementation of bytes)
* - array
*/
const lengthCoder = (len: Length) => {
if (len !== null && typeof len !== 'string' && !isCoder(len) && !isBytes(len) && !isNum(len)) {
// Constructor argument validation uses TypeError.
// Stream/data failures keep contextual Error paths.
throw new TypeError(
`lengthCoder: expected null | number | Uint8Array | CoderType, got ${len} (${typeof len})`
);
}
if (typeof len === 'number' && len < 0) throw new Error(`lengthCoder: wrong length=${len}`);
if (isBytes(len) && !len.length) throw new Error('lengthCoder: empty terminator');
return {
encodeStream(w: TArg<Writer>, value: number | null) {
if (len === null) return;
if (isCoder(len)) return len.encodeStream(w, value);
let byteLen;
if (typeof len === 'number') byteLen = len;
else if (typeof len === 'string') byteLen = Path.resolve((w as _Writer).stack, len);
if (typeof byteLen === 'bigint') byteLen = Number(byteLen);
if (!isNum(byteLen) || byteLen < 0 || byteLen !== value)
throw w.err(`Wrong length: ${byteLen} len=${len} exp=${value} (${typeof value})`);
},
decodeStream(r: TArg<Reader>) {
let byteLen;
if (isCoder(len)) byteLen = len.decodeStream(r);
else if (typeof len === 'number') byteLen = len;
else if (typeof len === 'string') byteLen = Path.resolve((r as _Reader).stack, len);
if (typeof byteLen === 'bigint') byteLen = Number(byteLen);
else if (typeof byteLen !== 'number') throw r.err(`Wrong length: ${byteLen}`);
if (!isNum(byteLen) || byteLen < 0) throw r.err(`Wrong length: ${byteLen}`);
return byteLen;
},
};
};
type ArrLike<T> = Array<T> | ReadonlyArray<T>;
// prettier-ignore
/** Typed arrays supported by the utility helper types. */
export type TypedArray =
| Uint8Array | Int8Array | Uint8ClampedArray
| Uint16Array | Int16Array
| Uint32Array | Int32Array;
/** Writable version of a type, where readonly properties are made writable. */
export type Writable<T> = T extends {}
? T extends TypedArray
? T
: {
-readonly [P in keyof T]: Writable<T[P]>;
}
: T;
/** Union of object value types. */
export type Values<T> = T[keyof T];
/** Key helper that removes fields whose values are exactly `undefined`. */
export type NonUndefinedKey<T, K extends keyof T> = T[K] extends undefined ? never : K;
/** Key helper that keeps only nullable fields. */
export type NullableKey<T, K extends keyof T> = T[K] extends NonNullable<T[K]> ? never : K;
// Opt: value !== undefined, but value === T|undefined
/** Key helper for optional-but-present struct fields. */
export type OptKey<T, K extends keyof T> = NullableKey<T, K> & NonUndefinedKey<T, K>;
/** Key helper for required struct fields. */
export type ReqKey<T, K extends keyof T> = T[K] extends NonNullable<T[K]> ? K : never;
/** Object containing only optional keys from a struct shape. */
export type OptKeys<T> = Pick<T, { [K in keyof T]: OptKey<T, K> }[keyof T]>;
/** Object containing only required keys from a struct shape. */
export type ReqKeys<T> = Pick<T, { [K in keyof T]: ReqKey<T, K> }[keyof T]>;
/** Input object type accepted by `struct()`. */
export type StructInput<T extends Record<string, any>> = { [P in keyof ReqKeys<T>]: T[P] } & {
[P in keyof OptKeys<T>]?: T[P];
};
/** Record of field names to coder instances for `struct()`. */
export type StructRecord<T extends Record<string, any>> = {
[P in keyof T]: CoderType<T[P]>;
};
/** Generic decoded object bag used internally by nested coders. */
export type StructOut = Record<string, any>;
/** Padding function that takes an index and returns a padding value. */
export type PadFn = (i: number) => number;
/**
* Small bitset structure to store position of ranges that have been read.
* Can be more efficient when internal trees are utilized at the cost of complexity.
* Needs `O(N/8)` memory for parsing.
* Purpose: if there are pointers in parsed structure,
* they can cause read of two distinct ranges:
* [0-32, 64-128], which means 'pos' is not enough to handle them
*/
const Bitset = /* @__PURE__ */ Object.freeze({
BITS: 32,
FULL_MASK: -1 >>> 0, // 1<<32 will overflow
len: (len: number) => {
if (!isNum(len) || len < 0) throw new Error(`wrong len=${len}`);
return Math.ceil(len / 32);
},
create: (len: number) => new Uint32Array(Bitset.len(len)),
clean: (bs: TArg<Uint32Array>) => bs.fill(0),
debug: (bs: TArg<Uint32Array>) =>
Array.from(bs).map((i) => (i >>> 0).toString(2).padStart(32, '0')),
checkLen: (bs: TArg<Uint32Array>, len: number) => {
if (Bitset.len(len) === bs.length) return;
throw new Error(`wrong length=${bs.length}. Expected: ${Bitset.len(len)}`);
},
chunkLen: (bsLen: number, pos: number, len: number) => {
if (!isNum(bsLen) || bsLen < 0) throw new Error(`wrong bsLen=${bsLen}`);
if (!isNum(pos) || pos < 0) throw new Error(`wrong pos=${pos}`);
if (!isNum(len) || len < 0) throw new Error(`wrong len=${len}`);
if (pos > bsLen - len) throw new Error(`wrong range=${pos}/${len} of ${bsLen}`);
},
set: (bs: TArg<Uint32Array>, chunk: number, value: number, allowRewrite = true) => {
if (!isNum(chunk) || chunk < 0 || chunk >= bs.length) return false;
if (!allowRewrite && (bs[chunk] & value) !== 0) return false;
bs[chunk] |= value;
return true;
},
pos: (pos: number, i: number) => ({
chunk: Math.floor((pos + i) / 32),
mask: 1 << (32 - ((pos + i) % 32) - 1),
}),
indices: (bs: TArg<Uint32Array>, len: number, invert = false) => {
Bitset.checkLen(bs, len);
const { FULL_MASK, BITS } = Bitset;
const left = BITS - (len % BITS);
const lastMask = left ? (FULL_MASK >>> left) << left : FULL_MASK;
const res = [];
for (let i = 0; i < bs.length; i++) {
let c = bs[i];
if (invert) c = ~c; // allows to gen unset elements
// apply mask to last element, so we won't iterate non-existent items
if (i === bs.length - 1) c &= lastMask;
if (c === 0) continue; // fast-path
for (let j = 0; j < BITS; j++) {
const m = 1 << (BITS - j - 1);
if (c & m) res.push(i * BITS + j);
}
}
return res;
},
range: (arr: number[]) => {
// Bitset.indices() returns sorted unique positions; this helper only merges adjacent runs.
const res = [];
let cur;
for (const i of arr) {
if (cur === undefined || i !== cur.pos + cur.length) res.push((cur = { pos: i, length: 1 }));
else cur.length += 1;
}
return res;
},
rangeDebug: (bs: TArg<Uint32Array>, len: number, invert = false) =>
`[${Bitset.range(Bitset.indices(bs, len, invert))
.map((i) => `(${i.pos}/${i.length})`)
.join(', ')}]`,
setRange: (
bs: TArg<Uint32Array>,
bsLen: number,
pos: number,
len: number,
allowRewrite = true
) => {
Bitset.chunkLen(bsLen, pos, len);
// Empty ranges are valid reader-bookkeeping no-ops; mask math below assumes at least one bit.
if (len === 0) return true;
const { FULL_MASK, BITS } = Bitset;
// Try to set range with maximum efficiency:
// - first chunk is always '0000[1111]' (only right ones)
// - middle chunks are set to '[1111 1111]' (all ones)
// - last chunk is always '[1111]0000' (only left ones)
// - max operations: (N/32) + 2 (first and last)
const first = pos % BITS ? Math.floor(pos / BITS) : undefined;
const lastPos = pos + len;
const last = lastPos % BITS ? Math.floor(lastPos / BITS) : undefined;
const canSet = (chunk: number, value: number) =>
chunk >= 0 && chunk < bs.length && (bs[chunk] & value) === 0;
if (!allowRewrite) {
// Check the whole range before writing so a late overlap cannot leave earlier chunks mutated.
if (first !== undefined && first === last) {
if (!canSet(first, (FULL_MASK >>> (BITS - len)) << (BITS - len - pos))) return false;
} else {
if (first !== undefined && !canSet(first, FULL_MASK >>> pos % BITS)) return false;
const start = first !== undefined ? first + 1 : pos / BITS;
const end = last !== undefined ? last : lastPos / BITS;
for (let i = start; i < end; i++) if (!canSet(i, FULL_MASK)) return false;
if (last !== undefined && first !== last)
if (!canSet(last, FULL_MASK << (BITS - (lastPos % BITS)))) return false;
}
}
// special case, whole range inside single chunk
if (first !== undefined && first === last)
return Bitset.set(
bs,
first,
(FULL_MASK >>> (BITS - len)) << (BITS - len - pos),
allowRewrite
);
if (first !== undefined) {
// first chunk
if (!Bitset.set(bs, first, FULL_MASK >>> pos % BITS, allowRewrite)) return false;
}
// middle chunks
const start = first !== undefined ? first + 1 : pos / BITS;
const end = last !== undefined ? last : lastPos / BITS;
for (let i = start; i < end; i++) if (!Bitset.set(bs, i, FULL_MASK, allowRewrite)) return false;
if (last !== undefined && first !== last)
if (!Bitset.set(bs, last, FULL_MASK << (BITS - (lastPos % BITS)), allowRewrite))
// last chunk
return false;
return true;
},
});
/** Path related utils (internal) */
// `field` stays a number for array/tuple indices and is stringified lazily in path():
// hot decode/encode loops must not allocate a string per element just for diagnostics.
type Path = { obj: StructOut; field?: string | number };
type PathStack = Path[];
export type _PathObjFn = () => void;
type PathUtils = {
pushObj: (stack: PathStack, obj: StructOut, objFn: _PathObjFn) => void;
path: (stack: PathStack) => string;
err: (name: string, stack: PathStack, msg: string | Error) => Error;
resolve: (stack: PathStack, path: string) => StructOut | undefined;
};
const Path: PathUtils = /* @__PURE__ */ Object.freeze({
/**
* Internal method for handling stack of paths (debug, errors, dynamic fields via path)
* `.pop()` always happens after the wrapped function.
* Fields inside the object are tracked via Reader/Writer enterField()/exitField(),
* which the debugger overrides to observe per-field byte ranges.
* NOTE: we don't want to do '.pop' on error!
*/
pushObj: (stack: PathStack, obj: StructOut, objFn: _PathObjFn): void => {
stack.push({ obj });
objFn();
stack.pop();
},
path: (stack: PathStack): string => {
const res = [];
for (const i of stack)
if (i.field !== undefined) res.push(i.field === '' ? '""' : `${i.field}`);
// Path.err() uses this string for user-visible context. Empty keys need explicit rendering so
// field("") is distinguishable from the root path; slash-containing keys are still raw.
return res.join('/');
},
err: (name: string, stack: PathStack, msg: string | Error): Error => {
const text = `${name}(${Path.path(stack)}): ${typeof msg === 'string' ? msg : msg.message}`;
// Path context is the primary diagnostic. Do not attach `cause`: Node inspection expands nested
// cause stacks and makes the original, path-prefixed failure harder to scan.
// Keep specific validation classes after adding the path prefix. Otherwise public coder
// APIs flatten inner TypeError / RangeError guards back to plain Error.
const err =
msg instanceof TypeError
? new TypeError(text)
: msg instanceof RangeError
? new RangeError(text)
: new Error(text);
if (msg instanceof Error && msg.stack) {
const from = `${msg.name}: ${msg.message}`;
const to = `${err.name}: ${err.message}`;
err.stack = msg.stack.startsWith(from) ? `${to}${msg.stack.slice(from.length)}` : msg.stack;
}
return err;
},
resolve: (stack: PathStack, path: string): StructOut | undefined => {
const parts = path.split('/');
// Leading '..' segments mean parent traversal and '/' separates nested fields, so literal
// keys using those tokens are not addressable through this helper.
const objPath = stack.map((i) => i.obj);
let i = 0;
for (; i < parts.length; i++) {
if (parts[i] === '..') objPath.pop();
else break;
}
let cur = objPath.pop();
for (; i < parts.length; i++) {
if (!cur || cur[parts[i]] === undefined) return undefined;
cur = cur[parts[i]];
}
return cur;
},
});
/** Options for the Reader class. */
export type ReaderOpts = {
/** Allow decoding to finish with unread trailing bytes. */
allowUnreadBytes?: boolean;
/** Allow the same byte range to be read more than once through pointers. */
allowMultipleReads?: boolean;
};
// These are safe API for external usage
/** Reader interface passed into stream decoders. */
export type Reader = {
// Utils
/** Current position in the buffer. */
readonly pos: number;
/** Number of bytes left in the buffer. */
readonly leftBytes: number;
/** Total number of bytes in the buffer. */
readonly totalBytes: number;
/**
* Checks if the end of the buffer has been reached.
* @returns `true` when the reader consumed the whole buffer.
*/
isEnd(): boolean;
/**
* Creates an error with the given message. Adds information about current field path.
* If Error object provided, saves original stack trace.
* @param msg - The error message or an Error object.
* @returns The created Error object.
*/
err(msg: string | Error): Error;
/**
* Reads a specified number of bytes from the buffer.
*
* WARNING: Uint8Array is subarray of original buffer. Do not modify.
* @param n - The number of bytes to read.
* @param peek - If `true`, the bytes are read without advancing the position.
* @returns The read bytes as a Uint8Array.
*/
bytes(n: number, peek?: boolean): Uint8Array;
/**
* Reads a single byte from the buffer.
* @param peek - If `true`, the byte is read without advancing the position.
* @returns The read byte as a number.
*/
byte(peek?: boolean): number;
/**
* Reads a specified number of bits from the buffer.
* @param bits - The number of bits to read.
* @returns The read bits as a number.
*/
bits(bits: number): number;
/**
* Finds the first occurrence of a needle in the buffer.
* @param needle - The needle to search for.
* @param pos - The starting position for the search.
* @returns The position of the first occurrence of the needle, or `undefined` if not found.
*/
find(needle: Bytes, pos?: number): number | undefined;
/**
* Creates a new Reader instance at the specified offset.
* Complex and unsafe API: currently only used in eth ABI parsing of pointers.
* Required to break pointer boundaries inside arrays for complex structure.
* Please use only if absolutely necessary!
* @param n - The offset to create the new Reader at.
* @returns A new Reader instance at the specified offset.
*/
offsetReader(n: number): Reader;
};
/** Writer interface passed into stream encoders. */
export type Writer = {
/**
* Creates an error with the given message. Adds information about current field path.
* If Error object provided, saves original stack trace.
* @param msg - The error message or an Error object.
* @returns The created Error object.
*/
err(msg: string | Error): Error;
/**
* Writes a byte array to the buffer.
* @param b - The byte array to write.
*/
bytes(b: Bytes): void;
/**
* Writes a single byte to the buffer.
* @param b - The byte to write.
*/
byte(b: number): void;
/**
* Writes a specified number of bits to the buffer.
* @param value - The value to write.
* @param bits - The number of bits to write.
*/
bits(value: number, bits: number): void;
};
/**
* Internal structure. Reader class for reading from a byte array.
* `stack` is internal: for debugger and logging
* @class Reader
*/
class _Reader implements Reader {
pos = 0;
readonly data: Bytes;
readonly opts: ReaderOpts;
readonly stack: PathStack;
private parent: _Reader | undefined;
private parentOffset: number;
private bitBuf = 0;
private bitPos = 0;
private bs: Uint32Array | undefined; // bitset
private view: DataView;
constructor(
data: Bytes,
opts: ReaderOpts = {},
stack: PathStack = [],
parent: _Reader | undefined = undefined,
parentOffset: number = 0
) {
if (!isBytes(data)) throw new TypeError(`Reader: expected Uint8Array, got ${typeof data}`);
if (!isPlainObject(opts)) throw new TypeError(`ReaderOpts: expected plain object, got ${opts}`);
if (opts.allowUnreadBytes !== undefined && typeof opts.allowUnreadBytes !== 'boolean')
throw new TypeError(
`ReaderOpts.allowUnreadBytes: expected boolean, got ${typeof opts.allowUnreadBytes}`
);
if (opts.allowMultipleReads !== undefined && typeof opts.allowMultipleReads !== 'boolean')
throw new TypeError(
`ReaderOpts.allowMultipleReads: expected boolean, got ${typeof opts.allowMultipleReads}`
);
this.data = data;
this.opts = opts;
this.stack = stack;
this.parent = parent;
this.parentOffset = parentOffset;
this.view = createView(data);
}
/** Internal method for pointers. */
_enablePointers(): void {
// Pointer decoding enables tracking before the pointed child reader starts consuming bytes, so
// only the root reader owns the bitset and seeds it from the already-consumed prefix.
if (this.parent) return this.parent._enablePointers();
if (this.bs) return;
this.bs = Bitset.create(this.data.length);
Bitset.setRange(this.bs, this.data.length, 0, this.pos, this.opts.allowMultipleReads);
}
private markBytesBS(pos: number, len: number): boolean {
if (this.parent) return this.parent.markBytesBS(this.parentOffset + pos, len);
if (!len) return true;
// Before pointers are enabled there is no bitset yet, so linear cursor checks remain the only
// guard; overlap tracking starts only after _enablePointers() allocates the root bitset.
if (!this.bs) return true;
return Bitset.setRange(this.bs, this.data.length, pos, len, false);
}
private markBytes(len: number): boolean {
const pos = this.pos;
const res = this.markBytesBS(pos, len);
// Keep failed overlap reads at their start so diagnostics point at the repeated span.
if (!this.opts.allowMultipleReads && !res)
throw this.err(`multiple read pos=${pos} len=${len}`);
this.pos += len;
return res;
}
pushObj(obj: StructOut, objFn: _PathObjFn): void {
return Path.pushObj(this.stack, obj, objFn);
}
enterField(field: string | number): void {
const last = this.stack[this.stack.length - 1];
// A field outside any pushObj() scope or nested inside another field is a coder bug.
if (last === undefined || last.field !== undefined)
throw this.err(`enterField: invalid stack state, field=${field}`);
last.field = field;
}
// Intentionally not called on throw, so Path.err() can report the failing leaf.
exitField(): void {
this.stack[this.stack.length - 1].field = undefined;
}
readView(n: number, fn: (view: DataView, pos: number) => number): number {
if (!isNum(n) || n < 0) throw this.err(`readView: wrong length=${n}`);
if (this.pos + n > this.data.length) throw this.err('readView: Unexpected end of buffer');
const res = fn(this.view, this.pos);
this.markBytes(n);
return res;
}
// read bytes by absolute offset
absBytes(n: number): Uint8Array {
if (!isNum(n) || n < 0 || n > this.data.length) throw new Error('Unexpected end of buffer');
return this.data.subarray(n);
}
finish(): void {
// ReaderOpts documents allowUnreadBytes as "Allow decoding to finish with unread trailing
// bytes." Prefix decoders may intentionally parse only a value from a larger byte array, so
// this skips all final unread-input checks, including residual bits and pointer-read gaps.
if (this.opts.allowUnreadBytes) return;
if (this.bitPos) {
throw this.err(
`${this.bitPos} bits left after unpack: ${baseHex.encode(this.data.subarray(this.pos))}`
);
}
if (this.bs && !this.parent) {
const notRead = Bitset.indices(this.bs, this.data.length, true);
if (notRead.length) {
const formatted = Bitset.range(notRead)
.map(
({ pos, length }) =>
`(${pos}/${length})[${baseHex.encode(this.data.subarray(pos, pos + length))}]`
)
.join(', ');
throw this.err(`unread byte ranges: ${formatted} (total=${this.data.length})`);
} else return; // all bytes read, everything is ok
}
// Default: no pointers enabled
if (!this.isEnd()) {
throw this.err(
`${this.leftBytes} bytes ${this.bitPos} bits left after unpack: ${baseHex.encode(
this.data.subarray(this.pos)
)}`
);
}
}
// User methods
err(msg: string | Error): Error {
return Path.err('Reader', this.stack, msg);
}
offsetReader(n: number): _Reader {
if (!isNum(n) || n < 0 || n > this.data.length)
throw this.err('offsetReader: Unexpected end of buffer');
return new _Reader(this.absBytes(n), this.opts, this.stack, this, n);
}
bytes(n: number, peek = false): Uint8Array {
if (this.bitPos) throw this.err('readBytes: bitPos not empty');
if (!isNum(n) || n < 0) throw this.err(`readBytes: wrong length=${n}`);
if (this.pos + n > this.data.length) throw this.err('readBytes: Unexpected end of buffer');
const slice = this.data.subarray(this.pos, this.pos + n);
if (!peek) this.markBytes(n);
return slice;
}
byte(peek = false): number {
if (this.bitPos) throw this.err('readByte: bitPos not empty');
if (this.pos + 1 > this.data.length) throw this.err('readByte: Unexpected end of buffer');
const data = this.data[this.pos];
if (!peek) this.markBytes(1);
return data;
}
get leftBytes(): number {
return this.data.length - this.pos;
}
get totalBytes(): number {
return this.data.length;
}
isEnd(): boolean {
return this.pos >= this.data.length && !this.bitPos;
}
progress(): number {
return this.pos * 8 - this.bitPos;
}
// bits are read in BE mode (left to right): (0b1000_0000).readBits(1) == 1
bits(bits: number): number {
// Reject before bitwise shifts: JS coerces negative/fractional shift counts into other widths.
if (!isNum(bits) || bits < 0) throw this.err(`BitReader: wrong length=${bits}`);
if (bits > 32) throw this.err('BitReader: cannot read more than 32 bits in single call');
let out = 0;
while (bits) {
if (!this.bitPos) {
this.bitBuf = this.byte();
this.bitPos = 8;
}
const take = Math.min(bits, this.bitPos);
this.bitPos -= take;
out = (out << take) | ((this.bitBuf >> this.bitPos) & (2 ** take - 1));
this.bitBuf &= 2 ** this.bitPos - 1;
bits -= take;
}
// Fix signed integers
return out >>> 0;
}
find(needle: Bytes, pos: number = this.pos): number | undefined {
if (!isBytes(needle)) throw this.err(`find: needle is not bytes! ${needle}`);
if (this.bitPos) throw this.err('find: bitPos not empty');
if (!needle.length) throw this.err(`find: needle is empty`);
if (!isNum(pos) || pos < 0) throw this.err(`find: wrong pos=${pos}`);
return findBytes(needle, this.data, pos);
}
}
/**
* Internal structure. Writer class for writing to a byte array.
* The `stack` argument of constructor is internal, for debugging and logs.
* @class Writer
*/
// A contiguous span of writer-owned bytes carved out of a chunk. Kept mutable so consecutive
// byte()/writeView() calls extend the open run instead of pushing one buffer per write.
type ChunkRun = { chunk: Uint8Array; start: number; end: number };
class _Writer implements Writer {
pos: number = 0;
readonly stack: PathStack;
// Small writes are carved out of writer-owned chunks; caller-provided bytes() buffers are kept
// by reference between them. A single realloc'd buffer was measured slower for tiny encodes
// (basic/encode bench: 395ns -> 560ns), copy-on-grow is what chunking avoids.
private buffers: (Bytes | ChunkRun)[] = [];
// Every chunk ever allocated by this writer, so finish(clean) can zeroize them whole.
private chunks: Bytes[] = [];
private chunk: Uint8Array | undefined;
private chunkView: DataView | undefined; // lazy: only writeView() needs it
private chunkPos = 0;
private run: ChunkRun | undefined;
private nextChunkSize = 0; // 0 = size the first chunk to the first write
ptrs: { pos: number; ptr: CoderType<number>; buffer: Bytes }[] = [];
private bitBuf = 0;
private bitPos = 0;
private finished = false;
constructor(stack: PathStack = []) {
this.stack = stack;
}
// Reserves `len` contiguous writer-owned bytes and returns their offset inside `this.chunk`.
private carve(len: number): number {
if (this.chunk === undefined || this.chunk.length - this.chunkPos < len) {
// Chunks grow geometrically to amortize allocations, capped to bound over-allocation and
// zeroization work. The 64-byte floor keeps small encodes in one chunk (one lazy DataView).
const size = Math.max(len, this.nextChunkSize, 64);
this.nextChunkSize = Math.min(size * 8, 4096);
this.chunk = new Uint8Array(size);
this.chunkView = undefined;
this.chunkPos = 0;
this.run = undefined;
this.chunks.push(this.chunk);
}
const pos = this.chunkPos;
if (this.run === undefined) {
this.run = { chunk: this.chunk, start: pos, end: pos + len };
this.buffers.push(this.run);
} else this.run.end += len;
this.chunkPos = pos + len;
this.pos += len;
return pos;
}
pushObj(obj: StructOut, objFn: _PathObjFn): void {
return Path.pushObj(this.stack, obj, objFn);
}
enterField(field: string | number): void {
const last = this.stack[this.stack.length - 1];
// A field outside any pushObj() scope or nested inside another field is a coder bug.
if (last === undefined || last.field !== undefined)
throw this.err(`enterField: invalid stack state, field=${field}`);
last.field = field;
}
// Intentionally not called on throw, so Path.err() can report the failing leaf.
exitField(): void {
this.stack[this.stack.length - 1].field = undefined;
}
writeView(len: number, fn: (view: DataView, pos: number) => void): void {
if (this.finished) throw this.err('buffer: finished');
if (!isNum(len) || len < 0 || len > 8) throw new Error(`wrong writeView length=${len}`);
if (this.bitPos) throw this.err('writeBytes: ends with non-empty bit buffer');
const pos = this.carve(len);
if (this.chunkView === undefined) this.chunkView = createView(this.chunk!);
fn(this.chunkView, pos);
}
// User methods
err(msg: string | Error): Error {
// Finished-state guards call err('buffer: finished'), so err itself must not recurse there.
return Path.err('Writer', this.stack, msg);
}
bytes(b: Bytes): void {
if (this.finished) throw this.err('buffer: finished');
if (this.bitPos) throw this.err('writeBytes: ends with non-empty bit buffer');
if (!isBytes(b)) throw this.err(`writeBytes: expected Uint8Array, got ${typeof b}`);
// Keep caller-provided buffers by reference until finish(); mutating them afterwards changes
// the encoded output.
this.buffers.push(b);
// Later carved writes must not merge into a run that would be emitted before `b`.
this.run = undefined;
this.pos += b.length;
}
byte(b: number): void {
if (this.finished) throw this.err('buffer: finished');
if (this.bitPos) throw this.err('writeByte: ends with non-empty bit buffer');
if (!isNum(b) || b < 0 || b > 255) throw this.err(`writeByte: wrong value=${b}`);
// carve() first: it may allocate/replace this.chunk.
const pos = this.carve(1);
this.chunk![pos] = b;
}
finish(clean = true): Bytes {
if (this.finished) throw this.err('buffer: finished');
if (this.bitPos) throw this.err('buffer: ends with non-empty bit buffer');
// Can't use concatBytes, because it limits amount of arguments (65K).
const buffers = this.buffers;
let sum = 0;
for (let i = 0; i < buffers.length; i++) {
const b = buffers[i];
sum += isBytes(b) ? b.length : b.end - b.start;
}
for (let i = 0; i < this.ptrs.length; i++) sum += this.ptrs[i].buffer.length;
const buf = new Uint8Array(sum);
let pad = 0;
for (let i = 0; i < buffers.length; i++) {
const b = buffers[i];
if (isBytes(b)) {
buf.set(b, pad);
pad += b.length;
} else {
buf.set(b.chunk.subarray(b.start, b.end), pad);
pad += b.end - b.start;
}
}
for (let i = 0; i < this.ptrs.length; i++) {
buf.set(this.ptrs[i].buffer, pad);
pad += this.ptrs[i].buffer.length;
}
for (let pos = this.pos, i = 0; i < this.ptrs.length; i++) {
const ptr = this.ptrs[i];
buf.set(ptr.ptr.encode(pos), ptr.pos);
pos += ptr.buffer.length;
}
// Cleanup
if (clean) {
// bytes() keeps caller-provided buffers by reference, so only writer-owned chunks are
// zeroized — whole, including unused tails: same hygiene as per-buffer cleanup.
for (const c of this.chunks) c.fill(0);
this.chunks = [];
this.chunk = undefined;
this.chunkView = undefined;
this.chunkPos = 0;
this.run = undefined;
this.buffers = [];
for (const p of this.ptrs) p.buffer.fill(0);
this.ptrs = [];
this.finished = true;
this.bitBuf = 0;
}
return buf;
}
bits(value: number, bits: number): void {
if (this.finished) throw this.err('buffer: finished');
// Reject before bitwise shifts: JS coerces negative/fractional values and widths.
if (!isNum(bits) || bits < 0) throw this.err(`writeBits: wrong length=${bits}`);
if (bits > 32) throw this.err('writeBits: cannot write more than 32 bits in single call');
if (!isNum(value) || value < 0) throw this.err(`writeBits: wrong value=${value}`);
if (value >= 2 ** bits) throw this.err(`writeBits: value (${value}) >= 2**bits (${bits})`);
while (bits) {
const take = Math.min(bits, 8 - this.bitPos);
this.bitBuf = (this.bitBuf << take) | (value >> (bits - take));
this.bitPos += take;
bits -= take;
value &= 2 ** bits - 1;
if (this.bitPos === 8) {
this.bitPos = 0;
// carve() first: it may allocate/replace this.chunk. The Uint8Array store wraps mod 256,
// matching the previous `new Uint8Array([bitBuf])` semantics for int32-coerced values.
const pos = this.carve(1);
this.chunk![pos] = this.bitBuf;
}
}
}
}
// Immutable LE<->BE
const swapEndianness = (b: TArg<Bytes>): TRet<Bytes> => Uint8Array.from(b).reverse();
function checkMinimalBigintBytes(
value: TArg<Bytes>,
le: boolean,
signed: boolean,
err: (msg: string) => Error
): void {
if (!value.length) return;
const b = le ? swapEndianness(value) : value;
const fail = () => {
throw err('bigint: non-minimal encoding');
};
if (!signed) {
if (b[0] === 0) fail();
return;
}
// Encoders emit the shortest signed two's-complement byte string that preserves the sign bit.
if (b[0] === 0 && (b.length === 1 || (b[1] & 128) === 0)) fail();
if (b.length > 1 && b[0] === 255 && (b[1] & 128) !== 0) fail();
}
/** Internal function for checking bit bounds of bigint in signed/unsinged form */
function checkBounds(value: bigint, bits: bigint, signed: boolean): void {
if (signed) {
if (bits <= _0n) throw new Error(`checkBounds: signed bits must be positive, got ${bits}`);
// [-(2**(32-1)), 2**(32-1)-1]
const signBit = _2n ** (bits - _1n);
if (value < -signBit || value >= signBit)
throw new Error(`value out of signed bounds. Expected ${-signBit} <= ${value} < ${signBit}`);
} else {
// [0, 2**32-1]
const max = _2n ** bits;
if (_0n > value || value >= max)
throw new Error(`value out of unsigned bounds. Expected 0 <= ${value} < ${max}`);
}
}
function _wrap<T>(inner: TArg<BytesCoderStream<T>>): CoderType<T> {
const _inner = inner as BytesCoderStream<T>;
return {
// NOTE: we cannot export validate here, since it is likely mistake.
// Raw inner throws propagate unchanged; path-aware errors must use w.err/r.err or validate().
encodeStream: _inner.encodeStream,
decodeStream: _inner.decodeStream,
size: _inner.size,
encode: (value: T): TRet<Bytes> => {
const w = new _Writer();
_inner.encodeStream(w, value);
return w.finish() as TRet<Bytes>;
},
decode: (data: TArg<Bytes>, opts: ReaderOpts = {}): T => {
if (!isBytes(data)) throw new TypeError(`decode: expected Uint8Array, got ${typeof data}`);
const r = new _Reader(data, opts);
const res = _inner.decodeStream(r);
r.finish();
return res;
},
};
}
/**
* Validates a value before encoding and after decoding using a provided function.
* @param inner - The inner CoderType.
* @param fn - The validation function.
* @returns CoderType which check value with validation function.
* @throws On wrong inner coder or validator argument types. {@link TypeError}
* @example
* Reject values outside the accepted range during both encode and decode.
* ```ts
* import * as P from 'micro-packed';
* const val = (n: number) => {
* if (n > 10) throw new Error(`${n} > 10`);
* return n;
* };
*
* // Checks that values are <= 10 during encoding and decoding.
* const RangedInt = P.validate(P.U32LE, val);
* ```
*/
export function validate<T>(inner: CoderType<T>, fn: Validate<T>): CoderType<T> {
if (!isCoder(inner)) throw new TypeError(`validate: invalid inner value ${inner}`);
if (typeof fn !== 'function') throw new TypeError('validate: fn should be function');
return _wrap({
size: inner.size,
encodeStream: (w: TArg<Writer>, value: T) => {
let res;
try {
res = fn(value);
} catch (e) {
// Validator callbacks are caller code: if they throw non-Error garbage, diagnostics are on
// them. Review policy: "if they throw garbage, then it is on them".
throw w.err(e as Error);
}
inner.encodeStream(w, res);
},
decodeStream: (r: TArg<Reader>): T => {
const res = inner.decodeStream(r);
try {
return fn(res);
} catch (e) {
throw r.err(e as Error);
}
},
});
}
/**
* Wraps a stream encoder into a generic encoder and optionally validation function
* @param inner - Stream coder with optional validation hook.
* @returns The wrapped CoderType.
* @throws On wrong wrapped stream-coder shapes. {@link TypeError}
* @example
* Start from stream methods, then add validation if needed.
* ```ts
* import * as P from 'micro-packed';
* const U8 = P.wrap({
* encodeStream: (w, value) => w.byte(value),
* decodeStream: (r) => r.byte(),
* });
* const checkedU8 = P.wrap({
* encodeStream: (w, value) => w.byte(value),
* decodeStream: (r) => r.byte(),
* validate: (n: number) => {
* if (n > 10) throw new Error(`${n} > 10`);
* return n;
* }
* });
* ```
*/
// Keep this as a plain contextual object type instead of TArg<>/TRet<> helpers:
// recursive object mapping breaks unannotated encodeStream/decodeStream parameters,
// and _wrap() already returns the CoderType<T> shape without byte-array normalization.
export const wrap = <T>(inner: {
size?: number;
encodeStream: (w: Writer, value: T) => void;
decodeStream: (r: Reader) => T;
validate?: Validate<T>;
}): CoderType<T> => {
const _inner = inner as BytesCoderStream<T> & { validate?: Validate<T> };
// Public wrap() is the boundary for raw stream coders; reject malformed shapes before a
// half-constructed coder fails later during encode/decode.
if (!isPlainObject(_inner)) throw new TypeError(`wrap: invalid inner value ${_inner}`);
if (typeof _inner.encodeStream !== 'function')
throw new TypeError('wrap: encodeStream should be function');
if (typeof _inner.decodeStream !== 'function')
throw new TypeError('wrap: decodeStream should be function');
if (_inner.size !== undefined && (!isNum(_inner.size) || _inner.size < 0))
throw new TypeError(`wrap: invalid size ${_inner.size}`);
if (_inner.validate !== undefined && typeof _inner.validate !== 'function')
throw new TypeError('wrap: validate should be function');
const res = _wrap(_inner);
return _inner.validate !== undefined ? validate(res, _inner.validate) : res;
};
const isBaseCoder = (elm: any) =>
isPlainObject(elm) && typeof elm.decode === 'function' && typeof elm.encode === 'function';
/**
* Checks if the given value is a CoderType.
* @param elm - The value to check.
* @returns True if the value is a CoderType, false otherwise.
* @example
* Guard unknown values before calling encode/decode helpers on them.
* ```ts
* import { isCoder, U8 } from 'micro-packed';
* isCoder(U8);
* ```
*/
export function isCoder<T>(elm: any): elm is CoderType<T> {
return (
isPlainObject(elm) &&
isBaseCoder(elm) &&
typeof elm.encodeStream === 'function' &&
typeof elm.decodeStream === 'function' &&
(elm.size === undefined || (isNum(elm.size) && elm.size >= 0))
);
}
// Coders (like in @scure/base) for common operations
/**
* Base coder for working with dictionaries (records, objects, key-value map)
* Dictionary is dynamic type like: `[key: string, value: any][]`
* @returns base coder that encodes/decodes between arrays of key-value tuples and dictionaries.
* @example
* Convert between tuple entries and a plain object record.
* ```ts
* import * as P from 'micro-packed';
* const dict: P.CoderType<Record<string, number>> = P.apply(
* P.array(P.U16BE, P.tuple([P.cstring, P.U32LE] as const)),
* P.coders.dict()
* );
* ```
*/
function dict<T>(): BaseCoder<[string, T][], Record<string, T>> {
return {
encode: (from: [string, T][]): Record<string, T> => {
if (!Array.isArray(from)) throw new Error('array expected');
const to: Record<string, T> = {};
const seen = new Set<string>();
for (const item of from) {
if (!Array.isArray(item) || item.length !== 2)
throw new Error(`array of two elements expected`);
const name = item[0];
const value = item[1];
validateFieldName(name, 'dict: key');
// Stored undefined is still a present key, so duplicate detection cannot inspect values.
if (seen.has(name)) throw new Error(`key(${name}) appears twice in struct`);
seen.add(name);
to[name] = value;
}
return to;
},
decode: (to: Record<string, T>): [string, T][] => {
if (!isPlainObject(to)) throw new Error(`expected plain object, got ${to}`);
for (const name in to) validateFieldName(name, 'dict: key');
return Object.entries(to);
},
};
}
/**
* Safely converts bigint to number.
* Sometimes pointers / tags use u64 or other big numbers which cannot be represented by number,
* but we still can use them since real value will be smaller than u32
*/
const numberBigint: BaseCoder<bigint, number> = /* @__PURE__ */ Object.freeze({
encode: (from: bigint): number => {
if (typeof from !== 'bigint') throw new Error(`expected bigint, got ${typeof from}`);
if (from > BigInt(Number.MAX_SAFE_INTEGER))
throw new Error(`element bigger than MAX_SAFE_INTEGER=${from}`);
// Number() silently rounds bigint values outside the safe integer range on either side.
if (from