UNPKG

micro-packed

Version:

Define complex binary structures using composable primitives

1,253 lines (1,220 loc) 129 kB
import type { TArg, TRet } from '@scure/base'; import { hex as baseHex, utf8, type Coder as BaseCoder } from '@scure/base'; export type { TArg, TRet } from '@scure/base'; /** * Define complex binary structures using composable primitives. * Main ideas: * - Encode / decode can be chained, same as in `scure-base` * - A complex structure can be created from an array and struct of primitive types * - Strings / bytes are arrays with specific optimizations: we can just read bytes directly * without creating plain array first and reading each byte separately. * - Types are inferred from definition * @module * @example * Define a struct with numbers, strings, bytes, and nested arrays. * ```ts * import * as P from 'micro-packed'; * const s = P.struct({ * field1: P.U32BE, // 32-bit unsigned big-endian integer * field2: P.string(P.U8), // String with U8 length prefix * field3: P.bytes(32), // 32 bytes * field4: P.array(P.U16BE, P.struct({ // Array of structs with U16BE length * subField1: P.U64BE, // 64-bit unsigned big-endian integer * subField2: P.string(10) // 10-byte string * })) * }); * ``` */ // TODO: remove dependency on scure-base & inline? /* Exports can be groupped like this: - Primitive types: P.bytes, P.string, P.hex, P.constant, P.pointer - Complex types: P.array, P.struct, P.tuple, P.map, P.tag, P.mappedTag - Padding, prefix, magic: P.padLeft, P.padRight, P.prefix, P.magic, P.magicBytes - Flags: P.flag, P.flagged, P.optional - Wrappers: P.apply, P.wrap, P.lazy - Bit fiddling: P.bits, P.bitset - utils: P.validate, coders.decimal - Debugger */ /** * Shortcut to zero-length (empty) byte array. * Keep public Bytes typing, not TRet<Bytes>, so variables inferred from this * constant can later accept caller-owned Bytes backed by any ArrayBufferLike. */ export const EMPTY: Bytes = /* @__PURE__ */ Uint8Array.of(); /** * Shortcut to one-element (element is 0) byte array. * Keep the same public Bytes typing rationale as EMPTY. */ export const NULL: Bytes = /* @__PURE__ */ Uint8Array.of(0); /** Prototype-sensitive names cannot roundtrip as normal fields on plain decoded objects. */ const restrictedKeys = /* @__PURE__ */ new Set(['__proto__', 'constructor', 'prototype']); const validateFieldName = (name: unknown, label: string): void => { if (typeof name !== 'string') throw new Error(`${label} should be string, got ${typeof name}`); if (name.includes('..')) throw new TypeError(`${label} ${name} cannot contain path parent ..`); if (name.includes('/')) throw new TypeError(`${label} ${name} cannot contain path separator /`); if (restrictedKeys.has(name)) throw new Error(`${label} ${name} is reserved`); }; /** Checks if two Uint8Arrays are equal. Not constant-time. */ function equalBytes(a: TArg<Uint8Array>, b: TArg<Uint8Array>): boolean { if (a.length !== b.length) return false; for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; return true; } type BytesFinder = (data: TArg<Uint8Array>, pos?: number) => number | undefined; function createFindBytes(needle: TArg<Uint8Array>): TRet<BytesFinder> { if (needle.length === 1) { const byte = needle[0]; return (data, pos = 0) => { const idx = data.indexOf(byte, pos); return idx === -1 ? undefined : idx; }; } // KMP avoids quadratic scans on repeated-prefix terminators. const back = new Uint32Array(needle.length); for (let i = 1, j = 0; i < needle.length; i++) { while (j && needle[i] !== needle[j]) j = back[j - 1]; if (needle[i] === needle[j]) back[i] = ++j; } return (data, pos = 0) => { for (let i = pos, j = 0; i < data.length; i++) { while (j && data[i] !== needle[j]) j = back[j - 1]; if (data[i] !== needle[j]) continue; if (++j === needle.length) return i - needle.length + 1; } return undefined; }; } const findBytes = (needle: TArg<Uint8Array>, data: TArg<Uint8Array>, pos = 0): number | undefined => createFindBytes(needle)(data, pos); /** Compares values used as encoded-domain constants; byte arrays compare by contents. */ function equal(a: unknown, b: unknown): boolean { const aBytes = isBytes(a); const bBytes = isBytes(b); if (aBytes || bBytes) return aBytes && bBytes && equalBytes(a, b); return a === b; } /** Checks if the given value is a Uint8Array. */ function isBytes(a: unknown): a is Bytes { // Plain `instanceof Uint8Array` is too strict for some Buffer / proxy / cross-realm cases. The // fallback still requires a real ArrayBuffer view, so plain JSON-deserialized // `{ constructor: ... }` spoofing is rejected. `BYTES_PER_ELEMENT === 1` keeps // the fallback on byte-oriented views. return ( a instanceof Uint8Array || (ArrayBuffer.isView(a) && a.constructor.name === 'Uint8Array' && 'BYTES_PER_ELEMENT' in a && a.BYTES_PER_ELEMENT === 1) ); } /** * Concatenates multiple Uint8Arrays. * Engines limit functions to 65K+ arguments. * @param arrays Array of Uint8Array elements * @returns Concatenated Uint8Array */ function concatBytes(...arrays: TArg<Uint8Array[]>): TRet<Uint8Array> { let sum = 0; for (let i = 0; i < arrays.length; i++) { const a = arrays[i]; if (!isBytes(a)) throw new Error('Uint8Array expected'); sum += a.length; } const res = new Uint8Array(sum); for (let i = 0, pad = 0; i < arrays.length; i++) { const a = arrays[i]; res.set(a, pad); pad += a.length; } return res; } /** * Creates DataView from Uint8Array * @param arr - bytes * @returns DataView */ const createView = (arr: TArg<Uint8Array>) => new DataView(arr.buffer, arr.byteOffset, arr.byteLength); const _0n = /* @__PURE__ */ BigInt(0); const _1n = /* @__PURE__ */ BigInt(1); const _2n = /* @__PURE__ */ BigInt(2); const _8n = /* @__PURE__ */ BigInt(8); const _10n = /* @__PURE__ */ BigInt(10); const _255n = /* @__PURE__ */ BigInt(255); /** * Checks if the provided value is object-like for option/schema bags. * This intentionally matches noble-curves and noble-hashes by using the * `[object Object]` tag instead of rejecting class/proxy/env objects by prototype; * stricter checks caused compatibility reports in proxied environments. * Array, Uint8Array and others are not plain objects. * @param obj - The value to be checked. */ function isPlainObject(obj: any): boolean { return Object.prototype.toString.call(obj) === '[object Object]'; } function isNum(num: unknown): num is number { return Number.isSafeInteger(num); } const hasOwn = (obj: object, key: PropertyKey): boolean => Object.prototype.hasOwnProperty.call(obj, key); /** * Miscellaneous helpers reused by the coder internals and tests. * @example * Reuse a couple of byte helpers without pulling in the full namespace. * ```ts * import { utils } from 'micro-packed'; * const left = Uint8Array.of(1); * const right = Uint8Array.of(2); * utils.equalBytes(utils.concatBytes(left, right), Uint8Array.of(1, 2)); * ``` */ export const utils: TRet<{ equalBytes: typeof equalBytes; isBytes: typeof isBytes; isCoder: typeof isCoder; checkBounds: typeof checkBounds; concatBytes: typeof concatBytes; createView: (arr: TArg<Uint8Array>) => DataView; isPlainObject: typeof isPlainObject; }> = /* @__PURE__ */ Object.freeze({ equalBytes, isBytes, isCoder, checkBounds, concatBytes, createView, isPlainObject, }); // Types /** Byte-array alias used throughout the public API. */ export type Bytes = Uint8Array; /** Optional value helper used by conditional coders. */ export type Option<T> = T | undefined; /** Coder encodes and decodes between two types. */ export interface Coder<F, T> { /** * Encodes (converts) a decoded value into its serialized representation. * @param from - Value to encode. * @returns Encoded representation. */ encode(from: F): T; /** * Decodes (converts) a serialized value back into its decoded representation. * @param to - Encoded representation to decode. * @returns Decoded value. */ decode(to: T): F; } /** BytesCoder converts value between a type and a byte array. */ export interface BytesCoder<T> extends Coder<T, Bytes> { /** Fixed-size hint in bytes, when known. */ size?: number; /** * Encodes a value into a byte array. * @param data - Value to encode. * @returns Encoded bytes. */ encode: (data: T) => Bytes; /** * Decodes a byte array into a value. * @param data - Bytes to decode. * @param opts - Reader options used while decoding. See {@link ReaderOpts}. * @returns Decoded value. */ decode: (data: Bytes, opts?: ReaderOpts) => T; } /** BytesCoderStream converts value between a type and a byte array, using streams. */ export interface BytesCoderStream<T> { /** Fixed-size hint in bytes, when known. */ size?: number; /** * Encodes a value into a Writer stream. * @param w - Writer stream. * @param value - Value to encode. */ encodeStream: (w: Writer, value: T) => void; /** * Decodes a value from a Reader stream. * @param r - Reader stream. * @returns Decoded value. */ decodeStream: (r: Reader) => T; } /** Full coder interface with both stream and byte-array helpers. */ export type CoderType<T> = BytesCoderStream<T> & BytesCoder<T>; /** CoderType with a known fixed byte size. */ export type Sized<T> = CoderType<T> & { size: number }; /** Extract the decoded value type from a coder. */ export type UnwrapCoder<T> = T extends CoderType<infer U> ? U : T; /** * Validation function. Should return value after validation. * Can be used to narrow types */ export type Validate<T> = (elm: T) => T; /** Length descriptor accepted by variable-size coders. */ export type Length = CoderType<number> | CoderType<bigint> | number | Bytes | string | null; // NOTE: we can't have terminator separate function, since it won't know about boundaries // E.g. array of U16LE ([1,2,3]) would be [1, 0, 2, 0, 3, 0] // But terminator will find array at index '1', which happens to be inside of an element itself /** * Can be: * - Dynamic (CoderType) * - Fixed (number) * - Terminated (usually zero): Uint8Array with terminator * - Field path to field with length (string) * - Infinity (null) - decodes until end of buffer * Used in: * - bytes (string, prefix is implementation of bytes) * - array */ const lengthCoder = (len: Length) => { if (len !== null && typeof len !== 'string' && !isCoder(len) && !isBytes(len) && !isNum(len)) { // Constructor argument validation uses TypeError. // Stream/data failures keep contextual Error paths. throw new TypeError( `lengthCoder: expected null | number | Uint8Array | CoderType, got ${len} (${typeof len})` ); } if (typeof len === 'number' && len < 0) throw new Error(`lengthCoder: wrong length=${len}`); if (isBytes(len) && !len.length) throw new Error('lengthCoder: empty terminator'); return { encodeStream(w: TArg<Writer>, value: number | null) { if (len === null) return; if (isCoder(len)) return len.encodeStream(w, value); let byteLen; if (typeof len === 'number') byteLen = len; else if (typeof len === 'string') byteLen = Path.resolve((w as _Writer).stack, len); if (typeof byteLen === 'bigint') byteLen = Number(byteLen); if (!isNum(byteLen) || byteLen < 0 || byteLen !== value) throw w.err(`Wrong length: ${byteLen} len=${len} exp=${value} (${typeof value})`); }, decodeStream(r: TArg<Reader>) { let byteLen; if (isCoder(len)) byteLen = len.decodeStream(r); else if (typeof len === 'number') byteLen = len; else if (typeof len === 'string') byteLen = Path.resolve((r as _Reader).stack, len); if (typeof byteLen === 'bigint') byteLen = Number(byteLen); else if (typeof byteLen !== 'number') throw r.err(`Wrong length: ${byteLen}`); if (!isNum(byteLen) || byteLen < 0) throw r.err(`Wrong length: ${byteLen}`); return byteLen; }, }; }; type ArrLike<T> = Array<T> | ReadonlyArray<T>; // prettier-ignore /** Typed arrays supported by the utility helper types. */ export type TypedArray = | Uint8Array | Int8Array | Uint8ClampedArray | Uint16Array | Int16Array | Uint32Array | Int32Array; /** Writable version of a type, where readonly properties are made writable. */ export type Writable<T> = T extends {} ? T extends TypedArray ? T : { -readonly [P in keyof T]: Writable<T[P]>; } : T; /** Union of object value types. */ export type Values<T> = T[keyof T]; /** Key helper that removes fields whose values are exactly `undefined`. */ export type NonUndefinedKey<T, K extends keyof T> = T[K] extends undefined ? never : K; /** Key helper that keeps only nullable fields. */ export type NullableKey<T, K extends keyof T> = T[K] extends NonNullable<T[K]> ? never : K; // Opt: value !== undefined, but value === T|undefined /** Key helper for optional-but-present struct fields. */ export type OptKey<T, K extends keyof T> = NullableKey<T, K> & NonUndefinedKey<T, K>; /** Key helper for required struct fields. */ export type ReqKey<T, K extends keyof T> = T[K] extends NonNullable<T[K]> ? K : never; /** Object containing only optional keys from a struct shape. */ export type OptKeys<T> = Pick<T, { [K in keyof T]: OptKey<T, K> }[keyof T]>; /** Object containing only required keys from a struct shape. */ export type ReqKeys<T> = Pick<T, { [K in keyof T]: ReqKey<T, K> }[keyof T]>; /** Input object type accepted by `struct()`. */ export type StructInput<T extends Record<string, any>> = { [P in keyof ReqKeys<T>]: T[P] } & { [P in keyof OptKeys<T>]?: T[P]; }; /** Record of field names to coder instances for `struct()`. */ export type StructRecord<T extends Record<string, any>> = { [P in keyof T]: CoderType<T[P]>; }; /** Generic decoded object bag used internally by nested coders. */ export type StructOut = Record<string, any>; /** Padding function that takes an index and returns a padding value. */ export type PadFn = (i: number) => number; /** * Small bitset structure to store position of ranges that have been read. * Can be more efficient when internal trees are utilized at the cost of complexity. * Needs `O(N/8)` memory for parsing. * Purpose: if there are pointers in parsed structure, * they can cause read of two distinct ranges: * [0-32, 64-128], which means 'pos' is not enough to handle them */ const Bitset = /* @__PURE__ */ Object.freeze({ BITS: 32, FULL_MASK: -1 >>> 0, // 1<<32 will overflow len: (len: number) => { if (!isNum(len) || len < 0) throw new Error(`wrong len=${len}`); return Math.ceil(len / 32); }, create: (len: number) => new Uint32Array(Bitset.len(len)), clean: (bs: TArg<Uint32Array>) => bs.fill(0), debug: (bs: TArg<Uint32Array>) => Array.from(bs).map((i) => (i >>> 0).toString(2).padStart(32, '0')), checkLen: (bs: TArg<Uint32Array>, len: number) => { if (Bitset.len(len) === bs.length) return; throw new Error(`wrong length=${bs.length}. Expected: ${Bitset.len(len)}`); }, chunkLen: (bsLen: number, pos: number, len: number) => { if (!isNum(bsLen) || bsLen < 0) throw new Error(`wrong bsLen=${bsLen}`); if (!isNum(pos) || pos < 0) throw new Error(`wrong pos=${pos}`); if (!isNum(len) || len < 0) throw new Error(`wrong len=${len}`); if (pos > bsLen - len) throw new Error(`wrong range=${pos}/${len} of ${bsLen}`); }, set: (bs: TArg<Uint32Array>, chunk: number, value: number, allowRewrite = true) => { if (!isNum(chunk) || chunk < 0 || chunk >= bs.length) return false; if (!allowRewrite && (bs[chunk] & value) !== 0) return false; bs[chunk] |= value; return true; }, pos: (pos: number, i: number) => ({ chunk: Math.floor((pos + i) / 32), mask: 1 << (32 - ((pos + i) % 32) - 1), }), indices: (bs: TArg<Uint32Array>, len: number, invert = false) => { Bitset.checkLen(bs, len); const { FULL_MASK, BITS } = Bitset; const left = BITS - (len % BITS); const lastMask = left ? (FULL_MASK >>> left) << left : FULL_MASK; const res = []; for (let i = 0; i < bs.length; i++) { let c = bs[i]; if (invert) c = ~c; // allows to gen unset elements // apply mask to last element, so we won't iterate non-existent items if (i === bs.length - 1) c &= lastMask; if (c === 0) continue; // fast-path for (let j = 0; j < BITS; j++) { const m = 1 << (BITS - j - 1); if (c & m) res.push(i * BITS + j); } } return res; }, range: (arr: number[]) => { // Bitset.indices() returns sorted unique positions; this helper only merges adjacent runs. const res = []; let cur; for (const i of arr) { if (cur === undefined || i !== cur.pos + cur.length) res.push((cur = { pos: i, length: 1 })); else cur.length += 1; } return res; }, rangeDebug: (bs: TArg<Uint32Array>, len: number, invert = false) => `[${Bitset.range(Bitset.indices(bs, len, invert)) .map((i) => `(${i.pos}/${i.length})`) .join(', ')}]`, setRange: ( bs: TArg<Uint32Array>, bsLen: number, pos: number, len: number, allowRewrite = true ) => { Bitset.chunkLen(bsLen, pos, len); // Empty ranges are valid reader-bookkeeping no-ops; mask math below assumes at least one bit. if (len === 0) return true; const { FULL_MASK, BITS } = Bitset; // Try to set range with maximum efficiency: // - first chunk is always '0000[1111]' (only right ones) // - middle chunks are set to '[1111 1111]' (all ones) // - last chunk is always '[1111]0000' (only left ones) // - max operations: (N/32) + 2 (first and last) const first = pos % BITS ? Math.floor(pos / BITS) : undefined; const lastPos = pos + len; const last = lastPos % BITS ? Math.floor(lastPos / BITS) : undefined; const canSet = (chunk: number, value: number) => chunk >= 0 && chunk < bs.length && (bs[chunk] & value) === 0; if (!allowRewrite) { // Check the whole range before writing so a late overlap cannot leave earlier chunks mutated. if (first !== undefined && first === last) { if (!canSet(first, (FULL_MASK >>> (BITS - len)) << (BITS - len - pos))) return false; } else { if (first !== undefined && !canSet(first, FULL_MASK >>> pos % BITS)) return false; const start = first !== undefined ? first + 1 : pos / BITS; const end = last !== undefined ? last : lastPos / BITS; for (let i = start; i < end; i++) if (!canSet(i, FULL_MASK)) return false; if (last !== undefined && first !== last) if (!canSet(last, FULL_MASK << (BITS - (lastPos % BITS)))) return false; } } // special case, whole range inside single chunk if (first !== undefined && first === last) return Bitset.set( bs, first, (FULL_MASK >>> (BITS - len)) << (BITS - len - pos), allowRewrite ); if (first !== undefined) { // first chunk if (!Bitset.set(bs, first, FULL_MASK >>> pos % BITS, allowRewrite)) return false; } // middle chunks const start = first !== undefined ? first + 1 : pos / BITS; const end = last !== undefined ? last : lastPos / BITS; for (let i = start; i < end; i++) if (!Bitset.set(bs, i, FULL_MASK, allowRewrite)) return false; if (last !== undefined && first !== last) if (!Bitset.set(bs, last, FULL_MASK << (BITS - (lastPos % BITS)), allowRewrite)) // last chunk return false; return true; }, }); /** Path related utils (internal) */ // `field` stays a number for array/tuple indices and is stringified lazily in path(): // hot decode/encode loops must not allocate a string per element just for diagnostics. type Path = { obj: StructOut; field?: string | number }; type PathStack = Path[]; export type _PathObjFn = () => void; type PathUtils = { pushObj: (stack: PathStack, obj: StructOut, objFn: _PathObjFn) => void; path: (stack: PathStack) => string; err: (name: string, stack: PathStack, msg: string | Error) => Error; resolve: (stack: PathStack, path: string) => StructOut | undefined; }; const Path: PathUtils = /* @__PURE__ */ Object.freeze({ /** * Internal method for handling stack of paths (debug, errors, dynamic fields via path) * `.pop()` always happens after the wrapped function. * Fields inside the object are tracked via Reader/Writer enterField()/exitField(), * which the debugger overrides to observe per-field byte ranges. * NOTE: we don't want to do '.pop' on error! */ pushObj: (stack: PathStack, obj: StructOut, objFn: _PathObjFn): void => { stack.push({ obj }); objFn(); stack.pop(); }, path: (stack: PathStack): string => { const res = []; for (const i of stack) if (i.field !== undefined) res.push(i.field === '' ? '""' : `${i.field}`); // Path.err() uses this string for user-visible context. Empty keys need explicit rendering so // field("") is distinguishable from the root path; slash-containing keys are still raw. return res.join('/'); }, err: (name: string, stack: PathStack, msg: string | Error): Error => { const text = `${name}(${Path.path(stack)}): ${typeof msg === 'string' ? msg : msg.message}`; // Path context is the primary diagnostic. Do not attach `cause`: Node inspection expands nested // cause stacks and makes the original, path-prefixed failure harder to scan. // Keep specific validation classes after adding the path prefix. Otherwise public coder // APIs flatten inner TypeError / RangeError guards back to plain Error. const err = msg instanceof TypeError ? new TypeError(text) : msg instanceof RangeError ? new RangeError(text) : new Error(text); if (msg instanceof Error && msg.stack) { const from = `${msg.name}: ${msg.message}`; const to = `${err.name}: ${err.message}`; err.stack = msg.stack.startsWith(from) ? `${to}${msg.stack.slice(from.length)}` : msg.stack; } return err; }, resolve: (stack: PathStack, path: string): StructOut | undefined => { const parts = path.split('/'); // Leading '..' segments mean parent traversal and '/' separates nested fields, so literal // keys using those tokens are not addressable through this helper. const objPath = stack.map((i) => i.obj); let i = 0; for (; i < parts.length; i++) { if (parts[i] === '..') objPath.pop(); else break; } let cur = objPath.pop(); for (; i < parts.length; i++) { if (!cur || cur[parts[i]] === undefined) return undefined; cur = cur[parts[i]]; } return cur; }, }); /** Options for the Reader class. */ export type ReaderOpts = { /** Allow decoding to finish with unread trailing bytes. */ allowUnreadBytes?: boolean; /** Allow the same byte range to be read more than once through pointers. */ allowMultipleReads?: boolean; }; // These are safe API for external usage /** Reader interface passed into stream decoders. */ export type Reader = { // Utils /** Current position in the buffer. */ readonly pos: number; /** Number of bytes left in the buffer. */ readonly leftBytes: number; /** Total number of bytes in the buffer. */ readonly totalBytes: number; /** * Checks if the end of the buffer has been reached. * @returns `true` when the reader consumed the whole buffer. */ isEnd(): boolean; /** * Creates an error with the given message. Adds information about current field path. * If Error object provided, saves original stack trace. * @param msg - The error message or an Error object. * @returns The created Error object. */ err(msg: string | Error): Error; /** * Reads a specified number of bytes from the buffer. * * WARNING: Uint8Array is subarray of original buffer. Do not modify. * @param n - The number of bytes to read. * @param peek - If `true`, the bytes are read without advancing the position. * @returns The read bytes as a Uint8Array. */ bytes(n: number, peek?: boolean): Uint8Array; /** * Reads a single byte from the buffer. * @param peek - If `true`, the byte is read without advancing the position. * @returns The read byte as a number. */ byte(peek?: boolean): number; /** * Reads a specified number of bits from the buffer. * @param bits - The number of bits to read. * @returns The read bits as a number. */ bits(bits: number): number; /** * Finds the first occurrence of a needle in the buffer. * @param needle - The needle to search for. * @param pos - The starting position for the search. * @returns The position of the first occurrence of the needle, or `undefined` if not found. */ find(needle: Bytes, pos?: number): number | undefined; /** * Creates a new Reader instance at the specified offset. * Complex and unsafe API: currently only used in eth ABI parsing of pointers. * Required to break pointer boundaries inside arrays for complex structure. * Please use only if absolutely necessary! * @param n - The offset to create the new Reader at. * @returns A new Reader instance at the specified offset. */ offsetReader(n: number): Reader; }; /** Writer interface passed into stream encoders. */ export type Writer = { /** * Creates an error with the given message. Adds information about current field path. * If Error object provided, saves original stack trace. * @param msg - The error message or an Error object. * @returns The created Error object. */ err(msg: string | Error): Error; /** * Writes a byte array to the buffer. * @param b - The byte array to write. */ bytes(b: Bytes): void; /** * Writes a single byte to the buffer. * @param b - The byte to write. */ byte(b: number): void; /** * Writes a specified number of bits to the buffer. * @param value - The value to write. * @param bits - The number of bits to write. */ bits(value: number, bits: number): void; }; /** * Internal structure. Reader class for reading from a byte array. * `stack` is internal: for debugger and logging * @class Reader */ class _Reader implements Reader { pos = 0; readonly data: Bytes; readonly opts: ReaderOpts; readonly stack: PathStack; private parent: _Reader | undefined; private parentOffset: number; private bitBuf = 0; private bitPos = 0; private bs: Uint32Array | undefined; // bitset private view: DataView; constructor( data: Bytes, opts: ReaderOpts = {}, stack: PathStack = [], parent: _Reader | undefined = undefined, parentOffset: number = 0 ) { if (!isBytes(data)) throw new TypeError(`Reader: expected Uint8Array, got ${typeof data}`); if (!isPlainObject(opts)) throw new TypeError(`ReaderOpts: expected plain object, got ${opts}`); if (opts.allowUnreadBytes !== undefined && typeof opts.allowUnreadBytes !== 'boolean') throw new TypeError( `ReaderOpts.allowUnreadBytes: expected boolean, got ${typeof opts.allowUnreadBytes}` ); if (opts.allowMultipleReads !== undefined && typeof opts.allowMultipleReads !== 'boolean') throw new TypeError( `ReaderOpts.allowMultipleReads: expected boolean, got ${typeof opts.allowMultipleReads}` ); this.data = data; this.opts = opts; this.stack = stack; this.parent = parent; this.parentOffset = parentOffset; this.view = createView(data); } /** Internal method for pointers. */ _enablePointers(): void { // Pointer decoding enables tracking before the pointed child reader starts consuming bytes, so // only the root reader owns the bitset and seeds it from the already-consumed prefix. if (this.parent) return this.parent._enablePointers(); if (this.bs) return; this.bs = Bitset.create(this.data.length); Bitset.setRange(this.bs, this.data.length, 0, this.pos, this.opts.allowMultipleReads); } private markBytesBS(pos: number, len: number): boolean { if (this.parent) return this.parent.markBytesBS(this.parentOffset + pos, len); if (!len) return true; // Before pointers are enabled there is no bitset yet, so linear cursor checks remain the only // guard; overlap tracking starts only after _enablePointers() allocates the root bitset. if (!this.bs) return true; return Bitset.setRange(this.bs, this.data.length, pos, len, false); } private markBytes(len: number): boolean { const pos = this.pos; const res = this.markBytesBS(pos, len); // Keep failed overlap reads at their start so diagnostics point at the repeated span. if (!this.opts.allowMultipleReads && !res) throw this.err(`multiple read pos=${pos} len=${len}`); this.pos += len; return res; } pushObj(obj: StructOut, objFn: _PathObjFn): void { return Path.pushObj(this.stack, obj, objFn); } enterField(field: string | number): void { const last = this.stack[this.stack.length - 1]; // A field outside any pushObj() scope or nested inside another field is a coder bug. if (last === undefined || last.field !== undefined) throw this.err(`enterField: invalid stack state, field=${field}`); last.field = field; } // Intentionally not called on throw, so Path.err() can report the failing leaf. exitField(): void { this.stack[this.stack.length - 1].field = undefined; } readView(n: number, fn: (view: DataView, pos: number) => number): number { if (!isNum(n) || n < 0) throw this.err(`readView: wrong length=${n}`); if (this.pos + n > this.data.length) throw this.err('readView: Unexpected end of buffer'); const res = fn(this.view, this.pos); this.markBytes(n); return res; } // read bytes by absolute offset absBytes(n: number): Uint8Array { if (!isNum(n) || n < 0 || n > this.data.length) throw new Error('Unexpected end of buffer'); return this.data.subarray(n); } finish(): void { // ReaderOpts documents allowUnreadBytes as "Allow decoding to finish with unread trailing // bytes." Prefix decoders may intentionally parse only a value from a larger byte array, so // this skips all final unread-input checks, including residual bits and pointer-read gaps. if (this.opts.allowUnreadBytes) return; if (this.bitPos) { throw this.err( `${this.bitPos} bits left after unpack: ${baseHex.encode(this.data.subarray(this.pos))}` ); } if (this.bs && !this.parent) { const notRead = Bitset.indices(this.bs, this.data.length, true); if (notRead.length) { const formatted = Bitset.range(notRead) .map( ({ pos, length }) => `(${pos}/${length})[${baseHex.encode(this.data.subarray(pos, pos + length))}]` ) .join(', '); throw this.err(`unread byte ranges: ${formatted} (total=${this.data.length})`); } else return; // all bytes read, everything is ok } // Default: no pointers enabled if (!this.isEnd()) { throw this.err( `${this.leftBytes} bytes ${this.bitPos} bits left after unpack: ${baseHex.encode( this.data.subarray(this.pos) )}` ); } } // User methods err(msg: string | Error): Error { return Path.err('Reader', this.stack, msg); } offsetReader(n: number): _Reader { if (!isNum(n) || n < 0 || n > this.data.length) throw this.err('offsetReader: Unexpected end of buffer'); return new _Reader(this.absBytes(n), this.opts, this.stack, this, n); } bytes(n: number, peek = false): Uint8Array { if (this.bitPos) throw this.err('readBytes: bitPos not empty'); if (!isNum(n) || n < 0) throw this.err(`readBytes: wrong length=${n}`); if (this.pos + n > this.data.length) throw this.err('readBytes: Unexpected end of buffer'); const slice = this.data.subarray(this.pos, this.pos + n); if (!peek) this.markBytes(n); return slice; } byte(peek = false): number { if (this.bitPos) throw this.err('readByte: bitPos not empty'); if (this.pos + 1 > this.data.length) throw this.err('readByte: Unexpected end of buffer'); const data = this.data[this.pos]; if (!peek) this.markBytes(1); return data; } get leftBytes(): number { return this.data.length - this.pos; } get totalBytes(): number { return this.data.length; } isEnd(): boolean { return this.pos >= this.data.length && !this.bitPos; } progress(): number { return this.pos * 8 - this.bitPos; } // bits are read in BE mode (left to right): (0b1000_0000).readBits(1) == 1 bits(bits: number): number { // Reject before bitwise shifts: JS coerces negative/fractional shift counts into other widths. if (!isNum(bits) || bits < 0) throw this.err(`BitReader: wrong length=${bits}`); if (bits > 32) throw this.err('BitReader: cannot read more than 32 bits in single call'); let out = 0; while (bits) { if (!this.bitPos) { this.bitBuf = this.byte(); this.bitPos = 8; } const take = Math.min(bits, this.bitPos); this.bitPos -= take; out = (out << take) | ((this.bitBuf >> this.bitPos) & (2 ** take - 1)); this.bitBuf &= 2 ** this.bitPos - 1; bits -= take; } // Fix signed integers return out >>> 0; } find(needle: Bytes, pos: number = this.pos): number | undefined { if (!isBytes(needle)) throw this.err(`find: needle is not bytes! ${needle}`); if (this.bitPos) throw this.err('find: bitPos not empty'); if (!needle.length) throw this.err(`find: needle is empty`); if (!isNum(pos) || pos < 0) throw this.err(`find: wrong pos=${pos}`); return findBytes(needle, this.data, pos); } } /** * Internal structure. Writer class for writing to a byte array. * The `stack` argument of constructor is internal, for debugging and logs. * @class Writer */ // A contiguous span of writer-owned bytes carved out of a chunk. Kept mutable so consecutive // byte()/writeView() calls extend the open run instead of pushing one buffer per write. type ChunkRun = { chunk: Uint8Array; start: number; end: number }; class _Writer implements Writer { pos: number = 0; readonly stack: PathStack; // Small writes are carved out of writer-owned chunks; caller-provided bytes() buffers are kept // by reference between them. A single realloc'd buffer was measured slower for tiny encodes // (basic/encode bench: 395ns -> 560ns), copy-on-grow is what chunking avoids. private buffers: (Bytes | ChunkRun)[] = []; // Every chunk ever allocated by this writer, so finish(clean) can zeroize them whole. private chunks: Bytes[] = []; private chunk: Uint8Array | undefined; private chunkView: DataView | undefined; // lazy: only writeView() needs it private chunkPos = 0; private run: ChunkRun | undefined; private nextChunkSize = 0; // 0 = size the first chunk to the first write ptrs: { pos: number; ptr: CoderType<number>; buffer: Bytes }[] = []; private bitBuf = 0; private bitPos = 0; private finished = false; constructor(stack: PathStack = []) { this.stack = stack; } // Reserves `len` contiguous writer-owned bytes and returns their offset inside `this.chunk`. private carve(len: number): number { if (this.chunk === undefined || this.chunk.length - this.chunkPos < len) { // Chunks grow geometrically to amortize allocations, capped to bound over-allocation and // zeroization work. The 64-byte floor keeps small encodes in one chunk (one lazy DataView). const size = Math.max(len, this.nextChunkSize, 64); this.nextChunkSize = Math.min(size * 8, 4096); this.chunk = new Uint8Array(size); this.chunkView = undefined; this.chunkPos = 0; this.run = undefined; this.chunks.push(this.chunk); } const pos = this.chunkPos; if (this.run === undefined) { this.run = { chunk: this.chunk, start: pos, end: pos + len }; this.buffers.push(this.run); } else this.run.end += len; this.chunkPos = pos + len; this.pos += len; return pos; } pushObj(obj: StructOut, objFn: _PathObjFn): void { return Path.pushObj(this.stack, obj, objFn); } enterField(field: string | number): void { const last = this.stack[this.stack.length - 1]; // A field outside any pushObj() scope or nested inside another field is a coder bug. if (last === undefined || last.field !== undefined) throw this.err(`enterField: invalid stack state, field=${field}`); last.field = field; } // Intentionally not called on throw, so Path.err() can report the failing leaf. exitField(): void { this.stack[this.stack.length - 1].field = undefined; } writeView(len: number, fn: (view: DataView, pos: number) => void): void { if (this.finished) throw this.err('buffer: finished'); if (!isNum(len) || len < 0 || len > 8) throw new Error(`wrong writeView length=${len}`); if (this.bitPos) throw this.err('writeBytes: ends with non-empty bit buffer'); const pos = this.carve(len); if (this.chunkView === undefined) this.chunkView = createView(this.chunk!); fn(this.chunkView, pos); } // User methods err(msg: string | Error): Error { // Finished-state guards call err('buffer: finished'), so err itself must not recurse there. return Path.err('Writer', this.stack, msg); } bytes(b: Bytes): void { if (this.finished) throw this.err('buffer: finished'); if (this.bitPos) throw this.err('writeBytes: ends with non-empty bit buffer'); if (!isBytes(b)) throw this.err(`writeBytes: expected Uint8Array, got ${typeof b}`); // Keep caller-provided buffers by reference until finish(); mutating them afterwards changes // the encoded output. this.buffers.push(b); // Later carved writes must not merge into a run that would be emitted before `b`. this.run = undefined; this.pos += b.length; } byte(b: number): void { if (this.finished) throw this.err('buffer: finished'); if (this.bitPos) throw this.err('writeByte: ends with non-empty bit buffer'); if (!isNum(b) || b < 0 || b > 255) throw this.err(`writeByte: wrong value=${b}`); // carve() first: it may allocate/replace this.chunk. const pos = this.carve(1); this.chunk![pos] = b; } finish(clean = true): Bytes { if (this.finished) throw this.err('buffer: finished'); if (this.bitPos) throw this.err('buffer: ends with non-empty bit buffer'); // Can't use concatBytes, because it limits amount of arguments (65K). const buffers = this.buffers; let sum = 0; for (let i = 0; i < buffers.length; i++) { const b = buffers[i]; sum += isBytes(b) ? b.length : b.end - b.start; } for (let i = 0; i < this.ptrs.length; i++) sum += this.ptrs[i].buffer.length; const buf = new Uint8Array(sum); let pad = 0; for (let i = 0; i < buffers.length; i++) { const b = buffers[i]; if (isBytes(b)) { buf.set(b, pad); pad += b.length; } else { buf.set(b.chunk.subarray(b.start, b.end), pad); pad += b.end - b.start; } } for (let i = 0; i < this.ptrs.length; i++) { buf.set(this.ptrs[i].buffer, pad); pad += this.ptrs[i].buffer.length; } for (let pos = this.pos, i = 0; i < this.ptrs.length; i++) { const ptr = this.ptrs[i]; buf.set(ptr.ptr.encode(pos), ptr.pos); pos += ptr.buffer.length; } // Cleanup if (clean) { // bytes() keeps caller-provided buffers by reference, so only writer-owned chunks are // zeroized — whole, including unused tails: same hygiene as per-buffer cleanup. for (const c of this.chunks) c.fill(0); this.chunks = []; this.chunk = undefined; this.chunkView = undefined; this.chunkPos = 0; this.run = undefined; this.buffers = []; for (const p of this.ptrs) p.buffer.fill(0); this.ptrs = []; this.finished = true; this.bitBuf = 0; } return buf; } bits(value: number, bits: number): void { if (this.finished) throw this.err('buffer: finished'); // Reject before bitwise shifts: JS coerces negative/fractional values and widths. if (!isNum(bits) || bits < 0) throw this.err(`writeBits: wrong length=${bits}`); if (bits > 32) throw this.err('writeBits: cannot write more than 32 bits in single call'); if (!isNum(value) || value < 0) throw this.err(`writeBits: wrong value=${value}`); if (value >= 2 ** bits) throw this.err(`writeBits: value (${value}) >= 2**bits (${bits})`); while (bits) { const take = Math.min(bits, 8 - this.bitPos); this.bitBuf = (this.bitBuf << take) | (value >> (bits - take)); this.bitPos += take; bits -= take; value &= 2 ** bits - 1; if (this.bitPos === 8) { this.bitPos = 0; // carve() first: it may allocate/replace this.chunk. The Uint8Array store wraps mod 256, // matching the previous `new Uint8Array([bitBuf])` semantics for int32-coerced values. const pos = this.carve(1); this.chunk![pos] = this.bitBuf; } } } } // Immutable LE<->BE const swapEndianness = (b: TArg<Bytes>): TRet<Bytes> => Uint8Array.from(b).reverse(); function checkMinimalBigintBytes( value: TArg<Bytes>, le: boolean, signed: boolean, err: (msg: string) => Error ): void { if (!value.length) return; const b = le ? swapEndianness(value) : value; const fail = () => { throw err('bigint: non-minimal encoding'); }; if (!signed) { if (b[0] === 0) fail(); return; } // Encoders emit the shortest signed two's-complement byte string that preserves the sign bit. if (b[0] === 0 && (b.length === 1 || (b[1] & 128) === 0)) fail(); if (b.length > 1 && b[0] === 255 && (b[1] & 128) !== 0) fail(); } /** Internal function for checking bit bounds of bigint in signed/unsinged form */ function checkBounds(value: bigint, bits: bigint, signed: boolean): void { if (signed) { if (bits <= _0n) throw new Error(`checkBounds: signed bits must be positive, got ${bits}`); // [-(2**(32-1)), 2**(32-1)-1] const signBit = _2n ** (bits - _1n); if (value < -signBit || value >= signBit) throw new Error(`value out of signed bounds. Expected ${-signBit} <= ${value} < ${signBit}`); } else { // [0, 2**32-1] const max = _2n ** bits; if (_0n > value || value >= max) throw new Error(`value out of unsigned bounds. Expected 0 <= ${value} < ${max}`); } } function _wrap<T>(inner: TArg<BytesCoderStream<T>>): CoderType<T> { const _inner = inner as BytesCoderStream<T>; return { // NOTE: we cannot export validate here, since it is likely mistake. // Raw inner throws propagate unchanged; path-aware errors must use w.err/r.err or validate(). encodeStream: _inner.encodeStream, decodeStream: _inner.decodeStream, size: _inner.size, encode: (value: T): TRet<Bytes> => { const w = new _Writer(); _inner.encodeStream(w, value); return w.finish() as TRet<Bytes>; }, decode: (data: TArg<Bytes>, opts: ReaderOpts = {}): T => { if (!isBytes(data)) throw new TypeError(`decode: expected Uint8Array, got ${typeof data}`); const r = new _Reader(data, opts); const res = _inner.decodeStream(r); r.finish(); return res; }, }; } /** * Validates a value before encoding and after decoding using a provided function. * @param inner - The inner CoderType. * @param fn - The validation function. * @returns CoderType which check value with validation function. * @throws On wrong inner coder or validator argument types. {@link TypeError} * @example * Reject values outside the accepted range during both encode and decode. * ```ts * import * as P from 'micro-packed'; * const val = (n: number) => { * if (n > 10) throw new Error(`${n} > 10`); * return n; * }; * * // Checks that values are <= 10 during encoding and decoding. * const RangedInt = P.validate(P.U32LE, val); * ``` */ export function validate<T>(inner: CoderType<T>, fn: Validate<T>): CoderType<T> { if (!isCoder(inner)) throw new TypeError(`validate: invalid inner value ${inner}`); if (typeof fn !== 'function') throw new TypeError('validate: fn should be function'); return _wrap({ size: inner.size, encodeStream: (w: TArg<Writer>, value: T) => { let res; try { res = fn(value); } catch (e) { // Validator callbacks are caller code: if they throw non-Error garbage, diagnostics are on // them. Review policy: "if they throw garbage, then it is on them". throw w.err(e as Error); } inner.encodeStream(w, res); }, decodeStream: (r: TArg<Reader>): T => { const res = inner.decodeStream(r); try { return fn(res); } catch (e) { throw r.err(e as Error); } }, }); } /** * Wraps a stream encoder into a generic encoder and optionally validation function * @param inner - Stream coder with optional validation hook. * @returns The wrapped CoderType. * @throws On wrong wrapped stream-coder shapes. {@link TypeError} * @example * Start from stream methods, then add validation if needed. * ```ts * import * as P from 'micro-packed'; * const U8 = P.wrap({ * encodeStream: (w, value) => w.byte(value), * decodeStream: (r) => r.byte(), * }); * const checkedU8 = P.wrap({ * encodeStream: (w, value) => w.byte(value), * decodeStream: (r) => r.byte(), * validate: (n: number) => { * if (n > 10) throw new Error(`${n} > 10`); * return n; * } * }); * ``` */ // Keep this as a plain contextual object type instead of TArg<>/TRet<> helpers: // recursive object mapping breaks unannotated encodeStream/decodeStream parameters, // and _wrap() already returns the CoderType<T> shape without byte-array normalization. export const wrap = <T>(inner: { size?: number; encodeStream: (w: Writer, value: T) => void; decodeStream: (r: Reader) => T; validate?: Validate<T>; }): CoderType<T> => { const _inner = inner as BytesCoderStream<T> & { validate?: Validate<T> }; // Public wrap() is the boundary for raw stream coders; reject malformed shapes before a // half-constructed coder fails later during encode/decode. if (!isPlainObject(_inner)) throw new TypeError(`wrap: invalid inner value ${_inner}`); if (typeof _inner.encodeStream !== 'function') throw new TypeError('wrap: encodeStream should be function'); if (typeof _inner.decodeStream !== 'function') throw new TypeError('wrap: decodeStream should be function'); if (_inner.size !== undefined && (!isNum(_inner.size) || _inner.size < 0)) throw new TypeError(`wrap: invalid size ${_inner.size}`); if (_inner.validate !== undefined && typeof _inner.validate !== 'function') throw new TypeError('wrap: validate should be function'); const res = _wrap(_inner); return _inner.validate !== undefined ? validate(res, _inner.validate) : res; }; const isBaseCoder = (elm: any) => isPlainObject(elm) && typeof elm.decode === 'function' && typeof elm.encode === 'function'; /** * Checks if the given value is a CoderType. * @param elm - The value to check. * @returns True if the value is a CoderType, false otherwise. * @example * Guard unknown values before calling encode/decode helpers on them. * ```ts * import { isCoder, U8 } from 'micro-packed'; * isCoder(U8); * ``` */ export function isCoder<T>(elm: any): elm is CoderType<T> { return ( isPlainObject(elm) && isBaseCoder(elm) && typeof elm.encodeStream === 'function' && typeof elm.decodeStream === 'function' && (elm.size === undefined || (isNum(elm.size) && elm.size >= 0)) ); } // Coders (like in @scure/base) for common operations /** * Base coder for working with dictionaries (records, objects, key-value map) * Dictionary is dynamic type like: `[key: string, value: any][]` * @returns base coder that encodes/decodes between arrays of key-value tuples and dictionaries. * @example * Convert between tuple entries and a plain object record. * ```ts * import * as P from 'micro-packed'; * const dict: P.CoderType<Record<string, number>> = P.apply( * P.array(P.U16BE, P.tuple([P.cstring, P.U32LE] as const)), * P.coders.dict() * ); * ``` */ function dict<T>(): BaseCoder<[string, T][], Record<string, T>> { return { encode: (from: [string, T][]): Record<string, T> => { if (!Array.isArray(from)) throw new Error('array expected'); const to: Record<string, T> = {}; const seen = new Set<string>(); for (const item of from) { if (!Array.isArray(item) || item.length !== 2) throw new Error(`array of two elements expected`); const name = item[0]; const value = item[1]; validateFieldName(name, 'dict: key'); // Stored undefined is still a present key, so duplicate detection cannot inspect values. if (seen.has(name)) throw new Error(`key(${name}) appears twice in struct`); seen.add(name); to[name] = value; } return to; }, decode: (to: Record<string, T>): [string, T][] => { if (!isPlainObject(to)) throw new Error(`expected plain object, got ${to}`); for (const name in to) validateFieldName(name, 'dict: key'); return Object.entries(to); }, }; } /** * Safely converts bigint to number. * Sometimes pointers / tags use u64 or other big numbers which cannot be represented by number, * but we still can use them since real value will be smaller than u32 */ const numberBigint: BaseCoder<bigint, number> = /* @__PURE__ */ Object.freeze({ encode: (from: bigint): number => { if (typeof from !== 'bigint') throw new Error(`expected bigint, got ${typeof from}`); if (from > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error(`element bigger than MAX_SAFE_INTEGER=${from}`); // Number() silently rounds bigint values outside the safe integer range on either side. if (from