UNPKG

sflow

Version:

sflow is a powerful and highly-extensible library designed for processing and manipulating streams of data effortlessly. Inspired by the functional programming paradigm, it provides a rich set of utilities for transforming streams, including chunking, fil

1,509 lines (1,438 loc) 46.9 kB
import { t as __exportAll } from "./chunk-BYypO7fO.js"; import { a as maps, i as flatMaps, n as lines, o as chunkIfs, r as throughs, t as skips } from "./skips-o3ebh_7B.js"; import DIE from "phpdie"; import { from as wseFrom, mergeAll as wseMerge, toArray as wseToArray, toPromise as wseToPromise } from "web-streams-extensions"; import replaceAsync from "string-replace-async"; import { unwind } from "unwind-array"; //#region src/andIgnoreError.ts function andIgnoreError(regex) { return (error) => error?.message?.match(regex) ? null : DIE(error); } //#endregion //#region src/asyncMaps.ts const asyncMaps = (fn, options = {}) => { let i = 0; const tasks = /* @__PURE__ */ new Map(); return new TransformStream({ transform: async (chunk, ctrl) => { const id = i++; tasks.set(id, (async () => fn(chunk, id))().then((data) => ({ id, data }))); if (tasks.size >= (options.concurrency ?? 16)) { const { id, data } = await Promise.race(tasks.values()); tasks.delete(id); ctrl.enqueue(data); } }, flush: async (ctrl) => { while (tasks.size) { const { id, data } = await Promise.race(tasks.values()); tasks.delete(id); ctrl.enqueue(data); } } }); }; //#endregion //#region src/never.ts const _never = new Promise(() => {}); const never = () => _never; //#endregion //#region src/cacheLists.ts /** * 1. cache whole list once upstream flushed * 2. Replace the stream with cached list if exist * * Set ttl in your store settings * * This step should place at near the output end. * @deprecated use cacheSkips to reduce cache size */ function cacheLists(store, _options) { const { key = (/* @__PURE__ */ new Error()).stack ?? DIE("missing cache key") } = typeof _options === "string" ? { key: _options } : _options ?? {}; const chunks = []; const cacheHitPromise = store.has?.(key) || store.get(key); let hitflag = false; return new TransformStream({ start: async (ctrl) => { if (!await cacheHitPromise) return; const cached = await store.get(key); if (!cached) return; cached.map((c) => ctrl.enqueue(c)); hitflag = true; }, transform: async (chunk, ctrl) => { if (await cacheHitPromise || hitflag) { ctrl.terminate(); return never(); } chunks.push(chunk); ctrl.enqueue(chunk); }, flush: async () => await store.set(key, chunks) }); } //#endregion //#region src/cacheSkips.ts /** * Assume Stream content is ordered plain json object, (class is not supported) * And new element always insert into head * * Only emit unmet contents * * Once flow done, cache latest contents in windowSize, and skip from cached content next time * * */ function cacheSkips(store, _options) { const { key = (/* @__PURE__ */ new Error()).stack ?? DIE("missing cache key"), windowSize = 1 } = typeof _options === "string" ? { key: _options } : _options ?? {}; const chunks = []; const cachePromise = store.get(key); return new TransformStream({ transform: async (chunk, ctrl) => { const cache = await cachePromise; const chunked = JSON.stringify(chunk); const inf404 = (idx) => idx == null || idx < 0 ? Infinity : idx; const hitCache = (item) => JSON.stringify(item) === chunked; const cachedContents = cache?.slice(inf404(cache.findIndex(hitCache))); if (cachedContents?.length) { await store.set(key, [...chunks, ...cachedContents].slice(0, windowSize)); ctrl.terminate(); return await never(); } chunks.push(chunk); ctrl.enqueue(chunk); }, flush: async () => { await store.set(key, chunks.slice(0, windowSize)); } }); } //#endregion //#region src/utils.ts /** Find minimum element by key function — O(N) instead of O(N log N) */ function minBy(fn, arr) { let min = arr[0]; for (let i = 1; i < arr.length; i++) if (fn(arr[i]) < fn(min)) min = arr[i]; return min; } /** Find maximum element by key function — O(N) instead of O(N log N) */ function maxBy(fn, arr) { let max = arr[0]; for (let i = 1; i < arr.length; i++) if (fn(arr[i]) > fn(max)) max = arr[i]; return max; } /** Deep equality check for plain JSON values (replaces rambda's equals) */ function deepEquals(a, b) { if (a === b) return true; if (a == null || b == null) return false; if (typeof a !== typeof b) return false; if (Array.isArray(a)) { if (!Array.isArray(b) || a.length !== b.length) return false; return a.every((v, i) => deepEquals(v, b[i])); } if (typeof a === "object") { const ka = Object.keys(a); const kb = Object.keys(b); if (ka.length !== kb.length) return false; return ka.every((k) => deepEquals(a[k], b[k])); } return false; } //#endregion //#region src/cacheTails.ts /** * Assume Stream content is ordered plain json object, (class is not supported) * And new element always insert into head * * Set ttl in your store settings * * 1. cache whole list once upstream flushed * 2. Stop upstream and Continue with cached list once head matched * * This step should place at near the output end. */ function cacheTails(store, _options) { const { key = (/* @__PURE__ */ new Error()).stack ?? DIE("missing cache key") } = typeof _options === "string" ? { key: _options } : _options ?? {}; const chunks = []; const cachePromise = Promise.withResolvers(); const t = new TransformStream(); const w = t.writable.getWriter(); return { writable: new WritableStream({ start: async () => cachePromise.resolve(await store.get(key) ?? []), write: async (chunk, ctrl) => { const cache = await cachePromise.promise; if (cache && deepEquals(chunk, cache[0])) { await store.set(key, [...chunks, ...cache]); for await (const item of cache) await w.write(item); await w.close(); ctrl.error(/* @__PURE__ */ new Error("cached")); return await never(); } chunks.push(chunk); await w.write(chunk); }, close: async () => { await store.set(key, [...chunks]); await w.close(); }, abort: () => w.abort() }), readable: t.readable }; } //#endregion //#region src/chunkBys.ts /** chunk items by compareFn, group items with same Ord */ function chunkBys(compareFn) { const chunks = []; let lastOrder; return new TransformStream({ transform: async (chunk, ctrl) => { const order = await compareFn(chunk); if (lastOrder && lastOrder !== order) ctrl.enqueue(chunks.splice(0, Infinity)); chunks.push(chunk); lastOrder = order; }, flush: async (ctrl) => void (chunks.length && ctrl.enqueue(chunks)) }); } //#endregion //#region src/chunkIntervals.ts /** * Collect items into lists, but collect item[] in interval (ms) * Note: will emit all */ function chunkIntervals(interval = 0) { const chunks = []; let id = null; return new TransformStream({ start: (ctrl) => { id = setInterval(() => ctrl.enqueue(chunks.splice(0, Infinity)), interval); }, transform: async (chunk) => { chunks.push(chunk); }, flush: async (ctrl) => { if (chunks.length) ctrl.enqueue(chunks.splice(0, Infinity)); if (id !== null) clearInterval(id); } }); } //#endregion //#region src/chunks.ts function chunks(n = Infinity) { const chunks = []; if (n <= 0) throw new Error("Buffer size must be greater than 0"); return new TransformStream({ transform: async (chunk, ctrl) => { chunks.push(chunk); if (chunks.length >= n) ctrl.enqueue(chunks.splice(0, Infinity)); }, flush: async (ctrl) => void (chunks.length && ctrl.enqueue(chunks)) }); } //#endregion //#region src/froms.ts const toStream = (src) => src instanceof ReadableStream ? src : wseFrom(src ?? []); //#endregion //#region src/nils.ts function nils() { return new WritableStream(); } function nil() { return null; } //#endregion //#region src/concats.ts /** * return a transform stream that concats streams from sources * don't get confused with mergeStream * concats : returns a TransformStream, which also concats upstream * concatStream: returns a ReadableStream, which doesnt have upstream */ const concats = (srcs) => { if (!srcs) return new TransformStream(); const upstream = new TransformStream(); return { writable: upstream.writable, readable: concatStream([upstream.readable, concatStream(srcs)]) }; }; /** * return a readable stream that concats streams from sources * don't get confused with concats * concatStream: returns a ReadableStream, which doesnt have upstream * concats : returns a TransformStream, which also concats upstream */ const concatStream = (srcs) => { if (!srcs) return new ReadableStream({ start: (c) => c.close() }); const t = new TransformStream(); const w = t.writable.getWriter(); toStream(srcs).pipeThrough(maps(toStream)).pipeThrough(maps(async (s) => { const r = s.getReader(); while (true) { const { value, done } = await r.read(); if (done) break; await w.write(value); } })).pipeTo(nils()).then(() => w.close()).catch((reason) => w.abort(reason)); return t.readable; }; //#endregion //#region src/confluences.ts /** Confluence of multiple flow sources by breadth first */ /** @deprecated use confluencesBy */ const confluences = ({ order = "breadth" } = {}) => { if (order !== "breadth") DIE("not implemented"); const { writable, readable: sources } = new TransformStream(); const srcsQueue = []; return { writable, readable: new ReadableStream({ async pull(ctrl) { while (true) { const src = await (async () => { const r = sources.getReader(); const { done, value: src } = await r.read(); r.releaseLock(); if (done) return srcsQueue.shift(); return src; })(); if (!src) return ctrl.close(); const r = src.getReader(); const { done, value } = await r.read(); r.releaseLock(); if (done) continue; srcsQueue.push(src); ctrl.enqueue(value); return; } } }) }; }; //#endregion //#region src/convolves.ts /** * @example * convolves(2) * [1,2,3,4] => [[1,2],[2,3],[3,4]] * convolves(3) * [1,2,3,4] => [[1,2,3],[2,3,4]] */ function convolves(n) { const buffer = []; return new TransformStream({ transform(chunk, controller) { buffer.push(chunk); if (buffer.length > n) buffer.shift(); if (buffer.length === n) controller.enqueue([...buffer]); }, flush(controller) { while (buffer.length > 1) { buffer.shift(); if (buffer.length === n) controller.enqueue([...buffer]); } } }); } //#endregion //#region src/debounces.ts function debounces(t) { let id = null; return new TransformStream({ transform: async (chunk, ctrl) => { if (id) clearTimeout(id); id = setTimeout(() => { ctrl.enqueue(chunk); id = null; }, t); }, flush: async () => { while (id) await new Promise((r) => setTimeout(r, t / 2)); } }); } //#endregion //#region src/filters.ts /** * Filter a stream, return a stream with non-null values. * If a function is provided, it will be called with each value, * and if it returns a truthy value, the value will be included in the output * stream. * If no function is provided, it will filter out null and undefined values. * This is useful for creating a stream that only contains non-null and non-undefined values. * * Note: empty string is considered a valid value and will not be filtered out. * * if you want to filter out empty strings, you can "Boolean" the filter function. * * @param fn Optional function to determine if a value should be included. * @returns A TransformStream that filters the input stream. * @template T The type of the input stream. * @template NonNullable<T> The type of the output stream, which will not include * null or undefined values. * @example */ const filters = (fn) => { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { if (fn) { if (await fn(chunk, i++)) ctrl.enqueue(chunk); } else if (!(void 0 === chunk || null === chunk)) ctrl.enqueue(chunk); } }); }; //#endregion //#region src/finds.ts function finds(predicate) { let index = 0; let found = false; return new TransformStream({ async transform(chunk, controller) { if (found) return; if (await predicate(chunk, index++)) { found = true; controller.enqueue(chunk); controller.terminate(); } } }); } //#endregion //#region src/flats.ts /** * Flattens an array of arrays into a stream of values. * If the input is an empty array, it will throw an error. * To avoid this error, you can add a `.filter(array => array.length)` stage before * * @returns A TransformStream that flattens an array of arrays into a stream of values. */ function flats() { return composers(filters((e) => e.length)).by(new TransformStream({ transform: async (a, ctrl) => { a.map((e) => ctrl.enqueue(e)); } })); } //#endregion //#region src/forEachs.ts /** For each loop on stream, you can modify the item by x.property = 123 */ function forEachs(fn, options) { const concurrency = options?.concurrency ?? 1; if (concurrency === 1) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const ret = fn(chunk, i++); if (ret instanceof Promise) await ret; ctrl.enqueue(chunk); } }); } let i = 0; const promises = []; const chunks = []; return new TransformStream({ transform: async (chunk, ctrl) => { promises.push(fn(chunk, i++)); chunks.push(chunk); if (promises.length >= concurrency) { await promises.shift(); const chunk = chunks.shift(); if (chunk === void 0) throw new Error("chunks.shift() returned undefined"); ctrl.enqueue(chunk); } }, flush: async (ctrl) => { while (promises.length) { await promises.shift(); const chunk = chunks.shift(); if (chunk === void 0) throw new Error("chunks.shift() returned undefined"); ctrl.enqueue(chunk); } } }); } //#endregion //#region src/heads.ts function heads(n = 1) { return new TransformStream({ transform: async (chunk, ctrl) => { return n-- > 0 ? ctrl.enqueue(chunk) : await never(); } }); } //#endregion //#region src/limits.ts /** Currently will not pipe down more items after count satisfied */ function limits(n, { terminate = true } = {}) { return new TransformStream({ transform: async (chunk, ctrl) => { ctrl.enqueue(chunk); if (--n === 0) { if (terminate) ctrl.terminate(); return never(); } }, flush: () => {} }, { highWaterMark: 1 }, { highWaterMark: 0 }); } //#endregion //#region src/unpromises.ts /** unwrap promises of readable stream */ function unpromises(promise) { const tr = new TransformStream(); (async () => { await (await promise).pipeTo(tr.writable); })().catch((error) => tr.writable.abort(error).catch(() => {})); return tr.readable; } //#endregion //#region src/bys.ts function bys(arg) { if (!arg) return new TransformStream(); if (typeof arg !== "function") return bys((s) => s.pipeThrough(arg)); const fn = arg; const { writable, readable } = new TransformStream(); return { writable, readable: unpromises(fn(readable)) }; } //#endregion //#region src/peeks.ts /** Note: peeks will not await peek fn, use forEachs if you want downstream tobe awaited */ function peeks(fn) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { ctrl.enqueue(chunk); const ret = fn(chunk, i++); if (ret instanceof Promise) await ret; } }); } //#endregion //#region src/logs.ts /** * log the value and index and return as original stream, handy to debug. * Note: stream won't await the log function. */ function logs(mapFn = (s, _i) => s) { return bys(peeks(async (e, i) => { const ret = mapFn(e, i); const val = ret instanceof Promise ? await ret : ret; console.log(typeof val === "string" ? val.replace(/\n$/, "") : val); })); } //#endregion //#region src/mapAddFields.ts function mapAddFields(key, fn) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => ctrl.enqueue({ ...chunk, [key]: await fn(chunk, i++) }) }); } //#endregion //#region src/mapMixins.ts function mapMixins(fn) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => ctrl.enqueue({ ...chunk, ...await fn(chunk, i++) }) }); } //#endregion //#region src/streamAsyncIterator.ts async function* streamAsyncIterator() { const reader = this.getReader(); try { while (true) { const { done, value } = await reader.read(); if (done) return; yield value; } } finally { reader.releaseLock(); } } //#endregion //#region src/mergeStream.ts /** * return a readable stream that merges streams from sources * don't get confused with merges * mergeStream: returns a ReadableStream, which doesnt have upstream * merges : returns a TransformStream, which also merges upstream */ const mergeStream = (...srcs) => { if (!srcs.length) return new ReadableStream({ start: (c) => c.close() }); if (srcs.length === 1) return toStream(srcs[0]); const t = new TransformStream(); const w = t.writable.getWriter(); const streams = srcs.map(toStream); Promise.all(streams.map(async (s) => { for await (const chunk of Object.assign(s, { [Symbol.asyncIterator]: streamAsyncIterator })) await w.write(chunk); })).then(async () => w.close()).catch((error) => { streams.forEach((e) => void e.cancel(error).catch(() => {})); return w.abort(error).catch(() => {}); }); return t.readable; }; //#endregion //#region src/emptyStream.ts const emptyStream = () => new ReadableStream({ start: (c) => c.close() }); //#endregion //#region src/mergeStreamsBy.ts function mergeStreamsBy(transform, sources) { if (!sources) return ((srcs) => mergeStreamsBy(transform, srcs)); if (!sources.length) return emptyStream(); const streams = sources.map((s) => toStream(s)); const readers = streams.map((stream) => stream.getReader()); let slots = streams.map(() => null); return new ReadableStream({ pull: async (ctrl) => { await Promise.all(readers.map(async (reader, i) => slots[i] ??= await reader.read())); slots = await transform([...slots], ctrl); if (slots.length !== streams.length) DIE("slot length mismatch"); } }); } function mergeStreamsByAscend(ordFn, sources) { if (!sources) return ((sources) => mergeStreamsByAscend(ordFn, sources)); let lastEmit = null; return mergeStreamsBy(async (slots, ctrl) => { const cands = slots.filter((e) => e?.done === false).map((e) => e.value); if (!cands.length) { ctrl.close(); return []; } const peak = minBy(ordFn, cands); const index = slots.findIndex((e) => e?.done === false && e?.value === peak); if (lastEmit && ordFn(lastEmit.value) > ordFn(peak)) throw new Error("MergeStreamError: one of sources is not ordered by ascending", { cause: { prevOrd: ordFn(lastEmit.value), currOrd: ordFn(peak), prev: lastEmit.value, curr: peak } }); lastEmit = { value: peak }; ctrl.enqueue(peak); return [ ...slots.slice(0, index), null, ...slots.slice(index + 1) ]; }, sources); } function mergeStreamsByDescend(ordFn, sources) { if (!sources) return ((srcs) => mergeStreamsByDescend(ordFn, srcs)); let lastEmit = null; return mergeStreamsBy(async (slots, ctrl) => { const cands = slots.filter((e) => e?.done === false).map((e) => e.value); if (!cands.length) { ctrl.close(); return []; } const peak = maxBy(ordFn, cands); const index = slots.findIndex((e) => e?.done === false && e?.value === peak); if (lastEmit && ordFn(lastEmit.value) < ordFn(peak)) DIE(new Error("MergeStreamError: one of sources is not ordered by descending", { cause: { prevOrd: ordFn(lastEmit.value), currOrd: ordFn(peak), prev: lastEmit.value, curr: peak } })); lastEmit = { value: peak }; ctrl.enqueue(peak); return [ ...slots.slice(0, index), null, ...slots.slice(index + 1) ]; }, sources); } //#endregion //#region src/wseMerges.ts const wseMerges = wseMerge; //#endregion //#region src/parallels.ts const parallels = (...srcs) => wseMerges()(wseFrom(srcs)); //#endregion //#region src/merges.ts /** * return a transform stream that merges streams from sources * don't get confused with mergeStreamw * merges : returns a TransformStream, which also merges upstream * mergeStream: returns a ReadableStream, which doesnt have upstream */ const merges = (...srcs) => { if (!srcs.length) return new TransformStream(); const upstream = new TransformStream(); return { writable: upstream.writable, readable: parallels(upstream.readable, ...srcs.map(toStream)) }; }; //#endregion //#region src/pMaps.ts const pMaps = (fn, options = {}) => { let i = 0; const promises = []; return new TransformStream({ transform: async (chunk, ctrl) => { promises.push(fn(chunk, i++)); if (promises.length >= (options.concurrency ?? 16)) ctrl.enqueue(await promises.shift()); }, flush: async (ctrl) => { while (promises.length) ctrl.enqueue(await promises.shift()); } }); }; //#endregion //#region src/portals.ts /** pipe upstream through a PortalStream, aka. TransformStream<T, T> */ const portals = (arg) => { if (!arg) return new TransformStream(); if (typeof arg !== "function") return throughs((s) => s.pipeThrough(arg)); const fn = arg; const { writable, readable } = new TransformStream(); return { writable, readable: fn(readable) }; }; //#endregion //#region src/reduceEmits.ts const reduceEmits = (fn, _state) => { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const { next, emit } = await fn(_state, chunk, i++); _state = next; ctrl.enqueue(emit); } }); }; //#endregion //#region src/reduces.ts /** return undefined to skip emit */ const reduces = (fn, state) => { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const ret = fn(state, chunk, i++); state = await (ret instanceof Promise ? await ret : ret); ctrl.enqueue(state); } }); }; //#endregion //#region src/riffles.ts function riffles(sep) { let last; return new TransformStream({ transform: (chunk, ctrl) => { if (last !== void 0) { ctrl.enqueue(last); ctrl.enqueue(sep); } last = chunk; }, flush: (ctrl) => ctrl.enqueue(last) }); } //#endregion //#region src/slices.ts function slices(start = 0, end = Infinity) { const count = end - start; const { readable, writable } = new TransformStream(); return { writable, readable: readable.pipeThrough(skips(start)).pipeThrough(limits(count)) }; } //#endregion //#region src/strings.ts const matchs = (matcher) => { return new TransformStream({ transform: (chunk, ctrl) => ctrl.enqueue(chunk.match(matcher)) }); }; const matchAlls = (matcher) => { return new TransformStream({ transform: (chunk, ctrl) => ctrl.enqueue(chunk.matchAll(matcher)) }); }; const replaces = (searchValue, replacement) => { return maps((s) => typeof replacement === "string" ? s.replace(searchValue, replacement) : replaceAsync(s, searchValue, replacement)); }; const replaceAlls = (searchValue, replacement) => { return maps((s) => typeof replacement === "string" ? s.replaceAll(searchValue, replacement) : replaceAsync(s, searchValue, replacement)); }; //#endregion //#region src/tails.ts function tails(n = 1) { const chunks = []; return new TransformStream({ transform: (chunk) => { chunks.push(chunk); if (chunks.length > n) chunks.shift(); }, flush: (ctrl) => { chunks.map((e) => ctrl.enqueue(e)); } }); } //#endregion //#region src/takeWhiles.ts /** * Takes elements from the stream while the predicate returns true. * Stops and terminates the stream when the predicate returns false. * * @param fn Predicate function that determines whether to continue taking elements * @returns A TransformStream that takes elements while the predicate is true * @template T The type of the input/output stream * @example * sflow([1, 2, 3, 4, 5]) * .takeWhile(x => x < 4) * .toArray() // [1, 2, 3] */ function takeWhiles(fn, { terminate = true } = {}) { let i = 0; let stopped = false; return new TransformStream({ transform: async (chunk, ctrl) => { if (stopped) return; if (await fn(chunk, i++)) ctrl.enqueue(chunk); else { stopped = true; if (terminate) { ctrl.terminate(); return never(); } } }, flush: () => {} }, { highWaterMark: 1 }, { highWaterMark: 0 }); } //#endregion //#region src/tees.ts /** * Create a TransformStream that tees (forks) the stream, passing one branch to the given function or writable stream. * @warning Uses `ReadableStream.tee()` internally. If the forked branch is consumed slower than the main branch, * the tee buffer will grow unboundedly in memory (no backpressure between tee branches). */ const tees = (arg) => { if (!arg) return new TransformStream(); if (arg instanceof WritableStream) return tees((s) => s.pipeTo(arg)); const fn = arg; const { writable, readable } = new TransformStream(); const [a, b] = readable.tee(); fn(a); return { writable, readable: b }; }; //#endregion //#region src/terminates.ts function terminates(signal) { return throughs((r) => r.pipeThrough(new TransformStream(), { signal })); } //#endregion //#region src/throttles.ts /** * ```ts * drop=true: Drop chunks by interval, will ensure last item emit by default. * set keepLast=false to drop last * * @example * if you dont want item drops, please use forEach to limit speed * sth like: * sflow().forEach(sleep(1000)).forEach(sleep(1000)).log().done() */ function throttles(interval, { drop = false, keepLast = true } = {}) { let timerId = null; let cdPromise = Promise.withResolvers(); let lasts = []; return new TransformStream({ transform: async (chunk, ctrl) => { if (timerId) { if (keepLast) lasts = [chunk]; if (drop) return; await cdPromise.promise; } lasts = []; ctrl.enqueue(chunk); [cdPromise, timerId] = [Promise.withResolvers(), setTimeout(() => { timerId = null; cdPromise.resolve(); }, interval)]; }, flush: async (ctrl) => { while (timerId) await new Promise((r) => setTimeout(r, interval / 2)); lasts.map((e) => ctrl.enqueue(e)); } }); } //#endregion //#region src/toLatest.ts /** * Convert a readable stream to it's latest value * * the return object is dynamic updated */ function toLatests(r) { let latest; let nextPromise = Promise.withResolvers(); r.pipeTo(new WritableStream({ write: (value) => { latest = value; nextPromise.resolve(value); nextPromise = Promise.withResolvers(); }, close: () => { nextPromise.resolve(void 0); } })); return { get latest() { if (latest === void 0) return nextPromise.promise; return Promise.resolve(latest); }, get next() { return nextPromise.promise; } }; } //#endregion //#region src/uniqs.ts /** uniq by a new Set(), note: use === to compare */ const uniqs = () => { const set = /* @__PURE__ */ new Set(); return throughs((s) => s.pipeThrough(filters((x) => { if (set.has(x)) return false; set.add(x); return true; }))); }; /** uniq by a new Map(), Note: use === to compare keys */ const uniqBys = (keyFn) => { const set = /* @__PURE__ */ new Set(); return throughs((s) => s.pipeThrough(filters(async (x) => { const key = await keyFn(x); if (set.has(key)) return false; set.add(key); return true; }))); }; //#endregion //#region src/unwinds.ts function unwinds(key) { return flatMaps((e) => unwind(e, { path: key })); } //#endregion //#region src/sflow.ts /** stream flow */ function sflow(...srcs) { let r = srcs.length === 1 ? toStream(srcs[0]) : concatStream(srcs); return Object.assign(r, { _type: null, get readable() { return r; }, portal: (...args) => sflow(r.pipeThrough(portals(...args))), through: (...args) => sflow(r.pipeThrough(_throughs(...args))), by: (...args) => sflow(r.pipeThrough(_throughs(...args))), byLazy: (t) => _byLazy(r, t), cacheSkip: (...args) => sflow(r).byLazy(cacheSkips(...args)), cacheList: (...args) => sflow(r).byLazy(cacheLists(...args)), cacheTail: (...args) => sflow(r).byLazy(cacheTails(...args)), chunkBy: (...args) => sflow(r.pipeThrough(chunkBys(...args))), chunkIf: (...args) => sflow(r.pipeThrough(chunkIfs(...args))), buffer: (...args) => sflow(r.pipeThrough(chunks(...args))), chunk: (...args) => sflow(r.pipeThrough(chunks(...args))), convolve: (...args) => sflow(r.pipeThrough(convolves(...args))), abort: (...args) => sflow(r.pipeThrough(terminates(...args))), chunkInterval: (...args) => sflow(r.pipeThrough(chunkIntervals(...args))), interval: (...args) => sflow(r.pipeThrough(chunkIntervals(...args))), debounce: (...args) => sflow(r.pipeThrough(debounces(...args))), filter: (...args) => sflow(r.pipeThrough(filters(...args))), find: (...args) => sflow(r.pipeThrough(finds(...args))), flatMap: (...args) => sflow(r.pipeThrough(flatMaps(...args))), flat: (...args) => sflow(r).by(flats(...args)), join: (...args) => sflow(r.pipeThrough(riffles(...args))), match: (...args) => sflow(r.pipeThrough(matchs(...args))), matchAll: (...args) => sflow(r.pipeThrough(matchAlls(...args))), replace: (...args) => sflow(r.pipeThrough(replaces(...args))), replaceAll: (...args) => sflow(r.pipeThrough(replaceAlls(...args))), merge: (...args) => sflow(r.pipeThrough(merges(...args))), concat: (srcs) => sflow(r.pipeThrough(concats(srcs))), confluence: (...args) => sflow(r.pipeThrough(confluences(...args))), confluenceByZip: () => sflow(r).by(confluences()), confluenceByConcat: () => sflow(r).by((srcs) => concatStream(srcs)), confluenceByParallel: () => sflow(r).by((srcs) => sflow(srcs).toArray().then((srcs) => mergeStream(...srcs))).confluence(), confluenceByAscend: (ordFn) => sflow(r).chunk().map((srcs) => mergeStreamsByAscend(ordFn, srcs)).confluence(), confluenceByDescend: (ordFn) => sflow(r).chunk().map((srcs) => mergeStreamsByDescend(ordFn, srcs)).confluence(), limit: (...args) => sflow(r).byLazy(limits(...args)), head: (...args) => sflow(r.pipeThrough(heads(...args))), map: (...args) => sflow(r.pipeThrough(maps(...args))), mapAddField: (...args) => sflow(r.pipeThrough(mapAddFields(...args))), mapMixin: (...args) => sflow(r.pipeThrough(mapMixins(...args))), log: (...args) => sflow(r.pipeThrough(logs(...args))), uniq: (...args) => sflow(r.pipeThrough(uniqs(...args))), unique: (...args) => sflow(r.pipeThrough(uniqs(...args))), uniqBy: (...args) => sflow(r.pipeThrough(uniqBys(...args))), unwind: (...args) => sflow(r.pipeThrough(unwinds(...args))), asyncMap: (...args) => sflow(r.pipeThrough(asyncMaps(...args))), pMap: (...args) => sflow(r.pipeThrough(pMaps(...args))), peek: (...args) => sflow(r.pipeThrough(peeks(...args))), riffle: (...args) => sflow(r.pipeThrough(riffles(...args))), forEach: (...args) => sflow(r.pipeThrough(forEachs(...args))), reduce: (...args) => sflow(r.pipeThrough(reduces(...args))), reduceEmit: (...args) => sflow(r.pipeThrough(reduceEmits(...args))), skip: (...args) => sflow(r.pipeThrough(skips(...args))), slice: (...args) => sflow(r.pipeThrough(slices(...args))), tail: (...args) => sflow(r.pipeThrough(tails(...args))), takeWhile: (...args) => sflow(r.pipeThrough(takeWhiles(...args))), tees: (...args) => sflow(r.pipeThrough(_tees(...args))), forkTo: (...args) => sflow(r.pipeThrough(_tees(...args))), fork: () => { let b; [r, b] = r.tee(); return sflow(b); }, throttle: (...args) => sflow(r.pipeThrough(throttles(...args))), preventAbort: () => sflow(r.pipeThrough(throughs(), { preventAbort: true })), preventClose: () => sflow(r.pipeThrough(throughs(), { preventClose: true })), preventCancel: () => sflow(r.pipeThrough(throughs(), { preventCancel: true })), onStart: (start) => sflow(r).by(new TransformStream({ start })), onTransform: (transform) => sflow(r).by(new TransformStream({ transform })), onFlush: (flush) => sflow(r).by(new TransformStream({ flush })), done: () => r.pipeTo(nils()), end: (dst = nils()) => r.pipeTo(dst), to: (dst = nils()) => r.pipeTo(dst), run: () => r.pipeTo(nils()), toEnd: () => r.pipeTo(nils()), toNil: () => r.pipeTo(nils()), toArray: () => wseToArray(r), toCount: async () => { let i = 0; const d = r.getReader(); while (!(await d.read()).done) i++; return i; }, toFirst: () => wseToPromise(sflow(r).limit(1, { terminate: true })), toFirstMatch: (predicate) => wseToPromise(sflow(r).find(predicate)), toLast: () => wseToPromise(sflow(r).tail(1)), toExactlyOne: async () => { const a = await wseToArray(r); if (a.length !== 1) DIE(`Expect exactly 1 Item, but got ${a.length}`); return a[0]; }, toOne: async () => { const a = await wseToArray(r); if (a.length > 1) DIE(`Expect only 1 Item, but got ${a.length}`); return a[0]; }, toAtLeastOne: async () => { const a = await wseToArray(r); if (a.length > 1) DIE(`Expect only 1 Item, but got ${a.length}`); if (a.length < 1) DIE(`Expect at least 1 Item, but got ${a.length}`); return a[0]; }, toLatest: () => toLatests(sflow(r)), toLog: (...args) => sflow(r.pipeThrough(logs(...args))).done(), lines: (...args) => sflow(r.pipeThrough(lines(...args))), toResponse: (init) => new Response(r, init), text: (init) => new Response(r.pipeThrough(new TextEncoderStream()), init).text(), json: (init) => new Response(r.pipeThrough(new TextEncoderStream()), init).json(), blob: (init) => new Response(sflow(r), init).blob(), arrayBuffer: (init) => new Response(r, init).arrayBuffer(), [Symbol.asyncIterator]: streamAsyncIterator }); } /** * Internal tees helper that wraps forked branches in sflow. * @warning Uses `ReadableStream.tee()` internally. If the forked branch is consumed slower than the main branch, * the tee buffer will grow unboundedly in memory (no backpressure between tee branches). */ const _tees = (arg) => { if (!arg) return new TransformStream(); if (arg instanceof WritableStream) return tees((s) => s.pipeTo(arg)); const fn = arg; const { writable, readable } = new TransformStream(); const [a, b] = readable.tee(); fn(sflow(a)); return { writable, readable: b }; }; const _throughs = (arg) => { if (!arg) return new TransformStream(); if (typeof arg !== "function") return throughs((s) => s.pipeThrough(arg)); const fn = arg; const { writable, readable } = new TransformStream(); return { writable, readable: sflow(fn(sflow(readable))) }; }; /** * byLazy is a lazy version of by, it only pull upstream when downstream pull. * * Warning: does not work with empty stream yet. */ function _byLazy(r, t) { const reader = r.getReader(); const tw = t.writable.getWriter(); const tr = t.readable.getReader(); return sflow(new ReadableStream({ start: async (ctrl) => { (async () => { while (true) { const { done, value } = await tr.read(); if (done) return ctrl.close(); ctrl.enqueue(value); } })(); }, pull: async (_ctrl) => { const { done, value } = await reader.read(); if (done) return tw.close(); await tw.write(value); }, cancel: async (r) => { reader.cancel(r); tr.cancel(r); } }, { highWaterMark: 0 })); } //#endregion //#region src/chunkOverlaps.ts function chunkOverlaps({ step, overlap }) { const chunks = []; if (step <= 0) throw new Error("step must be greater than 0"); if (overlap < 0) throw new Error("overlap must be greater than or equal to 0"); return new TransformStream({ transform: async (chunk, ctrl) => { chunks.push(chunk); if (chunks.length >= step + overlap) ctrl.enqueue([...chunks.splice(0, step), ...chunks.slice(0, overlap)]); }, flush: async (ctrl) => void (chunks.length && ctrl.enqueue(chunks)) }); } //#endregion //#region src/chunkTransforms.ts /** * Creates a TransformStream that processes chunks using provided transformers. * * @template T - The type of chunk that the TransformStream will process. * * @param {Object} options - The options object containing transformer functions. * @param {ChunkTransformer<T>} [options.start] - The transformer function to run when the stream is started. * @param {ChunkTransformer<T>} [options.transform] - The transformer function to run for each chunk, current chunk will be the last element. * @param {ChunkTransformer<T>} [options.flush] - The transformer function to run when the stream is flushed. * * @returns A new TransformStream that applies the provided transformers. */ function chunkTransforms(options) { let chunks = []; const { start, transform, flush } = options; return new TransformStream({ start: async (ctrl) => { if (start) chunks = await start(chunks, ctrl); }, transform: async (chunk, ctrl) => { chunks.push(chunk); if (transform) chunks = await transform(chunks, ctrl); }, flush: async (ctrl) => { if (flush) chunks = await flush(chunks, ctrl); } }); } //#endregion //#region src/distributeBys.ts const distributeBys = (groupFn) => { const streams = /* @__PURE__ */ new Map(); const { writable: srcs, readable } = new TransformStream(); const { writable, readable: chunks } = new TransformStream(); const w = srcs.getWriter(); chunks.pipeThrough(pMaps(async (chunk) => { const ord = await groupFn(chunk); if (!streams.has(ord)) await (async () => { const t = new TransformStream(); await w.write(t.readable); const r = { ...t, writer: t.writable.getWriter() }; streams.set(ord, r); return r; })(); const t = streams.get(ord); if (!t) throw new Error(`Stream not found for order ${ord}`); await t.writer.write(chunk); })).pipeTo(nils()).finally(() => { w.close(); [...streams.values()].map((e) => e.writer.close()); }); return { writable, readable }; }; //#endregion //#region src/mergeAscends.ts /** * merge multiple stream by ascend order, assume all input stream is sorted by ascend * output stream will be sorted by ascend too. * * if one of input stream is not sorted by ascend, it will throw an error. * * @param ordFn a function to get the order of input * @param srcs a list of input stream * @returns a new stream that merge all input stream by ascend order * @deprecated use {@link mergeStreamsByAscend}, this will be removed in next major version */ const mergeAscends = (ordFn, _srcs) => { if (!_srcs) return ((srcs) => mergeAscends(ordFn, srcs)); return sflow(new ReadableStream({ pull: async (ctrl) => { const srcs = await sflow(_srcs).toArray(); const slots = srcs.map(() => void 0); const pendingSlotRemoval = srcs.map(() => void 0); const drains = srcs.map(() => false); let lastMinValue; await Promise.all(srcs.map(async (src, i) => { for await (const value of sflow(src)) { while (slots[i] !== void 0) { if (shiftMinValueIfFull()) continue; pendingSlotRemoval[i] = Promise.withResolvers(); await pendingSlotRemoval[i]?.promise; } slots[i] = { value }; shiftMinValueIfFull(); } drains[i] = true; pendingSlotRemoval.map((e) => e?.resolve()); await Promise.all(pendingSlotRemoval.map((e) => e?.promise)); if (drains.every(Boolean)) { while (slots.some((e) => e !== void 0)) shiftMinValueIfFull(); ctrl.close(); } function shiftMinValueIfFull() { if (!slots.every((slot, i) => slot !== void 0 || drains[i])) return false; const minValue = minBy(ordFn, slots.flatMap((e) => e !== void 0 ? [e] : []).map((e) => e.value)); const minIndex = slots.findIndex((e) => e?.value === minValue); if (lastMinValue !== void 0) { if (ordFn(lastMinValue) > ordFn(minValue)) DIE(` MergeAscendError: one of source stream is not ascending ordered. stream index: ${minIndex} prev: ${ordFn(lastMinValue)} prev: ${JSON.stringify(lastMinValue)} curr: ${ordFn(minValue)} curr: ${JSON.stringify(minValue)} `); } lastMinValue = minValue; ctrl.enqueue(minValue); slots[minIndex] = void 0; pendingSlotRemoval[minIndex]?.resolve(); pendingSlotRemoval[minIndex] = void 0; return true; } })); } }, { highWaterMark: 0 })); }; /** * merge multiple stream by ascend order, assume all input stream is sorted by ascend * output stream will be sorted by ascend too. * * if one of input stream is not sorted by ascend, it will throw an error. * * @param ordFn a function to get the order of input * @param srcs a list of input stream * @returns a new stream that merge all input stream by ascend order * @deprecated use {@link mergeStreamsByAscend} */ const mergeDescends = (ordFn, _srcs) => { if (!_srcs) return ((srcs) => mergeDescends(ordFn, srcs)); return toStream(new ReadableStream({ pull: async (ctrl) => { const srcs = await sflow(_srcs).toArray(); const slots = srcs.map(() => void 0); const pendingSlotRemoval = srcs.map(() => void 0); const drains = srcs.map(() => false); let lastMaxValue; await Promise.all(srcs.map(async (src, i) => { const stream = toStream(src); const streamIterator = Object.assign(stream, { [Symbol.asyncIterator]: streamAsyncIterator }); for await (const value of streamIterator) { while (slots[i] !== void 0) { if (shiftMaxValueIfFull()) continue; pendingSlotRemoval[i] = Promise.withResolvers(); await pendingSlotRemoval[i]?.promise; } slots[i] = { value }; shiftMaxValueIfFull(); } drains[i] = true; pendingSlotRemoval.map((e) => e?.resolve()); await Promise.all(pendingSlotRemoval.map((e) => e?.promise)); if (drains.every(Boolean)) { while (slots.some((e) => e !== void 0)) shiftMaxValueIfFull(); ctrl.close(); } function shiftMaxValueIfFull() { if (!slots.every((slot, i) => slot !== void 0 || drains[i])) return false; const maxValue = maxBy(ordFn, slots.flatMap((e) => e !== void 0 ? [e] : []).map((e) => e.value)); const maxIndex = slots.findIndex((e) => e?.value === maxValue); if (lastMaxValue !== void 0) { if (ordFn(lastMaxValue) < ordFn(maxValue)) DIE(` MergeDescendError: one of source stream is not descending ordered. stream index: ${maxIndex} prev: ${ordFn(lastMaxValue)} prev: ${JSON.stringify(lastMaxValue)} curr: ${ordFn(maxValue)} curr: ${JSON.stringify(maxValue)} `); } lastMaxValue = maxValue; ctrl.enqueue(maxValue); slots[maxIndex] = void 0; pendingSlotRemoval[maxIndex]?.resolve(); pendingSlotRemoval[maxIndex] = void 0; return true; } })); } }, { highWaterMark: 0 })); }; //#endregion //#region src/pageStream.ts function pageStream(initialQuery, fetcher) { let query = null; return new ReadableStream({ pull: async (ctrl) => { if (query === null) query = { value: await initialQuery }; const ret = fetcher(query.value); const { data, next } = ret instanceof Promise ? await ret : ret; if (data !== void 0) ctrl.enqueue(data); if (null == next) return ctrl.close(); query.value = next; } }, { highWaterMark: 0 }); } //#endregion //#region src/pageFlow.ts function pageFlow(initialCursor, fetcher) { return sflow(pageStream(initialCursor, fetcher)); } //#endregion //#region src/rangeStream.ts function rangeStream(...args) { const [min, max] = args[1] != null ? [args[0], args[1]] : [0, args[0]]; let i = min; return new ReadableStream({ pull: (ctrl) => { ctrl.enqueue(i); if (++i >= max) ctrl.close(); } }, { highWaterMark: 0 }); } function rangeFlow(...args) { return sflow(rangeStream(...args)); } //#endregion //#region src/retry.ts function retry(onError, fn) { return async (...args) => { let attempt = 0; const retryable = async (...retryArgs) => { try { const ret = fn(...retryArgs); return ret instanceof Promise ? await ret : ret; } catch (error) { attempt++; const ret = onError(error, attempt, retryable, ...retryArgs); return ret instanceof Promise ? await ret : ret; } }; return retryable(...args); }; } //#endregion //#region src/sfTemplate.ts /** sflow Template */ function sfTemplate(tsa, ...args) { return sflow(...tsa.flatMap((str) => [sflow([str]), args.shift() || []])); } const sfT = sfTemplate; //#endregion //#region src/svector.ts /** stream vector */ const svector = (...src) => sflow(src); //#endregion //#region src/composers.ts function composers(stream) { return Object.assign(stream, { by: (appendStream) => composers({ writable: stream.writable, readable: stream.readable.pipeThrough(appendStream) }) }); } //#endregion //#region src/sf.ts var sf_exports = /* @__PURE__ */ __exportAll({ aborts: () => terminates, buffers: () => chunks, bys: () => bys, cacheLists: () => cacheLists, cacheSkips: () => cacheSkips, cacheTails: () => cacheTails, chunkBys: () => chunkBys, chunkIfs: () => chunkIfs, chunkIntervals: () => chunkIntervals, chunkOverlaps: () => chunkOverlaps, chunkTransforms: () => chunkTransforms, chunks: () => chunks, composers: () => composers, concatStream: () => concatStream, concats: () => concats, confluences: () => confluences, debounces: () => debounces, distributesBy: () => distributeBys, filters: () => filters, flatMaps: () => flatMaps, flats: () => flats, forEachs: () => forEachs, from: () => sflow, intervals: () => chunkIntervals, joins: () => merges, lines: () => lines, logs: () => logs, mapAddFields: () => mapAddFields, maps: () => maps, matchAlls: () => matchAlls, matchs: () => matchs, mergeAscends: () => mergeAscends, mergeDescends: () => mergeDescends, mergeStream: () => mergeStream, mergeStreamsBy: () => mergeStreamsBy, mergeStreamsByAscend: () => mergeStreamsByAscend, mergeStreamsByDescend: () => mergeStreamsByDescend, merges: () => merges, nil: () => nil, nils: () => nils, pMaps: () => pMaps, pageFlow: () => pageFlow, pageStream: () => pageStream, peeks: () => peeks, portals: () => portals, rangeFlow: () => rangeFlow, rangeStream: () => rangeStream, reduces: () => reduces, replaceAlls: () => replaceAlls, replaces: () => replaces, retry: () => retry, sf: () => sflow, sfT: () => sfT, sfTemplate: () => sfTemplate, sflow: () => sflow, skips: () => skips, slices: () => slices, snoflow: () => sflow, streamAsyncIterator: () => streamAsyncIterator, sv: () => svector, svector: () => svector, tails: () => tails, tees: () => tees, throttles: () => throttles, throughs: () => throughs, uniqBys: () => uniqBys, uniqs: () => uniqs, unpromises: () => unpromises, unwinds: () => unwinds }); //#endregion //#region src/index.ts var src_default = sflow; //#endregion export { terminates as aborts, andIgnoreError, chunks as buffers, chunks, bys, cacheLists, cacheSkips, cacheTails, chunkBys, chunkIfs, chunkIntervals, chunkIntervals as intervals, chunkOverlaps, chunkTransforms, concatStream, concats, confluences, debounces, src_default as default, distributeBys as distributesBy, filters, finds, flatMaps, flats, forEachs, merges as joins, merges, lines, logs, mapAddFields, maps, matchAlls, matchs, mergeAscends, mergeDescends, mergeStream, mergeStreamsBy, mergeStreamsByAscend, mergeStreamsByDescend, nil, nils, pMaps, pageFlow, pageStream, peeks, portals, rangeFlow, rangeStream, reduces, replaceAlls, replaces, retry, sf_exports as sf, sfT, sfTemplate, sflow, sflow as snoflow, skips, slices, streamAsyncIterator, svector as sv, svector, tails, takeWhiles, tees, throttles, throughs, uniqBys, uniqs, unpromises, unwinds }; //# sourceMappingURL=index.js.map