UNPKG

sflow

Version:

sflow is a powerful and highly-extensible library designed for processing and manipulating streams of data effortlessly. Inspired by the functional programming paradigm, it provides a rich set of utilities for transforming streams, including chunking, fil

1,578 lines (1,501 loc) 53.9 kB
var sflow = (function(exports, phpdie, string_replace_async, unwind_array) { Object.defineProperties(exports, { __esModule: { value: true }, [Symbol.toStringTag]: { value: 'Module' } }); //#region \0rolldown/runtime.js var __create = Object.create; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; var __getOwnPropNames = Object.getOwnPropertyNames; var __getProtoOf = Object.getPrototypeOf; var __hasOwnProp = Object.prototype.hasOwnProperty; var __exportAll = (all, no_symbols) => { let target = {}; for (var name in all) { __defProp(target, name, { get: all[name], enumerable: true }); } if (!no_symbols) { __defProp(target, Symbol.toStringTag, { value: "Module" }); } return target; }; var __copyProps = (to, from, except, desc) => { if (from && typeof from === "object" || typeof from === "function") { for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) { key = keys[i]; if (!__hasOwnProp.call(to, key) && key !== except) { __defProp(to, key, { get: ((k) => from[k]).bind(null, key), enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); } } } return to; }; var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, mod)); //#endregion phpdie = __toESM(phpdie); string_replace_async = __toESM(string_replace_async); //#region src/andIgnoreError.ts function andIgnoreError(regex) { return (error) => error?.message?.match(regex) ? null : (0, phpdie.default)(error); } //#endregion //#region src/asyncMaps.ts const asyncMaps = (fn, options = {}) => { let i = 0; const tasks = /* @__PURE__ */ new Map(); return new TransformStream({ transform: async (chunk, ctrl) => { const id = i++; tasks.set(id, (async () => fn(chunk, id))().then((data) => ({ id, data }))); if (tasks.size >= (options.concurrency ?? 16)) { const { id, data } = await Promise.race(tasks.values()); tasks.delete(id); ctrl.enqueue(data); } }, flush: async (ctrl) => { while (tasks.size) { const { id, data } = await Promise.race(tasks.values()); tasks.delete(id); ctrl.enqueue(data); } } }); }; //#endregion //#region src/never.ts const _never = new Promise(() => {}); const never = () => _never; //#endregion //#region src/cacheLists.ts /** * 1. cache whole list once upstream flushed * 2. Replace the stream with cached list if exist * * Set ttl in your store settings * * This step should place at near the output end. * @deprecated use cacheSkips to reduce cache size */ function cacheLists(store, _options) { const { key = (/* @__PURE__ */ new Error()).stack ?? (0, phpdie.default)("missing cache key") } = typeof _options === "string" ? { key: _options } : _options ?? {}; const chunks = []; const cacheHitPromise = store.has?.(key) || store.get(key); let hitflag = false; return new TransformStream({ start: async (ctrl) => { if (!await cacheHitPromise) return; const cached = await store.get(key); if (!cached) return; cached.map((c) => ctrl.enqueue(c)); hitflag = true; }, transform: async (chunk, ctrl) => { if (await cacheHitPromise || hitflag) { ctrl.terminate(); return never(); } chunks.push(chunk); ctrl.enqueue(chunk); }, flush: async () => await store.set(key, chunks) }); } //#endregion //#region src/cacheSkips.ts /** * Assume Stream content is ordered plain json object, (class is not supported) * And new element always insert into head * * Only emit unmet contents * * Once flow done, cache latest contents in windowSize, and skip from cached content next time * * */ function cacheSkips(store, _options) { const { key = (/* @__PURE__ */ new Error()).stack ?? (0, phpdie.default)("missing cache key"), windowSize = 1 } = typeof _options === "string" ? { key: _options } : _options ?? {}; const chunks = []; const cachePromise = store.get(key); return new TransformStream({ transform: async (chunk, ctrl) => { const cache = await cachePromise; const chunked = JSON.stringify(chunk); const inf404 = (idx) => idx == null || idx < 0 ? Infinity : idx; const hitCache = (item) => JSON.stringify(item) === chunked; const cachedContents = cache?.slice(inf404(cache.findIndex(hitCache))); if (cachedContents?.length) { await store.set(key, [...chunks, ...cachedContents].slice(0, windowSize)); ctrl.terminate(); return await never(); } chunks.push(chunk); ctrl.enqueue(chunk); }, flush: async () => { await store.set(key, chunks.slice(0, windowSize)); } }); } //#endregion //#region src/utils.ts /** Find minimum element by key function — O(N) instead of O(N log N) */ function minBy(fn, arr) { let min = arr[0]; for (let i = 1; i < arr.length; i++) if (fn(arr[i]) < fn(min)) min = arr[i]; return min; } /** Find maximum element by key function — O(N) instead of O(N log N) */ function maxBy(fn, arr) { let max = arr[0]; for (let i = 1; i < arr.length; i++) if (fn(arr[i]) > fn(max)) max = arr[i]; return max; } /** Deep equality check for plain JSON values (replaces rambda's equals) */ function deepEquals(a, b) { if (a === b) return true; if (a == null || b == null) return false; if (typeof a !== typeof b) return false; if (Array.isArray(a)) { if (!Array.isArray(b) || a.length !== b.length) return false; return a.every((v, i) => deepEquals(v, b[i])); } if (typeof a === "object") { const ka = Object.keys(a); const kb = Object.keys(b); if (ka.length !== kb.length) return false; return ka.every((k) => deepEquals(a[k], b[k])); } return false; } //#endregion //#region src/cacheTails.ts /** * Assume Stream content is ordered plain json object, (class is not supported) * And new element always insert into head * * Set ttl in your store settings * * 1. cache whole list once upstream flushed * 2. Stop upstream and Continue with cached list once head matched * * This step should place at near the output end. */ function cacheTails(store, _options) { const { key = (/* @__PURE__ */ new Error()).stack ?? (0, phpdie.default)("missing cache key") } = typeof _options === "string" ? { key: _options } : _options ?? {}; const chunks = []; const cachePromise = Promise.withResolvers(); const t = new TransformStream(); const w = t.writable.getWriter(); return { writable: new WritableStream({ start: async () => cachePromise.resolve(await store.get(key) ?? []), write: async (chunk, ctrl) => { const cache = await cachePromise.promise; if (cache && deepEquals(chunk, cache[0])) { await store.set(key, [...chunks, ...cache]); for await (const item of cache) await w.write(item); await w.close(); ctrl.error(/* @__PURE__ */ new Error("cached")); return await never(); } chunks.push(chunk); await w.write(chunk); }, close: async () => { await store.set(key, [...chunks]); await w.close(); }, abort: () => w.abort() }), readable: t.readable }; } //#endregion //#region src/chunkBys.ts /** chunk items by compareFn, group items with same Ord */ function chunkBys(compareFn) { const chunks = []; let lastOrder; return new TransformStream({ transform: async (chunk, ctrl) => { const order = await compareFn(chunk); if (lastOrder && lastOrder !== order) ctrl.enqueue(chunks.splice(0, Infinity)); chunks.push(chunk); lastOrder = order; }, flush: async (ctrl) => void (chunks.length && ctrl.enqueue(chunks)) }); } //#endregion //#region src/chunkIfs.ts /** chunk items if condition is true */ function chunkIfs(predicate, { inclusive = false } = {}) { const chunks = []; let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const cond = await predicate(chunk, i++, chunks); if (!inclusive && !cond) { if (chunks.length) ctrl.enqueue(chunks.splice(0, Infinity)); } chunks.push(chunk); if (!cond) ctrl.enqueue(chunks.splice(0, Infinity)); }, flush: async (ctrl) => { if (chunks.length) ctrl.enqueue(chunks); } }); } //#endregion //#region src/chunkIntervals.ts /** * Collect items into lists, but collect item[] in interval (ms) * Note: will emit all */ function chunkIntervals(interval = 0) { const chunks = []; let id = null; return new TransformStream({ start: (ctrl) => { id = setInterval(() => ctrl.enqueue(chunks.splice(0, Infinity)), interval); }, transform: async (chunk) => { chunks.push(chunk); }, flush: async (ctrl) => { if (chunks.length) ctrl.enqueue(chunks.splice(0, Infinity)); if (id !== null) clearInterval(id); } }); } //#endregion //#region src/chunks.ts function chunks(n = Infinity) { const chunks = []; if (n <= 0) throw new Error("Buffer size must be greater than 0"); return new TransformStream({ transform: async (chunk, ctrl) => { chunks.push(chunk); if (chunks.length >= n) ctrl.enqueue(chunks.splice(0, Infinity)); }, flush: async (ctrl) => void (chunks.length && ctrl.enqueue(chunks)) }); } //#endregion //#region src/froms.ts const toStream = (src) => src instanceof ReadableStream ? src : (0, web_streams_extensions.from)(src ?? []); //#endregion //#region src/maps.ts function maps(fn, options) { const concurrency = options?.concurrency ?? 1; if (concurrency === 1) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const ret = fn(chunk, i++); const val = ret instanceof Promise ? await ret : ret; ctrl.enqueue(val); } }); } let i = 0; const promises = []; return new TransformStream({ transform: async (chunk, ctrl) => { promises.push(fn(chunk, i++)); if (promises.length >= concurrency) ctrl.enqueue(await promises.shift()); }, flush: async (ctrl) => { while (promises.length) ctrl.enqueue(await promises.shift()); } }); } //#endregion //#region src/nils.ts function nils() { return new WritableStream(); } function nil() { return null; } //#endregion //#region src/concats.ts /** * return a transform stream that concats streams from sources * don't get confused with mergeStream * concats : returns a TransformStream, which also concats upstream * concatStream: returns a ReadableStream, which doesnt have upstream */ const concats = (srcs) => { if (!srcs) return new TransformStream(); const upstream = new TransformStream(); return { writable: upstream.writable, readable: concatStream([upstream.readable, concatStream(srcs)]) }; }; /** * return a readable stream that concats streams from sources * don't get confused with concats * concatStream: returns a ReadableStream, which doesnt have upstream * concats : returns a TransformStream, which also concats upstream */ const concatStream = (srcs) => { if (!srcs) return new ReadableStream({ start: (c) => c.close() }); const t = new TransformStream(); const w = t.writable.getWriter(); toStream(srcs).pipeThrough(maps(toStream)).pipeThrough(maps(async (s) => { const r = s.getReader(); while (true) { const { value, done } = await r.read(); if (done) break; await w.write(value); } })).pipeTo(nils()).then(() => w.close()).catch((reason) => w.abort(reason)); return t.readable; }; //#endregion //#region src/confluences.ts /** Confluence of multiple flow sources by breadth first */ /** @deprecated use confluencesBy */ const confluences = ({ order = "breadth" } = {}) => { if (order !== "breadth") (0, phpdie.default)("not implemented"); const { writable, readable: sources } = new TransformStream(); const srcsQueue = []; return { writable, readable: new ReadableStream({ async pull(ctrl) { while (true) { const src = await (async () => { const r = sources.getReader(); const { done, value: src } = await r.read(); r.releaseLock(); if (done) return srcsQueue.shift(); return src; })(); if (!src) return ctrl.close(); const r = src.getReader(); const { done, value } = await r.read(); r.releaseLock(); if (done) continue; srcsQueue.push(src); ctrl.enqueue(value); return; } } }) }; }; //#endregion //#region src/convolves.ts /** * @example * convolves(2) * [1,2,3,4] => [[1,2],[2,3],[3,4]] * convolves(3) * [1,2,3,4] => [[1,2,3],[2,3,4]] */ function convolves(n) { const buffer = []; return new TransformStream({ transform(chunk, controller) { buffer.push(chunk); if (buffer.length > n) buffer.shift(); if (buffer.length === n) controller.enqueue([...buffer]); }, flush(controller) { while (buffer.length > 1) { buffer.shift(); if (buffer.length === n) controller.enqueue([...buffer]); } } }); } //#endregion //#region src/debounces.ts function debounces(t) { let id = null; return new TransformStream({ transform: async (chunk, ctrl) => { if (id) clearTimeout(id); id = setTimeout(() => { ctrl.enqueue(chunk); id = null; }, t); }, flush: async () => { while (id) await new Promise((r) => setTimeout(r, t / 2)); } }); } //#endregion //#region src/filters.ts /** * Filter a stream, return a stream with non-null values. * If a function is provided, it will be called with each value, * and if it returns a truthy value, the value will be included in the output * stream. * If no function is provided, it will filter out null and undefined values. * This is useful for creating a stream that only contains non-null and non-undefined values. * * Note: empty string is considered a valid value and will not be filtered out. * * if you want to filter out empty strings, you can "Boolean" the filter function. * * @param fn Optional function to determine if a value should be included. * @returns A TransformStream that filters the input stream. * @template T The type of the input stream. * @template NonNullable<T> The type of the output stream, which will not include * null or undefined values. * @example */ const filters = (fn) => { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { if (fn) { if (await fn(chunk, i++)) ctrl.enqueue(chunk); } else if (!(void 0 === chunk || null === chunk)) ctrl.enqueue(chunk); } }); }; //#endregion //#region src/finds.ts function finds(predicate) { let index = 0; let found = false; return new TransformStream({ async transform(chunk, controller) { if (found) return; if (await predicate(chunk, index++)) { found = true; controller.enqueue(chunk); controller.terminate(); } } }); } //#endregion //#region src/flatMaps.ts function flatMaps(fn) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const ret = fn(chunk, i++); (ret instanceof Promise ? await ret : ret).map((e) => ctrl.enqueue(e)); } }); } //#endregion //#region src/flats.ts /** * Flattens an array of arrays into a stream of values. * If the input is an empty array, it will throw an error. * To avoid this error, you can add a `.filter(array => array.length)` stage before * * @returns A TransformStream that flattens an array of arrays into a stream of values. */ function flats() { return composers(filters((e) => e.length)).by(new TransformStream({ transform: async (a, ctrl) => { a.map((e) => ctrl.enqueue(e)); } })); } //#endregion //#region src/forEachs.ts /** For each loop on stream, you can modify the item by x.property = 123 */ function forEachs(fn, options) { const concurrency = options?.concurrency ?? 1; if (concurrency === 1) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const ret = fn(chunk, i++); if (ret instanceof Promise) await ret; ctrl.enqueue(chunk); } }); } let i = 0; const promises = []; const chunks = []; return new TransformStream({ transform: async (chunk, ctrl) => { promises.push(fn(chunk, i++)); chunks.push(chunk); if (promises.length >= concurrency) { await promises.shift(); const chunk = chunks.shift(); if (chunk === void 0) throw new Error("chunks.shift() returned undefined"); ctrl.enqueue(chunk); } }, flush: async (ctrl) => { while (promises.length) { await promises.shift(); const chunk = chunks.shift(); if (chunk === void 0) throw new Error("chunks.shift() returned undefined"); ctrl.enqueue(chunk); } } }); } //#endregion //#region src/heads.ts function heads(n = 1) { return new TransformStream({ transform: async (chunk, ctrl) => { return n-- > 0 ? ctrl.enqueue(chunk) : await never(); } }); } //#endregion //#region src/limits.ts /** Currently will not pipe down more items after count satisfied */ function limits(n, { terminate = true } = {}) { return new TransformStream({ transform: async (chunk, ctrl) => { ctrl.enqueue(chunk); if (--n === 0) { if (terminate) ctrl.terminate(); return never(); } }, flush: () => {} }, { highWaterMark: 1 }, { highWaterMark: 0 }); } //#endregion //#region src/throughs.ts /** pipe upstream through a transform stream */ const throughs = (arg) => { if (!arg) return new TransformStream(); if (typeof arg !== "function") return throughs((s) => s.pipeThrough(arg)); const fn = arg; const { writable, readable } = new TransformStream(); return { writable, readable: fn(readable) }; }; //#endregion //#region src/lines.ts /** split string stream into lines stream, handy to concat LLM's tokens stream into line by line stream or split a long string by lines */ const lines = ({ EOL = "KEEP" } = {}) => { const CRLFMap = { KEEP: "$1", LF: "\n", CRLF: "\r\n", NONE: "" }; return throughs((r) => r.pipeThrough(flatMaps((s) => s.split(/(?<=\n)/g))).pipeThrough(chunkIfs((ch) => ch.indexOf("\n") === -1, { inclusive: true })).pipeThrough(maps((chunks) => chunks.join("").replace(/(\r?\n?)$/, CRLFMap[EOL])))); }; //#endregion //#region src/unpromises.ts /** unwrap promises of readable stream */ function unpromises(promise) { const tr = new TransformStream(); (async () => { await (await promise).pipeTo(tr.writable); })().catch((error) => tr.writable.abort(error).catch(() => {})); return tr.readable; } //#endregion //#region src/bys.ts function bys(arg) { if (!arg) return new TransformStream(); if (typeof arg !== "function") return bys((s) => s.pipeThrough(arg)); const fn = arg; const { writable, readable } = new TransformStream(); return { writable, readable: unpromises(fn(readable)) }; } //#endregion //#region src/peeks.ts /** Note: peeks will not await peek fn, use forEachs if you want downstream tobe awaited */ function peeks(fn) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { ctrl.enqueue(chunk); const ret = fn(chunk, i++); if (ret instanceof Promise) await ret; } }); } //#endregion //#region src/logs.ts /** * log the value and index and return as original stream, handy to debug. * Note: stream won't await the log function. */ function logs(mapFn = (s, _i) => s) { return bys(peeks(async (e, i) => { const ret = mapFn(e, i); const val = ret instanceof Promise ? await ret : ret; console.log(typeof val === "string" ? val.replace(/\n$/, "") : val); })); } //#endregion //#region src/mapAddFields.ts function mapAddFields(key, fn) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => ctrl.enqueue({ ...chunk, [key]: await fn(chunk, i++) }) }); } //#endregion //#region src/mapMixins.ts function mapMixins(fn) { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => ctrl.enqueue({ ...chunk, ...await fn(chunk, i++) }) }); } //#endregion //#region src/streamAsyncIterator.ts async function* streamAsyncIterator() { const reader = this.getReader(); try { while (true) { const { done, value } = await reader.read(); if (done) return; yield value; } } finally { reader.releaseLock(); } } //#endregion //#region src/mergeStream.ts /** * return a readable stream that merges streams from sources * don't get confused with merges * mergeStream: returns a ReadableStream, which doesnt have upstream * merges : returns a TransformStream, which also merges upstream */ const mergeStream = (...srcs) => { if (!srcs.length) return new ReadableStream({ start: (c) => c.close() }); if (srcs.length === 1) return toStream(srcs[0]); const t = new TransformStream(); const w = t.writable.getWriter(); const streams = srcs.map(toStream); Promise.all(streams.map(async (s) => { for await (const chunk of Object.assign(s, { [Symbol.asyncIterator]: streamAsyncIterator })) await w.write(chunk); })).then(async () => w.close()).catch((error) => { streams.forEach((e) => void e.cancel(error).catch(() => {})); return w.abort(error).catch(() => {}); }); return t.readable; }; //#endregion //#region src/emptyStream.ts const emptyStream = () => new ReadableStream({ start: (c) => c.close() }); //#endregion //#region src/mergeStreamsBy.ts function mergeStreamsBy(transform, sources) { if (!sources) return ((srcs) => mergeStreamsBy(transform, srcs)); if (!sources.length) return emptyStream(); const streams = sources.map((s) => toStream(s)); const readers = streams.map((stream) => stream.getReader()); let slots = streams.map(() => null); return new ReadableStream({ pull: async (ctrl) => { await Promise.all(readers.map(async (reader, i) => slots[i] ??= await reader.read())); slots = await transform([...slots], ctrl); if (slots.length !== streams.length) (0, phpdie.default)("slot length mismatch"); } }); } function mergeStreamsByAscend(ordFn, sources) { if (!sources) return ((sources) => mergeStreamsByAscend(ordFn, sources)); let lastEmit = null; return mergeStreamsBy(async (slots, ctrl) => { const cands = slots.filter((e) => e?.done === false).map((e) => e.value); if (!cands.length) { ctrl.close(); return []; } const peak = minBy(ordFn, cands); const index = slots.findIndex((e) => e?.done === false && e?.value === peak); if (lastEmit && ordFn(lastEmit.value) > ordFn(peak)) throw new Error("MergeStreamError: one of sources is not ordered by ascending", { cause: { prevOrd: ordFn(lastEmit.value), currOrd: ordFn(peak), prev: lastEmit.value, curr: peak } }); lastEmit = { value: peak }; ctrl.enqueue(peak); return [ ...slots.slice(0, index), null, ...slots.slice(index + 1) ]; }, sources); } function mergeStreamsByDescend(ordFn, sources) { if (!sources) return ((srcs) => mergeStreamsByDescend(ordFn, srcs)); let lastEmit = null; return mergeStreamsBy(async (slots, ctrl) => { const cands = slots.filter((e) => e?.done === false).map((e) => e.value); if (!cands.length) { ctrl.close(); return []; } const peak = maxBy(ordFn, cands); const index = slots.findIndex((e) => e?.done === false && e?.value === peak); if (lastEmit && ordFn(lastEmit.value) < ordFn(peak)) (0, phpdie.default)(new Error("MergeStreamError: one of sources is not ordered by descending", { cause: { prevOrd: ordFn(lastEmit.value), currOrd: ordFn(peak), prev: lastEmit.value, curr: peak } })); lastEmit = { value: peak }; ctrl.enqueue(peak); return [ ...slots.slice(0, index), null, ...slots.slice(index + 1) ]; }, sources); } //#endregion //#region src/wseMerges.ts const wseMerges = web_streams_extensions.mergeAll; //#endregion //#region src/parallels.ts const parallels = (...srcs) => wseMerges()((0, web_streams_extensions.from)(srcs)); //#endregion //#region src/merges.ts /** * return a transform stream that merges streams from sources * don't get confused with mergeStreamw * merges : returns a TransformStream, which also merges upstream * mergeStream: returns a ReadableStream, which doesnt have upstream */ const merges = (...srcs) => { if (!srcs.length) return new TransformStream(); const upstream = new TransformStream(); return { writable: upstream.writable, readable: parallels(upstream.readable, ...srcs.map(toStream)) }; }; //#endregion //#region src/pMaps.ts const pMaps = (fn, options = {}) => { let i = 0; const promises = []; return new TransformStream({ transform: async (chunk, ctrl) => { promises.push(fn(chunk, i++)); if (promises.length >= (options.concurrency ?? 16)) ctrl.enqueue(await promises.shift()); }, flush: async (ctrl) => { while (promises.length) ctrl.enqueue(await promises.shift()); } }); }; //#endregion //#region src/portals.ts /** pipe upstream through a PortalStream, aka. TransformStream<T, T> */ const portals = (arg) => { if (!arg) return new TransformStream(); if (typeof arg !== "function") return throughs((s) => s.pipeThrough(arg)); const fn = arg; const { writable, readable } = new TransformStream(); return { writable, readable: fn(readable) }; }; //#endregion //#region src/reduceEmits.ts const reduceEmits = (fn, _state) => { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const { next, emit } = await fn(_state, chunk, i++); _state = next; ctrl.enqueue(emit); } }); }; //#endregion //#region src/reduces.ts /** return undefined to skip emit */ const reduces = (fn, state) => { let i = 0; return new TransformStream({ transform: async (chunk, ctrl) => { const ret = fn(state, chunk, i++); state = await (ret instanceof Promise ? await ret : ret); ctrl.enqueue(state); } }); }; //#endregion //#region src/riffles.ts function riffles(sep) { let last; return new TransformStream({ transform: (chunk, ctrl) => { if (last !== void 0) { ctrl.enqueue(last); ctrl.enqueue(sep); } last = chunk; }, flush: (ctrl) => ctrl.enqueue(last) }); } //#endregion //#region src/skips.ts function skips(n = 1) { return new TransformStream({ transform: async (chunk, ctrl) => { if (n <= 0) ctrl.enqueue(chunk); else n--; } }); } //#endregion //#region src/slices.ts function slices(start = 0, end = Infinity) { const count = end - start; const { readable, writable } = new TransformStream(); return { writable, readable: readable.pipeThrough(skips(start)).pipeThrough(limits(count)) }; } //#endregion //#region src/strings.ts const matchs = (matcher) => { return new TransformStream({ transform: (chunk, ctrl) => ctrl.enqueue(chunk.match(matcher)) }); }; const matchAlls = (matcher) => { return new TransformStream({ transform: (chunk, ctrl) => ctrl.enqueue(chunk.matchAll(matcher)) }); }; const replaces = (searchValue, replacement) => { return maps((s) => typeof replacement === "string" ? s.replace(searchValue, replacement) : (0, string_replace_async.default)(s, searchValue, replacement)); }; const replaceAlls = (searchValue, replacement) => { return maps((s) => typeof replacement === "string" ? s.replaceAll(searchValue, replacement) : (0, string_replace_async.default)(s, searchValue, replacement)); }; //#endregion //#region src/tails.ts function tails(n = 1) { const chunks = []; return new TransformStream({ transform: (chunk) => { chunks.push(chunk); if (chunks.length > n) chunks.shift(); }, flush: (ctrl) => { chunks.map((e) => ctrl.enqueue(e)); } }); } //#endregion //#region src/takeWhiles.ts /** * Takes elements from the stream while the predicate returns true. * Stops and terminates the stream when the predicate returns false. * * @param fn Predicate function that determines whether to continue taking elements * @returns A TransformStream that takes elements while the predicate is true * @template T The type of the input/output stream * @example * sflow([1, 2, 3, 4, 5]) * .takeWhile(x => x < 4) * .toArray() // [1, 2, 3] */ function takeWhiles(fn, { terminate = true } = {}) { let i = 0; let stopped = false; return new TransformStream({ transform: async (chunk, ctrl) => { if (stopped) return; if (await fn(chunk, i++)) ctrl.enqueue(chunk); else { stopped = true; if (terminate) { ctrl.terminate(); return never(); } } }, flush: () => {} }, { highWaterMark: 1 }, { highWaterMark: 0 }); } //#endregion //#region src/tees.ts /** * Create a TransformStream that tees (forks) the stream, passing one branch to the given function or writable stream. * @warning Uses `ReadableStream.tee()` internally. If the forked branch is consumed slower than the main branch, * the tee buffer will grow unboundedly in memory (no backpressure between tee branches). */ const tees = (arg) => { if (!arg) return new TransformStream(); if (arg instanceof WritableStream) return tees((s) => s.pipeTo(arg)); const fn = arg; const { writable, readable } = new TransformStream(); const [a, b] = readable.tee(); fn(a); return { writable, readable: b }; }; //#endregion //#region src/terminates.ts function terminates(signal) { return throughs((r) => r.pipeThrough(new TransformStream(), { signal })); } //#endregion //#region src/throttles.ts /** * ```ts * drop=true: Drop chunks by interval, will ensure last item emit by default. * set keepLast=false to drop last * * @example * if you dont want item drops, please use forEach to limit speed * sth like: * sflow().forEach(sleep(1000)).forEach(sleep(1000)).log().done() */ function throttles(interval, { drop = false, keepLast = true } = {}) { let timerId = null; let cdPromise = Promise.withResolvers(); let lasts = []; return new TransformStream({ transform: async (chunk, ctrl) => { if (timerId) { if (keepLast) lasts = [chunk]; if (drop) return; await cdPromise.promise; } lasts = []; ctrl.enqueue(chunk); [cdPromise, timerId] = [Promise.withResolvers(), setTimeout(() => { timerId = null; cdPromise.resolve(); }, interval)]; }, flush: async (ctrl) => { while (timerId) await new Promise((r) => setTimeout(r, interval / 2)); lasts.map((e) => ctrl.enqueue(e)); } }); } //#endregion //#region src/toLatest.ts /** * Convert a readable stream to it's latest value * * the return object is dynamic updated */ function toLatests(r) { let latest; let nextPromise = Promise.withResolvers(); r.pipeTo(new WritableStream({ write: (value) => { latest = value; nextPromise.resolve(value); nextPromise = Promise.withResolvers(); }, close: () => { nextPromise.resolve(void 0); } })); return { get latest() { if (latest === void 0) return nextPromise.promise; return Promise.resolve(latest); }, get next() { return nextPromise.promise; } }; } //#endregion //#region src/uniqs.ts /** uniq by a new Set(), note: use === to compare */ const uniqs = () => { const set = /* @__PURE__ */ new Set(); return throughs((s) => s.pipeThrough(filters((x) => { if (set.has(x)) return false; set.add(x); return true; }))); }; /** uniq by a new Map(), Note: use === to compare keys */ const uniqBys = (keyFn) => { const set = /* @__PURE__ */ new Set(); return throughs((s) => s.pipeThrough(filters(async (x) => { const key = await keyFn(x); if (set.has(key)) return false; set.add(key); return true; }))); }; //#endregion //#region src/unwinds.ts function unwinds(key) { return flatMaps((e) => (0, unwind_array.unwind)(e, { path: key })); } //#endregion //#region src/sflow.ts /** stream flow */ function sflow(...srcs) { let r = srcs.length === 1 ? toStream(srcs[0]) : concatStream(srcs); return Object.assign(r, { _type: null, get readable() { return r; }, portal: (...args) => sflow(r.pipeThrough(portals(...args))), through: (...args) => sflow(r.pipeThrough(_throughs(...args))), by: (...args) => sflow(r.pipeThrough(_throughs(...args))), byLazy: (t) => _byLazy(r, t), cacheSkip: (...args) => sflow(r).byLazy(cacheSkips(...args)), cacheList: (...args) => sflow(r).byLazy(cacheLists(...args)), cacheTail: (...args) => sflow(r).byLazy(cacheTails(...args)), chunkBy: (...args) => sflow(r.pipeThrough(chunkBys(...args))), chunkIf: (...args) => sflow(r.pipeThrough(chunkIfs(...args))), buffer: (...args) => sflow(r.pipeThrough(chunks(...args))), chunk: (...args) => sflow(r.pipeThrough(chunks(...args))), convolve: (...args) => sflow(r.pipeThrough(convolves(...args))), abort: (...args) => sflow(r.pipeThrough(terminates(...args))), chunkInterval: (...args) => sflow(r.pipeThrough(chunkIntervals(...args))), interval: (...args) => sflow(r.pipeThrough(chunkIntervals(...args))), debounce: (...args) => sflow(r.pipeThrough(debounces(...args))), filter: (...args) => sflow(r.pipeThrough(filters(...args))), find: (...args) => sflow(r.pipeThrough(finds(...args))), flatMap: (...args) => sflow(r.pipeThrough(flatMaps(...args))), flat: (...args) => sflow(r).by(flats(...args)), join: (...args) => sflow(r.pipeThrough(riffles(...args))), match: (...args) => sflow(r.pipeThrough(matchs(...args))), matchAll: (...args) => sflow(r.pipeThrough(matchAlls(...args))), replace: (...args) => sflow(r.pipeThrough(replaces(...args))), replaceAll: (...args) => sflow(r.pipeThrough(replaceAlls(...args))), merge: (...args) => sflow(r.pipeThrough(merges(...args))), concat: (srcs) => sflow(r.pipeThrough(concats(srcs))), confluence: (...args) => sflow(r.pipeThrough(confluences(...args))), confluenceByZip: () => sflow(r).by(confluences()), confluenceByConcat: () => sflow(r).by((srcs) => concatStream(srcs)), confluenceByParallel: () => sflow(r).by((srcs) => sflow(srcs).toArray().then((srcs) => mergeStream(...srcs))).confluence(), confluenceByAscend: (ordFn) => sflow(r).chunk().map((srcs) => mergeStreamsByAscend(ordFn, srcs)).confluence(), confluenceByDescend: (ordFn) => sflow(r).chunk().map((srcs) => mergeStreamsByDescend(ordFn, srcs)).confluence(), limit: (...args) => sflow(r).byLazy(limits(...args)), head: (...args) => sflow(r.pipeThrough(heads(...args))), map: (...args) => sflow(r.pipeThrough(maps(...args))), mapAddField: (...args) => sflow(r.pipeThrough(mapAddFields(...args))), mapMixin: (...args) => sflow(r.pipeThrough(mapMixins(...args))), log: (...args) => sflow(r.pipeThrough(logs(...args))), uniq: (...args) => sflow(r.pipeThrough(uniqs(...args))), unique: (...args) => sflow(r.pipeThrough(uniqs(...args))), uniqBy: (...args) => sflow(r.pipeThrough(uniqBys(...args))), unwind: (...args) => sflow(r.pipeThrough(unwinds(...args))), asyncMap: (...args) => sflow(r.pipeThrough(asyncMaps(...args))), pMap: (...args) => sflow(r.pipeThrough(pMaps(...args))), peek: (...args) => sflow(r.pipeThrough(peeks(...args))), riffle: (...args) => sflow(r.pipeThrough(riffles(...args))), forEach: (...args) => sflow(r.pipeThrough(forEachs(...args))), reduce: (...args) => sflow(r.pipeThrough(reduces(...args))), reduceEmit: (...args) => sflow(r.pipeThrough(reduceEmits(...args))), skip: (...args) => sflow(r.pipeThrough(skips(...args))), slice: (...args) => sflow(r.pipeThrough(slices(...args))), tail: (...args) => sflow(r.pipeThrough(tails(...args))), takeWhile: (...args) => sflow(r.pipeThrough(takeWhiles(...args))), tees: (...args) => sflow(r.pipeThrough(_tees(...args))), forkTo: (...args) => sflow(r.pipeThrough(_tees(...args))), fork: () => { let b; [r, b] = r.tee(); return sflow(b); }, throttle: (...args) => sflow(r.pipeThrough(throttles(...args))), preventAbort: () => sflow(r.pipeThrough(throughs(), { preventAbort: true })), preventClose: () => sflow(r.pipeThrough(throughs(), { preventClose: true })), preventCancel: () => sflow(r.pipeThrough(throughs(), { preventCancel: true })), onStart: (start) => sflow(r).by(new TransformStream({ start })), onTransform: (transform) => sflow(r).by(new TransformStream({ transform })), onFlush: (flush) => sflow(r).by(new TransformStream({ flush })), done: () => r.pipeTo(nils()), end: (dst = nils()) => r.pipeTo(dst), to: (dst = nils()) => r.pipeTo(dst), run: () => r.pipeTo(nils()), toEnd: () => r.pipeTo(nils()), toNil: () => r.pipeTo(nils()), toArray: () => (0, web_streams_extensions.toArray)(r), toCount: async () => { let i = 0; const d = r.getReader(); while (!(await d.read()).done) i++; return i; }, toFirst: () => (0, web_streams_extensions.toPromise)(sflow(r).limit(1, { terminate: true })), toFirstMatch: (predicate) => (0, web_streams_extensions.toPromise)(sflow(r).find(predicate)), toLast: () => (0, web_streams_extensions.toPromise)(sflow(r).tail(1)), toExactlyOne: async () => { const a = await (0, web_streams_extensions.toArray)(r); if (a.length !== 1) (0, phpdie.default)(`Expect exactly 1 Item, but got ${a.length}`); return a[0]; }, toOne: async () => { const a = await (0, web_streams_extensions.toArray)(r); if (a.length > 1) (0, phpdie.default)(`Expect only 1 Item, but got ${a.length}`); return a[0]; }, toAtLeastOne: async () => { const a = await (0, web_streams_extensions.toArray)(r); if (a.length > 1) (0, phpdie.default)(`Expect only 1 Item, but got ${a.length}`); if (a.length < 1) (0, phpdie.default)(`Expect at least 1 Item, but got ${a.length}`); return a[0]; }, toLatest: () => toLatests(sflow(r)), toLog: (...args) => sflow(r.pipeThrough(logs(...args))).done(), lines: (...args) => sflow(r.pipeThrough(lines(...args))), toResponse: (init) => new Response(r, init), text: (init) => new Response(r.pipeThrough(new TextEncoderStream()), init).text(), json: (init) => new Response(r.pipeThrough(new TextEncoderStream()), init).json(), blob: (init) => new Response(sflow(r), init).blob(), arrayBuffer: (init) => new Response(r, init).arrayBuffer(), [Symbol.asyncIterator]: streamAsyncIterator }); } /** * Internal tees helper that wraps forked branches in sflow. * @warning Uses `ReadableStream.tee()` internally. If the forked branch is consumed slower than the main branch, * the tee buffer will grow unboundedly in memory (no backpressure between tee branches). */ const _tees = (arg) => { if (!arg) return new TransformStream(); if (arg instanceof WritableStream) return tees((s) => s.pipeTo(arg)); const fn = arg; const { writable, readable } = new TransformStream(); const [a, b] = readable.tee(); fn(sflow(a)); return { writable, readable: b }; }; const _throughs = (arg) => { if (!arg) return new TransformStream(); if (typeof arg !== "function") return throughs((s) => s.pipeThrough(arg)); const fn = arg; const { writable, readable } = new TransformStream(); return { writable, readable: sflow(fn(sflow(readable))) }; }; /** * byLazy is a lazy version of by, it only pull upstream when downstream pull. * * Warning: does not work with empty stream yet. */ function _byLazy(r, t) { const reader = r.getReader(); const tw = t.writable.getWriter(); const tr = t.readable.getReader(); return sflow(new ReadableStream({ start: async (ctrl) => { (async () => { while (true) { const { done, value } = await tr.read(); if (done) return ctrl.close(); ctrl.enqueue(value); } })(); }, pull: async (_ctrl) => { const { done, value } = await reader.read(); if (done) return tw.close(); await tw.write(value); }, cancel: async (r) => { reader.cancel(r); tr.cancel(r); } }, { highWaterMark: 0 })); } //#endregion //#region src/chunkOverlaps.ts function chunkOverlaps({ step, overlap }) { const chunks = []; if (step <= 0) throw new Error("step must be greater than 0"); if (overlap < 0) throw new Error("overlap must be greater than or equal to 0"); return new TransformStream({ transform: async (chunk, ctrl) => { chunks.push(chunk); if (chunks.length >= step + overlap) ctrl.enqueue([...chunks.splice(0, step), ...chunks.slice(0, overlap)]); }, flush: async (ctrl) => void (chunks.length && ctrl.enqueue(chunks)) }); } //#endregion //#region src/chunkTransforms.ts /** * Creates a TransformStream that processes chunks using provided transformers. * * @template T - The type of chunk that the TransformStream will process. * * @param {Object} options - The options object containing transformer functions. * @param {ChunkTransformer<T>} [options.start] - The transformer function to run when the stream is started. * @param {ChunkTransformer<T>} [options.transform] - The transformer function to run for each chunk, current chunk will be the last element. * @param {ChunkTransformer<T>} [options.flush] - The transformer function to run when the stream is flushed. * * @returns A new TransformStream that applies the provided transformers. */ function chunkTransforms(options) { let chunks = []; const { start, transform, flush } = options; return new TransformStream({ start: async (ctrl) => { if (start) chunks = await start(chunks, ctrl); }, transform: async (chunk, ctrl) => { chunks.push(chunk); if (transform) chunks = await transform(chunks, ctrl); }, flush: async (ctrl) => { if (flush) chunks = await flush(chunks, ctrl); } }); } //#endregion //#region src/distributeBys.ts const distributeBys = (groupFn) => { const streams = /* @__PURE__ */ new Map(); const { writable: srcs, readable } = new TransformStream(); const { writable, readable: chunks } = new TransformStream(); const w = srcs.getWriter(); chunks.pipeThrough(pMaps(async (chunk) => { const ord = await groupFn(chunk); if (!streams.has(ord)) await (async () => { const t = new TransformStream(); await w.write(t.readable); const r = { ...t, writer: t.writable.getWriter() }; streams.set(ord, r); return r; })(); const t = streams.get(ord); if (!t) throw new Error(`Stream not found for order ${ord}`); await t.writer.write(chunk); })).pipeTo(nils()).finally(() => { w.close(); [...streams.values()].map((e) => e.writer.close()); }); return { writable, readable }; }; //#endregion //#region src/mergeAscends.ts /** * merge multiple stream by ascend order, assume all input stream is sorted by ascend * output stream will be sorted by ascend too. * * if one of input stream is not sorted by ascend, it will throw an error. * * @param ordFn a function to get the order of input * @param srcs a list of input stream * @returns a new stream that merge all input stream by ascend order * @deprecated use {@link mergeStreamsByAscend}, this will be removed in next major version */ const mergeAscends = (ordFn, _srcs) => { if (!_srcs) return ((srcs) => mergeAscends(ordFn, srcs)); return sflow(new ReadableStream({ pull: async (ctrl) => { const srcs = await sflow(_srcs).toArray(); const slots = srcs.map(() => void 0); const pendingSlotRemoval = srcs.map(() => void 0); const drains = srcs.map(() => false); let lastMinValue; await Promise.all(srcs.map(async (src, i) => { for await (const value of sflow(src)) { while (slots[i] !== void 0) { if (shiftMinValueIfFull()) continue; pendingSlotRemoval[i] = Promise.withResolvers(); await pendingSlotRemoval[i]?.promise; } slots[i] = { value }; shiftMinValueIfFull(); } drains[i] = true; pendingSlotRemoval.map((e) => e?.resolve()); await Promise.all(pendingSlotRemoval.map((e) => e?.promise)); if (drains.every(Boolean)) { while (slots.some((e) => e !== void 0)) shiftMinValueIfFull(); ctrl.close(); } function shiftMinValueIfFull() { if (!slots.every((slot, i) => slot !== void 0 || drains[i])) return false; const minValue = minBy(ordFn, slots.flatMap((e) => e !== void 0 ? [e] : []).map((e) => e.value)); const minIndex = slots.findIndex((e) => e?.value === minValue); if (lastMinValue !== void 0) { if (ordFn(lastMinValue) > ordFn(minValue)) (0, phpdie.default)(` MergeAscendError: one of source stream is not ascending ordered. stream index: ${minIndex} prev: ${ordFn(lastMinValue)} prev: ${JSON.stringify(lastMinValue)} curr: ${ordFn(minValue)} curr: ${JSON.stringify(minValue)} `); } lastMinValue = minValue; ctrl.enqueue(minValue); slots[minIndex] = void 0; pendingSlotRemoval[minIndex]?.resolve(); pendingSlotRemoval[minIndex] = void 0; return true; } })); } }, { highWaterMark: 0 })); }; /** * merge multiple stream by ascend order, assume all input stream is sorted by ascend * output stream will be sorted by ascend too. * * if one of input stream is not sorted by ascend, it will throw an error. * * @param ordFn a function to get the order of input * @param srcs a list of input stream * @returns a new stream that merge all input stream by ascend order * @deprecated use {@link mergeStreamsByAscend} */ const mergeDescends = (ordFn, _srcs) => { if (!_srcs) return ((srcs) => mergeDescends(ordFn, srcs)); return toStream(new ReadableStream({ pull: async (ctrl) => { const srcs = await sflow(_srcs).toArray(); const slots = srcs.map(() => void 0); const pendingSlotRemoval = srcs.map(() => void 0); const drains = srcs.map(() => false); let lastMaxValue; await Promise.all(srcs.map(async (src, i) => { const stream = toStream(src); const streamIterator = Object.assign(stream, { [Symbol.asyncIterator]: streamAsyncIterator }); for await (const value of streamIterator) { while (slots[i] !== void 0) { if (shiftMaxValueIfFull()) continue; pendingSlotRemoval[i] = Promise.withResolvers(); await pendingSlotRemoval[i]?.promise; } slots[i] = { value }; shiftMaxValueIfFull(); } drains[i] = true; pendingSlotRemoval.map((e) => e?.resolve()); await Promise.all(pendingSlotRemoval.map((e) => e?.promise)); if (drains.every(Boolean)) { while (slots.some((e) => e !== void 0)) shiftMaxValueIfFull(); ctrl.close(); } function shiftMaxValueIfFull() { if (!slots.every((slot, i) => slot !== void 0 || drains[i])) return false; const maxValue = maxBy(ordFn, slots.flatMap((e) => e !== void 0 ? [e] : []).map((e) => e.value)); const maxIndex = slots.findIndex((e) => e?.value === maxValue); if (lastMaxValue !== void 0) { if (ordFn(lastMaxValue) < ordFn(maxValue)) (0, phpdie.default)(` MergeDescendError: one of source stream is not descending ordered. stream index: ${maxIndex} prev: ${ordFn(lastMaxValue)} prev: ${JSON.stringify(lastMaxValue)} curr: ${ordFn(maxValue)} curr: ${JSON.stringify(maxValue)} `); } lastMaxValue = maxValue; ctrl.enqueue(maxValue); slots[maxIndex] = void 0; pendingSlotRemoval[maxIndex]?.resolve(); pendingSlotRemoval[maxIndex] = void 0; return true; } })); } }, { highWaterMark: 0 })); }; //#endregion //#region src/pageStream.ts function pageStream(initialQuery, fetcher) { let query = null; return new ReadableStream({ pull: async (ctrl) => { if (query === null) query = { value: await initialQuery }; const ret = fetcher(query.value); const { data, next } = ret instanceof Promise ? await ret : ret; if (data !== void 0) ctrl.enqueue(data); if (null == next) return ctrl.close(); query.value = next; } }, { highWaterMark: 0 }); } //#endregion //#region src/pageFlow.ts function pageFlow(initialCursor, fetcher) { return sflow(pageStream(initialCursor, fetcher)); } //#endregion //#region src/rangeStream.ts function rangeStream(...args) { const [min, max] = args[1] != null ? [args[0], args[1]] : [0, args[0]]; let i = min; return new ReadableStream({ pull: (ctrl) => { ctrl.enqueue(i); if (++i >= max) ctrl.close(); } }, { highWaterMark: 0 }); } function rangeFlow(...args) { return sflow(rangeStream(...args)); } //#endregion //#region src/retry.ts function retry(onError, fn) { return async (...args) => { let attempt = 0; const retryable = async (...retryArgs) => { try { const ret = fn(...retryArgs); return ret instanceof Promise ? await ret : ret; } catch (error) { attempt++; const ret = onError(error, attempt, retryable, ...retryArgs); return ret instanceof Promise ? await ret : ret; } }; return retryable(...args); }; } //#endregion //#region src/sfTemplate.ts /** sflow Template */ function sfTemplate(tsa, ...args) { return sflow(...tsa.flatMap((str) => [sflow([str]), args.shift() || []])); } const sfT = sfTemplate; //#endregion //#region src/svector.ts /** stream vector */ const svector = (...src) => sflow(src); //#endregion //#region src/composers.ts function composers(stream) { return Object.assign(stream, { by: (appendStream) => composers({ writable: stream.writable, readable: stream.readable.pipeThrough(appendStream) }) }); } //#endregion //#region src/sf.ts var sf_exports = /* @__PURE__ */ __exportAll({ aborts: () => terminates, buffers: () => chunks, bys: () => bys, cacheLists: () => cacheLists, cacheSkips: () => cacheSkips, cacheTails: () => cacheTails, chunkBys: () => chunkBys, chunkIfs: () => chunkIfs, chunkIntervals: () => chunkIntervals, chunkOverlaps: () => chunkOverlaps, chunkTransforms: () => chunkTransforms, chunks: () => chunks, composers: () => composers, concatStream: () => concatStream, concats: () => concats, confluences: () =>