@yoyo-org/progressive-json
Version:
Stream and render JSON data as it arrives - perfect for AI responses, large datasets, and real-time updates
283 lines (282 loc) • 9.83 kB
JavaScript
import { fetchJson } from "./fetch-json";
import { filterPlaceholders } from "./utils/filter-placeholders";
import { findPlaceholders, } from "./utils/find-placeholders";
import { isPlaceholder } from "./utils/is-placeholder";
import { produce } from "immer";
export class Processor {
constructor(options) {
this.decoder = new TextDecoder();
this.listeners = [];
this.refStore = {};
this.plugins = [];
this.isStreaming = false;
this.streamError = null;
this.options = options;
this.store = options.initialStore;
this.plugins = options.plugins || [];
if (this.store) {
this.updateRefStore(this.store);
this.updateTransformedStore();
}
// Only start fetching if enabled is not explicitly set to false
if (options.enabled !== false && options.url) {
this.startStreaming();
}
}
async startStreaming() {
var _a, _b;
if (this.isStreaming)
return;
this.isStreaming = true;
this.streamError = null;
(_b = (_a = this.options).onStreamStart) === null || _b === void 0 ? void 0 : _b.call(_a);
try {
// Use custom fetch function if provided, otherwise use default
if (this.options.customFetch) {
await this.options.customFetch(this.options.url, this, this.options);
}
else {
await fetchJson(this.options.url, this);
}
}
catch (error) {
this.doHandleStreamError(error);
}
}
doHandleStreamError(error) {
var _a, _b;
this.isStreaming = false;
this.streamError = error;
(_b = (_a = this.options).onStreamError) === null || _b === void 0 ? void 0 : _b.call(_a, error);
this.notifyListeners();
}
updateTransformedStore() {
if (!this.store) {
this.transformedStore = undefined;
this.selectedStore = undefined;
return;
}
// Apply transform if provided
this.transformedStore = this.options.transform
? this.options.transform(this.store)
: this.store;
// Apply selector if provided
this.selectedStore = this.options.select
? this.options.select(this.transformedStore)
: this.transformedStore;
}
// --- Public API ---
getStore() {
return this.selectedStore;
}
getRawStore() {
return this.store;
}
getTransformedStore() {
return this.transformedStore;
}
getRefStore() {
return this.refStore;
}
isCurrentlyStreaming() {
return this.isStreaming;
}
getStreamError() {
return this.streamError;
}
subscribe(listener) {
this.listeners.push(listener);
return () => {
this.listeners = this.listeners.filter((l) => l !== listener);
};
}
startFetching() {
if (this.options.url && this.options.enabled !== false) {
this.startStreaming();
}
}
updateOptions(newOptions) {
const oldEnabled = this.options.enabled;
const oldUrl = this.options.url;
this.options = { ...this.options, ...newOptions };
// Restart streaming if URL changed or enabled changed from false to true
if ((oldEnabled === false && this.options.enabled !== false) ||
(oldUrl !== this.options.url && this.options.enabled !== false)) {
this.startStreaming();
}
// Update transforms if store exists
if (this.store) {
this.updateTransformedStore();
this.notifyListeners();
}
}
processChunk(chunk) {
const text = this.decoder.decode(chunk, { stream: true });
for (const line of text.split("\n")) {
if (!this.store) {
this.store = {};
}
const prevStore = this.store;
this.store = this.handleStreamLine(line, this.store);
// Only update transformed store and notify if data actually changed
if (this.hasStoreChanged(prevStore, this.store)) {
this.updateTransformedStore();
this.notifyListeners();
}
}
}
// --- Internals ---
notifyListeners() {
this.listeners.forEach((listener) => listener());
}
hasStoreChanged(prev, next) {
// Use custom compare function if provided
if (this.options.compare) {
return !this.options.compare(prev, next);
}
// Default: simple reference equality
return prev !== next;
}
stop() {
this.isStreaming = false;
// Cleanup SSE connection if exists
if (this._sseCleanup) {
this._sseCleanup();
delete this._sseCleanup;
}
// Note: fetchJson implementation should handle abort signals
}
onStreamComplete(finalData) {
var _a, _b;
this.isStreaming = false;
this.updateTransformedStore();
(_b = (_a = this.options).onStreamEnd) === null || _b === void 0 ? void 0 : _b.call(_a, finalData);
this.notifyListeners();
}
handleStreamError(error) {
this.doHandleStreamError(error);
}
destroy() {
this.stop();
this.listeners = [];
}
normalizeRefKey(key) {
return key.startsWith("ref") ? key : `ref${key}`;
}
getRefIdFromKey(refKey) {
const match = refKey.match(/^ref\$(\d+)$/);
return match ? Number(match[1]) : null;
}
updateRefStore(store) {
const found = findPlaceholders(store);
Object.entries(found).forEach(([refId, path]) => {
this.refStore[Number(refId)] = path;
});
}
updateAtPath(store, key, updater) {
const refKey = this.normalizeRefKey(key);
const refId = this.getRefIdFromKey(refKey);
if (refId == null)
return store;
const path = this.refStore[refId];
if (!path)
return store;
return produce(store, (draft) => {
let obj = draft;
for (let i = 0; i < path.length - 1; i++) {
if (typeof obj === "object" && obj !== null) {
obj = obj[path[i]];
}
}
const lastKey = path[path.length - 1];
if (typeof obj === "object" && obj !== null) {
updater(obj, lastKey);
}
});
}
applyPushUpdate(store, key, value) {
return this.updateAtPath(store, key, (obj, lastKey) => {
if (!Array.isArray(obj[lastKey]))
obj[lastKey] = [];
obj[lastKey].push(value);
});
}
applyConcatUpdate(store, key, value) {
return this.updateAtPath(store, key, (obj, lastKey) => {
if (!Array.isArray(obj[lastKey]))
obj[lastKey] = [];
obj[lastKey].push(...value);
});
}
applyRefUpdate(store, key, value) {
const updatedStore = this.updateAtPath(store, key, (obj, lastKey) => {
obj[lastKey] = value;
});
this.updateRefStore(updatedStore);
return updatedStore;
}
applyStreamUpdate(store, key, value) {
return this.updateAtPath(store, key, (obj, lastKey) => {
var _a;
if (typeof obj[lastKey] === "string" &&
isPlaceholder(obj[lastKey])) {
obj[lastKey] = value;
}
else {
obj[lastKey] = ((_a = obj[lastKey]) !== null && _a !== void 0 ? _a : "") + value;
}
});
}
handleStreamLine(line, currentStore) {
if (!line.trim())
return currentStore;
const { onMessage } = this.options;
let updatedStore = currentStore;
try {
const msg = JSON.parse(line);
// Find a plugin whose type matches the message type
const plugin = this.plugins.find((p) => p.type === msg.type);
if (plugin) {
const context = {
updateAtPath: this.updateAtPath.bind(this),
normalizeRefKey: this.normalizeRefKey.bind(this),
getRefIdFromKey: this.getRefIdFromKey.bind(this),
refStore: this.refStore,
};
// Type-safe dispatch: cast msg to the plugin's message type
updatedStore = plugin.handleMessage(msg, currentStore, context);
}
else {
// Handle built-in message types
switch (msg.type) {
case "init":
updatedStore = this.handleInit(msg.data);
break;
case "value":
updatedStore = this.applyRefUpdate(updatedStore, msg.key, msg.value);
break;
case "text":
updatedStore = this.applyStreamUpdate(updatedStore, msg.key, String(msg.value));
break;
case "push":
updatedStore = this.applyPushUpdate(updatedStore, msg.key, msg.value);
break;
case "concat":
updatedStore = this.applyConcatUpdate(updatedStore, msg.key, msg.value);
break;
}
}
if (onMessage) {
onMessage(filterPlaceholders(updatedStore));
}
return updatedStore;
}
catch {
return currentStore;
}
}
handleInit(data) {
this.updateRefStore(data);
return data;
}
}