@statezero/core
Version:
The type-safe frontend client for StateZero - connect directly to your backend models with zero boilerplate
383 lines (382 loc) • 16.2 kB
JavaScript
import { getAllEventReceivers } from "../core/eventReceivers";
import { operationRegistry } from "./stores/operation";
import { initializeAllEventReceivers } from "../config";
import { getEventReceiver } from "../core/eventReceivers";
import { querysetStoreRegistry, QuerysetStoreRegistry, } from "./registries/querysetStoreRegistry";
import { modelStoreRegistry, ModelStoreRegistry, } from "./registries/modelStoreRegistry";
import { metricRegistry, MetricRegistry } from "./registries/metricRegistry";
import { getModelClass, getConfig } from "../config";
import { isNil } from "lodash-es";
import { QuerysetStore } from "./stores/querysetStore";
import PQueue from "p-queue";
export class EventPayload {
constructor(data) {
this.event = data.event;
this.model = data.model;
this.operation_id = data.operation_id;
this.pk_field_name = data.pk_field_name;
this.configKey = data.configKey;
// Parse PK fields to numbers in instances
this.instances = data.instances?.map(instance => {
if (instance && this.pk_field_name && instance[this.pk_field_name] != null) {
return {
...instance,
[this.pk_field_name]: Number(instance[this.pk_field_name])
};
}
return instance;
}) || data.instances;
this._cachedInstances = null;
}
get modelClass() {
return getModelClass(this.model, this.configKey);
}
async getFullInstances() {
if (this.event === "delete") {
throw new Error("Cannot fetch full instances for delete operation bozo...");
}
if (isNil(this._cachedInstances)) {
this._cachedInstances = await this.modelClass.objects
.filter({
[`${this.modelClass.primaryKeyField}__in`]: this.instances.map((instance) => instance[this.modelClass.primaryKeyField]),
})
.fetch();
}
return this._cachedInstances;
}
}
export class SyncManager {
constructor() {
this.handleEvent = (event) => {
let payload = new EventPayload(event);
let isLocalOperation = operationRegistry.has(payload.operation_id);
// Always process metrics immediately (they're lightweight)
if (this.registries.has(MetricRegistry)) {
this.processMetrics(payload);
}
if (isLocalOperation) {
return;
}
// Add to batch for queryset/model processing
const key = `${event.model}::${event.configKey}`;
const now = Date.now();
// If this is the first event in the batch, start max wait timer
if (this.eventBatch.size === 0) {
this.batchStartTime = now;
this.maxWaitTimer = setTimeout(() => {
this.processBatch("maxWait");
}, this.maxWaitMs);
}
this.eventBatch.set(key, {
event: payload,
firstSeen: this.eventBatch.has(key)
? this.eventBatch.get(key).firstSeen
: now,
});
// Reset debounce timer
if (this.debounceTimer) {
clearTimeout(this.debounceTimer);
}
this.debounceTimer = setTimeout(() => {
this.processBatch("debounce");
}, this.debounceMs);
};
this.registries = new Map();
// Map of registries to model sets
this.followedModels = new Map();
// Map of querysets to keep synced
this.followAllQuerysets = true;
this.followedQuerysets = new Map();
this.periodicSyncTimer = null;
// Batching for event processing
this.eventBatch = new Map(); // model::configKey -> { event, firstSeen }
this.debounceTimer = null;
this.maxWaitTimer = null;
this.debounceMs = 100; // Wait for rapid events to settle
this.maxWaitMs = 2000; // Maximum time to hold events
this.batchStartTime = null;
// SyncQueue
this.syncQueue = new PQueue({ concurrency: 1 });
}
withTimeout(promise, ms) {
// If no timeout specified, use 2x the periodic sync interval, or 30s as fallback
if (!ms) {
try {
const config = getConfig();
const intervalSeconds = config.periodicSyncIntervalSeconds;
ms = intervalSeconds ? intervalSeconds * 2000 : 30000; // 2x interval in ms, or 30s default
}
catch {
ms = 30000; // 30s fallback if no config
}
}
return Promise.race([
promise,
new Promise((_, reject) => setTimeout(() => reject(new Error(`Sync timeout after ${ms}ms`)), ms)),
]);
}
/**
* Initialize event handlers for all event receivers
*/
initialize() {
// Initialize all event receivers
initializeAllEventReceivers();
// Get all registered event receivers
const eventReceivers = getAllEventReceivers();
// Register the event handlers with each receiver
eventReceivers.forEach((receiver, configKey) => {
if (receiver) {
// Model events go to handleEvent
receiver.addModelEventHandler(this.handleEvent.bind(this));
}
});
this.startPeriodicSync();
}
startPeriodicSync() {
if (this.periodicSyncTimer)
return;
try {
const config = getConfig();
const intervalSeconds = config.periodicSyncIntervalSeconds;
// If null or undefined, don't start periodic sync
if (!intervalSeconds) {
console.log("[SyncManager] Periodic sync disabled (set to null)");
return;
}
const intervalMs = intervalSeconds * 1000;
this.periodicSyncTimer = setInterval(() => {
this.syncStaleQuerysets();
}, intervalMs);
console.log(`[SyncManager] Periodic sync started: ${intervalSeconds}s intervals`);
}
catch (error) {
// If no config, don't start periodic sync by default
console.log("[SyncManager] No config found, periodic sync disabled by default");
}
}
syncStaleQuerysets() {
let syncedCount = 0;
// Sync all followed querysets - keep it simple
const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
if (querysetRegistry) {
for (const [semanticKey, store] of querysetRegistry._stores.entries()) {
// Only sync if this store is actually being followed
const isFollowed = this.isStoreFollowed(querysetRegistry, semanticKey);
if (this.followAllQuerysets || isFollowed) {
this.syncQueue.add(() => this.withTimeout(store.sync()));
syncedCount++;
}
}
}
if (syncedCount > 0) {
console.log(`[SyncManager] Periodic sync: ${syncedCount} stores pushed to the sync queue`);
}
// Prune unreferenced model instances
this.pruneUnreferencedModels();
}
pruneUnreferencedModels() {
const modelRegistry = this.registries.get(ModelStoreRegistry);
const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
if (!modelRegistry || !querysetRegistry)
return;
// Prune each model store
for (const [modelClass, store] of modelRegistry._stores.entries()) {
store.pruneUnreferencedInstances(querysetRegistry);
}
}
isStoreFollowed(registry, semanticKey) {
const followingQuerysets = registry.followingQuerysets.get(semanticKey);
if (!followingQuerysets)
return false;
return [...followingQuerysets].some((queryset) => {
return this.isQuerysetFollowed(queryset);
});
}
cleanup() {
if (this.periodicSyncTimer) {
clearInterval(this.periodicSyncTimer);
this.periodicSyncTimer = null;
}
// Clean up batch timers
if (this.debounceTimer) {
clearTimeout(this.debounceTimer);
this.debounceTimer = null;
}
if (this.maxWaitTimer) {
clearTimeout(this.maxWaitTimer);
this.maxWaitTimer = null;
}
}
followModel(registry, modelClass) {
const models = this.followedModels.get(registry) || new Set();
this.followedModels.set(registry, models);
if (models.has(modelClass))
return;
const alreadyFollowed = [...this.followedModels.values()].some((set) => set.has(modelClass));
models.add(modelClass);
if (!alreadyFollowed) {
getEventReceiver(modelClass.configKey)?.subscribe(modelClass.modelName, this.handleEvent);
}
}
unfollowModel(registry, modelClass) {
const models = this.followedModels.get(registry);
if (!models)
return;
models.delete(modelClass);
const stillFollowed = [...this.followedModels.values()].some((set) => set.has(modelClass));
if (!stillFollowed) {
getEventReceiver(modelClass.configKey)?.unsubscribe(modelClass.modelName, this.handleEvent);
}
}
manageRegistry(registry) {
this.registries.set(registry.constructor, registry);
registry.setSyncManager(this);
}
removeRegistry(registry) {
this.registries.delete(registry.constructor);
}
processBatch(reason = "unknown") {
if (this.eventBatch.size === 0)
return;
// Clear timers
if (this.debounceTimer) {
clearTimeout(this.debounceTimer);
this.debounceTimer = null;
}
if (this.maxWaitTimer) {
clearTimeout(this.maxWaitTimer);
this.maxWaitTimer = null;
}
const events = Array.from(this.eventBatch.values()).map((item) => item.event);
const waitTime = Date.now() - this.batchStartTime;
this.eventBatch.clear();
this.batchStartTime = null;
console.log(`[SyncManager] Processing batch of ${events.length} events (reason: ${reason}, waited: ${waitTime}ms)`);
// Group events by model for efficient processing
const eventsByModel = new Map();
events.forEach((event) => {
const key = `${event.model}::${event.configKey}`;
if (!eventsByModel.has(key)) {
eventsByModel.set(key, []);
}
eventsByModel.get(key).push(event);
});
// Process each model's events as a batch
eventsByModel.forEach((modelEvents, modelKey) => {
const event = modelEvents[0]; // Use first event as representative
if (this.registries.has(QuerysetStoreRegistry)) {
this.processQuerysetsBatch(event, modelEvents);
}
if (this.registries.has(ModelStoreRegistry)) {
this.processModels(event);
}
});
}
isQuerysetFollowed(queryset) {
const activeSemanticKeys = new Set([...this.followedQuerysets].map((qs) => qs.semanticKey));
let current = queryset;
while (current) {
if (activeSemanticKeys.has(current.semanticKey)) {
return true;
}
current = current.__parent;
}
return false;
}
processQuerysetsBatch(representativeEvent, allEvents) {
const registry = this.registries.get(QuerysetStoreRegistry);
// Collect all stores that need syncing for this model
const storesToSync = [];
for (const [semanticKey, store] of registry._stores.entries()) {
if (store.modelClass.modelName === representativeEvent.model &&
store.modelClass.configKey === representativeEvent.configKey) {
if (this.followAllQuerysets) {
storesToSync.push(store);
continue;
}
const followingQuerysets = registry.followingQuerysets.get(semanticKey);
if (followingQuerysets) {
const shouldSync = [...followingQuerysets].some((queryset) => {
return this.isQuerysetFollowed(queryset);
});
if (shouldSync) {
storesToSync.push(store);
}
}
}
}
// Sync all relevant stores for this model
console.log(`[SyncManager] Syncing ${storesToSync.length} queryset stores for ${representativeEvent.model}`);
storesToSync.forEach((store) => {
this.syncQueue.add(() => this.withTimeout(store.sync()));
});
}
processMetrics(event) {
const registry = this.registries.get(MetricRegistry);
for (const [key, entry] of registry._stores.entries()) {
// Check if the store has a queryset with the model class AND correct backend
if (entry.queryset &&
entry.queryset.modelClass.modelName === event.model &&
entry.queryset.modelClass.configKey === event.configKey) {
if (this.followAllQuerysets) {
entry.store.sync();
continue;
}
// Check if this queryset (or any parent) is being followed
if (this.isQuerysetFollowed(entry.queryset)) {
console.log(`syncing metric store for key: ${key}`);
entry.store.sync();
}
}
}
}
processModels(event) {
const registry = this.registries.get(ModelStoreRegistry);
if (!registry)
return;
const modelStore = registry.getStore(event.modelClass);
if (!modelStore)
return;
// Get PKs from the event
const eventPks = new Set((event.instances || [])
.filter(inst => inst && inst[event.pk_field_name] != null)
.map(inst => inst[event.pk_field_name]));
if (eventPks.size === 0)
return;
// Get PKs that are already in the model store's ground truth
const groundTruthPks = new Set(modelStore.groundTruthPks);
// Get PKs that are top-level in ANY queryset for this model (not just followed ones)
const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
const topLevelQuerysetPks = new Set();
if (querysetRegistry) {
for (const [semanticKey, store] of querysetRegistry._stores.entries()) {
// Check ALL querysets for this model (permanent and temporary)
if (store.modelClass.modelName === event.model &&
store.modelClass.configKey === event.configKey) {
// Get this queryset's ground truth PKs (top-level instances)
// Don't use render() as it applies operations - we just want ground truth
store.groundTruthPks.forEach(pk => topLevelQuerysetPks.add(pk));
}
}
}
// Find PKs that are:
// 1. In the event
// 2. Already in ground truth (we're tracking them)
// 3. NOT top-level in any queryset (they're only nested/included)
const pksToSync = [];
eventPks.forEach(pk => {
if (groundTruthPks.has(pk) && !topLevelQuerysetPks.has(pk)) {
pksToSync.push(pk);
}
});
if (pksToSync.length > 0) {
console.log(`[SyncManager] Syncing ${pksToSync.length} nested-only PKs for ${event.model}: ${pksToSync}`);
this.syncQueue.add(() => this.withTimeout(modelStore.sync(pksToSync)));
}
}
}
const syncManager = new SyncManager();
syncManager.manageRegistry(querysetStoreRegistry);
syncManager.manageRegistry(modelStoreRegistry);
syncManager.manageRegistry(metricRegistry);
export { syncManager };