@signaldb/sync
Version:
This is the sync implementation of [SignalDB](https://github.com/maxnowack/signaldb). SignalDB is a local-first JavaScript database with real-time sync, enabling optimistic UI with signal-based reactivity across multiple frameworks.
609 lines (607 loc) • 20.3 kB
JavaScript
var E = Object.defineProperty;
var M = (c, e, t) => e in c ? E(c, e, { enumerable: !0, configurable: !0, writable: !0, value: t }) : c[e] = t;
var u = (c, e, t) => M(c, typeof e != "symbol" ? e + "" : e, t);
import { isEqual as S, modify as C, randomId as D, Collection as m, createIndex as y } from "@signaldb/core";
function x(c, e, t = {}) {
let s, n;
const { leading: o = !1, trailing: a = !0 } = t;
function r(...h) {
const i = o && !s, l = a && !s;
return s && clearTimeout(s), s = setTimeout(() => {
s = null, a && !i && (n = c.apply(this, h));
}, e), i ? n = c.apply(this, h) : l || (n = null), n;
}
return r;
}
class R {
constructor() {
u(this, "queue", []);
u(this, "pendingPromise", !1);
}
/**
* Method to add a new promise to the queue and returns a promise that resolves when this task is done
* @param task Function that returns a promise that will be added to the queue
* @returns Promise that resolves when the task is done
*/
add(e) {
return new Promise((t, s) => {
this.queue.push(() => e().then(t).catch((n) => {
throw s(n), n;
})), this.dequeue();
});
}
/**
* Method to check if there is a pending promise in the queue
* @returns True if there is a pending promise, false otherwise
*/
hasPendingPromise() {
return this.pendingPromise;
}
/**
* Method to process the queue
*/
dequeue() {
if (this.pendingPromise || this.queue.length === 0)
return;
const e = this.queue.shift();
e && (this.pendingPromise = !0, e().then(() => {
this.pendingPromise = !1, this.dequeue();
}).catch(() => {
this.pendingPromise = !1, this.dequeue();
}));
}
}
function N(c, e) {
const t = [], s = Object.keys(c), n = Object.keys(e), o = /* @__PURE__ */ new Set([...s, ...n]);
for (const a of o)
if (e[a] !== c[a])
if (typeof e[a] == "object" && typeof c[a] == "object" && e[a] != null && c[a] != null) {
const r = N(c[a], e[a]);
for (const h of r)
t.push(`${a}.${h}`);
} else
t.push(a);
return t;
}
function g(c, e) {
const t = [], s = [], n = /* @__PURE__ */ new Map(), o = [], a = new Map(c.map((h) => [h.id, h])), r = new Map(e.map((h) => [h.id, h]));
for (const [h, i] of a) {
const l = r.get(h);
l ? S(l, i) || (n.set(l.id, N(i, l)), s.push(l)) : o.push(i);
}
for (const [h, i] of r)
a.has(h) || t.push(i);
return {
added: t,
modified: s,
modifiedFields: n,
removed: o
};
}
function P(c, e) {
if (e.items != null)
return e.items;
const t = c || [];
return e.changes.added.forEach((s) => {
const n = t.findIndex((o) => o.id === s.id);
n === -1 ? t.push(s) : t[n] = s;
}), e.changes.modified.forEach((s) => {
const n = t.findIndex((o) => o.id === s.id);
n === -1 ? t.push(s) : t[n] = s;
}), e.changes.removed.forEach((s) => {
const n = t.findIndex((o) => o.id === s.id);
n !== -1 && t.splice(n, 1);
}), t;
}
function v(c, e) {
const t = new Map(c.map((s) => [s.id, s]));
return e.forEach((s) => {
if (s.type === "remove")
t.delete(s.data);
else if (s.type === "insert") {
const n = t.get(s.data.id);
t.set(s.data.id, n ? { ...n, ...s.data } : s.data);
} else {
const n = t.get(s.data.id);
t.set(s.data.id, n ? C(n, s.data.modifier) : C({ id: s.data.id }, s.data.modifier));
}
}), [...t.values()];
}
function O(c) {
return c.added.length > 0 || c.modified.length > 0 || c.removed.length > 0;
}
function b(c, e) {
return O(g(c, e));
}
async function I({ changes: c, lastSnapshot: e, data: t, pull: s, push: n, insert: o, update: a, remove: r, batch: h }) {
let i = t, l = e || [], f = P(e, i);
if (c.length > 0) {
const p = v(l, c);
if (b(l, p)) {
const $ = v(f, c), w = g(f, $);
O(w) && (await n(w), i = await s(), f = P(f, i)), l = p;
}
}
const d = i.changes == null ? g(l, i.items) : i.changes;
return h(() => {
d.added.forEach((p) => o(p)), d.modified.forEach((p) => a(p.id, { $set: p })), d.removed.forEach((p) => r(p.id));
}), f;
}
class T {
/**
* @param options Collection options
* @param options.pull Function to pull data from remote source.
* @param options.push Function to push data to remote source.
* @param [options.registerRemoteChange] Function to register a callback for remote changes.
* @param [options.id] Unique identifier for this sync manager. Only nessesary if you have multiple sync managers.
* @param [options.persistenceAdapter] Persistence adapter to use for storing changes, snapshots and sync operations.
* @param [options.reactivity] Reactivity adapter to use for reactivity.
* @param [options.onError] Function to handle errors that occur async during syncing.
* @param [options.autostart] Whether to automatically start syncing new collections.
* @param [options.debounceTime] The time in milliseconds to debounce push operations.
*/
constructor(e) {
u(this, "options");
u(this, "collections", /* @__PURE__ */ new Map());
u(this, "changes");
u(this, "snapshots");
u(this, "syncOperations");
u(this, "scheduledPushes", /* @__PURE__ */ new Set());
u(this, "remoteChanges", []);
u(this, "syncQueues", /* @__PURE__ */ new Map());
u(this, "persistenceReady");
u(this, "isDisposed", !1);
u(this, "instanceId", D());
u(this, "id");
u(this, "debouncedFlush");
this.options = {
autostart: !0,
...e
}, this.id = this.options.id || "default-sync-manager";
const { reactivity: t } = this.options, s = this.createPersistenceAdapter("changes"), n = this.createPersistenceAdapter("snapshots"), o = this.createPersistenceAdapter("sync-operations");
this.changes = new m({
name: `${this.options.id}-changes`,
persistence: s == null ? void 0 : s.adapter,
indices: [y("collectionName")],
reactivity: t
}), this.snapshots = new m({
name: `${this.options.id}-snapshots`,
persistence: n == null ? void 0 : n.adapter,
indices: [y("collectionName")],
reactivity: t
}), this.syncOperations = new m({
name: `${this.options.id}-sync-operations`,
persistence: o == null ? void 0 : o.adapter,
indices: [y("collectionName"), y("status")],
reactivity: t
}), this.changes.on("persistence.error", (a) => s == null ? void 0 : s.handler(a)), this.snapshots.on("persistence.error", (a) => n == null ? void 0 : n.handler(a)), this.syncOperations.on("persistence.error", (a) => o == null ? void 0 : o.handler(a)), this.persistenceReady = Promise.all([
this.syncOperations.isReady(),
this.changes.isReady(),
this.snapshots.isReady()
]).then(() => {
}), this.changes.setMaxListeners(1e3), this.snapshots.setMaxListeners(1e3), this.syncOperations.setMaxListeners(1e3), this.debouncedFlush = x(this.flushScheduledPushes, this.options.debounceTime ?? 100);
}
createPersistenceAdapter(e) {
if (this.options.persistenceAdapter == null)
return;
let t = () => {
};
return {
adapter: this.options.persistenceAdapter(`${this.id}-${e}`, (n) => {
t = n;
}),
handler: (n) => t(n)
};
}
getSyncQueue(e) {
return this.syncQueues.get(e) == null && this.syncQueues.set(e, new R()), this.syncQueues.get(e);
}
/**
* Clears all internal data structures
*/
async dispose() {
this.collections.clear(), this.syncQueues.clear(), this.remoteChanges.splice(0), await Promise.all([
this.changes.dispose(),
this.snapshots.dispose(),
this.syncOperations.dispose()
]), this.isDisposed = !0;
}
/**
* Gets a collection with it's options by name
* @deprecated Use getCollectionProperties instead.
* @param name Name of the collection
* @throws Will throw an error if the name wasn't found
* @returns Tuple of collection and options
*/
getCollection(e) {
const { collection: t, options: s } = this.getCollectionProperties(e);
return [t, s];
}
/**
* Gets collection options by name
* @param name Name of the collection
* @throws Will throw an error if the name wasn't found
* @returns An object of all properties of the collection
*/
getCollectionProperties(e) {
const t = this.collections.get(e);
if (t == null)
throw new Error(`Collection with id '${e}' not found`);
return t;
}
/**
* Adds a collection to the sync manager.
* @param collection Collection to add
* @param options Options for the collection. The object needs at least a `name` property.
* @param options.name Unique name of the collection
*/
addCollection(e, t) {
if (this.isDisposed)
throw new Error("SyncManager is disposed");
this.collections.set(t.name, {
collection: e,
options: t,
readyPromise: e.isReady(),
syncPaused: !0
// always start paused as the autostart will start it
});
const s = (o) => {
for (const a of this.remoteChanges)
if (a != null && a.collectionName === o.collectionName && a.type === o.type && !(o.type === "remove" && a.data !== o.data) && a.data.id === o.data.id)
return !0;
return !1;
}, n = (o, a) => {
const r = [...this.remoteChanges];
for (let h = 0; h < r.length; h += 1) {
const i = r[h];
i != null && i.collectionName === o && (i.type === "remove" && i.data !== a || i.data.id === a && (r[h] = null));
}
this.remoteChanges = r.filter((h) => h != null);
};
e.on("added", (o) => {
if (s({ collectionName: t.name, type: "insert", data: o })) {
n(t.name, o.id);
return;
}
this.changes.insert({
collectionName: t.name,
time: Date.now(),
type: "insert",
data: o
}), !this.getCollectionProperties(t.name).syncPaused && this.schedulePush(t.name);
}), e.on("changed", ({ id: o }, a) => {
const r = { id: o, modifier: a };
if (s({ collectionName: t.name, type: "update", data: r })) {
n(t.name, o);
return;
}
this.changes.insert({
collectionName: t.name,
time: Date.now(),
type: "update",
data: r
}), !this.getCollectionProperties(t.name).syncPaused && this.schedulePush(t.name);
}), e.on("removed", ({ id: o }) => {
if (s({ collectionName: t.name, type: "remove", data: o })) {
n(t.name, o);
return;
}
this.changes.insert({
collectionName: t.name,
time: Date.now(),
type: "remove",
data: o
}), !this.getCollectionProperties(t.name).syncPaused && this.schedulePush(t.name);
}), this.options.autostart && this.startSync(t.name).catch((o) => {
this.options.onError && this.options.onError(this.getCollectionProperties(t.name).options, o);
});
}
flushScheduledPushes() {
this.scheduledPushes.forEach((e) => {
this.pushChanges(e).catch(() => {
});
}), this.scheduledPushes.clear();
}
schedulePush(e) {
this.scheduledPushes.add(e), this.debouncedFlush();
}
/**
* Setup all collections to be synced with remote changes
* and enable automatic pushing changes to the remote source.
*/
async startAll() {
await Promise.all([...this.collections.keys()].map((e) => this.startSync(e)));
}
/**
* Setup a collection to be synced with remote changes
* and enable automatic pushing changes to the remote source.
* @param name Name of the collection
*/
async startSync(e) {
const t = this.getCollectionProperties(e);
if (!t.syncPaused)
return;
this.schedulePush(e);
const s = this.options.registerRemoteChange ? await this.options.registerRemoteChange(t.options, async (n) => {
if (n == null)
await this.sync(e);
else {
const o = Date.now(), a = this.syncOperations.insert({
start: o,
collectionName: e,
instanceId: this.instanceId,
status: "active"
});
await this.syncWithData(e, n).then(() => {
this.syncOperations.removeMany({
id: { $ne: a },
collectionName: e,
$or: [
{ end: { $lte: o } },
{ status: "active" }
]
}), this.syncOperations.updateOne({ id: a }, {
$set: { status: "done", end: Date.now() }
});
}).catch((r) => {
throw this.options.onError && this.options.onError(this.getCollectionProperties(e).options, r), this.syncOperations.updateOne({ id: a }, {
$set: { status: "error", end: Date.now(), error: r.stack || r.message }
}), r;
});
}
}) : void 0;
this.collections.set(e, {
...t,
syncPaused: !1,
cleanupFunction: s
});
}
/**
* Pauses the sync process for all collections.
* This means that the collections will not be synced with remote changes
* and changes will not automatically be pushed to the remote source.
*/
async pauseAll() {
await Promise.all([...this.collections.keys()].map((e) => this.pauseSync(e)));
}
/**
* Pauses the sync process for a collection.
* This means that the collection will not be synced with remote changes
* and changes will not automatically be pushed to the remote source.
* @param name Name of the collection
*/
async pauseSync(e) {
const t = this.getCollectionProperties(e);
t.syncPaused || (t.cleanupFunction && await t.cleanupFunction(), this.collections.set(e, {
...t,
cleanupFunction: void 0,
syncPaused: !0
}));
}
/**
* Starts the sync process for all collections
*/
async syncAll() {
if (this.isDisposed)
throw new Error("SyncManager is disposed");
const e = [];
if (await Promise.all([...this.collections.keys()].map((t) => this.sync(t).catch((s) => {
e.push({ id: t, error: s });
}))), e.length > 0)
throw new Error(`Error while syncing collections:
${e.map((t) => `${t.id}: ${t.error.message}`).join(`
`)}`);
}
/**
* Checks if a collection is currently beeing synced
* @param [name] Name of the collection. If not provided, it will check if any collection is currently beeing synced.
* @returns True if the collection is currently beeing synced, false otherwise.
*/
isSyncing(e) {
return this.syncOperations.findOne({
...e ? { collectionName: e } : {},
status: "active"
}, { fields: { status: 1 } }) != null;
}
/**
* Checks if the sync manager is ready to sync.
* @returns A promise that resolves when the sync manager is ready to sync.
*/
async isReady() {
await this.persistenceReady;
}
/**
* Starts the sync process for a collection
* @param name Name of the collection
* @param options Options for the sync process.
* @param options.force If true, the sync process will be started even if there are no changes and onlyWithChanges is true.
* @param options.onlyWithChanges If true, the sync process will only be started if there are changes.
*/
async sync(e, t = {}) {
if (this.isDisposed)
throw new Error("SyncManager is disposed");
await this.isReady();
const { options: s, readyPromise: n } = this.getCollectionProperties(e);
await n;
const o = this.syncOperations.find({
collectionName: e,
instanceId: this.instanceId,
status: "active"
}, {
reactive: !1
}).count() > 0, a = Date.now();
let r = null;
await new Promise((i) => {
setTimeout(i, 0);
});
const h = async () => {
const i = this.syncOperations.findOne({
collectionName: e,
status: "done"
}, {
sort: { end: -1 },
reactive: !1
});
if (t != null && t.onlyWithChanges && this.changes.find({
collectionName: e,
time: { $lte: a }
}, {
sort: { time: 1 },
reactive: !1
}).count() === 0)
return;
o || (r = this.syncOperations.insert({
start: a,
collectionName: e,
instanceId: this.instanceId,
status: "active"
}));
const l = await this.options.pull(s, {
lastFinishedSyncStart: i == null ? void 0 : i.start,
lastFinishedSyncEnd: i == null ? void 0 : i.end
});
await this.syncWithData(e, l);
};
await (t != null && t.force ? h() : this.getSyncQueue(e).add(h)).catch((i) => {
throw r != null && (this.options.onError && this.options.onError(s, i), this.syncOperations.updateOne({ id: r }, {
$set: { status: "error", end: Date.now(), error: i.stack || i.message }
})), i;
}), r != null && (this.syncOperations.removeMany({
id: { $ne: r },
collectionName: e,
$or: [
{ end: { $lte: a } },
{ status: "active" }
]
}), this.syncOperations.updateOne({ id: r }, {
$set: { status: "done", end: Date.now() }
}));
}
/**
* Starts the push process for a collection (sync process but only if there are changes)
* @param name Name of the collection
*/
async pushChanges(e) {
await this.sync(e, {
onlyWithChanges: !0
});
}
async syncWithData(e, t) {
const { collection: s, options: n } = this.getCollectionProperties(e), o = Date.now(), a = this.syncOperations.findOne({
collectionName: e,
status: "done"
}, {
sort: { end: -1 },
reactive: !1
}), r = this.snapshots.findOne({
collectionName: e
}, {
sort: { time: -1 },
reactive: !1
}), h = this.changes.find({
collectionName: e,
time: { $lte: o }
}, {
sort: { time: 1 },
reactive: !1
}).fetch();
await I({
changes: h,
lastSnapshot: r == null ? void 0 : r.items,
data: t,
pull: () => this.options.pull(n, {
lastFinishedSyncStart: a == null ? void 0 : a.start,
lastFinishedSyncEnd: a == null ? void 0 : a.end
}),
push: (i) => this.options.push(n, {
changes: i,
rawChanges: h
}),
insert: (i) => {
this.remoteChanges.push({
collectionName: e,
type: "insert",
data: i
}, {
collectionName: e,
type: "update",
data: { id: i.id, modifier: { $set: i } }
}), s.replaceOne({ id: i.id }, i, { upsert: !0 });
},
update: (i, l) => {
this.remoteChanges.push({
collectionName: e,
type: "insert",
data: { id: i, ...l.$set }
}, {
collectionName: e,
type: "update",
data: { id: i, modifier: l }
}), s.updateOne({ id: i }, {
...l,
$setOnInsert: { id: i }
}, { upsert: !0 });
},
remove: (i) => {
s.findOne({
id: i
}, { reactive: !1 }) && (this.remoteChanges.push({
collectionName: e,
type: "remove",
data: i
}), s.removeOne({ id: i }));
},
batch: (i) => {
s.batch(() => {
i();
});
}
}).then(async (i) => {
if (this.snapshots.removeMany({
collectionName: e,
time: { $lte: o }
}), this.changes.removeMany({
collectionName: e,
id: { $in: h.map((d) => d.id) }
}), this.snapshots.insert({
time: o,
collectionName: e,
items: i
}), await new Promise((d) => {
setTimeout(d, 0);
}), this.changes.find({
collectionName: e
}, { reactive: !1 }).count() > 0) {
await this.sync(e, {
force: !0,
onlyWithChanges: !0
});
return;
}
const f = s.find({
id: { $nin: i.map((d) => d.id) }
}, {
reactive: !1
}).map((d) => d.id);
s.batch(() => {
i.forEach((d) => {
this.remoteChanges.push({
collectionName: e,
type: "insert",
data: d
}, {
collectionName: e,
type: "update",
data: { id: d.id, modifier: { $set: d } }
}), s.replaceOne({ id: d.id }, d, { upsert: !0 });
}), f.forEach((d) => {
s.removeOne({ id: d });
});
});
});
}
}
export {
T as SyncManager
};
//# sourceMappingURL=index.mjs.map