@durable-streams/y-durable-streams
Version:
Yjs provider for Durable Streams - sync Yjs documents over append-only streams
527 lines (525 loc) • 15.9 kB
JavaScript
import * as Y from "yjs";
import * as awarenessProtocol from "y-protocols/awareness";
import { ObservableV2 } from "lib0/observable";
import * as decoding from "lib0/decoding";
import * as encoding from "lib0/encoding";
import { DurableStream, DurableStreamError, FetchError, IdempotentProducer } from "@durable-streams/client";
//#region src/yjs-provider.ts
/**
* Valid state transitions - documents the state machine at a glance.
* disconnected -> connecting (connect() called)
* connecting -> connected (initial sync complete)
* connecting -> disconnected (error or disconnect() called)
* connected -> disconnected (disconnect() or error)
*/
const VALID_TRANSITIONS = {
disconnected: [`connecting`],
connecting: [`connected`, `disconnected`],
connected: [`disconnected`]
};
/**
* Interval for awareness heartbeats (15 seconds).
*/
const AWARENESS_HEARTBEAT_INTERVAL = 15e3;
/**
* YjsProvider for the Yjs Durable Streams Protocol.
*/
var YjsProvider = class YjsProvider extends ObservableV2 {
doc;
awareness;
baseUrl;
docId;
headers;
liveMode;
_state = `disconnected`;
_connectionId = 0;
_ctx = null;
_synced = false;
updatesStreamGeneration = 0;
updatesSubscription = null;
sendingAwareness = false;
pendingAwareness = null;
awarenessHeartbeat = null;
constructor(options) {
super();
this.doc = options.doc;
this.awareness = options.awareness;
this.baseUrl = options.baseUrl.replace(/\/$/, ``);
this.docId = options.docId;
this.headers = options.headers ?? {};
this.liveMode = options.liveMode ?? `sse`;
this.doc.on(`update`, this.handleDocumentUpdate);
if (this.awareness) this.awareness.on(`update`, this.handleAwarenessUpdate);
if (options.connect !== false) this.connect();
}
get synced() {
return this._synced;
}
set synced(state) {
if (this._synced !== state) {
this._synced = state;
this.emit(`synced`, [state]);
}
}
/** True when connected to the server */
get connected() {
return this._state === `connected`;
}
/** True when connection is in progress */
get connecting() {
return this._state === `connecting`;
}
/**
* Transition to a new connection state.
* Returns false if the transition is invalid (logs a warning).
*/
transition(to) {
const allowed = VALID_TRANSITIONS[this._state];
if (!allowed.includes(to)) {
console.warn(`[YjsProvider] Invalid transition: ${this._state} -> ${to}`);
return false;
}
this._state = to;
this.emit(`status`, [to]);
return true;
}
/**
* Create a new connection context with a unique ID.
*/
createConnectionContext() {
this._connectionId += 1;
const ctx = {
id: this._connectionId,
controller: new AbortController(),
startOffset: `-1`,
producer: null
};
this._ctx = ctx;
return ctx;
}
/**
* Check if a connection context is stale (disconnected or replaced).
* Use this after every await to detect race conditions.
*/
isStale(ctx) {
return this._ctx !== ctx || ctx.controller.signal.aborted;
}
async connect() {
if (this._state !== `disconnected`) return;
if (!this.transition(`connecting`)) return;
const ctx = this.createConnectionContext();
try {
await this.ensureDocument(ctx);
if (this.isStale(ctx)) return;
await this.discoverSnapshot(ctx);
if (this.isStale(ctx)) return;
this.createUpdatesProducer(ctx);
await this.startUpdatesStream(ctx, ctx.startOffset);
if (this.isStale(ctx)) return;
if (this.awareness) this.startAwareness(ctx);
} catch (err) {
const isAborted = err instanceof Error && err.name === `AbortError`;
if (!isAborted && !this.isStale(ctx)) {
this.emit(`error`, [err instanceof Error ? err : new Error(String(err))]);
this.disconnect();
}
}
}
async disconnect() {
const ctx = this._ctx;
if (!ctx || this._state === `disconnected`) return;
this.transition(`disconnected`);
this._ctx = null;
this.synced = false;
if (this.awarenessHeartbeat) {
clearInterval(this.awarenessHeartbeat);
this.awarenessHeartbeat = null;
}
if (this.awareness) this.broadcastAwarenessRemoval();
this.updatesStreamGeneration += 1;
if (this.updatesSubscription) {
this.updatesSubscription();
this.updatesSubscription = null;
}
await this.closeUpdatesProducer(ctx);
ctx.controller.abort();
this.pendingAwareness = null;
}
destroy() {
this.disconnect().catch(() => {});
this.doc.off(`update`, this.handleDocumentUpdate);
if (this.awareness) this.awareness.off(`update`, this.handleAwarenessUpdate);
super.destroy();
}
/**
* Flush any pending updates to the server.
*
* @internal This method is primarily for testing to ensure all batched
* updates have been sent before making assertions. In production, updates
* are sent automatically via the IdempotentProducer's batching/linger mechanism.
*/
async flush() {
if (this._ctx?.producer) await this._ctx.producer.flush();
}
/**
* Get the document URL.
*/
docUrl() {
return `${this.baseUrl}/docs/${this.docId}`;
}
/**
* Get the awareness URL for a named stream.
*/
awarenessUrl(name = `default`) {
return `${this.docUrl()}?awareness=${encodeURIComponent(name)}`;
}
/**
* Create the document on the server via PUT.
* Idempotent: succeeds if document already exists with matching config.
*/
async ensureDocument(ctx) {
const url = this.docUrl();
const response = await fetch(url, {
method: `PUT`,
headers: {
...this.headers,
"content-type": `application/octet-stream`
},
signal: ctx.controller.signal
});
if (response.status === 201 || response.status === 200) {
await response.arrayBuffer();
return;
}
if (response.status === 409) {
await response.arrayBuffer();
return;
}
const text = await response.text().catch(() => ``);
throw new Error(`Failed to create document: ${response.status} ${text}`);
}
/**
* Discover the current snapshot state via ?offset=snapshot.
* Handles 307 redirect to determine starting offset.
*/
async discoverSnapshot(ctx) {
const url = `${this.docUrl()}?offset=snapshot`;
const response = await fetch(url, {
method: `GET`,
headers: this.headers,
redirect: `manual`,
signal: ctx.controller.signal
});
if (response.status === 307) {
const location = response.headers.get(`location`);
if (location) {
const redirectUrl = new URL(location, url);
const offset = redirectUrl.searchParams.get(`offset`);
if (offset) {
if (offset.endsWith(`_snapshot`)) await this.loadSnapshot(ctx, offset);
else ctx.startOffset = offset;
return;
}
}
}
ctx.startOffset = `-1`;
}
/**
* Load a snapshot from the server.
*/
async loadSnapshot(ctx, snapshotOffset) {
const url = `${this.docUrl()}?offset=${encodeURIComponent(snapshotOffset)}`;
try {
const response = await fetch(url, {
method: `GET`,
headers: this.headers,
signal: ctx.controller.signal
});
if (!response.ok) {
if (response.status === 404) {
await this.discoverSnapshot(ctx);
return;
}
throw new Error(`Failed to load snapshot: ${response.status}`);
}
const data = new Uint8Array(await response.arrayBuffer());
if (data.length > 0) Y.applyUpdate(this.doc, data, `server`);
const nextOffset = response.headers.get(`stream-next-offset`);
ctx.startOffset = nextOffset ?? `-1`;
} catch (err) {
if (this.isNotFoundError(err)) {
await this.discoverSnapshot(ctx);
return;
}
throw err;
}
}
createUpdatesProducer(ctx) {
const stream = new DurableStream({
url: this.docUrl(),
headers: this.headers,
contentType: `application/octet-stream`
});
const producerId = `${this.docId}-${this.doc.clientID}`;
ctx.producer = new IdempotentProducer(stream, producerId, {
autoClaim: true,
signal: ctx.controller.signal,
onError: (err) => {
if (err instanceof Error && err.name === `AbortError`) return;
console.error(`[YjsProvider] Producer error:`, err);
this.emit(`error`, [err]);
if (!this.isAuthError(err)) {
this.disconnect();
this.connect();
}
}
});
}
async closeUpdatesProducer(ctx) {
if (!ctx.producer) return;
try {
await ctx.producer.close();
} catch {}
ctx.producer = null;
}
startUpdatesStream(ctx, offset) {
if (ctx.controller.signal.aborted) return Promise.resolve();
this.updatesStreamGeneration += 1;
const generation = this.updatesStreamGeneration;
this.updatesSubscription?.();
this.updatesSubscription = null;
let settled = false;
let resolveInitial;
let rejectInitial;
const initialPromise = new Promise((resolve, reject) => {
resolveInitial = () => {
if (!settled) {
settled = true;
resolve();
}
};
rejectInitial = (error) => {
if (!settled) {
settled = true;
reject(error);
}
};
});
this.runUpdatesStream(ctx, offset, generation, resolveInitial, rejectInitial).catch((err) => {
rejectInitial(err instanceof Error ? err : new Error(String(err)));
});
return initialPromise;
}
async runUpdatesStream(ctx, offset, generation, resolveInitialSync, rejectInitialSync) {
let currentOffset = offset;
let initialSyncPending = true;
const markSynced = () => {
if (!initialSyncPending) return;
initialSyncPending = false;
if (this._state === `connecting`) this.transition(`connected`);
this.synced = true;
resolveInitialSync();
};
const isStale = () => this.isStale(ctx) || this.updatesStreamGeneration !== generation;
while (this.updatesStreamGeneration === generation) {
if (ctx.controller.signal.aborted) {
markSynced();
return;
}
const stream = new DurableStream({
url: this.docUrl(),
headers: this.headers,
contentType: `application/octet-stream`
});
try {
const response = await stream.stream({
offset: currentOffset,
live: this.liveMode,
signal: ctx.controller.signal
});
this.updatesSubscription?.();
this.updatesSubscription = response.subscribeBytes(async (chunk) => {
if (isStale()) return;
currentOffset = chunk.offset;
if (chunk.data.length > 0) this.applyUpdates(chunk.data);
if (initialSyncPending && chunk.upToDate) markSynced();
else if (chunk.data.length > 0) this.synced = true;
});
await response.closed;
markSynced();
continue;
} catch (err) {
if (isStale()) {
markSynced();
return;
}
if (this.isNotFoundError(err)) {
if (initialSyncPending) {
rejectInitialSync(err instanceof Error ? err : new Error(String(err)));
return;
}
this.emit(`error`, [err instanceof Error ? err : new Error(String(err))]);
this.disconnect();
return;
}
if (initialSyncPending) {
rejectInitialSync(err instanceof Error ? err : new Error(String(err)));
return;
}
await new Promise((resolve) => setTimeout(resolve, 1e3));
} finally {
if (this.updatesSubscription) {
this.updatesSubscription();
this.updatesSubscription = null;
}
}
}
}
/**
* Frame data with lib0 length-prefix encoding for transport.
*/
static frameUpdate(data) {
const encoder = encoding.createEncoder();
encoding.writeVarUint8Array(encoder, data);
return encoding.toUint8Array(encoder);
}
/**
* Apply lib0-framed updates from the server.
*/
applyUpdates(data) {
if (data.length === 0) return;
const decoder = decoding.createDecoder(data);
while (decoding.hasContent(decoder)) {
const update = decoding.readVarUint8Array(decoder);
Y.applyUpdate(this.doc, update, `server`);
}
}
/**
* Apply lib0-framed awareness updates from the server.
*/
applyAwarenessUpdates(data) {
if (data.length === 0 || !this.awareness) return;
try {
const decoder = decoding.createDecoder(data);
while (decoding.hasContent(decoder)) {
const update = decoding.readVarUint8Array(decoder);
try {
awarenessProtocol.applyAwarenessUpdate(this.awareness, update, `server`);
} catch {}
}
} catch {}
}
handleDocumentUpdate = (update, origin) => {
if (origin === `server`) return;
const producer = this._ctx?.producer;
if (!producer || !this.connected) return;
this.synced = false;
producer.append(YjsProvider.frameUpdate(update));
};
startAwareness(ctx) {
if (!this.awareness) return;
if (ctx.controller.signal.aborted) return;
this.broadcastAwareness();
this.awarenessHeartbeat = setInterval(() => {
this.broadcastAwareness();
}, AWARENESS_HEARTBEAT_INTERVAL);
this.subscribeAwareness(ctx);
}
handleAwarenessUpdate = (update, origin) => {
if (!this.awareness || origin === `server` || origin === this) return;
const { added, updated, removed } = update;
const changedClients = added.concat(updated).concat(removed);
if (!changedClients.includes(this.awareness.clientID)) return;
this.pendingAwareness = update;
this.sendAwareness();
};
broadcastAwareness() {
if (!this.awareness) return;
this.pendingAwareness = {
added: [this.awareness.clientID],
updated: [],
removed: []
};
this.sendAwareness();
}
broadcastAwarenessRemoval() {
if (!this.awareness) return;
try {
this.awareness.setLocalState(null);
const encoded = awarenessProtocol.encodeAwarenessUpdate(this.awareness, [this.awareness.clientID]);
const stream = new DurableStream({
url: this.awarenessUrl(),
headers: this.headers,
contentType: `application/octet-stream`
});
stream.append(YjsProvider.frameUpdate(encoded), { contentType: `application/octet-stream` }).catch(() => {});
} catch {}
}
async sendAwareness() {
if (!this.awareness || !this.connected && !this.connecting || this.sendingAwareness) return;
this.sendingAwareness = true;
try {
while (this.pendingAwareness) {
const update = this.pendingAwareness;
this.pendingAwareness = null;
const { added, updated, removed } = update;
const changedClients = added.concat(updated).concat(removed);
const encoded = awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients);
const stream = new DurableStream({
url: this.awarenessUrl(),
headers: this.headers,
contentType: `application/octet-stream`
});
await stream.append(YjsProvider.frameUpdate(encoded), { contentType: `application/octet-stream` });
}
} catch (err) {
console.error(`[YjsProvider] Failed to send awareness:`, err);
} finally {
this.sendingAwareness = false;
}
}
async subscribeAwareness(ctx) {
if (!this.awareness) return;
const signal = ctx.controller.signal;
if (signal.aborted) return;
const stream = new DurableStream({
url: this.awarenessUrl(),
headers: this.headers,
contentType: `application/octet-stream`
});
try {
const response = await stream.stream({
offset: `now`,
live: `sse`,
signal
});
response.closed.catch(() => {});
response.subscribeBytes(async (chunk) => {
if (signal.aborted) return;
if (chunk.data.length > 0) this.applyAwarenessUpdates(chunk.data);
});
await response.closed;
if (this.connected && !signal.aborted) {
await new Promise((r) => setTimeout(r, 250));
this.subscribeAwareness(ctx);
}
} catch (err) {
if (signal.aborted || !this.connected && !this.connecting) return;
if (this.isNotFoundError(err)) {
console.error(`[YjsProvider] Awareness stream not found`);
return;
}
console.error(`[YjsProvider] Awareness stream error:`, err);
await new Promise((resolve) => setTimeout(resolve, 1e3));
if (this.connected) this.subscribeAwareness(ctx);
}
}
isNotFoundError(err) {
return err instanceof DurableStreamError && err.code === `NOT_FOUND` || err instanceof FetchError && err.status === 404;
}
isAuthError(err) {
return err instanceof DurableStreamError && (err.code === `UNAUTHORIZED` || err.code === `FORBIDDEN`) || err instanceof FetchError && (err.status === 401 || err.status === 403);
}
};
//#endregion
export { AWARENESS_HEARTBEAT_INTERVAL, YjsProvider };