@durable-streams/y-durable-streams
Version:
Yjs provider for Durable Streams - sync Yjs documents over append-only streams
1,024 lines (1,020 loc) • 35.2 kB
JavaScript
import * as Y from "yjs";
import * as decoding from "lib0/decoding";
import "lib0/encoding";
import { DurableStream, DurableStreamError, FetchError } from "@durable-streams/client";
import { createServer } from "node:http";
//#region src/server/types.ts
/**
* Headers used by the Yjs protocol layer (lowercase per protocol spec).
*/
const YJS_HEADERS = {
STREAM_NEXT_OFFSET: `stream-next-offset`,
STREAM_UP_TO_DATE: `stream-up-to-date`,
STREAM_CURSOR: `stream-cursor`
};
/**
* Stream path builders for consistent path generation.
* All operations use the same document URL with query parameters.
*
* Internal streams use `.` prefixed segments (e.g., `.updates`, `.index`, `.snapshots`)
* which are safe from user collisions since document paths reject `.` characters.
*/
const YjsStreamPaths = {
doc(service, docPath) {
return `/v1/yjs/${service}/docs/${docPath}`;
},
dsStream(service, docPath) {
return `/v1/stream/yjs/${service}/docs/${docPath}/.updates`;
},
indexStream(service, docPath) {
return `/v1/stream/yjs/${service}/docs/${docPath}/.index`;
},
snapshotStream(service, docPath, snapshotKey) {
return `/v1/stream/yjs/${service}/docs/${docPath}/.snapshots/${snapshotKey}`;
},
awarenessStream(service, docPath, name) {
return `/v1/stream/yjs/${service}/docs/${docPath}/.awareness/${name}`;
},
awarenessIndexStream(service, docPath) {
return `/v1/stream/yjs/${service}/docs/${docPath}/.awareness/.index`;
},
snapshotKey(offset) {
return `${offset}_snapshot`;
},
parseSnapshotOffset(key) {
const match = key.match(/^(.+)_snapshot$/);
return match ? match[1] : null;
}
};
/**
* Path normalization utilities.
*/
const PathUtils = { normalize(path) {
let decoded;
try {
decoded = decodeURIComponent(path);
} catch {
return null;
}
const normalized = decoded.replace(/\/+/g, `/`);
const trimmed = normalized.replace(/^\/|\/$/g, ``);
const segments = trimmed.split(`/`);
for (const segment of segments) if (segment === `..` || segment === `.`) return null;
if (!/^[a-zA-Z0-9_\-/]*$/.test(normalized)) return null;
if (normalized.length > 256) return null;
return normalized;
} };
//#endregion
//#region src/server/compaction.ts
/**
* Check if an error is a 404 Not Found error.
*/
function isNotFoundError$1(err) {
return err instanceof DurableStreamError && err.code === `NOT_FOUND` || err instanceof FetchError && err.status === 404;
}
/**
* Handles document compaction.
*/
var Compactor = class {
server;
constructor(server) {
this.server = server;
}
/**
* Trigger compaction for a document.
* Uses atomic check-and-set to prevent concurrent compactions.
*/
async triggerCompaction(service, docPath) {
if (!this.server.tryStartCompaction(service, docPath)) return;
try {
await this.performCompaction(service, docPath);
} finally {
this.server.setCompacting(service, docPath, false);
}
}
/**
* Perform the actual compaction.
*/
async performCompaction(service, docPath) {
const state = this.server.getDocumentState(service, docPath);
if (!state) throw new Error(`Document state not found for ${service}/${docPath}`);
const dsServerUrl = this.server.getDsServerUrl();
const dsHeaders = this.server.getDsServerHeaders();
const doc = new Y.Doc();
try {
if (state.snapshotOffset) {
const snapshotKey$1 = YjsStreamPaths.snapshotKey(state.snapshotOffset);
const snapshotUrl = `${dsServerUrl}${YjsStreamPaths.snapshotStream(service, docPath, snapshotKey$1)}`;
const stream = new DurableStream({
url: snapshotUrl,
headers: dsHeaders,
contentType: `application/octet-stream`
});
try {
const response = await stream.stream({ offset: `-1` });
const snapshot = await response.body();
if (snapshot.length > 0) Y.applyUpdate(doc, snapshot);
} catch (err) {
if (!isNotFoundError$1(err)) throw err;
}
}
const updatesUrl = `${dsServerUrl}${YjsStreamPaths.dsStream(service, docPath)}`;
const updatesStream = new DurableStream({
url: updatesUrl,
headers: dsHeaders,
contentType: `application/octet-stream`
});
const updatesOffset = state.snapshotOffset ? incrementOffset(state.snapshotOffset) : `-1`;
let currentEndOffset = state.snapshotOffset ?? `-1`;
try {
const response = await updatesStream.stream({ offset: updatesOffset });
const updatesData = await response.body();
if (updatesData.length > 0) {
const decoder = decoding.createDecoder(updatesData);
while (decoding.hasContent(decoder)) {
const update = decoding.readVarUint8Array(decoder);
Y.applyUpdate(doc, update);
}
}
currentEndOffset = response.offset;
} catch (err) {
if (isNotFoundError$1(err)) console.error(`[Compactor] Updates stream not found for ${service}/${docPath} during compaction`);
throw err;
}
const newSnapshot = Y.encodeStateAsUpdate(doc);
const snapshotKey = YjsStreamPaths.snapshotKey(currentEndOffset);
const newSnapshotUrl = `${dsServerUrl}${YjsStreamPaths.snapshotStream(service, docPath, snapshotKey)}`;
const snapshotStream = await DurableStream.create({
url: newSnapshotUrl,
headers: dsHeaders,
contentType: `application/octet-stream`
});
await snapshotStream.append(newSnapshot, { contentType: `application/octet-stream` });
const oldSnapshotOffset = state.snapshotOffset;
await this.writeIndexEntry(service, docPath, currentEndOffset);
this.server.updateSnapshotOffset(service, docPath, currentEndOffset);
this.server.resetUpdateCounters(service, docPath);
if (oldSnapshotOffset) this.deleteOldSnapshot(service, docPath, oldSnapshotOffset).catch((err) => {
console.error(`[Compactor] Error deleting old snapshot for ${service}/${docPath}:`, err);
});
const result = {
snapshotOffset: currentEndOffset,
snapshotSizeBytes: newSnapshot.length,
oldSnapshotOffset
};
console.log(`[Compactor] Compacted ${service}/${docPath}: snapshot=${newSnapshot.length} bytes, offset=${currentEndOffset}`);
return result;
} finally {
doc.destroy();
}
}
/**
* Write a new entry to the internal index stream.
* This persists the snapshot offset so it survives server restarts.
*/
async writeIndexEntry(service, docPath, snapshotOffset) {
const indexPath = YjsStreamPaths.indexStream(service, docPath);
const indexEntry = {
snapshotOffset,
createdAt: Date.now()
};
await this.server.appendToIndexStream(indexPath, indexEntry);
}
/**
* Delete old snapshot.
*/
async deleteOldSnapshot(service, docPath, snapshotOffset) {
const dsServerUrl = this.server.getDsServerUrl();
const dsHeaders = this.server.getDsServerHeaders();
const snapshotKey = YjsStreamPaths.snapshotKey(snapshotOffset);
const snapshotUrl = `${dsServerUrl}${YjsStreamPaths.snapshotStream(service, docPath, snapshotKey)}`;
try {
await DurableStream.delete({
url: snapshotUrl,
headers: dsHeaders
});
} catch (err) {
if (!isNotFoundError$1(err)) throw err;
}
}
};
/**
* Increment an offset string by 1 in the sequence portion.
* Offsets are formatted as "{timestamp}_{sequence}" with zero-padded parts.
*/
function incrementOffset(offset) {
const parts = offset.split(`_`);
if (parts.length !== 2) return offset;
const seq = parseInt(parts[1], 10);
if (isNaN(seq)) return offset;
const nextSeq = (seq + 1).toString().padStart(parts[1].length, `0`);
return `${parts[0]}_${nextSeq}`;
}
//#endregion
//#region src/server/yjs-server.ts
const DEFAULT_COMPACTION_THRESHOLD = 1024 * 1024;
/**
* Check if an error is a 404 Not Found error.
*/
function isNotFoundError(err) {
return err instanceof DurableStreamError && err.code === `NOT_FOUND` || err instanceof FetchError && err.status === 404;
}
/**
* Parse the URL path and extract route parameters.
* Expected format: /v1/yjs/:service/docs/:docPath
* where docPath can include forward slashes.
*/
function parseRoute(path) {
const match = path.match(/^\/v1\/yjs\/([^/]+)\/docs\/(.+)$/);
if (!match) return null;
const docPath = PathUtils.normalize(match[2]);
if (!docPath) return null;
return {
service: match[1],
docPath
};
}
/**
* HTTP server implementing the Yjs Durable Streams Protocol.
*/
var YjsServer = class {
dsServerUrl;
dsServerHeaders;
compactionThreshold;
port;
host;
compactor;
documentStates = new Map();
stateKey(service, docPath) {
return `${service}/${docPath}`;
}
server = null;
_url = null;
constructor(options) {
this.dsServerUrl = options.dsServerUrl;
this.dsServerHeaders = options.dsServerHeaders ?? {};
this.compactionThreshold = options.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
this.port = options.port ?? 0;
this.host = options.host ?? `127.0.0.1`;
this.compactor = new Compactor(this);
}
async start() {
if (this.server) throw new Error(`Server already started`);
return new Promise((resolve, reject) => {
this.server = createServer((req, res) => {
this.handleRequest(req, res).catch((err) => {
console.error(`[YjsServer] Request error:`, err);
if (!res.headersSent) {
res.writeHead(500, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INTERNAL_ERROR`,
message: `Internal error`
} }));
}
});
});
this.server.on(`error`, reject);
this.server.listen(this.port, this.host, () => {
const addr = this.server.address();
if (typeof addr === `string`) this._url = addr;
else if (addr) this._url = `http://${this.host}:${addr.port}`;
resolve(this._url);
});
});
}
async stop() {
if (!this.server) return;
this.server.closeAllConnections();
return new Promise((resolve, reject) => {
this.server.close((err) => {
if (err) {
reject(err);
return;
}
this.server = null;
this._url = null;
resolve();
});
});
}
get url() {
if (!this._url) throw new Error(`Server not started`);
return this._url;
}
async handleRequest(req, res) {
const rawUrl = req.url ?? `/`;
const url = new URL(rawUrl, `http://${req.headers.host}`);
const path = url.pathname;
const method = req.method?.toUpperCase();
res.setHeader(`access-control-allow-origin`, `*`);
res.setHeader(`access-control-allow-methods`, `GET, POST, PUT, DELETE, OPTIONS`);
res.setHeader(`access-control-allow-headers`, `authorization, content-type, stream-offset, stream-live, producer-id, producer-epoch, producer-seq, stream-producer-id, stream-producer-epoch, stream-producer-seq`);
res.setHeader(`access-control-expose-headers`, `stream-next-offset, stream-up-to-date, stream-cursor, location`);
if (method === `OPTIONS`) {
res.writeHead(204);
res.end();
return;
}
if (rawUrl.includes(`/..`) || rawUrl.includes(`/./`) || rawUrl.endsWith(`/.`)) {
res.writeHead(400, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INVALID_REQUEST`,
message: `Invalid document path`
} }));
return;
}
const route = parseRoute(path);
if (!route) {
if (path.startsWith(`/v1/yjs/`)) {
res.writeHead(400, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INVALID_REQUEST`,
message: `Invalid document path`
} }));
return;
}
await this.proxyToDsServer(req, res, rawUrl);
return;
}
const offset = url.searchParams.get(`offset`);
const awareness = url.searchParams.get(`awareness`);
const live = url.searchParams.get(`live`);
try {
if (awareness !== null) {
await this.handleAwareness(req, res, route, awareness, url);
return;
}
if (offset === `snapshot`) {
await this.handleSnapshotDiscovery(res, route, url);
return;
}
if (offset && offset.endsWith(`_snapshot`)) {
await this.handleSnapshotRead(req, res, route, offset, url);
return;
}
if (method === `GET`) await this.handleUpdatesRead(req, res, route, offset, live, url);
else if (method === `HEAD`) await this.handleDocumentHead(req, res, route);
else if (method === `POST`) await this.handleUpdateWrite(req, res, route);
else if (method === `PUT`) await this.handleDocumentCreate(req, res, route);
else if (method === `DELETE`) await this.handleDocumentDelete(res, route);
else {
res.writeHead(405, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INVALID_REQUEST`,
message: `Method not allowed`
} }));
}
} catch (err) {
console.error(`[YjsServer] Error handling ${method} ${path}:`, err);
if (!res.headersSent) {
res.writeHead(500, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INTERNAL_ERROR`,
message: `Internal error`
} }));
}
}
}
/**
* Proxy requests that don't match Yjs routes to the underlying DS server.
* This allows clients to use a single endpoint for both Yjs and raw DS operations.
*/
async proxyToDsServer(req, res, path) {
const targetUrl = `${this.dsServerUrl}${path}`;
const method = req.method ?? `GET`;
let body;
if (method !== `GET` && method !== `HEAD`) {
const chunks = [];
for await (const chunk of req) chunks.push(chunk);
if (chunks.length > 0) body = Buffer.concat(chunks);
}
const headers = { ...this.dsServerHeaders };
for (const [key, value] of Object.entries(req.headers)) if (key.toLowerCase() !== `host` && value) headers[key] = Array.isArray(value) ? value.join(`, `) : value;
try {
const response = await fetch(targetUrl, {
method,
headers,
body: body ? new Uint8Array(body) : void 0
});
const responseHeaders = {};
response.headers.forEach((value, key) => {
if (key === `content-encoding` || key === `content-length`) return;
responseHeaders[key] = value;
});
res.writeHead(response.status, responseHeaders);
if (response.body) for await (const chunk of response.body) res.write(chunk);
res.end();
} catch (err) {
console.error(`[YjsServer] Proxy error for ${method} ${path}:`, err);
if (!res.headersSent) {
res.writeHead(502, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `PROXY_ERROR`,
message: `Failed to proxy request`
} }));
}
}
}
/**
* POST with auto-create on 404: try POST, if stream doesn't exist
* create it via PUT and retry. Handles awareness streams that may
* have expired due to TTL.
*/
async postWithAutoCreate(req, res, dsPath, docDsPath) {
const body = await this.readBody(req);
const headers = {
...this.dsServerHeaders,
"content-type": req.headers[`content-type`] ?? `application/octet-stream`
};
const targetUrl = `${this.dsServerUrl}${dsPath}`;
const response = await fetch(targetUrl, {
method: `POST`,
headers,
body: body.length > 0 ? new Uint8Array(body) : void 0
});
if (response.status === 404) {
await response.arrayBuffer();
if (docDsPath) {
const headUrl = `${this.dsServerUrl}${docDsPath}`;
const headResponse = await fetch(headUrl, {
method: `HEAD`,
headers: this.dsServerHeaders
});
if (headResponse.status === 404) {
res.writeHead(404, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `DOCUMENT_NOT_FOUND`,
message: `Document does not exist`
} }));
return;
}
if (!headResponse.ok) throw new Error(`Document existence check failed: ${headResponse.status}`);
}
await this.tryCreateStream(dsPath);
const retryResponse = await fetch(targetUrl, {
method: `POST`,
headers,
body: body.length > 0 ? new Uint8Array(body) : void 0
});
await this.forwardResponse(res, retryResponse);
} else await this.forwardResponse(res, response);
}
async handleSnapshotDiscovery(res, route, originalUrl) {
const dsPath = YjsStreamPaths.dsStream(route.service, route.docPath);
const headUrl = `${this.dsServerUrl}${dsPath}`;
try {
const headResponse = await fetch(headUrl, {
method: `HEAD`,
headers: this.dsServerHeaders
});
if (headResponse.status === 404) {
res.writeHead(404, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `DOCUMENT_NOT_FOUND`,
message: `Document does not exist`
} }));
return;
}
} catch {}
const state = this.getOrCreateDocumentState(route.service, route.docPath);
if (!state.snapshotOffset) {
const loadedOffset = await this.loadSnapshotOffsetFromIndex(route.service, route.docPath);
if (loadedOffset) state.snapshotOffset = loadedOffset;
}
const redirectUrl = new URL(originalUrl.href);
if (state.snapshotOffset) redirectUrl.searchParams.set(`offset`, YjsStreamPaths.snapshotKey(state.snapshotOffset));
else redirectUrl.searchParams.set(`offset`, `-1`);
const redirectPath = `${redirectUrl.pathname}${redirectUrl.search}`;
res.writeHead(307, {
location: redirectPath,
"cache-control": `private, max-age=5`
});
res.end();
}
/**
* Load the latest snapshot offset from the internal index stream.
* Returns null if no index exists or it's empty.
*/
async loadSnapshotOffsetFromIndex(service, docPath) {
const indexUrl = `${this.dsServerUrl}${YjsStreamPaths.indexStream(service, docPath)}`;
try {
const stream = new DurableStream({
url: indexUrl,
headers: this.dsServerHeaders,
contentType: `application/json`
});
const response = await stream.stream({ offset: `-1` });
const body = await response.text();
if (!body || body.trim().length === 0) return null;
try {
const parsed = JSON.parse(body);
if (Array.isArray(parsed)) {
const last = parsed[parsed.length - 1];
if (last?.snapshotOffset) return last.snapshotOffset;
} else if (parsed && typeof parsed === `object` && `snapshotOffset` in parsed) return parsed.snapshotOffset;
} catch {}
const lines = body.trim().split(`\n`);
for (let i = lines.length - 1; i >= 0; i -= 1) {
const line = lines[i]?.trim();
if (!line) continue;
try {
const entry = JSON.parse(line);
if (entry.snapshotOffset) return entry.snapshotOffset;
} catch {}
}
return null;
} catch (err) {
if (isNotFoundError(err)) return null;
console.error(`[YjsServer] Error loading index for ${service}/${docPath}:`, err);
return null;
}
}
async handleSnapshotRead(req, res, route, offset, _url) {
const snapshotOffset = YjsStreamPaths.parseSnapshotOffset(offset);
if (!snapshotOffset) {
res.writeHead(400, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INVALID_REQUEST`,
message: `Invalid snapshot offset`
} }));
return;
}
const dsPath = YjsStreamPaths.snapshotStream(route.service, route.docPath, offset);
const dsUrl = new URL(dsPath, this.dsServerUrl);
try {
const dsResponse = await fetch(dsUrl.toString(), {
method: `GET`,
headers: {
...this.dsServerHeaders,
"stream-offset": `-1`
}
});
if (!dsResponse.ok) {
if (dsResponse.status === 404) {
res.writeHead(404, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `SNAPSHOT_NOT_FOUND`,
message: `Snapshot not found`
} }));
return;
}
throw new Error(`DS server returned ${dsResponse.status}`);
}
const responseHeaders = {
"content-type": `application/octet-stream`,
[YJS_HEADERS.STREAM_NEXT_OFFSET]: this.incrementOffset(snapshotOffset)
};
res.writeHead(200, responseHeaders);
if (dsResponse.body) {
const reader = dsResponse.body.getReader();
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
res.write(value);
}
} finally {
reader.releaseLock();
}
}
res.end();
} catch (err) {
if (isNotFoundError(err)) {
res.writeHead(404, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `SNAPSHOT_NOT_FOUND`,
message: `Snapshot not found`
} }));
return;
}
throw err;
}
}
/**
* Increment an offset string for the next read position.
* Offsets are formatted as "{timestamp}_{sequence}" padded strings.
* Increments the sequence portion by 1 so the client reads from the
* position after the snapshot, not from the snapshot offset itself.
*/
incrementOffset(offset) {
const parts = offset.split(`_`);
if (parts.length !== 2) return offset;
const seq = parseInt(parts[1], 10);
if (isNaN(seq)) return offset;
const nextSeq = (seq + 1).toString().padStart(parts[1].length, `0`);
return `${parts[0]}_${nextSeq}`;
}
/**
* GET - Proxy to read from .updates stream.
*/
async handleUpdatesRead(req, res, route, offset, live, _url) {
let dsPath = YjsStreamPaths.dsStream(route.service, route.docPath);
const params = new URLSearchParams();
if (offset !== null) params.set(`offset`, offset);
if (live) params.set(`live`, live === `true` ? `long-poll` : live);
const query = params.toString();
if (query) dsPath = `${dsPath}?${query}`;
if (live === `sse`) await this.proxyWithSseFlush(req, res, dsPath);
else await this.proxyToDsServer(req, res, dsPath);
}
/**
* HEAD - Proxy to check .updates stream existence.
*/
async handleDocumentHead(req, res, route) {
const dsPath = YjsStreamPaths.dsStream(route.service, route.docPath);
await this.proxyToDsServer(req, res, dsPath);
}
/**
* PUT - Create document: creates both .updates and .awareness.default streams.
*/
async handleDocumentCreate(req, res, route) {
const dsPath = YjsStreamPaths.dsStream(route.service, route.docPath);
const targetUrl = `${this.dsServerUrl}${dsPath}`;
const headers = { ...this.dsServerHeaders };
for (const [key, value] of Object.entries(req.headers)) if (key.toLowerCase() !== `host` && value) headers[key] = Array.isArray(value) ? value.join(`, `) : value;
if (!headers[`content-type`]) headers[`content-type`] = `application/octet-stream`;
const body = await this.readBody(req);
const dsResponse = await fetch(targetUrl, {
method: `PUT`,
headers,
body: body.length > 0 ? new Uint8Array(body) : void 0
});
if (dsResponse.status === 201 || dsResponse.status === 200) {
const awarenessPath = YjsStreamPaths.awarenessStream(route.service, route.docPath, `default`);
await this.tryCreateStream(awarenessPath).catch((err) => {
console.error(`[YjsServer] Failed to create awareness stream:`, err);
});
}
await this.forwardResponse(res, dsResponse);
}
/**
* POST - Streaming proxy to write to .updates stream.
* Client sends lib0-framed updates; we pass through directly.
* Returns 404 if the document does not exist.
*/
async handleUpdateWrite(req, res, route) {
const stateKey = this.stateKey(route.service, route.docPath);
const body = await this.readBody(req);
const dsPath = YjsStreamPaths.dsStream(route.service, route.docPath);
const targetUrl = `${this.dsServerUrl}${dsPath}`;
const headers = {
...this.dsServerHeaders,
"content-type": `application/octet-stream`
};
for (const h of [
`producer-id`,
`producer-epoch`,
`producer-seq`,
`stream-producer-id`,
`stream-producer-epoch`,
`stream-producer-seq`
]) {
const v = req.headers[h];
if (typeof v === `string`) headers[h] = v;
}
const dsResponse = await fetch(targetUrl, {
method: `POST`,
headers,
body: body.length > 0 ? new Uint8Array(body) : void 0
});
await this.forwardResponse(res, dsResponse);
if (dsResponse.status >= 200 && dsResponse.status < 300) {
if (!this.documentStates.has(stateKey)) this.documentStates.set(stateKey, {
snapshotOffset: null,
updatesSizeBytes: 0,
compacting: false
});
const state = this.documentStates.get(stateKey);
state.updatesSizeBytes += body.length;
if (this.shouldTriggerCompaction(state)) this.compactor.triggerCompaction(route.service, route.docPath).catch((err) => {
console.error(`[YjsServer] Compaction error:`, err);
});
}
}
/**
* DELETE - Delete document and cascade to associated streams.
*/
async handleDocumentDelete(res, route) {
const { service, docPath } = route;
const dsPath = YjsStreamPaths.dsStream(service, docPath);
const dsUrl = `${this.dsServerUrl}${dsPath}`;
const response = await fetch(dsUrl, {
method: `DELETE`,
headers: this.dsServerHeaders
});
if (response.status === 404) {
await response.arrayBuffer();
res.writeHead(404, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `DOCUMENT_NOT_FOUND`,
message: `Document does not exist`
} }));
return;
}
if (!response.ok) {
const text = await response.text().catch(() => ``);
throw new Error(`Failed to delete document stream: ${response.status} ${text}`);
}
await response.arrayBuffer();
const stateKey = this.stateKey(service, docPath);
this.documentStates.delete(stateKey);
await this.cascadeDeleteStreams(service, docPath).catch((err) => {
console.error(`[YjsServer] Cascade delete failed for ${docPath}:`, err);
});
res.writeHead(204);
res.end();
}
/**
* Best-effort cascade delete of snapshot and awareness streams.
* Errors are logged but do not propagate.
*/
async cascadeDeleteStreams(service, docPath) {
const deleteStream = async (dsPath) => {
try {
await DurableStream.delete({
url: `${this.dsServerUrl}${dsPath}`,
headers: this.dsServerHeaders
});
} catch {}
};
const [snapshotOffsets, awarenessNames] = await Promise.all([this.loadIndexEntries(YjsStreamPaths.indexStream(service, docPath), (entry) => entry.snapshotOffset), this.loadIndexEntries(YjsStreamPaths.awarenessIndexStream(service, docPath), (entry) => entry.name)]);
const pathsToDelete = [];
for (const offset of snapshotOffsets) {
const snapshotKey = YjsStreamPaths.snapshotKey(offset);
pathsToDelete.push(YjsStreamPaths.snapshotStream(service, docPath, snapshotKey));
}
pathsToDelete.push(YjsStreamPaths.indexStream(service, docPath));
pathsToDelete.push(YjsStreamPaths.awarenessStream(service, docPath, `default`));
for (const name of awarenessNames) pathsToDelete.push(YjsStreamPaths.awarenessStream(service, docPath, name));
pathsToDelete.push(YjsStreamPaths.awarenessIndexStream(service, docPath));
await Promise.allSettled(pathsToDelete.map(deleteStream));
}
/**
* Load entries from an index stream, extracting a value from each entry.
* Returns deduplicated values. Returns empty array if index doesn't exist.
*/
async loadIndexEntries(dsPath, extractValue) {
const indexUrl = `${this.dsServerUrl}${dsPath}`;
try {
const stream = new DurableStream({
url: indexUrl,
headers: this.dsServerHeaders,
contentType: `application/json`
});
const response = await stream.stream({ offset: `-1` });
const body = await response.text();
if (!body || body.trim().length === 0) return [];
const values = new Set();
try {
const parsed = JSON.parse(body);
if (Array.isArray(parsed)) {
for (const entry of parsed) {
const val = extractValue(entry);
if (val) values.add(val);
}
return [...values];
}
} catch {}
const lines = body.trim().split(`\n`);
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) continue;
try {
const entry = JSON.parse(trimmed);
const val = extractValue(entry);
if (val) values.add(val);
} catch {}
}
return [...values];
} catch (err) {
if (!isNotFoundError(err)) console.error(`[YjsServer] Error loading index ${dsPath}:`, err);
return [];
}
}
async handleAwareness(req, res, route, awarenessName, url) {
const method = req.method?.toUpperCase();
const dsPath = YjsStreamPaths.awarenessStream(route.service, route.docPath, awarenessName);
if (method === `PUT`) {
const docDsPath = YjsStreamPaths.dsStream(route.service, route.docPath);
const headUrl = `${this.dsServerUrl}${docDsPath}`;
try {
const headResponse = await fetch(headUrl, {
method: `HEAD`,
headers: this.dsServerHeaders
});
if (headResponse.status === 404) {
res.writeHead(404, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `DOCUMENT_NOT_FOUND`,
message: `Document does not exist`
} }));
return;
}
} catch (err) {
console.error(`[YjsServer] HEAD check for document existence failed:`, err);
}
try {
const created = await this.tryCreateStream(dsPath);
if (created && awarenessName !== `default`) {
const indexPath = YjsStreamPaths.awarenessIndexStream(route.service, route.docPath);
await this.appendToIndexStream(indexPath, {
name: awarenessName,
createdAt: Date.now()
}).catch((err) => {
console.error(`[YjsServer] Failed to append to awareness index:`, err);
});
}
res.writeHead(created ? 201 : 200, { "content-type": `application/json` });
res.end();
} catch (err) {
console.error(`[YjsServer] Failed to create awareness stream:`, err);
if (!res.headersSent) {
res.writeHead(500, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INTERNAL_ERROR`,
message: `Failed to create awareness stream`
} }));
}
}
} else if (method === `POST`) {
const docDsPath = YjsStreamPaths.dsStream(route.service, route.docPath);
await this.postWithAutoCreate(req, res, dsPath, docDsPath);
} else if (method === `GET`) {
const offset = url.searchParams.get(`offset`);
const live = url.searchParams.get(`live`);
const params = new URLSearchParams();
if (offset !== null) params.set(`offset`, offset);
if (live) params.set(`live`, live === `true` ? `sse` : live);
const query = params.toString();
const fullPath = query ? `${dsPath}?${query}` : dsPath;
if (live === `sse` || live === `true`) await this.proxyWithSseFlush(req, res, fullPath);
else await this.proxyToDsServer(req, res, fullPath);
} else if (method === `HEAD`) await this.proxyToDsServer(req, res, dsPath);
else if (method === `DELETE`) {
const response = await fetch(`${this.dsServerUrl}${dsPath}`, {
method: `DELETE`,
headers: this.dsServerHeaders
});
if (response.status === 404) {
await response.arrayBuffer();
res.writeHead(404, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `STREAM_NOT_FOUND`,
message: `Awareness stream not found`
} }));
return;
}
if (!response.ok) {
const text = await response.text().catch(() => ``);
throw new Error(`Failed to delete awareness stream: ${response.status} ${text}`);
}
await response.arrayBuffer();
res.writeHead(204);
res.end();
} else {
res.writeHead(405, { "content-type": `application/json` });
res.end(JSON.stringify({ error: {
code: `INVALID_REQUEST`,
message: `Method not allowed`
} }));
}
}
/**
* Forward a fetch Response to the client ServerResponse.
*/
async forwardResponse(res, response) {
const responseHeaders = {};
response.headers.forEach((value, key) => {
if (key === `content-encoding` || key === `content-length`) return;
responseHeaders[key] = value;
});
res.writeHead(response.status, responseHeaders);
if (response.body) for await (const chunk of response.body) res.write(chunk);
res.end();
}
/**
* Proxy with SSE-specific handling: flush after each chunk for immediate delivery.
*/
async proxyWithSseFlush(req, res, path) {
const targetUrl = `${this.dsServerUrl}${path}`;
const headers = { ...this.dsServerHeaders };
for (const [key, value] of Object.entries(req.headers)) if (key.toLowerCase() !== `host` && value) headers[key] = Array.isArray(value) ? value.join(`, `) : value;
const response = await fetch(targetUrl, {
method: `GET`,
headers
});
const responseHeaders = {
"cache-control": `no-cache`,
connection: `keep-alive`
};
response.headers.forEach((value, key) => {
if (key === `content-encoding` || key === `content-length`) return;
responseHeaders[key] = value;
});
res.writeHead(response.status, responseHeaders);
if (response.body) for await (const chunk of response.body) {
res.write(chunk);
const flushable = res;
flushable.flush?.();
}
res.end();
}
/**
* Try to create a stream at the given DS path.
* Returns true if the stream was created, false if it already existed.
*/
async tryCreateStream(dsPath, contentType = `application/octet-stream`) {
const url = `${this.dsServerUrl}${dsPath}`;
const response = await fetch(url, {
method: `PUT`,
headers: {
...this.dsServerHeaders,
"content-type": contentType
}
});
if (response.status === 201) {
await response.arrayBuffer();
return true;
}
if (response.status === 200 || response.status === 409) {
await response.arrayBuffer();
return false;
}
const text = await response.text().catch(() => ``);
throw new Error(`Failed to create stream ${dsPath}: ${response.status} ${text}`);
}
/**
* Append a JSON entry to an index stream, creating the stream if needed.
*/
async appendToIndexStream(dsPath, entry) {
await this.tryCreateStream(dsPath, `application/json`);
const stream = new DurableStream({
url: `${this.dsServerUrl}${dsPath}`,
headers: this.dsServerHeaders,
contentType: `application/json`
});
await stream.append(JSON.stringify(entry) + `\n`, { contentType: `application/json` });
}
getOrCreateDocumentState(service, docPath) {
const stateKey = this.stateKey(service, docPath);
let state = this.documentStates.get(stateKey);
if (!state) {
state = {
snapshotOffset: null,
updatesSizeBytes: 0,
compacting: false
};
this.documentStates.set(stateKey, state);
}
return state;
}
shouldTriggerCompaction(state) {
return !state.compacting && state.updatesSizeBytes >= this.compactionThreshold;
}
getDocumentState(service, docPath) {
return this.documentStates.get(this.stateKey(service, docPath));
}
/**
* Atomically check if compaction can start and set compacting=true if so.
* Returns true if compaction was started, false if already compacting or state not found.
*/
tryStartCompaction(service, docPath) {
const state = this.documentStates.get(this.stateKey(service, docPath));
if (!state || state.compacting) return false;
state.compacting = true;
return true;
}
setCompacting(service, docPath, compacting) {
const state = this.documentStates.get(this.stateKey(service, docPath));
if (state) state.compacting = compacting;
}
resetUpdateCounters(service, docPath) {
const state = this.documentStates.get(this.stateKey(service, docPath));
if (state) state.updatesSizeBytes = 0;
}
updateSnapshotOffset(service, docPath, offset) {
const state = this.documentStates.get(this.stateKey(service, docPath));
if (state) state.snapshotOffset = offset;
}
getDsServerUrl() {
return this.dsServerUrl;
}
getDsServerHeaders() {
return this.dsServerHeaders;
}
readBody(req) {
return new Promise((resolve, reject) => {
const chunks = [];
req.on(`data`, (chunk) => {
chunks.push(chunk);
});
req.on(`end`, () => {
const body = Buffer.concat(chunks);
resolve(new Uint8Array(body));
});
req.on(`error`, reject);
});
}
};
//#endregion
export { Compactor, YJS_HEADERS, YjsServer, YjsStreamPaths };