UNPKG

@substrate-system/mergeparty

Version:
8 lines (7 loc) 19.4 kB
{ "version": 3, "sources": ["../../src/server/with-storage.ts"], "sourcesContent": ["// src/server/with-storage.ts\nimport type * as Party from 'partykit/server'\nimport {\n type PeerId,\n Repo,\n type StorageAdapterInterface,\n type StorageKey,\n} from '@substrate-system/automerge-repo-slim'\nimport { decode as cborDecode } from 'cborg'\nimport { Relay } from './relay.js'\nimport './polyfill.js' // need this for cloudflare environment\n\nexport class WithStorage\n extends Relay\n implements Party.Server, StorageAdapterInterface\n{ // eslint-disable-line brace-style\n readonly isStorageServer:boolean = true /* This is used by the relay,\n to decide if we should be announced as a peer. */\n _repo:Repo\n\n constructor (room:Party.Room, repo?:Repo) {\n super(room)\n\n if (!repo) {\n this._repo = new Repo({\n storage: this,\n network: [this],\n // server accepts new documents from clients\n sharePolicy: async () => {\n // Always accept and request documents from any peer\n return true\n },\n // Set a stable peer ID for the server\n peerId: `server:${this.room.id}` as PeerId,\n })\n } else {\n // repo should already have a storage adapter added\n this._repo = repo\n }\n\n // Set up event-driven storage persistence\n this.setupStoragePersistence()\n\n this._log = this._baseLog.extend('storage') // mergeparty:storage\n\n // Initialize the network adapter connection\n // The repo should call this automatically, but let's ensure it happens\n this.connect(this.serverPeerId as any, {})\n }\n\n async onMessage (\n raw:ArrayBuffer|string,\n conn:Party.Connection\n ):Promise<void> {\n if (!this.byConn.get(conn)?.joined) {\n // has not joined yet\n return super.onMessage(raw, conn)\n }\n\n // Check if this is a sync message for a new document\n try {\n if (raw instanceof ArrayBuffer) {\n const decoded = cborDecode(new Uint8Array(raw))\n if (decoded && decoded.type === 'sync' && decoded.documentId) {\n const documentId = decoded.documentId\n\n // Check if we already have this document\n const existingHandle = this._repo.handles[documentId]\n if (!existingHandle) {\n // Create a handle for this document so the repo knows\n // about it\n // This will trigger the sync process where the\n // server requests the document\n this._repo.find(documentId)\n }\n }\n }\n } catch (_e) {\n // If we can't decode the message,\n // just continue with normal processing\n }\n\n // Feed the frame to the repo via Relay\n // this should automatically handle storage\n await super.onMessage(raw, conn)\n }\n\n /**\n * Loads a value from PartyKit storage by key.\n * @param {StorageKey} key The storage key\n * @returns {Promise<Uint8Array|undefined>}\n */\n async load (key:StorageKey):Promise<Uint8Array|undefined> {\n const keyStr = this.keyToString(key)\n this._log(`Loading from storage: key=${keyStr}`)\n\n const value = await this.room.storage.get(keyStr)\n if (!value) {\n this._log(`No value found for key: ${keyStr}`)\n return\n }\n\n this._log(`Found value for key: ${keyStr}, type=${typeof value}`)\n\n if (value instanceof Uint8Array) return value\n if (value instanceof ArrayBuffer) return new Uint8Array(value)\n if (\n typeof value === 'object' && value !== null &&\n Object.keys(value).every(k => !isNaN(Number(k)))\n ) {\n return new Uint8Array(Object.values(value))\n }\n throw new Error('Unsupported value type from storage')\n }\n\n /**\n * Saves a value to PartyKit storage by key.\n * @param {StorageKey} key The storage key\n * @param {Uint8Array} value The value to store (Uint8Array)\n */\n async save (key:StorageKey, value:Uint8Array):Promise<void> {\n const keyStr = this.keyToString(key)\n this._log(`Saving to storage: key=${keyStr}, valueLength=${value.length}`)\n\n await this.room.storage.put(keyStr, value)\n this._log(`Successfully saved key: ${keyStr}`)\n }\n\n /**\n * Removes a value from PartyKit storage by key.\n * @param key The storage key\n */\n async remove (key:StorageKey):Promise<void> {\n await this.room.storage.delete(this.keyToString(key))\n }\n\n /**\n * Loads a range of values from PartyKit storage by prefix.\n * @param prefix The key prefix\n * @returns {Promise<{ key:StorageKey, data:Uint8Array|undefined }[]>}\n */\n async loadRange (prefix:StorageKey):Promise<{\n key:StorageKey;\n data:Uint8Array | undefined;\n }[]> {\n const key = this.keyToString(prefix)\n const entries:{ key:StorageKey, data:Uint8Array | undefined }[] = []\n const map = await this.room.storage.list({ prefix: key })\n\n for (const [k, v] of [...map.entries()].sort(([a], [b]) => {\n return a.localeCompare(b)\n })) {\n let u8:Uint8Array | undefined\n if (v instanceof Uint8Array) u8 = v\n else if (v instanceof ArrayBuffer) u8 = new Uint8Array(v)\n else if (\n typeof v === 'object' &&\n v !== null &&\n Object.keys(v).every(k => !isNaN(Number(k)))\n ) {\n u8 = new Uint8Array(Object.values(v))\n } else {\n u8 = undefined\n }\n\n entries.push({ key: this.stringToKey(k), data: u8 })\n }\n\n return entries\n }\n\n /**\n * Removes a range of values from PartyKit storage by prefix.\n * @param prefix The key prefix\n */\n async removeRange (prefix:StorageKey):Promise<void> {\n const key = this.keyToString(prefix)\n const map = await this.room.storage.list({ prefix: key })\n for (const key of map.keys()) {\n await this.room.storage.delete(key)\n }\n }\n\n async onStart ():Promise<void> {\n this._log('**Stateful sync server started (Automerge peer w/' +\n ' PartyKit storage)**')\n\n // Store the storage adapter ID to ensure storage is initialized\n await this.save(\n ['storage-adapter-id'],\n new TextEncoder().encode(this.peerId || 'server')\n )\n this._log('Storage adapter initialized')\n }\n\n // HTTP endpoints\n async onRequest (req:Party.Request):Promise<Response> {\n const url = new URL(req.url)\n\n // Debug endpoint to view storage contents\n if (url.pathname.includes('/debug/storage')) {\n const storageMap = await this.room.storage.list()\n const result:Record<string, any> = {}\n for (const [key, value] of storageMap) {\n result[key] = value\n }\n return Response.json(result, {\n status: 200,\n headers: {\n 'Access-Control-Allow-Origin': '*',\n 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',\n 'Access-Control-Allow-Headers': 'Content-Type'\n }\n })\n }\n\n // Test endpoint to verify storage functionality\n if (url.pathname.includes('/test/storage')) {\n this._log('[WithStorage] Storage test endpoint called')\n try {\n this._log('[WithStorage] Starting basic storage operations test...')\n\n // Test basic storage operations (this works immediately)\n const testKey = 'test-manual-storage'\n const testValue = new TextEncoder().encode('test-value')\n\n this._log('[WithStorage] Calling save...')\n await this.save([testKey], testValue)\n this._log('[WithStorage] Calling load...')\n const retrieved = await this.load([testKey])\n\n if (\n !retrieved ||\n new TextDecoder().decode(retrieved) !== 'test-value'\n ) {\n throw new Error('Storage test failed')\n }\n\n this._log('[WithStorage] Basic storage operations successful')\n\n // Get repo state for debugging - be very careful here\n this._log('[WithStorage] Storage test: getting repo handles...')\n let totalHandles = 0\n let readyHandles = 0\n let handleIds: string[] = []\n\n try {\n handleIds = Object.keys(this._repo.handles)\n totalHandles = handleIds.length\n this._log(`[WithStorage] Found ${totalHandles} handles`)\n\n if (totalHandles > 0) {\n const handles = Object.values(this._repo.handles)\n this._log('[WithStorage] Checking readiness of handles...')\n readyHandles = handles.filter(handle => {\n try {\n return handle.isReady()\n } catch (e: any) {\n this._log('[WithStorage] Error checking handle' +\n ` readiness: ${e.message}`)\n return false\n }\n }).length\n }\n } catch (e: any) {\n this._log(\n `[WithStorage] Error accessing repo handles: ${e.message}`\n )\n }\n\n this._log(`[WithStorage] Storage test: found ${totalHandles}` +\n ` total handles, ${readyHandles} ready`)\n\n return Response.json({\n success: true,\n message: 'Storage operations successful ' +\n '- Automerge handles persistence automatically',\n repoHandles: handleIds,\n readyHandles,\n totalHandles,\n storageKeys: await this.room.storage.list().then(map => {\n return [...map.keys()]\n })\n }, {\n status: 200,\n headers: {\n 'Access-Control-Allow-Origin': '*',\n 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',\n 'Access-Control-Allow-Headers': 'Content-Type'\n }\n })\n } catch (error: any) {\n this._log(`[WithStorage] Storage test failed: ${error.message}`)\n return Response.json({\n success: false,\n error: error.message\n }, {\n status: 500,\n headers: {\n 'Access-Control-Allow-Origin': '*',\n 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',\n 'Access-Control-Allow-Headers': 'Content-Type'\n }\n })\n }\n }\n\n // Fall back to parent implementation for health checks\n return super.onRequest(req)\n }\n\n async onConnect (conn:Party.Connection):Promise<void> {\n // Call parent onConnect first\n super.onConnect(conn)\n\n // Trigger a flush when a new client connects to ensure\n // any existing documents are available\n try {\n await this._repo.flush()\n this._log('Flushed on client connect')\n } catch (e) {\n this._log(`Failed to flush on connect: ${e}`)\n }\n }\n\n protected unicastByPeerId (peerId:string, data:Uint8Array) {\n const conn:Party.Connection|undefined = this.sockets[peerId]\n if (conn) conn.send(data)\n }\n\n private keyToString (key:string[]):string {\n return key.join('.')\n }\n\n private stringToKey (key:string):string[] {\n return key.split('.')\n }\n\n private setupStoragePersistence ():void {\n this._log('[WithStorage] Setting up storage persistence ' +\n '- Automerge should handle this automatically')\n\n // Log repo state periodically for debugging\n setInterval(() => {\n const handleCount = Object.keys(this._repo.handles).length\n if (handleCount > 0) {\n const handles = Object.values(this._repo.handles)\n const readyHandles = handles.filter(handle => handle.isReady())\n this._log(`[WithStorage] Repo state: ${handleCount}` +\n ` total handles, ${readyHandles.length} ready`)\n this._log(\n '[WithStorage] Handle IDs:',\n Object.keys(this._repo.handles)\n )\n }\n }, 5000) // Log every 5 seconds for debugging\n }\n}\n"], "mappings": ";;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAEA,iCAKO;AACP,mBAAqC;AACrC,mBAAsB;AACtB,sBAAO;AAEA,MAAM,oBACD,mBAEZ;AAAA,EAfA,OAeA;AAAA;AAAA;AAAA;AAAA,EACa,kBAA0B;AAAA;AAAA;AAAA,EAEnC;AAAA,EAEA,YAAa,MAAiB,MAAY;AACtC,UAAM,IAAI;AAEV,QAAI,CAAC,MAAM;AACP,WAAK,QAAQ,IAAI,gCAAK;AAAA,QAClB,SAAS;AAAA,QACT,SAAS,CAAC,IAAI;AAAA;AAAA,QAEd,aAAa,mCAAY;AAErB,iBAAO;AAAA,QACX,GAHa;AAAA;AAAA,QAKb,QAAQ,UAAU,KAAK,KAAK,EAAE;AAAA,MAClC,CAAC;AAAA,IACL,OAAO;AAEH,WAAK,QAAQ;AAAA,IACjB;AAGA,SAAK,wBAAwB;AAE7B,SAAK,OAAO,KAAK,SAAS,OAAO,SAAS;AAI1C,SAAK,QAAQ,KAAK,cAAqB,CAAC,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,UACF,KACA,MACY;AACZ,QAAI,CAAC,KAAK,OAAO,IAAI,IAAI,GAAG,QAAQ;AAEhC,aAAO,MAAM,UAAU,KAAK,IAAI;AAAA,IACpC;AAGA,QAAI;AACA,UAAI,eAAe,aAAa;AAC5B,cAAM,cAAU,aAAAA,QAAW,IAAI,WAAW,GAAG,CAAC;AAC9C,YAAI,WAAW,QAAQ,SAAS,UAAU,QAAQ,YAAY;AAC1D,gBAAM,aAAa,QAAQ;AAG3B,gBAAM,iBAAiB,KAAK,MAAM,QAAQ,UAAU;AACpD,cAAI,CAAC,gBAAgB;AAKjB,iBAAK,MAAM,KAAK,UAAU;AAAA,UAC9B;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ,SAAS,IAAI;AAAA,IAGb;AAIA,UAAM,MAAM,UAAU,KAAK,IAAI;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,KAAM,KAA8C;AACtD,UAAM,SAAS,KAAK,YAAY,GAAG;AACnC,SAAK,KAAK,6BAA6B,MAAM,EAAE;AAE/C,UAAM,QAAQ,MAAM,KAAK,KAAK,QAAQ,IAAI,MAAM;AAChD,QAAI,CAAC,OAAO;AACR,WAAK,KAAK,2BAA2B,MAAM,EAAE;AAC7C;AAAA,IACJ;AAEA,SAAK,KAAK,wBAAwB,MAAM,UAAU,OAAO,KAAK,EAAE;AAEhE,QAAI,iBAAiB,WAAY,QAAO;AACxC,QAAI,iBAAiB,YAAa,QAAO,IAAI,WAAW,KAAK;AAC7D,QACI,OAAO,UAAU,YAAY,UAAU,QACvC,OAAO,KAAK,KAAK,EAAE,MAAM,OAAK,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,GACjD;AACE,aAAO,IAAI,WAAW,OAAO,OAAO,KAAK,CAAC;AAAA,IAC9C;AACA,UAAM,IAAI,MAAM,qCAAqC;AAAA,EACzD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,KAAM,KAAgB,OAAgC;AACxD,UAAM,SAAS,KAAK,YAAY,GAAG;AACnC,SAAK,KAAK,0BAA0B,MAAM,iBAAiB,MAAM,MAAM,EAAE;AAEzE,UAAM,KAAK,KAAK,QAAQ,IAAI,QAAQ,KAAK;AACzC,SAAK,KAAK,2BAA2B,MAAM,EAAE;AAAA,EACjD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,OAAQ,KAA8B;AACxC,UAAM,KAAK,KAAK,QAAQ,OAAO,KAAK,YAAY,GAAG,CAAC;AAAA,EACxD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,UAAW,QAGZ;AACD,UAAM,MAAM,KAAK,YAAY,MAAM;AACnC,UAAM,UAA4D,CAAC;AACnE,UAAM,MAAM,MAAM,KAAK,KAAK,QAAQ,KAAK,EAAE,QAAQ,IAAI,CAAC;AAExD,eAAW,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM;AACvD,aAAO,EAAE,cAAc,CAAC;AAAA,IAC5B,CAAC,GAAG;AACA,UAAI;AACJ,UAAI,aAAa,WAAY,MAAK;AAAA,eACzB,aAAa,YAAa,MAAK,IAAI,WAAW,CAAC;AAAA,eAEpD,OAAO,MAAM,YACb,MAAM,QACN,OAAO,KAAK,CAAC,EAAE,MAAM,CAAAC,OAAK,CAAC,MAAM,OAAOA,EAAC,CAAC,CAAC,GAC7C;AACE,aAAK,IAAI,WAAW,OAAO,OAAO,CAAC,CAAC;AAAA,MACxC,OAAO;AACH,aAAK;AAAA,MACT;AAEA,cAAQ,KAAK,EAAE,KAAK,KAAK,YAAY,CAAC,GAAG,MAAM,GAAG,CAAC;AAAA,IACvD;AAEA,WAAO;AAAA,EACX;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,YAAa,QAAiC;AAChD,UAAM,MAAM,KAAK,YAAY,MAAM;AACnC,UAAM,MAAM,MAAM,KAAK,KAAK,QAAQ,KAAK,EAAE,QAAQ,IAAI,CAAC;AACxD,eAAWC,QAAO,IAAI,KAAK,GAAG;AAC1B,YAAM,KAAK,KAAK,QAAQ,OAAOA,IAAG;AAAA,IACtC;AAAA,EACJ;AAAA,EAEA,MAAM,UAAyB;AAC3B,SAAK,KAAK,uEACgB;AAG1B,UAAM,KAAK;AAAA,MACP,CAAC,oBAAoB;AAAA,MACrB,IAAI,YAAY,EAAE,OAAO,KAAK,UAAU,QAAQ;AAAA,IACpD;AACA,SAAK,KAAK,6BAA6B;AAAA,EAC3C;AAAA;AAAA,EAGA,MAAM,UAAW,KAAqC;AAClD,UAAM,MAAM,IAAI,IAAI,IAAI,GAAG;AAG3B,QAAI,IAAI,SAAS,SAAS,gBAAgB,GAAG;AACzC,YAAM,aAAa,MAAM,KAAK,KAAK,QAAQ,KAAK;AAChD,YAAM,SAA6B,CAAC;AACpC,iBAAW,CAAC,KAAK,KAAK,KAAK,YAAY;AACnC,eAAO,GAAG,IAAI;AAAA,MAClB;AACA,aAAO,SAAS,KAAK,QAAQ;AAAA,QACzB,QAAQ;AAAA,QACR,SAAS;AAAA,UACL,+BAA+B;AAAA,UAC/B,gCAAgC;AAAA,UAChC,gCAAgC;AAAA,QACpC;AAAA,MACJ,CAAC;AAAA,IACL;AAGA,QAAI,IAAI,SAAS,SAAS,eAAe,GAAG;AACxC,WAAK,KAAK,4CAA4C;AACtD,UAAI;AACA,aAAK,KAAK,yDAAyD;AAGnE,cAAM,UAAU;AAChB,cAAM,YAAY,IAAI,YAAY,EAAE,OAAO,YAAY;AAEvD,aAAK,KAAK,+BAA+B;AACzC,cAAM,KAAK,KAAK,CAAC,OAAO,GAAG,SAAS;AACpC,aAAK,KAAK,+BAA+B;AACzC,cAAM,YAAY,MAAM,KAAK,KAAK,CAAC,OAAO,CAAC;AAE3C,YACI,CAAC,aACD,IAAI,YAAY,EAAE,OAAO,SAAS,MAAM,cAC1C;AACE,gBAAM,IAAI,MAAM,qBAAqB;AAAA,QACzC;AAEA,aAAK,KAAK,mDAAmD;AAG7D,aAAK,KAAK,qDAAqD;AAC/D,YAAI,eAAe;AACnB,YAAI,eAAe;AACnB,YAAI,YAAsB,CAAC;AAE3B,YAAI;AACA,sBAAY,OAAO,KAAK,KAAK,MAAM,OAAO;AAC1C,yBAAe,UAAU;AACzB,eAAK,KAAK,uBAAuB,YAAY,UAAU;AAEvD,cAAI,eAAe,GAAG;AAClB,kBAAM,UAAU,OAAO,OAAO,KAAK,MAAM,OAAO;AAChD,iBAAK,KAAK,gDAAgD;AAC1D,2BAAe,QAAQ,OAAO,YAAU;AACpC,kBAAI;AACA,uBAAO,OAAO,QAAQ;AAAA,cAC1B,SAAS,GAAQ;AACb,qBAAK,KAAK,kDACS,EAAE,OAAO,EAAE;AAC9B,uBAAO;AAAA,cACX;AAAA,YACJ,CAAC,EAAE;AAAA,UACP;AAAA,QACJ,SAAS,GAAQ;AACb,eAAK;AAAA,YACD,+CAA+C,EAAE,OAAO;AAAA,UAC5D;AAAA,QACJ;AAEA,aAAK,KAAK,qCAAqC,YAAY,mBACpC,YAAY,QAAQ;AAE3C,eAAO,SAAS,KAAK;AAAA,UACjB,SAAS;AAAA,UACT,SAAS;AAAA,UAET,aAAa;AAAA,UACb;AAAA,UACA;AAAA,UACA,aAAa,MAAM,KAAK,KAAK,QAAQ,KAAK,EAAE,KAAK,SAAO;AACpD,mBAAO,CAAC,GAAG,IAAI,KAAK,CAAC;AAAA,UACzB,CAAC;AAAA,QACL,GAAG;AAAA,UACC,QAAQ;AAAA,UACR,SAAS;AAAA,YACL,+BAA+B;AAAA,YAC/B,gCAAgC;AAAA,YAChC,gCAAgC;AAAA,UACpC;AAAA,QACJ,CAAC;AAAA,MACL,SAAS,OAAY;AACjB,aAAK,KAAK,sCAAsC,MAAM,OAAO,EAAE;AAC/D,eAAO,SAAS,KAAK;AAAA,UACjB,SAAS;AAAA,UACT,OAAO,MAAM;AAAA,QACjB,GAAG;AAAA,UACC,QAAQ;AAAA,UACR,SAAS;AAAA,YACL,+BAA+B;AAAA,YAC/B,gCAAgC;AAAA,YAChC,gCAAgC;AAAA,UACpC;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAGA,WAAO,MAAM,UAAU,GAAG;AAAA,EAC9B;AAAA,EAEA,MAAM,UAAW,MAAqC;AAElD,UAAM,UAAU,IAAI;AAIpB,QAAI;AACA,YAAM,KAAK,MAAM,MAAM;AACvB,WAAK,KAAK,2BAA2B;AAAA,IACzC,SAAS,GAAG;AACR,WAAK,KAAK,+BAA+B,CAAC,EAAE;AAAA,IAChD;AAAA,EACJ;AAAA,EAEU,gBAAiB,QAAe,MAAiB;AACvD,UAAM,OAAkC,KAAK,QAAQ,MAAM;AAC3D,QAAI,KAAM,MAAK,KAAK,IAAI;AAAA,EAC5B;AAAA,EAEQ,YAAa,KAAqB;AACtC,WAAO,IAAI,KAAK,GAAG;AAAA,EACvB;AAAA,EAEQ,YAAa,KAAqB;AACtC,WAAO,IAAI,MAAM,GAAG;AAAA,EACxB;AAAA,EAEQ,0BAAgC;AACpC,SAAK,KAAK,2FACwC;AAGlD,gBAAY,MAAM;AACd,YAAM,cAAc,OAAO,KAAK,KAAK,MAAM,OAAO,EAAE;AACpD,UAAI,cAAc,GAAG;AACjB,cAAM,UAAU,OAAO,OAAO,KAAK,MAAM,OAAO;AAChD,cAAM,eAAe,QAAQ,OAAO,YAAU,OAAO,QAAQ,CAAC;AAC9D,aAAK,KAAK,6BAA6B,WAAW,mBAC3B,aAAa,MAAM,QAAQ;AAClD,aAAK;AAAA,UACD;AAAA,UACA,OAAO,KAAK,KAAK,MAAM,OAAO;AAAA,QAClC;AAAA,MACJ;AAAA,IACJ,GAAG,GAAI;AAAA,EACX;AACJ;", "names": ["cborDecode", "k", "key"] }