@electric-sql/pglite-sync/dist/index.cjs

ElectricSQL Sync for PGlite

"use strict";var F=Object.defineProperty;var oe=Object.getOwnPropertyDescriptor;var re=Object.getOwnPropertyNames;var ie=Object.prototype.hasOwnProperty;var ce=(t,n)=>{for(var e in n)F(t,e,{get:n[e],enumerable:!0})},le=(t,n,e,a)=>{if(n&&typeof n=="object"||typeof n=="function")for(let r of re(n))!ie.call(t,r)&&r!==e&&F(t,r,{get:()=>n[r],enumerable:!(a=oe(n,r))||a.enumerable});return t};var pe=t=>le(F({},"__esModule",{value:!0}),t);var be={};ce(be,{electricSync:()=>ge});module.exports=pe(be);var v=require("@electric-sql/client"),ne=require("@electric-sql/experimental");var ue="subscriptions_metadata";async function J({pg:t,metadataSchema:n,subscriptionKey:e}){let a=await t.query(`
      SELECT key, shape_metadata, last_lsn
      FROM ${B(n)}
      WHERE key = $1
    `,[e]);if(a.rows.length===0)return null;if(a.rows.length>1)throw new Error(`Multiple subscriptions found for key: ${e}`);let r=a.rows[0];if(typeof r.last_lsn=="string")return{...r,last_lsn:BigInt(r.last_lsn)};throw new Error(`Invalid last_lsn type: ${typeof r.last_lsn}`)}async function Q({pg:t,metadataSchema:n,subscriptionKey:e,shapeMetadata:a,lastLsn:r,debug:b}){b&&console.log("updating subscription state",e,a,r),await t.query(`
      INSERT INTO ${B(n)}
        (key, shape_metadata, last_lsn)
      VALUES
        ($1, $2, $3)
      ON CONFLICT(key)
      DO UPDATE SET
        shape_metadata = EXCLUDED.shape_metadata,
        last_lsn = EXCLUDED.last_lsn;
    `,[e,a,r.toString()])}async function z({pg:t,metadataSchema:n,subscriptionKey:e}){await t.query(`DELETE FROM ${B(n)} WHERE key = $1`,[e])}async function Z({pg:t,metadataSchema:n}){await t.exec(`
      SET ${n}.syncing = false;
      CREATE SCHEMA IF NOT EXISTS "${n}";
      CREATE TABLE IF NOT EXISTS ${B(n)} (
        key TEXT PRIMARY KEY,
        shape_metadata JSONB NOT NULL,
        last_lsn TEXT NOT NULL
      );
    `)}function B(t){return`"${t}"."${ue}"`}async function ee({pg:t,table:n,schema:e="public",message:a,mapColumns:r,primaryKey:b,debug:m}){let u=r?U(r,a):a.value;switch(a.headers.operation){case"insert":{m&&console.log("inserting",u);let f=Object.keys(u);return await t.query(`
            INSERT INTO "${e}"."${n}"
            (${f.map(o=>'"'+o+'"').join(", ")})
            VALUES
            (${f.map((o,y)=>"$"+(y+1)).join(", ")})
          `,f.map(o=>u[o]))}case"update":{m&&console.log("updating",u);let f=Object.keys(u).filter(o=>!b.includes(o));return f.length===0?void 0:await t.query(`
            UPDATE "${e}"."${n}"
            SET ${f.map((o,y)=>'"'+o+'" = $'+(y+1)).join(", ")}
            WHERE ${b.map((o,y)=>'"'+o+'" = $'+(f.length+y+1)).join(" AND ")}
          `,[...f.map(o=>u[o]),...b.map(o=>u[o])])}case"delete":return m&&console.log("deleting",u),await t.query(`
            DELETE FROM "${e}"."${n}"
            WHERE ${b.map((f,o)=>'"'+f+'" = $'+(o+1)).join(" AND ")}
          `,[...b.map(f=>u[f])])}}async function K({pg:t,table:n,schema:e="public",messages:a,mapColumns:r,debug:b}){let m=a.map(s=>r?U(r,s):s.value);b&&console.log("inserting",m);let u=Object.keys(m[0]),f=s=>{if(s===null)return 0;if(s instanceof ArrayBuffer)return s.byteLength;if(s instanceof Blob)return s.size;if(s instanceof Uint8Array||s instanceof DataView||ArrayBuffer.isView(s))return s.byteLength;switch(typeof s){case"string":return s.length;case"number":return 8;case"boolean":return 1;default:return s instanceof Date?8:s?.toString()?.length||0}},o=s=>u.reduce((d,_)=>{let g=s[_];if(g===null)return d;if(Array.isArray(g)){if(g.length===0)return d;let w=g[0];switch(typeof w){case"number":return d+g.length*8;case"string":return d+g.reduce((P,$)=>P+$.length,0);case"boolean":return d+g.length;default:return w instanceof Date?d+g.length*8:d+g.reduce((P,$)=>P+f($),0)}}return d+f(g)},0),y=32e3,I=50*1024*1024,O=async s=>{let d=`
      INSERT INTO "${e}"."${n}"
      (${u.map(g=>`"${g}"`).join(", ")})
      VALUES
      ${s.map((g,w)=>`(${u.map((P,$)=>"$"+(w*u.length+$+1)).join(", ")})`).join(", ")}
    `,_=s.flatMap(g=>u.map(w=>g[w]));await t.query(d,_)},i=[],h=0,N=0;for(let s=0;s<m.length;s++){let d=m[s],_=o(d),g=u.length;i.length>0&&(h+_>I||N+g>y)&&(b&&h+_>I&&console.log("batch size limit exceeded, executing batch"),b&&N+g>y&&console.log("batch params limit exceeded, executing batch"),await O(i),i=[],h=0,N=0),i.push(d),h+=_,N+=g}i.length>0&&await O(i),b&&console.log(`Inserted ${a.length} rows using INSERT`)}async function te({pg:t,table:n,schema:e="public",messages:a,mapColumns:r,debug:b}){b&&console.log("applying messages with json_to_recordset");let m=a.map(o=>r?U(r,o):o.value),u=(await t.query(`
        SELECT column_name, udt_name, data_type
        FROM information_schema.columns
        WHERE table_name = $1 AND table_schema = $2
      `,[n,e])).rows.filter(o=>Object.prototype.hasOwnProperty.call(m[0],o.column_name)),f=1e4;for(let o=0;o<m.length;o+=f){let y=m.slice(o,o+f);await t.query(`
        INSERT INTO "${e}"."${n}"
        SELECT x.* from json_to_recordset($1) as x(${u.map(I=>`${I.column_name} ${I.udt_name.replace(/^_/,"")}`+(I.data_type==="ARRAY"?"[]":"")).join(", ")})
      `,[y])}b&&console.log(`Inserted ${a.length} rows using json_to_recordset`)}async function W({pg:t,table:n,schema:e="public",messages:a,mapColumns:r,debug:b}){b&&console.log("applying messages with COPY");let m=a.map(y=>r?U(r,y):y.value),u=Object.keys(m[0]),f=m.map(y=>u.map(I=>{let O=y[I];return typeof O=="string"&&(O.includes(",")||O.includes('"')||O.includes(`
`))?`"${O.replace(/"/g,'""')}"`:O===null?"\\N":O}).join(",")).join(`
`),o=new Blob([f],{type:"text/csv"});await t.query(`
      COPY "${e}"."${n}" (${u.map(y=>`"${y}"`).join(", ")})
      FROM '/dev/blob'
      WITH (FORMAT csv, NULL '\\N')
    `,[],{blob:o}),b&&console.log(`Inserted ${a.length} rows using COPY`)}function U(t,n){if(typeof t=="function")return t(n);let e={};for(let[a,r]of Object.entries(t))e[a]=n.value[r];return e}async function fe(t,n){let e=n?.debug??!1,a=n?.metadataSchema??"electric",r=[],b=new Map,m=!1,u=async()=>{m||(m=!0,await Z({pg:t,metadataSchema:a}))},f=async({key:i,shapes:h,useCopy:N=!1,initialInsertMethod:s="insert",onInitialSync:d,onError:_})=>{let g=!1;await u(),Object.values(h).filter(c=>!c.onMustRefetch).forEach(c=>{if(b.has(c.table))throw new Error("Already syncing shape for table "+c.table);b.set(c.table)});let w=null;i!==null&&(w=await J({pg:t,metadataSchema:a,subscriptionKey:i}),e&&w&&console.log("resuming from subscription state",w));let P=w===null;N&&s==="insert"&&(s="csv",console.warn("The useCopy option is deprecated and will be removed in a future version. Use initialInsertMethod instead."));let $=!P||s==="insert",X=!1,q=new Map(Object.keys(h).map(c=>[c,new Map])),k=new Map(Object.keys(h).map(c=>[c,BigInt(-1)])),x=new Set,H=w?.last_lsn??BigInt(-1),D=new AbortController;Object.values(h).filter(c=>!!c.shape.signal).forEach(c=>{c.shape.signal.addEventListener("abort",()=>D.abort(),{once:!0})});let R=new ne.MultiShapeStream({shapes:Object.fromEntries(Object.entries(h).map(([c,L])=>{let S=w?.shape_metadata[c];return[c,{...L.shape,...S?{offset:S.offset,handle:S.handle}:{},signal:D.signal}]}))}),se={json:te,csv:W,useCopy:W,insert:K},ae=async c=>{let L=new Map(Object.keys(h).map(S=>[S,[]]));for(let[S,M]of q.entries()){let l=L.get(S);for(let p of M.keys())if(p<=c){for(let T of M.get(p))l.push(T);M.delete(p)}}await t.transaction(async S=>{e&&console.time("commit"),await S.exec(`SET LOCAL ${a}.syncing = true;`);for(let[M,l]of L.entries()){let p=h[M],T=l;if(x.has(M)){if(e&&console.log("truncating table",p.table),p.onMustRefetch)await p.onMustRefetch(S);else{let E=p.schema||"public";await S.exec(`DELETE FROM "${E}"."${p.table}";`)}x.delete(M)}if(!$){let E=[],A=[],V=!1;for(let G of T)!V&&G.headers.operation==="insert"?E.push(G):(V=!0,A.push(G));E.length>0&&s==="csv"&&A.unshift(E.pop()),T=A,E.length>0&&(await se[s]({pg:S,table:p.table,schema:p.schema,messages:E,mapColumns:p.mapColumns,debug:e}),$=!0)}let C=[],j=null,Y=T.length;for(let E=0;E<Y;E++){let A=T[E];A.headers.operation==="insert"?C.push(A):j=A,(j||E===Y-1)&&(C.length>0&&(await K({pg:S,table:p.table,schema:p.schema,messages:C,mapColumns:p.mapColumns,debug:e}),C.length=0),j&&(await ee({pg:S,table:p.table,schema:p.schema,message:j,mapColumns:p.mapColumns,primaryKey:p.primaryKey,debug:e}),j=null))}}i&&await Q({pg:S,metadataSchema:a,subscriptionKey:i,shapeMetadata:Object.fromEntries(Object.keys(h).map(M=>[M,{handle:R.shapes[M].shapeHandle,offset:R.shapes[M].lastOffset}])),lastLsn:c,debug:e}),g&&await S.rollback()}),e&&console.timeEnd("commit"),d&&!X&&R.isUpToDate&&(d(),X=!0)};return R.subscribe(async c=>{if(g)return;e&&console.log("received messages",c.length),c.forEach(l=>{let p=k.get(l.shape)??BigInt(-1);if((0,v.isChangeMessage)(l)){let T=q.get(l.shape),C=typeof l.headers.lsn=="string"?BigInt(l.headers.lsn):BigInt(0);if(C<=p)return;let j=l.headers.last??!1;T.has(C)||T.set(C,[]),T.get(C).push(l),j&&k.set(l.shape,C)}else if((0,v.isControlMessage)(l))switch(l.headers.control){case"up-to-date":{if(e&&console.log("received up-to-date",l),typeof l.headers.global_last_seen_lsn!="string")throw new Error("global_last_seen_lsn is not a string");let T=BigInt(l.headers.global_last_seen_lsn);if(T<=p)return;k.set(l.shape,T);break}case"must-refetch":{e&&console.log("received must-refetch",l),q.get(l.shape).clear(),k.set(l.shape,BigInt(-1)),x.add(l.shape);break}}});let L=Array.from(k.values()).reduce((l,p)=>p<l?p:l),S=L>H,M=L>=H&&x.size>0;(S||M)&&(ae(L),await new Promise(l=>setTimeout(l)))},_),r.push({stream:R,aborter:D}),{unsubscribe:()=>{e&&console.log("unsubscribing"),g=!0,R.unsubscribeAll(),D.abort();for(let c of Object.values(h))b.delete(c.table)},get isUpToDate(){return R.isUpToDate},streams:Object.fromEntries(Object.keys(h).map(c=>[c,R.shapes[c]]))}};return{namespaceObj:{initMetadataTables:u,syncShapesToTables:f,syncShapeToTable:async i=>{let h=await f({shapes:{shape:{shape:i.shape,table:i.table,schema:i.schema,mapColumns:i.mapColumns,primaryKey:i.primaryKey,onMustRefetch:i.onMustRefetch}},key:i.shapeKey,useCopy:i.useCopy,initialInsertMethod:i.initialInsertMethod,onInitialSync:i.onInitialSync,onError:i.onError});return{unsubscribe:h.unsubscribe,get isUpToDate(){return h.isUpToDate},stream:h.streams.shape}},deleteSubscription:async i=>{await z({pg:t,metadataSchema:a,subscriptionKey:i})}},close:async()=>{for(let{stream:i,aborter:h}of r)i.unsubscribeAll(),h.abort()}}}function ge(t){return{name:"ElectricSQL Sync",setup:async n=>{let{namespaceObj:e,close:a}=await fe(n,t);return{namespaceObj:e,close:a}}}}0&&(module.exports={electricSync});
//# sourceMappingURL=index.cjs.map