protobuf-array-stream/dist/index.mjs

Stream and decode Protocol Buffer arrays without memory issues - perfect for processing large datasets

import{Transform as F}from"stream";import E from"protobufjs";var t={INT_32:"int32",INT_64:"int64",UINT_32:"uint32",UINT_64:"uint64",SINT_32:"sint32",SINT_64:"sint64",BOOL:"bool",ENUM:"enum",FIXED_64:"fixed64",SFIXED_64:"sfixed64",DOUBLE:"double",STRING:"string",BYTES:"bytes",FIXED_32:"fixed32",SFIXED_32:"sfixed32",FLOAT:"float"},o={VARINT:0,I64:1,LEN:2,I32:5},f={TAG:0,LENGTH:1,PAYLOAD:3},A={MSB:127,WIRE_TYPE:7},h={MESSAGE:{NOT_EXISTS:a=>`Invalid message: '${a}' does not exist in the proto file.`,EMPTY_FIELD:a=>`Invalid message: '${a}' does not have any fields.`,MORE_FIELDS:(a,e)=>`Invalid message: '${a}' should only have one field, but has ${e} fields.`,NOT_REPEATED:(a,e)=>`Invalid message: The field '${e}' in '${a}' must be 'repeated'.`},TAG:(a,e)=>`Expected array-like structure: Tag should repeat as ${a}, but encountered a different tag: ${e}.`,PAYLOAD:a=>`Failed to decode protobuf payload: ${a instanceof Error?a.message:String(a)}`};var O=class extends Error{constructor(r,s){super(r);this.cause=s;this.name="RepeatedFieldStreamError"}},T=O;var n=class n extends F{constructor(e){super({readableObjectMode:!0,writableObjectMode:!1});let{root:r,messageName:s}=e;n.validateMessage(r,s),this.repeatedField=n.getRepeatedFieldDetails(r,s),this.currentPayloadSize=n.getPayloadSizeInBytes(this.repeatedField.wireType),this.buffer=Buffer.alloc(0),this.tag=null,this.readOffset=0,this.nextToken=f.TAG}decodePayload(e){try{return this.repeatedField.decode(e)}catch(r){throw new T(h.PAYLOAD(r))}}cleanup(){this.repeatedField={name:"",wireType:0,isPacked:!1,decode:()=>({})},this.buffer=Buffer.alloc(0)}detectVarint(){let e=!0;for(;e&&this.readOffset<this.buffer.length;){let r=this.buffer.at(this.readOffset);this.readOffset+=1,r>>7===0&&(e=!1)}return!e}detectPayload(){for(;this.readOffset<this.currentPayloadSize&&this.readOffset<this.buffer.length;)this.readOffset+=1;return this.readOffset===this.currentPayloadSize}decode(){if(this.nextToken===f.TAG&&this.detectVarint()){let e=this.buffer.subarray(0,this.readOffset),r=n.decodeVarint(e);if(!this.tag)this.tag=r;else if(r!==this.tag)throw new T(h.TAG(this.tag,r));this.buffer=this.buffer.subarray(this.readOffset),this.readOffset=0,this.nextToken=this.repeatedField.wireType===o.LEN||this.repeatedField.isPacked?f.LENGTH:f.PAYLOAD}if(this.nextToken===f.LENGTH&&this.detectVarint()){let e=this.buffer.subarray(0,this.readOffset);this.repeatedField.isPacked||(this.currentPayloadSize=n.decodeVarint(e)),this.buffer=this.buffer.subarray(this.readOffset),this.readOffset=0,this.nextToken=f.PAYLOAD}if(this.nextToken===f.PAYLOAD&&(this.repeatedField.wireType===o.VARINT?this.detectVarint():this.detectPayload())){let e=this.buffer.subarray(0,this.readOffset),r=this.decodePayload(e);this.push(r),this.buffer=this.buffer.subarray(this.readOffset),this.readOffset=0,this.repeatedField.isPacked?this.nextToken=f.PAYLOAD:(this.currentPayloadSize=0,this.nextToken=f.TAG)}}_transform(e,r,s){try{for(this.buffer=Buffer.concat([this.buffer,e]);this.readOffset<this.buffer.length;)this.decode();s()}catch(l){s(l)}}_final(e){this.cleanup(),e()}_flush(e){try{for(;this.readOffset<this.buffer.length;)this.decode();e()}catch(r){e(r)}}};n.loadRootFromFileSync=E.loadSync,n.loadRootFromFile=E.load,n.loadRootFromJSON=E.Root.fromJSON,n.loadRootFromSource=(e,r)=>E.parse(e,r).root,n.validateMessage=(e,r)=>{var p,b;let s;try{s=e.lookupType(r)}catch(S){throw new T(h.MESSAGE.NOT_EXISTS(r))}let l=((p=s==null?void 0:s.fieldsArray)==null?void 0:p.length)||0,c=(b=s==null?void 0:s.fieldsArray)==null?void 0:b.at(0);if(!c)throw new T(h.MESSAGE.EMPTY_FIELD(r));if(l>1)throw new T(h.MESSAGE.MORE_FIELDS(r,l));if(!(c!=null&&c.repeated))throw new T(h.MESSAGE.NOT_REPEATED(r,c.name))},n.getRepeatedFieldDetails=(e,r)=>{let s=u=>{switch(u){case t.INT_32:case t.INT_64:case t.UINT_32:case t.UINT_64:case t.SINT_32:case t.SINT_64:case t.BOOL:case t.ENUM:return o.VARINT;case t.FIXED_64:case t.SFIXED_64:case t.DOUBLE:return o.I64;case t.FIXED_32:case t.SFIXED_32:case t.FLOAT:return o.I32;case t.STRING:case t.BYTES:default:return o.LEN}},l=(u,y)=>{let d=u;return Object.values(t).includes(u)||y.lookupTypeOrEnum(u)instanceof E.Enum&&(d=t.INT_32),d},c=(u,y)=>{if(!Object.values(t).includes(u)){let d=y.lookupType(u);return i=>{let D=d.decode(i);return d.toObject(D)}}return d=>{let i=E.Reader.create(d);switch(I){case t.INT_32:return i.int32();case t.INT_64:return i.int64();case t.UINT_32:return i.uint32();case t.UINT_64:return i.uint64();case t.SINT_32:return i.sint32();case t.SINT_64:return i.sint64();case t.BOOL:return i.bool();case t.FIXED_64:return i.fixed64();case t.SFIXED_64:return i.sfixed64();case t.DOUBLE:return i.double();case t.FIXED_32:return i.fixed32();case t.SFIXED_32:return i.sfixed32();case t.FLOAT:return i.float();case t.STRING:return d.toString("utf8");case t.BYTES:default:return d}}},b=e.lookupType(r).fieldsArray.at(0),{name:S,packed:m}=b,I=l(b.type,e),g=s(I),P=[o.VARINT,o.I64,o.I32].includes(g)&&m!==!1,_=c(I,e);return{name:S,wireType:g,isPacked:P,decode:_}},n.getPayloadSizeInBytes=e=>{switch(e){case o.I32:return 4;case o.I64:return 8;default:return 0}},n.decodeVarint=e=>{let r=0;for(let s=0;s<e.length;s++){let c=e[s]&A.MSB,p=s*7;r=r|c<<p}return r};var N=n;export{N as RepeatedFieldStream};