jsonarrayfs/dist/index.js

Specialized Node.js library for memory-efficient operations on JSON arrays. Stream individual elements from large JSON arrays (files, network responses etc.) and append elements to array files without loading the entire array into memory. Perfect for proc

"use strict";var I=Object.defineProperty;var M=Object.getOwnPropertyDescriptor;var D=Object.getOwnPropertyNames;var J=Object.prototype.hasOwnProperty;var K=(a,e)=>{for(var s in e)I(a,s,{get:e[s],enumerable:!0})},H=(a,e,s,r)=>{if(e&&typeof e=="object"||typeof e=="function")for(let n of D(e))!J.call(a,n)&&n!==s&&I(a,n,{get:()=>e[n],enumerable:!(r=M(e,n))||r.enumerable});return a};var Q=a=>H(I({},"__esModule",{value:!0}),a);var w=(a,e,s)=>new Promise((r,n)=>{var i=f=>{try{o(s.next(f))}catch(h){n(h)}},l=f=>{try{o(s.throw(f))}catch(h){n(h)}},o=f=>f.done?r(f.value):Promise.resolve(f.value).then(i,l);o((s=s.apply(a,e)).next())});var U={};K(U,{JsonArrayStream:()=>B,appendToJsonArrayFile:()=>L});module.exports=Q(U);var g=require("stream");var t={BRACKET:{OPEN:"[",CLOSE:"]"},BRACE:{OPEN:"{",CLOSE:"}"},QUOTE:'"',ESCAPE:"\\",SPACE:" ",COMMA:",",TAB:"	",CARRIAGE_RETURN:"\r",NEW_LINE:`
`,ZERO_WIDTH_NO_BREAK_SPACE:"\uFEFF"},C=[t.SPACE,t.TAB,t.NEW_LINE,t.CARRIAGE_RETURN,t.ZERO_WIDTH_NO_BREAK_SPACE],_=1e3,P=1024*4,E={NOT_AN_ARRAY:'Input must be a valid JSON array starting with "["',INVALID_JSON_ARRAY:"Invalid JSON array format - array is incomplete or malformed",MISSING_ELEMENT:"Missing element between commas in JSON array",TRAILING_COMMA:"Trailing comma is not allowed in JSON array",INVALID_ELEMENT:a=>`Invalid JSON element found in array: ${a}`};var u=class extends Error{constructor(e){super(e),this.name="JsonArrayStreamError"}},S=class extends Error{constructor(e){super(e),this.name="JsonArrayAppendError"}};var T=class T extends g.Transform{constructor(e){super({readableObjectMode:!0,writableObjectMode:!1}),e||console.warn("Warning: Encoding not specified. Defaulting to UTF-8 to prevent issues."),this.state={encoding:e||"utf-8",rootDetected:!1,elementDetected:!1,elementType:"others",elementParser:null,elementEnclosureCount:0,isCharInsideQuotes:!1,isCharEscaped:!1,commaSkipped:!1,buffer:"",parsedElements:[]}}resetParser(){this.state.elementDetected=!1,this.state.elementType="others",this.state.elementParser=null,this.state.elementEnclosureCount=0,this.state.isCharInsideQuotes=!1,this.state.isCharEscaped=!1,this.state.buffer=""}parseElement(){try{return JSON.parse(this.state.buffer)}catch(e){throw new u(E.INVALID_ELEMENT(this.state.buffer))}}parseStringElement(e){if(this.state.buffer=`${this.state.buffer}${e}`,e===t.QUOTE){if(this.state.isCharInsideQuotes&&!this.state.isCharEscaped){let s=this.parseElement();return this.state.parsedElements.push(s),!0}this.state.buffer===t.QUOTE?this.state.isCharInsideQuotes=!0:this.state.isCharEscaped&&(this.state.isCharEscaped=!1)}else e===t.ESCAPE?this.state.isCharEscaped=!this.state.isCharEscaped:this.state.isCharEscaped&&(this.state.isCharEscaped=!1);return!1}parsePrimitiveElement(e){if([t.COMMA,t.BRACKET.CLOSE,...C].includes(e)){let s=this.parseElement();return this.state.parsedElements.push(s),!0}return this.state.buffer=`${this.state.buffer}${e}`,!1}parseContainerElement(e){let s=this.state.elementType==="array"?t.BRACKET:t.BRACE;if(e===s.OPEN&&!this.state.isCharInsideQuotes)this.state.buffer=`${this.state.buffer}${e}`,this.state.elementEnclosureCount+=1;else if(e===s.CLOSE&&!this.state.isCharInsideQuotes){if(this.state.buffer=`${this.state.buffer}${e}`,this.state.elementEnclosureCount-=1,this.state.elementEnclosureCount===0){let r=this.parseElement();return this.state.parsedElements.push(r),!0}}else this.state.buffer.length&&(this.state.buffer=`${this.state.buffer}${e}`,e===t.ESCAPE?this.state.isCharEscaped=!this.state.isCharEscaped:e===t.QUOTE&&!this.state.isCharEscaped?this.state.isCharInsideQuotes=!this.state.isCharInsideQuotes:this.state.isCharEscaped&&(this.state.isCharEscaped=!1));return!1}_transform(e,s,r){try{let n=Buffer.from(e,this.state.encoding).toString("utf-8");for(let i of n){if(!this.state.rootDetected){if(![...C,t.BRACKET.OPEN].includes(i))throw new u(E.NOT_AN_ARRAY);this.state.rootDetected=i===t.BRACKET.OPEN;continue}if(!this.state.elementDetected){if(i===t.BRACKET.CLOSE){if(this.state.buffer.length>0)throw new u(E.INVALID_JSON_ARRAY);if(this.state.commaSkipped)throw new u(E.TRAILING_COMMA);break}else if(i===t.COMMA){if(this.state.commaSkipped)throw new u(E.MISSING_ELEMENT);this.state.commaSkipped=!0}this.state.elementDetected=![t.COMMA,...C].includes(i)}this.state.elementDetected&&(this.state.elementParser||(i===t.BRACKET.OPEN?(this.state.elementType="array",this.state.elementParser=this.parseContainerElement.bind(this)):i===t.BRACE.OPEN?(this.state.elementType="object",this.state.elementParser=this.parseContainerElement.bind(this)):i===t.QUOTE?(this.state.elementType="string",this.state.elementParser=this.parseStringElement.bind(this)):(this.state.elementType="others",this.state.elementParser=this.parsePrimitiveElement.bind(this)),this.state.commaSkipped=!1),this.state.elementParser(i)&&(this.resetParser(),i===t.COMMA&&(this.state.commaSkipped=!0)))}if(this.state.parsedElements.length){for(let i of this.state.parsedElements)this.push(i===null?T.NULL:i);this.state.parsedElements=[]}r()}catch(n){r(n)}}_flush(e){this.state.buffer.length>0||!this.state.rootDetected?e(new u(E.INVALID_JSON_ARRAY)):e()}};T.NULL=Symbol("JsonArrayStream.NULL");var b=T,B=b;var N=require("fs/promises");var y=(a,e,s,r=0,n=!1)=>w(void 0,null,function*(){let i=Buffer.from(",",s),l=r;n&&(yield a.write(i,0,1,l),l++);for(let o=0;o<e.length;o+=_){let f=Math.min(o+_,e.length),h=e.slice(o,f),c=[];for(let d=0;d<h.length;d++)o+d>0&&c.push(i),c.push(Buffer.from(JSON.stringify(h[d]),s));let p=Buffer.concat(c);yield a.write(p,0,p.length,l),l+=p.length}return l}),v=(a,e,...s)=>w(void 0,null,function*(){if(s.length===0)return;let r=null;try{let i=(yield(0,N.stat)(a).catch(m=>{if(m.code==="ENOENT")return{size:0};throw m})).size;if(i===0){r=yield(0,N.open)(a,"w+"),yield r.write(Buffer.from("[",e));let m=yield y(r,s,e,1);yield r.write(Buffer.from("]",e),0,1,m);return}r=yield(0,N.open)(a,"r+");let l=i,o=!1,f=!1,h=Buffer.alloc(P),c=i;for(;c>0&&!o;){let m=Math.min(P,c),O=c-m;yield r.read(h,0,m,O);let R=h.toString(e,0,m);for(let A=m-1;A>=0;A--)if(!C.includes(R[A]))if(o){if(!f){f=R[A]!==t.BRACKET.OPEN;break}}else if(R[A]===t.BRACKET.CLOSE)o=!0,l=O+A;else throw new S(E.INVALID_JSON_ARRAY);c=O}if(!o)throw new S(E.INVALID_JSON_ARRAY);let p=Buffer.alloc(i-l);yield r.read(p,0,p.length,l);let d=yield y(r,s,e,l,f);yield r.write(p,0,p.length,d)}finally{yield r==null?void 0:r.close()}}),L=v;0&&(module.exports={JsonArrayStream,appendToJsonArrayFile});