UNPKG

@d3x0r/sack-gui

Version:

SACK abstraction library exposed to JS to provide low level system services.

1,509 lines (1,355 loc) 52.4 kB
let currentStorage = null; //const sack = require( "sack.vfs" ); //console.log( "sack?", sack ); export default function( sack ) { const _debug = false; const _debug_dangling = false; const _debug_output = _debug || false; const _debug_object_convert = _debug || false; const _debug_ll = false; // remote receive message logging. const _debug_map = false; const _debug_replace = false; if( "undefined" === typeof log ) { const os = require( "os" ); //const os = require( "os" ); sack.SaltyRNG.setSigningThreads( os.cpus().length ); } // save original object. const _objectStorage = sack.ObjectStorage; const nativeVol = sack.Volume(); const path = import.meta?.url?.replace(/file\:\/\/\//, '' ).split("/") || "."; //console.log( "path?", path ); //const path = import.meta.url.replace(/file\:\/\/\//, '' ).split("/"); const root = path;// __dirname;//(path.splice(path.length-1,1),path.join('/')+"/"); const remoteExtensionsSrc = nativeVol.read( root+"/object-storage-remote.js" ); //const remoteExtensionsSrc = nativeVol.read( __dirname+"/object-storage-remote.js" ); const remoteExtensions = remoteExtensionsSrc?remoteExtensionsSrc.toString():"// No COde Found"; const jsonRemoteExtensions = JSON.stringify( remoteExtensions ); function shortId( s ) { return sack.Id( s ); } let allDangling = new Map(); let dangling = []; let objectRefs = 0; let currentContainer = null; let preloadStorage = null; // storage thrown from main thread const updatedPrototypes = new WeakMap(); // ---- This block of variables is for parsing JSOX object and foriegn reference revivals during map const fullObjectMaps = new WeakMap(); let rootContainer = false; let rootObjectSet = false; let rootObjectContainer = null; //let mapFullObject = false; // ------ end foreign reference tracking ^^^ let currentReadId ; let loading = null; // mulitple requests for getRoot stack here until the first is resolved.... // these are used during store events because they are delay-flushed the same object may get re-written. const pendingStore = []; let lastStore = 0; const storedPromise = Promise.resolve(undefined); let mapping = false; // default options used when map is not passed options. const defaultMapOpts = {depth:0}; class DbStorage { // this is a super simple key value store in a (postgresql) database. // some support to fall back to sqlite - but will require manually specifying options // need to update personality detection in C library. #db = null; #psql = false; #maria = false; #mysql = false; #sqlite = false; constructor(db, opts ) { console.trace( "Options:", db, db.provider, opts ); this.#db = db; if( db.provider === 3 ) this.#psql = true; else if( db.provider === 2 ) this.#mysql = true; else if( db.provider === 5 ) this.#maria = true; // really is mariadb if( db.provider === 1 ) this.#sqlite = true; { if( this.#psql ) { const table1 = "create table if not exists os (id char(45) primary key,value varchar(4096),updated datetime default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP)"; db.makeTable( table1 ); } if( this.#maria || this.#mysql ) { const table1 = "create table if not exists os (id char(45) primary key,value LONGTEXT,updated datetime default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP)"; db.makeTable( table1 ); } else if( !this.#sqlite ) { db.makeTable( "create table os (id char(45) primary key,value char,updated datetime default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP)" ); } } } read( obj, ...args ) { let version = 0; let parser = null; let cb = null; //console.log( "Read in object storage interface? should it have been a file?", obj ); for( let arg of args ) { if( "function" === typeof arg ) { cb = arg; } else if( "object" === typeof arg ) { parser = arg; } else if( "number" === typeof arg ) { version = arg; } } const records = this.#db.do( "select value from os where id=?", obj ); //console.log( "Thing?", obj, records ); if( records.length ) { const o = parser.parse( records[0].value ); cb( o ); } else cb( undefined ); } writeRaw( opts, obj ) { //console.log( "Write Raw:", opts, obj ); if( this.#psql ) this.#db.do( "insert into os (id,value)values(?,?) ON CONFLICT (id) DO UPDATE SET value=?", opts.id, obj,obj ); else if( this.#mysql ) { this.#db.do( "insert into os (id,value)values(?,?) ON CONFLICT (id) DO UPDATE SET value=?", opts.id, obj,obj ); } else if( this.#maria ) { try { this.#db.do( "insert into os (id,value)values(?,?) ON DUPLICATE KEY UPDATE value=?", opts.id, obj, obj ); //this.#db.do( "insert into os (id,value)values(?,?) ON CONFLICT (id) DO UPDATE SET value=?", opts.id, obj, obj ); } catch( err ) { console.log( "Insert error:", err ); } } else this.#db.do( "insert into os (id,value)values(?,?) ON CONFLICT (id) DO UPDATE SET value=excluded.value", opts.id, obj ); } } // manufacture a JS interface to _objectStorage. class ObjectStorage { cached = new Map(); cachedContainer = new Map(); stored = new WeakMap(); // objects which have alredy been through put()/get() storedIn = new WeakMap(); // objects which have alredy been through put()/get() decoding = []; pending = []; stringifier = sack.JSOX.stringifier(); parser = null; // this will be filled when .get() is called. currentParser = null; root = null; // this gets filled when the root file system is enabled. encoders = []; decoders = []; constructor (...args) { if( !( this instanceof ObjectStorage ) ) return new ObjectStorage( args ); const arg0 = args[0]; if( arg0 instanceof sack.Sqlite ) { _debug && console.log( "Using a db storage interface..." ); preloadStorage = new DbStorage( arg0 ); }else _debug && console.log( "Using a file interface?" ); const newStorage = preloadStorage || _objectStorage(...args); if( newStorage === preloadStorage ) preloadStorage = null; const thisStorage = this; this.storage = newStorage; setupStringifier( this, this.stringifier ); this.handleMessage = handleMessage; this.StoredObject = StoredObject; function handleMessage( ws, msg ) { _debug_ll && console.log( "Storage Remote Message:", msg ); try { if( msg.op === "connect" ) { ws.send( `{op:connected,code:${jsonRemoteExtensions}}` ); return true; } if( msg.op === "get" ) { newStorage.readRaw( currentReadId = msg.opts.id , (data)=>{ const msgout = newStorage.stringifier.stringify( { op:"GET", id:msg.id, data:data } ); //console.log( "Read ID:", msg.opts.id, typeof data, msgout ); ws.send( msgout ); } ) return true; } if( msg.op === "put" ) { // want to get back a file ID if possible... // and/or use the data for encoding/salting/etc... which can determine the result ID. //console.log( "PUT THIGN:", msg.id ); newStorage.storage.writeRaw( msg.rid, msg.data); ws.send( { op:"PUT", id:msg.id, r:msg.rid } ); return true; } }catch(err) { console.log( "Failed?", err ) } return false; } //console.log( "constructed storage..." ); } setupStringifier(s) { return setupStringifier( this, s ); } async map( o, opts ) { // // options = { // depth : maximum distance to load... // } // //thisStorage.mapping = true; //this.parser. if( !opts ) opts = defaultMapOpts; //if( opts && opts.depth === 0 ) return; //console.log( "External API invoked map..."); const rootId = this.stored.get( o ); if( !rootId ) { console.trace( "Object was not stored, cannot map", o ); return Promise.resolve( o ); } if( rootId[0] === '~' ) { //console.log( "this is mapping a directly referenced field; which means we have to resolve this at least.", rootId ); const loading = this.storedIn.get( o ); //console.log( "Get result:", loading ); return loadPending( this, loading ); } //console.trace( "cached container on root ID check", rootId ); return this.cachedContainer.get( rootId ).map( opts ); } async getRoot() { console.log( "Getting root...!!!!" ); const this_ = this; if( this_.root ) return this_.root; if( loading ) { return new Promise( (res,rej)=>{ loading.push( {res:res, rej:rej} ); } ); } console.log( "Getting root..." ); this_.addEncoders( [ { tag: "d", p:FileDirectory, f:null}, { tag: "f", p:FileEntry, f:null} ] ); this_.addDecoders( [ {tag: "d", p:FileDirectory }, {tag: "f", p:FileEntry } ] ) const result = new FileDirectory( this, "?" ); //console.trace( "Who gets this promise for getting root?"); return this_.get( { id:result.id } ) .then( (dir)=>{ if( !dir ) { return result.store(true) .then( function(id){ //console.log( "1) Assigning ID to directory", id ); Object.defineProperty( result, "id", { value:id } ); finishLoad( result ); return dir; } ) } // foreach file, set file.folder else{ for( var file of dir.files ) { //console.log( "File:", dir, file ); Object.defineProperty( file, "folder", {value:dir} ); } Object.defineProperty( dir, "id", { value:result.id } ); finishLoad(dir); } function finishLoad(dir) { //console.log( "2) Assigning ID to directory(sub)", result.id ); Object.defineProperty( dir, "volume", {value:this_} ); this_.root = dir; } return dir; } ); } async dir() { return (await this.getRoot).files; } // define a class... to be handled by stringification defineClasss (a,b) { this.stringifier.defineClass(a,b); } scan( from ) { var fromTime = ( from.getTime() * 256 ); //this.loadSince( fromTime ); } getContainerRef( obj, options ) { var containerId = this.stored.get( obj ); if( containerId ) { const container = this.storedIn.get( obj ); //console.log( "ID:", containerId, container ); return container; } return null; } getContainer( obj, options ) { var container = this.stored.get( obj ); var storage; if( container ) { container = this.cachedContainer.get( container ); return container; } //console.trace( "Getting a new container...", options ); container = new ObjectStorageContainer(this,obj,options); //console.log( " *** SETTING ID HERE", container.id); //if( container.id === undefined ) throw new Error( "Error along path of setting container ID"); if( container.id ) { this.stored.set( obj, container.id ); this.cached.set( container.id, container.data ); console.log( "Container storage gets updated here ---------------------------"); container.storage = this; this.cachedContainer.set( container.id, container ); }else { // should fix 'undefined' as a index in stored //console.log( "Storage still has to come from real storage operation... not setting cached ID"); console.log( "Container storage NOT gets updated here ---------------------------"); } } delete( opts ) { return this.storage.delete(opts); } remove( opts ) { if( "string" === typeof opts ) { var container = this.cached.get( opts ); if( !container ) throw new Error( "This is not a tracked object." ); this.delete( opts ); } else if( "object" === typeof opts ) { var container = this.stored.get( opts ); if( !container ) throw new Error( "This is not a tracked object." ); this.delete( container); } } addEncoders(encoderList) { const this_ = this; encoderList.forEach( f=>{ this_.encoders.push( f ); //console.log( "encoders:", encoderList ); this_.stringifier.registerToJSOX( f.tag, f.p, f.f ) ; }); } addDecoders(encoderList) { const this_ = this; encoderList.forEach( f=>{ function redirect(f) { return function redirect(field,val) { // call loaded anyway. loaded can't change the object. if( !field && ( this instanceof StoredObject ) ) this.loaded( this_, currentReadId ); // call registered handler; handler can change what this object is. if(f) { if( !field ) { return f.call( this, field, this_ ); } else return f.call(this,field,val ); }else { if( !field ) return this; else return val; } } } f.f = redirect(f.f); this_.decoders.push( f ); if( this.parser ) this.parser.fromJSOX( f.tag, f.p, f.f ); }); } getCurrentParseRef() { if( this.currentParser ){ return this.currentParser.currentRef(); } return null; } stringify( obj ) { const containerId = this.stored.get( obj ); if( containerId ) { const container = this.cachedContainer.get( containerId ); const stringifier = this.stringifier; if( container ) { container.encoding = true; const storage = stringifier.stringify( container ); container.encoding = false; return storage; } } const output = stringifier.stringify( container ); //console.log( "stringify default:", output, container ) ; return output; } // this hides the original 'put' put( obj, opts ) { const this_ = this; if( currentContainer && currentContainer.data === this ) { saveObject( null, null ); _debug && console.log( "Returning same id; queued save to background...") return Promise.resolve( currentContainer.id ); }else { } return new Promise( saveObject ); function saveObject(res,rej) { if( "string" === typeof obj && opts.id ) { console.log( "SAVING A STRING OBJECT" ); // this isn't cached on this side. // we don't know the real object. this_.storage.writeRaw( opts.id, obj ); return res?res( opts.id ):null; } var container = this_.stored.get( obj ); if( !container && opts && opts.id ) { const oldObj = this_.cached.get( opts.id ); if( oldObj ) container = this_.stored.get( oldObj ); } let storage; _debug && console.log( "Put found object?", container, obj, opts ); if( container ) { container = this_.cachedContainer.get( container ); if( obj !== container.data ) { //console.log( "Overwrite old data with new?", container.data, obj ); if( val === this ){ console.trace( "This is so bad(3)..."); process.exit(0); } container.data = obj; } if( !container.nonce ) { // make sure every item that is in an index // has been written... if( this_.def ) this_.def.indexes.forEach( index=>{ index.data.forEach( (item)=>{ this_.put( item ); }); }) var stringifier; if( opts && opts.extraEncoders ) { stringifier = sack.JSOX.stringifier(); this_.setupStringifier( stringifier ); opts.extraEncoders.forEach( f=>{ stringifier.registerToJSOX( f.tag, f.p, f.f ) }); }else { stringifier = this_.stringifier; } container.encoding = true; //console.log( "Setting root container mode....(object already contained)"); rootContainer = true; storage = stringifier.stringify( container ); container.encoding = false; if( !container.id || container.id === "null" ) { console.trace( "0) Container has no ID or is nUll", container ); } _debug_output && console.trace( "WRite:", container.id, storage ); this_.storage.writeRaw( { id:container.id, version:true }, storage ); return res?res( container.id ):null; } else { throw new Error( "record is signed, cannot put" ); } } if( opts && opts.id ) { var stringifier; if( opts.extraEncoders ) { stringifier = sack.JSOX.stringifier(); this_.setupStringifier( stringifier ); opts.extraEncoders.forEach( f=>{ stringifier.registerToJSOX( f.tag, f.p, f.f ) }); }else { stringifier = this_.stringifier; } // raw object; no encoding to set. console.log( "Set rootObjectSet(1)...."); //rootObjectSet = true; storage = stringifier.stringify( obj ); if( !opts.id || opts.id === "null" ) { throw new Error( "Container has no ID or is null" ); } _debug_output && console.trace( "Write(root set):", opts, storage ); if( this_.versioned ) { const id = opts.id.split( '.' ); //console.log( "Storing with an ID already?", obj, opts ); const newVersion = this_.storage.writeRaw( {id:id[0]}, storage ); opts.id = id[0] + '.' + newVersion; } else this_.storage.writeRaw( opts, storage ); res && res( opts.id ); } else if( !opts || !opts.id ) { // need a new ID for an object (not string) _debug && console.log( "New bare object, create a container...", opts ); if( !opts ) opts = { id : shortId() } else opts.id = shortId(); //console.log( "Storage ID is picked here:", opts ); if( "object" === typeof obj ) { container = new ObjectStorageContainer(this_,obj,opts); if( !container.id || "string" !== typeof container.id ) throw new Error( "Failed to get a conatiner ID for some reason" ); //console.log( "New container looks like... ", container.id, container ); //console.log( "saving stored container.id", typeof obj, obj, container.id ); //this_.stored.delete( obj ); //console.log( " *** SETTING ID HERE", container.id); this_.stored.set( obj, container.id ); if( container.id === undefined ) throw new Error( "Error along path of setting container ID"); this_.cached.set( container.id, container.data ); //console.log( "3CACHE IS SET HERE ------------------------- ", obj) this_.cachedContainer.set( container.id, container ); var stringifier; if( opts.extraEncoders ) { stringifier = sack.JSOX.stringifier(); this_.setupStringifier( stringifier ); opts.extraEncoders.forEach( f=>{ stringifier.registerToJSOX( f.tag, f.p, f.f ) }); }else { stringifier = this_.stringifier; } opts.id = container.id container.encoding = true; console.log( "Setting root container mode(2)...."); rootContainer = true; storage = stringifier.stringify( container ); container.encoding = false; } else { container = new ObjectStorageContainer(this_,obj,opts); storage = obj; } if( !container.id || container.id === "null" ) { console.trace( "Container has no ID or is nUll", container ); } _debug_output && console.log( "Output container to storage... ", container, container.storage, container.id ); try { this_.storage.writeRaw( opts, storage ); if( container.id === undefined ) throw new Error( "Error along path of setting container ID"); this_.cached.set( container.id, container.data ); //console.log( "2CACHE IS SET HERE ------------------------- ", obj) this_.cachedContainer.set( container.id, container ); }catch(err) { console.log( "WRITE RAW?", this_ )} //console.log( "OUTPUT:", storage ); res && res( container.id ); } } /* sack.ObjectStorage.prototype.update( objId, obj ) { var container = new ObjectStorageContainer(this,sack.JSOX.stringify(obj),sign); this.stored.set( obj, container.id ); this.cached.set( container.id, container ); return container.id; } */ } // end of put() method get( opts ) { //this.parser. let resolve; //let reject; const os = this; if( "string" === typeof opts ) { opts = { id:opts , extraDecoders : null }; } if( !opts ){ console.trace( "Must specify options ", opts); return null; } { const priorLoad = this.cachedContainer.get( opts.id ); //console.log( "already found?", priorLoad ); if( priorLoad ) return Promise.resolve( priorLoad.data ); } //console.trace( "TEST going to get:", opts ) const priorDecode = this.decoding.find( d=>d.opts.id === opts.id ); if( priorDecode ){ //console.trace( "already decoding...(use same promised result) things?", priorDecode ); return priorDecode.p; } if( !this.parser ){ this.parser = sack.JSOX.begin(); //console.log( "ADDING ~os"); this.parser.fromJSOX( "~os", ObjectStorageContainer, reviveContainer ); // I don't know ahead of time which this is. this.parser.fromJSOX( "~or", objectStorageContainerRef, reviveContainerRef ); // I don't know ahead of time which this is. for( let f of this.decoders ) this.parser.fromJSOX( f.tag, f.p, f.f ); } const parser = this.parser; if( opts.extraDecoders ) { parser = sack.JSOX.begin( ); //console.log( "Adding ~os handler"); parser.fromJSOX( "~os", ObjectStorageContainer, reviveContainer ); // I don't know ahead of time which this is. parser.fromJSOX( "~or", objectStorageContainerRef, reviveContainerRef ); // I don't know ahead of time which this is. //console.log( "this has no decoders? ", this ); if( this.decoders && this.decoders.length ) this.decoders.forEach( f=>parser.fromJSOX( f.tag, f.p, f.f ) ); // allow extra to override default. if( opts && opts.extraDecoders && opts.extraDecoders.length ) { opts.extraDecoders.forEach( f=>parser.fromJSOX( f.tag, f.p, f.f ) ); } //console.log( "Created a parser for revival..." ); } this.currentParser = parser; //console.log( "(get)Read Key:", opts.id ); const p = new Promise( function(res,rej) { resolve = res; //reject = rej; const priorReadId = currentReadId; try { currentStorage = os; // this part is synchronous... but leaves JS heap from os.read(), does callbacks using parser, but results with a parsed object // all synchronously. const parts = opts.id.split('.'); os.storage.read( currentReadId = parts[0], Number( parts[1] ) , parser, (obj)=>resultDecode(opts, priorReadId, obj,res) ); }catch(err) { console.log( "ERROR:", err ); currentReadId = priorReadId; rej(err); } } ); //console.log( "PUSHING THING TO LOAD LATER (decoding)?", opts ); this.decoding.push( { p:p, opts:opts } ); p.then( (r)=>{ //console.log( "Removing decoding object with then...", r); const doneDecoding = os.decoding.findIndex( d=>d.opts === opts ); //console.log( "decode should already be resolved:", doneDecoding, os.decoding, opts, r ); if( doneDecoding >=0 ) os.decoding.splice( doneDecoding, 1 ); else console.log( "Failed to find decoding object?" ); return r; } ); return p; function resultDecode( opts,priorReadId, obj,resolve) { // with a new parser, only a partial decode before revive again... try { _debug && console.log( "Read resulted with an object:", opts.id, obj ); if( obj instanceof ObjectStorageContainer ) { //console.log( "SHOULD HAVE FIXED IT ALREDY!"); obj.storage = os; } let deleteId = -1; currentStorage = null; //const extraResolutions = []; for( let n = 0; n < os.decoding.length; n++ ) { const decode = os.decoding[n]; //console.log( "pending decode?", decode, opts, decode.opts === opts ) if( decode.opts === opts ) deleteId = n; else if( decode.opts.id === opts.id ) { console.log( "pending decode resolve?"); decode.res( obj ); } } if( deleteId >= 0 ) { console.log( "Something else is alwso waiting for this result...", os.decoding, deleteId); os.decoding.splice( deleteId, 1 ); } var found; do { var found = os.pending.findIndex( pending=>{ console.log( "what is in pending?", key, opts.id, pending ); return pending.id === opts.id } ); if( found >= 0 ) { os.pending[found].ref.o[os.pending[found].ref.f] = obj.data; os.pending.splice( found, 1 ); } } while( found >= 0 ); //console.log( "Object is what?", obj ); if( obj && ( obj instanceof ObjectStorageContainer ) ){ //console.log( "GOTzz:", obj, obj.id, obj.data ); if( !("id" in obj )) Object.defineProperty( obj, "id", { value:currentReadId } ); //console.log( "Why does object have no data?", obj.data, obj ); os.stored.set( obj.data, obj.id ); os.cachedContainer.set( obj.id, obj ); currentReadId = priorReadId; //for( let res in extraResolutions ) res.res(obj.data); resolve(obj.data); } else { currentReadId = priorReadId; //for( let res in extraResolutions ) res.res(obj); //console.log( "RESOLVE WITH OBJECT NEED DATA?", ( obj instanceof ObjectStorageContainer ) ); resolve(obj) } }catch( err) { console.log( "Less fatal error:", err ); } } function objectStorageContainerRef( s ) { //_debug_dangling && try { const existing = os.cachedContainer.get(s); const here = os.getCurrentParseRef(); const thisDangling = dangling; //console.log( "Conainer ref, this will resolve in-place") this.d = {id:s,p:null,res:null,rej:null,r:{o:here.o,f:here.f},rootMap:null }; if( !existing ) { _debug_dangling && console.log( "PUSHING DANGLING REFERNCE", this ); const requests = allDangling.get( this.d.id ) || []; if( !requests.length ) allDangling.set( this.d.id, requests ); const p = this.d.p = new Promise( (res,rej)=>{ _debug_dangling && console.log( "setting up pending promise to resolve:", s ); this.d.res = res; this.d.rej = rej; }).then( (obj)=>{ _debug_dangling && console.log( "replace value at reference with real value:", here, obj, thisDangling.length, s ) //const dr = thisDangling.findIndex( d=> d.d.id === s ); //if( dr >= 0 ) thisDangling.splice( dr, 1 ); //else _debug_dangling && console.log( "FAILED TO FIND DANGLING REFERENCE 1" ); // this is the last time the promise exists... os.storedIn.delete( p ); os.stored.delete( p ); const inObject = dangling.find( d=>d === this ); if(inObject >= 0 ) { _debug_replace && console.log( "Removing request from container dangling" ); dangling.spice( inObject, 1 ); } _debug_replace && console.log( "-------- REPLACE VALUE HERE:", this.d.r.o[this.d.r.f], this.d.r.o, this.d.r.f, obj ); return (this.d.r.o[this.d.r.f] = obj); }).catch( (err)=>{ console.log( "CATCH UNCAUGHT PLEASE DO", err); }); os.storedIn.set( this.d.p, this ); //console.log( " *** SETTING ID HERE", s); os.stored.set( this.d.p, '~or"'+s+'"' ); requests.push( this ); dangling.push( this ); objectRefs++; } else { _debug_dangling && console.log( "NOT DANGLING; existing object already exists... " ); this.d.p = Promise.resolve( existing.data ); // this will still have to be swapped. } //console.log( "Container ref:", s, this.d.p ); } catch(err) { console.log( "Init ObjectContainerRef failed:", err)} } function reviveContainer( field, val ) { //console.log( "(JS)Revive container:", this, field, val ); if( !field ) { // finished. if( objectRefs ) { /* sets dangling property on container */ _debug_dangling && console.log( "Collapse dangling:", dangling.length ); if( !this.dangling ) Object.defineProperty( this, "dangling", { value:dangling } ); else{ this.dangling.push.apply(this.dangling, dangling ) //console.log( "Added more to the dangling things..." ); } // resets to store new objects for next load. dangling = []; objectRefs = 0; } Object.defineProperty( this, "id", { value:currentReadId } ); //console.log( "handle object ??", this ); if( this.nonce ) { let s = null; var v = sack.SaltyRNG.verify( s = os.stringifier.stringify(this.data), this.nonce, this.sign.pad1, this.sign.pad2 ); if( v.key !== currentReadId ) { throw new Error( `Data corrupted: s:'${s}' v:${v}, this Id:${currentReadId}` ); } else console.log( "Valid record.", v, this.sign ); } const request = allDangling.get( currentReadId ); //console.log( "Revive container final pass... does this resolve?", this, request, allDangling, currentReadId ); if( request ) { _debug_replace && console.log( "Dispatching all resolutions waiting on this object..." ); allDangling.delete( currentReadId ); for( let load of request ) { load.d.res( this.data ); } } return this; } else { const this_ = this; // new value isn't anything special; just set the value. //console.log( "This sort of thing... val is just a thing - like a key part identifier...; but that should have been a container."); if( val instanceof Promise ) { //_debug_dangling && console.log( "Value is a promise, and needs resolution.") var dangle = dangling.find( d=>d.d.p===val ); if( dangle ) dangle.d.n = field; this_.data[field] = val return val.then( (val)=>{ _debug_dangling && console.log( "(DOESN'T HAPPEN NOW?) THIS SHOULD BE WHAT REPLACES THE VALUE", field, val ); this_.data[field] = val }); } // a custom type might want something else... if( field === "data" ) { //console.log( "This should have fixed the 'pending value' value" ); if( val === this ){ console.trace( "This is so bad..."); process.exit(0); } this.data = val; return undefined; } this[field] = val; return undefined; } } function reviveContainerRef( field, val ) { //console.trace( "Revival of a container reference:", this, field, val ); if( !field ) { const existing = os.cachedContainer.get( this.d.id ); if( existing ){ //console.log( "duplicate reference... return the existing promise(but this is still used later)", existing, this.d.id); // even better, don't even store the reference, return the real //console.log( "So, just return with the real object to assign. (and remove fom dangling)"); const id = dangling.find( d=>d.d === this.d ); objectRefs--; if( id >= 0 ) dangling.splice( id, 1 ); //console.log( "this is the real value anyway1...", existing.data ); return existing.data; } os.stored.set( this.d.p, '~or"'+this.d.id+'"' ); //console.log( "this is the real value anyway2...", this.d.p ); return this.d.p; } else { //console.log( "this is the real value anyway3...", val ); return val; } } } // end of get() method flush() { // this doesn't really cache anything to flush? } } // end of ObjectStorage class function loadPending(store, load, opts) { //console.log( "doing real get, which results with ANOTHER promise", load, store ); return store.get( {id:load.d.id}).then( (obj)=>{ //console.log( "Storage requested:", load.d.id ); if( load.d.res ) { load.d.res(obj); // result with real value. } else { load.d.p.then( o2=>{if( obj!==o2) throw new Error( "resolved and loaded object mismatch");return o2}) return load.d.p; } //const existobj = thisStorage.stored.get( load.d.p ); //const exist = thisStorage.stored.get( obj ); //console.log( "This is the promise object?", load.d.id, exist, existobj ); const objc = store.cachedContainer.get( load.d.id ); _debug_dangling && console.log( "Replaced promise with container", objc, obj ); // resolving this promis on load.d will set this. if( opts && opts.depth ) { _debug_replace && console.log( "Mapping the object, if there is any more" ); objc.map( {depth:opts.depth-1} ).then( (objc)=>{ _debug_replace && console.log( "mapped sub-object resultinged...", objc ); load.d.waiting--; if( !load.d.waiting ) { _debug_dangling && console.log( "1map is resolving with : ", objc, load.d.rootMap.data ); _debug_replace && console.log( "Resolving map request...", load.d.rootMap.data ); }else console.log( "waiting for something..." ); }); } else { _debug_replace && console.log( "waiting for something, not resolving" ); } //console.log( "This is just an object - from a then callback" ); return obj; }).catch( (err)=>{ console.log( "Error:", err ); }) } function objectToJSOX( stringifier, storage, obj ){ _debug_object_convert && console.log( "Convert Any Object to JSOX:", typeof obj, obj ); if( rootContainer ) { //console.log( "know the next thing is is a new container, get the data..." ); rootContainer = false; //rootObject = obj.data; rootObjectContainer = obj; fullObjectMaps.set( rootObjectContainer, obj.id ); }else { if( rootObjectSet ) { //console.log( "know the next thing is is a new container, get the data..." ); //const newId = shortId(); console.trace( "setting root container ID to ?", rootObjectSet, rootContainer, rootObjectContainer , "\nbecomes", obj ); //rootObject = obj; rootObjectContainer = obj; // I don't know this has to be in this map... it's a // un-named part of an object to encode, it might have a path // reference, but that's not what this is for. //fullObjectMaps.set( rootObjectContainer, newId ); } //else // console.log( "Sub Object of something...", this !== rootObjectContainer ); } if( false /*mapFullObject*/ ) if( obj !== rootObjectContainer ) { //const curMap = fullObjectMaps.get( rootObjectContainer ); const ref = stringifier.getReference( obj ); console.log( "(full map)saving internal reference:", ref, obj ); fullObjectMaps.set( obj, ref ); } //const isRoot = (rootObjectContainer === obj); if( _debug_object_convert ) if(obj instanceof Promise ) { console.log( "This is still a pending object reference(?)", obj ); } // see if we alread stored this... (or are currently storing this.) (back references container) //console.log( "this?", obj, rootObjectContainer ); var exist = !rootObjectSet && rootObjectContainer && storage.stored.get( obj ); //console.log( "Exist after promise?", exist ); _debug_object_convert && console.log( "THIS GOT CALLED?", obj, Object.getPrototypeOf( obj ), exist ); if( exist ) { //console.trace( "Is it this sort of exist?", exist ); if( exist[0] == '~' ) { // this is a dangling promise return exist; } var objContainer = storage.cachedContainer.get( exist ); _debug_object_convert && console.log( "Object stored independantly... RECOVERED:", exist, obj&&obj.encoding ); _debug_object_convert && console.log( "existing reference...", exist ); //console.log( "Cached container:", objContainer.encoding, obj ) if( objContainer.encoding ) return obj; else { return '~or"'+exist+'"'; } } else { if( rootObjectContainer !== obj ) { exist = fullObjectMaps.get( obj ); if( exist ) { // sometimes it's a storage container, other times it's a raw object... console.trace( "Deep remote reference.", rootObjectSet, obj, exist ); //return '~or"'+exist+'"'; } else { _debug_object_convert && console.log( "not a stored object, encode itself." ); } } } return obj; } function storageObjectToJSOX( stringifier, storage, obj ){ // see if we alread stored this... (or are currently storing this.) (back references container) _debug_object_convert && console.trace( "THIS GOT CALLED?", obj, Object.getPrototypeOf( obj ) ); var exist = storage.stored.get( obj ); if( exist ) { console.log( "Think this already is existing?", exist ); if( exist[0] == '~' ) { // is a pending promise... return exist; } var obj = storage.cachedContainer.get( exist ); _debug_object_convert && console.log( "Object stored independantly... RECOVERED:", exist, obj.encoding ); if( obj.encoding ) return obj; else { //console.log( "Why is this an ~or and not ~os?"); return '~or"'+exist+'"'; } } else { if( this instanceof ObjectStorageContainer ) { //console.log( "THIS SHOULD ALREADY BE IN THE STORAGE!", this, this.stored.get( this.data ) ); // this is probably the final result when encoding this object. //this.stored.set( this.data, this.id ); // maybe update the ID though. if( obj.nonce ) return {data:obj.data,nonce:obj.nonce,sign:obj.sign}; // this object will be stringified, and have our type prefix prepended. else return {data:obj.data}; // this object will be stringified, and have our type prefix prepended. } } //console.log( "not a container..."); return obj; } // associates object data with storage data for later put(obj) to re-use the same informations. class ObjectStorageContainer { #storage = null; #stored_data = null; data = "pending reference"; // reviving we get null opts. constructor(storage,o,opts) { //if( !this instanceof ObjectStorageContainer ) return new ObjectStorageContainer(thisStorage,o,opts); // created from JSOX; there aare no parameters passed on many usages... //console.trace( "This should always get storage...", storage ) if( storage ) { this.#storage = storage; //_debug_object_convert && console.log( "Something creating a new container..", o, opts ); if( "string" === typeof o ){ if( !o ) throw new Error( "Blank string lookup" ); // still, if the proir didn't resolve, need to resolve this.. let existing = storage.cachedContainer.get( o ); if( existing ) { o = existing.data; if( existing.resolve ) return; } } if( o ) { if( o === this ){ console.trace( "This is so bad(2)..."); process.exit(0); } this.data = o; // reviving we get null opts. } if( opts && opts.sign ) { if( !this.nonce ) { //console.log( "SIGNING FAILS?" ); let s; const signature = sack.SaltyRNG.sign( s = this.#storage.stringifier.stringify(o), opts.sign.pad1, opts.sign.pad2 ); this.nonce = signature.key; //console.log( "Signing returned:", signature ); this.sign = Object.assign( {e:signature.extent,c:signature.classifier},opts.sign ); //console.log( "MAKING A NONCE:", s ); //var v = sack.SaltyRNG.verify( this.#storage.stringifier.stringify(o), this.nonce, opts.sign.pad1, opts.sign.pad2 ); Object.defineProperty( this, "id", { value:signature.id } ); //console.trace( "This makes it go boom?", opts, this.nonce, o, v.key ); } else { // var v = sack.SaltyRNG.verify( thisStorage.stringifier.stringify(o), this.nonce, opts.sign.pad1, opts.sign.pad2 ); // Object.defineProperty( this, "id", { value:v.key } ); } } else { if( opts && opts.id ) { Object.defineProperty( this, "id", { value:opts.id } ); } } } // pass this through a global for jsox fixup at end. // currentContainer = this; Object.defineProperty( this, "encoding", { writable:true, value:false } ); } get stored() { return this.#storage.stored; } set storage( val ) { this.#storage = val; } get storage( ) { return this.#storage; } getStore () { return this.#storage; } async map( opts ) { const dangling = this.dangling; /* this is a property set dynamically */ if( dangling && dangling.length ) { opts = opts || { depth:0, paths:[] }; const rootMap = this; _debug_replace && console.log( "doing a map on a thing... amybe this trace?", this.id ); return new Promise( (res,rej)=>{ const waiting = []; _debug_map && console.log( "Map Object called... dangling:", dangling.length) if( ( "paths" in opts ) && opts.paths.length ){ let nPath = 0; const handled = []; while(nPath < opts.paths.length ){ var path = opts.paths[nPath++]; let part = 0; var obj = this.data; for( var step of path ){ if( obj ) { obj = obj[step]; part++; } else break; } if( obj ) { for( let load of dangling ){ if( load.d.p === obj ){ //console.log( "This should get marked resolved maybe?") handled.push( load ); load.d.rootMap = rootMap; waiting.push( loadPending( this.storage, load, {depth:opts.depth,paths:part<path.length?[path.slice(part)]:undefined} ) ); obj = null; break; } } // obj will probably also be resolved here, because it's not in pending... if( obj ) { const ref = this.#storage.storedIn.get( obj ); console.log( "Failed to find promise?", obj, ref ); } } else console.log( "Failed to find path in object:", this.data, path ); for( let h of handled ) { _debug_replace && console.log( "in map - removing dangling reference?" ); var idx = dangling.findIndex( p=>p===h ); if( idx >= 0 ) { dangling.splice( idx, 1 ); _debug_replace && console.log( "in map - removed dangling reference?" ); } else _debug_replace && console.log( "resolved load was not found???" ); } } } else{ // load everything that's pending on this object. _debug_replace && console.log( "Load all dangling things anyway:", dangling ); for( let load of dangling ){ if( !load.d.rootMap ) { load.d.rootMap = rootMap; waiting.push( loadPending( this.#storage,load,opts, res) ); } else { //console.log( "Dangling has already been triggered to load", load ); } } // wait until the promise resolves to delete this //dangling.length = 0; } if( !waiting.length ){ //console.log( "Nothing scheduled to really wait, go ahead and resolve"); res( rootMap.data ); } else { _debug_replace && console.log( "(AWAIT!)Waiting on some events to resolve...", waiting ); Promise.all( waiting ).then( ( last)=>{ _debug_replace && console.log( "And now we properly resolve.", last ); res( rootMap.data ) } ); } }) } if(0) // untested. if( opts.paths ) { // if the first level is a promised value - it would definitely be in dangling and have been checked above. // so here, we check path step + 1... const nextMaps = opts.paths.reduce( (acc,path)=>path.length>1?(acc.push(path.slice(1)),acc):acc, [] ); if( nextMaps.length ) { // if any paths more than one level... const checks = opts.paths.reduce( (acc,path)=>path.length>1?(acc.push(this.data),acc):acc, [] ); const nextPaths = opts.paths.reduce( (acc,path)=>path.length>1?(acc.push(path[0]),acc):acc, [] ); for( let n = 0; n < nextPaths.length; n++ ) { if( !checks[n] ) continue; // already traced to the end. const step = checks[n][nextPaths[n]]; // this is a field dereference... N is an object. if( step instanceof Promise ) { const isObject = this.#storage.storedIn.get( step ); if( isObject ) { console.log( "This thing should be loaded too?? (ignoring this promise, it is also probalby already handled above..." ); checks[n] = null; } } } } } _debug_dangling && console.log( "Nothing dangling found on object"); return this.data; // this function is async, just return. } } function setupStringifier( newStorage, stringifier ) { stringifier.setDefaultObjectToJSOX( function tojsox(stringifier){ return objectToJSOX( stringifier, newStorage, this ); } ); stringifier.registerToJSOX( "~os", ObjectStorageContainer, function tojsox(stringifier) { return storageObjectToJSOX( stringifier, newStorage, this ) } ); stringifier.store = newStorage; for( let f of newStorage.encoders ) stringifier.registerToJSOX( f.tag, f.p, f.f ) ; } ObjectStorage.getRemoteFragment = function() { return remoteExtensions; } ObjectStorage.Thread = { post: _objectStorage.Thread.post, accept(cb) { _objectStorage.Thread.accept((a,b)=>{ // posted from core thread ( worker can't access disk itself? ) preloadStorage = b; cb(a, new ObjectStorage(b) ); }); } } class FileEntry { constructor ( d ) { this.name = null; this.id = null; // object identifier of this. this.contents = null; this.created = new Date(); this.updated = new Date(); if( d ) Object.defineProperty( this, "folder", {value:d} ); } getLength () { return this.data.length; } read( from, len ) { const this_ = this; //console.trace( "READ RESULTING A PROMISE... PLEASE CATCH"); return new Promise( (res,rej)=>{ //console.log( "reading:", this ); if( this_.isFolder ) { this_.folder.volume.get( { id:this_.id } ) .catch( rej ) .then( (dir)=>{ //console.log( "Read directory, set folder property..." ); for( var file of dir.files ) Object.defineProperty( file, "folder", {value:dir} ); res(dir) } ); } else { //console.log( "Reading ...", this_.id ); if( this_.id ) return this_.folder.volume.get( {id:this_.id} ).then( res ).catch( rej ); //console.log( "Rejecting, no ID, (no data)", this_ ); //console.log( "no file data in it?" ); res( undefined ) // no data } } ); } write( o ) { if( this.id ) try { if( "string" === typeof o ) { console.log( "direct write of string data:", this.id, o ); this.folder.volume.storage.writeRaw( this.id, o ); return Promise.resolve( this.id ); } else if( o instanceof ArrayBuffer ) { console.log( "Write raw buffer", this ); this.folder.volume.storage.writeRaw( this.id, o ); return Promise.resolve( this.id ); } else if( "object" === typeof o ) ;//console.log( "expected encode of object:", o ); } catch(err) { console.log( "Did the above do something?" ) } /* if( !("number"===typeof(len))) { if( "number"===typeof(at) ) { len = at; at = 0; } else { len = undefined; at = undefined; } } */ const this_ = this; //console.log( "Returning a promised volume put:", o, this ); if( this.id ) return this.folder.volume.put( o, this ); else return this.folder.volume.put( o, this ) .then( id=>{ //console.log( "Re-setting id?", this_, id ); this_.id=id; this_.folder.store(); return id; } ) .catch( (err)=>{ console.log( "Error doing put?", err ); }) } async open( fileName ) { if( this.root ) return file.root.open( fileName ); if( this.contents ) { return this.folder.volume.get( {id:file.contents } ) .then( async (dir)=>{ Object.defineProperty( file, "root", {value:dir} ); return dir; } ); } return this.folder.open( file.name ); } } class FileDirectory { files = []; #id = null; #volume = null; constructor( v, id ) { this.#volume = v; this.#id = id; } get id() { return this.#id; } find( file ) { // simple check; for a more advanced pathname matching use has(file) instead. return !!this.files.find( (f)=>(f.name == file ) ); } async create( fileName ) { var file = this.files.find( (f)=>(f.name == fileName ) ); if( file ) { //console.log( "File already exists, not creating." ); return null; // can't creeate already exists. } else { file = new FileEntry( this ); file.name = fileName; this.files.push(file); this.store(); return file; } } async open( fileName ) { const file = this.files.find( (f)=>(f.name == fileName ) ); const _this = this; //console.log( "OPEN?", file ); if( !file ) { return Promise.reject( new Error( "File not found:" + fileName ) ); } return Promise.resolve( file ); } store(force) { // save changes to this. if( !force ) { if( !pendingStore.find( p=>p===this)) pendingStore.push( this ); if( !lastStore ) { checkPendingStore(); } lastStore = Date.now(); return storedPromise; } return this.#volume.put( this, { id:this.#id } ); } remove ( fileName ) { var parts = splitPath( fileName ); var part; var pathIndex = 0; var dir = this; async function getOnePath() { if( pathIndex >= parts.length ) return false; if( !dir ) return false; part = parts[pathIndex++]; const fileId = dir.files.findIndex( (f)=>( f.name == part ) ); if( fileId >= 0 && pathIndex >= parts.length ) { const file = dir.files[fileId]; dir.#volume.remove( file.id ); dir.files.splice( fileId, 1 ); //if( !dir.files.length ) { // dir.volume.remove( dir.id ); //} return true; } if( file.root ) dir = file.root; else { if( file.contents ) { return dir.#volume.get( {id:file.contents } ) .then( async (readdir)=>{ Object.defineProperty( file, "root", {value:readdir} ); dir = readdir; return getOnePath(); }); } else dir = null; } return getOnePath(); } return getOnePath(); } async has ( fileName ) { var parts = splitPath( fileName ); var part; var pathIndex = 0; var dir = this; async function getOnePath() { console.log( "Checking for file:", fileName, dir ); if( pathIndex >= parts.length ) return true; if( !dir ) return false; part = parts[pathIndex++]; var file = dir.files.find( (f)=>( f.name == part ) ); console.log( "Checking for file:", fileName, file, dir ); if( !file ) return false; if( file.root ) dir = file.root; else { if( file.contents ) { return dir.#volume.get( {id:file.contents } ) .then( async (readdir)=>{ Object.defineProperty( file, "root", {value:readdir} ); dir = readdir; return getOnePath(); }); } else dir = null; } return getOnePath(); } return getOnePath(); } folder( fileName ) { const _this = this; return new Promise( (res,rej)=>{ const path = splitPath( fileName ); const pathIndex = 0; const here = _this; function getOnePath() { const part = path[pathIndex++]; let file = here.files.find( (f)=>(f.name == part ) ); if( file ) { if( file.contents ) { _this.#volume.get( {id:file.contents } ) .then( (dir)=>{ if( pathIndex < path.length ) { here = dir; return getOnePath(); } else res( dir ); } ); } else return Promise.reject( "Folder not found" ); }else { file = new FileEntry( this ); file.name = part; var newDir = new FileDirectory( _this.#volume, file.contents );