UNPKG

global_persistence

Version:

A link between LRU and messaging to peers which may store or cache LRU entries remotely

977 lines (948 loc) 24.6 kB
const Repository = require('repository-bridge') const fs = require('fs') // const DEFAULT_P2P_REPO = 'local' let g_media_types = { "audio" : { "encrypted" : true, "store_local" : true, "store_repo" : true }, "video" : { "encrypted" : false, "store_local" : true, "store_repo" : true }, "image" : { "encrypted" : true, "store_local" : true, "store_repo" : true }, "text" : { "encrypted" : true, "store_local" : true, "store_repo" : false } } let types_to_general_media_types = { "3dm": "x-world", "x-3dmf": "x-world", "3dmf": "x-world", "a": "application", "octet-stream": "application", "aab": "application", "x-authorware-bin": "application", "aam": "application", "x-authorware-map": "application", "aas": "application", "x-authorware-seg": "application", "abc": "text", "vnd.abc": "text", "acgi": "text", "html": "text", "afl": "video", "animaflex": "video", "ai": "application", "postscript": "application", "aif": "audio", "aiff": "audio", "x-aiff": "audio", "aifc": "audio", "aim": "application", "x-aim": "application", "aip": "text", "x-audiosoft-intra": "text", "ani": "application", "x-navi-animation": "application", "aos": "application", "x-nokia-9000-communicator-add-on-software": "application", "aps": "application", "mime": "www", "arc": "application", "arj": "application", "art": "image", "x-jg": "image", "asf": "video", "x-ms-asf": "video", "asm": "text", "x-asm": "text", "asp": "text", "asx": "video", "x-mplayer2": "application", "x-ms-asf-plugin": "video", "au": "audio", "basic": "audio", "x-au": "audio", "avi": "video", "x-troff-msvideo": "application", "msvideo": "video", "x-msvideo": "video", "avs": "video", "avs-video": "video", "bcpio": "application", "x-bcpio": "application", "bin": "application", "mac-binary": "application", "macbinary": "application", "x-binary": "application", "x-macbinary": "application", "bm": "image", "bmp": "image", "x-windows-bmp": "image", "boo": "application", "book": "application", "boz": "application", "x-bzip2": "application", "bsh": "application", "x-bsh": "application", "bz": "application", "x-bzip": "application", "bz2": "application", "c": "text", "plain": "text", "x-c": "text", "c++": "text", "cat": "application", "vnd.ms-pki.seccat": "application", "cc": "text", "ccad": "application", "clariscad": "application", "cco": "application", "x-cocoa": "application", "cdf": "application", "x-cdf": "application", "x-netcdf": "application", "cer": "application", "pkix-cert": "application", "x-x509-ca-cert": "application", "cha": "application", "x-chat": "application", "chat": "application", "class": "application", "java": "text", "java-byte-code": "application", "x-java-class": "application", "com": "text", "conf": "text", "cpio": "application", "x-cpio": "application", "cpp": "text", "cpt": "application", "mac-compactpro": "application", "x-compactpro": "application", "x-cpt": "application", "crl": "application", "pkcs-crl": "application", "pkix-crl": "application", "crt": "application", "x-x509-user-cert": "application", "csh": "text", "x-csh": "application", "x-script.csh": "text", "css": "text", "x-pointplus": "application", "cxx": "text", "dcr": "application", "x-director": "application", "deepv": "application", "x-deepv": "application", "def": "text", "der": "application", "dif": "video", "x-dv": "video", "dir": "application", "dl": "video", "x-dl": "video", "doc": "application", "msword": "application", "dot": "application", "dp": "application", "commonground": "application", "drw": "application", "drafting": "application", "dump": "application", "dv": "video", "dvi": "application", "x-dvi": "application", "dwf": "model", "(old)": "drawing", "vnd.dwf": "model", "dwg": "image", "acad": "application", "vnd.dwg": "image", "x-dwg": "image", "dxf": "image", "dxr": "application", "el": "text", "x-script.elisp": "text", "elc": "application", "(compiled": "application", "x-elc": "application", "env": "application", "x-envoy": "application", "eps": "application", "es": "application", "x-esrehber": "application", "etx": "text", "x-setext": "text", "evy": "application", "envoy": "application", "exe": "application", "f": "text", "x-fortran": "text", "f77": "text", "f90": "text", "fdf": "application", "vnd.fdf": "application", "fif": "image", "fractals": "application", "fli": "video", "x-fli": "video", "flo": "image", "florian": "image", "flx": "text", "vnd.fmi.flexstor": "text", "fmf": "video", "x-atomic3d-feature": "video", "for": "text", "fpx": "image", "vnd.fpx": "image", "vnd.net-fpx": "image", "frl": "application", "freeloader": "application", "funk": "audio", "make": "audio", "g": "text", "g3": "image", "g3fax": "image", "gif": "image", "gl": "video", "x-gl": "video", "gsd": "audio", "x-gsm": "audio", "gsm": "audio", "gsp": "application", "x-gsp": "application", "gss": "application", "x-gss": "application", "gtar": "application", "x-gtar": "application", "gz": "application", "x-compressed": "application", "x-gzip": "multipart", "gzip": "multipart", "h": "text", "x-h": "text", "hdf": "application", "x-hdf": "application", "help": "application", "x-helpfile": "application", "hgl": "application", "vnd.hp-hpgl": "application", "hh": "text", "hlb": "text", "x-script": "text", "hlp": "application", "x-winhelp": "application", "hpg": "application", "hpgl": "application", "hqx": "application", "binhex": "application", "binhex4": "application", "mac-binhex": "application", "mac-binhex40": "application", "x-binhex40": "application", "x-mac-binhex40": "application", "hta": "application", "htc": "text", "x-component": "text", "htm": "text", "htmls": "text", "htt": "text", "webviewhtml": "text", "htx": "text", "ice": "x-conference", "x-cooltalk": "x-conference", "ico": "image", "x-icon": "image", "idc": "text", "ief": "image", "iefs": "image", "iges": "model", "igs": "model", "ima": "application", "x-ima": "application", "imap": "application", "x-httpd-imap": "application", "inf": "application", "ins": "application", "x-internett-signup": "application", "ip": "application", "x-ip2": "application", "isu": "video", "x-isvideo": "video", "it": "audio", "iv": "application", "x-inventor": "application", "ivr": "i-world", "i-vrml": "i-world", "ivy": "application", "x-livescreen": "application", "jam": "audio", "x-jam": "audio", "jav": "text", "x-java-source": "text", "jcm": "application", "x-java-commerce": "application", "jfif": "image", "jpeg": "image", "pjpeg": "image", "jfif-tbnl": "image", "jpe": "image", "jpg": "image", "jps": "image", "x-jps": "image", "js": "text", "x-javascript": "application", "javascript": "text", "ecmascript": "text", "jut": "image", "jutvision": "image", "kar": "music", "midi": "x-music", "x-karaoke": "music", "ksh": "text", "x-ksh": "application", "x-script.ksh": "text", "la": "audio", "nspaudio": "audio", "x-nspaudio": "audio", "lam": "audio", "x-liveaudio": "audio", "latex": "application", "x-latex": "application", "lha": "application", "x-lha": "application", "lhx": "application", "list": "text", "lma": "audio", "log": "text", "lsp": "text", "x-lisp": "application", "x-script.lisp": "text", "lst": "text", "lsx": "text", "x-la-asf": "text", "ltx": "application", "lzh": "application", "x-lzh": "application", "lzx": "application", "x-lzx": "application", "m": "text", "x-m": "text", "m1v": "video", "mpeg": "audio", "m2a": "audio", "m2v": "video", "m3u": "audio", "x-mpequrl": "audio", "man": "application", "x-troff-man": "application", "map": "application", "x-navimap": "application", "mar": "text", "mbd": "application", "mbedlet": "application", "mc$": "application", "x-magic-cap-package-1.0": "application", "mcd": "application", "mcad": "application", "x-mathcad": "application", "mcf": "text", "vasa": "image", "mcp": "application", "netmc": "application", "me": "application", "x-troff-me": "application", "mht": "message", "rfc822": "message", "mhtml": "message", "mid": "audio", "x-midi": "x-music", "x-mid": "audio", "crescendo": "music", "mif": "application", "x-frame": "application", "x-mif": "application", "mjf": "audio", "x-vnd.audioexplosion.mjuicemediafile": "audio", "mjpg": "video", "x-motion-jpeg": "video", "mm": "application", "base64": "application", "x-meme": "application", "mme": "application", "mod": "audio", "x-mod": "audio", "moov": "video", "quicktime": "video", "mov": "video", "movie": "xgl", "x-sgi-movie": "video", "mp2": "video", "x-mpeg": "video", "x-mpeq2a": "video", "mp3": "video", "mpeg3": "audio", "x-mpeg-3": "audio", "mpa": "video", "mpc": "application", "x-project": "application", "mpe": "video", "mpg": "video", "mpga": "audio", "mpp": "application", "vnd.ms-project": "application", "mpt": "application", "mpv": "application", "mpx": "application", "mrc": "application", "marc": "application", "ms": "application", "x-troff-ms": "application", "mv": "video", "my": "audio", "mzz": "application", "x-vnd.audioexplosion.mzz": "application", "nap": "image", "naplps": "image", "nc": "application", "ncm": "application", "vnd.nokia.configuration-message": "application", "nif": "image", "x-niff": "image", "niff": "image", "nix": "application", "x-mix-transfer": "application", "nsc": "application", "x-conference": "application", "nvd": "application", "x-navidoc": "application", "o": "application", "oda": "application", "omc": "application", "x-omc": "application", "omcd": "application", "x-omcdatamaker": "application", "omcr": "application", "x-omcregerator": "application", "p": "text", "x-pascal": "text", "p10": "application", "pkcs10": "application", "x-pkcs10": "application", "p12": "application", "pkcs-12": "application", "x-pkcs12": "application", "p7a": "application", "x-pkcs7-signature": "application", "p7c": "application", "pkcs7-mime": "application", "x-pkcs7-mime": "application", "p7m": "application", "p7r": "application", "x-pkcs7-certreqresp": "application", "p7s": "application", "pkcs7-signature": "application", "part": "application", "pro_eng": "application", "pas": "text", "pascal": "text", "pbm": "image", "x-portable-bitmap": "image", "pcl": "application", "vnd.hp-pcl": "application", "x-pcl": "application", "pct": "image", "x-pict": "image", "pcx": "image", "x-pcx": "image", "pdb": "chemical", "x-pdb": "chemical", "pdf": "application", "pfunk": "audio", "make.my.funk": "audio", "pgm": "image", "x-portable-graymap": "image", "x-portable-greymap": "image", "pic": "image", "pict": "image", "pkg": "application", "x-newton-compatible-pkg": "application", "pko": "application", "vnd.ms-pki.pko": "application", "pl": "text", "x-script.perl": "text", "plx": "application", "x-pixclscript": "application", "pm": "text", "x-xpixmap": "image", "x-script.perl-module": "text", "pm4": "application", "x-pagemaker": "application", "pm5": "application", "png": "image", "pnm": "image", "x-portable-anymap": "image", "pot": "application", "mspowerpoint": "application", "vnd.ms-powerpoint": "application", "pov": "model", "x-pov": "model", "ppa": "application", "ppm": "image", "x-portable-pixmap": "image", "pps": "application", "ppt": "application", "powerpoint": "application", "x-mspowerpoint": "application", "ppz": "application", "pre": "application", "x-freelance": "application", "prt": "application", "ps": "application", "psd": "application", "pvu": "paleovu", "x-pv": "paleovu", "pwz": "application", "py": "text", "x-script.phyton": "text", "pyc": "application", "x-bytecode.python": "application", "qcp": "audio", "vnd.qcelp": "audio", "qd3": "x-world", "qd3d": "x-world", "qif": "image", "x-quicktime": "image", "qt": "video", "qtc": "video", "x-qtc": "video", "qti": "image", "qtif": "image", "ra": "audio", "x-pn-realaudio": "audio", "x-pn-realaudio-plugin": "audio", "x-realaudio": "audio", "ram": "audio", "ras": "image", "x-cmu-raster": "image", "cmu-raster": "image", "rast": "image", "rexx": "text", "x-script.rexx": "text", "rf": "image", "vnd.rn-realflash": "image", "rgb": "image", "x-rgb": "image", "rm": "audio", "vnd.rn-realmedia": "application", "rmi": "audio", "rmm": "audio", "rmp": "audio", "rng": "application", "ringing-tones": "application", "vnd.nokia.ringing-tone": "application", "rnx": "application", "vnd.rn-realplayer": "application", "roff": "application", "x-troff": "application", "rp": "image", "vnd.rn-realpix": "image", "rpm": "audio", "rt": "text", "richtext": "text", "vnd.rn-realtext": "text", "rtf": "application", "x-rtf": "application", "rtx": "text", "rv": "video", "vnd.rn-realvideo": "video", "s": "text", "s3m": "audio", "saveme": "application", "sbk": "application", "x-tbook": "application", "scm": "video", "x-lotusscreencam": "application", "x-script.guile": "text", "x-script.scheme": "text", "x-scm": "video", "sdml": "text", "sdp": "application", "x-sdp": "application", "sdr": "application", "sounder": "application", "sea": "application", "x-sea": "application", "set": "application", "sgm": "text", "sgml": "text", "x-sgml": "text", "sh": "text", "x-sh": "application", "x-shar": "application", "x-script.sh": "text", "shar": "application", "shtml": "text", "x-server-parsed-html": "text", "sid": "audio", "x-psid": "audio", "sit": "application", "x-sit": "application", "x-stuffit": "application", "skd": "application", "x-koan": "application", "skm": "application", "skp": "application", "skt": "application", "sl": "application", "x-seelogo": "application", "smi": "application", "smil": "application", "snd": "audio", "x-adpcm": "audio", "sol": "application", "solids": "application", "spc": "text", "x-pkcs7-certificates": "application", "x-speech": "text", "spl": "application", "futuresplash": "application", "spr": "application", "x-sprite": "application", "sprite": "application", "src": "application", "x-wais-source": "application", "ssi": "text", "ssm": "application", "streamingmedia": "application", "sst": "application", "vnd.ms-pki.certstore": "application", "step": "application", "stl": "application", "sla": "application", "vnd.ms-pki.stl": "application", "x-navistyle": "application", "stp": "application", "sv4cpio": "application", "x-sv4cpio": "application", "sv4crc": "application", "x-sv4crc": "application", "svf": "image", "svr": "x-world", "x-world": "application", "x-svr": "x-world", "swf": "application", "x-shockwave-flash": "application", "t": "application", "talk": "text", "tar": "application", "x-tar": "application", "tbk": "application", "toolbook": "application", "tcl": "text", "x-tcl": "application", "x-script.tcl": "text", "tcsh": "text", "x-script.tcsh": "text", "tex": "application", "x-tex": "application", "texi": "application", "x-texinfo": "application", "texinfo": "application", "text": "text", "tgz": "application", "gnutar": "application", "tif": "image", "tiff": "image", "x-tiff": "image", "tr": "application", "tsi": "audio", "tsp-audio": "audio", "tsp": "audio", "dsptype": "application", "tsplayer": "audio", "tsv": "text", "tab-separated-values": "text", "turbot": "image", "txt": "text", "uil": "text", "x-uil": "text", "uni": "text", "uri-list": "text", "unis": "text", "unv": "application", "i-deas": "application", "uri": "text", "uris": "text", "ustar": "multipart", "x-ustar": "multipart", "uu": "text", "x-uuencode": "text", "uue": "text", "vcd": "application", "x-cdlink": "application", "vcs": "text", "x-vcalendar": "text", "vda": "application", "vdo": "video", "vew": "application", "groupwise": "application", "viv": "video", "vivo": "video", "vnd.vivo": "video", "vmd": "application", "vocaltec-media-desc": "application", "vmf": "application", "vocaltec-media-file": "application", "voc": "audio", "x-voc": "audio", "vos": "video", "vosaic": "video", "vox": "audio", "voxware": "audio", "vqe": "audio", "x-twinvq-plugin": "audio", "vqf": "audio", "x-twinvq": "audio", "vql": "audio", "vrml": "model", "x-vrml": "x-world", "vrt": "x-world", "x-vrt": "x-world", "vsd": "application", "x-visio": "application", "vst": "application", "vsw": "application", "w60": "application", "wordperfect6.0": "application", "w61": "application", "wordperfect6.1": "application", "w6w": "application", "wav": "audio", "x-wav": "audio", "wb1": "application", "x-qpro": "application", "wbmp": "image", "vnd.wap.wbmp": "image", "web": "application", "vnd.xara": "application", "wiz": "application", "wk1": "application", "x-123": "application", "wmf": "windows", "metafile": "windows", "wml": "text", "vnd.wap.wml": "text", "wmlc": "application", "vnd.wap.wmlc": "application", "wmls": "text", "vnd.wap.wmlscript": "text", "wmlsc": "application", "vnd.wap.wmlscriptc": "application", "word": "application", "wp": "application", "wordperfect": "application", "wp5": "application", "wp6": "application", "wpd": "application", "x-wpwin": "application", "wq1": "application", "x-lotus": "application", "wri": "application", "mswrite": "application", "x-wri": "application", "wrl": "x-world", "wrz": "x-world", "wsc": "text", "scriplet": "text", "wsrc": "application", "wtk": "application", "x-wintalk": "application", "xbm": "image", "x-xbitmap": "image", "x-xbm": "image", "xdr": "video", "x-amt-demorun": "video", "xgz": "xgl", "drawing": "xgl", "xif": "image", "vnd.xiff": "image", "xl": "application", "excel": "application", "xla": "application", "x-excel": "application", "x-msexcel": "application", "xlb": "application", "vnd.ms-excel": "application", "xlc": "application", "xld": "application", "xlk": "application", "xll": "application", "xlm": "application", "xls": "application", "xlt": "application", "xlv": "application", "xlw": "application", "xm": "audio", "xml": "text", "xmz": "xgl", "xpix": "application", "x-vnd.ls-xpix": "application", "xpm": "image", "x-png": "image", "xsr": "video", "x-amt-showrun": "video", "xwd": "image", "x-xwd": "image", "x-xwindowdump": "image", "xyz": "chemical", "z": "application", "x-compress": "application", "zip": "multipart", "x-zip-compressed": "application", "x-zip": "multipart", "zoo": "application", "zsh": "text", "x-script.zsh": "text" } // decryption in back directory.... class MediaHandler { // constructor(conf) { // this.media_dir = conf.media_dir this.entries_dir = conf.entries_dir this.default_repo = DEFAULT_P2P_REPO // this.link_manager = false // this.media_types = conf && conf.media_types ? conf.media_types : g_media_types if ( typeof conf.media_encryption_selection === 'object' ) { for ( let mtype in this.media_types ) { let sel_mtype = conf.media_encryption_selection[mtype] if ( sel_mtype && (typeof sel_mtype.encrypted !== "boolean") ) { this.media_types[mtype].encrypted = sel_mtype.encrypted } } } this.repository = false if ( this.media_dir ) { for ( let mt in this.media_types ) { this.media_types[mt].dir = this.media_dir.replace('$media_type',mt) } } // } async init_repository(conf) { if ( conf.default_p2p_repo !== undefined ) { this.default_repo = conf.default_p2p_repo // config `default_p2p_repo` becomes member `default_repo` } if ( ( conf.accepted_repos !== undefined ) && (Array.isArray(conf.accepted_repos) ) ) { this.accepted_repos = conf.accepted_repos if ( this.accepted_repos.indexOf(this.default_repo) < 0 ) { this.accepted_repos.push(this.default_repo) } } else { this.accepted_repos = [this.default_repo] // if the accepted repos are not listed, this becomes a list of one } this.repository = new Repository(conf.repos,this.accepted_repos) if ( this.link_manager ) { this.repository.set_link_mamagers(conf,this.link_manager) } await this.repository.init_repos() } /** * set_link_manager * * @param {object} conf * @param {object} link_m */ set_link_manager(conf,link_m) { this.link_manager = link_m this.repo_link_conf = conf } is_accepted_rep(repo) { let i = this.accepted_repos.indexOf(repo) return ( i >= 0 ) } supported_media_type(media_type) { return (this.media_types[media_type] !== undefined) } nearest_media_type(media_type) { let possible = types_to_general_media_types[media_type] if ( possible === undefined ) return("text") if ( this.supported_media_type(possible) ) { return possible } return "text" } // _media_storage // FOR IMAGE AND STREAM // media_name is for local disk storage only... // What goes to the ipfs like repository is the encode blob of data. The same as the local file... // async _media_storage(repo_kind,media_name,media_type,enc_blob) { // try { let media_dir = this.media_types[media_type].dir let out_file = media_dir + media_name // store to the local drive let store_local = this.media_types[media_type].store_local if ( store_local ) { fs.writeFileSync(out_file,enc_blob) // write binary encrypted data } // store to the local repo let store_repo = this.media_types[media_type].store_repo if ( store_repo && this.repository ) { // add locally to the p2p subsytem... -> this will be pinned in the persistence manager... const repo_id = await this.repository.add(repo_kind,enc_blob) // put the same data in the repo (com and P2P storage) if ( repo_id !== false ) { return { "protocol" : repo_kind, "id" : repo_id } } } return true } catch (e) { return false } // } storable(source) { // turn data into something that can be processed further down the line let blob_data = source.blob_url delete source.blob_url // this is only used to go from the interface to storage // let bdata_parts = blob_data.split(',') let blob_bytes = bdata_parts[1] let blob = Buffer.from(blob_bytes, 'base64'); return blob } // store_media // ---- store the actual data... and edit the fields of the meta data object // ---- this is the api interface for storing across the repository bridge from here. async store_media(blob,media,media_name,media_type) { // if ( media_type in this.media_types ) { let protocol = media.protocol if ( !this.is_accepted_rep(repo) ) { protocol = this.default_repo } let result = await this._media_storage(protocol,media_name,media_type,blob) if ( typeof result === 'boolean' ) { return result } else { media.protocol = result.protocol media[media.protocol] = result.id } return true } return false } } // for external global use (not done as a static of the class...) function nearest_media_type(media_type) { let possible = types_to_general_media_types[media_type] if ( possible === undefined ) return("text") if ( g_media_types[possible] !== undefined ) { return possible } return "text" } module.exports = MediaHandler module.nearest_media_type = nearest_media_type