global_persistence
Version:
A link between LRU and messaging to peers which may store or cache LRU entries remotely
977 lines (948 loc) • 24.6 kB
JavaScript
const Repository = require('repository-bridge')
const fs = require('fs')
//
const DEFAULT_P2P_REPO = 'local'
let g_media_types = {
"audio" : { "encrypted" : true, "store_local" : true, "store_repo" : true },
"video" : { "encrypted" : false, "store_local" : true, "store_repo" : true },
"image" : { "encrypted" : true, "store_local" : true, "store_repo" : true },
"text" : { "encrypted" : true, "store_local" : true, "store_repo" : false }
}
let types_to_general_media_types = {
"3dm": "x-world",
"x-3dmf": "x-world",
"3dmf": "x-world",
"a": "application",
"octet-stream": "application",
"aab": "application",
"x-authorware-bin": "application",
"aam": "application",
"x-authorware-map": "application",
"aas": "application",
"x-authorware-seg": "application",
"abc": "text",
"vnd.abc": "text",
"acgi": "text",
"html": "text",
"afl": "video",
"animaflex": "video",
"ai": "application",
"postscript": "application",
"aif": "audio",
"aiff": "audio",
"x-aiff": "audio",
"aifc": "audio",
"aim": "application",
"x-aim": "application",
"aip": "text",
"x-audiosoft-intra": "text",
"ani": "application",
"x-navi-animation": "application",
"aos": "application",
"x-nokia-9000-communicator-add-on-software": "application",
"aps": "application",
"mime": "www",
"arc": "application",
"arj": "application",
"art": "image",
"x-jg": "image",
"asf": "video",
"x-ms-asf": "video",
"asm": "text",
"x-asm": "text",
"asp": "text",
"asx": "video",
"x-mplayer2": "application",
"x-ms-asf-plugin": "video",
"au": "audio",
"basic": "audio",
"x-au": "audio",
"avi": "video",
"x-troff-msvideo": "application",
"msvideo": "video",
"x-msvideo": "video",
"avs": "video",
"avs-video": "video",
"bcpio": "application",
"x-bcpio": "application",
"bin": "application",
"mac-binary": "application",
"macbinary": "application",
"x-binary": "application",
"x-macbinary": "application",
"bm": "image",
"bmp": "image",
"x-windows-bmp": "image",
"boo": "application",
"book": "application",
"boz": "application",
"x-bzip2": "application",
"bsh": "application",
"x-bsh": "application",
"bz": "application",
"x-bzip": "application",
"bz2": "application",
"c": "text",
"plain": "text",
"x-c": "text",
"c++": "text",
"cat": "application",
"vnd.ms-pki.seccat": "application",
"cc": "text",
"ccad": "application",
"clariscad": "application",
"cco": "application",
"x-cocoa": "application",
"cdf": "application",
"x-cdf": "application",
"x-netcdf": "application",
"cer": "application",
"pkix-cert": "application",
"x-x509-ca-cert": "application",
"cha": "application",
"x-chat": "application",
"chat": "application",
"class": "application",
"java": "text",
"java-byte-code": "application",
"x-java-class": "application",
"com": "text",
"conf": "text",
"cpio": "application",
"x-cpio": "application",
"cpp": "text",
"cpt": "application",
"mac-compactpro": "application",
"x-compactpro": "application",
"x-cpt": "application",
"crl": "application",
"pkcs-crl": "application",
"pkix-crl": "application",
"crt": "application",
"x-x509-user-cert": "application",
"csh": "text",
"x-csh": "application",
"x-script.csh": "text",
"css": "text",
"x-pointplus": "application",
"cxx": "text",
"dcr": "application",
"x-director": "application",
"deepv": "application",
"x-deepv": "application",
"def": "text",
"der": "application",
"dif": "video",
"x-dv": "video",
"dir": "application",
"dl": "video",
"x-dl": "video",
"doc": "application",
"msword": "application",
"dot": "application",
"dp": "application",
"commonground": "application",
"drw": "application",
"drafting": "application",
"dump": "application",
"dv": "video",
"dvi": "application",
"x-dvi": "application",
"dwf": "model",
"(old)": "drawing",
"vnd.dwf": "model",
"dwg": "image",
"acad": "application",
"vnd.dwg": "image",
"x-dwg": "image",
"dxf": "image",
"dxr": "application",
"el": "text",
"x-script.elisp": "text",
"elc": "application",
"(compiled": "application",
"x-elc": "application",
"env": "application",
"x-envoy": "application",
"eps": "application",
"es": "application",
"x-esrehber": "application",
"etx": "text",
"x-setext": "text",
"evy": "application",
"envoy": "application",
"exe": "application",
"f": "text",
"x-fortran": "text",
"f77": "text",
"f90": "text",
"fdf": "application",
"vnd.fdf": "application",
"fif": "image",
"fractals": "application",
"fli": "video",
"x-fli": "video",
"flo": "image",
"florian": "image",
"flx": "text",
"vnd.fmi.flexstor": "text",
"fmf": "video",
"x-atomic3d-feature": "video",
"for": "text",
"fpx": "image",
"vnd.fpx": "image",
"vnd.net-fpx": "image",
"frl": "application",
"freeloader": "application",
"funk": "audio",
"make": "audio",
"g": "text",
"g3": "image",
"g3fax": "image",
"gif": "image",
"gl": "video",
"x-gl": "video",
"gsd": "audio",
"x-gsm": "audio",
"gsm": "audio",
"gsp": "application",
"x-gsp": "application",
"gss": "application",
"x-gss": "application",
"gtar": "application",
"x-gtar": "application",
"gz": "application",
"x-compressed": "application",
"x-gzip": "multipart",
"gzip": "multipart",
"h": "text",
"x-h": "text",
"hdf": "application",
"x-hdf": "application",
"help": "application",
"x-helpfile": "application",
"hgl": "application",
"vnd.hp-hpgl": "application",
"hh": "text",
"hlb": "text",
"x-script": "text",
"hlp": "application",
"x-winhelp": "application",
"hpg": "application",
"hpgl": "application",
"hqx": "application",
"binhex": "application",
"binhex4": "application",
"mac-binhex": "application",
"mac-binhex40": "application",
"x-binhex40": "application",
"x-mac-binhex40": "application",
"hta": "application",
"htc": "text",
"x-component": "text",
"htm": "text",
"htmls": "text",
"htt": "text",
"webviewhtml": "text",
"htx": "text",
"ice": "x-conference",
"x-cooltalk": "x-conference",
"ico": "image",
"x-icon": "image",
"idc": "text",
"ief": "image",
"iefs": "image",
"iges": "model",
"igs": "model",
"ima": "application",
"x-ima": "application",
"imap": "application",
"x-httpd-imap": "application",
"inf": "application",
"ins": "application",
"x-internett-signup": "application",
"ip": "application",
"x-ip2": "application",
"isu": "video",
"x-isvideo": "video",
"it": "audio",
"iv": "application",
"x-inventor": "application",
"ivr": "i-world",
"i-vrml": "i-world",
"ivy": "application",
"x-livescreen": "application",
"jam": "audio",
"x-jam": "audio",
"jav": "text",
"x-java-source": "text",
"jcm": "application",
"x-java-commerce": "application",
"jfif": "image",
"jpeg": "image",
"pjpeg": "image",
"jfif-tbnl": "image",
"jpe": "image",
"jpg": "image",
"jps": "image",
"x-jps": "image",
"js": "text",
"x-javascript": "application",
"javascript": "text",
"ecmascript": "text",
"jut": "image",
"jutvision": "image",
"kar": "music",
"midi": "x-music",
"x-karaoke": "music",
"ksh": "text",
"x-ksh": "application",
"x-script.ksh": "text",
"la": "audio",
"nspaudio": "audio",
"x-nspaudio": "audio",
"lam": "audio",
"x-liveaudio": "audio",
"latex": "application",
"x-latex": "application",
"lha": "application",
"x-lha": "application",
"lhx": "application",
"list": "text",
"lma": "audio",
"log": "text",
"lsp": "text",
"x-lisp": "application",
"x-script.lisp": "text",
"lst": "text",
"lsx": "text",
"x-la-asf": "text",
"ltx": "application",
"lzh": "application",
"x-lzh": "application",
"lzx": "application",
"x-lzx": "application",
"m": "text",
"x-m": "text",
"m1v": "video",
"mpeg": "audio",
"m2a": "audio",
"m2v": "video",
"m3u": "audio",
"x-mpequrl": "audio",
"man": "application",
"x-troff-man": "application",
"map": "application",
"x-navimap": "application",
"mar": "text",
"mbd": "application",
"mbedlet": "application",
"mc$": "application",
"x-magic-cap-package-1.0": "application",
"mcd": "application",
"mcad": "application",
"x-mathcad": "application",
"mcf": "text",
"vasa": "image",
"mcp": "application",
"netmc": "application",
"me": "application",
"x-troff-me": "application",
"mht": "message",
"rfc822": "message",
"mhtml": "message",
"mid": "audio",
"x-midi": "x-music",
"x-mid": "audio",
"crescendo": "music",
"mif": "application",
"x-frame": "application",
"x-mif": "application",
"mjf": "audio",
"x-vnd.audioexplosion.mjuicemediafile": "audio",
"mjpg": "video",
"x-motion-jpeg": "video",
"mm": "application",
"base64": "application",
"x-meme": "application",
"mme": "application",
"mod": "audio",
"x-mod": "audio",
"moov": "video",
"quicktime": "video",
"mov": "video",
"movie": "xgl",
"x-sgi-movie": "video",
"mp2": "video",
"x-mpeg": "video",
"x-mpeq2a": "video",
"mp3": "video",
"mpeg3": "audio",
"x-mpeg-3": "audio",
"mpa": "video",
"mpc": "application",
"x-project": "application",
"mpe": "video",
"mpg": "video",
"mpga": "audio",
"mpp": "application",
"vnd.ms-project": "application",
"mpt": "application",
"mpv": "application",
"mpx": "application",
"mrc": "application",
"marc": "application",
"ms": "application",
"x-troff-ms": "application",
"mv": "video",
"my": "audio",
"mzz": "application",
"x-vnd.audioexplosion.mzz": "application",
"nap": "image",
"naplps": "image",
"nc": "application",
"ncm": "application",
"vnd.nokia.configuration-message": "application",
"nif": "image",
"x-niff": "image",
"niff": "image",
"nix": "application",
"x-mix-transfer": "application",
"nsc": "application",
"x-conference": "application",
"nvd": "application",
"x-navidoc": "application",
"o": "application",
"oda": "application",
"omc": "application",
"x-omc": "application",
"omcd": "application",
"x-omcdatamaker": "application",
"omcr": "application",
"x-omcregerator": "application",
"p": "text",
"x-pascal": "text",
"p10": "application",
"pkcs10": "application",
"x-pkcs10": "application",
"p12": "application",
"pkcs-12": "application",
"x-pkcs12": "application",
"p7a": "application",
"x-pkcs7-signature": "application",
"p7c": "application",
"pkcs7-mime": "application",
"x-pkcs7-mime": "application",
"p7m": "application",
"p7r": "application",
"x-pkcs7-certreqresp": "application",
"p7s": "application",
"pkcs7-signature": "application",
"part": "application",
"pro_eng": "application",
"pas": "text",
"pascal": "text",
"pbm": "image",
"x-portable-bitmap": "image",
"pcl": "application",
"vnd.hp-pcl": "application",
"x-pcl": "application",
"pct": "image",
"x-pict": "image",
"pcx": "image",
"x-pcx": "image",
"pdb": "chemical",
"x-pdb": "chemical",
"pdf": "application",
"pfunk": "audio",
"make.my.funk": "audio",
"pgm": "image",
"x-portable-graymap": "image",
"x-portable-greymap": "image",
"pic": "image",
"pict": "image",
"pkg": "application",
"x-newton-compatible-pkg": "application",
"pko": "application",
"vnd.ms-pki.pko": "application",
"pl": "text",
"x-script.perl": "text",
"plx": "application",
"x-pixclscript": "application",
"pm": "text",
"x-xpixmap": "image",
"x-script.perl-module": "text",
"pm4": "application",
"x-pagemaker": "application",
"pm5": "application",
"png": "image",
"pnm": "image",
"x-portable-anymap": "image",
"pot": "application",
"mspowerpoint": "application",
"vnd.ms-powerpoint": "application",
"pov": "model",
"x-pov": "model",
"ppa": "application",
"ppm": "image",
"x-portable-pixmap": "image",
"pps": "application",
"ppt": "application",
"powerpoint": "application",
"x-mspowerpoint": "application",
"ppz": "application",
"pre": "application",
"x-freelance": "application",
"prt": "application",
"ps": "application",
"psd": "application",
"pvu": "paleovu",
"x-pv": "paleovu",
"pwz": "application",
"py": "text",
"x-script.phyton": "text",
"pyc": "application",
"x-bytecode.python": "application",
"qcp": "audio",
"vnd.qcelp": "audio",
"qd3": "x-world",
"qd3d": "x-world",
"qif": "image",
"x-quicktime": "image",
"qt": "video",
"qtc": "video",
"x-qtc": "video",
"qti": "image",
"qtif": "image",
"ra": "audio",
"x-pn-realaudio": "audio",
"x-pn-realaudio-plugin": "audio",
"x-realaudio": "audio",
"ram": "audio",
"ras": "image",
"x-cmu-raster": "image",
"cmu-raster": "image",
"rast": "image",
"rexx": "text",
"x-script.rexx": "text",
"rf": "image",
"vnd.rn-realflash": "image",
"rgb": "image",
"x-rgb": "image",
"rm": "audio",
"vnd.rn-realmedia": "application",
"rmi": "audio",
"rmm": "audio",
"rmp": "audio",
"rng": "application",
"ringing-tones": "application",
"vnd.nokia.ringing-tone": "application",
"rnx": "application",
"vnd.rn-realplayer": "application",
"roff": "application",
"x-troff": "application",
"rp": "image",
"vnd.rn-realpix": "image",
"rpm": "audio",
"rt": "text",
"richtext": "text",
"vnd.rn-realtext": "text",
"rtf": "application",
"x-rtf": "application",
"rtx": "text",
"rv": "video",
"vnd.rn-realvideo": "video",
"s": "text",
"s3m": "audio",
"saveme": "application",
"sbk": "application",
"x-tbook": "application",
"scm": "video",
"x-lotusscreencam": "application",
"x-script.guile": "text",
"x-script.scheme": "text",
"x-scm": "video",
"sdml": "text",
"sdp": "application",
"x-sdp": "application",
"sdr": "application",
"sounder": "application",
"sea": "application",
"x-sea": "application",
"set": "application",
"sgm": "text",
"sgml": "text",
"x-sgml": "text",
"sh": "text",
"x-sh": "application",
"x-shar": "application",
"x-script.sh": "text",
"shar": "application",
"shtml": "text",
"x-server-parsed-html": "text",
"sid": "audio",
"x-psid": "audio",
"sit": "application",
"x-sit": "application",
"x-stuffit": "application",
"skd": "application",
"x-koan": "application",
"skm": "application",
"skp": "application",
"skt": "application",
"sl": "application",
"x-seelogo": "application",
"smi": "application",
"smil": "application",
"snd": "audio",
"x-adpcm": "audio",
"sol": "application",
"solids": "application",
"spc": "text",
"x-pkcs7-certificates": "application",
"x-speech": "text",
"spl": "application",
"futuresplash": "application",
"spr": "application",
"x-sprite": "application",
"sprite": "application",
"src": "application",
"x-wais-source": "application",
"ssi": "text",
"ssm": "application",
"streamingmedia": "application",
"sst": "application",
"vnd.ms-pki.certstore": "application",
"step": "application",
"stl": "application",
"sla": "application",
"vnd.ms-pki.stl": "application",
"x-navistyle": "application",
"stp": "application",
"sv4cpio": "application",
"x-sv4cpio": "application",
"sv4crc": "application",
"x-sv4crc": "application",
"svf": "image",
"svr": "x-world",
"x-world": "application",
"x-svr": "x-world",
"swf": "application",
"x-shockwave-flash": "application",
"t": "application",
"talk": "text",
"tar": "application",
"x-tar": "application",
"tbk": "application",
"toolbook": "application",
"tcl": "text",
"x-tcl": "application",
"x-script.tcl": "text",
"tcsh": "text",
"x-script.tcsh": "text",
"tex": "application",
"x-tex": "application",
"texi": "application",
"x-texinfo": "application",
"texinfo": "application",
"text": "text",
"tgz": "application",
"gnutar": "application",
"tif": "image",
"tiff": "image",
"x-tiff": "image",
"tr": "application",
"tsi": "audio",
"tsp-audio": "audio",
"tsp": "audio",
"dsptype": "application",
"tsplayer": "audio",
"tsv": "text",
"tab-separated-values": "text",
"turbot": "image",
"txt": "text",
"uil": "text",
"x-uil": "text",
"uni": "text",
"uri-list": "text",
"unis": "text",
"unv": "application",
"i-deas": "application",
"uri": "text",
"uris": "text",
"ustar": "multipart",
"x-ustar": "multipart",
"uu": "text",
"x-uuencode": "text",
"uue": "text",
"vcd": "application",
"x-cdlink": "application",
"vcs": "text",
"x-vcalendar": "text",
"vda": "application",
"vdo": "video",
"vew": "application",
"groupwise": "application",
"viv": "video",
"vivo": "video",
"vnd.vivo": "video",
"vmd": "application",
"vocaltec-media-desc": "application",
"vmf": "application",
"vocaltec-media-file": "application",
"voc": "audio",
"x-voc": "audio",
"vos": "video",
"vosaic": "video",
"vox": "audio",
"voxware": "audio",
"vqe": "audio",
"x-twinvq-plugin": "audio",
"vqf": "audio",
"x-twinvq": "audio",
"vql": "audio",
"vrml": "model",
"x-vrml": "x-world",
"vrt": "x-world",
"x-vrt": "x-world",
"vsd": "application",
"x-visio": "application",
"vst": "application",
"vsw": "application",
"w60": "application",
"wordperfect6.0": "application",
"w61": "application",
"wordperfect6.1": "application",
"w6w": "application",
"wav": "audio",
"x-wav": "audio",
"wb1": "application",
"x-qpro": "application",
"wbmp": "image",
"vnd.wap.wbmp": "image",
"web": "application",
"vnd.xara": "application",
"wiz": "application",
"wk1": "application",
"x-123": "application",
"wmf": "windows",
"metafile": "windows",
"wml": "text",
"vnd.wap.wml": "text",
"wmlc": "application",
"vnd.wap.wmlc": "application",
"wmls": "text",
"vnd.wap.wmlscript": "text",
"wmlsc": "application",
"vnd.wap.wmlscriptc": "application",
"word": "application",
"wp": "application",
"wordperfect": "application",
"wp5": "application",
"wp6": "application",
"wpd": "application",
"x-wpwin": "application",
"wq1": "application",
"x-lotus": "application",
"wri": "application",
"mswrite": "application",
"x-wri": "application",
"wrl": "x-world",
"wrz": "x-world",
"wsc": "text",
"scriplet": "text",
"wsrc": "application",
"wtk": "application",
"x-wintalk": "application",
"xbm": "image",
"x-xbitmap": "image",
"x-xbm": "image",
"xdr": "video",
"x-amt-demorun": "video",
"xgz": "xgl",
"drawing": "xgl",
"xif": "image",
"vnd.xiff": "image",
"xl": "application",
"excel": "application",
"xla": "application",
"x-excel": "application",
"x-msexcel": "application",
"xlb": "application",
"vnd.ms-excel": "application",
"xlc": "application",
"xld": "application",
"xlk": "application",
"xll": "application",
"xlm": "application",
"xls": "application",
"xlt": "application",
"xlv": "application",
"xlw": "application",
"xm": "audio",
"xml": "text",
"xmz": "xgl",
"xpix": "application",
"x-vnd.ls-xpix": "application",
"xpm": "image",
"x-png": "image",
"xsr": "video",
"x-amt-showrun": "video",
"xwd": "image",
"x-xwd": "image",
"x-xwindowdump": "image",
"xyz": "chemical",
"z": "application",
"x-compress": "application",
"zip": "multipart",
"x-zip-compressed": "application",
"x-zip": "multipart",
"zoo": "application",
"zsh": "text",
"x-script.zsh": "text"
}
// decryption in back directory....
class MediaHandler {
//
constructor(conf) {
//
this.media_dir = conf.media_dir
this.entries_dir = conf.entries_dir
this.default_repo = DEFAULT_P2P_REPO
//
this.link_manager = false
//
this.media_types = conf && conf.media_types ? conf.media_types : g_media_types
if ( typeof conf.media_encryption_selection === 'object' ) {
for ( let mtype in this.media_types ) {
let sel_mtype = conf.media_encryption_selection[mtype]
if ( sel_mtype && (typeof sel_mtype.encrypted !== "boolean") ) {
this.media_types[mtype].encrypted = sel_mtype.encrypted
}
}
}
this.repository = false
if ( this.media_dir ) {
for ( let mt in this.media_types ) {
this.media_types[mt].dir = this.media_dir.replace('$media_type',mt)
}
} //
}
async init_repository(conf) {
if ( conf.default_p2p_repo !== undefined ) {
this.default_repo = conf.default_p2p_repo // config `default_p2p_repo` becomes member `default_repo`
}
if ( ( conf.accepted_repos !== undefined ) && (Array.isArray(conf.accepted_repos) ) ) {
this.accepted_repos = conf.accepted_repos
if ( this.accepted_repos.indexOf(this.default_repo) < 0 ) {
this.accepted_repos.push(this.default_repo)
}
} else {
this.accepted_repos = [this.default_repo] // if the accepted repos are not listed, this becomes a list of one
}
this.repository = new Repository(conf.repos,this.accepted_repos)
if ( this.link_manager ) {
this.repository.set_link_mamagers(conf,this.link_manager)
}
await this.repository.init_repos()
}
/**
* set_link_manager
*
* @param {object} conf
* @param {object} link_m
*/
set_link_manager(conf,link_m) {
this.link_manager = link_m
this.repo_link_conf = conf
}
is_accepted_rep(repo) {
let i = this.accepted_repos.indexOf(repo)
return ( i >= 0 )
}
supported_media_type(media_type) {
return (this.media_types[media_type] !== undefined)
}
nearest_media_type(media_type) {
let possible = types_to_general_media_types[media_type]
if ( possible === undefined ) return("text")
if ( this.supported_media_type(possible) ) {
return possible
}
return "text"
}
// _media_storage
// FOR IMAGE AND STREAM
// media_name is for local disk storage only...
// What goes to the ipfs like repository is the encode blob of data. The same as the local file...
//
async _media_storage(repo_kind,media_name,media_type,enc_blob) {
//
try {
let media_dir = this.media_types[media_type].dir
let out_file = media_dir + media_name
// store to the local drive
let store_local = this.media_types[media_type].store_local
if ( store_local ) {
fs.writeFileSync(out_file,enc_blob) // write binary encrypted data
}
// store to the local repo
let store_repo = this.media_types[media_type].store_repo
if ( store_repo && this.repository ) {
// add locally to the p2p subsytem... -> this will be pinned in the persistence manager...
const repo_id = await this.repository.add(repo_kind,enc_blob) // put the same data in the repo (com and P2P storage)
if ( repo_id !== false ) {
return {
"protocol" : repo_kind,
"id" : repo_id
}
}
}
return true
} catch (e) {
return false
}
//
}
storable(source) { // turn data into something that can be processed further down the line
let blob_data = source.blob_url
delete source.blob_url // this is only used to go from the interface to storage
//
let bdata_parts = blob_data.split(',')
let blob_bytes = bdata_parts[1]
let blob = Buffer.from(blob_bytes, 'base64');
return blob
}
// store_media
// ---- store the actual data... and edit the fields of the meta data object
// ---- this is the api interface for storing across the repository bridge from here.
async store_media(blob,media,media_name,media_type) {
//
if ( media_type in this.media_types ) {
let protocol = media.protocol
if ( !this.is_accepted_rep(repo) ) {
protocol = this.default_repo
}
let result = await this._media_storage(protocol,media_name,media_type,blob)
if ( typeof result === 'boolean' ) {
return result
} else {
media.protocol = result.protocol
media[media.protocol] = result.id
}
return true
}
return false
}
}
// for external global use (not done as a static of the class...)
function nearest_media_type(media_type) {
let possible = types_to_general_media_types[media_type]
if ( possible === undefined ) return("text")
if ( g_media_types[possible] !== undefined ) {
return possible
}
return "text"
}
module.exports = MediaHandler
module.nearest_media_type = nearest_media_type