UNPKG

@x5e/gink

Version:

an eventually consistent database

715 lines 26.9 kB
"use strict"; /** * Herein lay a bunch of utility functions, mostly for creating and * manipulating the types defined in typedefs.ts. */ Object.defineProperty(exports, "__esModule", { value: true }); exports.verifyBundle = exports.signBundle = exports.mergeBytes = exports.getType = exports.isAlive = exports.getIdentity = exports.getActorId = exports.rehydrate = exports.dehydrate = exports.entryToEdgeData = exports.sameData = exports.logToStdErr = exports.isPathDangerous = exports.muidTupleToMuid = exports.muidToTuple = exports.valueToJson = exports.timestampToString = exports.hexToBytes = exports.parseByte = exports.bytesToHex = exports.oneByteToHex = exports.intToHex = exports.strToMuid = exports.strToMuidTuple = exports.muidTupleToString = exports.muidToString = exports.pairKeyToArray = exports.matches = exports.isDate = exports.wrapValue = exports.encodeToken = exports.decodeToken = exports.unwrapValue = exports.unwrapKey = exports.wrapKey = exports.builderToMuid = exports.muidToBuilder = exports.makeMedallion = exports.fromStorageKey = exports.generateTimestamp = exports.ensure = exports.toLastWithPrefixBeforeSuffix = exports.noOp = exports.signingBundles = exports.librariesReady = exports.digest = exports.shorterHash = exports.safeMask = exports.getShortHashKey = exports.emptyBytes = void 0; exports.decryptMessage = exports.encryptMessage = exports.getSig = exports.createKeyPair = void 0; const builders_1 = require("./builders"); const os_1 = require("os"); const libsodium_wrappers_1 = require("libsodium-wrappers"); exports.emptyBytes = new Uint8Array(0); let shorthashKey = exports.emptyBytes; function getShortHashKey() { if (shorthashKey.length === 0) shorthashKey = new Uint8Array(Array(libsodium_wrappers_1.crypto_shorthash_KEYBYTES).fill(0x5e)); return shorthashKey; } exports.getShortHashKey = getShortHashKey; exports.safeMask = BigInt(2 ** 52 - 1); function shorterHash(data) { // I'm using this truncated shorthash because the Google protobuf library can't handle bignums. const out1 = (0, libsodium_wrappers_1.crypto_shorthash)(data, getShortHashKey()); const asBigNum = new DataView(out1.buffer).getBigUint64(0, true); return Number(asBigNum & exports.safeMask); } exports.shorterHash = shorterHash; const digest = (data) => (0, libsodium_wrappers_1.crypto_generichash)(libsodium_wrappers_1.crypto_generichash_BYTES, data); exports.digest = digest; exports.librariesReady = libsodium_wrappers_1.ready; exports.signingBundles = true; function noOp(_) { ensure(arguments.length > 0); } exports.noOp = noOp; function toLastWithPrefixBeforeSuffix(map, prefix, suffix = "~") { const iterator = map.upperBound(prefix + suffix); iterator.prev(); if (!iterator.key) return undefined; if (!iterator.key.startsWith(prefix)) return undefined; return iterator; } exports.toLastWithPrefixBeforeSuffix = toLastWithPrefixBeforeSuffix; // Since find-process uses child-process, we can't load this if gink // is running in a browser // TODO: only install this package when you will be using as a backend? const findProcess = typeof window === "undefined" ? eval("require('find-process')") : undefined; function ensure(x, msg) { if (!x) throw new Error(msg !== null && msg !== void 0 ? msg : "assert failed"); return x; } exports.ensure = ensure; let lastTime = 0; function generateTimestamp() { // TODO: there's probably a better way ... let current = Date.now() * 1000; if (lastTime >= current) { current = lastTime + 20; } lastTime = current; return current; } exports.generateTimestamp = generateTimestamp; /** * Converts a storage key (which is the key used in EntryBuilders) to a * key usable by addEntry, etc. * @param storageKey * @returns */ function fromStorageKey(storageKey) { let newKey; if (Array.isArray(storageKey)) { if (storageKey.length === 3) { newKey = muidTupleToMuid(storageKey); } else if (storageKey.length === 2) { newKey = [ muidTupleToMuid(storageKey[0]), muidTupleToMuid(storageKey[1]), ]; } else { throw new Error("Invalid key length?"); } } else { newKey = storageKey; } return newKey; } exports.fromStorageKey = fromStorageKey; /** * Randomly selects a number that can be used as a medallion. * Note that this doesn't actually have to be cryptographically secure; * as long as it's unique within an organization there won't be problems. * This is unlikely to cause collisions as long as an organization * has fewer than a million instances, after that some tracking is warranted. * https://en.wikipedia.org/wiki/Birthday_problem#Probability_table * @returns Random number between 2**48 and 2**49 (exclusive) */ function makeMedallion() { const crypto = globalThis["crypto"]; if (crypto) { const getRandomValues = crypto["getRandomValues"]; // defined in browsers if (getRandomValues) { const array = new Uint16Array(3); globalThis.crypto.getRandomValues(array); return 2 ** 48 + array[0] * 2 ** 32 + array[1] * 2 ** 16 + array[2]; } const randomInt = crypto["randomInt"]; // defined in some versions of node if (randomInt) { return randomInt(2 ** 48 + 1, 2 ** 49 - 1); } } return Math.floor(Math.random() * 2 ** 48) + 1 + 2 ** 48; } exports.makeMedallion = makeMedallion; function muidToBuilder(address, relativeTo) { const muid = new builders_1.MuidBuilder(); if (address.medallion && address.medallion !== relativeTo) muid.setMedallion(address.medallion); if (address.timestamp) // not set if also pending muid.setTimestamp(address.timestamp); muid.setOffset(address.offset); return muid; } exports.muidToBuilder = muidToBuilder; function builderToMuid(muidBuilder, relativeTo) { // If a MuidBuilder in a message has a zero medallion and/or timestamp, it should be // interpreted that those values are the same as the trxn it comes from. return { timestamp: muidBuilder.getTimestamp() || relativeTo.timestamp, medallion: muidBuilder.getMedallion() || relativeTo.medallion, offset: ensure(muidBuilder.getOffset(), "zero offset"), }; } exports.builderToMuid = builderToMuid; /** * Converts from a KeyType (number or string) to a Gink Proto * @param key * @returns */ function wrapKey(key) { const keyBuilder = new builders_1.KeyBuilder(); if (typeof key === "string") { keyBuilder.setCharacters(key); return keyBuilder; } if (typeof key === "number") { ensure(Number.isSafeInteger(key), `key=${key} not a safe integer`); keyBuilder.setNumber(key); return keyBuilder; } if (key instanceof Uint8Array) { keyBuilder.setOctets(key); return keyBuilder; } throw new Error(`key not a number or string or bytes: ${key}`); } exports.wrapKey = wrapKey; /** * Convert from a Gink Proto known to contain a string or number * into the equiv Javascript object. * @param keyBuilder * @returns */ function unwrapKey(keyBuilder) { ensure(keyBuilder); if (keyBuilder.hasCharacters()) { return keyBuilder.getCharacters(); } if (keyBuilder.hasNumber()) { return keyBuilder.getNumber(); } if (keyBuilder.hasOctets()) { return new Uint8Array(keyBuilder.getOctets_asU8()); } throw new Error("value isn't a number or string!"); } exports.unwrapKey = unwrapKey; /** * Convert from a Gink Proto (Builder) for a Value to the corresponding JS object. * @param valueBuilder Gink Proto for Value * @returns */ function unwrapValue(valueBuilder) { ensure(valueBuilder instanceof builders_1.ValueBuilder); if (valueBuilder.hasCharacters()) { return valueBuilder.getCharacters(); } if (valueBuilder.hasFloating()) { return valueBuilder.getFloating(); } if (valueBuilder.hasInteger()) { return BigInt(valueBuilder.getInteger()); } if (valueBuilder.hasSpecial()) { const special = valueBuilder.getSpecial(); if (special === builders_1.Special.NULL) return null; if (special === builders_1.Special.TRUE) return true; if (special === builders_1.Special.FALSE) return false; throw new Error("bad special"); } if (valueBuilder.hasOctets()) { return new Uint8Array(valueBuilder.getOctets_asU8()); } if (valueBuilder.hasDocument()) { const document = valueBuilder.getDocument(); const keys = document.getKeysList(); const values = document.getValuesList(); const result = new Map(); for (let i = 0; i < keys.length; i++) { result.set(unwrapKey(keys[i]), unwrapValue(values[i])); } return result; } if (valueBuilder.hasTuple()) { const tuple = valueBuilder.getTuple(); return tuple.getValuesList().map(unwrapValue); } if (valueBuilder.hasTimestamp()) { const epochMicros = valueBuilder.getTimestamp(); const epochMillis = epochMicros / 1000; const date = new Date(epochMillis); return date; } throw new Error("haven't implemented unwrap for this Value"); } exports.unwrapValue = unwrapValue; /** * Converts a hex string (presumably encoded previously) to * an authentication token, prefixed with 'token ' * @param {string} hex hexadecimal string to convert * @returns a string 'token {token}' */ function decodeToken(hex) { ensure(hex.substring(0, 2) === "0x", "Hex string should start with 0x"); let token = ""; for (let i = 0; i < hex.length; i += 2) { let hexValue = hex.substring(i, i + 2); token += String.fromCharCode(parseInt(hexValue, 16)); } ensure(token.includes("token "), `Token '${token}' does not begin with 'token '`); return token; } exports.decodeToken = decodeToken; /** * Encodes an authentication token as hexadecimal, prefixed by '0x'. * @param {string} token the token to encode * @returns an encoded hexadecimal string */ function encodeToken(token) { let result = "0x"; if (!token.includes("token ")) { token = "token " + token; } for (let i = 0; i < token.length; i++) { let hex = token.charCodeAt(i).toString(16); result += hex.padStart(2, "0"); } return result; } exports.encodeToken = encodeToken; /** * Converts from any javascript value Gink can store into the corresponding proto builder. * @param arg Any Javascript value Gink can store * @returns */ function wrapValue(arg) { ensure(arg !== undefined); const valueBuilder = new builders_1.ValueBuilder(); if (arg instanceof Uint8Array) { return valueBuilder.setOctets(arg); } if (arg instanceof Date) { return valueBuilder.setTimestamp(arg.getTime() * 1000); } if (arg === null) { return valueBuilder.setSpecial(builders_1.Special.NULL); } if (arg === true) { return valueBuilder.setSpecial(builders_1.Special.TRUE); } if (arg === false) { return valueBuilder.setSpecial(builders_1.Special.FALSE); } if (typeof arg === "string") { return valueBuilder.setCharacters(arg); } if (typeof arg === "number") { return valueBuilder.setFloating(arg); } if (typeof arg === "bigint") { return valueBuilder.setInteger(arg.toString()); } if (Array.isArray(arg)) { const tupleBuilder = new builders_1.TupleBuilder(); tupleBuilder.setValuesList(arg.map(wrapValue)); return valueBuilder.setTuple(tupleBuilder); } if (arg instanceof Map) { const documentBuilder = new builders_1.DocumentBuilder(); for (const [key, val] of arg.entries()) { if (typeof key !== "number" && typeof key !== "string") { throw new Error("keys in documents must be numbers or strings"); } documentBuilder.addKeys(wrapKey(key)); documentBuilder.addValues(wrapValue(val)); } return valueBuilder.setDocument(documentBuilder); } if (typeof arg === "object") { if (Object.getPrototypeOf(arg) !== Object.prototype) { throw new Error(`Don't know how to serialize: ${arg}`); } const documentBuilder = new builders_1.DocumentBuilder(); for (const [key, val] of Object.entries(arg)) { documentBuilder.addKeys(wrapKey(key)); documentBuilder.addValues(wrapValue(val)); } return valueBuilder.setDocument(documentBuilder); } throw new Error(`don't know how to wrap: ${arg}`); } exports.wrapValue = wrapValue; function isDate(value) { return (typeof value === "object" && Object.prototype.toString.call(value) === "[object Date]"); } exports.isDate = isDate; function matches(a, b) { if (a.length !== b.length) return false; for (let i = 0; i < a.length; i++) { if (a[i] !== b[i]) return false; } return true; } exports.matches = matches; function pairKeyToArray(storageKey) { const split = storageKey.split(","); ensure(split.length === 2); return [strToMuid(split[0]), strToMuid(split[1])]; } exports.pairKeyToArray = pairKeyToArray; /** * Converts a Muid object to its canonical string representation * Refer to docs/muid.md * @param muid * @returns a string of the canonical string representation */ function muidToString(muid) { let timestamp = intToHex(muid.timestamp, 14); let medallion = intToHex(muid.medallion, 13); let offset = intToHex(muid.offset, 5); let result = `${timestamp}-${medallion}-${offset}`; ensure(result.length === 34); return result; } exports.muidToString = muidToString; function muidTupleToString(muidTuple) { let timestamp; if (muidTuple[0] === Infinity) { timestamp = "FFFFFFFFFFFFFF"; } else { timestamp = intToHex(muidTuple[0], 14); } let medallion = intToHex(muidTuple[1], 13); let offset = intToHex(muidTuple[2], 5); return `${timestamp}-${medallion}-${offset}`; } exports.muidTupleToString = muidTupleToString; function strToMuidTuple(value) { const nums = value.split("-"); return [ muidHexToInt(nums[0]), muidHexToInt(nums[1]), muidHexToInt(nums[2]), ]; } exports.strToMuidTuple = strToMuidTuple; function strToMuid(value) { const nums = value.split("-"); return { timestamp: muidHexToInt(nums[0]), medallion: muidHexToInt(nums[1]), offset: muidHexToInt(nums[2]), }; } exports.strToMuid = strToMuid; /** * Converts a hexadecimal string to an integer. String should * not contain more than 14 characters. * @param hexString hexadecimal string <= 14 characters * @returns a signed integer. */ function muidHexToInt(hexString) { ensure(hexString.length <= 14); let beginningAddition = BigInt(0); if (hexString.length === 14) { let beginning = hexString.substring(0, 1); hexString = hexString.substring(1); if (beginning === "1") { beginningAddition = BigInt(16) ** BigInt(14); } } let len = hexString.length; let mod = BigInt(16) ** BigInt(len); let num = BigInt(parseInt(hexString, 16)); mod = mod * (num > mod >> BigInt(1) ? BigInt(1) : BigInt(0)); return Number(num + beginningAddition - mod); } /** * Converts a number to its hexadecimal equivalent. * @param value * @param padding maximum size of hex string, padded by 0s. * @returns a hexadecimal string */ function intToHex(value, padding) { const digits = padding || 0; const twosComplement = value < 0 ? BigInt(16) ** BigInt(digits) + BigInt(value) : value; return twosComplement.toString(16).padStart(digits, "0").toUpperCase(); } exports.intToHex = intToHex; const oneByteToHex = (byte) => byte.toString(16).padStart(2, "0").toUpperCase(); exports.oneByteToHex = oneByteToHex; const bytesToHex = (bytes) => Array.from(bytes).map(exports.oneByteToHex).join(""); exports.bytesToHex = bytesToHex; const parseByte = (twoHexDigits) => parseInt(twoHexDigits, 16); exports.parseByte = parseByte; const hexToBytes = (hex) => Uint8Array.from(hex.match(/.{1,2}/g).map(exports.parseByte)); exports.hexToBytes = hexToBytes; function timestampToString(timestamp) { return intToHex(timestamp, 14); } exports.timestampToString = timestampToString; function valueToJson(value) { // Note that this function doesn't check for circular references or anything like that, but // I think this is okay because circular objects can't be encoded into the database in the first place. if (value instanceof Uint8Array) { value = Array.from(value).map(intToHex).join(""); } const type = typeof value; if (type === "string" || type === "number" || value === true || value === false || value === null) { return JSON.stringify(value); } if ("function" === typeof value["toISOString"]) { return `"${value.toISOString()}"`; } if (Array.isArray(value)) { return "[" + value.map(valueToJson).join(",") + "]"; } if (value instanceof Map || value[Symbol.toStringTag] === "Map") { const entries = Array.from(value["entries"]()); entries.sort(); return ("{" + entries .map(function (pair) { return `"${pair[0]}":` + valueToJson(pair[1]); }) .join(",") + "}"); } throw new Error(`value not recognized: ${value}`); } exports.valueToJson = valueToJson; function muidToTuple(muid) { return [muid.timestamp, muid.medallion, muid.offset]; } exports.muidToTuple = muidToTuple; function muidTupleToMuid(tuple) { return { timestamp: tuple[0], medallion: tuple[1], offset: tuple[2], }; } exports.muidTupleToMuid = muidTupleToMuid; /** * Checks the resource path to ensure that it will resolve to a sensible file. * Specifically, it will require that each path component start with [a-zA-Z0-9_], * and only allow [a-zA-Z0-9_.@-] for following characters. This is to prevent * users from accessing hidden files with a dot prefix and traversing up with dot-dot * @param path resource requested * @returns True if the path doesn't look like something we should let users access. */ function isPathDangerous(path) { const pathParts = path.split(/\/+/).filter((part) => part.length > 0); return (pathParts.length === 0 || !pathParts.every((part) => /^\w[\w.@-]*$/.test(part))); } exports.isPathDangerous = isPathDangerous; /** * Uses `console.error` to log messages to stderr in a form like: * [04:07:03.227Z CommandLineInterface.ts:51] got chain manager, using medallion=383316229311328 * That is to say, it's: * [<Timestamp> <SourceFileName>:<SourceLine>] <Message> * @param msg message to log */ function logToStdErr(msg) { var _a; const stackString = new Error().stack; const callerLine = stackString ? stackString.split("\n")[2] : ""; const caller = (_a = callerLine .split(/\//) .pop()) === null || _a === void 0 ? void 0 : _a.replace(/:\d+\)/, ""); const timestamp = new Date().toISOString().split("T").pop(); // using console.error because I want to write to stderr const procId = process ? process.pid : 0; console.error(`[${timestamp} ${caller} ${procId}] ${msg}`); } exports.logToStdErr = logToStdErr; function sameData(key1, key2) { if (key1 instanceof Uint8Array && key2 instanceof Uint8Array) { if (key1.byteLength !== key2.byteLength) return false; for (let i = 0; i < key1.byteLength; i++) { if (key1[i] !== key2[i]) return false; } return true; } if (Array.isArray(key1) && Array.isArray(key2)) { if (key1.length !== key2.length) return false; for (let i = 0; i < key1.length; i++) { if (!sameData(key1[i], key2[i])) return false; } return true; } if (typeof key1 === "number" || typeof key1 === "string" || typeof key1 === "undefined") { return key1 === key2; } return false; } exports.sameData = sameData; function entryToEdgeData(entry) { return { source: muidTupleToMuid(entry.sourceList[0]), target: muidTupleToMuid(entry.targetList[0]), value: entry.value, action: muidTupleToMuid(entry.containerId), effective: entry.storageKey, }; } exports.entryToEdgeData = entryToEdgeData; exports.dehydrate = muidToTuple; exports.rehydrate = muidTupleToMuid; function getActorId() { if (typeof window === "undefined") return process.pid; else { // So we don't assign multiple gink instances in different windows the same actorId if (!window.localStorage.getItem(`gink-current-window`)) { window.localStorage.setItem(`gink-current-window`, "1"); } let currentWindow = Number(window.localStorage.getItem(`gink-current-window`)); // Using 2^22 since that is the max pid for any process on a 64 bit machine. const aId = 2 ** 22 + currentWindow; currentWindow++; window.localStorage.setItem(`gink-current-window`, String(currentWindow)); window.localStorage.setItem(`gink-${aId}`, `${Date.now()}`); // Heartbeat the browser's localStorage every 1 second with the current time. // This is to tell isAlive() that the window is still alive. setInterval(() => { window.localStorage.setItem(`gink-${aId}`, `${Date.now()}`); }, 1000); window.onunload = () => { window.localStorage.removeItem(`gink-${aId}`); }; return aId; } } exports.getActorId = getActorId; /** * Used to (attempt to) identify the user who starts a gink chain. * @returns either the 'username@hostname' of the process running gink, * or a generic 'browser-client' if gink is running in a browser. */ function getIdentity() { if (typeof window === "undefined") return `${(0, os_1.userInfo)().username}@${(0, os_1.hostname)()}`; else { return ("browser-client-" + "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (c) => (+c ^ (crypto.getRandomValues(new Uint8Array(1))[0] & (15 >> (+c / 4)))).toString(16))); } } exports.getIdentity = getIdentity; /** * This function exists to determine if the process or window that previously wrote to a chain is still around. * If not, then it's safe to append to that chain (to reduce the number of chain starts). If the creator of a * chain is still active, then you can't assume that the chain is free for reuse. * @param actorId * @returns */ async function isAlive(actorId) { if (typeof window === "undefined") { ensure(findProcess, "find-process library didn't load in browser"); const found = await findProcess("pid", actorId); ensure(found.length === 0 || found.length === 1); return found.length === 1; } else { const lastPinged = window.localStorage.getItem(`gink-${actorId}`); if (!lastPinged) return false; const lastPingedTime = Number(lastPinged); const currentTime = Date.now(); // Compare current time to the last window heartbeat // Using 5 seconds here for a bit of a buffer return currentTime - lastPingedTime < 5000; } } exports.isAlive = isAlive; function getType(extension) { const types = { html: "text/html", css: "text/css", js: "application/javascript", png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif", json: "application/json", xml: "application/xml", }; const result = types[extension]; if (!result) { throw new Error(`type not found for extension: ${extension}`); } return result; } exports.getType = getType; function mergeBytes(arrayOne, arrayTwo) { const mergedArray = new Uint8Array(arrayOne.length + arrayTwo.length); mergedArray.set(arrayOne); mergedArray.set(arrayTwo, arrayOne.length); return mergedArray; } exports.mergeBytes = mergeBytes; function signBundle(message, secretKey) { if (secretKey.length != 64) throw new Error("secret key not appropriate length!"); if (exports.signingBundles) { //return mergeBytes(secretKey, message); return (0, libsodium_wrappers_1.crypto_sign)(message, secretKey); } else return message; } exports.signBundle = signBundle; function verifyBundle(signedBundle, verifyKey) { ensure(verifyKey.length == 32); if (exports.signingBundles) { (0, libsodium_wrappers_1.crypto_sign_open)(signedBundle, verifyKey); } } exports.verifyBundle = verifyBundle; function createKeyPair() { const result = (0, libsodium_wrappers_1.crypto_sign_keypair)(); return { publicKey: result.publicKey, secretKey: result.privateKey }; /* uncomment for deterministic debugging const x = '5FF46DD6A05CCA09822D96CA4AF957D4ED22E059B1D82AA8DD692FF092B5A15C'; const y = '26F20F23EB12D508DF46DB9EE51BCA3E005AD00845F8A92A1E0E3E2440FE35E0'; return { secretKey: hexToBytes( x + y), publicKey: hexToBytes(y), } */ } exports.createKeyPair = createKeyPair; function getSig(bytes) { let result = 0; for (let i = 0; i < bytes.byteLength; i++) { result = result ^ bytes[i]; } return result; } exports.getSig = getSig; function encryptMessage(message, key) { let nonce = (0, libsodium_wrappers_1.randombytes_buf)(libsodium_wrappers_1.crypto_secretbox_NONCEBYTES); nonce = (0, libsodium_wrappers_1.randombytes_buf)(libsodium_wrappers_1.crypto_secretbox_NONCEBYTES); const ciphertext = (0, libsodium_wrappers_1.crypto_secretbox_easy)(message, nonce, key); return mergeBytes(nonce, ciphertext); } exports.encryptMessage = encryptMessage; function decryptMessage(message, key) { if (message.length < libsodium_wrappers_1.crypto_secretbox_NONCEBYTES + libsodium_wrappers_1.crypto_secretbox_MACBYTES) { throw new Error("Message length shorter than nonce + MAC"); } let nonce = message.slice(0, libsodium_wrappers_1.crypto_secretbox_NONCEBYTES), ciphertext = message.slice(libsodium_wrappers_1.crypto_secretbox_NONCEBYTES); return (0, libsodium_wrappers_1.crypto_secretbox_open_easy)(ciphertext, nonce, key); } exports.decryptMessage = decryptMessage; //# sourceMappingURL=utils.js.map