@x5e/gink
Version:
an eventually consistent database
620 lines • 28.4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Database = void 0;
const Peer_1 = require("./Peer");
const utils_1 = require("./utils");
const ChainTracker_1 = require("./ChainTracker");
const Bundler_1 = require("./Bundler");
const PairSet_1 = require("./PairSet");
const PairMap_1 = require("./PairMap");
const KeySet_1 = require("./KeySet");
const Directory_1 = require("./Directory");
const Box_1 = require("./Box");
const Sequence_1 = require("./Sequence");
const Group_1 = require("./Group");
const builders_1 = require("./builders");
const Property_1 = require("./Property");
const Vertex_1 = require("./Vertex");
const EdgeType_1 = require("./EdgeType");
const Decomposition_1 = require("./Decomposition");
const MemoryStore_1 = require("./MemoryStore");
const factories_1 = require("./factories");
/**
* This is an instance of the Gink database that can be run inside a web browser or via
* ts-node on a server. Because of the need to work within a browser it doesn't do any port
* listening (see SimpleServer for that capability).
*/
class Database {
constructor(store = new MemoryStore_1.MemoryStore(true), identity = (0, utils_1.getIdentity)(), logger = utils_1.noOp) {
this.store = store;
this.logger = logger;
this.peers = new Map();
this.listeners = new Map();
this.countConnections = 0; // Includes disconnected clients.
this.chainGetter = undefined;
this.identity = identity;
this.ready = this.initialize();
}
async initialize() {
await this.store.ready;
this.iHave = await this.store.getChainTracker();
const innerMap = new Map();
innerMap.set("all_bundles", []);
innerMap.set("remote_only", []);
this.listeners.set("all", innerMap);
const callback = async (bundle) => {
for (const [peerId, peer] of this.peers) {
peer._sendIfNeeded(bundle);
}
// Send to listeners subscribed to all containers.
for (const listener of this.getListeners()) {
listener(bundle);
}
};
this.store.addFoundBundleCallBack(callback);
}
/**
* Starts a chain or finds one to reuse, then sets myChain.
*/
getChain() {
if (!this.chainGetter)
this.chainGetter = this.getChainHelper();
return this.chainGetter;
}
async getChainHelper() {
if (this.lastLinkToExtend)
return this.lastLinkToExtend;
this.logger("calling getChain()");
const claimedChains = await this.store.getClaimedChains();
let toReuse;
for (let value of claimedChains.values()) {
const chainId = await this.store.getChainIdentity([
value.medallion,
value.chainStart,
]);
this.logger(`considering chain: ${JSON.stringify(value)}`);
if (chainId !== this.identity) {
this.logger(`identities don't match: ${chainId} ${this.identity}`);
continue;
}
if (await (0, utils_1.isAlive)(value.actorId)) {
this.logger(`actor is still alive`);
continue;
}
// TODO: check to see if meta-data matches, and overwrite if not
toReuse = value;
if (typeof window !== "undefined") {
// If we are running in a browser and take over a chain,
// start a new heartbeat.
setInterval(() => {
window.localStorage.setItem(`gink-${value.actorId}`, `${Date.now()}`);
}, 1000);
}
break;
}
if (toReuse) {
(0, utils_1.ensure)(toReuse.medallion > 0);
const publicKey = await this.store.getVerifyKey([
toReuse.medallion,
toReuse.chainStart,
]);
(0, utils_1.ensure)(publicKey);
this.keyPair = (0, utils_1.ensure)(await this.store.pullKeyPair(publicKey));
this.lastLinkToExtend = this.iHave.getBundleInfo([
toReuse.medallion,
toReuse.chainStart,
]);
}
else {
const medallion = (0, utils_1.makeMedallion)();
const chainStart = (0, utils_1.generateTimestamp)();
const keyPair = (0, utils_1.createKeyPair)();
await this.store.saveKeyPair(keyPair);
this.keyPair = keyPair;
const bundler = new Bundler_1.Bundler(undefined, medallion);
// Starting a new chain, so don't have/need a prior_hash.
bundler.seal({
medallion,
timestamp: chainStart,
chainStart,
}, keyPair, undefined, this.identity);
(0, utils_1.ensure)(bundler.builder.getIdentity() === this.identity);
await this.store.addBundle(bundler, true);
this.lastLinkToExtend = bundler.info;
(0, utils_1.ensure)(this.lastLinkToExtend.hashCode &&
this.lastLinkToExtend.hashCode.length == 32);
this.iHave.markAsHaving(bundler.info);
this.logger(`started chain with ${JSON.stringify(bundler.info, ["medallion", "chainStart"])}`);
// If there is already a connection before we claim a chain, ensure the
// peers get this bundle as well so future bundles will be valid extensions.
for (const peer of this.peers.values()) {
peer._sendIfNeeded(bundler);
}
}
(0, utils_1.ensure)(this.lastLinkToExtend, "myChain wasn't set.");
(0, utils_1.ensure)(this.lastLinkToExtend.hashCode &&
this.lastLinkToExtend.hashCode.length == 32);
return this.lastLinkToExtend;
}
/**
* Reset all containers in the database to a previous time.
* @param toTime optional timestamp to reset to. If not provided, each container
* will be cleared.
* @param bundlerOrComment optional bundler to add this change to, or a string to
* add a comment to a new bundle.
*/
async reset(toTime, bundlerOrComment) {
let immediate = false;
let bundler;
if (bundlerOrComment instanceof Bundler_1.Bundler) {
bundler = bundlerOrComment;
}
else {
immediate = true;
bundler = new Bundler_1.Bundler(bundlerOrComment);
}
// Leaving off Behavior.PROPERTY since each individual property will get reset
// with the other container reset calls
const globalBehaviors = [
builders_1.Behavior.BOX,
builders_1.Behavior.SEQUENCE,
builders_1.Behavior.PAIR_MAP,
builders_1.Behavior.DIRECTORY,
builders_1.Behavior.KEY_SET,
builders_1.Behavior.GROUP,
builders_1.Behavior.PAIR_SET,
];
const globalContainers = [];
for (const behavior of globalBehaviors) {
globalContainers.push([-1, -1, behavior]);
}
const containers = await this.store.getAllContainerTuples();
for (const muidTuple of containers) {
const container = await (0, factories_1.construct)(this, (0, utils_1.muidTupleToMuid)(muidTuple));
if (container instanceof Property_1.Property)
continue;
await container.reset({ toTime, bundlerOrComment: bundler });
}
for (const muidTuple of globalContainers) {
const container = await (0, factories_1.construct)(this, (0, utils_1.muidTupleToMuid)(muidTuple));
await container.reset({ toTime, bundlerOrComment: bundler });
}
if (immediate) {
await this.addBundler(bundler);
}
}
/*
* Returns a handle to the magic global directory. Primarily intended for testing.
* @returns a "magic" global directory that always exists and is accessible by all instances
*/
getGlobalDirectory() {
return new Directory_1.Directory(this, {
timestamp: -1,
medallion: -1,
offset: builders_1.Behavior.DIRECTORY,
});
}
getGlobalProperty() {
return new Property_1.Property(this, {
timestamp: -1,
medallion: -1,
offset: builders_1.Behavior.PROPERTY,
});
}
getMedallionDirectory() {
return new Directory_1.Directory(this, {
timestamp: -1,
medallion: this.lastLinkToExtend[0],
offset: builders_1.Behavior.DIRECTORY,
});
}
/**
* Creates a new box container.
* @param change either the bundler to add this box creation to, or a comment for an immediate change
* @returns promise that resolves to the Box container (immediately if a bundler is passed in, otherwise after the bundle)
*/
async createBox(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.BOX, change);
return new Box_1.Box(this, muid, containerBuilder);
}
/**
* Creates a new List container.
* @param change either the bundler to add this box creation to, or a comment for an immediate change
* @returns promise that resolves to the List container (immediately if a bundler is passed in, otherwise after the bundle)
*/
async createSequence(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.SEQUENCE, change);
return new Sequence_1.Sequence(this, muid, containerBuilder);
}
/**
* Creates a new Key Set container.
* @param change either the bundler to add this box creation to, or a comment for an immediate change
* @returns promise that resolves to the Key Set container (immediately if a bundler is passed in, otherwise after the bundle)
*/
async createKeySet(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.KEY_SET, change);
return new KeySet_1.KeySet(this, muid, containerBuilder);
}
/**
* Creates a new Group container.
* @param change either the bundler to add this box creation to, or a comment for an immediate change
* @returns promise that resolves to the Group container (immediately if a bundler is passed in, otherwise after the bundle)
*/
async createGroup(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.GROUP, change);
return new Group_1.Group(this, muid, containerBuilder);
}
/**
* Creates a new PairSet container.
* @param change either the bundler to add this box creation to, or a comment for an immediate change
* @returns promise that resolves to the PairSet container (immediately if a bundler is passed in, otherwise after the bundle)
*/
async createPairSet(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.PAIR_SET, change);
return new PairSet_1.PairSet(this, muid, containerBuilder);
}
/**
* Creates a new PairMap container.
* @param change either the bundler to add this box creation to, or a comment for an immediate change
* @returns promise that resolves to the PairMap container (immediately if a bundler is passed in, otherwise after the bundle)
*/
async createPairMap(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.PAIR_MAP, change);
return new PairMap_1.PairMap(this, muid, containerBuilder);
}
/**
* Creates a new Directory container (like a javascript map or a python dict).
* @param change either the bundler to add this box creation to, or a comment for an immediate change
* @returns promise that resolves to the Directory container (immediately if a bundler is passed in, otherwise after the bundle)
*/
// TODO: allow user to specify the types allowed for keys and values
async createDirectory(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.DIRECTORY, change);
return new Directory_1.Directory(this, muid, containerBuilder);
}
async createVertex(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.VERTEX, change);
return new Vertex_1.Vertex(this, muid, containerBuilder);
}
async createEdgeType(change) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.EDGE_TYPE, change);
return new EdgeType_1.EdgeType(this, muid, containerBuilder);
}
async createProperty(bundlerOrComment) {
const [muid, containerBuilder] = await this.createContainer(builders_1.Behavior.PROPERTY, bundlerOrComment);
return new Property_1.Property(this, muid, containerBuilder);
}
async createContainer(behavior, change) {
let immediate = false;
if (!(change instanceof Bundler_1.Bundler)) {
immediate = true;
change = new Bundler_1.Bundler(change);
}
const containerBuilder = new builders_1.ContainerBuilder();
containerBuilder.setBehavior(behavior);
const address = change.addContainer(containerBuilder);
if (immediate) {
await this.addBundler(change);
}
return [address, containerBuilder];
}
/**
* Returns an array of Muids of containers that have the provided name.
* @param name
* @param asOf optional timestamp to look back to.
* @returns an array of Muids.
*/
async getContainersWithName(name, asOf) {
return await this.store.getContainersByName(name, asOf);
}
/**
* Adds a listener that will be called every time a bundle is received with the
* BundleInfo (which contains chain information, timestamp, and bundle comment).
* @param listener a callback to be invoked when a change occurs in the database or container
* @param containerMuid the Muid of a container to subscribe to. If left out, subscribe to all containers.
*/
addListener(listener, containerMuid, remoteOnly = false) {
const key = containerMuid ? (0, utils_1.muidToString)(containerMuid) : "all";
if (!this.listeners.has(key)) {
const innerMap = new Map();
innerMap.set("all_bundles", []);
innerMap.set("remote_only", []);
this.listeners.set(key, innerMap);
}
const which = remoteOnly ? "remote_only" : "all_bundles";
this.listeners.get(key).get(which).push(listener);
}
/**
* Gets a list of bundle listeners per container, listening to all bundles or just remote.
* @param remoteOnly true if looking for listeners only subscribed to remote bundles.
* @param containerMuid optional container muid to find listeners subscribed to a specific container.
*/
getListeners(remoteOnly = false, containerMuid) {
const key = containerMuid ? (0, utils_1.muidToString)(containerMuid) : "all";
const containerMap = this.listeners.get(key);
if (!containerMap)
return [];
const innerMap = remoteOnly
? containerMap.get("remote_only")
: containerMap.get("all_bundles");
return innerMap || [];
}
/**
* Adds a bundle to a chain, setting the medallion and timestamps on the bundle in the process.
*
* @param bundler a PendingBundle ready to be sealed
* @returns A promise that will resolve to the bundle timestamp once it's persisted/sent.
*/
addBundler(bundler) {
return this.ready.then(() => this.getChain().then(() => {
const nowMicros = (0, utils_1.generateTimestamp)();
const seenThrough = this.lastLinkToExtend.timestamp;
const newTimestamp = nowMicros > seenThrough ? nowMicros : seenThrough + 10;
(0, utils_1.ensure)(seenThrough > 0 && seenThrough < nowMicros);
const bundleInfo = {
medallion: this.lastLinkToExtend.medallion,
chainStart: this.lastLinkToExtend.chainStart,
timestamp: newTimestamp,
priorTime: seenThrough,
};
bundler.seal(bundleInfo, this.keyPair, this.lastLinkToExtend.hashCode);
// The bundle is seralized then deserialized to catch problems before broadcasting.
const decomposition = new Decomposition_1.Decomposition(bundler.bytes);
this.lastLinkToExtend = decomposition.info;
return this.receiveBundle(decomposition);
}));
}
/**
* Closes connections to peers and closes the store.
*/
async close() {
for (const peer of this.peers.values()) {
try {
peer.close();
}
catch (problem) {
console.error(`problem closing peer: ${problem}`);
}
}
await this.store.close();
}
/**
* @returns a truthy number that can be used to identify connections
*/
createConnectionId() {
return ++this.countConnections;
}
/**
* Tries to add a bundle to the local store. If successful (i.e. it hasn't seen it before)
* then it will also publish that bundle to the connected peers.
*
* This is called both from addPendingBundle (for locally produced bundles) and
* being called by receiveMessage.
*
* @param bundleBytes The bytes that correspond to this transaction.
* @param fromConnectionId The (truthy) connectionId if it came from a peer.
* @returns
*/
receiveBundle(bundle, fromConnectionId) {
return this.store.addBundle(bundle).then((added) => {
var _a;
if (!added)
return;
let summary;
if (bundle.info.chainStart === bundle.info.timestamp) {
summary = JSON.stringify(bundle.info, [
"medallion",
"timestamp",
"chainStart",
]);
}
else {
summary = JSON.stringify(bundle.info, [
"medallion",
"timestamp",
"priorTime",
]);
}
this.logger(`added bundle from ${fromConnectionId}: ${summary}`);
this.iHave.markAsHaving(bundle.info);
const peer = this.peers.get(fromConnectionId);
if (peer) {
(_a = peer.hasMap) === null || _a === void 0 ? void 0 : _a.markAsHaving(bundle.info);
peer._sendAck(bundle.info);
}
for (const [peerId, peer] of this.peers) {
if (peerId !== fromConnectionId)
peer._sendIfNeeded(bundle);
}
// Send to listeners subscribed to all containers.
for (const listener of this.getListeners()) {
listener(bundle);
}
if (this.listeners.size > 1) {
// Loop through changes and gather a set of changed containers.
const changedContainers = new Set();
const changesList = bundle.builder.getChangesList();
for (let index = 0; index < changesList.length; index++) {
const offset = index + 1;
const changeBuilder = changesList[index];
const entry = changeBuilder.getEntry();
const clearance = changeBuilder.getClearance();
let container;
if (entry) {
container = entry.getContainer();
}
else if (clearance) {
container = clearance.getContainer();
}
if (container &&
container.getTimestamp() &&
container.getMedallion() &&
container.getOffset()) {
const muid = (0, utils_1.builderToMuid)(container, {
timestamp: bundle.info.timestamp,
medallion: bundle.info.medallion,
offset: offset,
});
changedContainers.add(muid);
}
}
// Send to listeners specifically subscribed to each container.
for (const muid of changedContainers) {
const containerListeners = this.getListeners(false, muid);
const remoteOnlyListeners = this.getListeners(true, muid);
for (const listener of containerListeners) {
listener(bundle);
}
if (fromConnectionId) {
for (const remoteListener of remoteOnlyListeners) {
remoteListener(bundle);
}
}
}
}
return bundle.info;
});
}
/**
* @param messageBytes Bytes received from a peer.
* @param fromConnectionId Local name of the peer the data was received from.
* @returns
*/
async receiveMessage(messageBytes, fromConnectionId) {
var _a, _b, _c;
await this.ready;
const peer = this.peers.get(fromConnectionId);
if (!peer)
throw Error("Got a message from a peer I don't have a proxy for?");
try {
const parsed = (builders_1.SyncMessageBuilder.deserializeBinary(messageBytes));
if (parsed.hasBundle()) {
const bundleBytes = parsed.getBundle_asU8();
const decomposition = new Decomposition_1.Decomposition(bundleBytes);
await this.receiveBundle(decomposition, fromConnectionId);
return;
}
if (parsed.hasGreeting()) {
this.logger(`got greeting from ${fromConnectionId}`);
const greeting = parsed.getGreeting();
peer._receiveHasMap(new ChainTracker_1.ChainTracker({ greeting }));
await this.store.getBundles(peer._sendIfNeeded.bind(peer));
return;
}
if (parsed.hasAck()) {
const ack = parsed.getAck();
const info = {
medallion: ack.getMedallion(),
timestamp: ack.getTimestamp(),
chainStart: ack.getChainStart(),
};
this.logger(`got ack from ${fromConnectionId}: ${JSON.stringify(info)}`);
(_b = (_a = this.peers.get(fromConnectionId)) === null || _a === void 0 ? void 0 : _a.hasMap) === null || _b === void 0 ? void 0 : _b.markAsHaving(info);
}
}
catch (e) {
//TODO: Send some sensible code to the peer to say what went wrong.
console.error(e);
(_c = this.peers.get(fromConnectionId)) === null || _c === void 0 ? void 0 : _c.close();
this.peers.delete(fromConnectionId);
}
finally {
//unlockingFunction();
}
}
/**
* Initiates a websocket connection to a peer.
* @param target a websocket uri, e.g. "ws://127.0.0.1:8080/"
* @param onClose optional callback to invoke when the connection is closed
* @param resolveOnOpen if true, resolve when the connection is established, otherwise wait for greeting
* @param retryOnDisconnect if true, try to reconnect (with backoff) if the server closes the connection
* @returns a promise to the peer
*/
async connectTo(target, options) {
//TODO(https://github.com/google/gink/issues/69): have the default be to wait for databases to sync
const onClose = options && options.onClose ? options.onClose : utils_1.noOp;
const resolveOnOpen = options && options.resolveOnOpen ? options.resolveOnOpen : false;
const retryOnDisconnect = options && options.retryOnDisconnect === false ? false : true;
const authToken = options && options.authToken ? options.authToken : undefined;
await this.ready;
const thisClient = this;
return new Promise((resolve, reject) => {
let protocols = [Database.PROTOCOL];
if (authToken)
protocols.push((0, utils_1.encodeToken)(authToken));
const connectionId = this.createConnectionId();
let websocketClient = new Database.W3cWebSocket(target, protocols);
websocketClient.binaryType = "arraybuffer";
const peer = new Peer_1.Peer(websocketClient.send.bind(websocketClient), websocketClient.close.bind(websocketClient));
websocketClient.onopen = function (_ev) {
// called once the new connection has been established
websocketClient.send(thisClient.iHave.getGreetingMessageBytes());
thisClient.peers.set(connectionId, peer);
if (resolveOnOpen)
resolve(peer);
else
peer.ready.then(resolve);
};
websocketClient.onerror = function (ev) {
// if/when this is called depends on the details of the websocket implementation
console.error(`error on connection ${connectionId} to ${target}, ${ev}`);
reject(ev);
};
websocketClient.onclose = async function (ev) {
// this should always be called once the peer disconnects, including in cases of error
onClose(`closed connection ${connectionId} to ${target}`);
// If the connection was never successfully established, then
// reject the promise returned from the outer connectTo.
reject(ev);
// I'm intentionally leaving the peer object in the peers map just in case we get data from them.
// thisClient.peers.delete(connectionId); // might still be processing data from peer
if (retryOnDisconnect) {
let peer;
let pow = 0;
let retry_ms = 1000;
let jitter = Math.floor(Math.random() * 1000);
while (!peer) {
await new Promise((resolve) => setTimeout(resolve, retry_ms + jitter));
try {
console.log(`retrying connection to ${target}`);
peer = await thisClient.connectTo(target, options);
if (peer) {
console.log(`reconnected to ${target}`);
break;
}
}
catch (e) {
console.error(`retry failed: ${e.message}`);
}
finally {
if (retry_ms < 30000) {
pow += 1;
retry_ms = 1000 * Math.pow(2, pow);
jitter = Math.floor(Math.random() * 1000 * Math.pow(2, pow));
}
}
}
}
};
websocketClient.onmessage = function (ev) {
// Called when any protocol messages are received.
const data = ev.data;
if (data instanceof ArrayBuffer) {
const uint8View = new Uint8Array(data);
thisClient.receiveMessage(uint8View, connectionId);
}
else {
// We don't expect any non-binary text messages.
console.error(`got non-arraybuffer message: ${data}`);
}
};
});
}
}
exports.Database = Database;
Database.PROTOCOL = "gink";
//TODO: centralize platform dependent code
Database.W3cWebSocket = typeof WebSocket === "function"
? WebSocket
: eval("require('websocket').w3cwebsocket");
//# sourceMappingURL=Database.js.map