@x5e/gink
Version:
an eventually consistent database
305 lines (304 loc) • 12.8 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LogBackedStore = void 0;
const MemoryStore_1 = require("./MemoryStore");
const PromiseChainLock_1 = require("./PromiseChainLock");
const LockableLog_1 = require("./LockableLog");
const fs_1 = require("fs");
const ChainTracker_1 = require("./ChainTracker");
const builders_1 = require("./builders");
const utils_1 = require("./utils");
const Decomposition_1 = require("./Decomposition");
/*
At time of writing, there's only an in-memory implementation of
IndexedDB available for Node.js. This subclass will append all
transactions it receives to a log file, making it possible to
recreate the same in-memory database in the future by simply
replaying the receipt of each bundle.
This is obviously not ideal; eventually want to move to either
a durable server side indexedDB implementation or create an
implementation of Store using some other system (e.g. LMDB).
*/
class LogBackedStore extends LockableLog_1.LockableLog {
/**
*
* @param filename file to store transactions and chain ownership information
* @param exclusive if true, lock the file until closing store, otherwise only lock as-needed.
*/
constructor(filename, exclusive = false, internalStore = new MemoryStore_1.MemoryStore()) {
super(filename, exclusive);
this.filename = filename;
this.exclusive = exclusive;
this.internalStore = internalStore;
this.bundlesProcessed = 0;
this.chainTracker = new ChainTracker_1.ChainTracker({});
this.claimedChains = [];
this.identities = new Map(); // Medallion,ChainStart => identity
// While the operating system lock prevents other processes from writing to this file,
// we need to prevent multiple async tasks within this process from trying to interleave operations.
// The memory lock accomplishes this. It might not strictly be necessary, because of how the
// rest of the system is designed, but the overhead is expected to be minimal and may prevent
// some foot shooting.
this.memoryLock = new PromiseChainLock_1.PromiseChainLock();
this.redTo = 0;
this.foundBundleCallBacks = [];
this.opened = false;
this.closed = false;
this.logBackedStoreReady = super.ready.then(() => this.initializeLogBackedStore());
}
getVerifyKey(chainInfo) {
return this.internalStore.getVerifyKey(chainInfo);
}
async saveKeyPair(keyPair) {
await this.ready;
const unlockingFunction = await this.memoryLock.acquireLock();
if (!this.exclusive)
await this.lockFile(true);
if (this.redTo === 0)
this.redTo += await this.writeMagicNumber();
const keyPairBuilder = new builders_1.KeyPairBuilder();
keyPairBuilder.setPublicKey(keyPair.publicKey);
keyPairBuilder.setSecretKey(keyPair.secretKey);
const logFragment = new builders_1.LogFileBuilder();
logFragment.setKeyPairsList([keyPairBuilder]);
this.redTo += await this.writeLogFragment(logFragment, true);
if (!this.exclusive)
await this.unlockFile();
unlockingFunction();
await this.internalStore.saveKeyPair(keyPair);
}
async pullKeyPair(publicKey) {
return await this.internalStore.pullKeyPair(publicKey);
}
async saveSymmetricKey(symmetricKey) {
return await this.internalStore.saveSymmetricKey(symmetricKey);
}
async getSymmetricKey(keyId) {
return await this.internalStore.getSymmetricKey(keyId);
}
get ready() {
return this.logBackedStoreReady;
}
async initializeLogBackedStore() {
await this.internalStore.ready;
const unlockingFunction = await this.memoryLock.acquireLock();
await this.pullDataFromFile();
const thisLogBackedStore = this;
this.fileWatcher = (0, fs_1.watch)(this.filename, async (eventType, filename) => {
await new Promise((r) => setTimeout(r, 10));
if (thisLogBackedStore.closed || !thisLogBackedStore.opened)
return;
let size = await thisLogBackedStore.getFileLength();
if (eventType === "change" && size > this.redTo) {
const unlockingFunction = await this.memoryLock.acquireLock();
if (!this.exclusive)
await this.lockFile(true);
await this.pullDataFromFile();
if (!this.exclusive)
await this.unlockFile();
unlockingFunction();
}
});
unlockingFunction();
this.opened = true;
}
async close() {
this.closed = true;
if (this.fileWatcher)
this.fileWatcher.close();
if (this.fileHandle)
await this.fileHandle.close().catch();
if (this.internalStore)
await this.internalStore.close().catch();
}
async pullDataFromFile() {
if (this.closed)
return;
const totalSize = await this.getFileLength();
if (this.redTo < totalSize) {
const logFileBuilder = await this.getLogContents(this.redTo, totalSize);
if (this.redTo === 0) {
(0, utils_1.ensure)(logFileBuilder.getMagicNumber() === 1263421767, "log file doesn't have magic number");
}
const bundles = logFileBuilder.getBundlesList();
for (const bundleBytes of bundles) {
const bundle = new Decomposition_1.Decomposition(bundleBytes);
const added = await this.internalStore.addBundle(bundle);
if (!added)
throw new Error("unexpected not added");
const info = bundle.info;
const identity = bundle.builder.getIdentity();
this.chainTracker.markAsHaving(bundle.info);
// This is the start of a chain, and we need to keep track of the identity.
if (info.timestamp === info.chainStart && !info.priorTime) {
(0, utils_1.ensure)(identity, "chain start bundle has no identity");
this.identities.set(`${info.medallion},${info.chainStart}`, identity);
}
else {
(0, utils_1.ensure)(!identity, "non-chain-start bundle has identity");
}
for (const callback of this.foundBundleCallBacks) {
callback(bundle);
}
this.bundlesProcessed += 1;
}
const claims = logFileBuilder.getClaimsList();
for (let i = 0; i < claims.length; i++) {
this.claimedChains.push({
medallion: claims[i].getMedallion(),
chainStart: claims[i].getChainStart(),
actorId: claims[i].getProcessId(),
claimTime: claims[i].getClaimTime(),
});
}
const keyPairs = logFileBuilder.getKeyPairsList();
for (let i = 0; i < keyPairs.length; i++) {
this.internalStore.saveKeyPair({
publicKey: keyPairs[i].getPublicKey_asU8(),
secretKey: keyPairs[i].getSecretKey_asU8(),
});
}
this.redTo = totalSize;
}
}
async getContainerProperties(containerMuid, asOf) {
await this.ready;
return this.internalStore.getContainerProperties(containerMuid, asOf);
}
async getOrderedEntries(container, through = Infinity, asOf) {
await this.ready;
return this.internalStore.getOrderedEntries(container, through, asOf);
}
async getEntriesBySourceOrTarget(vertex, source, asOf) {
await this.ready;
return this.internalStore.getEntriesBySourceOrTarget(vertex, source, asOf);
}
async getBundlesProcessed() {
await this.ready;
return this.bundlesProcessed;
}
async getLocation(entry, asOf) {
await this.ready;
return await this.internalStore.getLocation(entry, asOf);
}
async addBundle(bundle, claimChain) {
// TODO(https://github.com/x5e/gink/issues/182): delay unlocking the file to give better throughput
await this.ready;
let added = false;
const unlockingFunction = await this.memoryLock.acquireLock();
if (!this.exclusive)
await this.lockFile(true);
try {
await this.pullDataFromFile();
if (this.redTo === 0)
this.redTo += await this.writeMagicNumber();
const info = bundle.info;
added = await this.internalStore.addBundle(bundle);
const identity = bundle.builder.getIdentity();
if (claimChain) {
if (!added)
throw new Error("can't claim chain on old bundle");
await this.claimChain(info.medallion, info.chainStart, (0, utils_1.getActorId)());
if (info.timestamp === info.chainStart && !info.priorTime) {
(0, utils_1.ensure)(identity, "chain start bundle has no identity");
this.identities.set(`${info.medallion},${info.chainStart}`, bundle.builder.getIdentity());
}
else {
(0, utils_1.ensure)(!identity, "non-chain-start bundle has identity");
}
}
this.chainTracker.markAsHaving(info);
if (added) {
(0, utils_1.ensure)(this.fileLocked);
await this.pullDataFromFile();
const logFragment = new builders_1.LogFileBuilder();
logFragment.setBundlesList([bundle.bytes]);
this.redTo += await this.writeLogFragment(logFragment, true);
}
}
finally {
unlockingFunction();
if (!this.exclusive)
await this.unlockFile();
}
return added;
}
async getClaimedChains() {
await this.ready;
const result = new Map();
for (let chain of this.claimedChains) {
result.set(chain.medallion, chain);
}
return result;
}
async claimChain(medallion, chainStart, actorId) {
await this.ready;
await this.pullDataFromFile();
const claimTime = (0, utils_1.generateTimestamp)();
const fragment = new builders_1.LogFileBuilder();
const claim = new builders_1.ClaimBuilder();
claim.setChainStart(chainStart);
claim.setMedallion(medallion);
claim.setProcessId(actorId);
claim.setClaimTime(claimTime);
fragment.setClaimsList([claim]);
this.redTo += await this.writeLogFragment(fragment);
const chain = {
medallion,
chainStart,
actorId: actorId || 0,
claimTime,
};
this.claimedChains.push(chain);
return chain;
}
async getChainIdentity(chainInfo) {
await this.ready;
return this.identities.get(`${chainInfo[0]},${chainInfo[1]}`);
}
async getChainTracker() {
await this.ready;
return await this.internalStore.getChainTracker();
}
async getBundles(callBack) {
await this.ready;
await this.internalStore.getBundles(callBack);
}
async getContainerBytes(address) {
await this.ready;
return this.internalStore.getContainerBytes(address);
}
async getEntryByKey(container, key, asOf) {
await this.ready;
return this.internalStore.getEntryByKey(container, key, asOf);
}
async getKeyedEntries(container, asOf) {
await this.ready;
return this.internalStore.getKeyedEntries(container, asOf);
}
async getEntryById(entryMuid, asOf) {
await this.ready;
return this.internalStore.getEntryById(entryMuid, asOf);
}
async getAllEntries() {
await this.ready;
return this.internalStore.getAllEntries();
}
async getContainersByName(name, asOf) {
return await this.internalStore.getContainersByName(name, asOf);
}
async getAllContainerTuples() {
return await this.internalStore.getAllContainerTuples();
}
/**
* Add a callback if you want another function to run when a new
* bundle is pulled from the log file.
* @param callback a function to be called when a new bundle has been
* received from the log file. It needs to take one argument, bundleInfo
*/
addFoundBundleCallBack(callback) {
this.foundBundleCallBacks.push(callback);
}
}
exports.LogBackedStore = LogBackedStore;
//# sourceMappingURL=LogBackedStore.js.map