ram64
Version:
Multi-threaded 64bit memory cache database inspired by redis-like features
475 lines (474 loc) • 15.2 kB
JavaScript
"use strict";
var _workerThreads = require("worker_threads");
var _commands = require("../commands");
var _processRequest = require("./process-request");
var _hash = require("../util/hash");
var _processServerRequest = require("./process-server-request");
var _processResponse = require("./process-response");
var _ramFunction = require("../ram-function");
var _promises = require("fs/promises");
var _isRam64Message = require("./is-ram64-message");
var _pLimit = _interopRequireDefault(require("p-limit"));
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : {
default: obj
};
}
class RAM64 {
constructor({ connectKey , workers , ports , shardCount , concurrency }){
this.#connectKey = connectKey;
this.#cacheWorkers = workers;
this.#ports = ports;
this.#shardCount = shardCount;
this.#concurrency = Math.max(1, concurrency);
this.#limit = this.#concurrency ? (0, _pLimit).default(this.#concurrency) : (handler)=>handler();
this.workerPorts.forEach((port)=>{
port.unref();
port.on('message', (msg)=>(0, _processResponse).processResponse(msg));
port.on('messageerror', (err)=>console.error('Worker message error:', err)); // TODO: need better handling
port.on('error', (err)=>console.error('Worker error:', err)); // TODO: need better handling
// TODO! handle 'close' event
});
}
#connectKey;
#cacheWorkers;
#ports;
#shardCount;
#concurrency;
#limit;
get connectKey() {
return this.#connectKey;
}
getPort(key) {
return this.getPortFromHash((0, _hash).getHash(key));
}
getPortFromHash(hash) {
return this.workerPorts[hash % this.workerPorts.length];
}
get workerPorts() {
return this.#cacheWorkers || this.#ports || [];
}
get shardCount() {
return this.#shardCount;
}
get shardsPerWorker() {
return Math.ceil(this.shardCount / this.workerPorts.length);
}
get concurrency() {
return this.#concurrency;
}
get limit() {
return this.#limit;
}
get isMain() {
return this.#cacheWorkers?.length > 0;
}
spawnWorker(filePath, workerData = {}, options = {}) {
const worker = new _workerThreads.Worker(filePath, {
workerData: {
...workerData,
connectKey: this.connectKey
}
});
worker.unref();
this.registerWorker(worker, options);
return worker;
}
registerWorker(worker, options = {}) {
worker.on('message', (msg)=>{
if ((0, _isRam64Message).isRAM64Message(msg)) {
(0, _processServerRequest).processServerRequest(this, worker, msg);
} else if (options.onMessage) {
options.onMessage.call(this, msg);
}
});
return worker;
}
// throw everything away
async shutdown() {
if (!this.isMain) return Promise.reject(new Error(`RAM64.shutdown() must be called from the main instance`));
// TODO: perhaps something more graceful in the future? like waiting for each worker to exit gracefully on its own
// especially useful once we support pending operations like save/load/dump
if (this.#cacheWorkers) {
this.#cacheWorkers.forEach((worker)=>worker.terminate());
this.#cacheWorkers = [];
}
if (this.#ports) {
this.#ports.forEach((port)=>port.close());
this.#ports = [];
}
this.#connectKey = '';
}
save(dirPath) {
return (0, _promises).mkdir(dirPath, {
recursive: true
}).then(()=>(0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.save.index,
args: {
dirPath
}
}));
}
load(dirPath) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.load.index,
args: {
dirPath
}
});
}
registerFunction(fn) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.registerFunction.index,
args: {
fnCode: fn.code
}
}).then(()=>fn);
}
exists(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.exists.index,
key
});
}
get(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.get.index,
key
});
}
async getKeyCount() {
const keys = await Promise.all(this.workerPorts.map((port)=>(0, _processRequest).processRequest(this, {
workerOrPort: port,
commandIndex: _commands.commandsDict.getKeyCount.index
})));
return keys.reduce((acc, val)=>acc + val, 0);
}
getMany(keys) {
return Promise.all(keys.map((key)=>this.get(key)));
}
getAndSet(key, staleFn) {
return this.getWithOptions(key).then((obj)=>{
if (!obj || obj?.staleAt && obj?.staleAt <= Date.now()) {
staleFn(obj).then((newObj)=>this.setWithOptions(key, newObj));
}
return obj;
});
}
getSet(key, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.getSet.index,
key,
args: {
value
}
});
}
getWithOptions(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.getWithOptions.index,
key
});
}
touch(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.touch.index,
key
});
}
set(key, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.set.index,
key,
args: {
value
}
});
}
setIfValue(key, expectedValue, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setIfValue.index,
key,
args: {
expectedValue,
value
}
});
}
setFn(key, fn, params) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setFn.index,
key,
args: {
fnId: fn.id,
params
}
});
}
setMany(sets) {
return Promise.all(sets.map((set)=>this.set(set[0], set[1]))).then(()=>void 0);
}
setOptions(key, options) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setOptions.index,
key,
args: options
});
}
setWithOptions(key, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setWithOptions.index,
key,
args: value
});
}
insert(key, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.insert.index,
key,
args: {
value
}
});
}
del(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.del.index,
key
});
}
deleteAll() {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.deleteAll.index
});
}
strAppend(key, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.strAppend.index,
key,
args: {
value
}
});
}
strPrepend(key, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.strPrepend.index,
key,
args: {
value
}
});
}
strLength(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.strLength.index,
key
});
}
strSetRange(key, offset, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.strSetRange.index,
key,
args: {
offset,
value
}
});
}
strGetRange(key, start, end) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.strGetRange.index,
key,
args: {
start,
end
}
});
}
strReplace(key, replace, value) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.strReplace.index,
key,
args: {
replace,
replaceType: typeof replace,
value
}
});
}
numAdd(key, value, defaultValue = 0) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.numAdd.index,
key,
args: {
value,
defaultValue
}
});
}
numSub(key, value, defaultValue = 0) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.numSub.index,
key,
args: {
value,
defaultValue
}
});
}
numMult(key, value, defaultValue = 0) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.numMult.index,
key,
args: {
value,
defaultValue
}
});
}
numDiv(key, value, defaultValue = 0) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.numDiv.index,
key,
args: {
value,
defaultValue
}
});
}
setGetMembers(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setGetMembers.index,
key
});
}
setAddMembers(key, members) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setAddMembers.index,
key,
args: {
members
}
});
}
setRemoveMembers(key, members) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setRemoveMembers.index,
key,
args: {
members
}
});
}
setGetMemberCount(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setGetMemberCount.index,
key
});
}
setHasMembers(key, members) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.setHasMembers.index,
key,
args: {
members
}
});
}
mapGetKeys(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.mapGetKeys.index,
key
});
}
mapGetValues(key, keys) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.mapGetValues.index,
key,
args: {
keys
}
});
}
mapGetFields(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.mapGetFields.index,
key
});
}
mapAddFields(key, fields) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.mapAddFields.index,
key,
args: {
fields
}
});
}
mapRemoveKeys(key, keys) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.mapRemoveKeys.index,
key,
args: {
keys
}
});
}
mapGetCount(key) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.mapGetCount.index,
key
});
}
mapHasKeys(key, keys) {
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.mapHasKeys.index,
key,
args: {
keys
}
});
}
async scan({ limit =1000 , filter , resumeKey: resumeKey1 , resumeCb } = {}) {
const handler = (resumeKey)=>{
return (0, _processRequest).processRequest(this, {
commandIndex: _commands.commandsDict.scan.index,
workerOrPort: !resumeKey ? this.workerPorts[0] : undefined,
resumeKey,
args: {
limit: Math.max(10, Math.min(limit, 10000)),
resumeKey,
filterExp: filter instanceof RegExp && filter,
filterFn: filter instanceof _ramFunction.RAMFunction && filter.id
}
});
};
let lastResult;
let allKeys = [];
do {
lastResult = await handler(resumeKey1);
allKeys = allKeys.concat(lastResult.keys);
resumeKey1 = lastResult.resumeKey;
}while (resumeCb && lastResult.resumeKey && await resumeCb(lastResult))
return {
keys: allKeys,
resumeKey: lastResult.resumeKey
};
}
scanSplit(resumeKeySplits) {
resumeKeySplits = Math.max(1, Math.min(resumeKeySplits, this.shardCount));
const shardsPerResumeKey = Math.ceil(this.shardCount / resumeKeySplits); // evenly split shards across resume keys
const resumeKeys = [];
let currentShard = 0;
while(currentShard < this.shardCount){
const maxShardIndex = Math.min(currentShard + shardsPerResumeKey, this.shardCount);
const workerIndex = Math.floor(currentShard / this.shardsPerWorker);
resumeKeys.push(`${workerIndex}:${currentShard % this.shardsPerWorker}::${maxShardIndex}`);
currentShard = maxShardIndex + 1;
}
return resumeKeys;
}
}
exports.RAM64 = RAM64;
//# sourceMappingURL=index.js.map