andresusandi-storage/ConsistentHashStore.js

storage utilities for distributed systems

const { name: packageName } = require('./package.json');
const ConsistentHashShard = require('./ConsistentHashShard');

const SHARDS_MAX_COUNT = 100000;
const ITEMS_MAX_PER_SHARD = 1000;

class ConsistentHashStore {
    
    constructor(storageConstructor, maxShardCount, maxItemsPerShard, counterConstructor) {
        this.maxShardCount = maxShardCount || SHARDS_MAX_COUNT;
        this.maxItemsPerShard = maxItemsPerShard || ITEMS_MAX_PER_SHARD;
        this.storageConstructor = storageConstructor;
        this.counterConstructor = counterConstructor;
        this.circle = new Map();
        console.log(`${packageName}/${this.constructor.name}: Created Consistant Hash Store with max ${this.maxShardCount} shards and ${this.maxItemsPerShard} items`);
    }

    async init(existingShardIndices) {
        console.log(`${packageName}/${this.constructor.name}: Initializing Consistent Hash Store`);
        if (existingShardIndices && existingShardIndices.length > 0) {
            for (var i = 0; i < existingShardIndices.length; i++) {
                this.circle.set(existingShardIndices[i], (await this._instantiateShard(existingShardIndices[i])));
                console.log(`${packageName}/${this.constructor.name}: Adding existing shard ${existingShardIndices[i]}`);
            }
        } else {
            console.log(`${packageName}/${this.constructor.name}: Adding default first shard`);
            this.circle.set(0, (await this._instantiateShard(0)));
        }
    }

    async save(item) {
        if (!item)
            throw new Error("item cannot be a null or empty string");

        const shardIndex = this._getShardIndex(item);
        const shard = this.circle.get(shardIndex);
        await shard.save(item, item);
        await shard.modifyCount(1);
        console.log(`${packageName}/${this.constructor.name}: Saved item ${item} to shard ${shardIndex}`);

        const shardCount = await shard.getCount();
        if (shardCount >= this.maxItemsPerShard) {
            console.log(`${packageName}/${this.constructor.name}: Item count for shard ${shardIndex} is ${shardCount} which is greater than ${this.maxItemsPerShard}, splitting...`);
            await this._splitShard(shardIndex);
        }
    }

    async exists(item) {
        const shardIndex = this._getShardIndex(item);
        const shard = this.circle.get(shardIndex);
        const storedItem = await shard.get(item);
        return (storedItem !== null && storedItem !== undefined);
    }

    async _splitShard(shardIndex) {
        
        var shardIndexNumber = Number(shardIndex);
        var newShardIndex = Array.from(this.circle.keys()).find(k => Number(k) > shardIndexNumber);
        newShardIndex = !newShardIndex ? this.maxShardCount : Number(newShardIndex);
        newShardIndex = shardIndexNumber + Math.floor((Number(newShardIndex) - shardIndexNumber) / 2);

        console.log(`${packageName}/${this.constructor.name}: Splitting shard ${shardIndex} with ${newShardIndex}`);

        this.circle.set(newShardIndex, (await this._instantiateShard(newShardIndex)));

        await this._recalculateShardLocation(shardIndex, newShardIndex);

        console.log(`${packageName}/${this.constructor.name}: Finished Shard Split`);
    }

    async _recalculateShardLocation(currentShardIndex, newShardIndex) {

        const currentShard = this.circle.get(currentShardIndex);
        const newShard = this.circle.get(newShardIndex);

        const items = await currentShard.getAllItems();
        console.log(`${packageName}/${this.constructor.name}: Moving ${items.length} items from ${currentShardIndex} to ${newShardIndex}`);
        for (var i = 0; i < items.length; i++) {
            const item = items[i].value;
            const calculatedShardIndex = this._getShardIndex(item);
            if (calculatedShardIndex != currentShardIndex && calculatedShardIndex != newShardIndex)
                console.log("throw new Error(`Shard index ${calculatedShardIndex} was not expected, ${newShardIndex} was`)");

            if (calculatedShardIndex == newShardIndex) {
                await newShard.save(item, item);
                await newShard.modifyCount(1);
                console.log(`${packageName}/${this.constructor.name}: Copied item ${item} from ${currentShardIndex} to ${newShardIndex}`);
            }
        }

        const newItems = await newShard.getAllItems();
        console.log(`${packageName}/${this.constructor.name}: Deleting ${newItems.length} items from ${currentShardIndex}`);
        for (var i = 0; i < newItems.length; i++) {
            await currentShard.delete(newItems[i].value);
            await currentShard.modifyCount(-1);
            console.log(`${packageName}/${this.constructor.name}: Deleted item ${newItems[i].value} from ${currentShardIndex}`);
        }
    }

    async _instantiateShard(shardIndex) {
        const shard = new ConsistentHashShard(shardIndex, this.storageConstructor, this.counterConstructor);
        await shard.init();
        return (shard);
    }

    _getShardIndex(item) {
        const rawIndex = this._gethashCode(item) % this.maxShardCount;
        //rawIndex = rawIndex < 0 ? rawIndex * -1 : rawIndex;
        if (rawIndex < 0)
            throw new Error("hash result is less than 0.");
        const keysDesc = [...Array.from(this.circle.keys())].sort((a, b) => b - a);
        const actualIndex = keysDesc.find(k => k <= rawIndex);
        return (actualIndex);
    }

    _gethashCode(str) {
        var hash = 0;
        for (var i = 0; i < str.length; i++) {
            var character = str.charCodeAt(i);
            hash = ((hash << 5) - hash) + character;
            hash = hash & hash; // Convert to 32bit integer
        }
        return hash + 0x7FFFFFFF; // make it unsigned
    }
}

module.exports = ConsistentHashStore;