mem100x
Version:
⚡ The FASTEST MCP memory server ever built - 66k+ entities/sec with intelligent context detection
263 lines • 9.41 kB
JavaScript
"use strict";
/**
* High-Performance Counting Bloom Filter
* Supports both addition and deletion of items
* Uses counters instead of bits to track item frequency
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CountingBloomFilter = void 0;
const xxhash_wasm_1 = __importDefault(require("xxhash-wasm"));
// Initialize xxhash instance
let hashInstance = null;
class CountingBloomFilter {
size;
numHashes;
counters; // Using 8-bit counters (max 255)
itemCount = 0;
initialized = false;
constructor(expectedItems = 10000, falsePositiveRate = 0.01) {
// Calculate optimal size and hash functions
this.size = Math.ceil(-expectedItems * Math.log(falsePositiveRate) / (Math.log(2) ** 2));
this.numHashes = Math.ceil(this.size / expectedItems * Math.log(2));
this.counters = new Uint8Array(this.size);
}
// Initialize the hash functions
async init() {
if (!this.initialized) {
hashInstance = await (0, xxhash_wasm_1.default)();
this.initialized = true;
}
}
// Synchronous initialization (uses fallback hash)
initSync() {
this.initialized = true;
}
// Synchronous fallback for immediate use
murmur3(str, seed) {
let h1 = seed;
const c1 = 0xcc9e2d51;
const c2 = 0x1b873593;
const r1 = 15;
const r2 = 13;
const m = 5;
const n = 0xe6546b64;
for (let i = 0; i < str.length; i++) {
let k1 = str.charCodeAt(i);
k1 = Math.imul(k1, c1);
k1 = (k1 << r1) | (k1 >>> (32 - r1));
k1 = Math.imul(k1, c2);
h1 ^= k1;
h1 = (h1 << r2) | (h1 >>> (32 - r2));
h1 = Math.imul(h1, m) + n;
}
h1 ^= str.length;
h1 ^= h1 >>> 16;
h1 = Math.imul(h1, 0x85ebca6b);
h1 ^= h1 >>> 13;
h1 = Math.imul(h1, 0xc2b2ae35);
h1 ^= h1 >>> 16;
return h1 >>> 0;
}
add(item) {
const indices = this.getIndices(item);
for (const index of indices) {
// Increment counter, but cap at 255 to prevent overflow
if (this.counters[index] < 255) {
this.counters[index]++;
}
}
this.itemCount++;
}
remove(item) {
// First check if item might be in the filter
if (!this.contains(item)) {
return false;
}
const indices = this.getIndices(item);
// Check if we can safely decrement all counters
for (const index of indices) {
if (this.counters[index] === 0) {
return false; // Item wasn't actually in the filter
}
}
// Decrement all counters
for (const index of indices) {
this.counters[index]--;
}
this.itemCount--;
return true;
}
contains(item) {
const indices = this.getIndices(item);
for (const index of indices) {
if (this.counters[index] === 0) {
return false;
}
}
return true;
}
getIndices(item) {
const indices = [];
if (hashInstance && this.initialized) {
// Use ultra-fast xxhash when available
const hash1 = hashInstance.h32(item, 0);
const hash2 = hashInstance.h32(item, 1);
// Use double hashing technique
for (let i = 0; i < this.numHashes; i++) {
indices.push(Math.abs(hash1 + i * hash2) % this.size);
}
}
else {
// Fallback to fast synchronous hash
const hash1 = this.murmur3(item, 0);
const hash2 = this.murmur3(item, 1);
for (let i = 0; i < this.numHashes; i++) {
indices.push(Math.abs(hash1 + i * hash2) % this.size);
}
}
return indices;
}
clear() {
this.counters.fill(0);
this.itemCount = 0;
}
getStats() {
let nonZeroCounters = 0;
let saturatedCounters = 0;
let totalCount = 0;
for (const counter of this.counters) {
if (counter > 0) {
nonZeroCounters++;
totalCount += counter;
}
if (counter === 255) {
saturatedCounters++;
}
}
return {
size: this.size,
numHashes: this.numHashes,
items: this.itemCount,
fillRate: nonZeroCounters / this.size,
saturatedCounters,
averageCounter: nonZeroCounters > 0 ? totalCount / nonZeroCounters : 0
};
}
// Serialize the Counting Bloom filter to a buffer for persistence
serialize() {
// Create header with metadata
const header = Buffer.alloc(16);
header.writeUInt32BE(this.size, 0);
header.writeUInt32BE(this.numHashes, 4);
header.writeUInt32BE(this.itemCount, 8);
header.writeUInt32BE(this.counters.length, 12);
// Combine header and counters
return Buffer.concat([header, Buffer.from(this.counters)]);
}
// Deserialize a Counting Bloom filter from a buffer
static deserialize(buffer) {
// Read header
const size = buffer.readUInt32BE(0);
const numHashes = buffer.readUInt32BE(4);
const itemCount = buffer.readUInt32BE(8);
const countersLength = buffer.readUInt32BE(12);
// Create a new filter with the same parameters
const filter = Object.create(CountingBloomFilter.prototype);
filter.size = size;
filter.numHashes = numHashes;
filter.itemCount = itemCount;
filter.initialized = false;
// Copy counters
filter.counters = new Uint8Array(countersLength);
buffer.copy(filter.counters, 0, 16, 16 + countersLength);
return filter;
}
// Save to file
async saveToFile(filePath) {
const fs = await Promise.resolve().then(() => __importStar(require('fs/promises')));
const buffer = this.serialize();
await fs.writeFile(filePath, buffer);
}
// Synchronous save to file
saveToFileSync(filePath) {
const fs = require('fs');
const buffer = this.serialize();
fs.writeFileSync(filePath, buffer);
}
// Load from file
static async loadFromFile(filePath) {
const fs = await Promise.resolve().then(() => __importStar(require('fs/promises')));
try {
const buffer = await fs.readFile(filePath);
const filter = CountingBloomFilter.deserialize(buffer);
// Initialize hash functions
await filter.init();
return filter;
}
catch (error) {
// File doesn't exist or is corrupted
return null;
}
}
// Synchronous load from file
static loadFromFileSync(filePath) {
const fs = require('fs');
try {
const buffer = fs.readFileSync(filePath);
const filter = CountingBloomFilter.deserialize(buffer);
filter.initSync();
return filter;
}
catch (error) {
// File doesn't exist or is corrupted
return null;
}
}
// Migrate from regular Bloom filter
static async migrateFromBloomFilter(oldFilter, expectedItems = 10000, falsePositiveRate = 0.01) {
const newFilter = new CountingBloomFilter(expectedItems, falsePositiveRate);
await newFilter.init();
// Note: We can't directly migrate the data, but we can create a new filter
// with the same parameters
console.log('Created new Counting Bloom Filter. Previous filter data cannot be migrated.');
return newFilter;
}
}
exports.CountingBloomFilter = CountingBloomFilter;
//# sourceMappingURL=counting-bloom-filter.js.map