UNPKG

plug-n-play-ws

Version:

A plug-and-play WebSocket layer on top of Socket.IO with full TypeScript support, zero manual wiring, and production-ready features

822 lines (816 loc) 26.5 kB
import Redis from 'ioredis'; // src/utils/text-processing.ts function buildNGrams(text, n) { const normalized = text.toLowerCase().replace(/[^\w\s]/g, " ").replace(/\s+/g, " ").trim(); const words = normalized.split(" "); const ngrams = []; for (const word of words) { if (word.length >= n) { for (let i = 0; i <= word.length - n; i++) { ngrams.push(word.substring(i, i + n)); } } } return [...new Set(ngrams)]; } function buildEdgeGrams(text, minGram, maxGram) { if (minGram > maxGram) { return []; } const normalized = text.toLowerCase().replace(/[^\w\s]/g, " ").replace(/\s+/g, " ").trim(); const words = normalized.split(" "); const edgegrams = []; for (const word of words) { for (let len = minGram; len <= Math.min(maxGram, word.length); len++) { edgegrams.push(word.substring(0, len)); } } return [...new Set(edgegrams)]; } function generateHighlights(content, searchTerms, maxHighlights = 3, contextLength = 30) { const highlights = []; const contentLower = content.toLowerCase(); for (const term of searchTerms) { const termLower = term.toLowerCase(); let index = contentLower.indexOf(termLower); while (index !== -1 && highlights.length < maxHighlights) { const start = Math.max(0, index - contextLength); const end = Math.min(content.length, index + term.length + contextLength); const snippet = content.substring(start, end); const highlightedSnippet = snippet.replace( new RegExp(term, "gi"), `<mark>$&</mark>` ); highlights.push( (start > 0 ? "..." : "") + highlightedSnippet + (end < content.length ? "..." : "") ); index = contentLower.indexOf(termLower, index + 1); } } return highlights; } // src/adapters/base-search.ts var DEFAULT_SEARCH_CONFIG = { ngramSize: 3, minEdgegram: 2, maxEdgegram: 10, exactMatchBoost: 100, ngramWeight: 0.5, edgegramWeight: 1, minScore: 0.1 }; var BaseSearchAdapter = class { searchConfig; constructor(searchConfig = {}) { this.searchConfig = { ...DEFAULT_SEARCH_CONFIG, ...searchConfig }; } /** * Generate search terms for indexing a document */ generateSearchTerms(content) { const ngrams = buildNGrams(content, this.searchConfig.ngramSize); const edgegrams = buildEdgeGrams( content, this.searchConfig.minEdgegram, this.searchConfig.maxEdgegram ); return { ngrams, edgegrams }; } /** * Calculate relevance score for a document based on search terms */ calculateRelevanceScore(documentContent, searchTerms, ngramMatches, edgegramMatches) { let score = 0; const contentLower = documentContent.toLowerCase(); const words = contentLower.split(/\s+/); for (const term of searchTerms) { if (words.includes(term.toLowerCase())) { score += this.searchConfig.exactMatchBoost; } } score += ngramMatches * this.searchConfig.ngramWeight; score += edgegramMatches * this.searchConfig.edgegramWeight; return score; } /** * Process search results: score, sort, and paginate */ processSearchResults(query, searchTerms, documentScores, startTime) { const filteredResults = []; for (const [docId, { score, data }] of documentScores.entries()) { if (score < this.searchConfig.minScore) continue; const highlights = generateHighlights(data.content, searchTerms); filteredResults.push({ id: docId, score, data: { content: data.content, ...data.metadata }, highlights }); } filteredResults.sort((a, b) => b.score - a.score); const offset = query.offset || 0; const limit = query.limit || 10; const paginatedResults = filteredResults.slice(offset, offset + limit); return { query: query.query, results: paginatedResults, total: filteredResults.length, took: Date.now() - startTime, hasMore: offset + limit < filteredResults.length }; } /** * Validate and normalize search query */ normalizeSearchQuery(query) { const searchTerms = query.query.toLowerCase().split(/\s+/).filter((term) => term.length > 0); return { searchTerms, isValid: searchTerms.length > 0 }; } /** * Create empty search response for invalid queries */ createEmptyResponse(query, startTime) { return { query: query.query, results: [], total: 0, took: Date.now() - startTime, hasMore: false }; } }; // src/adapters/memory.ts var MemoryAdapter = class extends BaseSearchAdapter { sessions = /* @__PURE__ */ new Map(); documents = /* @__PURE__ */ new Map(); documentAccessOrder = /* @__PURE__ */ new Map(); // Track access order for LRU ngramIndex = /* @__PURE__ */ new Map(); // ngram -> document IDs edgegramIndex = /* @__PURE__ */ new Map(); // edgegram -> document IDs maxDocuments; sessionCleanupHours; accessCounter = 0; // Monotonic counter for access order constructor(config = {}) { super(config.searchConfig); this.maxDocuments = config.maxDocuments || 1e4; this.sessionCleanupHours = config.sessionCleanupInterval || 24; } async setSession(sessionId, metadata) { this.sessions.set(sessionId, { ...metadata }); } async getSession(sessionId) { const session = this.sessions.get(sessionId); return session ? { ...session } : null; } async deleteSession(sessionId) { this.sessions.delete(sessionId); } async getAllSessions() { return Array.from(this.sessions.values()).map((session) => ({ ...session })); } async updateLastSeen(sessionId) { const session = this.sessions.get(sessionId); if (session) { session.lastSeenAt = /* @__PURE__ */ new Date(); } } async indexDocument(id, content, metadata) { await this.removeFromIndex(id); while (this.documents.size >= this.maxDocuments) { const lruDocumentId = this.findLRUDocument(); if (lruDocumentId) { await this.removeDocument(lruDocumentId); } else { break; } } this.documents.set(id, { content, ...metadata && { metadata } }); this.documentAccessOrder.set(id, ++this.accessCounter); const { ngrams, edgegrams } = this.generateSearchTerms(content); for (const ngram of ngrams) { if (!this.ngramIndex.has(ngram)) { this.ngramIndex.set(ngram, /* @__PURE__ */ new Set()); } this.ngramIndex.get(ngram).add(id); } for (const edgegram of edgegrams) { if (!this.edgegramIndex.has(edgegram)) { this.edgegramIndex.set(edgegram, /* @__PURE__ */ new Set()); } this.edgegramIndex.get(edgegram).add(id); } } /** * Find the least recently used document for eviction */ findLRUDocument() { let lruId; let lruAccessTime = Infinity; for (const [docId, accessTime] of this.documentAccessOrder.entries()) { if (accessTime < lruAccessTime) { lruAccessTime = accessTime; lruId = docId; } } return lruId; } /** * Update access time for a document (for LRU tracking) */ updateDocumentAccess(id) { if (this.documents.has(id)) { this.documentAccessOrder.set(id, ++this.accessCounter); } } async removeDocument(id) { this.documents.delete(id); this.documentAccessOrder.delete(id); await this.removeFromIndex(id); } async removeFromIndex(id) { for (const [ngram, docIds] of this.ngramIndex.entries()) { docIds.delete(id); if (docIds.size === 0) { this.ngramIndex.delete(ngram); } } for (const [edgegram, docIds] of this.edgegramIndex.entries()) { docIds.delete(id); if (docIds.size === 0) { this.edgegramIndex.delete(edgegram); } } } async search(query) { const startTime = Date.now(); const { searchTerms, isValid } = this.normalizeSearchQuery(query); if (!isValid) { return this.createEmptyResponse(query, startTime); } const documentScores = /* @__PURE__ */ new Map(); for (const term of searchTerms) { const { ngrams, edgegrams } = this.generateSearchTerms(term); for (const ngram of ngrams) { const matchingDocs = this.ngramIndex.get(ngram); if (matchingDocs) { for (const docId of matchingDocs) { if (!documentScores.has(docId)) { const doc = this.documents.get(docId); if (doc) { documentScores.set(docId, { score: 0, data: doc }); } } const current = documentScores.get(docId); if (current) { current.score += this.searchConfig.ngramWeight; } } } } for (const edgegram of edgegrams) { const matchingDocs = this.edgegramIndex.get(edgegram); if (matchingDocs) { for (const docId of matchingDocs) { if (!documentScores.has(docId)) { const doc = this.documents.get(docId); if (doc) { documentScores.set(docId, { score: 0, data: doc }); } } const current = documentScores.get(docId); if (current) { current.score += this.searchConfig.edgegramWeight; } } } } } for (const [docId, scoreData] of documentScores.entries()) { this.updateDocumentAccess(docId); const finalScore = this.calculateRelevanceScore( scoreData.data.content, searchTerms, scoreData.score, // Use actual calculated match score 0 // No additional boost for memory adapter ); documentScores.set(docId, { ...scoreData, score: finalScore }); } return this.processSearchResults(query, searchTerms, documentScores, startTime); } async cleanup() { const cutoffTime = new Date(Date.now() - this.sessionCleanupHours * 60 * 60 * 1e3); for (const [sessionId, session] of this.sessions.entries()) { if (session.lastSeenAt < cutoffTime) { this.sessions.delete(sessionId); } } } async disconnect() { this.sessions.clear(); this.documents.clear(); this.documentAccessOrder.clear(); this.ngramIndex.clear(); this.edgegramIndex.clear(); this.accessCounter = 0; } }; // src/adapters/redis.ts var UnifiedRedisAdapter = class extends BaseSearchAdapter { redis; keyPrefix; logger; ttl; constructor(config) { super(config.searchConfig); this.redis = config.redis; this.keyPrefix = config.keyPrefix || "pnp-ws:"; this.logger = config.logger; this.ttl = { session: 24 * 60 * 60, // 24 hours document: 7 * 24 * 60 * 60, // 7 days index: 7 * 24 * 60 * 60, // 7 days ...config.ttl }; } getKey(type, id) { return `${this.keyPrefix}${type}:${id}`; } // Session Management async setSession(sessionId, metadata) { const key = this.getKey("session", sessionId); const flatData = []; flatData.push("id", metadata.id); flatData.push("connectedAt", metadata.connectedAt.toISOString()); flatData.push("lastSeenAt", metadata.lastSeenAt.toISOString()); if (metadata.userId) flatData.push("userId", metadata.userId); if (metadata.tabId) flatData.push("tabId", metadata.tabId); if (metadata.userAgent) flatData.push("userAgent", metadata.userAgent); if (metadata.ip) flatData.push("ip", metadata.ip); if (metadata.metadata) flatData.push("metadata", JSON.stringify(metadata.metadata)); await this.redis.pipeline([ ["HSET", key, ...flatData], ["EXPIRE", key, this.ttl.session.toString()], ["SADD", this.getKey("sessions", "active"), sessionId] ]); } async getSession(sessionId) { const key = this.getKey("session", sessionId); const data = await this.redis.hgetall(key); if (!data || Object.keys(data).length === 0 || !data.id) { return null; } return { id: data.id, ...data.userId && { userId: data.userId }, ...data.tabId && { tabId: data.tabId }, ...data.userAgent && { userAgent: data.userAgent }, ...data.ip && { ip: data.ip }, connectedAt: new Date(data.connectedAt || Date.now()), lastSeenAt: new Date(data.lastSeenAt || Date.now()), ...data.metadata && { metadata: JSON.parse(data.metadata) } }; } async deleteSession(sessionId) { const key = this.getKey("session", sessionId); await this.redis.pipeline([ ["DEL", key], ["SREM", this.getKey("sessions", "active"), sessionId] ]); } async getAllSessions() { const sessionIds = await this.redis.smembers(this.getKey("sessions", "active")); const sessions = []; for (const sessionId of sessionIds) { const session = await this.getSession(sessionId); if (session) { sessions.push(session); } else { await this.redis.srem(this.getKey("sessions", "active"), sessionId); } } return sessions; } async updateLastSeen(sessionId) { const key = this.getKey("session", sessionId); await this.redis.pipeline([ ["HSET", key, "lastSeenAt", (/* @__PURE__ */ new Date()).toISOString()], ["EXPIRE", key, this.ttl.session.toString()] ]); } // Document Indexing async indexDocument(id, content, metadata) { await this.removeDocument(id); const docKey = this.getKey("doc", id); const { ngrams, edgegrams } = this.generateSearchTerms(content); const allTerms = [...ngrams, ...edgegrams]; const docData = [ "content", content, "indexedAt", (/* @__PURE__ */ new Date()).toISOString(), "terms", JSON.stringify(allTerms) // Store terms for efficient removal ]; if (metadata) { docData.push("metadata", JSON.stringify(metadata)); } const commands = [ ["HSET", docKey, ...docData], ["EXPIRE", docKey, this.ttl.document.toString()], ["SADD", this.getKey("docs", "all"), id] ]; for (const ngram of ngrams) { const ngramKey = this.getKey("ngram", ngram); commands.push(["SADD", ngramKey, id]); commands.push(["EXPIRE", ngramKey, this.ttl.index.toString()]); } for (const edgegram of edgegrams) { const edgegramKey = this.getKey("edgegram", edgegram); commands.push(["SADD", edgegramKey, id]); commands.push(["EXPIRE", edgegramKey, this.ttl.index.toString()]); } await this.redis.pipeline(commands); } async removeDocument(id) { const docKey = this.getKey("doc", id); const doc = await this.redis.hgetall(docKey); if (doc && doc.terms) { try { const terms = JSON.parse(doc.terms); const commands = []; for (const term of terms) { const ngramKey = this.getKey("ngram", term); const edgegramKey = this.getKey("edgegram", term); commands.push(["SREM", ngramKey, id]); commands.push(["SREM", edgegramKey, id]); } commands.push(["DEL", docKey]); commands.push(["SREM", this.getKey("docs", "all"), id]); await this.redis.pipeline(commands); } catch (error) { this.logger?.warn?.("Failed to parse indexed terms, using fallback removal", { id, error }); await this.removeDocumentFromIndexesFallback(id); } } else { await this.redis.pipeline([ ["DEL", docKey], ["SREM", this.getKey("docs", "all"), id] ]); } } async removeDocumentFromIndexesFallback(id) { const [ngramKeys, edgegramKeys] = await Promise.all([ this.redis.keys(this.getKey("ngram", "*")), this.redis.keys(this.getKey("edgegram", "*")) ]); const commands = []; for (const key of [...ngramKeys, ...edgegramKeys]) { commands.push(["SREM", key, id]); } commands.push(["DEL", this.getKey("doc", id)]); commands.push(["SREM", this.getKey("docs", "all"), id]); if (commands.length > 0) { await this.redis.pipeline(commands); } } // Search Implementation async search(query) { const startTime = Date.now(); const searchTimeout = 1e4; const timeoutPromise = new Promise((_, reject) => { setTimeout(() => reject(new Error("Search timeout")), searchTimeout); }); try { const searchPromise = this.performSearch(query, startTime); return await Promise.race([searchPromise, timeoutPromise]); } catch (error) { if (error instanceof Error && error.message === "Search timeout") { this.logger?.error?.("Search timeout exceeded", { query: query.query, timeout: searchTimeout }); } throw error; } } async performSearch(query, startTime) { const { searchTerms, isValid } = this.normalizeSearchQuery(query); if (!isValid) { return this.createEmptyResponse(query, startTime); } const documentScores = /* @__PURE__ */ new Map(); for (const term of searchTerms) { const { ngrams, edgegrams } = this.generateSearchTerms(term); const ngramCommands = ngrams.map( (ngram) => ["SMEMBERS", this.getKey("ngram", ngram)] ); const edgegramCommands = edgegrams.map( (edgegram) => ["SMEMBERS", this.getKey("edgegram", edgegram)] ); const [ngramResults, edgegramResults] = await Promise.all([ this.redis.pipeline(ngramCommands), this.redis.pipeline(edgegramCommands) ]); ngramResults.forEach((matchingDocs) => { for (const docId of matchingDocs) { if (!documentScores.has(docId)) { documentScores.set(docId, { score: 0, data: { content: "" } }); } documentScores.get(docId).score += this.searchConfig.ngramWeight; } }); edgegramResults.forEach((matchingDocs, index2) => { const edgegram = edgegrams[index2]; if (!edgegram) return; const boost = edgegram.length / this.searchConfig.maxEdgegram; for (const docId of matchingDocs) { if (!documentScores.has(docId)) { documentScores.set(docId, { score: 0, data: { content: "" } }); } documentScores.get(docId).score += boost * this.searchConfig.edgegramWeight; } }); } const docCommands = Array.from(documentScores.keys()).map( (docId) => ["HGETALL", this.getKey("doc", docId)] ); if (docCommands.length === 0) { return this.createEmptyResponse(query, startTime); } const docResults = await this.redis.pipeline(docCommands); let index = 0; for (const [docId, scoreData] of documentScores.entries()) { const docData = docResults[index]; index++; if (docData && docData.content) { const documentData = { content: docData.content, ...docData.metadata && { metadata: JSON.parse(docData.metadata) } }; const finalScore = this.calculateRelevanceScore( docData.content, searchTerms, 0, // n-gram matches already counted 0 // edge-gram matches already counted ) + scoreData.score; documentScores.set(docId, { score: finalScore, data: documentData }); } else { documentScores.delete(docId); } } return this.processSearchResults(query, searchTerms, documentScores, startTime); } // Cleanup and Maintenance async cleanup() { const sessionIds = await this.redis.smembers(this.getKey("sessions", "active")); for (const sessionId of sessionIds) { const key = this.getKey("session", sessionId); const exists = await this.redis.exists(key); if (!exists) { await this.redis.srem(this.getKey("sessions", "active"), sessionId); } } } async disconnect() { await this.redis.disconnect(); } }; // src/adapters/redis-clients.ts var IoRedisAdapter = class { constructor(redis) { this.redis = redis; } async hset(key, ...args) { if (args.length === 2 && args[0] && args[1]) { await this.redis.hset(key, args[0], args[1]); } else { await this.redis.hset(key, ...args); } } async hgetall(key) { return this.redis.hgetall(key); } async del(key) { await this.redis.del(key); } async exists(key) { return this.redis.exists(key); } async expire(key, seconds) { await this.redis.expire(key, seconds); } async sadd(key, member) { await this.redis.sadd(key, member); } async smembers(key) { return this.redis.smembers(key); } async srem(key, member) { await this.redis.srem(key, member); } async pipeline(commands) { const pipeline = this.redis.pipeline(); for (const [command, ...args] of commands) { const pipelineMethod = pipeline[command.toLowerCase()]; if (pipelineMethod) { pipelineMethod.call(pipeline, ...args); } } const results = await pipeline.exec(); const processedResults = []; if (results) { for (let i = 0; i < results.length; i++) { const result = results[i]; if (!result) continue; const [error, value] = result; if (error) { let commandInfo = "unknown"; if (i < commands.length) { const cmd = commands[i]; if (cmd && cmd.length > 0) { commandInfo = cmd[0]; } } throw new Error(`Pipeline command ${i} (${commandInfo}) failed: ${error.message}`); } processedResults.push(value); } } return processedResults; } async keys(pattern) { return this.redis.keys(pattern); } async disconnect() { await this.redis.quit(); } }; var UpstashRedisAdapter = class { baseUrl; token; constructor(config) { this.baseUrl = config.url.endsWith("/") ? config.url.slice(0, -1) : config.url; this.token = config.token; } async request(command) { const response = await fetch(`${this.baseUrl}`, { method: "POST", headers: { "Authorization": `Bearer ${this.token}`, "Content-Type": "application/json" }, body: JSON.stringify(command) }); if (!response.ok) { throw new Error(`Upstash Redis request failed: ${response.statusText}`); } const data = await response.json(); if (data.error) { throw new Error(`Upstash Redis error: ${data.error}`); } return data.result; } async hset(key, ...args) { await this.request(["HSET", key, ...args]); } async hgetall(key) { return await this.request(["HGETALL", key]); } async del(key) { await this.request(["DEL", key]); } async exists(key) { return await this.request(["EXISTS", key]); } async expire(key, seconds) { await this.request(["EXPIRE", key, seconds.toString()]); } async sadd(key, member) { await this.request(["SADD", key, member]); } async smembers(key) { return await this.request(["SMEMBERS", key]); } async srem(key, member) { await this.request(["SREM", key, member]); } async pipeline(commands) { const response = await fetch(`${this.baseUrl}/pipeline`, { method: "POST", headers: { "Authorization": `Bearer ${this.token}`, "Content-Type": "application/json" }, body: JSON.stringify(commands) }); if (!response.ok) { throw new Error(`Upstash Redis pipeline request failed: ${response.statusText}`); } const data = await response.json(); return data.map((item) => { if (item.error) { throw new Error(`Upstash Redis error: ${item.error}`); } return item.result; }); } async keys(pattern) { return await this.request(["KEYS", pattern]); } async disconnect() { } }; function buildAdapterConfig(redis, config) { const adapterConfig = { redis }; if (config.keyPrefix) { adapterConfig.keyPrefix = config.keyPrefix; } if (config.searchConfig) { adapterConfig.searchConfig = config.searchConfig; } if (config.ttl) { adapterConfig.ttl = { session: config.ttl.session ?? 24 * 60 * 60, document: config.ttl.document ?? 7 * 24 * 60 * 60, index: config.ttl.index ?? 7 * 24 * 60 * 60 }; } return adapterConfig; } function createRedisAdapter(config) { let redis; if (config.url) { redis = new Redis(config.url); } else { const redisConfig = { host: config.host || "localhost", port: config.port || 6379, db: config.db || 0, maxRetriesPerRequest: 3, lazyConnect: true }; if (config.password) { redisConfig.password = config.password; } redis = new Redis(redisConfig); } const unifiedRedis = new IoRedisAdapter(redis); const adapterConfig = buildAdapterConfig(unifiedRedis, config); return new UnifiedRedisAdapter(adapterConfig); } function createUpstashRedisAdapter(config) { const upstashRedis = new UpstashRedisAdapter({ url: config.url, token: config.token }); const adapterConfig = buildAdapterConfig(upstashRedis, config); return new UnifiedRedisAdapter(adapterConfig); } function createRedisAdapterFromEnv(keyPrefix, searchConfig) { const upstashUrl = process.env.UPSTASH_REDIS_REST_URL; const upstashToken = process.env.UPSTASH_REDIS_REST_TOKEN; if (upstashUrl && upstashToken) { const config2 = { url: upstashUrl, token: upstashToken, ...keyPrefix && { keyPrefix }, ...searchConfig && { searchConfig } }; return createUpstashRedisAdapter(config2); } const redisUrl = process.env.REDIS_URL; if (redisUrl) { const config2 = { url: redisUrl, ...keyPrefix && { keyPrefix }, ...searchConfig && { searchConfig } }; return createRedisAdapter(config2); } const config = { host: process.env.REDIS_HOST || "localhost", port: parseInt(process.env.REDIS_PORT || "6379"), db: parseInt(process.env.REDIS_DB || "0"), ...process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD }, ...keyPrefix && { keyPrefix }, ...searchConfig && { searchConfig } }; return createRedisAdapter(config); } export { BaseSearchAdapter, DEFAULT_SEARCH_CONFIG, IoRedisAdapter, MemoryAdapter, UnifiedRedisAdapter, UpstashRedisAdapter as UpstashRedisClient, buildEdgeGrams, buildNGrams, createRedisAdapter, createRedisAdapterFromEnv, createUpstashRedisAdapter, generateHighlights }; //# sourceMappingURL=index.mjs.map //# sourceMappingURL=index.mjs.map