UNPKG

@keyv/redis

Version:

Redis storage adapter for Keyv

894 lines (892 loc) 29.8 kB
"use strict"; var __create = Object.create; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; var __getOwnPropNames = Object.getOwnPropertyNames; var __getProtoOf = Object.getPrototypeOf; var __hasOwnProp = Object.prototype.hasOwnProperty; var __export = (target, all) => { for (var name in all) __defProp(target, name, { get: all[name], enumerable: true }); }; var __copyProps = (to, from, except, desc) => { if (from && typeof from === "object" || typeof from === "function") { for (let key of __getOwnPropNames(from)) if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); } return to; }; var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( // If the importer is in node compatibility mode or this is not an ESM // file that has been converted to a CommonJS file using a Babel- // compatible transform (i.e. "__esModule" has not been set), then set // "default" to the CommonJS "module.exports" for node compatibility. isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, mod )); var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); // src/index.ts var index_exports = {}; __export(index_exports, { Keyv: () => import_keyv2.Keyv, RedisErrorMessages: () => RedisErrorMessages, createClient: () => import_client2.createClient, createCluster: () => import_client2.createCluster, createKeyv: () => createKeyv, createKeyvNonBlocking: () => createKeyvNonBlocking, createSentinel: () => import_client2.createSentinel, default: () => KeyvRedis, defaultReconnectStrategy: () => defaultReconnectStrategy }); module.exports = __toCommonJS(index_exports); var import_client = require("@redis/client"); var import_cluster_key_slot = __toESM(require("cluster-key-slot"), 1); var import_hookified = require("hookified"); var import_keyv = require("keyv"); var import_client2 = require("@redis/client"); var import_keyv2 = require("keyv"); var RedisErrorMessages = /* @__PURE__ */ ((RedisErrorMessages2) => { RedisErrorMessages2["RedisClientNotConnectedThrown"] = "Redis client is not connected or has failed to connect. This is thrown because throwOnConnectError is set to true."; return RedisErrorMessages2; })(RedisErrorMessages || {}); var defaultReconnectStrategy = (attempts) => { const backoff = Math.min(2 ** attempts * 100, 2e3); const jitter = (Math.random() - 0.5) * 100; return backoff + jitter; }; var KeyvRedis = class extends import_hookified.Hookified { _client = (0, import_client.createClient)(); _namespace; _keyPrefixSeparator = "::"; _clearBatchSize = 1e3; _useUnlink = true; _noNamespaceAffectsAll = false; _throwOnConnectError = true; _throwOnErrors = false; _connectionTimeout; /** * KeyvRedis constructor. * @param {string | RedisClientOptions | RedisClientType} [connect] How to connect to the Redis server. If string pass in the url, if object pass in the options, if RedisClient pass in the client. * @param {KeyvRedisOptions} [options] Options for the adapter such as namespace, keyPrefixSeparator, and clearBatchSize. */ constructor(connect, options) { super(); const socket = { reconnectStrategy: defaultReconnectStrategy // Default timeout for the connection }; if (connect) { if (typeof connect === "string") { this._client = (0, import_client.createClient)({ url: connect, socket }); } else if (connect.connect !== void 0) { if (this.isClientSentinel(connect)) { this._client = connect; } else if (this.isClientCluster(connect)) { this._client = connect; } else { this._client = connect; } } else if (connect instanceof Object) { if (connect.sentinelRootNodes !== void 0) { this._client = (0, import_client.createSentinel)( connect ); } else if (connect.rootNodes === void 0) { this._client = (0, import_client.createClient)( connect ); } else { this._client = (0, import_client.createCluster)(connect); } } } this.setOptions(options); this.initClient(); } /** * Get the Redis client. */ get client() { return this._client; } /** * Set the Redis client. */ set client(value) { this._client = value; this.initClient(); } /** * Get the options for the adapter. */ get opts() { let url = "redis://localhost:6379"; if (this._client.options) { const redisUrl = this._client.options?.url; if (redisUrl) { url = redisUrl; } } const results = { namespace: this._namespace, keyPrefixSeparator: this._keyPrefixSeparator, clearBatchSize: this._clearBatchSize, noNamespaceAffectsAll: this._noNamespaceAffectsAll, useUnlink: this._useUnlink, throwOnConnectError: this._throwOnConnectError, throwOnErrors: this._throwOnErrors, connectionTimeout: this._connectionTimeout, dialect: "redis", url }; return results; } /** * Set the options for the adapter. */ set opts(options) { this.setOptions(options); } /** * Get the namespace for the adapter. If undefined, it will not use a namespace including keyPrefixing. * @default undefined */ get namespace() { return this._namespace; } /** * Set the namespace for the adapter. If undefined, it will not use a namespace including keyPrefixing. */ set namespace(value) { this._namespace = value; } /** * Get the separator between the namespace and key. * @default '::' */ get keyPrefixSeparator() { return this._keyPrefixSeparator; } /** * Set the separator between the namespace and key. */ set keyPrefixSeparator(value) { this._keyPrefixSeparator = value; } /** * Get the number of keys to delete in a single batch. * @default 1000 */ get clearBatchSize() { return this._clearBatchSize; } /** * Set the number of keys to delete in a single batch. */ set clearBatchSize(value) { if (value > 0) { this._clearBatchSize = value; } else { this.emit("error", "clearBatchSize must be greater than 0"); } } /** * Get if Unlink is used instead of Del for clearing keys. This is more performant but may not be supported by all Redis versions. * @default true */ get useUnlink() { return this._useUnlink; } /** * Set if Unlink is used instead of Del for clearing keys. This is more performant but may not be supported by all Redis versions. */ set useUnlink(value) { this._useUnlink = value; } /** * Get if no namespace affects all keys. * Whether to allow clearing all keys when no namespace is set. * If set to true and no namespace is set, iterate() will return all keys. * @default false */ get noNamespaceAffectsAll() { return this._noNamespaceAffectsAll; } /** * Set if not namespace affects all keys. */ set noNamespaceAffectsAll(value) { this._noNamespaceAffectsAll = value; } /** * Get if throwOnConnectError is set to true. * This is used to throw an error if the client is not connected when trying to connect. By default, this is * set to true so that it throws an error when trying to connect to the Redis server fails. * @default true */ get throwOnConnectError() { return this._throwOnConnectError; } /** * Set if throwOnConnectError is set to true. * This is used to throw an error if the client is not connected when trying to connect. By default, this is * set to true so that it throws an error when trying to connect to the Redis server fails. */ set throwOnConnectError(value) { this._throwOnConnectError = value; } /** * Get if throwOnErrors is set to true. * This is used to throw an error if at any point there is a failure. Use this if you want to * ensure that all operations are successful and you want to handle errors. By default, this is * set to false so that it does not throw an error on every operation and instead emits an error event * and returns no-op responses. * @default false */ get throwOnErrors() { return this._throwOnErrors; } /** * Set if throwOnErrors is set to true. * This is used to throw an error if at any point there is a failure. Use this if you want to * ensure that all operations are successful and you want to handle errors. By default, this is * set to false so that it does not throw an error on every operation and instead emits an error event * and returns no-op responses. */ set throwOnErrors(value) { this._throwOnErrors = value; } /** * Get the connection timeout in milliseconds such as 5000 (5 seconds). Default is undefined. If undefined, it will use the default. * @default undefined */ get connectionTimeout() { return this._connectionTimeout; } /** * Set the connection timeout in milliseconds such as 5000 (5 seconds). Default is undefined. If undefined, it will use the default. * @default undefined */ set connectionTimeout(value) { this._connectionTimeout = value; } /** * Get the Redis URL used to connect to the server. This is used to get a connected client. */ async getClient() { if (this._client.isOpen) { return this._client; } try { if (this._connectionTimeout === void 0) { await this._client.connect(); } else { await Promise.race([ this._client.connect(), this.createTimeoutPromise(this._connectionTimeout) ]); } } catch (error) { this.emit("error", error); await this.disconnect(true); if (this._throwOnConnectError) { throw new Error("Redis client is not connected or has failed to connect. This is thrown because throwOnConnectError is set to true." /* RedisClientNotConnectedThrown */); } } this.initClient(); return this._client; } /** * Set a key value pair in the store. TTL is in milliseconds. * @param {string} key - the key to set * @param {string} value - the value to set * @param {number} [ttl] - the time to live in milliseconds */ async set(key, value, ttl) { const client = await this.getClient(); try { key = this.createKeyPrefix(key, this._namespace); if (ttl) { await client.set(key, value, { PX: ttl }); } else { await client.set(key, value); } } catch (error) { this.emit("error", error); if (this._throwOnErrors) { throw error; } } } /** * Will set many key value pairs in the store. TTL is in milliseconds. This will be done as a single transaction. * @param {KeyvEntry[]} entries - the key value pairs to set with optional ttl */ async setMany(entries) { try { if (this.isCluster()) { await this.getClient(); const slotMap = /* @__PURE__ */ new Map(); for (const entry of entries) { const prefixedKey = this.createKeyPrefix(entry.key, this._namespace); const slot = (0, import_cluster_key_slot.default)(prefixedKey); const slotEntries = slotMap.get(slot) ?? []; slotEntries.push(entry); slotMap.set(slot, slotEntries); } await Promise.all( Array.from(slotMap.entries(), async ([slot, slotEntries]) => { const client = await this.getSlotMaster(slot); const multi = client.multi(); for (const { key, value, ttl } of slotEntries) { const prefixedKey = this.createKeyPrefix(key, this._namespace); if (ttl) { multi.set(prefixedKey, value, { PX: ttl }); } else { multi.set(prefixedKey, value); } } await multi.exec(); }) ); } else { const client = await this.getClient(); const multi = client.multi(); for (const { key, value, ttl } of entries) { const prefixedKey = this.createKeyPrefix(key, this._namespace); if (ttl) { multi.set(prefixedKey, value, { PX: ttl }); } else { multi.set(prefixedKey, value); } } await multi.exec(); } } catch (error) { this.emit("error", error); if (this._throwOnConnectError && error.message === "Redis client is not connected or has failed to connect. This is thrown because throwOnConnectError is set to true." /* RedisClientNotConnectedThrown */) { throw error; } if (this._throwOnErrors) { throw error; } } } /** * Check if a key exists in the store. * @param {string} key - the key to check * @returns {Promise<boolean>} - true if the key exists, false if not */ async has(key) { const client = await this.getClient(); try { key = this.createKeyPrefix(key, this._namespace); const exists = await client.exists(key); return exists === 1; } catch (error) { this.emit("error", error); if (this._throwOnErrors) { throw error; } return false; } } /** * Check if many keys exist in the store. This will be done as a single transaction. * @param {Array<string>} keys - the keys to check * @returns {Promise<Array<boolean>>} - array of booleans for each key if it exists */ async hasMany(keys) { try { const prefixedKeys = keys.map( (key) => this.createKeyPrefix(key, this._namespace) ); if (this.isCluster()) { const slotMap = this.getSlotMap(prefixedKeys); const resultMap = /* @__PURE__ */ new Map(); await Promise.all( Array.from(slotMap.entries(), async ([slot, slotKeys]) => { const client = await this.getSlotMaster(slot); const multi = client.multi(); for (const key of slotKeys) { multi.exists(key); } const results = await multi.exec(); for (const [index, result] of results.entries()) { resultMap.set( slotKeys[index], typeof result === "number" && result === 1 ); } }) ); return prefixedKeys.map((key) => resultMap.get(key) ?? false); } else { const client = await this.getClient(); const multi = client.multi(); for (const key of prefixedKeys) { multi.exists(key); } const results = await multi.exec(); return results.map( (result) => typeof result === "number" && result === 1 ); } } catch (error) { this.emit("error", error); if (this._throwOnConnectError && error.message === "Redis client is not connected or has failed to connect. This is thrown because throwOnConnectError is set to true." /* RedisClientNotConnectedThrown */) { throw error; } if (this._throwOnErrors) { throw error; } return Array.from({ length: keys.length }).fill(false); } } /** * Get a value from the store. If the key does not exist, it will return undefined. * @param {string} key - the key to get * @returns {Promise<string | undefined>} - the value or undefined if the key does not exist */ async get(key) { const client = await this.getClient(); try { key = this.createKeyPrefix(key, this._namespace); const value = await client.get(key); if (value === null) { return void 0; } return value; } catch (error) { this.emit("error", error); if (this._throwOnErrors) { throw error; } return void 0; } } /** * Get many values from the store. If a key does not exist, it will return undefined. * @param {Array<string>} keys - the keys to get * @returns {Promise<Array<string | undefined>>} - array of values or undefined if the key does not exist */ async getMany(keys) { if (keys.length === 0) { return []; } keys = keys.map((key) => this.createKeyPrefix(key, this._namespace)); try { const values = await this.mget(keys); return values; } catch (error) { this.emit("error", error); if (this._throwOnErrors) { throw error; } return Array.from({ length: keys.length }).fill(void 0); } } /** * Delete a key from the store. * @param {string} key - the key to delete * @returns {Promise<boolean>} - true if the key was deleted, false if not */ async delete(key) { const client = await this.getClient(); try { key = this.createKeyPrefix(key, this._namespace); let deleted = 0; deleted = await (this._useUnlink ? client.unlink(key) : client.del(key)); return deleted > 0; } catch (error) { this.emit("error", error); if (this._throwOnErrors) { throw error; } return false; } } /** * Delete many keys from the store. This will be done as a single transaction. * @param {Array<string>} keys - the keys to delete * @returns {Promise<boolean>} - true if any key was deleted, false if not */ async deleteMany(keys) { let result = false; try { const prefixedKeys = keys.map( (key) => this.createKeyPrefix(key, this._namespace) ); if (this.isCluster()) { const slotMap = this.getSlotMap(prefixedKeys); await Promise.all( Array.from(slotMap.entries(), async ([slot, slotKeys]) => { const client = await this.getSlotMaster(slot); const multi = client.multi(); for (const key of slotKeys) { if (this._useUnlink) { multi.unlink(key); } else { multi.del(key); } } const results = await multi.exec(); for (const deleted of results) { if (typeof deleted === "number" && deleted > 0) { result = true; } } }) ); } else { const client = await this.getClient(); const multi = client.multi(); for (const key of prefixedKeys) { if (this._useUnlink) { multi.unlink(key); } else { multi.del(key); } } const results = await multi.exec(); for (const deleted of results) { if (typeof deleted === "number" && deleted > 0) { result = true; } } } } catch (error) { this.emit("error", error); if (this._throwOnConnectError && error.message === "Redis client is not connected or has failed to connect. This is thrown because throwOnConnectError is set to true." /* RedisClientNotConnectedThrown */) { throw error; } if (this._throwOnErrors) { throw error; } } return result; } /** * Disconnect from the Redis server. * @returns {Promise<void>} * @param {boolean} [force] - it will send a quit command if false, otherwise it will send a disconnect command to forcefully disconnect. * @see {@link https://github.com/redis/node-redis/tree/master/packages/redis#disconnecting} */ async disconnect(force) { if (this._client.isOpen) { await (force ? this._client.destroy() : this._client.close()); } } /** * Helper function to create a key with a namespace. * @param {string} key - the key to prefix * @param {string} namespace - the namespace to prefix the key with * @returns {string} - the key with the namespace such as 'namespace::key' */ createKeyPrefix(key, namespace) { if (namespace) { return `${namespace}${this._keyPrefixSeparator}${key}`; } return key; } /** * Helper function to get a key without the namespace. * @param {string} key - the key to remove the namespace from * @param {string} namespace - the namespace to remove from the key * @returns {string} - the key without the namespace such as 'key' */ getKeyWithoutPrefix(key, namespace) { if (namespace) { return key.replace(`${namespace}${this._keyPrefixSeparator}`, ""); } return key; } /** * Is the client a cluster. * @returns {boolean} - true if the client is a cluster, false if not */ isCluster() { return this.isClientCluster(this._client); } /** * Is the client a sentinel. * @returns {boolean} - true if the client is a sentinel, false if not */ isSentinel() { return this.isClientSentinel(this._client); } /** * Get the master nodes in the cluster. If not a cluster, it will return the single client. * * @returns {Promise<RedisClientType[]>} - array of master nodes */ async getMasterNodes() { if (this.isCluster()) { const cluster = await this.getClient(); const nodes = cluster.masters.map( async (main) => cluster.nodeClient(main) ); return Promise.all(nodes); } return [await this.getClient()]; } /** * Get an async iterator for the keys and values in the store. If a namespace is provided, it will only iterate over keys with that namespace. * @param {string} [namespace] - the namespace to iterate over * @returns {AsyncGenerator<[string, T | undefined], void, unknown>} - async iterator with key value pairs */ async *iterator(namespace) { const clients = await this.getMasterNodes(); for (const client of clients) { const match = namespace ? `${namespace}${this._keyPrefixSeparator}*` : "*"; let cursor = "0"; do { const result = await client.scan(cursor, { MATCH: match, TYPE: "string" }); cursor = result.cursor.toString(); let { keys } = result; if (!namespace && !this._noNamespaceAffectsAll) { keys = keys.filter((key) => !key.includes(this._keyPrefixSeparator)); } if (keys.length > 0) { const values = await this.mget(keys); for (const i of keys.keys()) { const key = this.getKeyWithoutPrefix(keys[i], namespace); const value = values[i]; yield [key, value]; } } } while (cursor !== "0"); } } /** * Clear all keys in the store. * IMPORTANT: this can cause performance issues if there are a large number of keys in the store and worse with clusters. Use with caution as not recommended for production. * If a namespace is not set it will clear all keys with no prefix. * If a namespace is set it will clear all keys with that namespace. * @returns {Promise<void>} */ async clear() { try { const clients = await this.getMasterNodes(); await Promise.all( clients.map(async (client) => { if (!this._namespace && this._noNamespaceAffectsAll) { await client.flushDb(); return; } let cursor = "0"; const batchSize = this._clearBatchSize; const match = this._namespace ? `${this._namespace}${this._keyPrefixSeparator}*` : "*"; const deletePromises = []; do { const result = await client.scan(cursor, { MATCH: match, COUNT: batchSize, TYPE: "string" }); cursor = result.cursor.toString(); let { keys } = result; if (keys.length === 0) { continue; } if (!this._namespace) { keys = keys.filter( (key) => !key.includes(this._keyPrefixSeparator) ); } deletePromises.push(this.clearWithClusterSupport(keys)); } while (cursor !== "0"); await Promise.all(deletePromises); }) ); } catch (error) { this.emit("error", error); } } /** * Get many keys. If the instance is a cluster, it will do multiple MGET calls * by separating the keys by slot to solve the CROSS-SLOT restriction. */ async mget(keys) { const valueMap = /* @__PURE__ */ new Map(); if (this.isCluster()) { const slotMap = this.getSlotMap(keys); await Promise.all( Array.from(slotMap.entries(), async ([slot, slotKeys]) => { const client = await this.getSlotMaster(slot); const values = await client.mGet(slotKeys); for (const [index, value] of values.entries()) { valueMap.set(slotKeys[index], value ?? void 0); } }) ); } else { const client = await this.getClient(); const values = await client.mGet(keys); for (const [index, value] of values.entries()) { valueMap.set(keys[index], value ?? void 0); } } return keys.map((key) => valueMap.get(key)); } /** * Clear all keys in the store with a specific namespace. If the instance is a cluster, it will clear all keys * by separating the keys by slot to solve the CROSS-SLOT restriction. */ async clearWithClusterSupport(keys) { if (keys.length > 0) { const slotMap = this.getSlotMap(keys); await Promise.all( Array.from(slotMap.entries(), async ([slot, keys2]) => { const client = await this.getSlotMaster(slot); return this._useUnlink ? client.unlink(keys2) : client.del(keys2); }) ); } } /** * Returns the master node client for a given slot or the instance's client if it's not a cluster. */ async getSlotMaster(slot) { const connection = await this.getClient(); if (this.isCluster()) { const cluster = connection; const mainNode = cluster.slots[slot].master; return cluster.nodeClient(mainNode); } return connection; } /** * Group keys by their slot. * * @param {string[]} keys - the keys to group * @returns {Map<number, string[]>} - map of slot to keys */ getSlotMap(keys) { const slotMap = /* @__PURE__ */ new Map(); if (this.isCluster()) { for (const key of keys) { const slot = (0, import_cluster_key_slot.default)(key); const slotKeys = slotMap.get(slot) ?? []; slotKeys.push(key); slotMap.set(slot, slotKeys); } } else { slotMap.set(0, keys); } return slotMap; } isClientCluster(client) { return client.slots !== void 0; } isClientSentinel(client) { return client.getSentinelNode !== void 0; } setOptions(options) { if (!options) { return; } if (options.namespace) { this._namespace = options.namespace; } if (options.keyPrefixSeparator !== void 0) { this._keyPrefixSeparator = options.keyPrefixSeparator; } if (options.clearBatchSize !== void 0 && options.clearBatchSize > 0) { this._clearBatchSize = options.clearBatchSize; } if (options.useUnlink !== void 0) { this._useUnlink = options.useUnlink; } if (options.noNamespaceAffectsAll !== void 0) { this._noNamespaceAffectsAll = options.noNamespaceAffectsAll; } if (options.throwOnConnectError !== void 0) { this._throwOnConnectError = options.throwOnConnectError; } if (options.throwOnErrors !== void 0) { this._throwOnErrors = options.throwOnErrors; } if (options.connectionTimeout !== void 0) { this._connectionTimeout = options.connectionTimeout; } } initClient() { this._client.on("connect", () => { this.emit("connect", this._client); }); this._client.on("disconnect", () => { this.emit("disconnect", this._client); }); this._client.on("reconnecting", (reconnectInfo) => { this.emit("reconnecting", reconnectInfo); }); } async createTimeoutPromise(timeoutMs) { return new Promise( (_, reject) => setTimeout(() => { reject(new Error(`Redis timed out after ${timeoutMs}ms`)); }, timeoutMs) ); } }; function createKeyv(connect, options) { connect ??= "redis://localhost:6379"; const adapter = new KeyvRedis(connect, options); if (options?.namespace) { adapter.namespace = options.namespace; const keyv2 = new import_keyv.Keyv(adapter, { namespace: options?.namespace, useKeyPrefix: false }); if (options?.throwOnConnectError) { keyv2.throwOnErrors = true; } if (options?.throwOnErrors) { keyv2.throwOnErrors = true; } return keyv2; } const keyv = new import_keyv.Keyv(adapter, { useKeyPrefix: false }); if (options?.throwOnConnectError) { keyv.throwOnErrors = true; } if (options?.throwOnErrors) { keyv.throwOnErrors = true; } keyv.namespace = void 0; return keyv; } function createKeyvNonBlocking(connect, options) { const keyv = createKeyv(connect, options); const keyvStore = keyv.store; keyvStore.throwOnConnectError = false; keyvStore.throwOnErrors = false; const redisClient = keyvStore.client; if (redisClient.options) { redisClient.options.disableOfflineQueue = true; if (redisClient.options.socket) { redisClient.options.socket.reconnectStrategy = false; } } keyv.throwOnErrors = false; return keyv; } // Annotate the CommonJS export names for ESM import in node: 0 && (module.exports = { Keyv, RedisErrorMessages, createClient, createCluster, createKeyv, createKeyvNonBlocking, createSentinel, defaultReconnectStrategy });