@imqueue/pg-cache/src/PgCache.js

PostgreSQL managed cache on Redis for @imqueue-based service methods

/*!
 * I'm Queue Software Project
 * Copyright (C) 2025  imqueue.com <support@imqueue.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * If you want to use this code in a closed source (commercial) project, you can
 * purchase a proprietary commercial license. Please contact us at
 * <support@imqueue.com> to get commercial licensing options.
 */
import { IMQService, RedisCache, } from '@imqueue/rpc';
import { signature } from './signature.js';
import { TagCache } from '@imqueue/tag-cache';
import { PgPubSub } from '@imqueue/pg-pubsub';
import { Client } from 'pg';
import { awaitInvalidation, DEFAULT_INVALIDATION_TIMEOUT, PG_CACHE_DEBUG, PG_CACHE_TRIGGER, } from './env.js';
const RX_TRIGGER = new RegExp('create\\s+(or\\s+replace)?function\\s+' +
    'post_change_notify_trigger\\s+\\([^)]*\\).*?returns\\s+trigger', 'i');
/**
 * Checks if a given definition valid. If not - will return default trigger
 * definition.
 *
 * @see {@link PG_CACHE_TRIGGER}
 */
function triggerDef(definition) {
    if (!RX_TRIGGER.test(definition + '')) {
        return PG_CACHE_TRIGGER;
    }
    return definition;
}
/**
 * Installs database triggers
 *
 */
async function install(channels, pg, triggerDefinition, logger) {
    try {
        await pg.query(triggerDefinition);
    }
    catch (err) {
        if (PG_CACHE_DEBUG) {
            logger.info('PgCache: create trigger function errored:', err);
        }
    }
    await Promise.all(channels.map(async (channel) => {
        try {
            await pg.query(`CREATE TRIGGER "post_change_notify"
                    AFTER INSERT OR UPDATE OR DELETE
                    ON "${channel}"
                    FOR EACH ROW
                EXECUTE PROCEDURE post_change_notify_trigger()`);
            if (PG_CACHE_DEBUG) {
                logger.info(`PgCache: trigger created on ${channel}!`);
            }
        }
        catch (err) {
            // 42P01 (undefined_table) means the channel does not name a
            // real table, so no NOTIFY will ever be emitted for it - a
            // silent dead end. Every other failure here is expected on a
            // warm database (42723: trigger already exists).
            if (err?.code === '42P01') {
                logger.warn(`PgCache: channel "${channel}" is not an existing ` +
                    'table - no trigger installed, this channel will ' +
                    'never fire');
            }
            else if (PG_CACHE_DEBUG) {
                logger.info(`PgCache: create trigger on ${channel} errored:`, err);
            }
        }
    }));
}
/**
 * The row-level operation that produced a change notification. Matches the
 * PostgreSQL trigger's `TG_OP`.
 */
export var ChannelOperation;
(function (ChannelOperation) {
    // noinspection JSUnusedGlobalSymbols
    /** A row was inserted. */
    ChannelOperation["INSERT"] = "INSERT";
    /** A row was updated. */
    ChannelOperation["UPDATE"] = "UPDATE";
    /** A row was deleted. */
    ChannelOperation["DELETE"] = "DELETE";
})(ChannelOperation || (ChannelOperation = {}));
function needInvalidate(payload, filter) {
    if (Array.isArray(filter)) {
        return !~filter.indexOf(payload.operation);
    }
    else if (typeof filter === 'function') {
        payload.timestamp = new Date(payload.timestamp);
        return !!filter(payload);
    }
    return true;
}
function publish(self, channel, payload, tag) {
    if (typeof self.publish !== 'function') {
        if (PG_CACHE_DEBUG) {
            self.logger.info(`PgCache: publish method does not exist on ${self.constructor.name}`);
        }
        return;
    }
    self
        .publish({ channel, payload, tag })
        .then((result) => {
        if (PG_CACHE_DEBUG) {
            self.logger.info(`PgCache: tag '${tag}' published to client with:`, channel);
        }
        return result;
    })
        .catch((err) => self.logger.warn(`PgCache: error publishing '${tag}':`, err));
}
function invalidate(self, tag) {
    self.taggedCache
        .invalidate(tag)
        .then((result) => {
        if (PG_CACHE_DEBUG) {
            self.logger.info(`PgCache: key '${tag}' invalidated!`);
        }
        return result;
    })
        .catch((err) => self.logger.warn(`PgCache: error invalidating '${tag}':`, err));
}
// noinspection JSUnusedGlobalSymbols
/**
 * Class decorator turning an `@imqueue` service into a PostgreSQL-invalidated
 * cache: method results are memoised in redis, and PostgreSQL itself tells the
 * service when to drop them.
 *
 * It installs a change-notify trigger on every table the service's
 * {@link cacheWith} and {@link cacheBy} decorators declare a dependency on, and
 * subscribes to one LISTEN/NOTIFY channel per table. When a row changes, the
 * matching cached results are invalidated by tag — so a cache entry lives exactly
 * as long as the data behind it is unchanged, rather than for a guessed TTL.
 *
 * ```typescript
 * import { PgCache, cacheWith } from '@imqueue/pg-cache';
 *
 * @PgCache({
 *     postgres: process.env.DB_URL!,
 *     redis: { host: 'localhost', port: 6379 },
 * })
 * class UserService extends IMQService {
 *     @cacheWith({ channels: ['users'] })
 *     public async list(): Promise<User[]> { ... }
 * }
 * ```
 *
 * Applied to the class, it wraps `start()`: the subscription and the triggers are
 * established there, after any existing `start()` implementation has run. So the
 * cache is inert until the service is started, and a service that never calls
 * `start()` is never cached.
 *
 * Awaiting `start()` is enough — it does not resolve until the triggers exist and
 * the channels are subscribed, so a row changed immediately afterwards cannot go
 * unnoticed. That costs a few tens of milliseconds at boot. If the setup fails,
 * or is not confirmed within {@link PgCacheOptions.invalidationTimeout}, the
 * service still caches and reports the failure loudly; pass
 * {@link PgCacheOptions.requireInvalidation} to have it run uncached instead.
 *
 * Works both as a standard (TC39) decorator and as a legacy
 * (`experimentalDecorators`) one, matching `@imqueue/rpc`, so it can be applied in
 * either compilation mode.
 *
 * Redis is resolved in order: `options.redisCache`, then `options.redis`, then a
 * `cache` property already on the service. If none is available `start()` throws.
 *
 * @param options - PostgreSQL and redis connection details, plus the cache-key
 *                  prefix, publication and trigger-definition overrides
 * @returns the class decorator to apply, which augments the class with
 *          {@link PgCacheable}
 */
export function PgCache(options) {
    // Dual-mode: standard (TC39) class decorators pass (value, context); legacy
    // ones pass just the constructor. In both cases the first argument is the
    // class, and the body augments its prototype in place. In standard mode the
    // per-method channel metadata is registered by initializers at construction
    // time, so it is read at runtime (see start(): `this.pgCacheChannels`)
    // rather than captured here at decoration time.
    return ((constructor, _context) => {
        const init = constructor.prototype.start;
        const pgCacheChannels = constructor.prototype.pgCacheChannels;
        class CachedService {
            taggedCache;
            pgCacheChannels;
            pubSub;
            async start(...args) {
                this.pubSub = new PgPubSub({
                    connectionString: options.postgres,
                });
                if (init && typeof init === 'function') {
                    await init.apply(this, args);
                }
                const logger = this.logger || console;
                const prefix = options.prefix || constructor.name;
                let cache;
                if (options.redisCache) {
                    cache = options.redisCache;
                }
                else if (options.redis) {
                    cache = await new RedisCache().init({
                        ...options.redis,
                        prefix,
                        logger,
                    });
                }
                else if (this.cache) {
                    cache = this.cache;
                }
                else {
                    throw new TypeError('PgCache: either one of redisCache or ' +
                        'redisConnectionString option must be provided!');
                }
                // built here, but deliberately NOT published on the instance
                // yet: the method decorators treat a missing taggedCache as
                // "not cacheable" and run the method, so withholding it is how
                // caching stays off until invalidation is live. start() does not
                // resolve until it is published one way or the other, so no
                // caller ever observes the gap
                const taggedCache = new TagCache(cache);
                // when invalidation cannot be established at all, caching still
                // happens unless the service asked for the stricter trade
                const cacheAnyway = options.requireInvalidation !== true;
                const className = constructor.name;
                const pgChannels = this.pgCacheChannels || pgCacheChannels || {};
                const channels = Object.keys(pgChannels);
                if (!(channels && channels.length)) {
                    logger.warn(`PgCache: ${className}: no channels registered - ` +
                        'nothing can ever invalidate this cache, so ' +
                        (cacheAnyway
                            ? 'cached reads will only expire by ttl'
                            : 'caching stays OFF ' +
                                '(requireInvalidation is on)'));
                    if (cacheAnyway) {
                        this.taggedCache = taggedCache;
                    }
                    return;
                }
                // a channel name is a table name; anything else (undefined,
                // empty, non-string) can only come from broken registration
                // and guarantees notifications will never arrive
                const invalidChannels = channels.filter(channel => !channel ||
                    channel === 'undefined' ||
                    channel === 'null');
                if (invalidChannels.length) {
                    logger.warn(`PgCache: ${className}: ${invalidChannels.length} ` +
                        'invalid channel name(s) registered: ' +
                        `${invalidChannels.join(', ')} - these are not ` +
                        'table names, so their notifications will never ' +
                        'fire. Usually means table names were read before ' +
                        'the models were initialized');
                }
                const maxListeners = channels.length * 2;
                this.pubSub.channels.setMaxListeners(maxListeners);
                this.pubSub.setMaxListeners(maxListeners);
                this.pubSub.pgClient.setMaxListeners(maxListeners);
                for (const channel of channels) {
                    this.pubSub.channels.on(channel, payload => {
                        if (PG_CACHE_DEBUG) {
                            logger.info('PgCache: database event caught:', channel, payload);
                        }
                        const methods = pgChannels[channel] || [];
                        const data = payload;
                        for (const [method, filter] of methods) {
                            const useTag = signature(className, method, []);
                            if (needInvalidate(data, filter)) {
                                invalidate(this, useTag);
                                if (options.publish !== false) {
                                    publish(this, channel, payload, useTag);
                                }
                            }
                        }
                    });
                }
                // PgPubSub.listen() stays silent when it decides not to
                // subscribe, so track the channels it actually confirmed
                const listened = new Set();
                this.pubSub.on('listen', (channel) => listened.add(channel));
                // installs the triggers and subscribes, then publishes the tag
                // cache: caching becomes live exactly when invalidation does
                const establish = async () => {
                    try {
                        await install(Object.keys(pgChannels), this.pubSub.pgClient, triggerDef(options.triggerDefinition), logger);
                        if (PG_CACHE_DEBUG) {
                            logger.info(`PgCache: triggers installed for ${className}`);
                        }
                        await Promise.all(channels.map(async (channel) => await this.pubSub.listen(channel)));
                        const missed = channels.filter(channel => !listened.has(channel));
                        logger.info(`PgCache: ${className}: listening ` +
                            `${listened.size}/${channels.length} ` +
                            `channels: ${[...listened].join(', ')}`);
                        if (missed.length) {
                            logger.warn(`PgCache: ${className}: NOT listening on ` +
                                `${missed.length} channel(s): ` +
                                `${missed.join(', ')} - writes to them ` +
                                'will not invalidate the cache in this ' +
                                'process');
                        }
                        this.taggedCache = taggedCache;
                    }
                    catch (err) {
                        logger.error(`PgCache: ${className}: failed to set up ` +
                            'invalidation - ' +
                            (cacheAnyway
                                ? 'caching is ENABLED but invalidation is ' +
                                    'DISABLED'
                                : 'caching stays OFF') +
                            ':', err);
                        if (cacheAnyway) {
                            this.taggedCache = taggedCache;
                        }
                    }
                };
                // this work runs from a `connect` handler, which an event
                // emitter cannot await, so it is captured for start() to await
                // below. The promise is created before connect() so it cannot
                // matter whether 'connect' is emitted before or after that call
                // resolves; an escaping rejection would be an unhandled one and
                // vanish without a trace, leaving cache on and invalidation off,
                // so establish() never rejects.
                let markReady = () => undefined;
                const invalidationReady = new Promise(resolve => (markReady = resolve));
                this.pubSub.on('connect', () => {
                    void establish().then(markReady);
                });
                await this.pubSub.connect();
                // without this, start() resolves while the cache is live and
                // nothing can invalidate it yet, and a row changed in that
                // window is never noticed - the entry then stands until the
                // next change to one of its tables, or the ttl
                const confirmed = await awaitInvalidation(invalidationReady, options.invalidationTimeout ?? DEFAULT_INVALIDATION_TIMEOUT, () => logger.warn(`PgCache: ${className}: invalidation was not ` +
                    'confirmed in time - ' +
                    (cacheAnyway
                        ? 'caching is ENABLED but invalidation is ' +
                            'DISABLED'
                        : 'caching stays OFF')));
                if (!confirmed && cacheAnyway) {
                    this.taggedCache = taggedCache;
                }
            }
        }
        const proto = new CachedService();
        for (const prop of Object.keys(proto)) {
            constructor.prototype[prop] = proto[prop];
        }
        constructor.prototype.start = CachedService.prototype.start;
        return constructor;
    });
}
//# sourceMappingURL=PgCache.js.map