ai-functions
Version:
Core AI primitives for building intelligent applications
433 lines • 13.1 kB
JavaScript
/**
* Caching layer for embeddings and generations
*
* Provides content-addressable caching for embeddings and parameter-aware
* caching for text/object generations with TTL support and LRU eviction.
*
* @packageDocumentation
*/
// ============================================================================
// Memory Cache Implementation
// ============================================================================
/**
* In-memory cache implementation with TTL and LRU eviction support
*
* @deprecated Phase C Week 1 — `MemoryCache` has zero production callers in
* primitives.org.ai (audited 2026-05-06; see `bd show aip-ibid`). New code
* should use `cacheMiddleware` (for `wrapLanguageModel`) or
* `embeddingCacheMiddleware` (for `wrapEmbeddingModel`) instead — both
* compose with AI SDK 6's `wrapLanguageModel` / `wrapEmbeddingModel` and
* carry per-call telemetry (TraceEvent emission) and budget tracking when
* paired via `wrapForV3`. The `MemoryCache` class will be removed in the
* Phase C semver bump alongside `EmbeddingCache` and `GenerationCache`.
*/
export class MemoryCache {
cache = new Map();
accessOrder = [];
options;
cleanupTimer;
constructor(options = {}) {
this.options = options;
// Start cleanup timer if interval is specified
if (options.cleanupInterval && options.cleanupInterval > 0) {
this.cleanupTimer = setInterval(() => {
this.cleanup();
}, options.cleanupInterval);
}
}
/**
* Get a value by key
*/
async get(key) {
const entry = this.cache.get(key);
if (!entry) {
return undefined;
}
// Check if expired
if (entry.expiresAt && Date.now() > entry.expiresAt) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
return undefined;
}
// Update access tracking
entry.lastAccessedAt = Date.now();
entry.accessCount++;
// Update access order for LRU
this.updateAccessOrder(key);
// Sliding expiration - refresh TTL on access
if (this.options.slidingExpiration && this.options.defaultTTL) {
entry.expiresAt = Date.now() + this.options.defaultTTL;
}
return entry.value;
}
/**
* Set a value by key
*/
async set(key, value, options) {
const now = Date.now();
const ttl = options?.ttl ?? this.options.defaultTTL;
const entry = {
value,
createdAt: now,
lastAccessedAt: now,
accessCount: 0,
...(ttl ? { expiresAt: now + ttl } : {}),
};
// Check if we need to evict (LRU)
if (this.options.maxSize && this.cache.size >= this.options.maxSize && !this.cache.has(key)) {
this.evictLRU();
}
this.cache.set(key, entry);
this.updateAccessOrder(key);
}
/**
* Check if a key exists
*/
async has(key) {
const entry = this.cache.get(key);
if (!entry) {
return false;
}
// Check if expired
if (entry.expiresAt && Date.now() > entry.expiresAt) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
return false;
}
return true;
}
/**
* Delete a key
*/
async delete(key) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
}
/**
* Clear all entries
*/
async clear() {
this.cache.clear();
this.accessOrder = [];
}
/**
* Get the number of entries (excluding expired)
*/
async size() {
await this.cleanup();
return this.cache.size;
}
/**
* Get all keys
*/
async keys() {
await this.cleanup();
return Array.from(this.cache.keys());
}
/**
* Get full entry with metadata
*/
async getEntry(key) {
const entry = this.cache.get(key);
if (!entry) {
return undefined;
}
// Check if expired
if (entry.expiresAt && Date.now() > entry.expiresAt) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
return undefined;
}
return entry;
}
/**
* Dispose cleanup timer
*/
dispose() {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
delete this.cleanupTimer;
}
}
/**
* Clean up expired entries
*/
async cleanup() {
const now = Date.now();
const keysToDelete = [];
for (const [key, entry] of this.cache) {
if (entry.expiresAt && now > entry.expiresAt) {
keysToDelete.push(key);
}
}
for (const key of keysToDelete) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
}
}
/**
* Evict the least recently used entry
*/
evictLRU() {
if (this.accessOrder.length === 0)
return;
const lruKey = this.accessOrder[0];
if (lruKey) {
this.cache.delete(lruKey);
this.accessOrder.shift();
}
}
/**
* Update access order for LRU tracking
*/
updateAccessOrder(key) {
this.removeFromAccessOrder(key);
this.accessOrder.push(key);
}
/**
* Remove a key from access order
*/
removeFromAccessOrder(key) {
const index = this.accessOrder.indexOf(key);
if (index > -1) {
this.accessOrder.splice(index, 1);
}
}
}
// ============================================================================
// Hash / Key Generation
// ============================================================================
/**
* Generate a hash for cache keys
* Uses a fast, non-cryptographic hash suitable for cache keys
*/
export function hashKey(input) {
const str = typeof input === 'string' ? input : stableStringify(input);
// Simple djb2 hash
let hash = 5381;
for (let i = 0; i < str.length; i++) {
hash = (hash << 5) + hash + str.charCodeAt(i);
hash = hash & hash; // Convert to 32bit integer
}
return (hash >>> 0).toString(36);
}
/**
* Stringify an object with sorted keys for stable hashing
*/
function stableStringify(obj) {
if (obj === null || typeof obj !== 'object') {
return JSON.stringify(obj);
}
if (Array.isArray(obj)) {
return '[' + obj.map(stableStringify).join(',') + ']';
}
const keys = Object.keys(obj).sort();
const pairs = keys.map((key) => {
return JSON.stringify(key) + ':' + stableStringify(obj[key]);
});
return '{' + pairs.join(',') + '}';
}
/**
* Create a cache key for a specific type and parameters
*/
export function createCacheKey(type, params) {
const hash = hashKey(params);
return `${type}:${hash}`;
}
/**
* Specialized cache for embedding vectors
*
* @deprecated Phase C Week 1 — `EmbeddingCache` has zero production callers
* in primitives.org.ai (audited 2026-05-06; see `bd show aip-ibid`). New
* code should use `embeddingCacheMiddleware` (for `wrapEmbeddingModel`) —
* it composes with AI SDK 6 directly and carries trace + budget telemetry
* when paired via `wrapForV3`. Note: `embeddingCacheMiddleware` keys on the
* whole batch, not per-text — callers wanting per-text caching should pass
* stable per-text batches. Will be removed in the Phase C semver bump.
*/
export class EmbeddingCache {
storage;
stats = { hits: 0, misses: 0 };
constructor(options) {
this.storage = new MemoryCache(options);
}
/**
* Get a cached embedding
*/
async get(content, options) {
const key = createCacheKey('embedding', { content, model: options.model });
const result = await this.storage.get(key);
if (result) {
this.stats.hits++;
}
else {
this.stats.misses++;
}
return result;
}
/**
* Set a cached embedding
*/
async set(content, embedding, options) {
const key = createCacheKey('embedding', { content, model: options.model });
await this.storage.set(key, embedding);
}
/**
* Set multiple embeddings at once
*/
async setMany(texts, embeddings, options) {
for (let i = 0; i < texts.length; i++) {
await this.set(texts[i], embeddings[i], options);
}
}
/**
* Get multiple embeddings, returning hits and misses
*/
async getMany(texts, options) {
const hits = {};
const misses = [];
for (const text of texts) {
const embedding = await this.get(text, options);
if (embedding) {
hits[text] = embedding;
// Note: hits counter already incremented in get()
this.stats.hits--; // Avoid double counting
}
else {
misses.push(text);
this.stats.misses--; // Avoid double counting
}
}
// Add back correct counts
this.stats.hits += Object.keys(hits).length;
this.stats.misses += misses.length;
return { hits, misses };
}
/**
* Get cache statistics
*/
getStats() {
const total = this.stats.hits + this.stats.misses;
return {
hits: this.stats.hits,
misses: this.stats.misses,
hitRate: total > 0 ? this.stats.hits / total : 0,
size: this.storage['cache'].size,
};
}
/**
* Clear the cache
*/
async clear() {
await this.storage.clear();
this.stats = { hits: 0, misses: 0 };
}
}
/**
* Specialized cache for generation results
*
* @deprecated Phase C Week 1 — `GenerationCache` has zero production callers
* in primitives.org.ai (audited 2026-05-06; see `bd show aip-ibid`). New
* code should use `cacheMiddleware` (for `wrapLanguageModel`) — it composes
* with AI SDK 6 directly and carries trace + budget telemetry when paired
* via `wrapForV3`. Will be removed in the Phase C semver bump.
*/
export class GenerationCache {
storage;
stats = { hits: 0, misses: 0 };
constructor(options) {
this.storage = new MemoryCache(options);
}
/**
* Get a cached generation result
*/
async get(params, options) {
if (options?.bypass) {
return undefined;
}
const key = this.createKey(params);
const result = await this.storage.get(key);
if (result) {
this.stats.hits++;
}
else {
this.stats.misses++;
}
return result;
}
/**
* Set a cached generation result
*/
async set(params, result) {
const key = this.createKey(params);
await this.storage.set(key, result);
}
/**
* Get cache statistics
*/
getStats() {
const total = this.stats.hits + this.stats.misses;
return {
hits: this.stats.hits,
misses: this.stats.misses,
hitRate: total > 0 ? this.stats.hits / total : 0,
size: this.storage['cache'].size,
};
}
/**
* Clear the cache
*/
async clear() {
await this.storage.clear();
this.stats = { hits: 0, misses: 0 };
}
/**
* Create a cache key from generation parameters
*/
createKey(params) {
const keyParams = {
prompt: params.prompt,
model: params.model,
};
if (params.system !== undefined) {
keyParams['system'] = params.system;
}
if (params.temperature !== undefined) {
keyParams['temperature'] = params.temperature;
}
if (params.schemaVersion !== undefined) {
keyParams['schemaVersion'] = params.schemaVersion;
}
return createCacheKey('generation', keyParams);
}
}
/**
* Wrap an async function with caching
*/
export function withCache(cache, fn, options) {
const { keyFn, ttl } = options;
const cachedFn = async (...args) => {
const key = keyFn(...args);
// Check cache
const cached = await cache.get(key);
if (cached !== undefined) {
return cached;
}
// Execute function
const result = await fn(...args);
// Cache result (don't cache errors - they throw before reaching here)
await cache.set(key, result, ttl !== undefined ? { ttl } : undefined);
return result;
};
// Add bypass method
cachedFn.bypass = async (...args) => {
const key = keyFn(...args);
// Execute function without checking cache
const result = await fn(...args);
// Update cache with new result
await cache.set(key, result, ttl !== undefined ? { ttl } : undefined);
return result;
};
return cachedFn;
}
//# sourceMappingURL=cache.js.map