UNPKG

chronos-db

Version:

Chronos-DB v2.0: Enterprise-grade MongoDB persistence layer with embedded multi-tenancy, tiered architecture, and big data capabilities. Features time-travel versioning, S3/Azure/local storage, enrichment API, and lineage tracking. Designed for scalable a

1,793 lines (1,771 loc) 114 kB
import { MongoClient, ObjectId, Db, ClientSession, Collection } from 'mongodb'; import { S3Client } from '@aws-sdk/client-s3'; import { Readable } from 'stream'; type VersionSpec = { ov: number; } | { at: string; }; type CollVersionSpec = { cv: number; } | { at: string; }; interface RestoreOptions { actor?: string; reason?: string; expectedHeadOv?: number; requestId?: string; } interface CollectionRestoreOptions { actor?: string; reason?: string; pageSize?: number; parallel?: number; dryRun?: boolean; requestId?: string; } interface RestoreResult$1 { id: string; newOv: number; newCv: number; } interface CollectionRestoreResult { target: CollVersionSpec & { resolvedCv: number; resolvedAt: string; }; planned: number; restored: number; } interface SnapshotInfo { jsonBucket: string; jsonKey: string; metaIndexed: Record<string, unknown>; size: number | null; checksum: string | null; ov: number; cv: number; at: Date; } /** * Restore an object to a specific version * @param router - Bridge router instance * @param ctx - Route context * @param id - Object ID (hex string) * @param to - Target version specification * @param opts - Restore options * @returns Restore result */ declare function restoreObject(router: BridgeRouter, ctx: RouteContext, id: string, to: VersionSpec, opts?: RestoreOptions): Promise<RestoreResult$1>; /** * Restore a collection to a specific version * @param router - Bridge router instance * @param ctx - Route context * @param to - Target version specification * @param opts - Restore options * @returns Collection restore result */ declare function restoreCollection(router: BridgeRouter, ctx: RouteContext, to: CollVersionSpec, opts?: CollectionRestoreOptions): Promise<CollectionRestoreResult>; /** * Local filesystem storage configuration (for development/testing) */ interface LocalStorageConfig { /** Base path for local storage */ basePath: string; /** Whether to enable this mode */ enabled: boolean; } /** * MongoDB connection configuration - define once, reference everywhere */ interface DbConnection { /** MongoDB connection URI */ mongoUri: string; } /** * S3-compatible storage connection configuration - define once, reference everywhere */ interface SpacesConnection { /** S3 endpoint URL */ endpoint: string; /** AWS region */ region: string; /** Access key */ accessKey: string; /** Secret key */ secretKey: string; /** Force path style (for MinIO) */ forcePathStyle?: boolean; } /** * Generic database configuration (no tenant/domain) */ interface GenericDatabase { /** MongoDB connection reference */ dbConnRef: string; /** S3 spaces connection reference */ spaceConnRef: string; /** S3 bucket name */ bucket: string; /** MongoDB database name */ dbName: string; } /** * Domain-specific database configuration */ interface DomainDatabase { /** Domain identifier */ domain: string; /** MongoDB connection reference */ dbConnRef: string; /** S3 spaces connection reference */ spaceConnRef: string; /** S3 bucket name */ bucket: string; /** MongoDB database name */ dbName: string; } /** * Tenant-specific database configuration */ interface TenantDatabase { /** Tenant identifier */ tenantId: string; /** MongoDB connection reference */ dbConnRef: string; /** S3 spaces connection reference */ spaceConnRef: string; /** S3 bucket name */ bucket: string; /** MongoDB database name */ dbName: string; } /** * Runtime tenant database configuration (includes analytics) */ interface RuntimeTenantDatabase { /** Tenant identifier */ tenantId: string; /** MongoDB connection reference */ dbConnRef: string; /** S3 spaces connection reference */ spaceConnRef: string; /** S3 bucket name */ bucket: string; /** MongoDB database name */ dbName: string; /** Analytics database name */ analyticsDbName: string; } /** * Logs database configuration (simple flat structure) */ interface LogsDatabase { /** MongoDB connection reference */ dbConnRef: string; /** S3 spaces connection reference */ spaceConnRef: string; /** S3 bucket name */ bucket: string; /** MongoDB database name */ dbName: string; } /** * Messaging database configuration (simple flat structure, like logs) * Simple MongoDB-only storage for Chronow integration (no versioning, no S3/Azure offload) */ interface MessagingDatabase { /** MongoDB connection reference */ dbConnRef: string; /** MongoDB database name */ dbName: string; /** Capture delivery attempts (default: false to save storage) */ captureDeliveries?: boolean; } /** * Identities database configuration (simple flat structure, like logs) * For users, accounts, authentication, permissions, roles (no versioning, no S3/Azure offload) */ interface IdentitiesDatabase { /** MongoDB connection reference */ dbConnRef: string; /** MongoDB database name */ dbName: string; } /** * Routing configuration */ interface RoutingConfig { /** Hash algorithm for routing */ hashAlgo: 'rendezvous' | 'jump'; /** Key selection strategy */ chooseKey?: string; } /** * Retention configuration */ interface RetentionConfig { /** Version retention settings */ ver?: { /** Days to retain versions */ days?: number; /** Maximum versions per item */ maxPerItem?: number; }; /** Counter retention settings */ counters?: { /** Days to retain counters */ days?: number; /** Weeks to retain counters */ weeks?: number; /** Months to retain counters */ months?: number; }; } /** * Rollup configuration */ interface RollupConfig { /** Whether rollup is enabled */ enabled: boolean; /** Manifest period */ manifestPeriod?: 'daily' | 'weekly' | 'monthly'; } /** * Collection map configuration */ interface CollectionMap { /** Properties to index */ indexedProps: string[]; /** Base64 properties configuration */ base64Props?: Record<string, { /** Content type */ contentType: string; /** Whether to prefer text */ preferredText?: boolean; /** Text charset */ textCharset?: string; }>; /** Validation rules */ validation?: { /** Required indexed properties */ requiredIndexed?: string[]; }; } /** * Counter rules configuration */ interface CountersRulesConfig { /** Counter rules */ rules?: Array<{ /** Rule name */ name: string; /** Events to trigger on */ on?: ('CREATE' | 'UPDATE' | 'DELETE')[]; /** Scope */ scope?: 'meta' | 'payload'; /** Condition */ when: Record<string, any>; }>; } /** * Dev shadow configuration */ interface DevShadowConfig { /** Whether dev shadow is enabled */ enabled: boolean; /** TTL in hours */ ttlHours: number; /** Maximum bytes per document */ maxBytesPerDoc?: number; } /** * Fallback configuration */ interface FallbackConfig { /** Whether fallback is enabled */ enabled: boolean; /** Maximum attempts */ maxAttempts: number; /** Base delay in milliseconds */ baseDelayMs: number; /** Maximum delay in milliseconds */ maxDelayMs: number; /** Dead letter collection name */ deadLetterCollection: string; } /** * Write optimization configuration */ interface WriteOptimizationConfig { /** Whether to batch S3 writes */ batchS3: boolean; /** Batch window in milliseconds */ batchWindowMs: number; /** Debounce counters in milliseconds */ debounceCountersMs: number; /** Whether to allow shadow skip */ allowShadowSkip: boolean; } /** * Logical delete configuration */ interface LogicalDeleteConfig { /** Whether logical delete is enabled (default: true) */ enabled: boolean; } /** * Versioning configuration */ interface VersioningConfig { /** Whether versioning is enabled (default: true) */ enabled: boolean; } /** * Transaction configuration */ interface TransactionConfig { /** Whether transactions are enabled */ enabled: boolean; /** Whether to auto-detect transaction support */ autoDetect?: boolean; } /** * Main Chronos configuration interface */ interface ChronosConfig { /** MongoDB connections - define once, reference by key */ dbConnections: Record<string, DbConnection>; /** S3-compatible storage connections - define once, reference by key */ spacesConnections: Record<string, SpacesConnection>; /** Database configuration with tiered structure */ databases: { /** Metadata databases with 3 tiers: generic, domains, tenants */ metadata?: { /** Generic metadata database (no tenant/domain) */ genericDatabase: GenericDatabase; /** Domain-specific metadata databases */ domainsDatabases: DomainDatabase[]; /** Tenant-specific metadata databases */ tenantDatabases: TenantDatabase[]; }; /** Knowledge databases with 3 tiers: generic, domains, tenants */ knowledge?: { /** Generic knowledge database (no tenant/domain) */ genericDatabase: GenericDatabase; /** Domain-specific knowledge databases */ domainsDatabases: DomainDatabase[]; /** Tenant-specific knowledge databases */ tenantDatabases: TenantDatabase[]; }; /** Runtime databases with tenant tier only */ runtime?: { /** Tenant-specific runtime databases (includes analytics) */ tenantDatabases: RuntimeTenantDatabase[]; }; /** Logs database (simple flat structure) */ logs?: LogsDatabase; /** Messaging database (simple flat structure, for Chronow integration) */ messaging?: MessagingDatabase; /** Identities database (simple flat structure, for users/accounts/auth/permissions) */ identities?: IdentitiesDatabase; }; /** Local filesystem storage (for development/testing, NOT recommended for production) */ localStorage?: LocalStorageConfig | undefined; /** How to route requests across backends */ routing: RoutingConfig; /** Retention policies for versions and counters (optional, defaults to disabled) */ retention?: RetentionConfig | undefined; /** Roll-up configuration for moving old data to manifests (optional, defaults to disabled) */ rollup?: RollupConfig | undefined; /** Collection definitions with indexing and externalization rules */ collectionMaps: Record<string, CollectionMap>; /** Analytics configuration (counters, time-based, cross-tenant) */ analytics?: AnalyticsConfig | undefined; /** Dev shadow configuration for full snapshots in Mongo */ devShadow?: DevShadowConfig | undefined; /** Enable hard delete functionality */ hardDeleteEnabled?: boolean | undefined; /** Fallback queue configuration */ fallback?: FallbackConfig | undefined; /** Write optimization configuration */ writeOptimization?: WriteOptimizationConfig | undefined; /** Transaction configuration */ transactions?: TransactionConfig | undefined; /** Logical delete configuration */ logicalDelete?: LogicalDeleteConfig | undefined; /** Versioning configuration */ versioning?: VersioningConfig | undefined; } /** * Route context for routing decisions */ interface RouteContext { /** Database name */ dbName: string; /** Collection name */ collection: string; /** Object ID */ objectId?: string; /** Forced index for admin override */ forcedIndex?: number; /** Key for direct routing */ key?: string; /** Database type */ databaseType?: 'metadata' | 'knowledge' | 'runtime' | 'logs' | 'messaging' | 'identities'; /** Tier */ tier?: 'generic' | 'domain' | 'tenant'; /** Tenant ID */ tenantId?: string; /** Domain identifier */ domain?: string; } /** * Validates transaction configuration against MongoDB capabilities * @param config - Configuration to validate * @throws Error if transaction configuration is invalid */ declare function validateTransactionConfig(config: Partial<ChronosConfig>): Promise<void>; /** * Validates a Chronos configuration object * @param config - Configuration object to validate * @returns Validated configuration with defaults applied * @throws ZodError if validation fails */ declare function validateChronosConfig(config: unknown): ChronosConfig; /** * Checks if an object is a valid Chronos configuration * @param obj - Object to check * @returns True if valid configuration */ declare function isValidChronosConfig(obj: unknown): obj is ChronosConfig; /** * Sets the global configuration * @param config - Configuration to set globally */ declare function setGlobalConfig(config: ChronosConfig): void; /** * Gets the global configuration * @returns Global configuration or null if not set */ declare function getGlobalConfig(): ChronosConfig | null; interface CounterRule { /** Rule name */ name: string; /** Events to trigger on */ on?: ('CREATE' | 'UPDATE' | 'DELETE')[]; /** Scope */ scope?: 'meta' | 'payload'; /** Condition */ when: Record<string, any>; /** Properties to count unique values for */ countUnique?: string[]; } /** * Time-based analytics rule for worker-driven analytics */ interface TimeBasedRule { /** Rule name */ name: string; /** Collection to query */ collection: string; /** Query filter */ query: Record<string, any>; /** Aggregation operation */ operation: 'count' | 'sum' | 'average' | 'max' | 'min' | 'median'; /** Field to aggregate (for sum, average, max, min, median) */ field?: string; /** Save mode */ saveMode: 'global' | 'timeframe'; /** Timeframe granularity (if saveMode is 'timeframe') */ timeframe?: 'hourly' | 'daily' | 'monthly'; /** Arguments (record keys for foreign key filtering) */ arguments?: string[]; /** Relative time arguments */ relativeTime?: { newerThan?: string; olderThan?: string; }; } /** * Cross-tenant analytics rule (master-slave aggregation) */ interface CrossTenantRule { /** Rule name */ name: string; /** Collection to query across tenants */ collection: string; /** Query filter */ query: Record<string, any>; /** Aggregation mode */ mode: 'boolean' | 'sum' | 'max' | 'min' | 'median'; /** Field to aggregate (for sum, max, min, median) */ field?: string; /** Master tenant ID (where results are stored) */ masterTenantId: string; /** Slave tenant IDs (data sources) */ slaveTenantIds: string[]; /** Relative time arguments */ relativeTime?: { newerThan?: string; olderThan?: string; }; } /** * Analytics configuration */ interface AnalyticsConfig { /** Standard counter rules */ counterRules?: CounterRule[]; /** Time-based analytics rules (worker-driven) */ timeBasedRules?: TimeBasedRule[]; /** Cross-tenant analytics rules */ crossTenantRules?: CrossTenantRule[]; /** List of all tenant IDs for cross-tenant operations */ tenants?: string[]; } /** * Unified storage interface that works with both S3 and local filesystem * This allows code to be storage-agnostic */ interface StorageAdapter { /** * Put JSON to storage */ putJSON(bucket: string, key: string, data: any): Promise<{ size: number | null; sha256: string | null; }>; /** * Put raw data to storage */ putRaw(bucket: string, key: string, body: Buffer | Uint8Array, contentType?: string): Promise<{ size: number | null; sha256: string | null; }>; /** * Get JSON from storage */ getJSON(bucket: string, key: string): Promise<Record<string, unknown>>; /** * Get raw data from storage */ getRaw(bucket: string, key: string): Promise<Buffer>; /** * Check if object exists */ head(bucket: string, key: string): Promise<{ exists: boolean; size?: number; }>; /** * Delete object from storage */ del(bucket: string, key: string): Promise<void>; /** * Generate presigned URL for GET */ presignGet(bucket: string, key: string, ttlSeconds: number): Promise<string>; /** * List objects with prefix */ list(bucket: string, prefix: string, opts?: { maxKeys?: number; continuationToken?: string; }): Promise<{ keys: string[]; nextToken?: string; }>; /** * Copy object */ copy(sourceBucket: string, sourceKey: string, destBucket: string, destKey: string): Promise<void>; } interface RouterInitArgs { dbConnections: Record<string, DbConnection>; spacesConnections: Record<string, SpacesConnection>; databases: { metadata?: { genericDatabase: GenericDatabase; domainsDatabases: DomainDatabase[]; tenantDatabases: TenantDatabase[]; }; knowledge?: { genericDatabase: GenericDatabase; domainsDatabases: DomainDatabase[]; tenantDatabases: TenantDatabase[]; }; runtime?: { tenantDatabases: RuntimeTenantDatabase[]; }; logs?: LogsDatabase; messaging?: MessagingDatabase; identities?: IdentitiesDatabase; }; localStorage?: LocalStorageConfig | undefined; hashAlgo?: 'rendezvous' | 'jump'; chooseKey?: string | ((ctx: RouteContext) => string); } interface BackendInfo { index: number; mongoUri: string; endpoint: string; region: string; bucket: string; analyticsDbName?: string; routingKey?: string; } declare class BridgeRouter { private readonly dbConnections; private readonly spacesConnections; private readonly localStorage; private readonly hashAlgo; private readonly chooseKey; private readonly backendIds; private readonly databases; private mongoClients; private s3Clients; private _isShutdown; constructor(args: RouterInitArgs); /** * Validate initialization arguments */ private validateArgs; /** * Find database connection by reference * @param ref - Database connection reference to search for * @returns Database connection config or null */ private findDbConnection; /** * Find spaces connection by reference * @param ref - Spaces connection reference to search for * @returns Spaces connection config or null */ private findSpacesConnection; /** * Get all MongoDB URIs from database connections */ getAllMongoUris(): string[]; /** * Generate backend IDs for consistent routing */ private generateBackendIds; /** * Resolve database connection based on context */ resolveDatabaseConnection(ctx: RouteContext): { mongoUri: string; dbName: string; analyticsDbName?: string; } | null; /** * Get MongoDB client for a specific URI */ getMongoClient(mongoUri: string): Promise<MongoClient>; /** * Get S3 storage adapter for a specific connection */ getSpaces(ctx: RouteContext): Promise<{ storage: StorageAdapter; bucket: string; analyticsDbName?: string; }>; /** * Generate routing key from context */ private generateRoutingKey; /** * Route request to appropriate backend */ route(ctx: RouteContext): BackendInfo; /** * Hash string for consistent mapping */ private hashString; /** * Shutdown all connections */ shutdown(): Promise<void>; /** * Check if router is shutdown */ get isShutdown(): boolean; } interface PresignOptions { presign?: boolean; ttlSeconds?: number; includeText?: boolean; } interface ReadOptions extends PresignOptions { projection?: string[]; includeDeleted?: boolean; includeMeta?: boolean; preferShadow?: boolean; ov?: number; at?: Date | string; } interface QueryOptions extends PresignOptions { projection?: string[]; includeDeleted?: boolean; includeMeta?: boolean; at?: Date | string; limit?: number; sort?: Array<{ field: string; dir: 1 | -1; }>; pageToken?: string; } interface MetaFilter { [key: string]: any; } interface ItemView { id: string; item: Record<string, unknown>; _meta?: { ov: number; cv: number; at: string; metaIndexed: Record<string, unknown>; deletedAt?: string; }; presigned?: { [jsonPath: string]: { blobUrl?: string; textUrl?: string; expiresIn?: number; }; }; } /** * Entity mapping configuration */ interface EntityMapping { /** Property name in the main record that holds the entity object */ property: string; /** Collection name where the entity should be stored */ collection: string; /** Property name in the entity object that acts as the key/ID */ keyProperty: string; /** Database type where the entity collection exists */ databaseType?: 'metadata' | 'knowledge' | 'runtime' | 'logs'; /** Tier for the entity collection */ tier?: 'generic' | 'domain' | 'tenant'; } /** * Options for insertWithEntities operation */ interface InsertWithEntitiesOptions { /** Main record to insert */ record: Record<string, any>; /** Collection name for the main record */ collection: string; /** Entity mappings configuration */ entityMappings: EntityMapping[]; /** Routing context */ ctx: RouteContext; /** Requester information */ requester: string; /** Reason for the operation */ reason: string; /** Router instance */ router: BridgeRouter; } /** * Result of insertWithEntities operation */ interface InsertWithEntitiesResult { /** ID of the main record created */ mainRecordId: string; /** Map of entity property names to their saved/updated entity IDs */ entityResults: Map<string, { id: string; operation: 'created' | 'updated' | 'unchanged'; }>; } /** * Insert a record with automatic entity relationship management * * This function: * 1. Extracts entity objects from the main record based on mappings * 2. For each entity, checks if it already exists in its collection * 3. Creates new entities or updates existing ones with changed properties * 4. Inserts the main record with references to the entities * * @param options - Insert with entities options * @returns Result with main record ID and entity operation results */ declare function insertWithEntities(options: InsertWithEntitiesOptions): Promise<InsertWithEntitiesResult>; /** * Options for getWithEntities operation */ interface GetWithEntitiesOptions { /** Main record ID to fetch */ id: string; /** Collection name for the main record */ collection: string; /** Entity mappings configuration */ entityMappings: EntityMapping[]; /** Routing context */ ctx: RouteContext; /** Router instance */ router: BridgeRouter; /** Read options */ readOptions?: ReadOptions; } /** * Result of getWithEntities operation */ interface GetWithEntitiesResult { /** Main record data */ mainRecord: Record<string, any> | null; /** Map of entity property names to their fetched entity records */ entityRecords: Map<string, Record<string, any> | null>; } /** * Get a record with automatic entity relationship fetching * * This function: * 1. Fetches the main record * 2. For each entity mapping, extracts the entity object from the main record * 3. Fetches the full entity record from its collection using the key property * 4. Returns the main record with all entity records * * @param options - Get with entities options * @returns Result with main record and entity records */ declare function getWithEntities(options: GetWithEntitiesOptions): Promise<GetWithEntitiesResult>; /** * Shared memory snapshot document * Stores key-value snapshots with append or latest-wins strategy */ interface SharedMemoryDoc<T = any> { _id?: ObjectId; tenantId: string; namespace: string; key: string; val: T; strategy: 'append' | 'latest'; version?: number; createdAt: Date; updatedAt: Date; } /** * Topic metadata document * Defines topics used in Chronow messaging */ interface TopicDoc { _id?: ObjectId; tenantId: string; topic: string; shards: number; createdAt: Date; } /** * Canonical message document * Stores published messages for audit and retrieval */ interface MessageDoc<P = any> { _id?: ObjectId; tenantId: string; topic: string; msgId: string; headers?: Record<string, string>; payload: P; firstSeenAt: Date; size: number; } /** * Delivery attempt document (optional, controlled by captureDeliveries flag) * Tracks per-subscription delivery attempts */ interface DeliveryDoc { _id?: ObjectId; tenantId: string; topic: string; subscription: string; msgId: string; attempt: number; status: 'pending' | 'ack' | 'nack' | 'dead'; consumerId?: string; reason?: string; ts: Date; } /** * Dead letter document * Stores terminally failed messages with audit trail */ interface DeadLetterDoc<P = any> { _id?: ObjectId; tenantId: string; topic: string; msgId: string; subscription?: string; headers?: Record<string, string>; payload: P; deliveries: number; failedAt: Date; reason?: string; } declare const MESSAGING_INDEXES: { shared_memory: ({ keys: { tenantId: number; namespace: number; key: number; strategy: number; version?: never; updatedAt?: never; }; options: { unique: boolean; partialFilterExpression: { strategy: string; }; name: string; }; } | { keys: { tenantId: number; namespace: number; key: number; version: number; strategy?: never; updatedAt?: never; }; options: { name: string; unique?: never; partialFilterExpression?: never; }; } | { keys: { updatedAt: number; tenantId?: never; namespace?: never; key?: never; strategy?: never; version?: never; }; options: { name: string; unique?: never; partialFilterExpression?: never; }; })[]; topics: { keys: { tenantId: number; topic: number; }; options: { unique: boolean; name: string; }; }[]; messages: ({ keys: { tenantId: number; topic: number; msgId: number; firstSeenAt?: never; }; options: { unique: boolean; name: string; }; } | { keys: { firstSeenAt: number; tenantId?: never; topic?: never; msgId?: never; }; options: { name: string; unique?: never; }; } | { keys: { tenantId: number; topic: number; firstSeenAt: number; msgId?: never; }; options: { name: string; unique?: never; }; })[]; deliveries: ({ keys: { tenantId: number; topic: number; subscription: number; msgId: number; attempt: number; ts?: never; }; options: { unique: boolean; name: string; }; } | { keys: { tenantId: number; topic: number; msgId: number; subscription?: never; attempt?: never; ts?: never; }; options: { name: string; unique?: never; }; } | { keys: { ts: number; tenantId?: never; topic?: never; subscription?: never; msgId?: never; attempt?: never; }; options: { name: string; unique?: never; }; })[]; dead_letters: ({ keys: { tenantId: number; topic: number; msgId: number; failedAt?: never; }; options: { name: string; }; } | { keys: { failedAt: number; tenantId?: never; topic?: never; msgId?: never; }; options: { name: string; }; } | { keys: { tenantId: number; topic: number; failedAt: number; msgId?: never; }; options: { name: string; }; })[]; }; interface MessagingApiOptions { mongoClient: MongoClient; dbName: string; tenantId: string; captureDeliveries?: boolean; } /** * Shared memory operations (KV snapshots with versioning) */ interface SharedApi { save(opts: { namespace: string; key: string; val: any; strategy?: 'append' | 'latest'; }): Promise<{ id: string; version?: number; }>; load(opts: { namespace: string; key: string; strategy?: 'append' | 'latest'; version?: number; }): Promise<any | null>; tombstone(opts: { namespace: string; key: string; reason?: string; }): Promise<void>; } /** * Topic operations (topic metadata & shard config) */ interface TopicsApi { ensure(opts: { topic: string; shards?: number; }): Promise<void>; get(opts: { topic: string; }): Promise<TopicDoc | null>; } /** * Message operations (canonical messages) */ interface MessagesApi { save(opts: { topic: string; msgId: string; headers?: Record<string, string>; payload: any; firstSeenAt?: Date; size?: number; }): Promise<void>; get(opts: { topic: string; msgId: string; }): Promise<MessageDoc | null>; list(opts: { topic: string; after?: Date; limit?: number; }): Promise<MessageDoc[]>; } /** * Delivery operations (optional, controlled by captureDeliveries) */ interface DeliveriesApi { append(opts: { topic: string; subscription: string; msgId: string; attempt: number; status: 'pending' | 'ack' | 'nack' | 'dead'; consumerId?: string; reason?: string; ts?: Date; }): Promise<void>; listByMessage(opts: { topic: string; subscription?: string; msgId: string; }): Promise<DeliveryDoc[]>; } /** * Dead letter operations (DLQ) */ interface DeadLettersApi { save(opts: { topic: string; subscription?: string; msgId: string; headers?: Record<string, string>; payload: any; deliveries: number; reason?: string; failedAt?: Date; }): Promise<void>; list(opts: { topic: string; after?: Date; limit?: number; }): Promise<DeadLetterDoc[]>; } /** * Main MessagingApi interface */ interface MessagingApi { shared: SharedApi; topics: TopicsApi; messages: MessagesApi; deliveries?: DeliveriesApi | undefined; deadLetters: DeadLettersApi; } declare class MessagingApiImpl implements MessagingApi { private db; private tenantId; private captureDeliveries; private indexesEnsured; constructor(opts: MessagingApiOptions); /** * Ensure indexes are created (lazy, one-time per instance) */ private ensureIndexes; /** * Shared memory API implementation */ shared: SharedApi; /** * Topics API implementation */ topics: TopicsApi; /** * Messages API implementation */ messages: MessagesApi; /** * Deliveries API implementation (optional) */ get deliveries(): DeliveriesApi | undefined; /** * Dead letters API implementation */ deadLetters: DeadLettersApi; } /** * Factory function to create MessagingApi instance */ declare function createMessagingApi(opts: MessagingApiOptions): MessagingApi; /** * Deep merge utility for Chronos-DB record objects * Merges data properties from multiple tiers (generic -> domain -> tenant) */ interface RecordMergeOptions { /** When true, remove duplicates from arrays. Default: true */ dedupeArrays?: boolean; /** When true, preserve existing values and only add new properties. Default: false */ preserveExisting?: boolean; } /** * Deep merge two record objects * Designed for merging data across tiers in Chronos-DB * * @param target - The base record object (from lower priority tier) * @param source - The record object to merge from (from higher priority tier) * @param opts - Merge configuration options * @returns Merged record object * * @example * // Merge generic -> domain -> tenant * const generic = { name: "Default", settings: { theme: "light" }, tags: ["base"] }; * const domain = { settings: { language: "en" }, tags: ["domain"] }; * const tenant = { settings: { notifications: true }, tags: ["tenant"] }; * * let result = deepMerge(generic, domain); * result = deepMerge(result, tenant); * // Result: { * // name: "Default", * // settings: { theme: "light", language: "en", notifications: true }, * // tags: ["base", "domain", "tenant"] * // } */ declare function deepMergeRecords(target: Record<string, any> | null | undefined, source: Record<string, any> | null | undefined, opts?: RecordMergeOptions): Record<string, any>; declare const deepMerge: typeof deepMergeRecords; /** * Options for tiered fetching (getKnowledge/getMetadata) */ interface TieredFetchOptions extends ReadOptions { /** Tenant ID for tenant-tier lookup */ tenantId?: string; /** Domain for domain-tier lookup */ domain?: string; /** If true, merge data from all tiers (generic -> domain -> tenant). If false, return first found. */ merge: boolean; /** Merge options for combining records */ mergeOptions?: RecordMergeOptions; } /** * Result of tiered fetch operation */ interface TieredFetchResult { /** Merged or first-found record data */ data: Record<string, any> | null; /** Which tiers had data */ tiersFound: Array<'generic' | 'domain' | 'tenant'>; /** Full records from each tier (if merge was true) */ tierRecords?: { generic?: Record<string, any>; domain?: Record<string, any>; tenant?: Record<string, any>; }; } /** * Fetch a record from the Knowledge database with tiered fallback/merge * * Tier priority: tenant -> domain -> generic * * @param router - Bridge router instance * @param collection - Collection name * @param filter - MongoDB filter to find the record * @param options - Tiered fetch options * @returns Tiered fetch result * * @example * // Fetch with fallback (first found) * const result = await getKnowledge(router, 'products', { sku: 'ABC123' }, { * tenantId: 'tenant-a', * domain: 'retail', * merge: false * }); * * @example * // Fetch with merge (combine all tiers) * const result = await getKnowledge(router, 'config', { key: 'settings' }, { * tenantId: 'tenant-a', * domain: 'retail', * merge: true, * mergeOptions: { dedupeArrays: true } * }); */ declare function getKnowledge(router: BridgeRouter, collection: string, filter: Record<string, any>, options: TieredFetchOptions): Promise<TieredFetchResult>; /** * Fetch a record from the Metadata database with tiered fallback/merge * * Tier priority: tenant -> domain -> generic * * @param router - Bridge router instance * @param collection - Collection name * @param filter - MongoDB filter to find the record * @param options - Tiered fetch options * @returns Tiered fetch result * * @example * // Fetch with fallback (first found) * const result = await getMetadata(router, 'schemas', { name: 'user' }, { * tenantId: 'tenant-a', * domain: 'saas', * merge: false * }); * * @example * // Fetch with merge (combine all tiers) * const result = await getMetadata(router, 'permissions', { resource: 'users' }, { * tenantId: 'tenant-a', * domain: 'saas', * merge: true * }); */ declare function getMetadata(router: BridgeRouter, collection: string, filter: Record<string, any>, options: TieredFetchOptions): Promise<TieredFetchResult>; type CounterOp = 'CREATE' | 'UPDATE' | 'DELETE'; interface CounterScope { tenant?: string; dbName: string; collection: string; } interface BumpTotalsInput { scope: CounterScope; op: CounterOp; metaIndexed?: Record<string, unknown>; payloadTransformed?: Record<string, unknown>; } interface CounterTotalsDoc { _id: string; tenant?: string; dbName: string; collection: string; created: number; updated: number; deleted: number; rules?: { [ruleName: string]: { created?: number; updated?: number; deleted?: number; unique?: { [propertyName: string]: number; }; }; }; lastAt: Date; } interface CounterUniqueDoc { _id: string; tenant?: string; dbName: string; collection: string; ruleName: string; propertyName: string; propertyValue: any; created: number; updated: number; deleted: number; lastAt: Date; } type AnalyticsDoc = CounterTotalsDoc | CounterUniqueDoc; interface GetTotalsQuery { dbName: string; collection: string; tenant?: string; includeRules?: boolean; rules?: string[]; } /** * Repository for managing counter totals */ declare class CounterTotalsRepo { private readonly collection; private readonly rules; constructor(countersDb: Db, rules?: CounterRule[]); /** * Ensure indexes are created */ ensureIndexes(): Promise<void>; /** * Generate document ID for a scope */ private generateDocId; /** * Get nested value from object using dot notation */ private getNestedValue; /** * Evaluate a counter rule against data */ private evaluateRule; /** * Evaluate a predicate against data */ private evaluatePredicate; /** * Evaluate complex condition with operators */ private evaluateComplexCondition; /** * Current operation being processed (for rule evaluation) */ private currentOp; /** * Bump totals for an operation */ bumpTotals(input: BumpTotalsInput): Promise<void>; /** * Get totals for a scope */ getTotals(query: GetTotalsQuery): Promise<CounterTotalsDoc | null>; /** * Reset totals for a scope (admin operation) */ resetTotals(query: { dbName: string; collection: string; tenant?: string; }): Promise<void>; /** * Get unique analytics results (one row per unique value) */ getUniqueAnalytics(query: { dbName: string; collection: string; tenant?: string; ruleName?: string; propertyName?: string; limit?: number; }): Promise<CounterUniqueDoc[]>; } interface EnrichContext { dbName: string; collection: string; objectId?: string; forcedIndex?: number; key?: string; databaseType?: 'metadata' | 'knowledge' | 'runtime' | 'logs' | 'messaging' | 'identities'; tier?: 'generic' | 'domain' | 'tenant'; tenantId?: string; domain?: string; } interface EnrichOptions { /** Optional optimistic lock. If provided, current head.ov must equal this. */ expectedOv?: number; /** Optional enrichment source tag to store under _system.functionIds[] (unique). */ functionId?: string | null; /** Who/why (recorded in _ver) */ actor?: string; reason?: string; /** Per-call override of devShadow config */ devShadowOverride?: boolean; } interface EnrichResult { id: string; ov: number; cv: number; } /** * Enrich a single record by deep-merging enrichment into the head payload * @param router - Bridge router instance * @param ctx - Enrichment context * @param id - Item ID (hex ObjectId) * @param enrichment - Enrichment data (single object or array of objects) * @param opts - Enrichment options * @param config - Configuration including devShadow settings * @returns Enrichment result */ declare function enrichRecord(router: BridgeRouter, ctx: EnrichContext, id: string, enrichment: Record<string, unknown> | Array<Record<string, unknown>>, opts?: EnrichOptions, config?: { devShadow?: DevShadowConfig; }): Promise<EnrichResult>; interface SmartInsertOptions { /** Unique key fields to check for existing record */ uniqueKeys: string[]; /** Actor performing the operation */ actor?: string; /** Reason for the operation */ reason?: string; /** Function ID for provenance (if merging) */ functionId?: string; /** Parent record for lineage tracking */ parentRecord?: { id: string; collection: string; dbName?: string; }; /** Dev shadow override */ devShadowOverride?: boolean; } interface SmartInsertResult { /** Record ID */ id: string; /** Object version */ ov: number; /** Collection version */ cv: number; /** Whether a new record was created (true) or existing merged (false) */ created: boolean; /** Timestamp of operation */ at: Date; } /** * Smart insert: Check if record exists by unique keys, merge if exists, create if not * * This is like MongoDB's upsert, but with Chronos-DB's enrichment merge semantics: * - If no matching record exists → CREATE new record * - If matching record exists → ENRICH (deep merge) into existing record * * @param router - Bridge router instance * @param ctx - Route context * @param data - Data to insert or merge * @param opts - Smart insert options with unique keys * @param config - Optional configuration * @returns Smart insert result * * @example * // Check by email, create or merge * const result = await smartInsert(router, ctx, * { email: 'john@example.com', name: 'John', tags: ['vip'] }, * { uniqueKeys: ['email'], functionId: 'importer@v1' } * ); * * if (result.created) { * console.log('Created new record:', result.id); * } else { * console.log('Merged into existing:', result.id); * } */ declare function smartInsert(router: BridgeRouter, ctx: RouteContext, data: Record<string, unknown>, opts: SmartInsertOptions, config?: { devShadow?: any; }): Promise<SmartInsertResult>; interface HealthReport { timestamp: string; router: { backends: Array<{ index: number; mongoUri: string; s3Endpoint: string; }>; }; mongoBackends: Array<{ index: number; ok: boolean; pingMs?: number; error?: string; }>; s3Backends: Array<{ index: number; ok: boolean; error?: string; }>; countersDb: { ok: boolean; pingMs?: number; error?: string; } | null; } /** * State management utilities for _system state transitions */ interface StateTransitionOptions { /** Dry run - don't actually update state */ dryRun?: boolean; /** Confirm the operation */ confirm?: boolean; /** Maximum number of items to process */ maxItems?: number; } interface StateTransitionResult { /** Collection name */ collection: string; /** Number of items processed */ itemsProcessed: number; /** Number of items marked as processed */ itemsMarkedAsProcessed: number; /** Whether this was a dry run */ dryRun: boolean; /** When the operation was performed */ processedAt: Date; } /** * Mark items as processed based on TTL expiration * @param router - Bridge router instance * @param ctx - Route context * @param ttlHours - TTL in hours * @param opts - State transition options * @returns State transition result */ declare function markItemsAsProcessedByTTL(router: BridgeRouter, ctx: RouteContext, ttlHours: number, opts?: StateTransitionOptions): Promise<StateTransitionResult>; /** * Mark a specific item as processed * @param router - Bridge router instance * @param ctx - Route context * @param id - Item ID * @param opts - State transition options * @returns Whether the item was marked as processed */ declare function markItemAsProcessed(router: BridgeRouter, ctx: RouteContext, id: string, opts?: StateTransitionOptions): Promise<boolean>; /** * Bucket management utilities for S3-compatible storage providers * Handles automatic bucket creation, validation, and health checks */ interface BucketManagerOptions { /** Dry run - don't actually create buckets */ dryRun?: boolean; /** Confirm the operation */ confirm?: boolean; /** Create buckets if they don't exist */ createIfMissing?: boolean; /** Validate bucket permissions */ validatePermissions?: boolean; } interface BucketStatus { /** Bucket name */ name: string; /** Whether bucket exists */ exists: boolean; /** Whether bucket is accessible */ accessible: boolean; /** Error message if bucket is not accessible */ error?: string; /** Whether bucket was created during this operation */ created?: boolean; } interface BucketManagerResult { /** Collection name */ collection: string; /** Number of buckets checked */ bucketsChecked: number; /** Number of buckets created */ bucketsCreated: number; /** Bucket status details */ bucketStatuses: BucketStatus[]; /** Whether this was a dry run */ dryRun: boolean; /** When the operation was performed */ processedAt: Date; } /** * Check and create required buckets for a collection * @param router - Bridge router instance * @param ctx - Route context * @param opts - Bucket manager options * @returns Bucket manager result */ declare function ensureBucketsExist(router: BridgeRouter, ctx: RouteContext, opts?: BucketManagerOptions): Promise<BucketManagerResult>; /** * Test S3 connectivity and list available buckets * @param router - Bridge router instance * @param ctx - Route context * @returns List of available buckets */ declare function testS3Connectivity(router: BridgeRouter, ctx: RouteContext): Promise<{ success: boolean; buckets: string[]; error?: string; }>; /** * Validate S3 configuration for DigitalOcean Spaces * @param router - Bridge router instance * @param ctx - Route context * @returns Validation result */ declare function validateSpacesConfiguration(router: BridgeRouter, ctx: RouteContext): Promise<{ valid: boolean; issues: string[]; recommendations: string[]; }>; interface FallbackOp { _id: ObjectId; type: 'CREATE' | 'UPDATE' | 'DELETE' | 'ENRICH' | 'RESTORE'; ctx: { dbName: string; collection: string; tenantId?: string; forcedIndex?: number; }; payload: any; opts: any; requestId: string; attempt: number; nextAttemptAt: Date; lastError?: string; createdAt: Date; } interface DeadLetterOp extends FallbackOp { finalError: string; failedAt: Date; } interface FallbackQueueOptions { /** Maximum number of retry attempts */ maxAttempts: number; /** Base delay in milliseconds for exponential backoff */ baseDelayMs: number; /** Maximum delay in milliseconds for exponential backoff */ maxDelayMs: number; /** Dead letter collection name */ deadLetterCollection: string; } interface EnqueueOptions { /** Request ID for idempotency */ requestId: string; /** Operation type */ type: FallbackOp['type']; /** Route context */ ctx: RouteContext; /** Operation payload */ payload: any; /** Operation options */ opts?: any; /** Immediate retry (skip delay) */ immediate?: boolean; } interface RetryResult { success: boolean; error?: string; shouldRetry: boolean; nextAttemptAt?: Date; } declare function isValidFallbackOp(op: any): op is FallbackOp; declare function isValidDeadLetterOp(op: any): op is DeadLetterOp; /** * Calculate next retry time with exponential backoff and jitter */ declare function calculateNextAttempt(attempt: number, baseDelayMs: number, maxDelayMs: number): Date; /** * Generate a unique request ID */ declare function generateRequestId(): string; /** * Check if an operation should be retried */ declare function shouldRetry(attempt: number, maxAttempts: number, lastError?: string): boolean; declare class FallbackQueue { private readonly queueCol; private readonly deadLetterCol; private readonly options; constructor(db: Db, options: FallbackQueueOptions); /** * Ensure indexes for fallback queue collections */ ensureIndexes(): Promise<void>; /**