@push.rocks/smartdata/ts/classes.doc.ts

An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.

import * as plugins from './plugins.js';

import { SmartdataDb } from './classes.db.js';
import { logger } from './logging.js';
import { SmartdataDbCursor } from './classes.cursor.js';
import { type IManager, SmartdataCollection } from './classes.collection.js';
import { SmartdataDbWatcher } from './classes.watcher.js';
import { SmartdataLuceneAdapter } from './classes.lucene.adapter.js';
/**
 * Search options for `.search()`:
 * - filter: additional MongoDB query to AND-merge
 * - validate: post-fetch validator, return true to keep a doc
 */
export interface SearchOptions<T> {
  /**
   * Additional MongoDB filter to AND‐merge into the query
   */
  filter?: Record<string, any>;
  /**
   * Post‐fetch validator; return true to keep each doc
   */
  validate?: (doc: T) => Promise<boolean> | boolean;
  /**
   * Optional MongoDB session for transactional operations
   */
  session?: plugins.mongodb.ClientSession;
}

export type TDocCreation = 'db' | 'new' | 'mixed';

// Type for decorator metadata - extends TypeScript's built-in DecoratorMetadataObject
interface ISmartdataDecoratorMetadata extends DecoratorMetadataObject {
  globalSaveableProperties?: string[];
  saveableProperties?: string[];
  uniqueIndexes?: string[];
  regularIndexes?: Array<{field: string, options: IIndexOptions}>;
  searchableFields?: string[];
  _svDbOptions?: Record<string, SvDbOptions>;
}

/**
 * Detects plain object literals only. Anything carrying a custom prototype
 * (Date, ObjectId, Binary, Buffer, RegExp, Decimal128, class instances, ...) is
 * deliberately excluded so that BSON types are never rebuilt or otherwise
 * altered while stripping undefined values.
 */
const isPlainObject = (valueArg: unknown): valueArg is Record<string, unknown> => {
  if (typeof valueArg !== 'object' || valueArg === null) {
    return false;
  }
  const prototype = Object.getPrototypeOf(valueArg);
  return prototype === Object.prototype || prototype === null;
};

/**
 * Recursively drops object properties whose value is `undefined` so MongoDB
 * stores them as absent rather than as BSON null.
 *
 * Notes on the deliberate boundaries of this traversal:
 * - Array positions are preserved: an `undefined` element stays in place and is
 *   serialized as null, exactly as `JSON.stringify` does. Removing it would
 *   shift every following index.
 * - Only plain objects are recursed into, so BSON types keep their identity.
 * - On a circular structure the offending value is returned untouched, letting
 *   BSON raise its usual "Cannot convert circular structure to BSON" error
 *   instead of overflowing the stack here.
 */
const stripUndefinedValues = <T>(valueArg: T, seenArg?: WeakSet<object>): T => {
  if (typeof valueArg !== 'object' || valueArg === null) {
    return valueArg;
  }
  const isArray = Array.isArray(valueArg);
  if (!isArray && !isPlainObject(valueArg)) {
    return valueArg;
  }
  const seen = seenArg ?? new WeakSet<object>();
  if (seen.has(valueArg as object)) {
    return valueArg;
  }
  seen.add(valueArg as object);
  if (isArray) {
    return (valueArg as unknown[]).map((entryArg) =>
      stripUndefinedValues(entryArg, seen),
    ) as unknown as T;
  }
  const strippedObject: Record<string, unknown> = {};
  for (const keyArg of Object.keys(valueArg as Record<string, unknown>)) {
    const entry = (valueArg as Record<string, unknown>)[keyArg];
    if (entry === undefined) {
      continue;
    }
    strippedObject[keyArg] = stripUndefinedValues(entry, seen);
  }
  return strippedObject as unknown as T;
};

export function globalSvDb() {
  return (value: undefined, context: ClassFieldDecoratorContext) => {
    if (context.kind !== 'field') {
      throw new Error('globalSvDb can only decorate fields');
    }

    // Store metadata at class level using Symbol.metadata
    const metadata = context.metadata as ISmartdataDecoratorMetadata;
    if (!metadata.globalSaveableProperties) {
      metadata.globalSaveableProperties = [];
    }
    metadata.globalSaveableProperties.push(String(context.name));

    // Use addInitializer to ensure prototype arrays are set up once
    context.addInitializer(function(this: any) {
      const proto = this.constructor.prototype;
      const metadata = this.constructor[Symbol.metadata];

      if (metadata && metadata.globalSaveableProperties && !proto.globalSaveableProperties) {
        // Initialize prototype array from metadata (runs once per class)
        proto.globalSaveableProperties = [...metadata.globalSaveableProperties];
      }
    });
  };
}

/**
 * Options for custom serialization/deserialization of a field.
 */
export interface SvDbOptions {
  /** Function to serialize the field value before saving to DB */
  serialize?: (value: any) => any;
  /** Function to deserialize the field value after reading from DB */
  deserialize?: (value: any) => any;
}

/**
 * saveable - saveable decorator to be used on class properties
 */
export function svDb(options?: SvDbOptions) {
  return (value: undefined, context: ClassFieldDecoratorContext) => {
    if (context.kind !== 'field') {
      throw new Error('svDb can only decorate fields');
    }

    const propName = String(context.name);

    // Store metadata at class level using Symbol.metadata
    const metadata = context.metadata as ISmartdataDecoratorMetadata;
    if (!metadata.saveableProperties) {
      metadata.saveableProperties = [];
    }
    metadata.saveableProperties.push(propName);

    // Store options in metadata
    if (options) {
      if (!metadata._svDbOptions) {
        metadata._svDbOptions = {};
      }
      metadata._svDbOptions[propName] = options;
    }

    // Use addInitializer to ensure prototype arrays are set up once
    context.addInitializer(function(this: any) {
      const proto = this.constructor.prototype;
      const ctor = this.constructor;
      const metadata = ctor[Symbol.metadata];

      if (metadata && metadata.saveableProperties && !proto.saveableProperties) {
        // Initialize prototype array from metadata (runs once per class)
        proto.saveableProperties = [...metadata.saveableProperties];
      }

      // Initialize svDbOptions from metadata
      if (metadata && metadata._svDbOptions && !ctor._svDbOptions) {
        ctor._svDbOptions = { ...metadata._svDbOptions };
      }
    });
  };
}

/**
 * searchable - marks a property as searchable with Lucene query syntax
 */
export function searchable() {
  return (value: undefined, context: ClassFieldDecoratorContext) => {
    if (context.kind !== 'field') {
      throw new Error('searchable can only decorate fields');
    }

    const propName = String(context.name);

    // Store metadata at class level
    const metadata = context.metadata as ISmartdataDecoratorMetadata;
    if (!metadata.searchableFields) {
      metadata.searchableFields = [];
    }
    metadata.searchableFields.push(propName);

    // Use addInitializer to set up constructor property once
    context.addInitializer(function(this: any) {
      const ctor = this.constructor as any;
      const metadata = ctor[Symbol.metadata];

      if (metadata && metadata.searchableFields && !Array.isArray(ctor.searchableFields)) {
        // Initialize from metadata (runs once per class)
        ctor.searchableFields = [...metadata.searchableFields];
      }
    });
  };
}

// Escape user input for safe use in MongoDB regular expressions
function escapeForRegex(input: string): string {
  return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

/**
 * unique index - decorator to mark a unique index
 */
export function unI() {
  return (value: undefined, context: ClassFieldDecoratorContext) => {
    if (context.kind !== 'field') {
      throw new Error('unI can only decorate fields');
    }

    const propName = String(context.name);

    // Store metadata at class level
    const metadata = context.metadata as ISmartdataDecoratorMetadata;
    if (!metadata.uniqueIndexes) {
      metadata.uniqueIndexes = [];
    }
    metadata.uniqueIndexes.push(propName);

    // Also mark as saveable
    if (!metadata.saveableProperties) {
      metadata.saveableProperties = [];
    }
    if (!metadata.saveableProperties.includes(propName)) {
      metadata.saveableProperties.push(propName);
    }

    // Use addInitializer to ensure prototype arrays are set up once
    context.addInitializer(function(this: any) {
      const proto = this.constructor.prototype;
      const metadata = this.constructor[Symbol.metadata];

      if (metadata && metadata.uniqueIndexes && !proto.uniqueIndexes) {
        proto.uniqueIndexes = [...metadata.uniqueIndexes];
      }

      if (metadata && metadata.saveableProperties && !proto.saveableProperties) {
        proto.saveableProperties = [...metadata.saveableProperties];
      }
    });
  };
}

/**
 * Options for MongoDB indexes
 */
export interface IIndexOptions {
  background?: boolean;
  unique?: boolean;
  sparse?: boolean;
  expireAfterSeconds?: number;
  [key: string]: any;
}

/**
 * index - decorator to mark a field for regular indexing
 */
export function index(options?: IIndexOptions) {
  return (value: undefined, context: ClassFieldDecoratorContext) => {
    if (context.kind !== 'field') {
      throw new Error('index can only decorate fields');
    }

    const propName = String(context.name);

    // Store metadata at class level
    const metadata = context.metadata as ISmartdataDecoratorMetadata;
    if (!metadata.regularIndexes) {
      metadata.regularIndexes = [];
    }
    metadata.regularIndexes.push({
      field: propName,
      options: options || {}
    });

    // Also mark as saveable
    if (!metadata.saveableProperties) {
      metadata.saveableProperties = [];
    }
    if (!metadata.saveableProperties.includes(propName)) {
      metadata.saveableProperties.push(propName);
    }

    // Use addInitializer to ensure prototype arrays are set up once
    context.addInitializer(function(this: any) {
      const proto = this.constructor.prototype;
      const metadata = this.constructor[Symbol.metadata];

      if (metadata && metadata.regularIndexes && !proto.regularIndexes) {
        proto.regularIndexes = [...metadata.regularIndexes];
      }

      if (metadata && metadata.saveableProperties && !proto.saveableProperties) {
        proto.saveableProperties = [...metadata.saveableProperties];
      }
    });
  };
}

// Helper type to extract element type from arrays or return T itself
type ElementOf<T> = T extends ReadonlyArray<infer U> ? U : T;

// Type for $in/$nin values - arrays of the element type
type InValues<T> = ReadonlyArray<ElementOf<T>>;

// Type that allows MongoDB operators on leaf values while maintaining nested type safety
export type MongoFilterCondition<T> = T | {
  $eq?: T;
  $ne?: T;
  $gt?: T;
  $gte?: T;
  $lt?: T;
  $lte?: T;
  $in?: InValues<T>;
  $nin?: InValues<T>;
  $exists?: boolean;
  $type?: string | number;
  $regex?: string | RegExp;
  $options?: string;
  $all?: T extends ReadonlyArray<infer U> ? ReadonlyArray<U> : never;
  $elemMatch?: T extends ReadonlyArray<infer U> ? MongoFilter<U> : never;
  $size?: T extends ReadonlyArray<any> ? number : never;
  $not?: MongoFilterCondition<T>;
};

export type MongoFilter<T> = {
  [K in keyof T]?: T[K] extends object 
    ? T[K] extends any[]
      ? MongoFilterCondition<T[K]>  // Arrays can have operators
      : MongoFilter<T[K]> | MongoFilterCondition<T[K]>  // Objects can be nested or have operators
    : MongoFilterCondition<T[K]>;  // Primitives get operators
} & {
  // Logical operators
  $and?: MongoFilter<T>[];
  $or?: MongoFilter<T>[];
  $nor?: MongoFilter<T>[];
  $not?: MongoFilter<T>;
  // Allow any string key for dot notation (we lose type safety here but maintain flexibility)
  [key: string]: any;
};

export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
  // SECURITY: Block $where to prevent server-side JS execution
  if (filterArg.$where !== undefined) {
    throw new Error('$where operator is not allowed for security reasons');
  }
  
  // Handle logical operators recursively
  const logicalOperators = ['$and', '$or', '$nor', '$not'];
  const processedFilter: { [key: string]: any } = {};
  
  for (const key of Object.keys(filterArg)) {
    if (logicalOperators.includes(key)) {
      if (key === '$not') {
        processedFilter[key] = convertFilterForMongoDb(filterArg[key]);
      } else if (Array.isArray(filterArg[key])) {
        processedFilter[key] = filterArg[key].map((subFilter: any) => convertFilterForMongoDb(subFilter));
      }
    }
  }
  
  // If only logical operators, return them
  const hasOnlyLogicalOperators = Object.keys(filterArg).every(key => logicalOperators.includes(key));
  if (hasOnlyLogicalOperators) {
    return processedFilter;
  }

  // Original conversion logic for non-MongoDB query objects
  const convertedFilter: { [key: string]: any } = {};
  
  // Helper to merge operator objects
  const mergeIntoConverted = (path: string, value: any) => {
    const existing = convertedFilter[path];
    if (!existing) {
      convertedFilter[path] = value;
    } else if (
      typeof existing === 'object' && !Array.isArray(existing) &&
      typeof value === 'object' && !Array.isArray(value) &&
      (Object.keys(existing).some(k => k.startsWith('$')) || Object.keys(value).some(k => k.startsWith('$')))
    ) {
      // Both have operators, merge them
      convertedFilter[path] = { ...existing, ...value };
    } else {
      // Otherwise later wins
      convertedFilter[path] = value;
    }
  };

  const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
    if (Array.isArray(filterArg2)) {
      // Arrays are typically used as values for operators like $in or as direct equality matches
      mergeIntoConverted(keyPathArg2, filterArg2);
      return;
    } else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
      // Check if this is an object with MongoDB operators
      const keys = Object.keys(filterArg2);
      const hasOperators = keys.some(key => key.startsWith('$'));
      
      if (hasOperators) {
        // This object contains MongoDB operators
        // Validate and pass through allowed operators
        const allowedOperators = [
          // Comparison operators
          '$eq', '$ne', '$gt', '$gte', '$lt', '$lte',
          // Array operators
          '$in', '$nin', '$all', '$elemMatch', '$size',
          // Element operators
          '$exists', '$type',
          // Evaluation operators (safe ones only)
          '$regex', '$options', '$text', '$mod',
          // Logical operators (nested)
          '$and', '$or', '$nor', '$not'
        ];
        
        // Check for dangerous operators
        if (keys.includes('$where')) {
          throw new Error('$where operator is not allowed for security reasons');
        }
        
        // Validate all operators are in the allowed list
        const invalidOperators = keys.filter(key => 
          key.startsWith('$') && !allowedOperators.includes(key)
        );
        
        if (invalidOperators.length > 0) {
          console.warn(`Warning: Unknown MongoDB operators detected: ${invalidOperators.join(', ')}`);
        }
        
        // For array operators, ensure the values are appropriate
        if (filterArg2.$in && !Array.isArray(filterArg2.$in)) {
          throw new Error('$in operator requires an array value');
        }
        if (filterArg2.$nin && !Array.isArray(filterArg2.$nin)) {
          throw new Error('$nin operator requires an array value');
        }
        if (filterArg2.$all && !Array.isArray(filterArg2.$all)) {
          throw new Error('$all operator requires an array value');
        }
        if (filterArg2.$size && typeof filterArg2.$size !== 'number') {
          throw new Error('$size operator requires a numeric value');
        }
        
        // Use merge helper to handle duplicate paths
        mergeIntoConverted(keyPathArg2, filterArg2);
        return;
      }
      
      // No operators, check for dots in keys
      for (const key of keys) {
        if (key.includes('.')) {
          throw new Error('keys cannot contain dots');
        }
      }
      
      // Recursively process nested objects
      for (const key of keys) {
        convertFilterArgument(`${keyPathArg2}.${key}`, filterArg2[key]);
      }
    } else {
      // Primitive values
      mergeIntoConverted(keyPathArg2, filterArg2);
    }
  };

  for (const key of Object.keys(filterArg)) {
    // Skip logical operators, they were already processed
    if (!logicalOperators.includes(key)) {
      convertFilterArgument(key, filterArg[key]);
    }
  }
  
  // Add back processed logical operators
  Object.assign(convertedFilter, processedFilter);
  
  return convertedFilter;
};

export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends IManager = any> {
  /**
   * the collection object an Doc belongs to
   */
  public static collection: SmartdataCollection<any>;
  declare public collection: SmartdataCollection<any>;
  public static defaultManager;
  public static manager;
  declare public manager: TManager;

  /**
   * Helper to get collection with fallback to static for Deno compatibility
   */
  private getCollectionSafe(): SmartdataCollection<any> {
    return this.collection || (this.constructor as any).collection;
  }

  // STATIC
  public static createInstanceFromMongoDbNativeDoc<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    mongoDbNativeDocArg: any,
  ): T {
    const newInstance = new this();
    (newInstance as any).creationStatus = 'db';
    for (const key of Object.keys(mongoDbNativeDocArg)) {
      const rawValue = mongoDbNativeDocArg[key];
      const optionsMap = (this as any)._svDbOptions || {};
      const opts = optionsMap[key];
      newInstance[key] = opts && typeof opts.deserialize === 'function'
        ? opts.deserialize(rawValue)
        : rawValue;
    }
    return newInstance;
  }

  /**
   * gets all instances as array
   * @param this
   * @param filterArg - Type-safe MongoDB filter with nested object support and operators
   * @returns
   */
  public static async getInstances<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    filterArg: MongoFilter<T>,
    opts?: { session?: plugins.mongodb.ClientSession }
  ): Promise<T[]> {
    // Pass session through to findAll for transactional queries
    const foundDocs = await (this as any).collection.findAll(
      convertFilterForMongoDb(filterArg),
      { session: opts?.session },
    );
    const returnArray: T[] = [];
    for (const foundDoc of foundDocs) {
      const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
      returnArray.push(newInstance);
    }
    return returnArray;
  }

  /**
   * Cursor-paged query with stable seek pagination — the org-standard way to
   * list unbounded collections. Orders by a comparable (ideally indexed)
   * sort field with a unique tiebreaker, returns one page plus the cursor
   * for the next, and never materializes the full result set.
   */
  public static async getPagedInstances<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    optionsArg: {
      filter?: MongoFilter<T>;
      /** Comparable, ideally indexed field that orders the pages (e.g. a numeric timestamp). */
      sortField: string;
      sortDirection?: 'asc' | 'desc';
      /** Unique tiebreaker field present on every document. Defaults to 'id'. */
      uniqueField?: string;
      /** Page size, default 100, capped at 1000. */
      limit?: number;
      /** Seek position returned as nextCursor by the previous page. */
      cursor?: { sortValue: number | string; uniqueValue: string };
      /** Also count all filter matches (extra query). */
      withTotal?: boolean;
      session?: plugins.mongodb.ClientSession;
    },
  ): Promise<{
    documents: T[];
    nextCursor?: { sortValue: number | string; uniqueValue: string };
    total?: number;
  }> {
    const collection: SmartdataCollection<T> = (this as any).collection;
    await collection.init();
    const sortDirection = optionsArg.sortDirection === 'asc' ? 1 : -1;
    const uniqueField = optionsArg.uniqueField || 'id';
    const requestedLimit = Number.isSafeInteger(optionsArg.limit) && optionsArg.limit! > 0
      ? optionsArg.limit!
      : 100;
    const limit = Math.min(requestedLimit, 1000);
    const baseSelector = convertFilterForMongoDb(optionsArg.filter || {});

    let selector: any = baseSelector;
    if (optionsArg.cursor) {
      const seekOperator = sortDirection === 1 ? '$gt' : '$lt';
      const seekSelector = {
        $or: [
          { [optionsArg.sortField]: { [seekOperator]: optionsArg.cursor.sortValue } },
          {
            [optionsArg.sortField]: optionsArg.cursor.sortValue,
            [uniqueField]: { [seekOperator]: optionsArg.cursor.uniqueValue },
          },
        ],
      };
      selector = Object.keys(baseSelector).length > 0
        ? { $and: [baseSelector, seekSelector] }
        : seekSelector;
    }

    const rawCursor = collection.mongoDbCollection
      .find(selector, { session: optionsArg.session })
      .sort({ [optionsArg.sortField]: sortDirection, [uniqueField]: sortDirection })
      .limit(limit + 1);
    try {
      const rawDocuments = await rawCursor.toArray();
      const hasMore = rawDocuments.length > limit;
      const pageRows = rawDocuments.slice(0, limit);
      const documents = pageRows.map(
        (rawDocument: any) => (this as any).createInstanceFromMongoDbNativeDoc(rawDocument) as T,
      );
      const lastRow: any = pageRows[pageRows.length - 1];
      const total = optionsArg.withTotal
        ? await collection.mongoDbCollection.countDocuments(baseSelector, {
            session: optionsArg.session,
          })
        : undefined;
      return {
        documents,
        ...(hasMore && lastRow
          ? {
              nextCursor: {
                sortValue: lastRow[optionsArg.sortField],
                uniqueValue: lastRow[uniqueField],
              },
            }
          : {}),
        ...(total !== undefined ? { total } : {}),
      };
    } finally {
      await rawCursor.close();
    }
  }

  /**
   * gets the first matching instance
   * @param this
   * @param filterArg
   * @returns
   */
  public static async getInstance<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    filterArg: MongoFilter<T>,
    opts?: { session?: plugins.mongodb.ClientSession }
  ): Promise<T> {
    // Retrieve one document, with optional session for transactions
    const foundDoc = await (this as any).collection.findOne(
      convertFilterForMongoDb(filterArg),
      { session: opts?.session },
    );
    if (foundDoc) {
      const newInstance: T = (this as any).createInstanceFromMongoDbNativeDoc(foundDoc);
      return newInstance;
    } else {
      return null as any;
    }
  }

  /**
   * get a unique id prefixed with the class name
   */
  public static async getNewId<T = any>(
    this: plugins.tsclass.typeFest.Class<T>,
    lengthArg: number = 20,
  ) {
    return `${(this as any).className}:${plugins.smartunique.shortId(lengthArg)}`;
  }

  /**
   * Get a cursor for streaming results, with optional session and native cursor modifiers.
   * @param filterArg Partial filter to apply
   * @param opts Optional session and modifier for the raw MongoDB cursor
   */
  public static async getCursor<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    filterArg: MongoFilter<T>,
    opts?: {
      session?: plugins.mongodb.ClientSession;
      modifier?: (cursorArg: plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>) => plugins.mongodb.FindCursor<plugins.mongodb.WithId<plugins.mongodb.BSON.Document>>;
    }
  ): Promise<SmartdataDbCursor<T>> {
    const collection: SmartdataCollection<T> = (this as any).collection;
    const { session, modifier } = opts || {};
    await collection.init();
    let rawCursor: plugins.mongodb.FindCursor<any> =
      collection.mongoDbCollection.find(convertFilterForMongoDb(filterArg), { session });
    if (modifier) {
      rawCursor = modifier(rawCursor);
    }
    return new SmartdataDbCursor<T>(rawCursor, this as any as typeof SmartDataDbDoc);
  }

  /**
   * watch the collection
   * @param this
   * @param filterArg
   * @param forEachFunction
   */
  /**
   * Watch the collection for changes, with optional buffering and change stream options.
   * @param filterArg MongoDB filter to select which changes to observe
   * @param opts optional ChangeStreamOptions plus bufferTimeMs
   */
  public static async watch<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    filterArg: MongoFilter<T>,
    opts?: plugins.mongodb.ChangeStreamOptions & { bufferTimeMs?: number },
  ): Promise<SmartdataDbWatcher<T>> {
    const collection: SmartdataCollection<T> = (this as any).collection;
    const watcher: SmartdataDbWatcher<T> = await collection.watch(
      convertFilterForMongoDb(filterArg),
      opts || {},
      this as any,
    );
    return watcher;
  }

  /**
   * run a function for all instances
   * @returns
   */
  public static async forEach<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    filterArg: MongoFilter<T>,
    forEachFunction: (itemArg: T) => Promise<any>,
  ) {
    const cursor: SmartdataDbCursor<T> = await (this as any).getCursor(filterArg);
    await cursor.forEach(forEachFunction);
  }

  /**
   * returns a count of the documents in the collection
   */
  public static async getCount<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    filterArg: MongoFilter<T> = {} as any,
  ) {
    const collection: SmartdataCollection<T> = (this as any).collection;
    return await collection.getCount(filterArg);
  }

  /**
   * Runs an integrity check on this collection.
   * Returns a summary with estimated vs actual counts and any duplicate unique fields.
   */
  public static async checkCollectionIntegrity<T>(
    this: plugins.tsclass.typeFest.Class<T>,
  ) {
    const collection: SmartdataCollection<T> = (this as any).collection;
    return await collection.checkCollectionIntegrity();
  }

  /**
   * Create a MongoDB filter from a Lucene query string
   * @param luceneQuery Lucene query string
   * @returns MongoDB query object
   */
  public static createSearchFilter<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    luceneQuery: string,
  ): any {
    const searchableFields = (this as any).getSearchableFields();
    if (searchableFields.length === 0) {
      throw new Error(`No searchable fields defined for class ${this.name}`);
    }
    const adapter = new SmartdataLuceneAdapter(searchableFields);
    return adapter.convert(luceneQuery);
  }
  /**
   * List all searchable fields defined on this class
   */
  public static getSearchableFields(): string[] {
    const ctor = this as any;
    return Array.isArray(ctor.searchableFields) ? ctor.searchableFields : [];
  }
  /**
   * Execute a query with optional hard filter and post-fetch validation
   */
  private static async execQuery<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    baseFilter: Record<string, any>,
    opts?: SearchOptions<T>
  ): Promise<T[]> {
    let mongoFilter = baseFilter || {};
    if (opts?.filter) {
      mongoFilter = { $and: [mongoFilter, opts.filter] };
    }
    // Fetch with optional session for transactions
    // Fetch within optional session
    let docs: T[] = await (this as any).getInstances(mongoFilter, { session: opts?.session });
    if (opts?.validate) {
      const out: T[] = [];
      for (const d of docs) {
        if (await opts.validate(d)) out.push(d);
      }
      docs = out;
    }
    return docs;
  }

  /**
   * Search documents by text or field:value syntax, with safe regex fallback
   * Supports additional filtering and post-fetch validation via opts
   * @param query A search term or field:value expression
   * @param opts Optional filter and validate hooks
   * @returns Array of matching documents
   */
  public static async search<T>(
    this: plugins.tsclass.typeFest.Class<T>,
    query: string,
    opts?: SearchOptions<T>,
  ): Promise<T[]> {
    const searchableFields = (this as any).getSearchableFields();
    if (searchableFields.length === 0) {
      throw new Error(`No searchable fields defined for class ${this.name}`);
    }
    // empty query -> return all
    const q = query.trim();
    if (!q) {
      // empty query: fetch all, apply opts
      return await (this as any).execQuery({}, opts);
    }
    // simple exact field:value (no spaces, no wildcards, no quotes)
    // simple exact field:value (no spaces, wildcards, quotes)
    const simpleExact = q.match(/^(\w+):([^"'\*\?\s]+)$/);
    if (simpleExact) {
      const field = simpleExact[1];
      const value = simpleExact[2];
      if (!searchableFields.includes(field)) {
        throw new Error(`Field '${field}' is not searchable for class ${this.name}`);
      }
      // simple field:value search
      return await (this as any).execQuery({ [field]: value }, opts);
    }
    // quoted phrase across all searchable fields: exact match of phrase
    const quoted = q.match(/^"(.+)"$|^'(.+)'$/);
    if (quoted) {
      const phrase = quoted[1] || quoted[2] || '';
      const parts = phrase.split(/\s+/).map((t) => escapeForRegex(t));
      const pattern = parts.join('\\s+');
      const orConds = searchableFields.map((f) => ({ [f]: { $regex: pattern, $options: 'i' } }));
      return await (this as any).execQuery({ $or: orConds }, opts);
    }
    // wildcard field:value (supports * and ?) -> direct regex on that field
    const wildcardField = q.match(/^(\w+):(.+[*?].*)$/);
    if (wildcardField) {
      const field = wildcardField[1];
      // Support quoted wildcard patterns: strip surrounding quotes
      let pattern = wildcardField[2];
      if ((pattern.startsWith('"') && pattern.endsWith('"')) ||
          (pattern.startsWith("'") && pattern.endsWith("'"))) {
        pattern = pattern.slice(1, -1);
      }
      if (!searchableFields.includes(field)) {
        throw new Error(`Field '${field}' is not searchable for class ${this.name}`);
      }
      // escape regex special chars except * and ?, then convert wildcards
      const escaped = pattern.replace(/([.+^${}()|[\\]\\])/g, '\\$1');
      const regexPattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
      return await (this as any).execQuery({ [field]: { $regex: regexPattern, $options: 'i' } }, opts);
    }
    // wildcard plain term across all fields (supports * and ?)
    if (!q.includes(':') && (q.includes('*') || q.includes('?'))) {
      // build wildcard regex pattern: escape all except * and ? then convert
      const escaped = q.replace(/([.+^${}()|[\\]\\])/g, '\\$1');
      const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
      const orConds = searchableFields.map((f) => ({ [f]: { $regex: pattern, $options: 'i' } }));
      return await (this as any).execQuery({ $or: orConds }, opts);
    }
    // implicit AND for multiple tokens: free terms, quoted phrases, and field:values
    {
      // Split query into tokens, preserving quoted substrings
      const rawTokens = q.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g) || [];
      // Only apply when more than one token and no boolean operators or grouping
      if (
        rawTokens.length > 1 &&
        !/(\bAND\b|\bOR\b|\bNOT\b|\(|\))/i.test(q) &&
        !/\[|\]/.test(q)
      ) {
        const andConds: any[] = [];
        for (let token of rawTokens) {
          // field:value token
          const fv = token.match(/^(\w+):(.+)$/);
          if (fv) {
            const field = fv[1];
            let value = fv[2];
            if (!searchableFields.includes(field)) {
              throw new Error(`Field '${field}' is not searchable for class ${this.name}`);
            }
            // Strip surrounding quotes if present
            if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
              value = value.slice(1, -1);
            }
            // Wildcard search?
            if (value.includes('*') || value.includes('?')) {
              const escaped = value.replace(/([.+^${}()|[\\]\\])/g, '\\$1');
              const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
              andConds.push({ [field]: { $regex: pattern, $options: 'i' } });
            } else {
              andConds.push({ [field]: value });
            }
          } else if ((token.startsWith('"') && token.endsWith('"')) || (token.startsWith("'") && token.endsWith("'"))) {
            // Quoted free phrase across all fields
            const phrase = token.slice(1, -1);
            const parts = phrase.split(/\s+/).map((t) => escapeForRegex(t));
            const pattern = parts.join('\\s+');
            andConds.push({ $or: searchableFields.map((f) => ({ [f]: { $regex: pattern, $options: 'i' } })) });
          } else {
            // Free term across all fields
            const esc = escapeForRegex(token);
            andConds.push({ $or: searchableFields.map((f) => ({ [f]: { $regex: esc, $options: 'i' } })) });
          }
        }
        return await (this as any).execQuery({ $and: andConds }, opts);
      }
    }
    // detect advanced Lucene syntax: field:value, wildcards, boolean, grouping
    const luceneSyntax = /(\w+:[^\s]+)|\*|\?|\bAND\b|\bOR\b|\bNOT\b|\(|\)/;
    if (luceneSyntax.test(q)) {
      const filter = (this as any).createSearchFilter(q);
      return await (this as any).execQuery(filter, opts);
    }
    // multi-term unquoted -> AND of regex across fields for each term
    const terms = q.split(/\s+/);
    if (terms.length > 1) {
      const andConds = terms.map((term) => {
        const esc = escapeForRegex(term);
        const ors = searchableFields.map((f) => ({ [f]: { $regex: esc, $options: 'i' } }));
        return { $or: ors };
      });
      return await (this as any).execQuery({ $and: andConds }, opts);
    }
    // single term -> regex across all searchable fields
    const esc = escapeForRegex(q);
    const orConds = searchableFields.map((f) => ({ [f]: { $regex: esc, $options: 'i' } }));
    return await (this as any).execQuery({ $or: orConds }, opts);
  }


  // INSTANCE

  // INSTANCE

  /**
   * how the Doc in memory was created, may prove useful later.
   */
  public creationStatus: TDocCreation = 'new';

  /**
   * updated from db in any case where doc comes from db
   */
  @globalSvDb()
  _createdAt: string = new Date().toISOString();

  /**
   * will be updated everytime the doc is saved
   */
  @globalSvDb()
  _updatedAt: string = new Date().toISOString();

  /**
   * an array of saveable properties of ALL doc
   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
   * Declared with definite assignment assertion to satisfy TypeScript without creating instance property
   */
  declare globalSaveableProperties: string[];

  /**
   * unique indexes
   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
   */
  declare uniqueIndexes: string[];

  /**
   * regular indexes with their options
   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
   */
  declare regularIndexes: Array<{field: string, options: IIndexOptions}>;

  /**
   * an array of saveable properties of a specific doc
   * Note: Set by decorators on prototype - NOT declared as instance property to avoid shadowing in Deno
   */
  declare saveableProperties: string[];

  /**
   * name
   */
  public name!: string;

  /**
   * primary id in the database
   */
  public dbDocUniqueId!: string;

  /**
   * class constructor
   */
  constructor() {}

  /**
   * saves this instance (optionally within a transaction)
   */
  public async save(opts?: { session?: plugins.mongodb.ClientSession }) {
    // allow hook before saving
    if (typeof (this as any).beforeSave === 'function') {
      await (this as any).beforeSave();
    }
    // tslint:disable-next-line: no-this-assignment
    const self: any = this;
    let dbResult: any;
    // update timestamp
    this._updatedAt = new Date().toISOString();
    // perform insert or update
    switch (this.creationStatus) {
      case 'db':
        dbResult = await this.getCollectionSafe().update(self, { session: opts?.session });
        break;
      case 'new':
        dbResult = await this.getCollectionSafe().insert(self, { session: opts?.session });
        this.creationStatus = 'db';
        break;
      default:
        logger.log('error', 'neither new nor in db?');
    }
    // allow hook after saving
    if (typeof (this as any).afterSave === 'function') {
      await (this as any).afterSave();
    }
    return dbResult;
  }

  /**
   * deletes a document from the database (optionally within a transaction)
   */
  public async delete(opts?: { session?: plugins.mongodb.ClientSession }) {
    // allow hook before deleting
    if (typeof (this as any).beforeDelete === 'function') {
      await (this as any).beforeDelete();
    }
    // perform deletion
    const result = await this.getCollectionSafe().delete(this, { session: opts?.session });
    // allow hook after delete
    if (typeof (this as any).afterDelete === 'function') {
      await (this as any).afterDelete();
    }
    return result;
  }

  /**
   * also store any referenced objects to DB
   * better for data consistency
   */
  public saveDeep(savedMapArg?: plugins.lik.ObjectMap<SmartDataDbDoc<any, any>>) {
    if (!savedMapArg) {
      savedMapArg = new plugins.lik.ObjectMap<SmartDataDbDoc<any, any>>();
    }
    savedMapArg.add(this);
    this.save();
    for (const propertyKey of Object.keys(this)) {
      const property: any = this[propertyKey];
      if (property instanceof SmartDataDbDoc && !savedMapArg.checkForObject(property)) {
        property.saveDeep(savedMapArg);
      }
    }
  }

  /**
   * updates an object from db
   */
  public async updateFromDb(): Promise<boolean> {
    const mongoDbNativeDoc = await this.getCollectionSafe().findOne(await this.createIdentifiableObject());
    if (!mongoDbNativeDoc) {
      return false; // Document not found in database
    }
    for (const key of Object.keys(mongoDbNativeDoc)) {
      const rawValue = mongoDbNativeDoc[key];
      const optionsMap = (this.constructor as any)._svDbOptions || {};
      const opts = optionsMap[key];
      this[key] = opts && typeof opts.deserialize === 'function'
        ? opts.deserialize(rawValue)
        : rawValue;
    }
    return true;
  }

  /**
   * creates a saveable object so the instance can be persisted as json in the database
   *
   * Properties whose value is `undefined` are omitted entirely rather than
   * written as BSON null, so "absent" stays representable. Explicit `null` is
   * untouched and remains storable wherever the field type allows it.
   */
  public async createSavableObject(): Promise<TImplements> {
    const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
    const globalProps = this.globalSaveableProperties || [];
    const specificProps = this.saveableProperties || [];
    const saveableProperties = [...globalProps, ...specificProps];
    // apply custom serialization if configured
    const optionsMap = (this.constructor as any)._svDbOptions || {};
    for (const propertyNameString of saveableProperties) {
      const rawValue = (this as any)[propertyNameString];
      const opts = optionsMap[propertyNameString];
      const serializedValue = opts && typeof opts.serialize === 'function'
        ? opts.serialize(rawValue)
        : rawValue;
      if (serializedValue === undefined) {
        // an undefined property means "absent" - skip it so the driver cannot
        // turn it into BSON null during serialization
        continue;
      }
      (saveableObject as any)[propertyNameString] = stripUndefinedValues(serializedValue);
    }
    return saveableObject as TImplements;
  }

  /**
   * creates an identifiable object for operations that require filtering
   */
  public async createIdentifiableObject() {
    const identifiableObject: any = {}; // is not exposed to outside, so any is ok here
    for (const propertyNameString of this.uniqueIndexes) {
      identifiableObject[propertyNameString] = this[propertyNameString];
    }
    return identifiableObject;
  }
}