@microsearch/lightning/dist/src/core/tokenizer.d.ts

Lightning fast text search for Node.js - blazing fast markdown and text search engine ⚡

/**
 * Tokenizes text based on configuration with query caching
 * @param text Input text
 * @param isQuery Whether this is a query (enables caching)
 * @returns Array of tokens
 */
export declare function tokenize(text: string, isQuery?: boolean): string[];
/**
 * Extracts a snippet from text containing query terms (optimized)
 * @param text Full text
 * @param queryTokens Query tokens
 * @returns Snippet with highlighted query terms
 */
export declare function generateSnippet(text: string, queryTokens: string[]): string;