chonkie/dist/chonkie/chunker/base.d.ts

🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.

/** Base Chunking Class. **/
import { Tokenizer } from "../tokenizer";
import { Chunk } from "../types/base";
/**
 * Base class for all chunking classes.
 *
 * This abstract class provides a common interface and shared logic for all chunking implementations.
 * It supports chunking a single text or a batch of texts, with optional concurrency and progress reporting.
 *
 * Subclasses must implement the `chunk` method to define how a single text is chunked.
 *
 * @template T - The type of chunk produced (usually `Chunk[]` or `string[]`).
 *
 * @property {Tokenizer} tokenizer - The tokenizer instance used for chunking operations.
 * @property {boolean} _useConcurrency - Whether to use concurrent processing for batch chunking (default: true).
 *
 * @example
 * class MyChunker extends BaseChunker {
 *   async chunk(text: string): Promise<Chunk[]> {
 *     // ... implementation ...
 *   }
 * }
 *
 * const chunker = new MyChunker(tokenizer);
 * const chunks = await chunker.call("Some text");
 * const batchChunks = await chunker.call(["Text 1", "Text 2"], true);
 */
export declare abstract class BaseChunker {
    protected tokenizer: Tokenizer;
    protected _useConcurrency: boolean;
    constructor(tokenizer: Tokenizer);
    /**
     * Returns a string representation of the chunker instance.
     *
     * @returns {string} The class name and constructor signature.
     */
    toString(): string;
    /**
     * Call the chunker with a single string or an array of strings.
     *
     * If a single string is provided, returns the result of `chunk(text)`.
     * If an array of strings is provided, returns the result of `chunkBatch(texts, showProgress)`.
     *
     * @param {string | string[]} textOrTexts - The text or array of texts to chunk.
     * @param {boolean} [showProgress=false] - Whether to display progress for batch operations (only applies to arrays).
     * @returns {Promise<Chunk[] | Chunk[][]>} The chunked result(s).
     * @throws {Error} If input is not a string or array of strings.
     */
    call(text: string, showProgress?: boolean): Promise<Chunk[]>;
    call(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
    /**
     * Process a batch of texts sequentially (one after another).
     *
     * @protected
     * @param {string[]} texts - The texts to chunk.
     * @param {boolean} [showProgress=false] - Whether to display progress in the console.
     * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
     */
    protected _sequential_batch_processing(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
    /**
     * Process a batch of texts concurrently using Promise.all.
     *
     * @protected
     * @param {string[]} texts - The texts to chunk.
     * @param {boolean} [showProgress=false] - Whether to display progress in the console.
     * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
     */
    protected _concurrent_batch_processing(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
    /**
     * Abstract method to chunk a single text. Must be implemented by subclasses.
     *
     * @param {string} text - The text to chunk.
     * @returns {Promise<Chunk[]>} The chunked representation of the input text.
     * @abstract
     */
    abstract chunk(text: string): Promise<Chunk[]>;
    /**
     * Chunk a batch of texts, using either concurrent or sequential processing.
     *
     * If only one text is provided, processes it directly without batch overhead.
     *
     * @param {string[]} texts - The texts to chunk.
     * @param {boolean} [showProgress=true] - Whether to display progress in the console.
     * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
     */
    chunkBatch(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
}