chonkie
Version:
🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.
200 lines (199 loc) • 10.2 kB
TypeScript
/** Module containing SentenceChunker class. */
import { Tokenizer } from "../tokenizer";
import { SentenceChunk } from "../types/sentence";
import { BaseChunker } from "./base";
/**
* Options for creating a SentenceChunker instance.
*
* @property {string | Tokenizer} [tokenizer] - The tokenizer to use for token counting. Can be a string (model name) or a Tokenizer instance. Default: 'Xenova/gpt2'.
* @property {number} [chunkSize] - Maximum number of tokens per chunk. Must be > 0. Default: 512.
* @property {number} [chunkOverlap] - Number of tokens to overlap between consecutive chunks. Must be >= 0 and < chunkSize. Default: 0.
* @property {number} [minSentencesPerChunk] - Minimum number of sentences per chunk. Must be > 0. Default: 1.
* @property {number} [minCharactersPerSentence] - Minimum number of characters for a valid sentence. Sentences shorter than this are merged. Must be > 0. Default: 12.
* @property {boolean} [approximate] - (Deprecated) Whether to use approximate token counting. Default: false. Will be removed in future versions.
* @property {string[]} [delim] - List of sentence delimiters to use for splitting. Default: ['. ', '! ', '? ', '\n'].
* @property {('prev' | 'next' | null)} [includeDelim] - Whether to include the delimiter with the previous sentence ('prev'), next sentence ('next'), or exclude it (null). Default: 'prev'.
*/
export interface SentenceChunkerOptions {
tokenizer?: string | Tokenizer;
chunkSize?: number;
chunkOverlap?: number;
minSentencesPerChunk?: number;
minCharactersPerSentence?: number;
approximate?: boolean;
delim?: string[];
includeDelim?: "prev" | "next" | null;
}
/**
* Options for creating a SentenceChunker instance from a recipe.
*
* @property {string} [name] - The name of the recipe to get. Default: 'default'.
* @property {string} [language] - The language of the recipe to get. Default: 'en'.
* @property {string} [filePath] - Optionally, provide the path to the recipe file.
* @property {string | Tokenizer} [tokenizer] - The tokenizer to use for token counting. Can be a string (model name) or a Tokenizer instance. Default: 'Xenova/gpt2'.
* @property {number} [chunkSize] - Maximum number of tokens per chunk. Must be > 0. Default: 512.
* @property {number} [chunkOverlap] - Number of tokens to overlap between consecutive chunks. Must be >= 0 and < chunkSize. Default: 0.
* @property {number} [minSentencesPerChunk] - Minimum number of sentences per chunk. Must be > 0. Default: 1.
* @property {number} [minCharactersPerSentence] - Minimum number of characters for a valid sentence. Sentences shorter than this are merged. Must be > 0. Default: 12.
* @property {boolean} [approximate] - (Deprecated) Whether to use approximate token counting. Default: false. Will be removed in future versions.
*/
export interface SentenceChunkerRecipeOptions {
name?: string;
language?: string;
filePath?: string;
tokenizer?: string | Tokenizer;
chunkSize?: number;
chunkOverlap?: number;
minSentencesPerChunk?: number;
minCharactersPerSentence?: number;
approximate?: boolean;
}
/**
* Represents a SentenceChunker instance that is also directly callable.
* This type combines the SentenceChunker class with a function interface,
* allowing the instance to be called directly like a function.
*
* When called, it executes the `call` method inherited from BaseChunker,
* which in turn calls either `chunk` (for single text) or `chunkBatch` (for multiple texts).
*
* @example
* const chunker = await SentenceChunker.create();
* // Single text processing
* const chunks = await chunker("This is a sample text.");
* // Batch processing
* const batchChunks = await chunker(["Text 1", "Text 2"]);
*
* @type {SentenceChunker & {
* (text: string, showProgress?: boolean): Promise<SentenceChunk[]>;
* (texts: string[], showProgress?: boolean): Promise<SentenceChunk[][]>;
* }}
*/
export type CallableSentenceChunker = SentenceChunker & {
(text: string, showProgress?: boolean): Promise<SentenceChunk[]>;
(texts: string[], showProgress?: boolean): Promise<SentenceChunk[][]>;
};
/**
* SentenceChunker is a class that implements the BaseChunker interface.
* It uses a tokenizer to split text into sentences and then creates chunks of text.
*
* @extends BaseChunker
*
* @property {number} chunkSize - Maximum number of tokens per chunk.
* @property {number} chunkOverlap - Number of tokens to overlap between consecutive chunks.
* @property {number} minSentencesPerChunk - Minimum number of sentences per chunk.
* @property {number} minCharactersPerSentence - Minimum number of characters for a valid sentence.
* @property {boolean} approximate - Whether to use approximate token counting.
* @property {string[]} delim - List of sentence delimiters to use for splitting.
* @property {('prev' | 'next' | null)} includeDelim - Whether to include the delimiter with the previous sentence ('prev'), next sentence ('next'), or exclude it (null).
*
* @method chunk - Chunk a single text string.
* @method chunkBatch - Chunk an array of text strings.
* @method call - (Inherited from BaseChunker) Chunk a single text string or an array of text strings.
* @method toString - Return a string representation of the SentenceChunker.
*
* @example
* const chunker = await SentenceChunker.create();
* const chunks = await chunker("This is a sample text.");
* const batchChunks = await chunker(["Text 1", "Text 2"]);
*
* @see BaseChunker
*/
export declare class SentenceChunker extends BaseChunker {
readonly chunkSize: number;
readonly chunkOverlap: number;
readonly minSentencesPerChunk: number;
readonly minCharactersPerSentence: number;
readonly approximate: boolean;
readonly delim: string[];
readonly includeDelim: "prev" | "next" | null;
readonly sep: string;
/**
* Private constructor. Use `SentenceChunker.create()` to instantiate.
*
* @param {Tokenizer} tokenizer - The tokenizer to use for token counting.
* @param {number} chunkSize - Maximum number of tokens per chunk.
* @param {number} chunkOverlap - Number of tokens to overlap between consecutive chunks.
* @param {number} minSentencesPerChunk - Minimum number of sentences per chunk.
* @param {number} minCharactersPerSentence - Minimum number of characters for a valid sentence.
* @param {boolean} approximate - Whether to use approximate token counting.
* @param {string[]} delim - List of sentence delimiters to use for splitting.
* @param {('prev' | 'next' | null)} includeDelim - Whether to include the delimiter with the previous sentence ('prev'), next sentence ('next'), or exclude it (null).
*/
private constructor();
/**
* Creates and initializes a SentenceChunker instance that is directly callable.
*
* This method is a static factory function that returns a Promise resolving to a CallableSentenceChunker instance.
* The returned instance is a callable function that can be used to chunk text strings or arrays of text strings.
*
* @param {SentenceChunkerOptions} [options] - Options for configuring the SentenceChunker.
* @returns {Promise<CallableSentenceChunker>} A promise that resolves to a callable SentenceChunker instance.
*
* @example
* const chunker = await SentenceChunker.create();
* const chunks = await chunker("This is a sample text.");
* const batchChunks = await chunker(["Text 1", "Text 2"]);
*
* @see SentenceChunkerOptions
*/
static create(options?: SentenceChunkerOptions): Promise<CallableSentenceChunker>;
/**
* Creates and initializes a SentenceChunker instance from a recipe that is directly callable.
*
* This method loads a recipe from the Chonkie hub and uses the recipe's delimiters and settings
* to configure the SentenceChunker. The recipe delimiters override the default delimiters.
*
* @param {SentenceChunkerRecipeOptions} [options] - Options for configuring the SentenceChunker with recipe settings.
* @returns {Promise<CallableSentenceChunker>} A promise that resolves to a callable SentenceChunker instance.
*
* @example
* const chunker = await SentenceChunker.fromRecipe({ name: 'default', language: 'en' });
* const chunks = await chunker("This is a sample text.");
*
* @see SentenceChunkerRecipeOptions
*/
static fromRecipe(options?: SentenceChunkerRecipeOptions): Promise<CallableSentenceChunker>;
/**
* Fast sentence splitting while maintaining accuracy.
*
* @param {string} text - The text to split into sentences.
* @returns {string[]} An array of sentences.
*/
private _splitText;
/**
* Split text into sentences and calculate token counts for each sentence.
*
* @param {string} text - The text to split into sentences.
* @returns {Promise<Sentence[]>} An array of Sentence objects.
*/
private _prepareSentences;
/**
* Create a chunk from a list of sentences.
*
* @param {Sentence[]} sentences - The sentences to create a chunk from.
* @returns {Promise<SentenceChunk>} A promise that resolves to a SentenceChunk object.
*/
private _createChunk;
/**
* Split text into overlapping chunks based on sentences while respecting token limits.
*
* @param {string} text - The text to split into chunks.
* @returns {Promise<SentenceChunk[]>} A promise that resolves to an array of SentenceChunk objects.
*/
chunk(text: string): Promise<SentenceChunk[]>;
/**
* Binary search to find the leftmost position where value should be inserted to maintain order.
*
* @param {number[]} arr - The array to search.
* @param {number} value - The value to search for.
* @param {number} [lo] - The starting index of the search.
* @returns {number} The index of the leftmost position where value should be inserted.
*/
private _bisectLeft;
/**
* Return a string representation of the SentenceChunker.
*
* @returns {string} A string representation of the SentenceChunker.
*/
toString(): string;
}