chonkie/dist/chonkie/chunker/code.d.ts

🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.

/** Module containing CodeChunker class. */
import { Tokenizer } from "../tokenizer";
import { CodeChunk } from "../types/code";
import { BaseChunker } from "./base";
/**
 * Options for creating a CodeChunker instance.
 */
export interface CodeChunkerOptions {
    tokenizer?: string | Tokenizer;
    chunkSize?: number;
    lang?: string;
    includeNodes?: boolean;
}
/**
 * Represents a CodeChunker instance that is also directly callable.
 * Calling it executes its `call` method (from BaseChunker), which
 * in turn calls `chunk` or `chunkBatch`.
 */
export type CallableCodeChunker = CodeChunker & {
    (text: string, showProgress?: boolean): Promise<CodeChunk[]>;
    (texts: string[], showProgress?: boolean): Promise<CodeChunk[][]>;
};
/**
 * CodeChunker class extends BaseChunker and provides functionality for chunking code.
 */
export declare class CodeChunker extends BaseChunker {
    readonly chunkSize: number;
    readonly lang?: string;
    readonly includeNodes: boolean;
    private parser;
    private language;
    private static treeSitterInitialized;
    /**
     * Private constructor. Use `CodeChunker.create()` to instantiate.
     */
    private constructor();
    /**
     * Creates and initializes a CodeChunker instance that is directly callable.
     */
    static create(options?: CodeChunkerOptions): Promise<CallableCodeChunker>;
    /**
     * Recursively finds the nearest node_modules directory from a starting directory.
     * @param startDir The directory to start searching from.
     * @returns The absolute path to the node_modules directory, or null if not found.
     */
    private static findNearestNodeModules;
    /**
     * Initialize the tree-sitter parser for the given language using WASM.
     */
    private _initParser;
    /**
     * Merge node groups together.
     */
    private _mergeNodeGroups;
    /**
     * Group child nodes based on their token counts.
     */
    private _groupChildNodes;
    /**
     * Binary search to find the first index where the value is greater than or equal to the target.
     */
    private _bisectLeft;
    /**
     * Get texts from node groups using original byte offsets.
     */
    private _getTextsFromNodeGroups;
    /**
     * Create CodeChunk objects from texts, token counts, and node groups.
     */
    private _createChunks;
    /**
     * Recursively chunks the code based on context from tree-sitter.
     */
    chunk(text: string): Promise<CodeChunk[]>;
    /**
     * Return a string representation of the CodeChunker.
     */
    toString(): string;
}