embeddings-splitter/lib/newImplementation.d.ts

A typescript library to split your long texts into smaller chunks to send them to OpenAI Embeddings API

import type * as tiktoken from "@dqbd/tiktoken";
import { Document } from "./document.js";
interface TextSplitterParams {
    chunkSize: number;
    chunkOverlap: number;
}
export declare abstract class TextSplitter implements TextSplitterParams {
    chunkSize: number;
    chunkOverlap: number;
    constructor(fields?: Partial<TextSplitterParams>);
    abstract splitText(text: string): Promise<string[]>;
    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Promise<Document[]>;
    splitDocuments(documents: Document[]): Promise<Document[]>;
    private joinDocs;
    mergeSplits(splits: string[], separator: string): string[];
}
export interface CharacterTextSplitterParams extends TextSplitterParams {
    separator: string;
}
export declare class CharacterTextSplitter extends TextSplitter implements CharacterTextSplitterParams {
    separator: string;
    constructor(fields?: Partial<CharacterTextSplitterParams>);
    splitText(text: string): Promise<string[]>;
}
export interface RecursiveCharacterTextSplitterParams extends TextSplitterParams {
    separators: string[];
}
export declare class RecursiveCharacterTextSplitter extends TextSplitter implements RecursiveCharacterTextSplitterParams {
    separators: string[];
    constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>);
    splitText(text: string): Promise<string[]>;
}
export interface TokenTextSplitterParams extends TextSplitterParams {
    encodingName: tiktoken.TiktokenEmbedding;
    allowedSpecial: "all" | Array<string>;
    disallowedSpecial: "all" | Array<string>;
}
/**
 * Implementation of splitter which looks at tokens.
 */
export declare class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {
    encodingName: tiktoken.TiktokenEmbedding;
    allowedSpecial: "all" | Array<string>;
    disallowedSpecial: "all" | Array<string>;
    private tokenizer;
    constructor(fields?: Partial<TokenTextSplitterParams>);
    splitText(text: string): Promise<string[]>;
    static imports(): Promise<typeof tiktoken>;
}
export {};