chonkie/dist/chonkie/cloud/neural.js

🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.

"use strict";
/** Neural chunker client for Chonkie API. */
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.NeuralChunker = void 0;
const base_1 = require("./base");
const base_2 = require("../types/base");
class NeuralChunker extends base_1.CloudClient {
    constructor(apiKey, config = {}) {
        super({ apiKey });
        this.config = {
            model: config.model || "mirth/chonky_modernbert_large_1",
            minCharactersPerChunk: config.minCharactersPerChunk || 10,
        };
    }
    chunk(input) {
        return __awaiter(this, void 0, void 0, function* () {
            const formData = new FormData();
            if (input.filepath) {
                formData.append("file", input.filepath);
            }
            else if (input.text) {
                // JSON encode the text
                formData.append("text", JSON.stringify(input.text));
                // Append empty file to ensure multipart form
                formData.append("file", new Blob(), "text_input.txt");
            }
            else {
                throw new Error("Either text or file must be provided");
            }
            formData.append("embedding_model", this.config.model);
            formData.append("min_characters_per_chunk", this.config.minCharactersPerChunk.toString());
            formData.append("return_type", "chunks");
            const data = yield this.request("/v1/chunk/neural", {
                method: "POST",
                body: formData,
            });
            // Convert from snake_case to camelCase
            const camelCaseData = data.map((chunk) => {
                return {
                    text: chunk.text,
                    startIndex: chunk.start_index,
                    endIndex: chunk.end_index,
                    tokenCount: chunk.token_count,
                    embedding: chunk.embedding || undefined,
                };
            });
            return camelCaseData.map((chunk) => base_2.Chunk.fromDict(chunk));
        });
    }
    chunkBatch(inputs) {
        return __awaiter(this, void 0, void 0, function* () {
            return Promise.all(inputs.map(input => this.chunk(input)));
        });
    }
}
exports.NeuralChunker = NeuralChunker;
//# sourceMappingURL=neural.js.map