chonkie/dist/chonkie/cloud/code.js

🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.

"use strict";
/** Code chunker client for Chonkie API. */
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CodeChunker = void 0;
const base_1 = require("./base");
const code_1 = require("../types/code");
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
class CodeChunker extends base_1.CloudClient {
    constructor(apiKey, config) {
        var _a;
        super({ apiKey });
        if (!config.language) {
            throw new Error("Language is required for code chunking");
        }
        this.config = {
            tokenizerOrTokenCounter: config.tokenizerOrTokenCounter || "gpt2",
            chunkSize: config.chunkSize || 1500,
            language: config.language,
            includeNodes: (_a = config.includeNodes) !== null && _a !== void 0 ? _a : false,
        };
    }
    chunk(input) {
        return __awaiter(this, void 0, void 0, function* () {
            const formData = new FormData();
            if (input.filepath) {
                const fileContent = fs.readFileSync(input.filepath);
                const fileName = path.basename(input.filepath) || 'file.txt';
                formData.append("file", new Blob([fileContent]), fileName);
            }
            else if (input.text) {
                formData.append("text", input.text);
                // Append empty file to ensure multipart form
                formData.append("file", new Blob(), "text_input.txt");
            }
            else {
                throw new Error("Either text or filepath must be provided");
            }
            formData.append("tokenizer_or_token_counter", this.config.tokenizerOrTokenCounter);
            formData.append("chunk_size", this.config.chunkSize.toString());
            formData.append("language", this.config.language);
            formData.append("include_nodes", this.config.includeNodes.toString());
            const data = yield this.request("/v1/chunk/code", {
                method: "POST",
                body: formData,
            });
            // Convert from snake_case to camelCase
            const camelCaseData = data.map((chunk) => {
                return {
                    text: chunk.text,
                    startIndex: chunk.start_index,
                    endIndex: chunk.end_index,
                    tokenCount: chunk.token_count,
                    nodes: chunk.nodes || undefined,
                    embedding: chunk.embedding || undefined,
                };
            });
            return camelCaseData.map((chunk) => code_1.CodeChunk.fromDict(chunk));
        });
    }
    chunkBatch(inputs) {
        return __awaiter(this, void 0, void 0, function* () {
            return Promise.all(inputs.map(input => this.chunk(input)));
        });
    }
}
exports.CodeChunker = CodeChunker;
//# sourceMappingURL=code.js.map