taskforce-aiagent/dist/cjs/fineTune/fineTuner.js

TaskForce is a modular, open-source, production-ready TypeScript agent framework for orchestrating AI agents, LLM-powered autonomous agents, task pipelines, dynamic toolchains, RAG workflows and memory/retrieval systems.

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.OpenAiFineTuner = exports.MODELS_PATH = void 0;
exports.trainAndRegisterFineTunedModel = trainAndRegisterFineTunedModel;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const openai_1 = require("openai");
const enum_js_1 = require("../configs/enum.js");
const aiConfig_js_1 = require("../configs/aiConfig.js");
const helper_js_1 = require("../helpers/helper.js");
exports.MODELS_PATH = path_1.default.join(process.cwd(), "fine-tuned/models.json");
class OpenAiFineTuner {
    static async modelTraining(options) {
        const { agentName, modelKey, baseModel, source, validation, suffix } = options;
        if (!agentName || !baseModel || !source) {
            throw new Error("Missing required fields: agentName, baseModel, and source are mandatory.");
        }
        console.log(`\n🚀 Starting model training for agent '${agentName}' with model '${baseModel}'...`);
        console.log("\n🎯 Training completed and registered successfully. Now waiting for model to be ready...");
        return await trainAndRegisterFineTunedModel({
            agentName,
            modelKey,
            baseModel,
            source,
            validation,
            suffix,
        });
    }
}
exports.OpenAiFineTuner = OpenAiFineTuner;
async function trainAndRegisterFineTunedModel(options) {
    const { agentName, baseModel, modelKey, source, validation, suffix } = options;
    const config = aiConfig_js_1.aiConfig[baseModel];
    if (!config || config.model.provider !== enum_js_1.SupportedModelProvider.OPENAI) {
        throw new Error(`Only OpenAI fine-tuning is supported. '${baseModel}' is not valid.`);
    }
    const openai = new openai_1.OpenAI({ apiKey: config.apiKey });
    // Upload training file(s)
    const trainingFile = await uploadFile(source, openai);
    const validationFile = validation
        ? await uploadFile(validation, openai)
        : undefined;
    const job = await openai.fineTuning.jobs.create({
        training_file: trainingFile.id,
        model: baseModel,
        validation_file: validationFile?.id,
        suffix: suffix || agentName.toLowerCase().replace(/\s+/g, "-"),
    });
    console.log("🎓 Fine-tuning started:", job.id);
    const finalJob = await waitForModelReady(openai, job.id);
    const fineTunedModel = finalJob.fine_tuned_model;
    const baseTokenLimit = helper_js_1.baseModelTokenLimits[baseModel] ??
        helper_js_1.baseModelTokenLimits[config.model.name] ??
        16000;
    const newEntry = {
        [modelKey]: {
            apiKey: config.apiKey,
            model: {
                name: fineTunedModel,
                provider: "openai",
                supportsTools: false,
                maxContextTokens: baseTokenLimit,
            },
        },
    };
    const fullPath = path_1.default.resolve(exports.MODELS_PATH);
    const existing = fs_1.default.existsSync(fullPath)
        ? JSON.parse(fs_1.default.readFileSync(fullPath, "utf-8"))
        : {};
    if (existing[modelKey]) {
        throw new Error(`Model key '${modelKey}' already exists in finetunedModels.json. Please choose a different key.`);
    }
    const updated = { ...existing, ...newEntry };
    fs_1.default.mkdirSync(path_1.default.dirname(fullPath), { recursive: true });
    fs_1.default.writeFileSync(fullPath, JSON.stringify(updated, null, 2), "utf-8");
    console.log(`✅ Fine-tuned model saved to models.json under key '${modelKey}'`);
}
async function waitForModelReady(openai, jobId, intervalMs = 5000) {
    console.log(`⏳ Waiting for fine-tuning job '${jobId}' to complete...`);
    while (true) {
        const job = await openai.fineTuning.jobs.retrieve(jobId);
        if (job.status === "succeeded") {
            console.log(`✅ Fine-tuning complete. Model: ${job.fine_tuned_model}`);
            return job;
        }
        if (job.status === "failed") {
            console.error(`❌ Fine-tuning failed.`);
            throw new Error("Fine-tuning job failed.");
        }
        console.log(`🟡 Status: ${job.status} (checking again in ${intervalMs / 1000}s)`);
        await new Promise((res) => setTimeout(res, intervalMs));
    }
}
async function uploadFile(input, openai) {
    if (input.startsWith("http")) {
        const res = await fetch(input);
        const data = await res.text();
        const tmpPath = path_1.default.join(".tmp", `remote-${Date.now()}.jsonl`);
        fs_1.default.mkdirSync(".tmp", { recursive: true });
        fs_1.default.writeFileSync(tmpPath, data, "utf-8");
        return await openai.files.create({
            file: fs_1.default.createReadStream(tmpPath),
            purpose: "fine-tune",
        });
    }
    else {
        return await openai.files.create({
            file: fs_1.default.createReadStream(input),
            purpose: "fine-tune",
        });
    }
}