throughmark/dist/llm/clients/openai.js

Find and Annotate Features in Images, From Objects to Concepts

import OpenAI from "openai";
import { env } from "../../utils/env.js";
import { INITIAL_ANALYSIS_PROMPT } from "../analyzer.js";
export class OpenAIClient {
    constructor(apiKey, model) {
        this.apiKey = apiKey;
        this.getModel = () => this.model;
        this.apiKey = apiKey || env.openaiApiKey;
        this.model = model || "gpt-4.1";
        this.client = new OpenAI({
            apiKey: this.apiKey,
        });
    }
    async analyze(imageBuffer, prompt) {
        console.log("Using OpenAI model:", this.model);
        const response = await this.client.chat.completions.create({
            model: this.model,
            response_format: { type: "json_object" },
            messages: [
                {
                    role: "system",
                    content: INITIAL_ANALYSIS_PROMPT,
                },
                {
                    role: "user",
                    content: [
                        {
                            type: "text",
                            text: prompt || "Highlight the damage.",
                        },
                        {
                            type: "image_url",
                            image_url: {
                                url: `data:image/png;base64,${imageBuffer.toString("base64")}`,
                            },
                        },
                    ],
                },
            ],
            max_tokens: 1024,
            temperature: 0.7,
            seed: 123, // For more consistent responses
            stream: false,
        });
        const message = response.choices[0]?.message?.content;
        if (!message) {
            throw new Error("No response from OpenAI");
        }
        return {
            text: message,
            usage: {
                input_tokens: response.usage?.prompt_tokens || 0,
                output_tokens: response.usage?.completion_tokens || 0,
            },
        };
    }
    async analyzePair(originalImage, gridImage, prompt, temperature = 0.7) {
        console.log(`Using OpenAI model: ${this.model} (temp: ${temperature})`);
        const response = await this.client.chat.completions.create({
            model: this.model,
            response_format: { type: "json_object" },
            temperature,
            messages: [
                {
                    role: "system",
                    content: INITIAL_ANALYSIS_PROMPT,
                },
                {
                    role: "user",
                    content: [
                        {
                            type: "text",
                            text: prompt || "Analyze the damage.",
                        },
                        {
                            type: "image_url",
                            image_url: {
                                url: `data:image/png;base64,${originalImage.toString("base64")}`,
                            },
                        },
                        {
                            type: "image_url",
                            image_url: {
                                url: `data:image/png;base64,${gridImage.toString("base64")}`,
                            },
                        },
                    ],
                },
            ],
        });
        return {
            text: response.choices[0]?.message?.content || "[]",
            usage: {
                input_tokens: response.usage?.prompt_tokens || 0,
                output_tokens: response.usage?.completion_tokens || 0,
            },
        };
    }
}