modelmix/test/images.test.js

🧬 Reliable interface with automatic fallback for AI LLMs.

const { expect } = require('chai');
const sinon = require('sinon');
const nock = require('nock');
const { ModelMix } = require('../index.js');

describe('Image Processing and Multimodal Support Tests', () => {

    afterEach(() => {
        nock.cleanAll();
        sinon.restore();
    });

    describe('Image Data Handling', () => {
        let model;

        const max_history = 2;

        beforeEach(() => {
            model = ModelMix.new({
                config: { debug: false },
                config: { max_history }
            });
        });

        it('should handle base64 image data correctly', async () => {
            const base64Image = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC';

            // Use gpt5mini (chat/completions) - gpt52 uses Responses API which has different image format
            model.gpt5mini()
                .addText('What do you see in this image?')
                .addImageFromUrl(base64Image);

            nock('https://api.openai.com')
                .post('/v1/chat/completions')
                .reply(function (uri, body) {
                    expect(body.messages[1].content).to.be.an('array');
                    expect(body.messages[1].content).to.have.length(max_history);
                    expect(body.messages[1].content[max_history - 1].image_url.url).to.equal(base64Image);

                    return [200, {
                        choices: [{
                            message: {
                                role: 'assistant',
                                content: 'I can see a small test image'
                            }
                        }]
                    }];
                });

            const response = await model.message();
            expect(response).to.include('I can see a small test image');
        });

        it('should support multimodal with sonnet46()', async () => {
            const base64Image = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC';

            model.sonnet46()
                .addText('Describe this image')
                .addImageFromUrl(base64Image);

            // Claude expects images as base64 in the content array
            // We'll check that the message is formatted as expected
            nock('https://api.anthropic.com')
                .post('/v1/messages')
                .reply(function (uri, body) {
                    expect(body.messages).to.be.an('array');
                    // Find the message with the image
                    const userMsg = body.messages.find(m => m.role === 'user');
                    expect(userMsg).to.exist;
                    const imageContent = userMsg.content.find(c => c.type === 'image');
                    expect(imageContent).to.exist;
                    expect(imageContent.source.type).to.equal('base64');
                    expect(imageContent.source.data).to.equal(base64Image.split(',')[1]);
                    expect(imageContent.source.media_type).to.equal('image/png');
                    return [200, {
                        content: [{ type: "text", text: "This is a small PNG test image." }],
                        role: "assistant"
                    }];
                });

            const response = await model.message();
            expect(response).to.include('small PNG test image');
        });

        it('should detect image mime type from buffer when content-type header is missing', async () => {
            const imageUrl = 'https://assets.example.com/test-image';
            const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC';
            const pngBuffer = Buffer.from(pngBase64, 'base64');

            model.sonnet46()
                .addText('Describe this image')
                .addImageFromUrl(imageUrl);

            // No content-type header on purpose: this forces buffer-based detection.
            nock('https://assets.example.com')
                .get('/test-image')
                .reply(200, pngBuffer);

            nock('https://api.anthropic.com')
                .post('/v1/messages')
                .reply(function (uri, body) {
                    const userMsg = body.messages.find(m => m.role === 'user');
                    expect(userMsg).to.exist;
                    const imageContent = userMsg.content.find(c => c.type === 'image');
                    expect(imageContent).to.exist;
                    expect(imageContent.source.type).to.equal('base64');
                    expect(imageContent.source.media_type).to.equal('image/png');
                    expect(imageContent.source.data).to.equal(pngBase64);
                    return [200, {
                        content: [{ type: "text", text: "Image received." }],
                        role: "assistant"
                    }];
                });

            const response = await model.message();
            expect(response).to.include('Image received.');
        });

    });
});