UNPKG

ai

Version:

AI SDK by Vercel - The AI Toolkit for TypeScript and JavaScript

1,192 lines (1,108 loc) • 33.9 kB
import { JSONParseError, SharedV3Warning, TypeValidationError, } from '@ai-sdk/provider'; import { jsonSchema } from '@ai-sdk/provider-utils'; import { convertReadableStreamToArray } from '@ai-sdk/provider-utils/test'; import assert, { fail } from 'node:assert'; import { afterEach, beforeEach, describe, expect, it, vitest, vi, } from 'vitest'; import { z } from 'zod/v4'; import { verifyNoObjectGeneratedError as originalVerifyNoObjectGeneratedError } from '../error/verify-no-object-generated-error'; import * as logWarningsModule from '../logger/log-warnings'; import { MockLanguageModelV3 } from '../test/mock-language-model-v3'; import { MockTracer } from '../test/mock-tracer'; import { generateObject } from './generate-object'; vi.mock('../version', () => { return { VERSION: '0.0.0-test', }; }); const dummyResponseValues = { finishReason: { unified: 'stop', raw: 'stop' } as const, usage: { inputTokens: { total: 10, noCache: 10, cacheRead: undefined, cacheWrite: undefined, }, outputTokens: { total: 20, text: 20, reasoning: undefined, }, }, response: { id: 'id-1', timestamp: new Date(123), modelId: 'm-1' }, warnings: [], }; describe('generateObject', () => { let logWarningsSpy: ReturnType<typeof vitest.spyOn>; beforeEach(() => { logWarningsSpy = vitest .spyOn(logWarningsModule, 'logWarnings') .mockImplementation(() => {}); }); afterEach(() => { logWarningsSpy.mockRestore(); }); describe('output = "object"', () => { describe('result.object', () => { it('should generate object', async () => { const model = new MockLanguageModelV3({ doGenerate: { ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], }, }); const result = await generateObject({ model, schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(result.object).toMatchInlineSnapshot(` { "content": "Hello, world!", } `); expect(model.doGenerateCalls[0].prompt).toMatchInlineSnapshot(` [ { "content": [ { "text": "prompt", "type": "text", }, ], "providerOptions": undefined, "role": "user", }, ] `); expect(model.doGenerateCalls[0].responseFormat).toMatchInlineSnapshot(` { "description": undefined, "name": undefined, "schema": { "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { "content": { "type": "string", }, }, "required": [ "content", ], "type": "object", }, "type": "json", } `); }); it('should use name and description', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({ prompt, responseFormat }) => { expect(responseFormat).toStrictEqual({ type: 'json', name: 'test-name', description: 'test description', schema: { $schema: 'http://json-schema.org/draft-07/schema#', additionalProperties: false, properties: { content: { type: 'string' } }, required: ['content'], type: 'object', }, }); expect(prompt).toStrictEqual([ { role: 'user', content: [{ type: 'text', text: 'prompt' }], providerOptions: undefined, }, ]); return { ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "Hello, world!" }' }, ], }; }, }), schema: z.object({ content: z.string() }), schemaName: 'test-name', schemaDescription: 'test description', prompt: 'prompt', }); assert.deepStrictEqual(result.object, { content: 'Hello, world!' }); }); }); it('should return warnings', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], warnings: [ { type: 'other', message: 'Setting is not supported', }, ], }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(result.warnings).toStrictEqual([ { type: 'other', message: 'Setting is not supported', }, ]); }); it('should call logWarnings with the correct warnings', async () => { const expectedWarnings: SharedV3Warning[] = [ { type: 'other', message: 'Setting is not supported', }, { type: 'unsupported', feature: 'temperature', details: 'Temperature parameter not supported', }, ]; await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], warnings: expectedWarnings, }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(logWarningsSpy).toHaveBeenCalledOnce(); expect(logWarningsSpy).toHaveBeenCalledWith({ warnings: expectedWarnings, provider: 'mock-provider', model: 'mock-model-id', }); }); it('should call logWarnings with empty array when no warnings are present', async () => { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], warnings: [], // no warnings }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(logWarningsSpy).toHaveBeenCalledOnce(); expect(logWarningsSpy).toHaveBeenCalledWith({ warnings: [], provider: 'mock-provider', model: 'mock-model-id', }); }); describe('result.request', () => { it('should contain request information', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "Hello, world!" }' }, ], request: { body: 'test body', }, }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(result.request).toStrictEqual({ body: 'test body', }); }); }); describe('result.response', () => { it('should contain response information', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "Hello, world!" }' }, ], response: { id: 'test-id-from-model', timestamp: new Date(10000), modelId: 'test-response-model-id', headers: { 'custom-response-header': 'response-header-value', 'user-agent': 'ai/0.0.0-test', }, body: 'test body', }, }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(result.response).toStrictEqual({ id: 'test-id-from-model', timestamp: new Date(10000), modelId: 'test-response-model-id', headers: { 'custom-response-header': 'response-header-value', 'user-agent': 'ai/0.0.0-test', }, body: 'test body', }); }); }); describe('zod schema', () => { it('should generate object when using zod transform', async () => { const model = new MockLanguageModelV3({ doGenerate: { ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], }, }); const result = await generateObject({ model, schema: z.object({ content: z .string() .transform(value => value.length) .pipe(z.number()), }), prompt: 'prompt', }); expect(result.object).toMatchInlineSnapshot(` { "content": 13, } `); expect(model.doGenerateCalls[0].prompt).toMatchInlineSnapshot(` [ { "content": [ { "text": "prompt", "type": "text", }, ], "providerOptions": undefined, "role": "user", }, ] `); expect(model.doGenerateCalls[0].responseFormat).toMatchInlineSnapshot(` { "description": undefined, "name": undefined, "schema": { "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { "content": { "type": "string", }, }, "required": [ "content", ], "type": "object", }, "type": "json", } `); }); it('should generate object when using zod prePreprocess', async () => { const model = new MockLanguageModelV3({ doGenerate: { ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], }, }); const result = await generateObject({ model, schema: z.object({ content: z.preprocess( val => (typeof val === 'number' ? String(val) : val), z.string(), ), }), prompt: 'prompt', }); expect(result.object).toMatchInlineSnapshot(` { "content": "Hello, world!", } `); expect(model.doGenerateCalls[0].prompt).toMatchInlineSnapshot(` [ { "content": [ { "text": "prompt", "type": "text", }, ], "providerOptions": undefined, "role": "user", }, ] `); expect(model.doGenerateCalls[0].responseFormat).toMatchInlineSnapshot(` { "description": undefined, "name": undefined, "schema": { "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { "content": { "type": "string", }, }, "required": [ "content", ], "type": "object", }, "type": "json", } `); }); }); describe('custom schema', () => { it('should generate object', async () => { const model = new MockLanguageModelV3({ doGenerate: { ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], }, }); const result = await generateObject({ model, schema: jsonSchema({ type: 'object', properties: { content: { type: 'string' } }, required: ['content'], additionalProperties: false, }), prompt: 'prompt', }); expect(result.object).toMatchInlineSnapshot(` { "content": "Hello, world!", } `); expect(model.doGenerateCalls[0].prompt).toMatchInlineSnapshot(` [ { "content": [ { "text": "prompt", "type": "text", }, ], "providerOptions": undefined, "role": "user", }, ] `); expect(model.doGenerateCalls[0].responseFormat).toMatchInlineSnapshot(` { "description": undefined, "name": undefined, "schema": { "additionalProperties": false, "properties": { "content": { "type": "string", }, }, "required": [ "content", ], "type": "object", }, "type": "json", } `); }); }); describe('result.toJsonResponse', () => { it('should return JSON response', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => ({ ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "Hello, world!" }' }, ], }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); const response = result.toJsonResponse(); assert.strictEqual(response.status, 200); assert.strictEqual( response.headers.get('Content-Type'), 'application/json; charset=utf-8', ); assert.deepStrictEqual( await convertReadableStreamToArray( response.body!.pipeThrough(new TextDecoderStream()), ), ['{"content":"Hello, world!"}'], ); }); }); describe('result.providerMetadata', () => { it('should contain provider metadata', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => ({ ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "Hello, world!" }' }, ], providerMetadata: { exampleProvider: { a: 10, b: 20, }, }, }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(result.providerMetadata).toStrictEqual({ exampleProvider: { a: 10, b: 20, }, }); }); }); describe('options.headers', () => { it('should pass headers to model', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({ headers }) => { expect(headers).toStrictEqual({ 'custom-request-header': 'request-header-value', 'user-agent': 'ai/0.0.0-test', }); return { ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "headers test" }' }, ], }; }, }), schema: z.object({ content: z.string() }), prompt: 'prompt', headers: { 'custom-request-header': 'request-header-value' }, }); expect(result.object).toStrictEqual({ content: 'headers test' }); }); }); describe('options.repairText', () => { it('should be able to repair a JSONParseError', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => { return { ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "provider metadata test" ', }, ], }; }, }), schema: z.object({ content: z.string() }), prompt: 'prompt', experimental_repairText: async ({ text, error }) => { expect(error).toBeInstanceOf(JSONParseError); expect(text).toStrictEqual( '{ "content": "provider metadata test" ', ); return text + '}'; }, }); expect(result.object).toStrictEqual({ content: 'provider metadata test', }); }); it('should be able to repair a TypeValidationError', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => { return { ...dummyResponseValues, content: [ { type: 'text', text: '{ "content-a": "provider metadata test" }', }, ], }; }, }), schema: z.object({ content: z.string() }), prompt: 'prompt', experimental_repairText: async ({ text, error }) => { expect(error).toBeInstanceOf(TypeValidationError); expect(text).toStrictEqual( '{ "content-a": "provider metadata test" }', ); return `{ "content": "provider metadata test" }`; }, }); expect(result.object).toStrictEqual({ content: 'provider metadata test', }); }); it('should be able to handle repair that returns null', async () => { const result = generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => { return { ...dummyResponseValues, content: [ { type: 'text', text: '{ "content-a": "provider metadata test" }', }, ], }; }, }), schema: z.object({ content: z.string() }), prompt: 'prompt', experimental_repairText: async ({ text, error }) => { expect(error).toBeInstanceOf(TypeValidationError); expect(text).toStrictEqual( '{ "content-a": "provider metadata test" }', ); return null; }, }); expect(result).rejects.toThrow( 'No object generated: response did not match schema.', ); }); }); describe('options.providerOptions', () => { it('should pass provider options to model', async () => { const result = await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({ providerOptions }) => { expect(providerOptions).toStrictEqual({ aProvider: { someKey: 'someValue' }, }); return { ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "provider metadata test" }', }, ], }; }, }), schema: z.object({ content: z.string() }), prompt: 'prompt', providerOptions: { aProvider: { someKey: 'someValue' }, }, }); expect(result.object).toStrictEqual({ content: 'provider metadata test', }); }); }); describe('error handling', () => { function verifyNoObjectGeneratedError( error: unknown, { message }: { message: string }, ) { originalVerifyNoObjectGeneratedError(error, { message, response: { id: 'id-1', timestamp: new Date(123), modelId: 'm-1', }, usage: { inputTokens: 10, inputTokenDetails: { noCacheTokens: 10, cacheReadTokens: undefined, cacheWriteTokens: undefined, }, outputTokens: 20, outputTokenDetails: { textTokens: 20, reasoningTokens: undefined, }, totalTokens: 30, reasoningTokens: undefined, cachedInputTokens: undefined, }, finishReason: 'stop', }); } it('should throw NoObjectGeneratedError when schema validation fails', async () => { try { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": 123 }' }], }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); fail('must throw error'); } catch (error) { verifyNoObjectGeneratedError(error, { message: 'No object generated: response did not match schema.', }); } }); it('should throw NoObjectGeneratedError when parsing fails', async () => { try { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ broken json' }], }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); fail('must throw error'); } catch (error) { verifyNoObjectGeneratedError(error, { message: 'No object generated: could not parse the response.', }); } }); it('should throw NoObjectGeneratedError when parsing fails with repairResponse', async () => { try { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ broken json' }], }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', experimental_repairText: async ({ text }) => text + '{', }); fail('must throw error'); } catch (error) { verifyNoObjectGeneratedError(error, { message: 'No object generated: could not parse the response.', }); } }); it('should throw NoObjectGeneratedError when no text is available', async () => { try { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async ({}) => ({ ...dummyResponseValues, content: [], }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); fail('must throw error'); } catch (error) { verifyNoObjectGeneratedError(error, { message: 'No object generated: the model did not return a response.', }); } }); }); }); describe('output = "array"', () => { it('should generate an array with 3 elements', async () => { const model = new MockLanguageModelV3({ doGenerate: { ...dummyResponseValues, content: [ { type: 'text', text: JSON.stringify({ elements: [ { content: 'element 1' }, { content: 'element 2' }, { content: 'element 3' }, ], }), }, ], }, }); const result = await generateObject({ model, schema: z.object({ content: z.string() }), output: 'array', prompt: 'prompt', }); expect(result.object).toMatchInlineSnapshot(` [ { "content": "element 1", }, { "content": "element 2", }, { "content": "element 3", }, ] `); expect(model.doGenerateCalls[0].prompt).toMatchInlineSnapshot(` [ { "content": [ { "text": "prompt", "type": "text", }, ], "providerOptions": undefined, "role": "user", }, ] `); expect(model.doGenerateCalls[0].responseFormat).toMatchInlineSnapshot(` { "description": undefined, "name": undefined, "schema": { "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { "elements": { "items": { "additionalProperties": false, "properties": { "content": { "type": "string", }, }, "required": [ "content", ], "type": "object", }, "type": "array", }, }, "required": [ "elements", ], "type": "object", }, "type": "json", } `); }); }); describe('output = "enum"', () => { it('should generate an enum value', async () => { const model = new MockLanguageModelV3({ doGenerate: { ...dummyResponseValues, content: [ { type: 'text', text: JSON.stringify({ result: 'sunny' }), }, ], }, }); const result = await generateObject({ model, output: 'enum', enum: ['sunny', 'rainy', 'snowy'], prompt: 'prompt', }); expect(result.object).toEqual('sunny'); expect(model.doGenerateCalls[0].prompt).toMatchInlineSnapshot(` [ { "content": [ { "text": "prompt", "type": "text", }, ], "providerOptions": undefined, "role": "user", }, ] `); expect(model.doGenerateCalls[0].responseFormat).toMatchInlineSnapshot(` { "description": undefined, "name": undefined, "schema": { "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { "result": { "enum": [ "sunny", "rainy", "snowy", ], "type": "string", }, }, "required": [ "result", ], "type": "object", }, "type": "json", } `); }); }); describe('output = "no-schema"', () => { it('should generate object', async () => { const model = new MockLanguageModelV3({ doGenerate: { ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], }, }); const result = await generateObject({ model, output: 'no-schema', prompt: 'prompt', }); expect(result.object).toMatchInlineSnapshot(` { "content": "Hello, world!", } `); expect(model.doGenerateCalls[0].prompt).toMatchInlineSnapshot(` [ { "content": [ { "text": "prompt", "type": "text", }, ], "providerOptions": undefined, "role": "user", }, ] `); expect(model.doGenerateCalls[0].responseFormat).toMatchInlineSnapshot(` { "description": undefined, "name": undefined, "schema": undefined, "type": "json", } `); }); }); describe('telemetry', () => { let tracer: MockTracer; beforeEach(() => { tracer = new MockTracer(); }); it('should not record any telemetry data when not explicitly enabled', async () => { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', }); assert.deepStrictEqual(tracer.jsonSpans, []); }); it('should record telemetry data when enabled', async () => { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], response: { id: 'test-id-from-model', timestamp: new Date(10000), modelId: 'test-response-model-id', }, providerMetadata: { testProvider: { testKey: 'testValue', }, }, }), }), schema: z.object({ content: z.string() }), schemaName: 'test-name', schemaDescription: 'test description', prompt: 'prompt', topK: 0.1, topP: 0.2, frequencyPenalty: 0.3, presencePenalty: 0.4, temperature: 0.5, headers: { header1: 'value1', header2: 'value2', }, experimental_telemetry: { isEnabled: true, functionId: 'test-function-id', metadata: { test1: 'value1', test2: false, }, tracer, }, }); expect(tracer.jsonSpans).toMatchSnapshot(); }); it('should not record telemetry inputs / outputs when disabled', async () => { await generateObject({ model: new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [{ type: 'text', text: '{ "content": "Hello, world!" }' }], response: { id: 'test-id-from-model', timestamp: new Date(10000), modelId: 'test-response-model-id', }, }), }), schema: z.object({ content: z.string() }), prompt: 'prompt', experimental_telemetry: { isEnabled: true, recordInputs: false, recordOutputs: false, tracer, }, }); expect(tracer.jsonSpans).toMatchSnapshot(); }); }); describe('options.messages', () => { it('should support models that use "this" context in supportedUrls', async () => { let supportedUrlsCalled = false; class MockLanguageModelWithImageSupport extends MockLanguageModelV3 { constructor() { super({ supportedUrls: () => { supportedUrlsCalled = true; // Reference 'this' to verify context return this.modelId === 'mock-model-id' ? ({ 'image/*': [/^https:\/\/.*$/] } as Record< string, RegExp[] >) : {}; }, doGenerate: async () => ({ ...dummyResponseValues, content: [ { type: 'text', text: '{ "content": "Hello, world!" }' }, ], }), }); } } const model = new MockLanguageModelWithImageSupport(); const result = await generateObject({ model, schema: z.object({ content: z.string() }), messages: [ { role: 'user', content: [{ type: 'image', image: 'https://example.com/test.jpg' }], }, ], }); expect(result.object).toStrictEqual({ content: 'Hello, world!' }); expect(supportedUrlsCalled).toBe(true); }); }); describe('reasoning', () => { it('should include reasoning in the result', async () => { const model = new MockLanguageModelV3({ doGenerate: async () => ({ ...dummyResponseValues, content: [ { type: 'reasoning', text: 'This is a test reasoning.' }, { type: 'reasoning', text: 'This is another test reasoning.' }, { type: 'text', text: '{ "content": "Hello, world!" }' }, ], }), }); const result = await generateObject({ model, schema: z.object({ content: z.string() }), prompt: 'prompt', }); expect(result.reasoning).toMatchInlineSnapshot(` "This is a test reasoning. This is another test reasoning." `); expect(result.object).toMatchInlineSnapshot(` { "content": "Hello, world!", } `); }); }); });