@firefliesai/schema-forge
Version:
Transform TypeScript classes into JSON Schema definitions with automatic support for OpenAI, Anthropic, and Google Gemini function calling (tool) formats
382 lines (381 loc) • 17.1 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
require("reflect-metadata");
const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
/** new Google AI Studio API which supports Gemini Developer API and Vertex AI.*/
const genai_1 = require("@google/genai");
/** old Google AI Studio API which supports Gemini Developer API.*/
const generative_ai_1 = require("@google/generative-ai");
const openai_1 = __importDefault(require("openai"));
const complex_class_tool_dto_1 = require("./fixture/complex-class.tool.dto");
const math_tool_dto_1 = require("./fixture/math.tool.dto");
const schema_forge_1 = require("./schema-forge");
const findCapitalToolName = 'find_capital';
const findCapitalToolDesc = 'Find the capital of a given state';
const userMessage = 'What is the capital of California?';
let CapitalTool = class CapitalTool {
};
__decorate([
(0, schema_forge_1.ToolProp)({
description: 'The name of the capital to find',
}),
__metadata("design:type", String)
], CapitalTool.prototype, "name", void 0);
__decorate([
(0, schema_forge_1.ToolProp)({
description: 'alternative names or nicknames of the capital',
isOptional: true,
}),
__metadata("design:type", String)
], CapitalTool.prototype, "alias", void 0);
CapitalTool = __decorate([
(0, schema_forge_1.ToolMeta)({
name: findCapitalToolName,
description: findCapitalToolDesc,
})
], CapitalTool);
const jsonSchema = (0, schema_forge_1.classToJsonSchema)(CapitalTool);
const openai = new openai_1.default();
const anthropic = new sdk_1.default();
const gemini = new genai_1.GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY });
const geminiOldApi = new generative_ai_1.GoogleGenerativeAI(process.env.GEMINI_API_KEY);
jest.setTimeout(60 * 1000);
describe('LLM Tool Call and Structured Output Tests', () => {
it('OpenAI (Chat Completions API) - Function calling with JSON Schema', async () => {
// Using the new helper function to convert JSON schema to OpenAI tool
const tool = (0, schema_forge_1.jsonSchemaToOpenAITool)(jsonSchema, {
name: findCapitalToolName,
description: findCapitalToolDesc,
});
const completion = await openai.chat.completions.create({
model: 'gpt-4o-mini',
messages: [
{
role: 'user',
content: userMessage,
},
],
tools: [tool],
tool_choice: 'required',
});
expect(completion.choices[0].message.tool_calls[0].function.name).toBe(findCapitalToolName);
const data = JSON.parse(completion.choices[0].message.tool_calls[0].function.arguments);
expect(data.name).toBeDefined();
});
it('OpenAI (Chat Completions API) - Function calling with direct class conversion', async () => {
const tool = (0, schema_forge_1.classToOpenAITool)(CapitalTool);
const completion = await openai.chat.completions.create({
model: 'gpt-4o-mini',
messages: [
{
role: 'user',
content: userMessage,
},
],
tools: [tool],
tool_choice: 'required',
});
expect(completion.choices[0].message.tool_calls[0].function.name).toBe(findCapitalToolName);
const data = JSON.parse(completion.choices[0].message.tool_calls[0].function.arguments);
expect(data.name).toBeDefined();
});
it('OpenAI (Chat Completions API) - Function calling with direct class conversion, with complex structured tool setup', async () => {
/** a simpler one first */
const messages = ['4+2=?'];
const mathTool = (0, schema_forge_1.classToOpenAITool)(math_tool_dto_1.MathToolDto);
const completion1 = await openai.chat.completions.create({
model: 'gpt-4o-mini',
messages: [
{
role: 'user',
content: messages[0],
},
],
tools: [mathTool],
tool_choice: 'required',
});
const jsonResp = JSON.parse(completion1.choices[0].message?.tool_calls[0].function.arguments);
expect(jsonResp.sum).toBe(6);
/** real complex one */
const userMessage = `You are a helpful AI assistant. Based on the following JSON schema for a game character, please generate a mock response that follows the schema exactly. Make the data realistic and consistent with a game character context.
`;
const expGameCharV2Tool = (0, schema_forge_1.classToOpenAITool)(complex_class_tool_dto_1.GameCharacterV2);
const completion = await openai.chat.completions.create({
model: 'gpt-4o-mini',
messages: [
{
role: 'user',
content: userMessage,
},
],
tools: [expGameCharV2Tool],
tool_choice: 'required',
});
const data = JSON.parse(completion.choices[0].message?.tool_calls[0].function.arguments);
expect(data.location).toBeDefined();
expect(data.location.city).toBeDefined();
expect(data.location.country).toBeDefined();
expect(data.banks[0].account).toBeDefined();
expect(data.banks[0].bankName).toBeDefined();
expect(data.level).toBeDefined();
expect(data.name).toBeDefined();
expect(data.rank).toBeDefined();
expect(data.status).toBeDefined();
expect(data.location).toBeDefined();
expect(data.titles[0]).toBeDefined();
expect(data.scores[0]).toBeGreaterThan(0);
expect(data.availableStatuses[0]).toBeDefined();
});
it('OpenAI (Chat Completions API) - Structured output with response_format', async () => {
const responseFormat = (0, schema_forge_1.classToOpenAIResponseFormatJsonSchema)(CapitalTool, {
// Enable structured output for OpenAI
forStructuredOutput: true,
});
const completion = await openai.chat.completions.create({
model: 'gpt-4o-mini',
messages: [
{
role: 'system',
content: 'You are a helpful assistant',
},
{
role: 'user',
content: userMessage,
},
],
response_format: responseFormat,
});
const data = JSON.parse(completion.choices[0].message.content);
expect(data.name).toBeDefined();
});
it('OpenAI (Response API) - Function calling with direct class conversion', async () => {
const tool = (0, schema_forge_1.classToOpenAIResponseApiTool)(CapitalTool);
const response = await openai.responses.create({
model: 'gpt-4o-mini',
/** it is equal to deprecated system role message */
instructions: 'You are a helpful assistant',
input: userMessage,
tools: [tool],
tool_choice: 'required',
});
if (response.output[0].type === 'function_call') {
const data = JSON.parse(response.output[0].arguments);
expect(data.name).toBeDefined();
}
else {
throw new Error('Tool use not found');
}
});
it('OpenAI (Response API) - Structured output with text.format', async () => {
const responseFormat = (0, schema_forge_1.classToOpenAIResponseApiTextSchema)(CapitalTool, {
forStructuredOutput: true,
});
const response = await openai.responses.create({
model: 'gpt-4o-mini',
/** it is equal to deprecated system role message */
instructions: 'You are a helpful assistant',
input: userMessage,
text: {
format: responseFormat,
},
});
if (response.output[0].type === 'message' &&
response.output[0].content[0].type === 'output_text') {
const data = JSON.parse(response.output[0].content[0].text);
expect(data.name).toBeDefined();
}
else {
throw new Error('Tool use not found');
}
});
it('Anthropic Claude - Tool use with JSON Schema conversion', async () => {
// Using the new helper function to convert JSON schema to Anthropic tool
const tool = (0, schema_forge_1.jsonSchemaToAnthropicTool)(jsonSchema, {
name: findCapitalToolName,
description: findCapitalToolDesc,
});
const message = await anthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 1000,
messages: [
{
role: 'user',
content: userMessage,
},
],
tools: [tool],
tool_choice: { type: 'any' },
});
if (message.content[0].type === 'tool_use') {
expect(message.content[0].name).toBe(findCapitalToolName);
const data = message.content[0].input;
expect(data.name).toBeDefined();
}
else {
throw new Error('Tool use not found');
}
});
it('Anthropic Claude - Tool use with direct class conversion', async () => {
const claudeTool = (0, schema_forge_1.classToAnthropicTool)(CapitalTool);
const message = await anthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 1000,
messages: [
{
role: 'user',
content: userMessage,
},
],
tools: [claudeTool],
tool_choice: { type: 'any' },
});
if (message.content[0].type === 'tool_use') {
expect(message.content[0].name).toBe(findCapitalToolName);
const data = message.content[0].input;
expect(data.name).toBeDefined();
}
else {
throw new Error('Tool use not found');
}
});
it('OpenAI to Anthropic - Convert OpenAI Chat Completions tool to Anthropic format', async () => {
// First create an OpenAI tool
const openaiTool = (0, schema_forge_1.classToOpenAITool)(CapitalTool);
// Convert OpenAI tool to JSON Schema
const { schema, metadata } = (0, schema_forge_1.openAIToolToJsonSchema)(openaiTool);
// Convert JSON Schema to Anthropic tool
const anthropicTool = (0, schema_forge_1.jsonSchemaToAnthropicTool)(schema, metadata);
// Verify the conversion maintains all necessary information
expect(anthropicTool.name).toBe(findCapitalToolName);
expect(anthropicTool.description).toBe(findCapitalToolDesc);
// Test the converted tool with Anthropic API
const message = await anthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 1000,
messages: [
{
role: 'user',
content: userMessage,
},
],
tools: [anthropicTool],
tool_choice: { type: 'any' },
});
if (message.content[0].type === 'tool_use') {
expect(message.content[0].name).toBe(findCapitalToolName);
const data = message.content[0].input;
expect(data.name).toBeDefined();
}
else {
throw new Error('Tool use not found');
}
});
it('OpenAI to Anthropic - Convert OpenAI Response API tool to Anthropic format', async () => {
// First create an OpenAI Response API tool
const responseApiTool = (0, schema_forge_1.classToOpenAIResponseApiTool)(CapitalTool);
// Convert OpenAI Response API tool to JSON Schema
const { schema, metadata } = (0, schema_forge_1.openAIResponseApiToolToJsonSchema)(responseApiTool);
// Convert JSON Schema to Anthropic tool
const anthropicTool = (0, schema_forge_1.jsonSchemaToAnthropicTool)(schema, metadata);
// Verify the conversion maintains all necessary information
expect(anthropicTool.name).toBe(findCapitalToolName);
expect(anthropicTool.description).toBe(findCapitalToolDesc);
// Test the converted tool with Anthropic API
const message = await anthropic.messages.create({
model: 'claude-3-7-sonnet-20250219',
max_tokens: 1000,
messages: [
{
role: 'user',
content: userMessage,
},
],
tools: [anthropicTool],
tool_choice: { type: 'any' },
});
if (message.content[0].type === 'tool_use') {
expect(message.content[0].name).toBe(findCapitalToolName);
const data = message.content[0].input;
expect(data.name).toBeDefined();
}
else {
throw new Error('Tool use not found');
}
});
it('Google Gemini (@google/genai) - Function calling with direct class conversion', async () => {
const tool = (0, schema_forge_1.classToGeminiTool)(CapitalTool);
const response = await gemini.models.generateContent({
model: 'gemini-2.0-flash-001',
contents: userMessage,
config: {
// we trust the auto choice of Gemini would work for the current case
// toolConfig: {
// functionCallingConfig: {
// // Force it to call any function
// mode: FunctionCallingConfigMode.ANY,
// allowedFunctionNames: [tool.name],
// },
// },
tools: [{ functionDeclarations: [tool] }],
},
});
expect(response.functionCalls[0].name).toBe(findCapitalToolName);
const data = response.functionCalls[0].args;
expect(data.name).toBeDefined();
});
it('Google Gemini (@google/generative-ai) - Function calling with direct class conversion', async () => {
const tool = (0, schema_forge_1.classToGeminiOldTool)(CapitalTool);
const model = geminiOldApi.getGenerativeModel({
model: 'gemini-2.0-flash-001',
tools: [
{
functionDeclarations: [tool],
},
],
});
const result = await model.generateContent([userMessage]);
expect(result.response.candidates[0].content.parts[0].functionCall.name).toBe(findCapitalToolName);
const data = result.response.candidates[0].content.parts[0].functionCall.args;
expect(data.name).toBeDefined();
});
it('Google Gemini (@google/genai) - Structured output with responseSchema', async () => {
// No forStructuredOutput needed for Gemini
const responseSchema = (0, schema_forge_1.classToGeminiResponseSchema)(CapitalTool);
const response = await gemini.models.generateContent({
model: 'gemini-2.0-flash-001',
contents: userMessage,
config: {
responseMimeType: 'application/json',
responseSchema: responseSchema,
},
});
const data = JSON.parse(response.text);
expect(data.name).toBeDefined();
});
it('Google Gemini (@google/generative-ai) - Structured output with responseSchema', async () => {
// No forStructuredOutput needed for Gemini
const responseSchema = (0, schema_forge_1.classToGeminiOldResponseSchema)(CapitalTool);
const model = geminiOldApi.getGenerativeModel({
model: 'gemini-2.0-flash-001',
generationConfig: {
responseMimeType: 'application/json',
responseSchema: responseSchema,
},
});
const result = await model.generateContent([userMessage]);
const data = JSON.parse(result.response.candidates[0].content.parts[0].text);
expect(data.name).toBeDefined();
});
});