@twilio-alpha/assistants-eval/src/providers/twilio-agent.ts

promptfoo extension for writing AI evaluations for Twilio AI Assistants

import {
  CallApiContextParams,
  CallApiOptionsParams,
  ProviderOptions,
} from 'promptfoo';
import type { MergeDeep } from 'type-fest';

import { AgentProvider } from './agent';
import TwilioProvider, { TwilioProviderOptions } from './twilio';

export type TwilioAgentProviderOptions = MergeDeep<
  TwilioProviderOptions,
  {
    config: {
      userProvider?: ProviderOptions;
    };
  }
>;

export class TwilioAgentProvider extends AgentProvider {
  constructor(options: TwilioAgentProviderOptions) {
    const { userProvider, ...agentProvider } = options.config ?? {};
    super({
      id: options.id,
      label: options.label,
      config: {
        userProvider: userProvider ?? 'openai:chat:gpt-4o-mini',
        agentProvider: {
          id: 'package:@twilio-alpha/assistants-eval:TwilioProvider',
          config: { ...agentProvider },
        },
        maxTurns: undefined,
      },
      env: options.env,
    });
  }

  get agentProviderInstance(): TwilioProvider {
    return this.agentProvider as TwilioProvider;
  }

  async callApi(
    prompt: string,
    context?: CallApiContextParams,
    callApiOptions?: CallApiOptionsParams,
  ) {
    await this.ensureProviders();

    if (!this.userProvider || !this.agentProvider) {
      throw new Error('Providers not loaded');
    }

    if (
      typeof context?.vars.maxTurns === 'number' &&
      context?.vars.maxTurns > 0
    ) {
      if (!context?.vars?.sessionId) {
        throw new Error(
          'sessionId variable is required when using maxTurns variable',
        );
      }
      return super.callApi(prompt, context, callApiOptions);
    }

    return this.agentProvider.callApi(prompt, context, callApiOptions);
  }
}

export default TwilioAgentProvider;