@twilio-alpha/assistants-eval/src/providers/twilio.ts

promptfoo extension for writing AI evaluations for Twilio AI Assistants

import { randomUUID } from 'crypto';

import {
  CallApiContextParams,
  CallApiOptionsParams,
  EnvOverrides,
  ProviderOptions,
  ProviderResponse,
} from 'promptfoo';

import { getOptionalEnv, getRequiredEnv } from '../utils/envs';
import ExternalProvider from './external';

type Realm = 'dev' | 'stage' | 'prod';

type BaseResponse = {
  aborted: boolean;
  account_sid: string;
  status: 'Success' | 'Failure';
  body: string;
};
type SuccessResponse = BaseResponse & {
  status: 'Success';
  body: string;
  flagged: boolean;
  session_id: string;
};
type FailureResponse = BaseResponse & {
  status: 'Failure';
  error: string;
};
type AssistantResponse = SuccessResponse | FailureResponse;

type EnvOverridesWithTwilio = EnvOverrides & {
  TWILIO_ACCOUNT_SID?: string;
  TWILIO_AUTH_TOKEN?: string;
  TWILIO_REGION?: string;
  TWILIO_ASSISTANT_ID?: string;
};

export type TwilioProviderOptions = ProviderOptions & {
  config: {
    twilioRegion?: Realm;
    assistantId?: string;
    accountSid?: string;
    authTokenKey?: string;
    sessionId?: string;
    identity?: string;
    constellationOverride?: string;
  };
  env: EnvOverridesWithTwilio;
};

export type TwilioProviderResponse = ProviderResponse & {
  metadata?: {
    sessionId: string;
    assistantId: string;
    flagged: boolean;
    accountSid: string;
  };
};

export function getAssistantUrl(assistantId: string, region: string) {
  const sub = region === 'prod' ? '' : `${region}.`;
  return `https://assistants.${sub}twilio.com/v1/Assistants/${assistantId}/Messages`;
}

export default class TwilioProvider extends ExternalProvider {
  private readonly sessionId?: string;

  private readonly userIdentity: string;

  private readonly region: string;

  constructor(options: TwilioProviderOptions) {
    const { config, id, label, env } = options;
    const region =
      config.twilioRegion ||
      env?.TWILIO_REGION ||
      getOptionalEnv('TWILIO_REGION') ||
      'prod';

    const assistantIdConfigValue =
      typeof config.assistantId === 'string'
        ? config.assistantId
        : config.assistantId?.[region];

    const assistantId =
      assistantIdConfigValue ||
      env?.TWILIO_ASSISTANT_ID ||
      getOptionalEnv<string>('TWILIO_ASSISTANT_ID') ||
      'invalid-assistant-id'; // not throwing an error so that you can use the provider by setting assistantId only as variable on a test

    const url = getAssistantUrl(assistantId, region);

    const username =
      config.accountSid ||
      env?.TWILIO_ACCOUNT_SID ||
      getRequiredEnv<string>('TWILIO_ACCOUNT_SID');

    const password = config.authTokenKey
      ? getRequiredEnv<string>(config.authTokenKey)
      : env?.TWILIO_AUTH_TOKEN || getRequiredEnv<string>('TWILIO_AUTH_TOKEN');

    const auth = Buffer.from(`${username}:${password}`).toString('base64');

    const headers = {};
    if (config.constellationOverride) {
      headers['User-Agent'] =
        `constellationName/${config.constellationOverride}`;
    }

    super({
      id,
      label,
      config: {
        url,
        identifier: 'twilio-provider',
        requestOptions: {
          method: 'POST',
          headers: {
            'Content-Type': 'application/json',
            Authorization: `Basic ${auth}`,
            ...headers,
          },
        },
      },
      env,
    });

    this.userIdentity = config.identity ?? 'test-evaluator';
    this.sessionId = config.sessionId;
    this.region = region;
  }

  protected getUrl(
    prompt: string,
    context?: CallApiContextParams,
    callApiOptions?: CallApiOptionsParams,
  ) {
    let url = super.getUrl(prompt, context, callApiOptions);
    if (typeof context?.vars.assistantId === 'string') {
      url = getAssistantUrl(context.vars.assistantId, this.region);
    } else if (typeof context?.vars.assistantId === 'object') {
      if (typeof context.vars.assistantId[this.region] === 'string') {
        url = getAssistantUrl(
          context.vars.assistantId[this.region],
          this.region,
        );
      }
    }
    return url;
  }

  protected getBody(
    prompt: string,
    context?: CallApiContextParams,
    _callApiOptions?: CallApiOptionsParams,
  ): string {
    let sessionId = this.sessionId ?? randomUUID();
    if (context?.vars.sessionId) {
      if (!context.vars.runId) {
        throw new Error(
          `
Invalid configuration. Can only use sessionId variable if you defined a runId as well. Make sure your promptfooconfig.yaml has the following set:
  defaultTest:
    vars:
      runId: package:@twilio-alpha/assistants-eval:variableHelpers.runId
`.trim(),
        );
      }
      sessionId = `${context.vars.sessionId}_${context.vars.runId}`;
    }

    let processedPrompt = prompt;

    try {
      // check if the input is actually an array of messages and just send the last one
      if (processedPrompt.trim().startsWith('[')) {
        const parsed = JSON.parse(processedPrompt);
        if (
          Array.isArray(parsed) &&
          parsed.every(
            (message) =>
              typeof message.role === 'string' &&
              typeof message.content === 'string',
          )
        ) {
          if (!context?.vars.sessionId) {
            throw new Error(
              'In order to use this feature you need to have a sessionId defined as variable.',
            );
          }
          processedPrompt = parsed[parsed.length - 1].content;
        }
      }
    } catch (err) {
      this.logger.debug(err);
    }

    const identity =
      typeof context?.vars.identity === 'string'
        ? context?.vars.identity
        : this.userIdentity;
    if (identity.includes('{{runId}}')) {
      if (typeof context?.vars.runId !== 'string') {
        throw new Error(
          `
Invalid configuration. Can only use {{runId}} in the identity variable if you defined a runId as well. Make sure your promptfooconfig.yaml has the following set:
  defaultTest:
    vars:
      runId: file://../../node_modules/@twilio-alpha/assistants-ai/dist/utils/evalRunId.js
`.trim(),
        );
      }
    }

    return JSON.stringify({
      body: processedPrompt,
      session_id: sessionId,
      identity: context?.vars.identity ?? this.userIdentity,
    });
  }

  protected async getResponse(
    response: Response,
  ): Promise<TwilioProviderResponse> {
    const data = (await response.json()) as AssistantResponse;
    this.logger.debug('AI Assistant response is %s', JSON.stringify(data));

    if (data.status === 'Success') {
      const url = new URL(response.url);
      const assistantId = url.pathname.split('/')[3];

      return {
        output: data.body,
        metadata: {
          sessionId: `webhook:${data.session_id}`,
          assistantId,
          flagged: data.flagged,
          accountSid: data.account_sid,
        },
      };
    }
    return {
      error: data.error,
    };
  }
}