@twilio-alpha/assistants-eval/dist/providers/twilio.js

promptfoo extension for writing AI evaluations for Twilio AI Assistants

import { randomUUID } from 'crypto';
import { getOptionalEnv, getRequiredEnv } from '../utils/envs.js';
import ExternalProvider from './external.js';
export function getAssistantUrl(assistantId, region) {
    const sub = region === 'prod' ? '' : `${region}.`;
    return `https://assistants.${sub}twilio.com/v1/Assistants/${assistantId}/Messages`;
}
export default class TwilioProvider extends ExternalProvider {
    sessionId;
    userIdentity;
    region;
    constructor(options) {
        const { config, id, label, env } = options;
        const region = config.twilioRegion ||
            env?.TWILIO_REGION ||
            getOptionalEnv('TWILIO_REGION') ||
            'prod';
        const assistantIdConfigValue = typeof config.assistantId === 'string'
            ? config.assistantId
            : config.assistantId?.[region];
        const assistantId = assistantIdConfigValue ||
            env?.TWILIO_ASSISTANT_ID ||
            getOptionalEnv('TWILIO_ASSISTANT_ID') ||
            'invalid-assistant-id'; // not throwing an error so that you can use the provider by setting assistantId only as variable on a test
        const url = getAssistantUrl(assistantId, region);
        const username = config.accountSid ||
            env?.TWILIO_ACCOUNT_SID ||
            getRequiredEnv('TWILIO_ACCOUNT_SID');
        const password = config.authTokenKey
            ? getRequiredEnv(config.authTokenKey)
            : env?.TWILIO_AUTH_TOKEN || getRequiredEnv('TWILIO_AUTH_TOKEN');
        const auth = Buffer.from(`${username}:${password}`).toString('base64');
        const headers = {};
        if (config.constellationOverride) {
            headers['User-Agent'] =
                `constellationName/${config.constellationOverride}`;
        }
        super({
            id,
            label,
            config: {
                url,
                identifier: 'twilio-provider',
                requestOptions: {
                    method: 'POST',
                    headers: {
                        'Content-Type': 'application/json',
                        Authorization: `Basic ${auth}`,
                        ...headers,
                    },
                },
            },
            env,
        });
        this.userIdentity = config.identity ?? 'test-evaluator';
        this.sessionId = config.sessionId;
        this.region = region;
    }
    getUrl(prompt, context, callApiOptions) {
        let url = super.getUrl(prompt, context, callApiOptions);
        if (typeof context?.vars.assistantId === 'string') {
            url = getAssistantUrl(context.vars.assistantId, this.region);
        }
        else if (typeof context?.vars.assistantId === 'object') {
            if (typeof context.vars.assistantId[this.region] === 'string') {
                url = getAssistantUrl(context.vars.assistantId[this.region], this.region);
            }
        }
        return url;
    }
    getBody(prompt, context, _callApiOptions) {
        let sessionId = this.sessionId ?? randomUUID();
        if (context?.vars.sessionId) {
            if (!context.vars.runId) {
                throw new Error(`
Invalid configuration. Can only use sessionId variable if you defined a runId as well. Make sure your promptfooconfig.yaml has the following set:
  defaultTest:
    vars:
      runId: package:@twilio-alpha/assistants-eval:variableHelpers.runId
`.trim());
            }
            sessionId = `${context.vars.sessionId}_${context.vars.runId}`;
        }
        let processedPrompt = prompt;
        try {
            // check if the input is actually an array of messages and just send the last one
            if (processedPrompt.trim().startsWith('[')) {
                const parsed = JSON.parse(processedPrompt);
                if (Array.isArray(parsed) &&
                    parsed.every((message) => typeof message.role === 'string' &&
                        typeof message.content === 'string')) {
                    if (!context?.vars.sessionId) {
                        throw new Error('In order to use this feature you need to have a sessionId defined as variable.');
                    }
                    processedPrompt = parsed[parsed.length - 1].content;
                }
            }
        }
        catch (err) {
            this.logger.debug(err);
        }
        const identity = typeof context?.vars.identity === 'string'
            ? context?.vars.identity
            : this.userIdentity;
        if (identity.includes('{{runId}}')) {
            if (typeof context?.vars.runId !== 'string') {
                throw new Error(`
Invalid configuration. Can only use {{runId}} in the identity variable if you defined a runId as well. Make sure your promptfooconfig.yaml has the following set:
  defaultTest:
    vars:
      runId: file://../../node_modules/@twilio-alpha/assistants-ai/dist/utils/evalRunId.js
`.trim());
            }
        }
        return JSON.stringify({
            body: processedPrompt,
            session_id: sessionId,
            identity: context?.vars.identity ?? this.userIdentity,
        });
    }
    async getResponse(response) {
        const data = (await response.json());
        this.logger.debug('AI Assistant response is %s', JSON.stringify(data));
        if (data.status === 'Success') {
            const url = new URL(response.url);
            const assistantId = url.pathname.split('/')[3];
            return {
                output: data.body,
                metadata: {
                    sessionId: `webhook:${data.session_id}`,
                    assistantId,
                    flagged: data.flagged,
                    accountSid: data.account_sid,
                },
            };
        }
        return {
            error: data.error,
        };
    }
}
//# sourceMappingURL=twilio.js.map