cloudflare/src/resources/ai-gateway/evaluations.ts

The official TypeScript library for the Cloudflare API

// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.

import { APIResource } from '../../core/resource';
import { APIPromise } from '../../core/api-promise';
import { PagePromise, V4PagePaginationArray, type V4PagePaginationArrayParams } from '../../core/pagination';
import { RequestOptions } from '../../internal/request-options';
import { path } from '../../internal/utils/path';

export class BaseEvaluations extends APIResource {
  static override readonly _key: readonly ['aiGateway', 'evaluations'] = Object.freeze([
    'aiGateway',
    'evaluations',
  ] as const);

  /**
   * Creates an evaluation that scores the logs in a dataset with the specified
   * evaluator types. Evaluations and datasets are deprecated and unavailable to new
   * accounts.
   *
   * @example
   * ```ts
   * const evaluation =
   *   await client.aiGateway.evaluations.create('my-gateway', {
   *     account_id: '3ebbcb006d4d46d7bb6a8c7f14676cb0',
   *     dataset_ids: ['string'],
   *     evaluation_type_ids: ['string'],
   *     name: 'name',
   *   });
   * ```
   */
  create(
    gatewayID: string,
    params: EvaluationCreateParams,
    options?: RequestOptions,
  ): APIPromise<EvaluationCreateResponse> {
    const { account_id, ...body } = params;
    return (
      this._client.post(path`/accounts/${account_id}/ai-gateway/gateways/${gatewayID}/evaluations`, {
        body,
        ...options,
      }) as APIPromise<{ result: EvaluationCreateResponse }>
    )._thenUnwrap((obj) => obj.result);
  }

  /**
   * Lists the evaluations run on an AI Gateway. Evaluations and datasets are
   * deprecated and unavailable to new accounts.
   *
   * @example
   * ```ts
   * // Automatically fetches more pages as needed.
   * for await (const evaluationListResponse of client.aiGateway.evaluations.list(
   *   'my-gateway',
   *   { account_id: '3ebbcb006d4d46d7bb6a8c7f14676cb0' },
   * )) {
   *   // ...
   * }
   * ```
   */
  list(
    gatewayID: string,
    params: EvaluationListParams,
    options?: RequestOptions,
  ): PagePromise<EvaluationListResponsesV4PagePaginationArray, EvaluationListResponse> {
    const { account_id, ...query } = params;
    return this._client.getAPIList(
      path`/accounts/${account_id}/ai-gateway/gateways/${gatewayID}/evaluations`,
      V4PagePaginationArray<EvaluationListResponse>,
      { query, ...options },
    );
  }

  /**
   * Deletes an evaluation and its results. Evaluations and datasets are deprecated
   * and unavailable to new accounts.
   *
   * @example
   * ```ts
   * const evaluation =
   *   await client.aiGateway.evaluations.delete('id', {
   *     account_id: '3ebbcb006d4d46d7bb6a8c7f14676cb0',
   *     gateway_id: 'my-gateway',
   *   });
   * ```
   */
  delete(
    id: string,
    params: EvaluationDeleteParams,
    options?: RequestOptions,
  ): APIPromise<EvaluationDeleteResponse> {
    const { account_id, gateway_id } = params;
    return (
      this._client.delete(
        path`/accounts/${account_id}/ai-gateway/gateways/${gateway_id}/evaluations/${id}`,
        options,
      ) as APIPromise<{ result: EvaluationDeleteResponse }>
    )._thenUnwrap((obj) => obj.result);
  }

  /**
   * Retrieves an evaluation and its results. Evaluations and datasets are deprecated
   * and unavailable to new accounts.
   *
   * @example
   * ```ts
   * const evaluation = await client.aiGateway.evaluations.get(
   *   'id',
   *   {
   *     account_id: '3ebbcb006d4d46d7bb6a8c7f14676cb0',
   *     gateway_id: 'my-gateway',
   *   },
   * );
   * ```
   */
  get(id: string, params: EvaluationGetParams, options?: RequestOptions): APIPromise<EvaluationGetResponse> {
    const { account_id, gateway_id } = params;
    return (
      this._client.get(
        path`/accounts/${account_id}/ai-gateway/gateways/${gateway_id}/evaluations/${id}`,
        options,
      ) as APIPromise<{ result: EvaluationGetResponse }>
    )._thenUnwrap((obj) => obj.result);
  }
}
export class Evaluations extends BaseEvaluations {}

export type EvaluationListResponsesV4PagePaginationArray = V4PagePaginationArray<EvaluationListResponse>;

export interface EvaluationCreateResponse {
  id: string;

  created_at: string;

  datasets: Array<EvaluationCreateResponse.Dataset>;

  /**
   * Unique identifier of the AI Gateway within the account.
   */
  gateway_id: string;

  modified_at: string;

  name: string;

  processed: boolean;

  results: Array<EvaluationCreateResponse.Result>;

  total_logs: number;
}

export namespace EvaluationCreateResponse {
  export interface Dataset {
    id: string;

    account_id: string;

    account_tag: string;

    created_at: string;

    enable: boolean;

    filters: Array<Dataset.Filter>;

    /**
     * Unique identifier of the AI Gateway within the account.
     */
    gateway_id: string;

    modified_at: string;

    name: string;
  }

  export namespace Dataset {
    export interface Filter {
      key:
        | 'created_at'
        | 'request_content_type'
        | 'response_content_type'
        | 'success'
        | 'cached'
        | 'provider'
        | 'model'
        | 'cost'
        | 'tokens'
        | 'tokens_in'
        | 'tokens_out'
        | 'duration'
        | 'feedback';

      operator: 'eq' | 'contains' | 'lt' | 'gt';

      value: Array<string | number | boolean>;
    }
  }

  export interface Result {
    id: string;

    created_at: string;

    evaluation_id: string;

    evaluation_type_id: string;

    modified_at: string;

    result: string;

    status: number;

    status_description: string;

    total_logs: number;
  }
}

export interface EvaluationListResponse {
  id: string;

  created_at: string;

  datasets: Array<EvaluationListResponse.Dataset>;

  /**
   * Unique identifier of the AI Gateway within the account.
   */
  gateway_id: string;

  modified_at: string;

  name: string;

  processed: boolean;

  results: Array<EvaluationListResponse.Result>;

  total_logs: number;
}

export namespace EvaluationListResponse {
  export interface Dataset {
    id: string;

    account_id: string;

    account_tag: string;

    created_at: string;

    enable: boolean;

    filters: Array<Dataset.Filter>;

    /**
     * Unique identifier of the AI Gateway within the account.
     */
    gateway_id: string;

    modified_at: string;

    name: string;
  }

  export namespace Dataset {
    export interface Filter {
      key:
        | 'created_at'
        | 'request_content_type'
        | 'response_content_type'
        | 'success'
        | 'cached'
        | 'provider'
        | 'model'
        | 'cost'
        | 'tokens'
        | 'tokens_in'
        | 'tokens_out'
        | 'duration'
        | 'feedback';

      operator: 'eq' | 'contains' | 'lt' | 'gt';

      value: Array<string | number | boolean>;
    }
  }

  export interface Result {
    id: string;

    created_at: string;

    evaluation_id: string;

    evaluation_type_id: string;

    modified_at: string;

    result: string;

    status: number;

    status_description: string;

    total_logs: number;
  }
}

export interface EvaluationDeleteResponse {
  id: string;

  created_at: string;

  datasets: Array<EvaluationDeleteResponse.Dataset>;

  /**
   * Unique identifier of the AI Gateway within the account.
   */
  gateway_id: string;

  modified_at: string;

  name: string;

  processed: boolean;

  results: Array<EvaluationDeleteResponse.Result>;

  total_logs: number;
}

export namespace EvaluationDeleteResponse {
  export interface Dataset {
    id: string;

    account_id: string;

    account_tag: string;

    created_at: string;

    enable: boolean;

    filters: Array<Dataset.Filter>;

    /**
     * Unique identifier of the AI Gateway within the account.
     */
    gateway_id: string;

    modified_at: string;

    name: string;
  }

  export namespace Dataset {
    export interface Filter {
      key:
        | 'created_at'
        | 'request_content_type'
        | 'response_content_type'
        | 'success'
        | 'cached'
        | 'provider'
        | 'model'
        | 'cost'
        | 'tokens'
        | 'tokens_in'
        | 'tokens_out'
        | 'duration'
        | 'feedback';

      operator: 'eq' | 'contains' | 'lt' | 'gt';

      value: Array<string | number | boolean>;
    }
  }

  export interface Result {
    id: string;

    created_at: string;

    evaluation_id: string;

    evaluation_type_id: string;

    modified_at: string;

    result: string;

    status: number;

    status_description: string;

    total_logs: number;
  }
}

export interface EvaluationGetResponse {
  id: string;

  created_at: string;

  datasets: Array<EvaluationGetResponse.Dataset>;

  /**
   * Unique identifier of the AI Gateway within the account.
   */
  gateway_id: string;

  modified_at: string;

  name: string;

  processed: boolean;

  results: Array<EvaluationGetResponse.Result>;

  total_logs: number;
}

export namespace EvaluationGetResponse {
  export interface Dataset {
    id: string;

    account_id: string;

    account_tag: string;

    created_at: string;

    enable: boolean;

    filters: Array<Dataset.Filter>;

    /**
     * Unique identifier of the AI Gateway within the account.
     */
    gateway_id: string;

    modified_at: string;

    name: string;
  }

  export namespace Dataset {
    export interface Filter {
      key:
        | 'created_at'
        | 'request_content_type'
        | 'response_content_type'
        | 'success'
        | 'cached'
        | 'provider'
        | 'model'
        | 'cost'
        | 'tokens'
        | 'tokens_in'
        | 'tokens_out'
        | 'duration'
        | 'feedback';

      operator: 'eq' | 'contains' | 'lt' | 'gt';

      value: Array<string | number | boolean>;
    }
  }

  export interface Result {
    id: string;

    created_at: string;

    evaluation_id: string;

    evaluation_type_id: string;

    modified_at: string;

    result: string;

    status: number;

    status_description: string;

    total_logs: number;
  }
}

export interface EvaluationCreateParams {
  /**
   * Path param
   */
  account_id: string;

  /**
   * Body param
   */
  dataset_ids: Array<string>;

  /**
   * Body param
   */
  evaluation_type_ids: Array<string>;

  /**
   * Body param
   */
  name: string;
}

export interface EvaluationListParams extends V4PagePaginationArrayParams {
  /**
   * Path param
   */
  account_id: string;

  /**
   * Query param
   */
  name?: string;

  /**
   * Query param
   */
  processed?: boolean;

  /**
   * Query param: Search by id, name
   */
  search?: string;
}

export interface EvaluationDeleteParams {
  account_id: string;

  /**
   * Unique identifier of the AI Gateway within the account.
   */
  gateway_id: string;
}

export interface EvaluationGetParams {
  account_id: string;

  /**
   * Unique identifier of the AI Gateway within the account.
   */
  gateway_id: string;
}

export declare namespace Evaluations {
  export {
    type EvaluationCreateResponse as EvaluationCreateResponse,
    type EvaluationListResponse as EvaluationListResponse,
    type EvaluationDeleteResponse as EvaluationDeleteResponse,
    type EvaluationGetResponse as EvaluationGetResponse,
    type EvaluationListResponsesV4PagePaginationArray as EvaluationListResponsesV4PagePaginationArray,
    type EvaluationCreateParams as EvaluationCreateParams,
    type EvaluationListParams as EvaluationListParams,
    type EvaluationDeleteParams as EvaluationDeleteParams,
    type EvaluationGetParams as EvaluationGetParams,
  };
}