eve/dist/src/evals/runner/reporters/datadog.d.ts

Filesystem-first framework for durable backend AI agents that run anywhere.

import type { EvalReporter } from "#evals/runner/reporters/types.js";
/** Configuration for the Datadog reporter. */
export interface DatadogReporterConfig {
    /** Datadog LLM Observability project name. Defaults to `DD_LLMOBS_PROJECT_NAME`, the configured ml_app, `DD_SERVICE`, or the first eval id. */
    readonly projectName?: string;
    /** Name for the dataset used by the experiment. Defaults to `<experimentName> dataset`. */
    readonly datasetName?: string;
    /** Name for the created experiment. Defaults to a timestamped eve eval run name. */
    readonly experimentName?: string;
    /** Optional experiment description. */
    readonly description?: string;
    /** Datadog service name used when the reporter initializes `dd-trace`. */
    readonly service?: string;
    /** Datadog env used when the reporter initializes `dd-trace`. */
    readonly env?: string;
    /** Datadog site used when the reporter initializes `dd-trace`. */
    readonly site?: string;
    /** LLM Observability ml_app used when the reporter initializes `dd-trace`. Defaults to `projectName`. */
    readonly mlApp?: string;
    /** Tags attached to the Datadog Experiment. */
    readonly tags?: Readonly<Record<string, string>>;
    /** JSON-serializable metadata attached to the Datadog Experiment. */
    readonly metadata?: Readonly<Record<string, unknown>>;
    /** JSON-serializable Datadog Experiment config. */
    readonly config?: Readonly<Record<string, unknown>>;
    /** Include the first sent message, falling back to the eval description, in the experiment row and a linked dataset record. Defaults to false. */
    readonly recordInputs?: boolean;
    /** Include eval outputs in the synthetic experiment row output. Defaults to false. */
    readonly recordOutputs?: boolean;
    /** Include `metadata.expectedOutput`, `metadata.expected`, or `metadata.expected_output` on experiment rows. Defaults to false. */
    readonly recordExpectedOutputs?: boolean;
    /** Include raw assertion names as metric tags and failure messages in row metadata. Defaults to false. */
    readonly recordAssertionDetails?: boolean;
    /** Include execution error messages, which may contain application data. Defaults to false. */
    readonly recordErrors?: boolean;
    /** Console hook used for tests. */
    readonly log?: (line: string) => void;
    /** eve-owned client seam for tests and custom Datadog SDK wiring. */
    readonly client?: DatadogExperimentsClient;
}
type DatadogJsonValue = string | number | boolean | null | DatadogJsonValue[] | {
    [key: string]: DatadogJsonValue;
};
interface DatadogDatasetRecordInput {
    inputData: DatadogJsonValue;
    expectedOutput?: DatadogJsonValue;
    metadata?: Record<string, DatadogJsonValue>;
    tags?: string[];
}
interface DatadogDatasetRecord {
    readonly id: string | null;
}
interface DatadogDataset {
    id(): string | null;
    name(): string;
    version(): number | null;
    records(): readonly DatadogDatasetRecord[];
    url(): string | null;
    push(): Promise<{
        pushedCount: number;
        totalCount: number;
    }>;
}
interface DatadogExperimentsClient {
    createDataset?(name: string, options?: {
        projectName?: string;
        description?: string;
        records?: DatadogDatasetRecordInput[];
    }): DatadogDataset;
    startExperiment(options: DatadogStartExperimentOptions): Promise<DatadogExternalExperiment>;
}
interface DatadogStartExperimentOptions {
    name: string;
    projectName?: string;
    description?: string;
    tags?: Record<string, string>;
    metadata?: Record<string, DatadogJsonValue>;
    config?: Record<string, DatadogJsonValue>;
    dataset?: {
        id?: string;
        version?: number;
        name?: string;
        description?: string;
    };
}
interface DatadogExternalExperiment {
    experimentId(): string;
    url(): string | null;
    submitSpan(row: DatadogExternalExperimentSpanInput): Promise<DatadogExternalExperimentSpan>;
    submitEvaluationMetrics(span: Pick<DatadogExternalExperimentSpan, "experimentId" | "spanId" | "traceId">, metrics: DatadogEvaluationMetricInput[]): Promise<void>;
    close(options?: {
        status?: string;
        error?: string | Error;
    }): Promise<void>;
}
interface DatadogExternalExperimentSpanInput {
    name?: string;
    input?: DatadogJsonValue;
    output?: DatadogJsonValue;
    expectedOutput?: DatadogJsonValue;
    metadata?: Record<string, DatadogJsonValue>;
    tags?: Record<string, string>;
    startedAt?: Date | string | number;
    completedAt?: Date | string | number;
    durationMs?: number;
    error?: string | Error | {
        type?: string;
        name?: string;
        message?: string;
        stack?: string;
    };
    datasetRecordId?: string;
    runId?: string;
    runIteration?: number;
}
interface DatadogExternalExperimentSpan {
    experimentId: string;
    spanId: string;
    traceId: string;
    url: string | null;
}
interface DatadogEvaluationMetricInput {
    label: string;
    value?: DatadogJsonValue;
    error?: string | Error;
    timestamp?: Date | string | number;
    tags?: Record<string, string>;
    source?: string;
}
/**
 * Creates an {@link EvalReporter} that uploads eval assertion scores to a
 * Datadog LLM Observability Experiment. Requires the optional `dd-trace`
 * package and `DD_API_KEY`/`DD_APP_KEY` credentials unless `config.client` is
 * provided.
 */
export declare function Datadog(config?: DatadogReporterConfig): EvalReporter;
export {};