genkit-cli/dist/commands/eval-extract-data.js

CLI for interacting with the Google Genkit AI framework

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.evalExtractData = void 0;
const utils_1 = require("@genkit-ai/tools-common/utils");
const clc = __importStar(require("colorette"));
const commander_1 = require("commander");
const promises_1 = require("fs/promises");
const manager_utils_1 = require("../utils/manager-utils");
exports.evalExtractData = new commander_1.Command('eval:extractData')
    .description('extract evaludation data for a given flow from the trace store')
    .argument('<flowName>', 'name of the flow to run')
    .option('--output <filename>', 'name of the output file to store the extracted data')
    .option('--maxRows <maxRows>', 'maximum number of rows', '100')
    .option('--label [label]', 'label flow run in this batch')
    .action(async (flowName, options) => {
    const dashDashIndex = process.argv.indexOf('--');
    let runtimeCommand;
    if (dashDashIndex !== -1) {
        runtimeCommand = process.argv.slice(dashDashIndex + 1);
    }
    const projectRoot = await (0, utils_1.findProjectRoot)();
    const runAction = async (manager) => {
        const extractors = await (0, utils_1.getEvalExtractors)(`/flow/${flowName}`);
        utils_1.logger.debug(`Extracting trace data '/flow/${flowName}'...`);
        let dataset = [];
        let continuationToken = undefined;
        while (dataset.length < Number.parseInt(options.maxRows)) {
            const response = await manager.listTraces({
                limit: Number.parseInt(options.maxRows),
                continuationToken,
            });
            continuationToken = response.continuationToken;
            const traces = response.traces;
            const batch = traces
                .map((t) => {
                const rootSpan = Object.values(t.spans).find((s) => s.attributes['genkit:metadata:subtype'] === 'flow' &&
                    (!options.label ||
                        s.attributes['batchRun'] === options.label) &&
                    s.attributes['genkit:name'] === flowName);
                if (!rootSpan) {
                    return undefined;
                }
                return t;
            })
                .filter((t) => !!t)
                .map((trace) => {
                return {
                    testCaseId: (0, utils_1.generateTestCaseId)(),
                    input: extractors.input(trace),
                    output: extractors.output(trace),
                    context: toArray(extractors.context(trace)),
                    traceIds: Array.from(new Set(Object.values(trace.spans).map((span) => span.traceId))),
                };
            })
                .filter((result) => !!result);
            batch.forEach((d) => dataset.push(d));
            if (dataset.length > Number.parseInt(options.maxRows)) {
                dataset = dataset.splice(0, Number.parseInt(options.maxRows));
                break;
            }
            if (!continuationToken) {
                break;
            }
        }
        if (options.output) {
            utils_1.logger.debug(`Writing data to '${options.output}'...`);
            await (0, promises_1.writeFile)(options.output, JSON.stringify(dataset, undefined, '  '));
        }
        else {
            utils_1.logger.debug(`Results will not be written to file.`);
            utils_1.logger.info(clc.green('Results:'));
            utils_1.logger.info(JSON.stringify(dataset, undefined, '  '));
        }
    };
    await (0, manager_utils_1.runWithManager)(projectRoot, runAction, { runtimeCommand });
});
function toArray(input) {
    return Array.isArray(input) ? input : [input];
}
//# sourceMappingURL=eval-extract-data.js.map