@autobe/agent/src/orchestrate/test/orchestrateTestScenario.ts

AI backend server code generator

import { IAgenticaController } from "@agentica/core";
import {
  AutoBeEventSource,
  AutoBeInterfaceAuthorization,
  AutoBeOpenApi,
  AutoBeProgressEventBase,
  AutoBeTestScenario,
  AutoBeTestScenarioEvent,
} from "@autobe/interface";
import { AutoBeOpenApiEndpointComparator } from "@autobe/utils";
import { NamingConvention } from "@typia/utils";
import { HashMap, HashSet, IPointer, Singleton } from "tstl";
import typia, { ILlmApplication, IValidation } from "typia";
import { v7 } from "uuid";

import { AutoBeContext } from "../../context/AutoBeContext";
import { buildAnalysisContextSections } from "../../utils/RAGRetrieval";
import { executeCachedBatch } from "../../utils/executeCachedBatch";
import { getEmbedder } from "../../utils/getEmbedder";
import { AutoBePreliminaryController } from "../common/AutoBePreliminaryController";
import { convertToSectionEntries } from "../common/internal/convertToSectionEntries";
import { IAnalysisSectionEntry } from "../common/structures/IAnalysisSectionEntry";
import { transformTestScenarioHistory } from "./histories/transformTestScenarioHistory";
import { AutoBeTestScenarioProgrammer } from "./programmers/AutoBeTestScenarioProgrammer";
import { IAutoBeTestScenarioApplication } from "./structures/IAutoBeTestScenarioApplication";
import { getPrerequisites } from "./utils/getPrerequisites";

/**
 * Orchestrate test scenario generation for all API operations.
 *
 * Following the InterfacePrerequisite pattern:
 *
 * - Generate one scenario per operation in parallel
 * - Review all generated scenarios in parallel
 * - Return final scenarios array
 *
 * @param ctx - AutoBe context
 * @param instruction - E2E-test-specific instructions from requirements
 * @returns Array of reviewed test scenarios
 */
export const orchestrateTestScenario = async (
  ctx: AutoBeContext,
  instruction: string,
): Promise<AutoBeTestScenario[]> => {
  const document: AutoBeOpenApi.IDocument | undefined =
    ctx.state().interface?.document;
  if (document === undefined) {
    throw new Error(
      "Cannot write test scenarios because there are no operations.",
    );
  }

  const dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation> =
    AutoBeTestScenarioProgrammer.associate(document.operations);
  const progress: AutoBeProgressEventBase = {
    total: document.operations.length,
    completed: 0,
  };

  const matrix: AutoBeTestScenario[][] = await executeCachedBatch(
    ctx,
    document.operations.map((operation) => async (promptCacheKey) => {
      const counter = new Singleton(() => ++progress.completed);
      try {
        return await process(ctx, {
          dict,
          document,
          operation,
          progress,
          counter,
          promptCacheKey,
          instruction,
        });
      } catch (error) {
        counter.get();
        console.log(operation, error);
        return [];
      }
    }),
  );
  const scenarios: AutoBeTestScenario[] = matrix.flat();

  // review removed — write agents self-review during rewrite loop
  return scenarios;
};

/**
 * Process single operation scenario generation.
 *
 * Following InterfacePrerequisite pattern:
 *
 * - Preliminary.orchestrate wrapper
 * - Conversate with controller
 * - Dispatch event
 * - Return scenario
 */
async function process(
  ctx: AutoBeContext,
  props: {
    dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>;
    operation: AutoBeOpenApi.IOperation;
    document: AutoBeOpenApi.IDocument;
    progress: AutoBeProgressEventBase;
    counter: Singleton<number>;
    promptCacheKey: string;
    instruction: string;
  },
): Promise<AutoBeTestScenario[]> {
  const allSections: IAnalysisSectionEntry[] = convertToSectionEntries(
    ctx.state().analyze?.files ?? [],
  );
  const pathSegments = props.operation.path
    .split("/")
    .filter((p) => p && !p.startsWith(":") && !p.startsWith("{"));
  const queryText: string = [
    "test",
    "scenario",
    props.operation.method,
    ...pathSegments,
  ].join(" ");

  const ragSections: IAnalysisSectionEntry[] =
    await buildAnalysisContextSections(
      getEmbedder(),
      allSections,
      queryText,
      "TOPK",
      { log: false, logPrefix: "testScenario" },
    );

  const authorizations: AutoBeInterfaceAuthorization[] =
    ctx.state().interface?.authorizations ?? [];
  const preliminary: AutoBePreliminaryController<
    "analysisSections" | "interfaceOperations" | "interfaceSchemas" | "complete"
  > = new AutoBePreliminaryController({
    application: typia.json.application<IAutoBeTestScenarioApplication>(),
    source: SOURCE,
    kinds: [
      "analysisSections",
      "interfaceOperations",
      "interfaceSchemas",
      "complete",
    ],
    dispatch: (e) => ctx.dispatch(e),
    state: ctx.state(),
    all: {
      interfaceOperations: props.document.operations,
    },
    local: {
      analysisSections: ragSections,
      interfaceOperations: (() => {
        const unique: HashSet<AutoBeOpenApi.IEndpoint> = new HashSet(
          AutoBeOpenApiEndpointComparator.hashCode,
          AutoBeOpenApiEndpointComparator.equals,
        );
        unique.insert({
          method: props.operation.method,
          path: props.operation.path,
        });
        for (const pr of getPrerequisites({
          document: props.document,
          endpoint: props.operation,
        }))
          unique.insert(pr.endpoint);
        return unique.toJSON().map((endpoint) => props.dict.get(endpoint));
      })(),
    },
  });

  const event: AutoBeTestScenarioEvent = await preliminary.orchestrate(
    ctx,
    async (out) => {
      const pointer: IPointer<AutoBeTestScenario[] | null> = {
        value: null,
      };
      const result: AutoBeContext.IResult = await ctx.conversate({
        source: SOURCE,
        controller: createController({
          dict: props.dict,
          operation: props.operation,
          authorizations,
          preliminary,
          build: (scenarios) => {
            // Normalize function name to snake_case
            for (const s of scenarios)
              s.functionName = NamingConvention.snake(s.functionName);
            pointer.value ??= [];
            pointer.value.push(...scenarios);
          },
        }),
        enforceFunctionCall: true,
        promptCacheKey: props.promptCacheKey,
        ...transformTestScenarioHistory({
          state: ctx.state(),
          operation: props.operation,
          instruction: props.instruction,
          preliminary,
        }),
      });
      if (pointer.value === null) return out(result)(null);

      pointer.value.splice(3);

      return out(result)({
        type: SOURCE,
        id: v7(),
        metric: result.metric,
        tokenUsage: result.tokenUsage,
        scenarios: pointer.value,
        acquisition: preliminary.getAcquisition(),
        total: props.progress.total,
        completed: props.counter.get(),
        step: ctx.state().interface?.step ?? 0,
        created_at: new Date().toISOString(),
      });
    },
  );
  ctx.dispatch(event);
  return event.scenarios;
}

function createController(props: {
  dict: HashMap<AutoBeOpenApi.IEndpoint, AutoBeOpenApi.IOperation>;
  authorizations: AutoBeInterfaceAuthorization[];
  operation: AutoBeOpenApi.IOperation;
  build: (scenarios: AutoBeTestScenario[]) => void;
  preliminary: AutoBePreliminaryController<
    "analysisSections" | "interfaceOperations" | "interfaceSchemas" | "complete"
  >;
}): IAgenticaController.IClass {
  const validate = (
    next: unknown,
  ): IValidation<IAutoBeTestScenarioApplication.IProps> => {
    const result: IValidation<IAutoBeTestScenarioApplication.IProps> =
      typia.validate<IAutoBeTestScenarioApplication.IProps>(next);
    if (result.success === false) return result;
    else if (result.data.request.type !== "write")
      return props.preliminary.validate({
        thinking: result.data.thinking,
        request: result.data.request,
      });

    const errors: IValidation.IError[] = [];
    result.data.request.scenarios.forEach((scenario, i) =>
      AutoBeTestScenarioProgrammer.validate({
        errors,
        dict: props.dict,
        operation: props.operation,
        scenario,
        accessor: `$input.request.scenarios[${i}]`,
      }),
    );

    return errors.length === 0
      ? result
      : {
          success: false,
          data: result.data,
          errors,
        };
  };

  const application: ILlmApplication = props.preliminary.fixApplication(
    typia.llm.application<IAutoBeTestScenarioApplication>({
      validate: {
        process: validate,
      },
    }),
  );

  return {
    protocol: "class",
    name: SOURCE,
    application,
    execute: {
      process: (next) => {
        if (next.request.type === "write") {
          // Fulfill missing authentication dependencies for each scenario
          for (const scenario of next.request.scenarios) {
            AutoBeTestScenarioProgrammer.fulfill({
              dict: props.dict,
              authorizations: props.authorizations,
              operation: props.operation,
              scenario,
            });
          }
          props.build(next.request.scenarios);
        }
      },
    } satisfies IAutoBeTestScenarioApplication,
  };
}

const SOURCE = "testScenario" satisfies AutoBeEventSource;