aws-cdk-neuronx-patterns/lib/vllm-nxd-inference/vllm-nxd-inference-ecs-patterns.d.ts

> [!WARNING] > This library is experimental module.

import * as ecs from "aws-cdk-lib/aws-ecs";
import { ApplicationListener, ApplicationLoadBalancer, ApplicationTargetGroup } from "aws-cdk-lib/aws-elasticloadbalancingv2";
import { Construct } from "constructs";
import { ApplicationLoadBalancedNeuronxServiceProps, NeuronxTaskDefinition, NeuronxTaskDefinitionPropsBase } from "../base/aws-ecs-patterns";
import { INeuronxImage } from "../base/neuronx";
import { INeuronxContainerImage } from "../base/neuronx-compiler";
import { VllmNxdInferenceCompiledModel } from "./vllm-nxd-inference-compiler";
export interface VllmNxdInferenceImageOptions {
    readonly vllmGitBranch?: string;
    readonly vllmGitCommitHash?: string;
}
/**
 * Base class for VllmNxdInferenceImage.
 */
export declare abstract class VllmNxdInferenceImageBase implements INeuronxContainerImage {
    /**
     * The container image.
     */
    abstract readonly image: ecs.ContainerImage;
    /**
     * The Git branch name of aws-neuron/upstreaming-to-vllm.
     * @see https://github.com/aws-neuron/upstreaming-to-vllm
     */
    readonly vllmGitBranch: string;
    /**
     * The Git commit fosh of aws-neuron/upstreaming-to-vllm.
     * @see https://github.com/aws-neuron/upstreaming-to-vllm
     */
    readonly vllmGitCommitHash: string;
    /**
     * The neuronx SDK version.
     */
    readonly sdkVersion: string;
    constructor(neruonxImage: INeuronxImage, options?: VllmNxdInferenceImageOptions);
}
/**
 * Inference container image for vLLM on NxD Inference.
 * @example new VllmNxdInferenceImage(PytorchTrainingNeuronxImage.LATEST)
 */
export declare class VllmNxdInferenceImage extends VllmNxdInferenceImageBase {
    readonly image: ecs.ContainerImage;
    constructor(neruonxImage: INeuronxImage, options?: VllmNxdInferenceImageOptions);
}
/**
 * Task definition for VllmNxdInference.
 */
export interface VllmNxdInferenceTaskDefinitionProps extends NeuronxTaskDefinitionPropsBase {
    /**
     * The model to be compiled.
     */
    readonly compiledModel: VllmNxdInferenceCompiledModel;
    /**
     * The image to be used for the container.
     * @default - latest VllmNxdInferenceImage
     */
    readonly image?: VllmNxdInferenceImage;
    /**
     * The environment variables to pass to the container.
     * This is only applicable when using container runtime.
     *
     * @default - No environment variables.
     */
    readonly environment?: {
        [key: string]: string;
    };
}
/**
 * Task definition for VllmNxdInference.
 */
export declare class VllmNxdInferenceTaskDefinition extends NeuronxTaskDefinition {
    constructor(scope: Construct, id: string, props: VllmNxdInferenceTaskDefinitionProps);
}
/**
 * Props for ApplicationLoadBalancedVllmNxDInferenceService.
 */
export interface ApplicationLoadBalancedVllmNxDInferenceServiceProps extends ApplicationLoadBalancedNeuronxServiceProps {
}
/**
 * ApplicationLoadBalancedVllmNxDInferenceService is a wrapper of ApplicationLoadBalancedNeuronxServiceBase.
 * It provides a simple way to deploy vLLM on NxD Inference.
 * @example
 * const compiler = new VllmNxdInferenceCompiler(this, "Compiler", {
 *   vpc,
 *   bucket,
 *   model: Model.fromHuggingFace("example/example-7b-chat"),
 * });
 * const compiledModel = compiler.compile();
 * const taskDefinition = new VllmNxdInferenceTaskDefinition(
 *   this,
 *   "TaskDefinition",
 *   {
 *     vpc,
 *     compiledModel,
 *   },
 * );
 * new ApplicationLoadBalancedVllmNxDInferenceService(this, "Service", {
 *   taskDefinition,
 * });
 */
export declare class ApplicationLoadBalancedVllmNxDInferenceService extends Construct {
    readonly loadBalancer: ApplicationLoadBalancer;
    readonly listener: ApplicationListener;
    readonly targetGroup: ApplicationTargetGroup;
    readonly service: ecs.Ec2Service;
    readonly taskDefinition: ecs.Ec2TaskDefinition;
    constructor(scope: Construct, id: string, props: ApplicationLoadBalancedVllmNxDInferenceServiceProps);
}