aws-cdk-neuronx-patterns/lib/sagemaker-inference-toolkit-tnx/sagemaker-inference-toolkit-tnx-sagemaker.d.ts

> [!WARNING] > This library is experimental module.

import * as sagemaker from "@aws-cdk/aws-sagemaker-alpha";
import { Duration, Size } from "aws-cdk-lib";
import { Grant, IGrantable } from "aws-cdk-lib/aws-iam";
import { IBucket } from "aws-cdk-lib/aws-s3";
import { ISource } from "aws-cdk-lib/aws-s3-deployment";
import { Construct } from "constructs";
import { INeuronxInstanceType, ModelConfig, OptLevel, Parameters, QuantDtype } from "../base/neuronx";
import { SageMakerInferenceToolkitTnxCompileOptions, SageMakerInferenceToolkitTnxCompiler } from "./sagemaker-inference-toolkit-tnx-compiler";
/**
 * Precompiled model options.
 */
export interface CompiledModelOptions {
    /**
     * Neuronx compile options.
     * @default - Each properties are set default.
     */
    readonly compileOptions?: SageMakerInferenceToolkitTnxCompileOptions;
    /**
     * Code used for inference
     * @default - using the predefined code
     */
    readonly code?: ISource;
    /**
     * Model ID or saved path
     * @default "./model"
     */
    readonly modelIdOrPath?: string;
    /**
     * The path where compiled artifacts (i.e. xxx.neff) are stored
     * @default "./compiled"
     */
    readonly compiledArtifactPath?: string;
}
export interface BucketCompiledModelOptions extends CompiledModelOptions {
    /**
     * The number of parameters of model.
     */
    readonly parameters: Parameters;
    readonly config?: ModelConfig;
}
/**
 * @deprecated
 */
export declare class SageMakerInferenceToolkitTnxSageMakerInferenceModelData {
    static fromBucket(bucket: IBucket, prefix: string, options: BucketCompiledModelOptions): SageMakerInferenceToolkitTnxSageMakerInferenceModelData;
    static fromNeuronxCompiler(compiler: SageMakerInferenceToolkitTnxCompiler, code?: ISource): SageMakerInferenceToolkitTnxSageMakerInferenceModelData;
    readonly bucket: IBucket;
    readonly compiledArtifactS3Prefix: string;
    readonly code: ISource;
    readonly tpDegree: number;
    readonly quantDtype?: QuantDtype;
    readonly nPositions: number;
    readonly optLevel: OptLevel;
    readonly modelIdOrPath?: string;
    readonly compiledArtifactPath?: string;
    readonly parameters: Parameters;
    private readonly dependables;
    private constructor();
    bind(scope: Construct, model: sagemaker.IModel): void;
}
export interface SageMakerInferenceToolkitTnxSageMakerRealtimeInferenceEndpointProps {
    /**
     * Model data for SageMaker inference.
     * The model data requires at least compiled artifacts.
     */
    readonly modelData: SageMakerInferenceToolkitTnxSageMakerInferenceModelData;
    /**
     * An image of the container where the inference job is executed.
     */
    readonly image?: sagemaker.ContainerImage;
    /**
     * A map of environment variables to pass into the container.
     * @default - Only the predefined environment variables required to use Neuronx have been set.
     */
    readonly environment?: {
        [key: string]: string;
    };
    /**
     * The instance type of compile worker instance.
     * @default - It is determined automatically according to the number of model parameters and compilation options.
     */
    readonly instanceType?: INeuronxInstanceType;
    /**
     * The size, of the ML storage volume attached to individual inference instance associated with the production variant.
     * Currently only Amazon EBS gp2 storage volumes are supported.
     * @see https://aws.amazon.com/jp/releasenotes/host-instance-storage-volumes-table
     * @default - 3 GB per billion parameter (Max 512 GB)
     */
    readonly volumeSize?: Size;
    /**
     * The timeout value, to download and extract the model that you want to host from Amazon S3
     * to the individual inference instance associated with this production variant.
     * @default - 60 seconds, when `volumeSize` larger than 30GB then 1GB x 15 seconds (max 60 minutes)
     */
    readonly modelDataDownloadTimeout?: Duration;
    /**
     * The timeout value, for your inference container to pass health check by SageMaker Hosting.
     * @see https://docs.aws.amazon.com/sagemaker/latest/dg/your-algorithms-inference-code.html#your-algorithms-inference-algo-ping-requests
     * @default - 60 seconds, when set the `modelDataDownloadTimeout` then use same value (max 60 minutes)
     */
    readonly containerStartupHealthCheckTimeout?: Duration;
}
/**
 * @deprecated
 */
export declare class SageMakerInferenceToolkitTnxSageMakerRealtimeInferenceEndpoint extends Construct {
    /**
     * The ARN of the endpoint.
     * @attribute
     */
    readonly endpointArn: string;
    /**
     * The name of the endpoint.
     * @attribute
     */
    readonly endpointName: string;
    private readonly endpoint;
    constructor(scope: Construct, id: string, props: SageMakerInferenceToolkitTnxSageMakerRealtimeInferenceEndpointProps);
    grantInvoke(grantee: IGrantable): Grant;
    private selectInstanceTypeByTpDegree;
}