@sap-ai-sdk/orchestration
Version:
SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**.
228 lines • 10.8 kB
JavaScript
import { executeRequest } from '@sap-ai-sdk/core';
import { createLogger } from '@sap-cloud-sdk/util';
import yaml from 'yaml';
import { RegistryControllerPromptControllerCreateUpdatePromptTemplateBody } from '@sap-ai-sdk/prompt-registry/internal.js';
import { getOrchestrationDeploymentId } from '@sap-ai-sdk/ai-api/internal.js';
import { OrchestrationStream } from './orchestration-stream.js';
import { OrchestrationStreamResponse } from './orchestration-stream-response.js';
import { OrchestrationResponse } from './orchestration-response.js';
import { constructCompletionPostRequest, constructCompletionPostRequestFromJsonModuleConfig, constructCompletionPostRequestFromConfigReference } from './util/index.js';
import { isConfigReference, isOrchestrationModuleConfigList, assertIsOrchestrationModuleConfigList } from './orchestration-types.js';
const logger = createLogger({
package: 'orchestration',
messageContext: 'orchestration-client'
});
/**
* Get the orchestration client.
*/
export class OrchestrationClient {
config;
deploymentConfig;
destination;
/**
* Creates an instance of the orchestration client.
* @param config - Orchestration configuration. Can be:
* - An `OrchestrationModuleConfig` object for inline configuration
* - An `OrchestrationModuleConfigList` array for module fallback (tries each config in order until one succeeds)
* - A JSON string obtained from AI Launchpad
* - An object of type`OrchestrationConfigRef` to reference a stored configuration by ID or name.
* @param deploymentConfig - Deployment configuration.
* @param destination - The destination to use for the request.
*/
constructor(config, deploymentConfig, destination) {
this.config = config;
this.deploymentConfig = deploymentConfig;
this.destination = destination;
if (typeof config === 'string') {
this.validateJsonConfig(config);
}
else if (Array.isArray(config)) {
assertIsOrchestrationModuleConfigList(config);
this.config = this.parseModuleConfigList(config);
}
else if (!isConfigReference(config)) {
this.config = this.parseTemplatingModule(config);
}
}
/**
* Send a chat completion request to the orchestration service.
* @param request - Request containing messages, placeholder values, and message history.
* @param requestConfig - Additional request configuration. Use `requestConfig.headers` to pass service-specific headers:
* - `AI-Object-Store-Secret-Name`: Name of the object store secret used by the feedback service.
* @returns The orchestration service response.
*/
async chatCompletion(request, requestConfig) {
requestConfig?.signal?.throwIfAborted();
if (isConfigReference(this.config) && request?.messages?.length) {
logger.debug('Messages provided with an orchestration config reference will be sent as messages_history.');
}
const response = await this.executeRequest({
request,
requestConfig,
stream: false
});
return new OrchestrationResponse(response);
}
/**
* Create a streaming chat completion request to the orchestration service.
* @param request - Request containing messages, placeholder values, and message history.
* @param signal - An abort signal to cancel the request.
* @param options - Streaming options, e.g., for input/output filtering.
* @param requestConfig - Additional request configuration. Use `requestConfig.headers` to pass service-specific headers:
* - `AI-Object-Store-Secret-Name`: Name of the object store secret used by the feedback service.
* @returns The orchestration stream response.
*/
async stream(request, signal, options, requestConfig) {
const controller = new AbortController();
if (signal) {
signal.throwIfAborted();
signal.addEventListener('abort', () => {
controller.abort();
});
}
try {
if (typeof this.config === 'string' && options) {
logger.warn('Stream options are not supported when using a JSON module config.');
}
if (isConfigReference(this.config)) {
if (options) {
logger.warn('Stream options are not supported when using an orchestration config reference. Streaming is only supported if the referenced config has streaming configured.');
}
if (request?.messages?.length) {
logger.debug('Messages provided with an orchestration config reference will be sent as messages_history.');
}
}
return await this.createStreamResponse({
request,
requestConfig,
stream: true,
streamOptions: options
}, controller);
}
catch (error) {
controller.abort();
throw error;
}
}
async executeRequest(options) {
const { request, requestConfig, stream, streamOptions } = options;
const body = typeof this.config === 'string'
? constructCompletionPostRequestFromJsonModuleConfig(JSON.parse(this.config), request, stream)
: isConfigReference(this.config)
? constructCompletionPostRequestFromConfigReference(this.config, request)
: isOrchestrationModuleConfigList(this.config)
? constructCompletionPostRequest(this.config, request, stream, streamOptions)
: constructCompletionPostRequest(this.config, request, stream, streamOptions);
const deploymentId = await getOrchestrationDeploymentId(this.deploymentConfig || {}, this.destination);
if (!deploymentId) {
throw new Error('Failed to resolve deployment ID');
}
const response = await executeRequest({
url: `/inference/deployments/${deploymentId}/v2/completion`,
...(this.deploymentConfig ?? {})
}, body, requestConfig, this.destination);
// Log summary when fallbacks were used
if (response.data?.intermediate_failures) {
const failureCount = response.data.intermediate_failures.length;
const successModel = response.data.final_result?.model;
logger.info(`Orchestration used ${failureCount} fallback(s) before success${successModel ? `. Succeeded with model: ${successModel}` : ''}.`);
}
return response;
}
async createStreamResponse(options, controller) {
const streamResponse = await this.executeRequest({
...options,
requestConfig: {
...options.requestConfig,
responseType: 'stream',
signal: controller.signal
}
});
const response = new OrchestrationStreamResponse(streamResponse);
const stream = OrchestrationStream._create(streamResponse, controller);
response.stream = stream
._pipe(OrchestrationStream._processChunk)
._pipe(OrchestrationStream._processOrchestrationStreamChunkResponse, response)
._pipe(OrchestrationStream._processStreamEnd, response);
return response;
}
/**
* Validate if a string is valid JSON.
* @param config - The JSON string to validate.
*/
validateJsonConfig(config) {
try {
JSON.parse(config);
}
catch (error) {
throw new Error(`Could not parse JSON: ${error}`, { cause: error });
}
}
/**
* Parse and merge templating into the config object.
* @param config - The orchestration module configuration with templating either as object or string.
* @returns The updated and merged orchestration module configuration.
* @throws Error if the YAML parsing fails or if the parsed object does not conform to the expected schema.
*/
parseAndMergeTemplating(config) {
let parsedObject;
if (typeof config.promptTemplating.prompt === 'string' &&
!config.promptTemplating.prompt.trim()) {
throw new Error('Templating YAML string must be non-empty.');
}
try {
parsedObject = yaml.parse(config.promptTemplating.prompt);
}
catch (error) {
throw new Error(`Error parsing YAML: ${error}`, { cause: error });
}
const result = RegistryControllerPromptControllerCreateUpdatePromptTemplateBody.safeParse(parsedObject);
if (!result.success) {
throw new Error(`Prompt Template YAML does not conform to the defined type. Validation errors: ${result.error}`);
}
const { template, defaults, response_format, tools } = result.data.spec;
return {
...config,
promptTemplating: {
...config.promptTemplating,
prompt: {
template: template,
...(defaults && { defaults: defaults }),
...(response_format && { response_format }),
...(tools && { tools })
}
}
};
}
/**
* Parse a single orchestration module config, handling YAML prompt templates.
* @param config - The orchestration module configuration.
* @returns The parsed configuration.
*/
parseTemplatingModule(config) {
return typeof config.promptTemplating.prompt === 'string'
? this.parseAndMergeTemplating(config)
: config;
}
/**
* Parse and validate a list of orchestration module configs for fallback.
* @param config - The array of configurations.
* @returns The validated and parsed configuration list.
* @throws {Error} If the array is empty or contains invalid elements.
*/
parseModuleConfigList(config) {
// Validate and assert it's a proper config list (throws if invalid)
assertIsOrchestrationModuleConfigList(config);
// Parse each config in the list
const parsedConfigs = config.map(c => this.parseTemplatingModule(c));
// Warn about duplicate models in fallback chain
const models = parsedConfigs.map(c => c.promptTemplating.model.name);
const uniqueModels = new Set(models);
if (uniqueModels.size < models.length) {
logger.warn(`Fallback configurations contain duplicate models: [${models.join(', ')}]. ` +
'Consider using different models for meaningful fallback behavior.');
}
return parsedConfigs;
}
}
//# sourceMappingURL=orchestration-client.js.map