@sap-ai-sdk/langchain
Version:
SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**.
145 lines • 6.65 kB
JavaScript
import { BaseChatModel } from '@langchain/core/language_models/chat_models';
import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orchestration';
import { ChatGenerationChunk } from '@langchain/core/outputs';
import { isTemplateRef, mapLangChainMessagesToOrchestrationMessages, mapOutputToChatResult, mapToolToChatCompletionTool, mapOrchestrationChunkToLangChainMessageChunk, setFinishReason, setTokenUsage, computeTokenIndices } from './util.js';
function isInputFilteringError(error) {
return (error.cause?.status === 400 &&
error.cause?.response?.data?.location?.includes('Input Filter'));
}
/**
* The Orchestration client.
*/
export class OrchestrationClient extends BaseChatModel {
orchestrationConfig;
langchainOptions;
deploymentConfig;
destination;
constructor(orchestrationConfig, langchainOptions = {}, deploymentConfig, destination) {
// Avoid retry if the error is due to input filtering
const { onFailedAttempt } = langchainOptions;
langchainOptions.onFailedAttempt = error => {
if (isInputFilteringError(error)) {
throw error;
}
onFailedAttempt?.(error);
};
super(langchainOptions);
this.orchestrationConfig = orchestrationConfig;
this.langchainOptions = langchainOptions;
this.deploymentConfig = deploymentConfig;
this.destination = destination;
}
_llmType() {
return 'orchestration';
}
/**
* Create a new runnable sequence that runs each individual runnable in series,
* piping the output of one runnable into another runnable or runnable-like.
* @param coerceable - A runnable, function, or object whose values are functions or runnables.
* @returns A new runnable sequence.
*/
pipe(coerceable) {
return super.pipe(coerceable);
}
async _generate(messages, options, runManager) {
const res = await this.caller.callWithOptions({
signal: options.signal
}, () => {
const { inputParams, customRequestConfig } = options;
const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options);
const orchestrationClient = new OrchestrationClientBase(mergedOrchestrationConfig, this.deploymentConfig, this.destination);
const allMessages = mapLangChainMessagesToOrchestrationMessages(messages);
return orchestrationClient.chatCompletion({
messages: allMessages,
inputParams
}, customRequestConfig);
});
const content = res.getContent();
await runManager?.handleLLMNewToken(typeof content === 'string' ? content : '');
return mapOutputToChatResult(res.data);
}
bindTools(tools, kwargs) {
let strict;
if (kwargs?.strict !== undefined) {
strict = kwargs.strict;
}
return this.withConfig({
tools: tools.map(tool => mapToolToChatCompletionTool(tool, strict)),
...kwargs
});
}
/**
* Stream response chunks from the Orchestration client.
* @param messages - The messages to send to the model.
* @param options - The call options.
* @param runManager - The callback manager for the run.
* @returns An async generator of chat generation chunks.
*/
async *_streamResponseChunks(messages, options, runManager) {
const orchestrationMessages = mapLangChainMessagesToOrchestrationMessages(messages);
const { inputParams, customRequestConfig } = options;
const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options);
const orchestrationClient = new OrchestrationClientBase(mergedOrchestrationConfig, this.deploymentConfig, this.destination);
const response = await this.caller.callWithOptions({
signal: options.signal
}, () => {
const controller = new AbortController();
if (options.signal) {
options.signal.addEventListener('abort', () => controller.abort());
}
return orchestrationClient.stream({ messages: orchestrationMessages, inputParams }, controller, options.streamOptions, customRequestConfig);
});
for await (const chunk of response.stream) {
const messageChunk = mapOrchestrationChunkToLangChainMessageChunk(chunk);
const tokenIndices = computeTokenIndices(chunk);
const finishReason = response.getFinishReason();
const tokenUsage = response.getTokenUsage();
setFinishReason(messageChunk, finishReason);
setTokenUsage(messageChunk, tokenUsage);
const content = chunk.getDeltaContent() ?? '';
const generationChunk = new ChatGenerationChunk({
message: messageChunk,
text: content,
generationInfo: { ...tokenIndices }
});
// Notify the run manager about the new token
// Some parameters(`_runId`, `_parentRunId`, `_tags`) are set as undefined as they are implicitly read from the context.
await runManager?.handleLLMNewToken(content, tokenIndices, undefined, undefined, undefined, { chunk: generationChunk });
yield generationChunk;
}
}
mergeOrchestrationConfig(options) {
const { tools = [], stop = [] } = options;
const config = {
...this.orchestrationConfig,
llm: {
...this.orchestrationConfig.llm,
model_params: {
...this.orchestrationConfig.llm.model_params,
...(stop.length && {
stop: [
...(this.orchestrationConfig.llm.model_params?.stop || []),
...stop
]
})
}
}
};
config.templating = this.orchestrationConfig.templating;
if (tools.length) {
if (!config.templating) {
config.templating = {};
}
if (!isTemplateRef(config.templating)) {
config.templating.tools = [
// Preserve existing tools configured in the templating module
...(config.templating.tools || []),
// Add new tools set with LangChain `bindTools()` or `invoke()` methods
...tools.map(t => mapToolToChatCompletionTool(t))
];
}
}
return config;
}
}
//# sourceMappingURL=client.js.map