@sap-ai-sdk/langchain
Version:
SAP Cloud SDK for AI is the official Software Development Kit (SDK) for **SAP AI Core**, **SAP Generative AI Hub**, and **Orchestration Service**.
172 lines • 8.11 kB
JavaScript
import { BaseChatModel } from '@langchain/core/language_models/chat_models';
import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orchestration';
import { ChatGenerationChunk } from '@langchain/core/outputs';
import { isTemplateRef, mapLangChainMessagesToOrchestrationMessages, mapOutputToChatResult, mapToolToChatCompletionTool, mapOrchestrationChunkToLangChainMessageChunk } from './util.js';
function isInputFilteringError(error) {
return (error.cause?.status === 400 &&
error.cause?.response?.data?.location?.includes('Input Filter'));
}
/**
* The Orchestration client.
*/
export class OrchestrationClient extends BaseChatModel {
orchestrationConfig;
langchainOptions;
deploymentConfig;
destination;
constructor(orchestrationConfig, langchainOptions = {}, deploymentConfig, destination) {
// Avoid retry if the error is due to input filtering
const { onFailedAttempt } = langchainOptions;
langchainOptions.onFailedAttempt = error => {
if (isInputFilteringError(error)) {
throw error;
}
onFailedAttempt?.(error);
};
super(langchainOptions);
this.orchestrationConfig = orchestrationConfig;
this.langchainOptions = langchainOptions;
this.deploymentConfig = deploymentConfig;
this.destination = destination;
}
_llmType() {
return 'orchestration';
}
/**
* Create a new runnable sequence that runs each individual runnable in series,
* piping the output of one runnable into another runnable or runnable-like.
* @param coerceable - A runnable, function, or object whose values are functions or runnables.
* @returns A new runnable sequence.
*/
pipe(coerceable) {
return super.pipe(coerceable);
}
async _generate(messages, options, runManager) {
const { placeholderValues, customRequestConfig } = options;
const allMessages = mapLangChainMessagesToOrchestrationMessages(messages);
const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options);
const res = await this.caller.callWithOptions({
signal: options.signal
}, () => {
const orchestrationClient = new OrchestrationClientBase(mergedOrchestrationConfig, this.deploymentConfig, this.destination);
return orchestrationClient.chatCompletion({
messages: allMessages,
placeholderValues
}, {
...customRequestConfig,
signal: options.signal
});
});
const content = res.getContent();
await runManager?.handleLLMNewToken(typeof content === 'string' ? content : '');
return mapOutputToChatResult(res._data);
}
bindTools(tools, kwargs) {
let strict;
if (kwargs?.strict !== undefined) {
strict = kwargs.strict;
}
return this.withConfig({
tools: tools.map(tool => mapToolToChatCompletionTool(tool, strict)),
...kwargs
});
}
/**
* Stream response chunks from the Orchestration client.
* @param messages - The messages to send to the model.
* @param options - The call options.
* @param runManager - The callback manager for the run.
* @returns An async generator of chat generation chunks.
*/
async *_streamResponseChunks(messages, options, runManager) {
const orchestrationMessages = mapLangChainMessagesToOrchestrationMessages(messages);
const { placeholderValues, customRequestConfig } = options;
const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options);
const orchestrationClient = new OrchestrationClientBase(mergedOrchestrationConfig, this.deploymentConfig, this.destination);
const response = await this.caller.callWithOptions({
signal: options.signal
}, () => orchestrationClient.stream({ messages: orchestrationMessages, placeholderValues }, options.signal, options.streamOptions, customRequestConfig));
for await (const chunk of response.stream) {
const orchestrationResult = chunk._data.final_result;
// There can be only none or one choice inside a chunk
const choice = orchestrationResult?.choices[0];
// Map the chunk to a LangChain message chunk
const messageChunk = mapOrchestrationChunkToLangChainMessageChunk(chunk);
// Create initial generation info with token indices
const newTokenIndices = {
prompt: options.promptIndex ?? 0,
completion: choice?.index ?? 0
};
const generationInfo = { ...newTokenIndices };
// Process finish reason
if (choice?.finish_reason && orchestrationResult) {
generationInfo.finish_reason = choice.finish_reason;
// Only include system fingerprint in the last chunk for now to avoid concatenation issues
generationInfo.system_fingerprint =
orchestrationResult.system_fingerprint;
generationInfo.model_name = orchestrationResult.model;
generationInfo.id = orchestrationResult.id;
generationInfo.created = orchestrationResult.created;
generationInfo.request_id = chunk._data.request_id;
}
// Process token usage
const tokenUsage = chunk.getTokenUsage();
if (tokenUsage) {
generationInfo.token_usage = tokenUsage;
messageChunk.usage_metadata = {
input_tokens: tokenUsage.prompt_tokens,
output_tokens: tokenUsage.completion_tokens,
total_tokens: tokenUsage.total_tokens
};
}
const content = chunk.getDeltaContent() ?? '';
const generationChunk = new ChatGenerationChunk({
message: messageChunk,
text: content,
generationInfo
});
// Notify the run manager about the new token
// Some parameters(`_runId`, `_parentRunId`, `_tags`) are set as undefined as they are implicitly read from the context.
await runManager?.handleLLMNewToken(content, newTokenIndices, undefined, undefined, undefined, { chunk: generationChunk });
yield generationChunk;
}
}
mergeOrchestrationConfig(options) {
const { tools = [], stop = [] } = options;
const config = {
...this.orchestrationConfig,
promptTemplating: {
...this.orchestrationConfig.promptTemplating,
model: {
...this.orchestrationConfig.promptTemplating.model,
params: {
...this.orchestrationConfig.promptTemplating.model.params,
...(stop.length && {
stop: [
...(this.orchestrationConfig.promptTemplating.model.params
?.stop || []),
...stop
]
})
}
}
}
};
if (tools.length) {
if (!config.promptTemplating.prompt) {
config.promptTemplating.prompt = {};
}
if (typeof config.promptTemplating.prompt === 'object' &&
!isTemplateRef(config.promptTemplating.prompt)) {
config.promptTemplating.prompt.tools = [
// Preserve existing tools configured in the templating module
...(config.promptTemplating.prompt.tools || []),
// Add new tools set with LangChain `bindTools()` or `invoke()` methods
...tools.map(t => mapToolToChatCompletionTool(t))
];
}
}
return config;
}
}
//# sourceMappingURL=client.js.map