chat-about-video
Version:
Chat about a video clip using ChatGPT hosted in OpenAI or Azure, or Gemini provided by Google
230 lines (229 loc) • 10.8 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ChatGptApi = void 0;
const tslib_1 = require("tslib");
const node_os_1 = tslib_1.__importDefault(require("node:os"));
const openai_1 = require("openai");
const chat_1 = require("../chat");
const utils_1 = require("../utils");
class ChatGptApi {
constructor(options) {
var _a;
this.options = options;
const { credential, endpoint, clientSettings } = options;
this.client = endpoint
? new openai_1.AzureOpenAI(Object.assign(Object.assign({}, clientSettings), { endpoint, apiKey: credential.key }))
: new openai_1.OpenAI(Object.assign(Object.assign({}, clientSettings), { apiKey: credential.key }));
this.storage = (0, utils_1.effectiveStorageOptions)(options.storage);
this.extractVideoFrames = (0, utils_1.effectiveExtractVideoFramesOptions)(options.extractVideoFrames);
this.tmpDir = (_a = options.tmpDir) !== null && _a !== void 0 ? _a : node_os_1.default.tmpdir();
}
async getClient() {
return this.client;
}
async generateContent(prompt, options) {
var _a;
const effectiveOptions = Object.assign(Object.assign({}, this.options.completionOptions), options);
const requestOptions = effectiveOptions;
if (effectiveOptions.jsonResponse) {
requestOptions.response_format =
effectiveOptions.jsonResponse === true
? { type: 'json_object' }
: {
type: 'json_schema',
json_schema: Object.assign(Object.assign({}, effectiveOptions.jsonResponse), { name: (_a = effectiveOptions.jsonResponse.name) !== null && _a !== void 0 ? _a : 'InlineResponseSchema' }),
};
}
// OpenAI does not allow unknown properties
delete effectiveOptions.backoffOnDownloadError;
delete effectiveOptions.backoffOnConnectivityError;
delete effectiveOptions.backoffOnServerError;
delete effectiveOptions.backoffOnThrottling;
delete effectiveOptions.systemPromptText;
delete effectiveOptions.startPromptText;
delete effectiveOptions.jsonResponse;
// These fields were used in previous versions of chat-about-video, just in case they are not removed from the config/options/settings.
delete effectiveOptions.maxTokens;
delete effectiveOptions.deploymentName;
delete effectiveOptions.extractVideoFrames;
return this.client.chat.completions.create(Object.assign(Object.assign({}, requestOptions), { messages: prompt, stream: false }));
}
async getResponseText(result) {
var _a, _b, _c, _d;
return (_d = (_c = (_b = (_a = result === null || result === void 0 ? void 0 : result.choices) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.message) === null || _c === void 0 ? void 0 : _c.content) !== null && _d !== void 0 ? _d : undefined;
}
async getUsageMetadata(result) {
if (result.usage) {
return {
totalTokens: result.usage.total_tokens,
promptTokens: result.usage.prompt_tokens,
completionTokens: result.usage.completion_tokens,
};
}
return undefined;
}
// Example error object:
// {
// "name": "Error",
// "message": "400 Invalid image data",
// "stack": "...",
// "status:": 400,
// "code": "BadRequest",
// "param": null,
// "type": null,
// "headers": {
// ...
// },
// "error": {
// "code": "BadRequest",
// "message": "Invalid image data",
// "param": null,
// "type": null,
// }
// }
//
// Another example error object:
// {
// "status": 400,
// "headers": {
// "api-supported-versions": "1",
// "apim-request-id": "aa24a9c4-d46b-47d6-b9bc-6f3ad0e2cc29",
// "content-length": "612",
// "content-type": "application/json; charset=utf-8",
// "date": "Sat, 26 Jul 2025 06:15:34 GMT",
// "request-id": "aa24a9c4-d46b-47d6-b9bc-6f3ad0e2cc29",
// "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
// "x-content-type-options": "nosniff",
// "x-envoy-upstream-service-time": "1084",
// "x-ms-deployment-name": "gpt-4o",
// "x-ms-region": "East US 2",
// "x-ratelimit-limit-requests": "4500",
// "x-ratelimit-limit-tokens": "450000",
// "x-ratelimit-remaining-requests": "4499",
// "x-ratelimit-remaining-tokens": "438897"
// },
// "error": {
// "inner_error": {
// "code": "ResponsibleAIPolicyViolation",
// "content_filter_results": {
// "sexual": {
// "filtered": true,
// "severity": "high"
// },
// "violence": {
// "filtered": false,
// "severity": "safe"
// },
// "hate": {
// "filtered": false,
// "severity": "safe"
// },
// "self_harm": {
// "filtered": false,
// "severity": "safe"
// }
// }
// },
// "code": "content_filter",
// "message": "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry. To learn more about our content filtering policies please read our documentation: \r\nhttps://go.microsoft.com/fwlink/?linkid=2198766.",
// "param": "prompt",
// "type": null
// },
// "code": "content_filter",
// "param": "prompt",
// "type": null
// }
isThrottlingError(error) {
var _a, _b, _c;
const code = String((_b = (_a = error === null || error === void 0 ? void 0 : error.status) !== null && _a !== void 0 ? _a : error === null || error === void 0 ? void 0 : error.code) !== null && _b !== void 0 ? _b : (_c = error === null || error === void 0 ? void 0 : error.error) === null || _c === void 0 ? void 0 : _c.code);
return code === 'TooManyRequests' || code === '429';
}
isServerError(error) {
var _a, _b, _c;
return ['500', 'InternalServerError', '502', 'BadGateway', '503', 'ServiceUnavailable', '504', 'GatewayTimeout'].includes(String((_b = (_a = error === null || error === void 0 ? void 0 : error.status) !== null && _a !== void 0 ? _a : error === null || error === void 0 ? void 0 : error.code) !== null && _b !== void 0 ? _b : (_c = error === null || error === void 0 ? void 0 : error.error) === null || _c === void 0 ? void 0 : _c.code));
}
isConnectivityError(error) {
return ['Request timed out.', 'Connection error.'].includes(error === null || error === void 0 ? void 0 : error.message);
}
isDownloadError(error) {
var _a, _b, _c, _d, _e, _f;
const type = String((_a = error === null || error === void 0 ? void 0 : error.type) !== null && _a !== void 0 ? _a : (_b = error === null || error === void 0 ? void 0 : error.error) === null || _b === void 0 ? void 0 : _b.type);
const code = String((_c = error === null || error === void 0 ? void 0 : error.code) !== null && _c !== void 0 ? _c : (_d = error === null || error === void 0 ? void 0 : error.error) === null || _d === void 0 ? void 0 : _d.code);
const message = String((_e = error === null || error === void 0 ? void 0 : error.message) !== null && _e !== void 0 ? _e : (_f = error === null || error === void 0 ? void 0 : error.error) === null || _f === void 0 ? void 0 : _f.message);
return (type === 'invalid_request_error' && code === 'invalid_image_url') || message.startsWith('Timeout while downloading ');
}
async appendToPrompt(newPromptOrResponse, prompt) {
var _a, _b;
prompt =
prompt !== null && prompt !== void 0 ? prompt : (((_a = this.options.completionOptions) === null || _a === void 0 ? void 0 : _a.systemPromptText)
? [
{
role: 'system',
content: this.options.completionOptions.systemPromptText,
},
]
: []);
if (isChatGptResponse(newPromptOrResponse)) {
const responseText = (_b = (await this.getResponseText(newPromptOrResponse))) !== null && _b !== void 0 ? _b : '';
prompt.push({
role: 'assistant',
content: responseText,
});
}
else {
prompt.push(...newPromptOrResponse);
}
return prompt;
}
async buildTextPrompt(text, _conversationId) {
return {
prompt: [
{
role: 'user',
content: text,
},
],
};
}
async buildVideoPrompt(videoFile, conversationId = (0, chat_1.generateTempConversationId)()) {
return (0, chat_1.buildImagesPromptFromVideo)(this, this.extractVideoFrames, this.tmpDir, videoFile, conversationId);
}
async buildImagesPrompt(imageInputs, conversationId = (0, chat_1.generateTempConversationId)()) {
const { commonParent, relativePaths } = (0, utils_1.findCommonParentPath)(imageInputs.map((imageInput) => imageInput.imageFile));
const { downloadUrls: frameImageUrls, cleanup: cleanupUploadedFrames } = await this.storage.uploader(commonParent, relativePaths, this.storage.storageContainerName, `${this.storage.storagePathPrefix}${conversationId}/`);
const messages = [];
messages.push(...frameImageUrls.map((url, i) => {
const content = [];
const { promptText } = imageInputs[i];
if (promptText) {
content.push({
type: 'text',
text: promptText,
});
}
content.push({
type: 'image_url',
image_url: {
url,
detail: 'auto',
},
});
return {
role: 'user',
content,
};
}));
return {
prompt: messages,
cleanup: async () => {
if (this.storage.deleteFilesWhenConversationEnds) {
await cleanupUploadedFrames();
}
},
};
}
}
exports.ChatGptApi = ChatGptApi;
function isChatGptResponse(obj) {
return Array.isArray(obj === null || obj === void 0 ? void 0 : obj.choices);
}