@aj-archipelago/cortex
Version:
Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.
145 lines (141 loc) • 6.27 kB
JavaScript
// sys_tool_readfile.js
// Entity tool that reads one or more files and answers questions about them
import { Prompt } from '../../../../server/prompt.js';
export default {
prompt:
[
new Prompt({ messages: [
{"role": "system", "content": `You are the part of an AI entity named {{aiName}} that can view, hear, and understand files of all sorts (images, videos, audio, pdfs, text, etc.) - you provide the capability to view and analyze files that the user provides.\nThe user has provided you with one or more files in this conversation - you should consider them for context when you respond.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.\n{{renderTemplate AI_DATETIME}}`},
"{{chatHistory}}",
]}),
],
inputParameters: {
chatHistory: [{role: '', content: []}],
contextId: ``,
aiName: "Jarvis",
language: "English",
},
max_tokens: 8192,
model: 'gemini-flash-20-vision',
useInputChunking: false,
enableDuplicateRequests: false,
timeout: 600,
toolDefinition: [{
type: "function",
icon: "📄",
function: {
name: "AnalyzePDF",
description: "Use specifically for reading, analyzing, and answering questions about PDF file content. Do not use this tool for analyzing and answering questions about other file types.",
parameters: {
type: "object",
properties: {
detailedInstructions: {
type: "string",
description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
},
userMessage: {
type: "string",
description: "A user-friendly message that describes what you're doing with this tool"
}
},
required: ["detailedInstructions", "userMessage"]
}
}
},
{
type: "function",
icon: "📝",
function: {
name: "AnalyzeText",
description: "Use specifically for reading, analyzing, and answering questions about text files (including csv, json, html, etc.).",
parameters: {
type: "object",
properties: {
detailedInstructions: {
type: "string",
description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
},
userMessage: {
type: "string",
description: "A user-friendly message that describes what you're doing with this tool"
}
},
required: ["detailedInstructions", "userMessage"]
}
}
},
{
type: "function",
icon: "📝",
function: {
name: "AnalyzeMarkdown",
description: "Use specifically for reading, analyzing, and answering questions about markdown files.",
parameters: {
type: "object",
properties: {
detailedInstructions: {
type: "string",
description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
},
userMessage: {
type: "string",
description: "A user-friendly message that describes what you're doing with this tool"
}
},
required: ["detailedInstructions", "userMessage"]
}
}
},
{
type: "function",
icon: "🖼️",
function: {
name: "AnalyzeImage",
description: "Use specifically for reading, analyzing, and answering questions about image files (jpg, gif, bmp, png, etc). This cannot be used for creating or transforming images.",
parameters: {
type: "object",
properties: {
detailedInstructions: {
type: "string",
description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
},
userMessage: {
type: "string",
description: "A user-friendly message that describes what you're doing with this tool"
}
},
required: ["detailedInstructions", "userMessage"]
}
}
},
{
type: "function",
icon: "🎥",
function: {
name: "AnalyzeVideo",
description: "Use specifically for reading, analyzing, and answering questions about video or audio file content. You MUST use this tool to look at video or audio files.",
parameters: {
type: "object",
properties: {
detailedInstructions: {
type: "string",
description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
},
userMessage: {
type: "string",
description: "A user-friendly message that describes what you're doing with this tool"
}
},
required: ["detailedInstructions", "userMessage"]
}
}
}],
executePathway: async ({args, runAllPrompts, resolver}) => {
if (args.detailedInstructions) {
args.chatHistory.push({role: "user", content: args.detailedInstructions});
}
const result = await runAllPrompts({ ...args });
resolver.tool = JSON.stringify({ toolUsed: "vision" });
return result;
}
}