node-agency
Version:
A node package for building AI agents
503 lines • 19.7 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getContext = exports.groupIntoNChunks = exports.generateOutput = exports.getManagerTools = exports.getCoworkerTools = exports.callFunction = exports.registerTool = exports.readableStreamAsyncIterable = exports.VectorStore = exports.getContent = exports.getEmbeddings = void 0;
const openai_1 = __importDefault(require("openai"));
const ml_distance_1 = require("ml-distance");
const pdf_parse_1 = __importDefault(require("pdf-parse"));
const axios_1 = __importDefault(require("axios"));
const cheerio = __importStar(require("cheerio"));
const readline_1 = __importDefault(require("readline"));
const tools = {};
const context = {};
const openai = new openai_1.default({
apiKey: process.env.OPENAI_API_KEY,
});
const getEmbeddings = (content) => __awaiter(void 0, void 0, void 0, function* () {
const embedding = yield openai.embeddings.create({
model: "text-embedding-3-small",
input: content,
encoding_format: "float",
});
return embedding;
});
exports.getEmbeddings = getEmbeddings;
let totalPages = [];
const pages = [];
const isValidLink = (link) => {
return (!link.includes(".css") &&
!link.includes(".js") &&
!link.includes(".png") &&
!link.includes(".jpg") &&
!link.includes(".jpeg") &&
!link.includes(".gif") &&
!link.includes(".svg") &&
!link.includes(".ico") &&
!link.includes(".xml") &&
!link.includes(".json"));
};
const cleanSourceText = (text) => {
return text
.trim()
.replace(/(\n){4,}/g, "\n\n\n")
.replace(/\n\n/g, " ")
.replace(/ {3,}/g, " ")
.replace(/\t/g, "")
.replace(/\n+(\s*\n)*/g, "\n");
};
const crawlWebsite = (url_1, depth_1, ...args_1) => __awaiter(void 0, [url_1, depth_1, ...args_1], void 0, function* (url, depth, currentDepth = 0) {
try {
const domain = new URL(url).origin;
const response = yield axios_1.default.get(url, {
headers: {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/109.0",
},
});
const $ = cheerio.load(response.data);
const body = $("body").text();
const links = $("a")
.map((_, el) => domain + $(el).attr("href"))
.get();
function onlyUnique(value, index, array) {
return array.indexOf(value) === index;
}
const unique = links.filter(onlyUnique);
const newPages = unique.filter((link) => isValidLink(link)) || [];
const filteredLinks = newPages.filter((link, idx) => {
try {
const domain = new URL(link).hostname;
if (link.includes(".pdf"))
return false;
const excludeList = [
"google",
"facebook",
"twitter",
"instagram",
"youtube",
"tiktok",
];
if (excludeList.some((site) => domain.includes(site)))
return false;
return link.includes(domain);
}
catch (e) {
return false;
}
});
if (newPages) {
totalPages = totalPages.concat(filteredLinks).filter(onlyUnique);
}
pages.push(cleanSourceText(body));
if (currentDepth >= depth) {
return pages;
}
const nextPage = totalPages.shift();
if (!nextPage) {
return pages;
}
return crawlWebsite(nextPage, depth, currentDepth + 1);
}
catch (e) {
const nextPage = totalPages.shift();
if (!nextPage) {
return pages;
}
return crawlWebsite(nextPage, depth, currentDepth + 1);
}
});
const isPDF = (url) => url.endsWith(".pdf");
// const stripHtml = (html: string) => html.replace(/<[^>]*>?/gm, " ");
// const onlyAplhaNumeric = (text: string) => text.replace(/[^a-zA-Z0-9 ]/g, " ");
const chunkText = (text, chunkSize) => {
const chunks = [];
const words = text
.replace(/\n/g, "")
.split(" ")
.filter((w) => w.length > 0);
let currentChunk = "";
for (let i = 0; i < words.length; i++) {
if (currentChunk.length + 1 + words[i].length < chunkSize) {
if (currentChunk.length > 0) {
currentChunk += " ";
}
currentChunk += words[i];
}
else {
chunks.push(currentChunk);
currentChunk = "";
if (i < words.length) {
currentChunk = words[i];
}
}
}
return chunks;
};
const getContent = (url) => __awaiter(void 0, void 0, void 0, function* () {
if (isPDF(url)) {
const pdf = yield fetch(url);
const pdfBuffer = yield pdf.arrayBuffer();
const buffer = Buffer.from(pdfBuffer);
const pdfText = yield (0, pdf_parse_1.default)(buffer);
const chunks = chunkText(pdfText.text, 500);
return chunks;
}
const pages = yield crawlWebsite(url, 10);
const chunks = chunkText(pages.join(" "), 800);
return chunks;
});
exports.getContent = getContent;
function VectorStore() {
let memoryVectors = [];
const similaritySearchVectorWithScore = (query, k) => {
const similarity = ml_distance_1.similarity.cosine;
const filter = (doc) => true;
const filterFunction = (memoryVector) => {
if (!filter) {
return true;
}
const doc = {
metadata: memoryVector.metadata,
pageContent: memoryVector.content,
};
return filter(doc);
};
const filteredMemoryVectors = memoryVectors.filter(filterFunction);
const searches = filteredMemoryVectors
.map((vector, index) => ({
similarity: similarity(query, vector.embedding),
index,
}))
.sort((a, b) => (a.similarity > b.similarity ? -1 : 0))
.slice(0, k);
const result = searches.map((search) => [
{
metadata: filteredMemoryVectors[search.index].metadata,
pageContent: filteredMemoryVectors[search.index].content,
},
search.similarity,
]);
return result;
};
return {
addVectors: (vectors, documents) => {
const memory = vectors.map((embedding, idx) => ({
content: documents[idx].pageContent,
embedding,
metadata: documents[idx].metadata,
}));
memoryVectors = memoryVectors.concat(memory);
},
similaritySearchVectorWithScore,
};
}
exports.VectorStore = VectorStore;
function readableStreamAsyncIterable(stream) {
if (stream[Symbol.asyncIterator])
return stream;
const reader = stream.getReader();
return {
next() {
return __awaiter(this, void 0, void 0, function* () {
try {
const result = yield reader.read();
if (result === null || result === void 0 ? void 0 : result.done)
reader.releaseLock(); // release lock when stream becomes closed
return result;
}
catch (e) {
reader.releaseLock(); // release lock when stream becomes errored
throw e;
}
});
},
return() {
return __awaiter(this, void 0, void 0, function* () {
const cancelPromise = reader.cancel();
reader.releaseLock();
yield cancelPromise;
return { done: true, value: undefined };
});
},
[Symbol.asyncIterator]() {
return this;
},
};
}
exports.readableStreamAsyncIterable = readableStreamAsyncIterable;
const registerTool = (name, execute) => {
tools[name] = execute;
};
exports.registerTool = registerTool;
const callFunction = (name, input) => __awaiter(void 0, void 0, void 0, function* () {
try {
const result = yield tools[name](input);
if (name in context) {
context[name] = result;
}
return result;
}
catch (e) {
let message = "Unknown error";
if (e instanceof Error) {
message = e.message;
console.warn("Error calling function:", name, e.message);
}
return "Error calling function " + name + ": " + message;
}
});
exports.callFunction = callFunction;
function askQuestion(query) {
const rl = readline_1.default.createInterface({
input: process.stdin,
output: process.stdout,
});
return new Promise((resolve) => rl.question(query + ": ", (ans) => {
rl.close();
resolve(ans);
}));
}
const getCoworkerTools = (agents, humanFeedback) => {
const tools = [
{
type: "function",
function: {
name: "ask_question",
description: `Ask a specific question to one of the following co-workers: ${agents
.map((a) => `"${a.role.replace(/\s/g, "_").toLowerCase()}"`)
.join(",")}\nThe input to this tool should be the co-worker, the question you have for them, and ALL necessary context to ask the question properly, they know nothing about the question, so share absolute everything you know, don't reference things but instead explain them.`,
parameters: {
type: "object",
properties: {
question: {
type: "string",
description: "The question to ask the coworker",
},
coworker: {
type: "string",
description: "The coworker to ask the question to",
},
context: {
type: "string",
description: "Any required context for the coworker",
},
},
required: ["question", "coworker", "context"],
},
},
},
{
type: "function",
function: {
name: "delegate_task",
description: `Delegate a specific task to one of the following co-workers: ${agents
.map((a) => `"${a.role.replace(/\s/g, "_").toLowerCase()}"`)
.join(",")}\nThe input to this tool should be the co-worker, the task you want them to do, and ALL necessary context to execute the task, they know nothing about the task, so share absolute everything you know, don't reference things but instead explain them.`,
parameters: {
type: "object",
properties: {
task: {
type: "string",
description: "The task to delegate to the coworker",
},
coworker: {
type: "string",
description: "The coworker to delegate task to",
},
context: {
type: "string",
description: "Any required context for the coworker",
},
},
required: ["task", "coworker", "context"],
},
},
},
];
(0, exports.registerTool)("ask_question", (prompt) => __awaiter(void 0, void 0, void 0, function* () {
try {
const { question, coworker, context } = JSON.parse(prompt);
const coworkerAgent = agents.find((a) => a.role.replace(/\s/g, "_").toLowerCase() === coworker);
if (!coworkerAgent) {
throw new Error(`Could not find coworker with role: ${coworker}`);
}
return yield coworkerAgent.execute(JSON.stringify({
task: `Answer the following question: ${question}`,
input: context,
}));
}
catch (e) {
console.warn("Error calling ask_question", e);
if (e instanceof Error)
return "Error calling ask_question: " + e.message;
return "Error calling ask_question";
}
}));
(0, exports.registerTool)("delegate_task", (prompt) => __awaiter(void 0, void 0, void 0, function* () {
try {
const { task: delegateTask, coworker, context } = JSON.parse(prompt);
const coworkerAgent = agents.find((a) => a.role.replace(/\s/g, "_").toLowerCase() === coworker);
if (!coworkerAgent) {
throw new Error(`Could not find coworker with role: ${coworker}`);
}
return yield coworkerAgent.execute(JSON.stringify({
task: delegateTask,
input: context,
}));
}
catch (e) {
console.warn("Error calling delegate_task", e);
if (e instanceof Error)
return "Error calling delegate_task: " + e.message;
return "Error calling delegate_task";
}
}));
if (humanFeedback) {
tools.push({
type: "function",
function: {
name: "human_feedback",
description: "Get the users feedback on next steps to take",
parameters: {
type: "object",
properties: {
question: {
type: "string",
description: "The question to ask the user for feedback",
},
},
required: ["question"],
},
},
});
(0, exports.registerTool)("human_feedback", (prompt) => __awaiter(void 0, void 0, void 0, function* () {
try {
const { question } = JSON.parse(prompt);
return askQuestion(question);
}
catch (e) {
console.warn("Error calling human_feedback", e);
if (e instanceof Error)
return "Error calling human_feedback: " + e.message;
return "Error calling human_feedback";
}
}));
context["human_feedback"] = "";
}
return tools;
};
exports.getCoworkerTools = getCoworkerTools;
const getManagerTools = (agents, humanFeedback) => {
const humanFeedBackTool = {
type: "function",
function: {
name: "human_feedback",
description: "Get the users feedback on next steps to take",
parameters: {
type: "object",
properties: {
question: {
type: "string",
description: "The question to ask the user for feedback",
},
},
required: ["question"],
},
},
};
const tools = agents.map((agent) => {
const toolName = agent.role.replace(/\s/g, "_").toLowerCase();
(0, exports.registerTool)(toolName, agent.execute);
context[toolName] = "";
return {
type: "function",
function: {
name: toolName,
description: agent.goal,
parameters: {
type: "object",
properties: {
task: {
type: "string",
description: "Task for the agent to complete",
},
input: {
type: "string",
description: "The required input for the Agent to complete their task",
},
},
required: ["task", "input"],
},
},
};
});
if (humanFeedback) {
(0, exports.registerTool)("human_feedback", (prompt) => __awaiter(void 0, void 0, void 0, function* () {
const { question } = JSON.parse(prompt);
return askQuestion(question);
}));
context["human_feedback"] = "";
tools.push(humanFeedBackTool);
}
return tools;
};
exports.getManagerTools = getManagerTools;
const generateOutput = (finalOutput, formattedRunTime) => {
return `\n\n-----------------\n\nFinalt Results: ${finalOutput}\n\n-----------------\n\nRun time: ${formattedRunTime}\n\n-----------------\n\n`;
};
exports.generateOutput = generateOutput;
const groupIntoNChunks = (arr, chunkSize) => {
const result = new Array(chunkSize).fill([]);
const amountPerChunk = arr.length / chunkSize;
const chunkAmount = amountPerChunk < 3 ? 3 : amountPerChunk;
for (let i = result.length; i > 0; i--) {
let beginPointer = (result.length - i) * chunkAmount;
result[result.length - i] = arr.slice(beginPointer, beginPointer + chunkAmount);
}
return result;
};
exports.groupIntoNChunks = groupIntoNChunks;
const getContext = () => {
let currentContext = "";
for (const key in context) {
if (context[key]) {
currentContext += `${key} Results: ${context[key]}\n`;
}
}
return currentContext;
};
exports.getContext = getContext;
//# sourceMappingURL=utils.js.map