@restnfeel/agentc-starter-kit
Version:
한국어 기업용 CMS 모듈 - Task Master AI와 함께 빠르게 웹사이트를 구현할 수 있는 재사용 가능한 컴포넌트 시스템
198 lines (195 loc) • 9.36 kB
JavaScript
import * as fs from 'fs/promises';
import * as path from 'path';
class MemoryVectorStore {
constructor(embeddingModel, storePath = "./store") {
this.vectors = [];
this.embeddingModel = embeddingModel;
this.storePath = storePath;
}
async initialize() {
// Ensure store directory exists
await fs.mkdir(this.storePath, { recursive: true });
// Try to load existing vectors
await this.loadVectors();
}
async addDocuments(documents) {
const newVectors = [];
for (const document of documents) {
if (!document.chunks)
continue;
// Generate embeddings for all chunks
const chunkTexts = document.chunks.map((chunk) => chunk.content);
const embeddings = await this.embeddingModel.embedBatch(chunkTexts);
// Create vector entries
for (let i = 0; i < document.chunks.length; i++) {
const chunk = document.chunks[i];
const embedding = embeddings[i];
// Update chunk with embedding
chunk.embedding = embedding;
newVectors.push({
id: chunk.id,
vector: embedding,
chunk,
document,
});
}
}
// Add to memory store
this.vectors.push(...newVectors);
// Persist to disk
await this.saveVectors();
}
async search(query, k = 5) {
if (this.vectors.length === 0) {
return [];
}
// Generate embedding for query
const queryEmbedding = await this.embeddingModel.embed(query);
// Calculate similarities
const similarities = this.vectors.map((vector) => ({
vector,
score: this.cosineSimilarity(queryEmbedding, vector.vector),
}));
// Sort by similarity and take top k
similarities.sort((a, b) => b.score - a.score);
return similarities.slice(0, k).map((item) => ({
chunk: item.vector.chunk,
score: item.score,
document: item.vector.document,
}));
}
async delete(documentId) {
// Remove all vectors for the document
this.vectors = this.vectors.filter((vector) => vector.document.id !== documentId);
// Persist changes
await this.saveVectors();
}
async update(document) {
// Delete existing vectors for this document
await this.delete(document.id);
// Add updated document
await this.addDocuments([document]);
}
async clear() {
this.vectors = [];
await this.saveVectors();
}
getCount() {
return this.vectors.length;
}
cosineSimilarity(a, b) {
if (a.length !== b.length) {
throw new Error("Vectors must have the same dimension");
}
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
if (magnitude === 0) {
return 0;
}
return dotProduct / magnitude;
}
async saveVectors() {
try {
const vectorsPath = path.join(this.storePath, "vectors.json");
const data = {
vectors: this.vectors.map((v) => ({
id: v.id,
vector: v.vector,
chunk: {
id: v.chunk.id,
content: v.chunk.content,
metadata: v.chunk.metadata,
},
document: {
id: v.document.id,
metadata: v.document.metadata,
source: v.document.source,
},
documentId: v.document.id,
documentSource: v.document.source,
})),
timestamp: new Date().toISOString(),
};
await fs.writeFile(vectorsPath, JSON.stringify(data, null, 2));
}
catch (error) {
console.warn("Failed to save vectors to disk:", error);
}
}
async loadVectors() {
try {
const vectorsPath = path.join(this.storePath, "vectors.json");
const data = await fs.readFile(vectorsPath, "utf-8");
const parsed = JSON.parse(data);
if (parsed.vectors && Array.isArray(parsed.vectors)) {
// Reconstruct the vectors with proper Document objects
this.vectors = parsed.vectors
.map((savedVector) => {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2;
// Create a minimal Document object if not fully saved
const document = {
id: savedVector.documentId || savedVector.id,
content: "", // We don't save full content, just reference
metadata: {
title: ((_b = (_a = savedVector.document) === null || _a === void 0 ? void 0 : _a.metadata) === null || _b === void 0 ? void 0 : _b.title) ||
((_c = savedVector.documentSource) === null || _c === void 0 ? void 0 : _c.split("/").pop()) ||
"Unknown Document",
author: (_e = (_d = savedVector.document) === null || _d === void 0 ? void 0 : _d.metadata) === null || _e === void 0 ? void 0 : _e.author,
createdAt: ((_g = (_f = savedVector.document) === null || _f === void 0 ? void 0 : _f.metadata) === null || _g === void 0 ? void 0 : _g.createdAt)
? new Date(savedVector.document.metadata.createdAt)
: new Date(),
updatedAt: ((_j = (_h = savedVector.document) === null || _h === void 0 ? void 0 : _h.metadata) === null || _j === void 0 ? void 0 : _j.updatedAt)
? new Date(savedVector.document.metadata.updatedAt)
: new Date(),
fileType: ((_l = (_k = savedVector.document) === null || _k === void 0 ? void 0 : _k.metadata) === null || _l === void 0 ? void 0 : _l.fileType) || "unknown",
fileSize: ((_o = (_m = savedVector.document) === null || _m === void 0 ? void 0 : _m.metadata) === null || _o === void 0 ? void 0 : _o.fileSize) || 0,
language: (_q = (_p = savedVector.document) === null || _p === void 0 ? void 0 : _p.metadata) === null || _q === void 0 ? void 0 : _q.language,
description: (_s = (_r = savedVector.document) === null || _r === void 0 ? void 0 : _r.metadata) === null || _s === void 0 ? void 0 : _s.description,
source: savedVector.documentSource || "unknown",
...(_t = savedVector.document) === null || _t === void 0 ? void 0 : _t.metadata,
},
source: savedVector.documentSource || "unknown",
};
// Reconstruct the chunk
const chunk = {
id: savedVector.id,
content: ((_u = savedVector.chunk) === null || _u === void 0 ? void 0 : _u.content) || "",
metadata: {
documentId: savedVector.documentId || savedVector.id,
chunkIndex: ((_w = (_v = savedVector.chunk) === null || _v === void 0 ? void 0 : _v.metadata) === null || _w === void 0 ? void 0 : _w.chunkIndex) || 0,
startOffset: ((_y = (_x = savedVector.chunk) === null || _x === void 0 ? void 0 : _x.metadata) === null || _y === void 0 ? void 0 : _y.startOffset) || 0,
endOffset: ((_0 = (_z = savedVector.chunk) === null || _z === void 0 ? void 0 : _z.metadata) === null || _0 === void 0 ? void 0 : _0.endOffset) || 0,
tokens: ((_2 = (_1 = savedVector.chunk) === null || _1 === void 0 ? void 0 : _1.metadata) === null || _2 === void 0 ? void 0 : _2.tokens) || 0,
source: savedVector.documentSource || "unknown",
},
embedding: savedVector.vector,
};
return {
id: savedVector.id,
vector: savedVector.vector,
chunk,
document,
};
})
.filter((vector) => vector.chunk && vector.document);
}
else {
this.vectors = [];
}
}
catch (error) {
// File doesn't exist or is corrupted - start fresh
this.vectors = [];
console.warn("Failed to load vectors, starting fresh:", error);
}
}
}
export { MemoryVectorStore };
//# sourceMappingURL=memory.js.map