@kareemaly/researcher/dist/services/fileSystemStorage.js

CLI tool for web research

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.FileSystemStorage = void 0;
const promises_1 = __importDefault(require("fs/promises"));
const path_1 = __importDefault(require("path"));
const logger_1 = require("../utils/logger");
const log = (0, logger_1.createLogger)("storage");
class FileSystemStorage {
    constructor({ basePath }) {
        this.basePath = basePath;
        this.searchesDir = path_1.default.join(basePath, "searches");
        this.contentDir = path_1.default.join(basePath, "content");
    }
    async initialize() {
        await promises_1.default.mkdir(this.searchesDir, { recursive: true });
        await promises_1.default.mkdir(this.contentDir, { recursive: true });
    }
    async saveSearch(search) {
        const id = `${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
        const searchWithId = { ...search, id };
        const filePath = path_1.default.join(this.searchesDir, `${id}.json`);
        await promises_1.default.writeFile(filePath, JSON.stringify(searchWithId, null, 2));
        return id;
    }
    async getSearch(id) {
        try {
            const filePath = path_1.default.join(this.searchesDir, `${id}.json`);
            const content = await promises_1.default.readFile(filePath, "utf-8");
            return JSON.parse(content);
        }
        catch (error) {
            if (error.code === "ENOENT") {
                return null;
            }
            throw error;
        }
    }
    async listSearches() {
        const files = await promises_1.default.readdir(this.searchesDir);
        const searches = [];
        for (const file of files) {
            if (!file.endsWith(".json"))
                continue;
            const content = await promises_1.default.readFile(path_1.default.join(this.searchesDir, file), "utf-8");
            searches.push(JSON.parse(content));
        }
        return searches.sort((a, b) => b.timestamp - a.timestamp);
    }
    async saveContent(searchId, content) {
        const searchDir = path_1.default.join(this.contentDir, searchId);
        await promises_1.default.mkdir(searchDir, { recursive: true });
        const fileName = Buffer.from(content.url).toString("base64url") + ".json";
        const filePath = path_1.default.join(searchDir, fileName);
        await promises_1.default.writeFile(filePath, JSON.stringify(content, null, 2));
    }
    async getContent(searchId, url) {
        try {
            const searchDir = path_1.default.join(this.contentDir, searchId);
            const fileName = Buffer.from(url).toString("base64url") + ".json";
            const filePath = path_1.default.join(searchDir, fileName);
            const content = await promises_1.default.readFile(filePath, "utf-8");
            return JSON.parse(content);
        }
        catch (error) {
            if (error.code === "ENOENT") {
                return null;
            }
            throw error;
        }
    }
    async getStats() {
        const searches = await this.listSearches();
        let contentCount = 0;
        let totalSizeBytes = 0;
        // Count searches and their size
        for (const search of searches) {
            const searchPath = path_1.default.join(this.searchesDir, `${search.id}.json`);
            const stats = await promises_1.default.stat(searchPath);
            totalSizeBytes += stats.size;
        }
        // Count content files and their size
        for (const search of searches) {
            const contentDir = path_1.default.join(this.contentDir, search.id);
            try {
                const files = await promises_1.default.readdir(contentDir);
                contentCount += files.length;
                for (const file of files) {
                    const stats = await promises_1.default.stat(path_1.default.join(contentDir, file));
                    totalSizeBytes += stats.size;
                }
            }
            catch (error) {
                if (error.code !== "ENOENT") {
                    throw error;
                }
            }
        }
        return {
            searchCount: searches.length,
            contentCount,
            totalSizeBytes
        };
    }
    async cleanAll() {
        const searches = await this.listSearches();
        await promises_1.default.rm(this.searchesDir, { recursive: true, force: true });
        await promises_1.default.rm(this.contentDir, { recursive: true, force: true });
        await this.initialize();
        return searches.length;
    }
    async cleanOlderThan(date) {
        const searches = await this.listSearches();
        const cutoffTime = date.getTime();
        let removed = 0;
        for (const search of searches) {
            if (search.timestamp < cutoffTime) {
                const searchPath = path_1.default.join(this.searchesDir, `${search.id}.json`);
                const contentDir = path_1.default.join(this.contentDir, search.id);
                await promises_1.default.rm(searchPath, { force: true });
                await promises_1.default.rm(contentDir, { recursive: true, force: true });
                removed++;
            }
        }
        return removed;
    }
}
exports.FileSystemStorage = FileSystemStorage;