@shirokuma-library/mcp-knowledge-base/dist/src/services/export-manager.js

MCP server for AI-powered knowledge management with semantic search, graph analysis, and automatic enrichment

import fs from 'fs/promises';
import path from 'path';
import { AppDataSource } from '../data-source.js';
import { Item } from '../entities/Item.js';
import { SystemState } from '../entities/SystemState.js';
import { Status } from '../entities/Status.js';
import { ItemTag } from '../entities/ItemTag.js';
import { ItemKeyword } from '../entities/ItemKeyword.js';
import { ItemConcept } from '../entities/ItemConcept.js';
import { ItemRelation } from '../entities/ItemRelation.js';
const SYSTEM_DIR = '.system';
const CURRENT_STATE_DIR = 'current_state';
const MAX_FILENAME_LENGTH = 100;
const DEFAULT_EXPORT_TIMEOUT = 2000;
export class ExportManager {
    itemRepo;
    stateRepo;
    statusRepo;
    itemTagRepo;
    itemKeywordRepo;
    itemConceptRepo;
    itemRelationRepo;
    autoExportConfig;
    constructor() {
        this.itemRepo = AppDataSource.getRepository(Item);
        this.stateRepo = AppDataSource.getRepository(SystemState);
        this.statusRepo = AppDataSource.getRepository(Status);
        this.itemTagRepo = AppDataSource.getRepository(ItemTag);
        this.itemKeywordRepo = AppDataSource.getRepository(ItemKeyword);
        this.itemConceptRepo = AppDataSource.getRepository(ItemConcept);
        this.itemRelationRepo = AppDataSource.getRepository(ItemRelation);
        this.autoExportConfig = this.loadAutoExportConfig();
    }
    loadAutoExportConfig() {
        const exportDir = process.env.SHIROKUMA_EXPORT_DIR;
        const timeout = process.env.SHIROKUMA_EXPORT_TIMEOUT;
        return {
            enabled: !!exportDir,
            baseDir: exportDir || '',
            timeout: timeout && !isNaN(Number(timeout)) ? Number(timeout) : DEFAULT_EXPORT_TIMEOUT
        };
    }
    getAutoExportConfig() {
        this.autoExportConfig = this.loadAutoExportConfig();
        return this.autoExportConfig;
    }
    async autoExportItem(item) {
        const config = this.loadAutoExportConfig();
        if (!config.enabled) {
            return;
        }
        try {
            this.autoExportConfig = config;
            await this.exportWithTimeout(item, config.timeout);
        }
        catch (error) {
            console.error('Auto export failed for item', { itemId: item.id, error });
        }
    }
    async autoExportCurrentState(state) {
        const config = this.loadAutoExportConfig();
        if (!config.enabled) {
            return;
        }
        try {
            this.autoExportConfig = config;
            await this.exportCurrentStateWithTimeout(state, config.timeout);
        }
        catch (error) {
            console.error('Current state auto export failed', { error });
        }
    }
    async exportWithTimeout(item, timeout) {
        return Promise.race([
            this.exportItemToFile(item),
            new Promise((_, reject) => setTimeout(() => reject(new Error('Export timeout')), timeout))
        ]);
    }
    async exportCurrentStateWithTimeout(state, timeout) {
        return Promise.race([
            this.exportSystemStateToFile(state),
            new Promise((_, reject) => setTimeout(() => reject(new Error('Export timeout')), timeout))
        ]);
    }
    async exportItemToFile(item) {
        const typeDir = path.join(this.autoExportConfig.baseDir, item.type);
        await fs.mkdir(typeDir, { recursive: true });
        const enrichedItem = await this.getEnrichedItem(item);
        const filename = `${item.id}-${this.sanitizeFilename(item.title)}.md`;
        const filepath = path.join(typeDir, filename);
        const files = await fs.readdir(typeDir).catch(() => []);
        const existingFile = files.find(f => f.startsWith(`${item.id}-`));
        if (existingFile && existingFile !== filename) {
            await fs.unlink(path.join(typeDir, existingFile)).catch(() => { });
        }
        const content = this.formatItemAsMarkdown(enrichedItem);
        await fs.writeFile(filepath, content, 'utf-8');
    }
    async exportSystemStateToFile(state) {
        const stateDir = path.join(this.autoExportConfig.baseDir, SYSTEM_DIR, CURRENT_STATE_DIR);
        await fs.mkdir(stateDir, { recursive: true });
        const filename = `${state.id}.md`;
        const filepath = path.join(stateDir, filename);
        const content = this.formatSystemStateAsMarkdown(state);
        await fs.writeFile(filepath, content, 'utf-8');
        const latestPath = path.join(stateDir, 'latest.md');
        await fs.unlink(latestPath).catch(() => { });
        await fs.copyFile(filepath, latestPath);
    }
    buildItemPath(item) {
        const config = this.loadAutoExportConfig();
        const filename = `${item.id}-${this.sanitizeFilename(item.title)}.md`;
        return path.join(config.baseDir, item.type, filename);
    }
    buildCurrentStatePath() {
        const config = this.loadAutoExportConfig();
        return path.join(config.baseDir, SYSTEM_DIR, CURRENT_STATE_DIR);
    }
    sanitizeFilename(title) {
        let sanitized = title
            .replace(/[<>:"/\\|?*\x00-\x1F]/g, '_')
            .replace(/[\s\t\r\n]+/g, '_')
            .replace(/_+/g, '_')
            .trim();
        if (sanitized.length > MAX_FILENAME_LENGTH) {
            sanitized = sanitized.substring(0, MAX_FILENAME_LENGTH);
        }
        sanitized = sanitized.replace(/^[._\s]+/, '').replace(/[._\s]+$/, '');
        return sanitized || 'untitled';
    }
    async exportItem(id) {
        const item = await this.itemRepo.findOne({
            where: { id }
        });
        if (!item) {
            throw new Error(`Item with ID ${id} not found`);
        }
        const baseDir = process.env.SHIROKUMA_EXPORT_DIR || 'docs/export';
        const typeDir = path.join(baseDir, item.type);
        await fs.mkdir(typeDir, { recursive: true });
        const enrichedItem = await this.getEnrichedItem(item);
        const filename = `${item.id}-${this.sanitizeFilename(item.title)}.md`;
        const filepath = path.join(typeDir, filename);
        const files = await fs.readdir(typeDir).catch(() => []);
        const existingFile = files.find(f => f.startsWith(`${item.id}-`));
        if (existingFile && existingFile !== filename) {
            await fs.unlink(path.join(typeDir, existingFile)).catch(() => { });
        }
        const content = this.formatItemAsMarkdown(enrichedItem);
        await fs.writeFile(filepath, content, 'utf-8');
        return {
            exported: 1,
            directory: baseDir,
            files: [path.relative(baseDir, filepath)]
        };
    }
    async exportItems(options = {}) {
        const baseDir = process.env.SHIROKUMA_EXPORT_DIR || 'docs/export';
        const query = this.itemRepo.createQueryBuilder('item')
            .leftJoinAndSelect('item.status', 'status');
        if (options.type) {
            query.andWhere('item.type = :type', { type: options.type });
        }
        if (options.status && options.status.length > 0) {
            query.andWhere('status.name IN (:...statuses)', { statuses: options.status });
        }
        if (options.tags && options.tags.length > 0) {
            query.innerJoin('item_tags', 'it', 'it.item_id = item.id')
                .innerJoin('tags', 't', 't.id = it.tag_id')
                .andWhere('t.name IN (:...tags)', { tags: options.tags });
        }
        if (options.limit) {
            query.limit(options.limit);
        }
        query.orderBy('item.updatedAt', 'DESC');
        const items = await query.getMany();
        const exportResult = {
            exported: 0,
            directory: baseDir,
            files: []
        };
        if (options.includeState) {
            const stateResult = await this.exportCurrentState(options.includeAllStates || false);
            exportResult.stateExported = stateResult.exported;
            if (stateResult.exported && stateResult.count) {
                const stateDir = path.join(SYSTEM_DIR, CURRENT_STATE_DIR);
                if (stateResult.count === 1) {
                    exportResult.files.push(stateResult.file);
                }
                else {
                    exportResult.files.push(`${stateDir}/ (${stateResult.count} states)`);
                }
            }
        }
        const itemsByType = new Map();
        for (const item of items) {
            if (!itemsByType.has(item.type)) {
                itemsByType.set(item.type, []);
            }
            itemsByType.get(item.type).push(item);
        }
        for (const [type, typeItems] of itemsByType) {
            const typeDir = path.join(baseDir, type);
            await fs.mkdir(typeDir, { recursive: true });
            for (const item of typeItems) {
                const enrichedItem = await this.getEnrichedItem(item);
                const filename = `${item.id}-${this.sanitizeFilename(item.title)}.md`;
                const filepath = path.join(typeDir, filename);
                const files = await fs.readdir(typeDir).catch(() => []);
                const existingFile = files.find(f => f.startsWith(`${item.id}-`));
                if (existingFile && existingFile !== filename) {
                    await fs.unlink(path.join(typeDir, existingFile)).catch(() => { });
                }
                const content = this.formatItemAsMarkdown(enrichedItem);
                await fs.writeFile(filepath, content, 'utf-8');
                exportResult.files.push(path.relative(baseDir, filepath));
                exportResult.exported++;
            }
        }
        return exportResult;
    }
    async exportCurrentState(exportAll = false) {
        const baseDir = process.env.SHIROKUMA_EXPORT_DIR || 'docs/export';
        const stateDir = path.join(baseDir, SYSTEM_DIR, CURRENT_STATE_DIR);
        await fs.mkdir(stateDir, { recursive: true });
        const states = exportAll
            ? await this.stateRepo.find({ order: { id: 'DESC' } })
            : await this.stateRepo.find({ order: { id: 'DESC' }, take: 1 });
        if (states.length === 0) {
            return {
                exported: false,
                directory: baseDir,
                file: null
            };
        }
        let latestFile = '';
        for (let i = 0; i < states.length; i++) {
            const state = states[i];
            const filename = `${state.id}.md`;
            const filepath = path.join(stateDir, filename);
            if (i === 0) {
                latestFile = filepath;
            }
            const content = this.formatSystemStateAsMarkdown(state);
            await fs.writeFile(filepath, content, 'utf-8');
        }
        if (latestFile) {
            const latestPath = path.join(stateDir, 'latest.md');
            await fs.unlink(latestPath).catch(() => { });
            await fs.copyFile(latestFile, latestPath);
        }
        return {
            exported: true,
            directory: baseDir,
            file: exportAll ? null : path.relative(baseDir, path.join(SYSTEM_DIR, CURRENT_STATE_DIR, `${states[0].id}.md`)),
            count: states.length
        };
    }
    async getEnrichedItem(item) {
        const status = await this.statusRepo.findOne({ where: { id: item.statusId } });
        const itemTags = await this.itemTagRepo.find({
            where: { itemId: item.id },
            relations: ['tag']
        });
        const itemKeywords = await this.itemKeywordRepo.find({
            where: { itemId: item.id },
            relations: ['keyword']
        });
        const itemConcepts = await this.itemConceptRepo.find({
            where: { itemId: item.id },
            relations: ['concept']
        });
        const relationsFrom = await this.itemRelationRepo.find({
            where: { sourceId: item.id }
        });
        const relationsTo = await this.itemRelationRepo.find({
            where: { targetId: item.id }
        });
        const relatedIds = [
            ...relationsFrom.map(r => r.targetId),
            ...relationsTo.map(r => r.sourceId)
        ];
        return {
            ...item,
            status: { name: status?.name || 'Open' },
            tags: itemTags.map(it => ({ tag: { name: it.tag.name } })),
            keywords: itemKeywords.map(ik => ({ keyword: { word: ik.keyword.word }, weight: ik.weight })),
            concepts: itemConcepts.map(ic => ({ concept: { name: ic.concept.name }, confidence: ic.confidence })),
            related: [...new Set(relatedIds)]
        };
    }
    formatItemAsMarkdown(item) {
        let md = '---\n';
        md += `id: ${item.id}\n`;
        md += `type: ${item.type}\n`;
        md += `title: "${item.title.replace(/"/g, '\\"')}"\n`;
        md += `status: ${item.status.name}\n`;
        md += `priority: ${item.priority || 'MEDIUM'}\n`;
        if (item.description) {
            md += `description: ${JSON.stringify(item.description)}\n`;
        }
        if (item.aiSummary) {
            md += `aiSummary: ${JSON.stringify(item.aiSummary)}\n`;
        }
        if (item.category) {
            md += `category: "${item.category}"\n`;
        }
        if (item.version) {
            md += `version: "${item.version}"\n`;
        }
        if (item.startDate) {
            md += `startDate: ${item.startDate.toISOString()}\n`;
        }
        if (item.endDate) {
            md += `endDate: ${item.endDate.toISOString()}\n`;
        }
        if (item.tags && item.tags.length > 0) {
            const tags = item.tags.map((t) => t.tag.name);
            md += `tags: ${JSON.stringify(tags)}\n`;
        }
        if (item.related && item.related.length > 0) {
            md += `related: ${JSON.stringify(item.related)}\n`;
        }
        if (item.keywords && item.keywords.length > 0) {
            const keywords = {};
            item.keywords
                .sort((a, b) => b.weight - a.weight)
                .slice(0, 5)
                .forEach((k) => {
                keywords[k.keyword.word] = parseFloat(k.weight.toFixed(2));
            });
            md += `keywords: ${JSON.stringify(keywords)}\n`;
        }
        if (item.concepts && item.concepts.length > 0) {
            const concepts = {};
            item.concepts
                .sort((a, b) => b.confidence - a.confidence)
                .slice(0, 5)
                .forEach((c) => {
                concepts[c.concept.name] = parseFloat(c.confidence.toFixed(2));
            });
            md += `concepts: ${JSON.stringify(concepts)}\n`;
        }
        if (item.embedding) {
            const embeddingBase64 = Buffer.from(item.embedding).toString('base64');
            md += `embedding: "${embeddingBase64}"\n`;
        }
        md += `createdAt: ${item.createdAt.toISOString()}\n`;
        md += `updatedAt: ${item.updatedAt.toISOString()}\n`;
        md += '---\n\n';
        if (item.content) {
            md += item.content;
        }
        return md;
    }
    formatSystemStateAsMarkdown(state) {
        let md = '---\n';
        md += `id: ${state.id}\n`;
        md += `type: system_state\n`;
        md += `version: "${state.version}"\n`;
        const tags = state.tags ? JSON.parse(state.tags) : [];
        if (tags.length > 0) {
            md += `tags: ${JSON.stringify(tags)}\n`;
        }
        const relatedItems = state.relatedItems ? JSON.parse(state.relatedItems) : [];
        if (relatedItems.length > 0) {
            md += `relatedItems: ${JSON.stringify(relatedItems)}\n`;
        }
        if (state.metrics) {
            try {
                const metrics = JSON.parse(state.metrics);
                md += `metrics: ${JSON.stringify(metrics)}\n`;
            }
            catch {
                md += `metrics: "${state.metrics}"\n`;
            }
        }
        if (state.context) {
            try {
                const context = JSON.parse(state.context);
                md += `context: ${JSON.stringify(context)}\n`;
            }
            catch {
                md += `context: "${state.context}"\n`;
            }
        }
        if (state.checkpoint) {
            md += `checkpoint: "${state.checkpoint}"\n`;
        }
        if (state.metadata) {
            try {
                const metadata = JSON.parse(state.metadata);
                if (Object.keys(metadata).length > 0) {
                    md += `metadata: ${JSON.stringify(metadata)}\n`;
                }
            }
            catch {
                md += `metadata: "${state.metadata}"\n`;
            }
        }
        md += `isActive: ${state.isActive}\n`;
        if (state.summary) {
            md += `summary: ${JSON.stringify(state.summary)}\n`;
        }
        md += `createdAt: ${state.createdAt.toISOString()}\n`;
        md += `updatedAt: ${state.updatedAt.toISOString()}\n`;
        md += '---\n\n';
        if (state.content) {
            md += state.content;
        }
        return md;
    }
}