csvlod-ai-mcp-server
Version:
CSVLOD-AI MCP Server v3.0 with Quantum Context Intelligence - Revolutionary Context Intelligence Engine and Multimodal Processor for sovereign AI development
667 lines • 23.9 kB
JavaScript
/**
* CSVLOD-AI Framework v3.0 - Multimodal Processor
* Process and understand non-text content including diagrams, images, and media
*/
import * as fs from 'fs';
import * as path from 'path';
export var ContentType;
(function (ContentType) {
ContentType["IMAGE"] = "image";
ContentType["DIAGRAM"] = "diagram";
ContentType["DOCUMENT"] = "document";
ContentType["VIDEO"] = "video";
ContentType["AUDIO"] = "audio";
})(ContentType || (ContentType = {}));
export var DiagramType;
(function (DiagramType) {
DiagramType["ARCHITECTURE"] = "architecture";
DiagramType["SEQUENCE"] = "sequence";
DiagramType["ENTITY_RELATIONSHIP"] = "entity_relationship";
DiagramType["FLOW_CHART"] = "flow_chart";
DiagramType["NETWORK"] = "network";
DiagramType["UML_CLASS"] = "uml_class";
DiagramType["C4_SYSTEM"] = "c4_system";
DiagramType["MERMAID"] = "mermaid";
})(DiagramType || (DiagramType = {}));
export var ProcessingStatus;
(function (ProcessingStatus) {
ProcessingStatus["PENDING"] = "pending";
ProcessingStatus["PROCESSING"] = "processing";
ProcessingStatus["COMPLETED"] = "completed";
ProcessingStatus["ERROR"] = "error";
})(ProcessingStatus || (ProcessingStatus = {}));
/**
* Multimodal Processor - Core implementation
*/
export class MultiModalProcessor {
constructor(projectRoot) {
this.projectRoot = projectRoot;
this.configPath = path.join(projectRoot, '.ai-v3', 'multimodal');
this.processingQueue = [];
this.cache = new Map();
this.supportedFormats = new Map([
// Images
['.png', ContentType.IMAGE],
['.jpg', ContentType.IMAGE],
['.jpeg', ContentType.IMAGE],
['.svg', ContentType.IMAGE],
['.webp', ContentType.IMAGE],
// Diagrams
['.mermaid', ContentType.DIAGRAM],
['.puml', ContentType.DIAGRAM],
['.drawio', ContentType.DIAGRAM],
['.lucid', ContentType.DIAGRAM],
// Documents
['.pdf', ContentType.DOCUMENT],
['.docx', ContentType.DOCUMENT],
['.md', ContentType.DOCUMENT],
['.html', ContentType.DOCUMENT],
// Media
['.mp4', ContentType.VIDEO],
['.webm', ContentType.VIDEO],
['.mp3', ContentType.AUDIO],
['.wav', ContentType.AUDIO]
]);
this.initializeProcessor();
}
/**
* Initialize the Multimodal Processor
*/
async initializeProcessor() {
try {
// Ensure multimodal directory exists
if (!fs.existsSync(this.configPath)) {
fs.mkdirSync(this.configPath, { recursive: true });
}
// Scan for existing multimodal content
await this.scanMultimodalContent();
console.log(`🎨 Multimodal Processor v3.0 initialized`);
console.log(`📁 Found ${this.cache.size} multimodal items`);
console.log(`🔧 Supporting ${this.supportedFormats.size} format types`);
}
catch (error) {
console.error('Failed to initialize Multimodal Processor:', error);
}
}
/**
* Scan project for multimodal content
*/
async scanMultimodalContent() {
const items = await this.findMultimodalFiles(this.projectRoot);
for (const item of items) {
await this.processContent(item);
}
}
/**
* Find multimodal files in project
*/
async findMultimodalFiles(directory) {
const files = [];
try {
const items = fs.readdirSync(directory);
for (const item of items) {
const fullPath = path.join(directory, item);
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
// Skip certain directories
if (!item.startsWith('.') && !['node_modules', 'dist', 'build'].includes(item)) {
const subFiles = await this.findMultimodalFiles(fullPath);
files.push(...subFiles);
}
}
else {
const ext = path.extname(item).toLowerCase();
if (this.supportedFormats.has(ext)) {
files.push(fullPath);
}
}
}
}
catch (error) {
// Skip directories we can't read
}
return files;
}
/**
* Process multimodal content
*/
async processContent(filePath) {
const id = this.generateContentId(filePath);
// Check cache first
if (this.cache.has(id)) {
return this.cache.get(id);
}
try {
const stat = fs.statSync(filePath);
const ext = path.extname(filePath).toLowerCase();
const contentType = this.supportedFormats.get(ext) || ContentType.DOCUMENT;
const content = {
id: id,
type: contentType,
filePath: filePath,
metadata: {
fileName: path.basename(filePath),
fileSize: stat.size,
mimeType: this.getMimeType(ext),
createdAt: stat.birthtime,
lastModified: stat.mtime
},
extractedData: {},
relationships: [],
processingStatus: ProcessingStatus.PENDING
};
// Process based on content type
content.processingStatus = ProcessingStatus.PROCESSING;
switch (contentType) {
case ContentType.IMAGE:
await this.processImage(content);
break;
case ContentType.DIAGRAM:
await this.processDiagram(content);
break;
case ContentType.DOCUMENT:
await this.processDocument(content);
break;
default:
content.extractedData = { text: `Unsupported content type: ${contentType}` };
}
content.processingStatus = ProcessingStatus.COMPLETED;
// Cache the result
this.cache.set(id, content);
// Save processing result
await this.saveProcessingResult(content);
return content;
}
catch (error) {
console.error(`Error processing ${filePath}:`, error);
const errorContent = {
id: id,
type: ContentType.DOCUMENT,
filePath: filePath,
metadata: {
fileName: path.basename(filePath),
fileSize: 0,
mimeType: 'application/octet-stream',
createdAt: new Date(),
lastModified: new Date()
},
extractedData: { text: `Error processing file: ${error}` },
relationships: [],
processingStatus: ProcessingStatus.ERROR
};
return errorContent;
}
}
/**
* Process image content
*/
async processImage(content) {
const filePath = content.filePath;
const ext = path.extname(filePath).toLowerCase();
if (ext === '.svg') {
// Process SVG as diagram
await this.processSVGDiagram(content);
}
else {
// For other images, extract basic metadata
content.extractedData = {
text: `Image file: ${content.metadata.fileName}`,
concepts: [
{
term: 'visual_content',
confidence: 1.0,
context: 'Image file in project',
category: 'media'
}
]
};
// Try to extract text if it's a screenshot or diagram
await this.extractImageText(content);
}
}
/**
* Process SVG diagrams
*/
async processSVGDiagram(content) {
try {
const svgContent = fs.readFileSync(content.filePath, 'utf8');
// Extract text elements from SVG
const textMatches = svgContent.match(/<text[^>]*>([^<]+)<\/text>/g) || [];
const extractedText = textMatches
.map(match => match.replace(/<[^>]*>/g, ''))
.join(' ');
// Identify diagram type based on content
const diagramType = this.identifyDiagramType(extractedText);
content.extractedData = {
text: extractedText,
structure: {
sections: [{
title: 'SVG Diagram',
content: extractedText,
level: 1,
subsections: []
}],
headings: [],
lists: [],
tables: [],
codeBlocks: []
},
concepts: this.extractConceptsFromText(extractedText)
};
}
catch (error) {
console.error('Error processing SVG diagram:', error);
content.extractedData = { text: 'Error processing SVG diagram' };
}
}
/**
* Extract text from images (OCR simulation)
*/
async extractImageText(content) {
// Simulated OCR - in production would use actual OCR library
const concepts = [
{
term: 'image_content',
confidence: 0.8,
context: `Image file: ${content.metadata.fileName}`,
category: 'visual'
}
];
// If filename suggests it's a diagram or screenshot
const fileName = content.metadata.fileName.toLowerCase();
if (fileName.includes('diagram') || fileName.includes('architecture') || fileName.includes('flow')) {
concepts.push({
term: 'diagram',
confidence: 0.9,
context: 'Likely contains diagrammatic content',
category: 'architecture'
});
}
if (!content.extractedData.concepts) {
content.extractedData.concepts = [];
}
content.extractedData.concepts.push(...concepts);
}
/**
* Process diagram content
*/
async processDiagram(content) {
const ext = path.extname(content.filePath).toLowerCase();
switch (ext) {
case '.mermaid':
await this.processMermaidDiagram(content);
break;
case '.puml':
await this.processPlantUMLDiagram(content);
break;
default:
content.extractedData = { text: `Unsupported diagram format: ${ext}` };
}
}
/**
* Process Mermaid diagrams
*/
async processMermaidDiagram(content) {
try {
const mermaidContent = fs.readFileSync(content.filePath, 'utf8');
// Parse Mermaid syntax (simplified)
const diagramType = this.identifyMermaidDiagramType(mermaidContent);
const components = this.extractMermaidComponents(mermaidContent);
const connections = this.extractMermaidConnections(mermaidContent);
const diagramStructure = {
type: diagramType,
components: components,
connections: connections,
metadata: {
title: this.extractMermaidTitle(mermaidContent),
tags: ['mermaid', diagramType.toLowerCase()],
description: `Mermaid ${diagramType} diagram`
}
};
content.extractedData = {
text: mermaidContent,
structure: this.convertDiagramToStructure(diagramStructure),
components: components,
relationships: connections.map(c => ({
source: c.source,
target: c.target,
type: c.type,
label: c.label
})),
concepts: this.extractConceptsFromText(mermaidContent)
};
}
catch (error) {
console.error('Error processing Mermaid diagram:', error);
content.extractedData = { text: 'Error processing Mermaid diagram' };
}
}
/**
* Process PlantUML diagrams
*/
async processPlantUMLDiagram(content) {
try {
const pumlContent = fs.readFileSync(content.filePath, 'utf8');
// Basic PlantUML parsing
const components = this.extractPlantUMLComponents(pumlContent);
content.extractedData = {
text: pumlContent,
components: components,
concepts: this.extractConceptsFromText(pumlContent)
};
}
catch (error) {
console.error('Error processing PlantUML diagram:', error);
content.extractedData = { text: 'Error processing PlantUML diagram' };
}
}
/**
* Process document content
*/
async processDocument(content) {
const ext = path.extname(content.filePath).toLowerCase();
switch (ext) {
case '.md':
await this.processMarkdownDocument(content);
break;
case '.html':
await this.processHTMLDocument(content);
break;
default:
content.extractedData = { text: `Unsupported document format: ${ext}` };
}
}
/**
* Process Markdown documents
*/
async processMarkdownDocument(content) {
try {
const markdownContent = fs.readFileSync(content.filePath, 'utf8');
const structure = this.parseMarkdownStructure(markdownContent);
const concepts = this.extractConceptsFromText(markdownContent);
content.extractedData = {
text: markdownContent,
structure: structure,
concepts: concepts
};
}
catch (error) {
console.error('Error processing Markdown document:', error);
content.extractedData = { text: 'Error processing Markdown document' };
}
}
/**
* Process HTML documents
*/
async processHTMLDocument(content) {
try {
const htmlContent = fs.readFileSync(content.filePath, 'utf8');
// Extract text content from HTML (simplified)
const textContent = htmlContent.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
content.extractedData = {
text: textContent,
concepts: this.extractConceptsFromText(textContent)
};
}
catch (error) {
console.error('Error processing HTML document:', error);
content.extractedData = { text: 'Error processing HTML document' };
}
}
/**
* Utility methods
*/
generateContentId(filePath) {
return `multimodal_${path.basename(filePath, path.extname(filePath))}_${Date.now()}`;
}
getMimeType(extension) {
const mimeTypes = {
'.png': 'image/png',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.svg': 'image/svg+xml',
'.webp': 'image/webp',
'.pdf': 'application/pdf',
'.md': 'text/markdown',
'.html': 'text/html',
'.mp4': 'video/mp4',
'.webm': 'video/webm',
'.mp3': 'audio/mpeg',
'.wav': 'audio/wav'
};
return mimeTypes[extension] || 'application/octet-stream';
}
identifyDiagramType(content) {
const contentLower = content.toLowerCase();
if (contentLower.includes('graph') || contentLower.includes('flowchart')) {
return DiagramType.FLOW_CHART;
}
else if (contentLower.includes('sequence')) {
return DiagramType.SEQUENCE;
}
else if (contentLower.includes('class') || contentLower.includes('uml')) {
return DiagramType.UML_CLASS;
}
else if (contentLower.includes('architecture') || contentLower.includes('system')) {
return DiagramType.ARCHITECTURE;
}
else {
return DiagramType.FLOW_CHART;
}
}
identifyMermaidDiagramType(content) {
if (content.includes('graph ') || content.includes('flowchart ')) {
return DiagramType.FLOW_CHART;
}
else if (content.includes('sequenceDiagram')) {
return DiagramType.SEQUENCE;
}
else if (content.includes('classDiagram')) {
return DiagramType.UML_CLASS;
}
else {
return DiagramType.FLOW_CHART;
}
}
extractMermaidComponents(content) {
const components = [];
// Extract nodes (simplified regex)
const nodeMatches = content.match(/\s+([A-Z0-9]+)\[([^\]]+)\]/g) || [];
nodeMatches.forEach((match, index) => {
const parts = match.match(/([A-Z0-9]+)\[([^\]]+)\]/);
if (parts) {
components.push({
id: parts[1],
name: parts[2],
type: 'node',
position: { x: index * 100, y: 100 },
size: { width: 100, height: 50 },
properties: {}
});
}
});
return components;
}
extractMermaidConnections(content) {
const connections = [];
// Extract connections (simplified regex)
const connectionMatches = content.match(/\s+([A-Z0-9]+)\s*-->?\s*([A-Z0-9]+)(\|[^|]*\|)?/g) || [];
connectionMatches.forEach((match, index) => {
const parts = match.match(/([A-Z0-9]+)\s*-->?\s*([A-Z0-9]+)(\|([^|]*)\|)?/);
if (parts) {
connections.push({
id: `conn_${index}`,
source: parts[1],
target: parts[2],
type: 'flow',
label: parts[4] || undefined
});
}
});
return connections;
}
extractMermaidTitle(content) {
const titleMatch = content.match(/title\s+([^\n]+)/);
return titleMatch ? titleMatch[1].trim() : undefined;
}
extractPlantUMLComponents(content) {
const components = [];
// Simple PlantUML component extraction
const componentMatches = content.match(/\[([^\]]+)\]/g) || [];
componentMatches.forEach((match, index) => {
const name = match.slice(1, -1);
components.push({
name: name,
type: 'component',
properties: {},
position: { x: index * 100, y: 100 }
});
});
return components;
}
parseMarkdownStructure(content) {
const lines = content.split('\n');
const headings = [];
const sections = [];
const codeBlocks = [];
let currentSection = null;
let inCodeBlock = false;
let currentCodeBlock = '';
let currentLanguage = '';
for (const line of lines) {
// Headings
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
if (headingMatch) {
const level = headingMatch[1].length;
const text = headingMatch[2];
headings.push({ text, level });
if (currentSection && level <= currentSection.level) {
sections.push(currentSection);
}
currentSection = {
title: text,
content: '',
level: level,
subsections: []
};
}
// Code blocks
const codeBlockMatch = line.match(/^```(\w+)?$/);
if (codeBlockMatch) {
if (!inCodeBlock) {
inCodeBlock = true;
currentLanguage = codeBlockMatch[1] || '';
currentCodeBlock = '';
}
else {
inCodeBlock = false;
codeBlocks.push({
language: currentLanguage,
code: currentCodeBlock
});
currentCodeBlock = '';
}
}
else if (inCodeBlock) {
currentCodeBlock += line + '\n';
}
else if (currentSection) {
currentSection.content += line + '\n';
}
}
if (currentSection) {
sections.push(currentSection);
}
return {
sections,
headings,
lists: [], // TODO: implement list parsing
tables: [], // TODO: implement table parsing
codeBlocks
};
}
convertDiagramToStructure(diagram) {
return {
sections: [{
title: diagram.metadata.title || 'Diagram',
content: diagram.metadata.description || '',
level: 1,
subsections: []
}],
headings: [{
text: diagram.metadata.title || 'Diagram',
level: 1
}],
lists: [],
tables: [],
codeBlocks: []
};
}
extractConceptsFromText(text) {
const concepts = [];
// Technical concepts
const technicalTerms = [
'api', 'database', 'security', 'performance', 'authentication',
'framework', 'architecture', 'design', 'pattern', 'component',
'service', 'interface', 'protocol', 'endpoint', 'data'
];
const textLower = text.toLowerCase();
for (const term of technicalTerms) {
if (textLower.includes(term)) {
concepts.push({
term: term,
confidence: 0.8,
context: `Found in multimodal content`,
category: 'technical'
});
}
}
return concepts;
}
async saveProcessingResult(content) {
const resultPath = path.join(this.configPath, `${content.id}.json`);
try {
fs.writeFileSync(resultPath, JSON.stringify(content, null, 2));
}
catch (error) {
console.error('Error saving processing result:', error);
}
}
/**
* Get all processed multimodal content
*/
getAllContent() {
return Array.from(this.cache.values());
}
/**
* Get content by ID
*/
getContent(id) {
return this.cache.get(id);
}
/**
* Search content by type
*/
getContentByType(type) {
return Array.from(this.cache.values()).filter(content => content.type === type);
}
/**
* Get processing statistics
*/
getProcessingStats() {
const content = Array.from(this.cache.values());
const byType = {};
const byStatus = {};
for (const item of content) {
byType[item.type] = (byType[item.type] || 0) + 1;
byStatus[item.processingStatus] = (byStatus[item.processingStatus] || 0) + 1;
}
return {
total: content.length,
byType,
byStatus
};
}
}
export default MultiModalProcessor;
//# sourceMappingURL=multimodal-processor.js.map