@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
903 lines (902 loc) • 34 kB
JavaScript
/**
* Data Export Service - Comprehensive data export capabilities
*
* Provides professional-grade data export functionality with support for
* multiple formats, custom templates, and automated report generation.
*/
import { structuredLogger } from './structured-logger.js';
import { globalAnalyticsEngine } from './analytics-engine.js';
export class DataExportService {
templates = new Map();
jobs = new Map();
EXPORT_DIR = '/tmp/hive-ai-exports';
JOB_RETENTION_HOURS = 24;
constructor() {
this.initializeDefaultTemplates();
this.startCleanupTimer();
}
/**
* Initialize default export templates
*/
initializeDefaultTemplates() {
const defaultTemplates = [
{
name: 'Executive Summary',
description: 'High-level executive summary with key metrics and insights',
format: 'pdf',
sections: [
{
id: 'exec_summary',
title: 'Executive Summary',
type: 'summary',
enabled: true,
config: { includeKPIs: true, includeAlerts: true }
},
{
id: 'key_metrics',
title: 'Key Performance Indicators',
type: 'metrics',
enabled: true,
config: {
metrics: ['successRate', 'qualityScore', 'costPerQuery', 'systemUptime'],
visualization: 'dashboard'
}
},
{
id: 'recommendations',
title: 'Strategic Recommendations',
type: 'recommendations',
enabled: true,
config: { priorityFilter: 'high', maxItems: 5 }
}
],
styling: {
theme: 'corporate',
colors: {
primary: '#2563eb',
secondary: '#64748b',
accent: '#f59e0b',
text: '#1e293b',
background: '#ffffff'
},
fonts: {
heading: 'Arial, sans-serif',
body: 'Arial, sans-serif',
code: 'Courier New, monospace'
},
layout: {
pageSize: 'A4',
orientation: 'portrait',
margins: { top: 25, right: 25, bottom: 25, left: 25 }
}
},
metadata: {
version: '1.0',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
author: 'Hive AI System'
}
},
{
name: 'Technical Operations Report',
description: 'Detailed technical metrics and operational data',
format: 'excel',
sections: [
{
id: 'performance_metrics',
title: 'Performance Metrics',
type: 'metrics',
enabled: true,
config: {
includeRawData: true,
timeSeriesData: true,
breakdown: 'hourly'
}
},
{
id: 'system_health',
title: 'System Health',
type: 'metrics',
enabled: true,
config: {
metrics: ['uptime', 'errorRate', 'latency', 'throughput'],
includeAlerts: true
}
},
{
id: 'raw_data_export',
title: 'Raw Data',
type: 'raw_data',
enabled: true,
config: {
tables: ['conversations', 'performance_metrics', 'consensus_metrics'],
includeTimestamps: true
}
}
],
styling: {
theme: 'minimal',
colors: {
primary: '#059669',
secondary: '#6b7280',
accent: '#dc2626',
text: '#111827',
background: '#ffffff'
},
fonts: {
heading: 'Arial, sans-serif',
body: 'Arial, sans-serif',
code: 'Consolas, monospace'
},
layout: {
pageSize: 'A4',
orientation: 'landscape',
margins: { top: 20, right: 20, bottom: 20, left: 20 }
}
},
metadata: {
version: '1.0',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
author: 'Hive AI System'
}
},
{
name: 'Quality Analytics Dashboard',
description: 'Comprehensive quality metrics and consensus effectiveness analysis',
format: 'html',
sections: [
{
id: 'quality_overview',
title: 'Quality Overview',
type: 'summary',
enabled: true,
config: { focusArea: 'quality' }
},
{
id: 'consensus_effectiveness',
title: 'Consensus vs Single Model Analysis',
type: 'charts',
enabled: true,
config: {
chartTypes: ['comparison', 'trend', 'breakdown'],
includeABTestData: true
}
},
{
id: 'quality_metrics',
title: 'Quality Metrics Detail',
type: 'metrics',
enabled: true,
config: {
metrics: ['accuracy', 'coherence', 'completeness', 'factualAccuracy'],
includeDistributions: true
}
}
],
styling: {
theme: 'modern',
colors: {
primary: '#7c3aed',
secondary: '#a1a1aa',
accent: '#06b6d4',
text: '#27272a',
background: '#fafafa'
},
fonts: {
heading: 'Inter, sans-serif',
body: 'Inter, sans-serif',
code: 'Fira Code, monospace'
},
layout: {
pageSize: 'A4',
orientation: 'portrait',
margins: { top: 30, right: 30, bottom: 30, left: 30 }
}
},
metadata: {
version: '1.0',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
author: 'Hive AI System'
}
},
{
name: 'Financial Analysis Report',
description: 'Cost analysis, budget tracking, and ROI measurements',
format: 'excel',
sections: [
{
id: 'cost_summary',
title: 'Cost Summary',
type: 'summary',
enabled: true,
config: { focusArea: 'cost' }
},
{
id: 'budget_analysis',
title: 'Budget vs Actual Analysis',
type: 'charts',
enabled: true,
config: {
chartTypes: ['budget_variance', 'cost_trends', 'model_breakdown'],
includeProjections: true
}
},
{
id: 'cost_optimization',
title: 'Cost Optimization Opportunities',
type: 'recommendations',
enabled: true,
config: { category: 'cost', priorityFilter: 'medium' }
},
{
id: 'financial_raw_data',
title: 'Detailed Cost Data',
type: 'raw_data',
enabled: true,
config: {
tables: ['conversation_costs'],
includeCalculations: true
}
}
],
styling: {
theme: 'professional',
colors: {
primary: '#dc2626',
secondary: '#4b5563',
accent: '#059669',
text: '#1f2937',
background: '#ffffff'
},
fonts: {
heading: 'Arial, sans-serif',
body: 'Arial, sans-serif',
code: 'Courier New, monospace'
},
layout: {
pageSize: 'A4',
orientation: 'portrait',
margins: { top: 25, right: 25, bottom: 25, left: 25 }
}
},
metadata: {
version: '1.0',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
author: 'Hive AI System'
}
}
];
defaultTemplates.forEach(template => {
const id = `template_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
this.templates.set(id, { ...template, id });
});
}
/**
* Create export job
*/
async createExportJob(templateId, timeframe, filters) {
try {
const template = this.templates.get(templateId);
if (!template) {
return {
success: false,
jobId: '',
error: `Template ${templateId} not found`
};
}
const jobId = `job_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const job = {
id: jobId,
status: 'pending',
progress: 0,
templateId,
timeframe,
filters,
createdAt: new Date().toISOString()
};
this.jobs.set(jobId, job);
// Start processing job asynchronously
this.processExportJob(jobId);
return {
success: true,
jobId,
filePath: undefined,
downloadUrl: `/api/exports/${jobId}/download`
};
}
catch (error) {
structuredLogger.error('Export job creation failed', { templateId, timeframe }, error);
return {
success: false,
jobId: '',
error: error.message
};
}
}
/**
* Process export job
*/
async processExportJob(jobId) {
const job = this.jobs.get(jobId);
if (!job)
return;
try {
job.status = 'processing';
job.progress = 10;
const template = this.templates.get(job.templateId);
if (!template) {
throw new Error(`Template ${job.templateId} not found`);
}
structuredLogger.info('Processing export job', { jobId, templateId: job.templateId });
// Generate analytics report
job.progress = 30;
const report = await globalAnalyticsEngine.generateAnalyticsReport('operational', // Default type, could be configurable
job.timeframe);
// Apply filters if specified
job.progress = 50;
const filteredReport = job.filters ? this.applyFilters(report, job.filters) : report;
// Generate export content based on template
job.progress = 70;
const exportContent = await this.generateExportContent(template, filteredReport);
// Save to file
job.progress = 90;
const filePath = await this.saveExportFile(jobId, template.format, exportContent);
const fileSize = Buffer.byteLength(exportContent, 'utf8');
// Complete job
job.status = 'completed';
job.progress = 100;
job.completedAt = new Date().toISOString();
job.filePath = filePath;
job.fileSize = fileSize;
structuredLogger.info('Export job completed', {
jobId,
filePath,
fileSize,
templateId: job.templateId
});
}
catch (error) {
job.status = 'failed';
job.error = error.message;
structuredLogger.error('Export job failed', { jobId }, error);
}
}
/**
* Generate export content based on template
*/
async generateExportContent(template, report) {
switch (template.format) {
case 'json':
return this.generateJSONExport(template, report);
case 'csv':
return this.generateCSVExport(template, report);
case 'excel':
return this.generateExcelExport(template, report);
case 'html':
return this.generateHTMLExport(template, report);
case 'pdf':
return this.generatePDFExport(template, report);
case 'markdown':
return this.generateMarkdownExport(template, report);
case 'xml':
return this.generateXMLExport(template, report);
default:
throw new Error(`Unsupported export format: ${template.format}`);
}
}
/**
* Generate JSON export
*/
generateJSONExport(template, report) {
const exportData = {
metadata: {
templateName: template.name,
generatedAt: new Date().toISOString(),
reportId: report.id,
timeframe: report.timeframe
},
sections: {}
};
template.sections.forEach(section => {
if (!section.enabled)
return;
switch (section.type) {
case 'summary':
exportData.sections[section.id] = {
title: section.title,
data: report.executiveSummary
};
break;
case 'metrics':
exportData.sections[section.id] = {
title: section.title,
data: this.extractMetrics(report.metrics, section.config)
};
break;
case 'charts':
exportData.sections[section.id] = {
title: section.title,
data: report.charts
};
break;
case 'recommendations':
exportData.sections[section.id] = {
title: section.title,
data: report.recommendations
};
break;
case 'raw_data':
exportData.sections[section.id] = {
title: section.title,
data: report.metrics
};
break;
}
});
return JSON.stringify(exportData, null, 2);
}
/**
* Generate CSV export
*/
generateCSVExport(template, report) {
const rows = [
['Section', 'Metric', 'Value', 'Category', 'Timestamp']
];
template.sections.forEach(section => {
if (!section.enabled)
return;
if (section.type === 'metrics') {
const metrics = this.extractMetrics(report.metrics, section.config);
this.flattenMetricsToRows(metrics, section.title, rows);
}
});
return rows.map(row => row.map(cell => `"${cell}"`).join(',')).join('\n');
}
/**
* Generate Excel export (placeholder)
*/
generateExcelExport(template, report) {
// In a real implementation, this would use a library like ExcelJS
// For now, return CSV-like data with Excel structure comments
return `# Excel Export for ${template.name}\n` + this.generateCSVExport(template, report);
}
/**
* Generate HTML export
*/
generateHTMLExport(template, report) {
const styling = template.styling || this.getDefaultStyling();
let html = `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>${template.name} - ${report.timeframe}</title>
<style>
body {
font-family: ${styling.fonts.body};
color: ${styling.colors.text};
background-color: ${styling.colors.background};
margin: 0;
padding: 20px;
line-height: 1.6;
}
.header {
background: linear-gradient(135deg, ${styling.colors.primary}, ${styling.colors.secondary});
color: white;
padding: 30px;
border-radius: 8px;
margin-bottom: 30px;
}
.header h1 {
margin: 0;
font-size: 2.5em;
font-weight: 300;
}
.header .subtitle {
margin-top: 10px;
opacity: 0.9;
font-size: 1.1em;
}
.section {
background: white;
border: 1px solid #e5e7eb;
border-radius: 8px;
padding: 25px;
margin-bottom: 25px;
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
}
.section h2 {
color: ${styling.colors.primary};
border-bottom: 2px solid ${styling.colors.accent};
padding-bottom: 10px;
margin-top: 0;
}
.metric-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 20px;
margin: 20px 0;
}
.metric-card {
background: ${styling.colors.background};
border: 1px solid #e5e7eb;
border-radius: 6px;
padding: 20px;
text-align: center;
}
.metric-value {
font-size: 2em;
font-weight: bold;
color: ${styling.colors.primary};
}
.metric-label {
color: ${styling.colors.secondary};
font-size: 0.9em;
margin-top: 5px;
}
.recommendations {
background: #f8fafc;
border-left: 4px solid ${styling.colors.accent};
padding: 20px;
margin: 20px 0;
}
.recommendation-item {
background: white;
border-radius: 6px;
padding: 15px;
margin: 10px 0;
border-left: 3px solid ${styling.colors.accent};
}
.priority-high { border-left-color: #dc2626; }
.priority-medium { border-left-color: #f59e0b; }
.priority-low { border-left-color: #059669; }
.footer {
text-align: center;
margin-top: 40px;
padding: 20px;
color: ${styling.colors.secondary};
font-size: 0.9em;
}
</style>
</head>
<body>
<div class="header">
<h1>${template.name}</h1>
<div class="subtitle">Generated on ${new Date().toLocaleDateString()} | Timeframe: ${report.timeframe}</div>
</div>
`;
template.sections.forEach(section => {
if (!section.enabled)
return;
html += ` <div class="section">\n <h2>${section.title}</h2>\n`;
switch (section.type) {
case 'summary':
html += this.generateHTMLSummarySection(report.executiveSummary);
break;
case 'metrics':
html += this.generateHTMLMetricsSection(report.metrics, section.config);
break;
case 'recommendations':
html += this.generateHTMLRecommendationsSection(report.recommendations);
break;
case 'charts':
html += ` <p><em>Chart visualizations would be rendered here with a charting library like Chart.js</em></p>\n`;
break;
}
html += ` </div>\n`;
});
html += `
<div class="footer">
<p>Generated by Hive AI Analytics Engine v1.0</p>
<p>Report ID: ${report.id}</p>
</div>
</body>
</html>`;
return html;
}
/**
* Generate PDF export (placeholder)
*/
generatePDFExport(template, report) {
// In a real implementation, this would use a library like PDFKit or Puppeteer
return `PDF Export placeholder for ${template.name}`;
}
/**
* Generate Markdown export
*/
generateMarkdownExport(template, report) {
let markdown = `# ${template.name}\n\n`;
markdown += `**Generated:** ${new Date().toISOString()}\n`;
markdown += `**Timeframe:** ${report.timeframe}\n`;
markdown += `**Report ID:** ${report.id}\n\n`;
template.sections.forEach(section => {
if (!section.enabled)
return;
markdown += `## ${section.title}\n\n`;
switch (section.type) {
case 'summary':
markdown += this.generateMarkdownSummarySection(report.executiveSummary);
break;
case 'metrics':
markdown += this.generateMarkdownMetricsSection(report.metrics, section.config);
break;
case 'recommendations':
markdown += this.generateMarkdownRecommendationsSection(report.recommendations);
break;
}
markdown += '\n';
});
return markdown;
}
/**
* Generate XML export
*/
generateXMLExport(template, report) {
return `<?xml version="1.0" encoding="UTF-8"?>
<analytics_report>
<metadata>
<template_name>${template.name}</template_name>
<generated_at>${new Date().toISOString()}</generated_at>
<timeframe>${report.timeframe}</timeframe>
<report_id>${report.id}</report_id>
</metadata>
<executive_summary>
${report.executiveSummary.keyFindings.map(finding => `<finding>${finding}</finding>`).join('\n ')}
</executive_summary>
<metrics>
<business_kpis>
<total_queries>${report.metrics.businessKPIs.totalQueries}</total_queries>
<success_rate>${report.metrics.businessKPIs.successRate}</success_rate>
<quality_score>${report.metrics.businessKPIs.averageQualityScore}</quality_score>
</business_kpis>
</metrics>
</analytics_report>`;
}
/**
* Helper methods for HTML generation
*/
generateHTMLSummarySection(summary) {
let html = '';
if (summary.keyFindings?.length > 0) {
html += ' <h3>Key Findings</h3>\n <ul>\n';
summary.keyFindings.forEach((finding) => {
html += ` <li>${finding}</li>\n`;
});
html += ' </ul>\n';
}
if (summary.performanceHighlights?.length > 0) {
html += ' <h3>Performance Highlights</h3>\n <ul>\n';
summary.performanceHighlights.forEach((highlight) => {
html += ` <li>${highlight}</li>\n`;
});
html += ' </ul>\n';
}
return html;
}
generateHTMLMetricsSection(metrics, config) {
let html = ' <div class="metric-grid">\n';
// Business KPIs
html += ` <div class="metric-card">
<div class="metric-value">${metrics.businessKPIs.totalQueries}</div>
<div class="metric-label">Total Queries</div>
</div>
<div class="metric-card">
<div class="metric-value">${(metrics.businessKPIs.successRate * 100).toFixed(1)}%</div>
<div class="metric-label">Success Rate</div>
</div>
<div class="metric-card">
<div class="metric-value">${metrics.businessKPIs.averageQualityScore.toFixed(1)}</div>
<div class="metric-label">Quality Score</div>
</div>
<div class="metric-card">
<div class="metric-value">$${metrics.businessKPIs.costPerQuery.toFixed(4)}</div>
<div class="metric-label">Cost Per Query</div>
</div>\n`;
html += ' </div>\n';
return html;
}
generateHTMLRecommendationsSection(recommendations) {
let html = ' <div class="recommendations">\n';
['immediate', 'shortTerm', 'longTerm'].forEach(timeframe => {
const items = recommendations[timeframe] || [];
if (items.length > 0) {
html += ` <h3>${timeframe === 'immediate' ? 'Immediate Actions' : timeframe === 'shortTerm' ? 'Short-term Improvements' : 'Long-term Strategy'}</h3>\n`;
items.forEach((item) => {
html += ` <div class="recommendation-item priority-${item.priority}">
<h4>${item.title}</h4>
<p>${item.description}</p>
<p><strong>Expected Impact:</strong> ${item.expectedImpact}</p>
<p><strong>Timeline:</strong> ${item.timeline}</p>
</div>\n`;
});
}
});
html += ' </div>\n';
return html;
}
/**
* Helper methods for Markdown generation
*/
generateMarkdownSummarySection(summary) {
let markdown = '';
if (summary.keyFindings?.length > 0) {
markdown += '### Key Findings\n\n';
summary.keyFindings.forEach((finding) => {
markdown += `- ${finding}\n`;
});
markdown += '\n';
}
if (summary.performanceHighlights?.length > 0) {
markdown += '### Performance Highlights\n\n';
summary.performanceHighlights.forEach((highlight) => {
markdown += `- ${highlight}\n`;
});
markdown += '\n';
}
return markdown;
}
generateMarkdownMetricsSection(metrics, config) {
return `| Metric | Value |
|--------|--------|
| Total Queries | ${metrics.businessKPIs.totalQueries} |
| Success Rate | ${(metrics.businessKPIs.successRate * 100).toFixed(1)}% |
| Quality Score | ${metrics.businessKPIs.averageQualityScore.toFixed(1)}/10 |
| Cost Per Query | $${metrics.businessKPIs.costPerQuery.toFixed(4)} |
| System Uptime | ${metrics.operational.systemUptime.toFixed(1)}% |
`;
}
generateMarkdownRecommendationsSection(recommendations) {
let markdown = '';
['immediate', 'shortTerm', 'longTerm'].forEach(timeframe => {
const items = recommendations[timeframe] || [];
if (items.length > 0) {
const title = timeframe === 'immediate' ? 'Immediate Actions' :
timeframe === 'shortTerm' ? 'Short-term Improvements' :
'Long-term Strategy';
markdown += `### ${title}\n\n`;
items.forEach((item, index) => {
markdown += `${index + 1}. **${item.title}** (${item.priority} priority)\n`;
markdown += ` - ${item.description}\n`;
markdown += ` - Expected Impact: ${item.expectedImpact}\n`;
markdown += ` - Timeline: ${item.timeline}\n\n`;
});
}
});
return markdown;
}
/**
* Utility methods
*/
extractMetrics(metrics, config) {
if (config.metrics) {
const extracted = {};
config.metrics.forEach((metricPath) => {
const value = this.getNestedValue(metrics, metricPath);
if (value !== undefined) {
extracted[metricPath] = value;
}
});
return extracted;
}
return metrics;
}
getNestedValue(obj, path) {
return path.split('.').reduce((current, key) => current?.[key], obj);
}
flattenMetricsToRows(metrics, sectionTitle, rows) {
const flatten = (obj, prefix = '') => {
Object.entries(obj).forEach(([key, value]) => {
const fullKey = prefix ? `${prefix}.${key}` : key;
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
flatten(value, fullKey);
}
else {
rows.push([
sectionTitle,
fullKey,
String(value),
this.categorizeMetric(fullKey),
new Date().toISOString()
]);
}
});
};
flatten(metrics);
}
categorizeMetric(metricPath) {
if (metricPath.includes('business'))
return 'Business';
if (metricPath.includes('cost'))
return 'Cost';
if (metricPath.includes('quality'))
return 'Quality';
if (metricPath.includes('operational'))
return 'Operational';
if (metricPath.includes('technical'))
return 'Technical';
return 'General';
}
applyFilters(report, filters) {
// Placeholder - would implement actual filtering logic
return report;
}
async saveExportFile(jobId, format, content) {
// Placeholder - would save to actual file system
const fileName = `${jobId}.${format}`;
const filePath = `${this.EXPORT_DIR}/${fileName}`;
return filePath;
}
getDefaultStyling() {
return {
theme: 'professional',
colors: {
primary: '#2563eb',
secondary: '#64748b',
accent: '#f59e0b',
text: '#1e293b',
background: '#ffffff'
},
fonts: {
heading: 'Arial, sans-serif',
body: 'Arial, sans-serif',
code: 'Courier New, monospace'
},
layout: {
pageSize: 'A4',
orientation: 'portrait',
margins: { top: 25, right: 25, bottom: 25, left: 25 }
}
};
}
startCleanupTimer() {
setInterval(() => {
this.cleanupOldJobs();
}, 60 * 60 * 1000); // Run every hour
}
cleanupOldJobs() {
const cutoffTime = Date.now() - (this.JOB_RETENTION_HOURS * 60 * 60 * 1000);
for (const [jobId, job] of this.jobs.entries()) {
const jobTime = new Date(job.createdAt).getTime();
if (jobTime < cutoffTime) {
this.jobs.delete(jobId);
// In a real implementation, would also delete the file
}
}
}
/**
* Public API methods
*/
getJobStatus(jobId) {
return this.jobs.get(jobId);
}
getAvailableTemplates() {
return Array.from(this.templates.values());
}
createCustomTemplate(template) {
const id = `template_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
this.templates.set(id, { ...template, id });
return id;
}
updateTemplate(id, updates) {
const template = this.templates.get(id);
if (!template)
return false;
this.templates.set(id, { ...template, ...updates, id });
return true;
}
deleteTemplate(id) {
return this.templates.delete(id);
}
getExportHistory() {
return Array.from(this.jobs.values()).sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
}
}
/**
* Global data export service instance
*/
export const globalDataExportService = new DataExportService();
//# sourceMappingURL=data-export-service.js.map