@invisiblecities/sidequest-cqo
Version:
Configuration-agnostic TypeScript and ESLint orchestrator with real-time watch mode, SQLite persistence, and intelligent terminal detection
536 lines • 21.7 kB
JavaScript
/**
* Storage Service for Code Quality Orchestrator
* Provides high-level database operations with optimized batch processing
*/
import { getDatabase } from "../database/connection.js";
import { sql } from "kysely";
import { violationsToDatabaseFormat, computeViolationDeltas, prepareDeltasForInsertion, chunk, formatDateTimeForDatabase, createPerformanceMetric, validateViolation, sanitizeViolation, } from "../database/utils.js";
import { debugLog } from "../utils/debug-logger.js";
// ============================================================================
// Storage Service Class
// ============================================================================
export class StorageService {
config;
batchSize;
maxHistoryAge; // Days
enableMetrics;
constructor(config = {}) {
this.config = {
database: config.database || { path: "./data/code-quality.db" },
batchSize: config.batchSize || 100,
maxHistoryAge: config.maxHistoryAge || 30,
enablePerformanceMetrics: config.enablePerformanceMetrics || true,
};
this.batchSize = this.config.batchSize;
this.maxHistoryAge = this.config.maxHistoryAge;
this.enableMetrics = this.config.enablePerformanceMetrics;
}
// ========================================================================
// Violation Management
// ========================================================================
/**
* Store violations with batch processing and deduplication
*/
async storeViolations(violations) {
const startTime = performance.now();
const database = getDatabase();
// Convert to database format
const databaseViolations = violationsToDatabaseFormat(violations);
// Validate all violations
const errors = [];
const validViolations = [];
for (const violation of databaseViolations) {
const validationErrors = validateViolation(violation);
if (validationErrors.length > 0) {
errors.push(`Violation ${violation.hash}: ${validationErrors.join(", ")}`);
}
else {
validViolations.push(sanitizeViolation(violation));
}
}
let inserted = 0;
let updated = 0;
if (validViolations.length > 0) {
// Process in batches for performance
const batches = chunk(validViolations, this.batchSize);
for (const batch of batches) {
const result = await database.transaction().execute(async (trx) => {
let batchInserted = 0;
let batchUpdated = 0;
for (const violation of batch) {
try {
// Try to insert, handle conflicts by updating last_seen_at
const insertResult = await trx
.insertInto("violations")
.values(violation)
.onConflict((oc) => oc.column("hash").doUpdateSet({
last_seen_at: formatDateTimeForDatabase(),
status: "active", // Reactivate if previously resolved
}))
.returning(["id"])
.executeTakeFirst();
if (insertResult) {
// Check if this was an insert or update by querying the violation
const existingViolation = await trx
.selectFrom("violations")
.select(["first_seen_at", "last_seen_at"])
.where("hash", "=", violation.hash)
.executeTakeFirst();
if (existingViolation?.first_seen_at ===
existingViolation?.last_seen_at) {
batchInserted++;
}
else {
batchUpdated++;
}
}
}
catch (error) {
errors.push(`Failed to store violation ${violation.hash}: ${error}`);
}
}
return { batchInserted, batchUpdated };
});
inserted += result.batchInserted;
updated += result.batchUpdated;
}
}
const executionTime = performance.now() - startTime;
// Record performance metric
if (this.enableMetrics) {
await this.recordPerformanceMetric("violation_storage", executionTime, "ms", `violations: ${violations.length}, batches: ${Math.ceil(validViolations.length / this.batchSize)}`);
}
return { inserted, updated, errors };
}
/**
* Get violations with flexible filtering
*/
async getViolations(parameters = {}) {
const database = getDatabase();
let query = database.selectFrom("violations").selectAll();
// Apply filters
if (parameters.status) {
query = query.where("status", "=", parameters.status);
}
if (parameters.categories && parameters.categories.length > 0) {
query = query.where("category", "in", parameters.categories);
}
if (parameters.sources && parameters.sources.length > 0) {
query = query.where("source", "in", parameters.sources);
}
if (parameters.severities && parameters.severities.length > 0) {
query = query.where("severity", "in", parameters.severities);
}
if (parameters.file_paths && parameters.file_paths.length > 0) {
query = query.where("file_path", "in", parameters.file_paths);
}
if (parameters.since) {
query = query.where("last_seen_at", ">=", parameters.since);
}
// Apply pagination
if (parameters.limit) {
query = query.limit(Math.min(parameters.limit, 1000)); // Cap at 1000
}
if (parameters.offset) {
query = query.offset(parameters.offset);
}
// Order by most recent first
query = query.orderBy("last_seen_at", "desc");
const results = await query.execute();
debugLog("StorageService", "Retrieved violations", {
count: results.length,
});
return results;
}
/**
* Get violation summary for dashboard
*/
async getViolationSummary() {
const database = getDatabase();
// Create summary from violations table
return (await database
.selectFrom("violations")
.select([
"rule_id",
"category",
"severity",
"source",
sql `COUNT(*)`.as("count"),
sql `COUNT(DISTINCT file_path)`.as("affected_files"),
sql `MIN(first_seen_at)`.as("first_occurrence"),
sql `MAX(last_seen_at)`.as("last_occurrence"),
])
.where("status", "=", "active")
.groupBy(["rule_id", "category", "severity", "source"])
.execute());
}
/**
* Mark violations as resolved
*/
async resolveViolations(hashes) {
const database = getDatabase();
const result = await database
.updateTable("violations")
.set({
status: "resolved",
last_seen_at: formatDateTimeForDatabase(),
})
.where("hash", "in", hashes)
.where("status", "=", "active")
.execute();
let sum = 0;
for (const rowResult of result) {
sum += Number(rowResult.numUpdatedRows || 0);
}
return sum;
}
// ========================================================================
// Rule Check Management
// ========================================================================
/**
* Start a new rule check
*/
async startRuleCheck(rule, engine) {
const database = getDatabase();
const result = await database
.insertInto("rule_checks")
.values({
rule_id: rule,
engine,
status: "running",
started_at: formatDateTimeForDatabase(),
})
.returning("id")
.executeTakeFirst();
return result?.id || 0;
}
/**
* Complete a rule check with results
*/
async completeRuleCheck(checkId, violationsFound, executionTimeMs, filesChecked = 0, filesWithViolations = 0) {
const database = getDatabase();
await database
.updateTable("rule_checks")
.set({
status: "completed",
completed_at: formatDateTimeForDatabase(),
violations_found: violationsFound,
execution_time_ms: executionTimeMs,
files_checked: filesChecked,
files_with_violations: filesWithViolations,
})
.where("id", "=", checkId)
.execute();
}
/**
* Mark rule check as failed
*/
async failRuleCheck(checkId, errorMessage) {
const database = getDatabase();
await database
.updateTable("rule_checks")
.set({
status: "failed",
completed_at: formatDateTimeForDatabase(),
error_message: errorMessage,
})
.where("id", "=", checkId)
.execute();
}
// ========================================================================
// Historical Analysis
// ========================================================================
/**
* Record violation deltas for historical tracking
*/
async recordViolationDeltas(checkId, currentViolationHashes) {
const database = getDatabase();
// Get previous active violation hashes
const previousViolations = await database
.selectFrom("violations")
.select("hash")
.where("status", "=", "active")
.execute();
const previousHashes = previousViolations.map((v) => v.hash);
// Compute deltas
const deltas = computeViolationDeltas(previousHashes, currentViolationHashes);
// Prepare for insertion
const deltaRecords = prepareDeltasForInsertion(checkId, deltas);
// Insert in batches
if (deltaRecords.length > 0) {
const batches = chunk(deltaRecords, this.batchSize);
for (const batch of batches) {
await database.insertInto("violation_history").values(batch).execute();
}
}
// Count by action type
const counts = {
added: deltas.filter((d) => d.action === "added").length,
removed: deltas.filter((d) => d.action === "removed").length,
unchanged: deltas.filter((d) => d.action === "unchanged").length,
};
return counts;
}
/**
* Get violation history for analysis
*/
async getViolationHistory(parameters = {}) {
const database = getDatabase();
let query = database
.selectFrom("violation_history")
.selectAll()
.orderBy("recorded_at", "desc");
// Apply filters
if (parameters.since) {
query = query.where("recorded_at", ">=", parameters.since);
}
if (parameters.until) {
query = query.where("recorded_at", "<=", parameters.until);
}
if (parameters.actions && parameters.actions.length > 0) {
query = query.where("action", "in", parameters.actions);
}
if (parameters.rule_ids && parameters.rule_ids.length > 0) {
query = query
.innerJoin("rule_checks", "violation_history.check_id", "rule_checks.id")
.where("rule_checks.rule_id", "in", parameters.rule_ids);
}
// Apply pagination
if (parameters.limit) {
query = query.limit(Math.min(parameters.limit, 1000));
}
if (parameters.offset) {
query = query.offset(parameters.offset);
}
return await query.execute();
}
// ========================================================================
// Rule Scheduling
// ========================================================================
/**
* Initialize or update rule schedule
*/
async upsertRuleSchedule(schedule) {
const database = getDatabase();
// Convert boolean to number for SQLite compatibility
const sqliteSchedule = {
...schedule,
enabled: (schedule.enabled ? 1 : 0),
};
const result = await database
.insertInto("rule_schedules")
.values({
...sqliteSchedule,
created_at: formatDateTimeForDatabase(),
updated_at: formatDateTimeForDatabase(),
})
.onConflict((oc) => oc.columns(["rule_id", "engine"]).doUpdateSet({
enabled: sqliteSchedule.enabled,
priority: schedule.priority,
check_frequency_ms: schedule.check_frequency_ms,
updated_at: formatDateTimeForDatabase(),
}))
.returning("id")
.executeTakeFirst();
return result?.id || 0;
}
/**
* Get next rules to check based on schedule
*/
async getNextRulesToCheck(limit = 5) {
const database = getDatabase();
const now = formatDateTimeForDatabase();
return await database
.selectFrom("rule_schedules")
.selectAll()
.where("enabled", "=", 1)
.where((eb) =>
// eslint-disable-next-line unicorn/no-null
eb.or([eb("next_run_at", "is", null), eb("next_run_at", "<=", now)]))
.orderBy("priority", "asc")
.orderBy("next_run_at", "asc")
.limit(limit)
.execute();
}
/**
* Get rule performance data
*/
async getRulePerformance() {
const database = getDatabase();
// Create performance stats from rule_checks table
return (await database
.selectFrom("rule_checks")
.select([
"rule_id",
"engine",
sql `COUNT(*)`.as("total_runs"),
sql `SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END)`.as("successful_runs"),
sql `SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END)`.as("failed_runs"),
sql `AVG(execution_time_ms)`.as("avg_execution_time"),
sql `AVG(violations_found)`.as("avg_violations_found"),
sql `MAX(started_at)`.as("last_run"),
])
.groupBy(["rule_id", "engine"])
.execute());
}
// ========================================================================
// Dashboard Data
// ========================================================================
/**
* Get comprehensive dashboard data
*/
async getDashboardData() {
const database = getDatabase();
const [summary, rulePerformance, recentHistory, activeViolationsResult, filesAffectedResult, lastCheckResult, nextCheckResult,] = await Promise.all([
this.getViolationSummary(),
this.getRulePerformance(),
this.getViolationHistory({ limit: 20 }),
database
.selectFrom("violations")
.select((eb) => eb.fn.count("id").as("count"))
.where("status", "=", "active")
.executeTakeFirst(),
database
.selectFrom("violations")
.select(sql `COUNT(DISTINCT file_path)`.as("count"))
.where("status", "=", "active")
.executeTakeFirst(),
database
.selectFrom("rule_checks")
.select("completed_at")
.where("status", "=", "completed")
.orderBy("completed_at", "desc")
.limit(1)
.executeTakeFirst(),
database
.selectFrom("rule_schedules")
.select("next_run_at")
.where("enabled", "=", 1)
// eslint-disable-next-line unicorn/no-null
.where("next_run_at", "is not", null)
.orderBy("next_run_at", "asc")
.limit(1)
.executeTakeFirst(),
]);
return {
summary,
rule_performance: rulePerformance,
recent_history: recentHistory,
active_violations: Number(activeViolationsResult?.count || 0),
total_files_affected: Number(filesAffectedResult?.count || 0),
last_check_time: lastCheckResult?.completed_at || undefined,
next_scheduled_check: nextCheckResult?.next_run_at || undefined,
};
}
// ========================================================================
// Performance and Maintenance
// ========================================================================
/**
* Record performance metric
*/
async recordPerformanceMetric(type, value, unit, context) {
if (!this.enableMetrics) {
return;
}
const database = getDatabase();
const metric = createPerformanceMetric(type, value, unit, context);
await database.insertInto("performance_metrics").values(metric).execute();
}
/**
* Clean up old historical data
*/
async cleanupOldData() {
const database = getDatabase();
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - this.maxHistoryAge);
const cutoffDateString = formatDateTimeForDatabase(cutoffDate);
const [historyResult, metricsResult, violationsResult] = await Promise.all([
// Clean old violation history
database
.deleteFrom("violation_history")
.where("recorded_at", "<", cutoffDateString)
.execute(),
// Clean old performance metrics
database
.deleteFrom("performance_metrics")
.where("recorded_at", "<", cutoffDateString)
.execute(),
// Clean old resolved violations
database
.deleteFrom("violations")
.where("status", "=", "resolved")
.where("last_seen_at", "<", cutoffDateString)
.execute(),
]);
return {
violationHistoryDeleted: historyResult.reduce((sum, result) => sum + Number(result.numDeletedRows || 0), 0),
performanceMetricsDeleted: metricsResult.reduce((sum, result) => sum + Number(result.numDeletedRows || 0), 0),
resolvedViolationsDeleted: violationsResult.reduce((sum, result) => sum + Number(result.numDeletedRows || 0), 0),
};
}
/**
* Get storage statistics
*/
async getStorageStats() {
const database = getDatabase();
const [totalViolationsResult, activeViolationsResult, ruleChecksResult, historyResult, oldestResult, newestResult,] = await Promise.all([
database
.selectFrom("violations")
.select((eb) => eb.fn.count("id").as("count"))
.executeTakeFirst(),
database
.selectFrom("violations")
.select((eb) => eb.fn.count("id").as("count"))
.where("status", "=", "active")
.executeTakeFirst(),
database
.selectFrom("rule_checks")
.select((eb) => eb.fn.count("id").as("count"))
.executeTakeFirst(),
database
.selectFrom("violation_history")
.select((eb) => eb.fn.count("id").as("count"))
.executeTakeFirst(),
database
.selectFrom("violations")
.select("first_seen_at")
.orderBy("first_seen_at", "asc")
.limit(1)
.executeTakeFirst(),
database
.selectFrom("violations")
.select("last_seen_at")
.orderBy("last_seen_at", "desc")
.limit(1)
.executeTakeFirst(),
]);
return {
totalViolations: Number(totalViolationsResult?.count || 0),
activeViolations: Number(activeViolationsResult?.count || 0),
totalRuleChecks: Number(ruleChecksResult?.count || 0),
totalHistoryRecords: Number(historyResult?.count || 0),
oldestViolation: oldestResult?.first_seen_at,
newestViolation: newestResult?.last_seen_at,
};
}
}
// ============================================================================
// Service Factory and Singleton
// ============================================================================
let storageServiceInstance;
/**
* Get or create storage service instance
*/
export function getStorageService(config) {
if (!storageServiceInstance) {
storageServiceInstance = new StorageService(config);
}
return storageServiceInstance;
}
/**
* Reset storage service instance (useful for testing)
*/
export function resetStorageService() {
storageServiceInstance = undefined;
}
//# sourceMappingURL=storage-service.js.map