signalk-parquet
Version:
SignalK plugin and webapp that archives SK data to Parquet files with a regimen control system, advanced querying, Claude integrated AI analysis, spatial capabilities, and REST API.
761 lines (675 loc) • 27.5 kB
JavaScript
import { getPluginPath } from './utils.js';
// Global variables for validation cancellation
let currentValidationController = null;
let currentRepairController = null;
let currentValidationJobId = null;
let validationCancelRequested = false;
let currentRepairJobId = null;
let repairCancelRequested = false;
// Cancel current validation
export function cancelValidation() {
if (validationCancelRequested) {
return;
}
const cancelBtn = document.getElementById('cancelValidationBtn');
const validateBtn = document.getElementById('validateBtn');
if (currentValidationJobId) {
validationCancelRequested = true;
if (cancelBtn) {
cancelBtn.disabled = true;
cancelBtn.textContent = 'Cancelling...';
}
if (validateBtn) {
validateBtn.textContent = '⏳ Cancelling validation...';
validateBtn.disabled = true;
}
fetch(
`${getPluginPath()}/api/validate-schemas/cancel/${currentValidationJobId}`,
{
method: 'POST',
}
)
.then(response => {
if (!response.ok) {
console.error('Cancel validation request failed:', response.status);
}
})
.catch(error => {
console.error('Cancel validation request error:', error);
});
} else if (currentValidationController) {
validationCancelRequested = true;
currentValidationController.abort();
currentValidationController = null;
if (cancelBtn) {
cancelBtn.disabled = true;
cancelBtn.textContent = 'Cancelling...';
}
if (validateBtn) {
validateBtn.textContent = '⏳ Cancelling validation...';
validateBtn.disabled = true;
}
}
}
// Cancel current repair
export function cancelRepair() {
if (repairCancelRequested) {
return;
}
repairCancelRequested = true;
const repairBtn = document.getElementById('repairBtn');
if (repairBtn) {
repairBtn.textContent = '⏳ Cancelling repair...';
repairBtn.disabled = true;
}
if (currentRepairJobId) {
fetch(
`${getPluginPath()}/api/repair-schemas/cancel/${currentRepairJobId}`,
{
method: 'POST',
}
).catch(error => {
console.error('Cancel repair request error:', error);
});
}
}
// Poll for validation progress
async function pollValidationProgress(
jobId,
timerElement,
startTime,
abortSignal
) {
let polling = true;
let lastProgress = null;
while (polling) {
try {
if (abortSignal?.aborted) {
throw new DOMException('Validation polling aborted', 'AbortError');
}
const response = await fetch(
`${getPluginPath()}/api/validate-schemas/progress/${jobId}`,
{
signal: abortSignal,
cache: 'no-store',
}
);
if (!response.ok) {
console.error('Failed to get progress:', response.status);
break;
}
const progress = await response.json();
lastProgress = progress;
const processed = progress.processed ?? 0;
const total = progress.total ?? 0;
const percent =
progress.percent ??
(total > 0 ? Math.round((processed / total) * 100) : 0);
const elapsed = (new Date() - startTime) / 1000;
const vessel = progress.currentVessel || 'Detecting vessel...';
const contextPath =
progress.currentRelativePath || progress.currentFile || 'Processing...';
const isCancelling =
progress.status === 'cancelling' || progress.cancelRequested;
const statusText =
progress.status === 'cancelled'
? 'Status: Cancelled'
: isCancelling
? 'Status: Cancelling...'
: 'Status: Running';
const statusColor =
progress.status === 'cancelled'
? '#d32f2f'
: isCancelling
? '#b36b00'
: '#666';
if (isCancelling) {
validationCancelRequested = true;
}
timerElement.innerHTML = `
<strong>⏱️ Validation Progress</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Elapsed: ${elapsed.toFixed(1)}s<br>
<strong>Files: ${processed.toLocaleString()} / ${total.toLocaleString()} (${percent}%)</strong><br>
<span style="font-size: 0.9em; color: #666;">Vessel: ${vessel}</span><br>
<span style="font-size: 0.9em; color: #666;">Current: ${contextPath}</span><br>
<span style="font-size: 0.9em; color: ${statusColor};">${statusText}</span>
`;
if (
progress.status === 'completed' ||
progress.status === 'cancelled' ||
progress.status === 'error'
) {
polling = false;
const endTime = new Date();
const totalTime = (endTime - startTime) / 1000;
if (progress.status === 'completed') {
timerElement.innerHTML = `
<strong>⏱️ Validation Complete</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Completed: ${endTime.toLocaleTimeString()}<br>
<strong>Total Time: ${totalTime.toFixed(1)}s</strong><br>
<strong>📁 Files: ${total.toLocaleString()} (100%)</strong>
`;
timerElement.style.background = '#e8f5e8';
timerElement.style.borderColor = '#4caf50';
} else if (progress.status === 'cancelled') {
timerElement.innerHTML = `
<strong>❌ Validation Cancelled</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Cancelled: ${endTime.toLocaleTimeString()}<br>
<strong>Time: ${totalTime.toFixed(1)}s</strong>
`;
timerElement.style.background = '#fff3cd';
timerElement.style.borderColor = '#ffc107';
} else if (progress.status === 'error') {
timerElement.innerHTML = `
<strong>⏱️ Validation Failed</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Failed: ${endTime.toLocaleTimeString()}<br>
<strong>Total Time: ${totalTime.toFixed(1)}s</strong>
`;
timerElement.style.background = '#ffeaea';
timerElement.style.borderColor = '#f44336';
}
}
await new Promise(resolve => setTimeout(resolve, 500));
} catch (error) {
if (error.name === 'AbortError') {
return lastProgress;
}
console.error('Progress polling error:', error);
break;
}
}
return lastProgress;
}
// Poll for repair progress
async function pollRepairProgress(jobId, timerElement, startTime, abortSignal) {
let polling = true;
let lastProgress = null;
while (polling) {
try {
if (abortSignal?.aborted) {
throw new DOMException('Repair polling aborted', 'AbortError');
}
const response = await fetch(
`${getPluginPath()}/api/repair-schemas/progress/${jobId}`,
{
signal: abortSignal,
cache: 'no-store',
}
);
if (!response.ok) {
console.error('Failed to get repair progress:', response.status);
break;
}
const progress = await response.json();
lastProgress = progress;
const processed = progress.processed ?? 0;
const total = progress.total ?? 0;
const percent =
progress.percent ??
(total > 0 ? Math.round((processed / total) * 100) : 0);
const elapsed = (new Date() - startTime) / 1000;
const statusText = progress.message || 'Repair in progress';
const statusColor =
progress.status === 'cancelled'
? '#d32f2f'
: progress.status === 'cancelling'
? '#b36b00'
: '#666';
timerElement.innerHTML = [
'<strong>🔧 Repair Progress</strong><br>',
`Started: ${startTime.toLocaleTimeString()}<br>`,
`Elapsed: ${elapsed.toFixed(1)}s<br>`,
`<strong>Files: ${processed.toLocaleString()} / ${total.toLocaleString()} (${percent}%)</strong><br>`,
progress.currentFile
? `<span style="font-size: 0.9em; color: #666;">Current: ${progress.currentFile}</span><br>`
: '',
`<span style="font-size: 0.9em; color: ${statusColor};">${statusText}</span>`,
]
.filter(Boolean)
.join('');
if (
progress.status === 'completed' ||
progress.status === 'cancelled' ||
progress.status === 'error'
) {
polling = false;
const endTime = new Date();
const totalTime = (endTime - startTime) / 1000;
lastProgress.completedAt = endTime;
if (progress.status === 'completed') {
timerElement.innerHTML = [
'<strong>✅ Repair Completed</strong><br>',
`Started: ${startTime.toLocaleTimeString()}<br>`,
`Ended: ${endTime.toLocaleTimeString()}<br>`,
`<strong>Total Time: ${totalTime.toFixed(1)}s</strong>`,
].join('');
timerElement.style.background = '#e8f5e8';
timerElement.style.borderColor = '#4caf50';
} else if (progress.status === 'cancelled') {
timerElement.innerHTML = [
'<strong>❌ Repair Cancelled</strong><br>',
`Started: ${startTime.toLocaleTimeString()}<br>`,
`Cancelled: ${endTime.toLocaleTimeString()}<br>`,
`<strong>Total Time: ${totalTime.toFixed(1)}s</strong>`,
].join('');
timerElement.style.background = '#fff3cd';
timerElement.style.borderColor = '#ffc107';
} else if (progress.status === 'error') {
timerElement.innerHTML = [
'<strong>⛔ Repair Failed</strong><br>',
`Started: ${startTime.toLocaleTimeString()}<br>`,
`Ended: ${endTime.toLocaleTimeString()}<br>`,
`<strong>Total Time: ${totalTime.toFixed(1)}s</strong>`,
].join('');
timerElement.style.background = '#ffeaea';
timerElement.style.borderColor = '#f44336';
}
}
await new Promise(resolve => setTimeout(resolve, 500));
} catch (error) {
if (error.name === 'AbortError') {
return lastProgress;
}
console.error('Repair progress polling error:', error);
break;
}
}
return lastProgress;
}
export async function runDataValidation() {
const statusDiv = document.getElementById('validationStatus');
const resultsDiv = document.getElementById('validationResults');
const summaryDiv = document.getElementById('validationSummary');
const detailsDiv = document.getElementById('validationDetails');
const summaryHeader = document.getElementById('validationSummaryHeader');
const detailsHeader = document.getElementById('validationDetailsHeader');
const button = document.getElementById('validateBtn');
const repairBtn = document.getElementById('repairBtn');
validationCancelRequested = false;
currentValidationJobId = null;
// Show loading state with cancel button and timer
button.textContent = '⏸️ Running Validation (click to cancel)';
button.style.background = '#ffc107';
button.onclick = cancelValidation;
if (repairBtn) {
repairBtn.style.display = 'inline-block';
repairBtn.disabled = true;
}
// Start timer
const startTime = new Date();
const timerElement = document.createElement('div');
timerElement.id = 'validationTimer';
timerElement.style.cssText =
'background: #e3f2fd; border: 1px solid #2196f3; border-radius: 5px; padding: 15px; margin-bottom: 15px; text-align: center; font-family: monospace; box-shadow: 0 2px 4px rgba(0,0,0,0.1);';
statusDiv.innerHTML = '';
statusDiv.appendChild(timerElement);
resultsDiv.style.display = 'none';
summaryDiv.innerHTML = '';
detailsDiv.innerHTML = '';
summaryDiv.style.display = 'none';
if (summaryHeader) summaryHeader.style.display = 'none';
if (detailsHeader) detailsHeader.style.display = 'none';
// Update timer every 100ms - NO FAKE PROGRESS
const timerInterval = setInterval(() => {
const elapsed = (new Date() - startTime) / 1000;
timerElement.innerHTML = `
<strong>⏱️ Validation Timer</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Elapsed: ${elapsed.toFixed(1)}s<br>
<span id="validationProgress" style="font-size: 0.9em; color: #666;">Running validation...</span>
`;
}, 100);
const controller = new AbortController();
currentValidationController = controller;
try {
// Set flag to track running validation
currentValidationController = true;
const response = await fetch(`${getPluginPath()}/api/validate-schemas`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
signal: controller.signal,
});
console.log('✅ Fetch completed, response status:', response.status);
const result = await response.json();
if (!response.ok) {
throw new Error(result.message || 'Validation failed');
}
let finalProgress = null;
// Start polling for progress if we got a jobId
if (result.jobId) {
console.log('📋 Starting progress polling for job:', result.jobId);
currentValidationJobId = result.jobId;
clearInterval(timerInterval); // Stop the basic timer since we're switching to progress polling
finalProgress = await pollValidationProgress(
result.jobId,
timerElement,
startTime,
controller.signal
);
} else {
// Old style response without polling
console.log('⚠️ No jobId in response, using old style completion');
}
// Stop timer and show final time
clearInterval(timerInterval);
const endTime = new Date();
const totalTime = (endTime - startTime) / 1000;
const summaryData = finalProgress?.result || result;
const overallTotal = finalProgress?.total ?? summaryData?.totalFiles ?? 0;
if (summaryData) {
const totalFiles = summaryData.totalFiles ?? 0;
const totalVessels = summaryData.totalVessels ?? 0;
const correctSchemas = summaryData.correctSchemas ?? 0;
const violations = summaryData.violations ?? 0;
const isCancelled = Boolean(
summaryData.cancelled ||
summaryData.error === 'Validation cancelled by user'
);
const isSuccess = Boolean(summaryData.success && !isCancelled);
const errorMessage = summaryData.error || summaryData.message || '';
const header = isCancelled
? '❌ Validation Cancelled'
: isSuccess
? '⏱️ Validation Complete'
: '⛔ Validation Failed';
const background = isCancelled
? '#fff3cd'
: isSuccess
? '#e8f5e8'
: '#ffeaea';
const border = isCancelled
? '#ffc107'
: isSuccess
? '#4caf50'
: '#f44336';
const successRate =
totalFiles > 0
? ((correctSchemas / totalFiles) * 100).toFixed(1)
: '0.0';
const filesLabel = isCancelled ? 'Processed files' : 'Total files';
const vesselsLabel = isCancelled
? 'Vessels encountered'
: 'Total vessels';
const correctLabel = isCancelled
? 'Correct schemas (processed)'
: 'Correct schemas';
const violationsLabel = isCancelled
? 'Detected schema issues'
: 'Schema violations';
const statsLines = [
`<div style="display: flex; justify-content: space-between; margin-bottom: 6px;"><span>📁 <strong>${filesLabel}:</strong></span> <span style="font-weight: bold;">${totalFiles.toLocaleString()}${isCancelled && overallTotal ? ` / ${overallTotal.toLocaleString()}` : isSuccess ? ' (100%)' : ''}</span></div>`,
`<div style="display: flex; justify-content: space-between; margin-bottom: 6px;"><span>🚢 <strong>${vesselsLabel}:</strong></span> <span style="font-weight: bold;">${totalVessels.toLocaleString()}</span></div>`,
`<div style="display: flex; justify-content: space-between; margin-bottom: 6px;"><span>✅ <strong>${correctLabel}:</strong></span> <span style="font-weight: bold; color: #4caf50;">${correctSchemas.toLocaleString()}</span></div>`,
`<div style="display: flex; justify-content: space-between; margin-bottom: 6px;"><span>❌ <strong>${violationsLabel}:</strong></span> <span style="font-weight: bold; color: ${violations > 0 ? '#f44336' : '#4caf50'};">${violations.toLocaleString()}</span></div>`,
`<div style="display: flex; justify-content: space-between;"><span>📊 <strong>${isCancelled ? 'Processed success rate' : 'Success rate'}:</strong></span> <span style="font-weight: bold; color: ${parseFloat(successRate) === 100 ? '#4caf50' : '#ff9800'};">${successRate}%</span></div>`,
].join('');
const statusNote =
isCancelled && overallTotal
? '<div style="margin-top: 8px; color: #b36b00;">Cancellation requested during processing. Only partial results are available.</div>'
: !isSuccess && !isCancelled && errorMessage
? `<div style="margin-top: 8px; color: #d32f2f;">${errorMessage}</div>`
: '';
timerElement.innerHTML = [
`<strong>${header}</strong><br>`,
`Started: ${startTime.toLocaleTimeString()}<br>`,
`Ended: ${endTime.toLocaleTimeString()}<br>`,
`<strong>Total Time: ${totalTime.toFixed(1)}s</strong>`,
`<div style="margin-top: 15px; padding: 12px; background: rgba(255,255,255,0.3); border-radius: 4px; text-align: left; line-height: 1.8; font-size: 14px;">${statsLines}</div>`,
statusNote,
]
.filter(Boolean)
.join('');
timerElement.style.background = background;
timerElement.style.borderColor = border;
if (isSuccess || isCancelled) {
const hasViolations =
Array.isArray(summaryData.violationDetails) &&
summaryData.violationDetails.length > 0;
if (hasViolations) {
if (summaryHeader) {
summaryHeader.style.display = 'none';
}
summaryDiv.style.display = 'none';
summaryDiv.innerHTML = '';
if (detailsHeader) {
detailsHeader.style.display = 'block';
detailsHeader.textContent = `Schema Violations (${violations.toLocaleString()})`;
}
detailsDiv.innerHTML = summaryData.violationDetails.join('\n');
resultsDiv.style.display = 'block';
} else {
if (summaryHeader) summaryHeader.style.display = 'none';
if (detailsHeader) detailsHeader.style.display = 'none';
summaryDiv.style.display = 'none';
summaryDiv.innerHTML = '';
detailsDiv.innerHTML = '';
resultsDiv.style.display = 'none';
}
if (repairBtn) {
repairBtn.style.display = 'inline-block';
repairBtn.disabled = violations === 0;
}
} else {
if (summaryHeader) {
summaryHeader.style.display = 'block';
summaryHeader.textContent = 'Validation Summary';
}
if (detailsHeader) detailsHeader.style.display = 'none';
summaryDiv.innerHTML = errorMessage
? `❌ ${errorMessage}`
: 'Validation failed.';
summaryDiv.style.display = summaryDiv.innerHTML ? 'block' : 'none';
detailsDiv.innerHTML = '';
resultsDiv.style.display = summaryDiv.innerHTML ? 'block' : 'none';
if (repairBtn) {
repairBtn.style.display = 'inline-block';
repairBtn.disabled = true;
}
}
}
} catch (error) {
// Stop timer on error
clearInterval(timerInterval);
const endTime = new Date();
const totalTime = (endTime - startTime) / 1000;
// Check if it was cancelled
if (error.name === 'AbortError') {
// Try to get progress info from the server response if available
let progressInfo = '';
let violationCount = 0;
try {
if (currentValidationJobId) {
const progressResponse = await fetch(
`${getPluginPath()}/api/validate-schemas/progress/${currentValidationJobId}`,
{
cache: 'no-store',
}
);
if (progressResponse.ok) {
const cancelData = await progressResponse.json();
if (
cancelData.result &&
Array.isArray(cancelData.result.violationFiles)
) {
violationCount = cancelData.result.violationFiles.length;
}
if (
cancelData.processed !== undefined &&
cancelData.total !== undefined
) {
const percentage =
cancelData.total > 0
? Math.round((cancelData.processed / cancelData.total) * 100)
: 0;
progressInfo = `<br><strong>Progress: ${cancelData.processed.toLocaleString()}/${cancelData.total.toLocaleString()} files (${percentage}%)</strong>`;
}
}
}
} catch (parseError) {
// Ignore parse errors, just show basic cancellation
}
timerElement.innerHTML = `
<strong>❌ Validation Cancelled</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Cancelled: ${endTime.toLocaleTimeString()}<br>
<strong>Time: ${totalTime.toFixed(1)}s</strong>${progressInfo}
`;
timerElement.style.background = '#fff3cd';
timerElement.style.borderColor = '#ffc107';
if (repairBtn) {
repairBtn.style.display = 'inline-block';
repairBtn.disabled = violationCount === 0;
}
} else {
timerElement.innerHTML = `
<strong>⏱️ Validation Failed</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Failed: ${endTime.toLocaleTimeString()}<br>
<strong>Total Time: ${totalTime.toFixed(1)}s</strong>
`;
timerElement.style.background = '#ffeaea';
timerElement.style.borderColor = '#f44336';
console.error('Validation error:', error);
}
if (repairBtn) {
repairBtn.style.display = 'inline-block';
repairBtn.disabled = true;
}
} finally {
currentValidationJobId = null;
validationCancelRequested = false;
currentValidationController = null;
button.onclick = runDataValidation;
button.disabled = false;
button.textContent = '🔍 Run Schema Validation';
button.style.background = '';
}
}
export async function repairSchemas() {
const statusDiv = document.getElementById('validationStatus');
const repairBtn = document.getElementById('repairBtn');
// If a job is already running, treat the click as a cancel request
if (currentRepairJobId || currentRepairController) {
cancelRepair();
return;
}
repairCancelRequested = false;
currentRepairJobId = null;
repairBtn.textContent = '⏸️ Repairing Schemas (click to cancel)';
repairBtn.style.background = '#ffc107';
repairBtn.onclick = cancelRepair;
repairBtn.disabled = false;
const startTime = new Date();
const timerElement = document.createElement('div');
timerElement.id = 'repairTimer';
timerElement.style.cssText =
'background: #e3f2fd; border: 1px solid #2196f3; border-radius: 5px; padding: 10px; margin-bottom: 15px; text-align: center; font-family: monospace;';
timerElement.innerHTML = `
<strong>🔧 Repair Timer</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Elapsed: 0.0s<br>
<span style="font-size: 0.9em; color: #666;">Preparing repair job...</span>
`;
statusDiv.innerHTML = '';
statusDiv.appendChild(timerElement);
const timerInterval = setInterval(() => {
const elapsed = (new Date() - startTime) / 1000;
timerElement.innerHTML = `
<strong>🔧 Repair Timer</strong><br>
Started: ${startTime.toLocaleTimeString()}<br>
Elapsed: ${elapsed.toFixed(1)}s<br>
<span style="font-size: 0.9em; color: #666;">Preparing repair job...</span>
`;
}, 100);
let finalProgress = null;
try {
// Set flag to track running repair
currentRepairController = true;
const response = await fetch(`${getPluginPath()}/api/repair-schemas`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
});
const result = await response.json();
if (!response.ok) {
throw new Error(result.message || 'Repair failed');
}
if (!result.jobId) {
timerElement.innerHTML =
'<strong>❌ Repair response missing job ID</strong>';
timerElement.style.background = '#ffeaea';
timerElement.style.borderColor = '#f44336';
return;
}
currentRepairJobId = result.jobId;
currentRepairController = new AbortController();
clearInterval(timerInterval);
finalProgress = await pollRepairProgress(
result.jobId,
timerElement,
startTime,
currentRepairController.signal
);
if (finalProgress && finalProgress.result) {
const {
repairedFiles,
backedUpFiles,
skippedFiles,
quarantinedFiles,
errors,
message,
} = finalProgress.result;
const summaryLines = [
`<strong>${message || 'Repair summary'}</strong>`,
`✅ Repaired: ${repairedFiles.toLocaleString()}`,
`📦 Backups created: ${backedUpFiles.toLocaleString()}`,
`⏭️ Skipped (already clean): ${skippedFiles.length.toLocaleString()}`,
`🚫 Quarantined: ${quarantinedFiles.length.toLocaleString()}`,
`⚠️ Errors: ${errors.length.toLocaleString()}`,
];
timerElement.innerHTML = summaryLines.join('<br>');
if (errors.length > 0) {
const errorList = document.createElement('div');
errorList.style.cssText =
'margin-top: 10px; background: #fff3f3; border: 1px solid #f44336; padding: 10px; font-size: 12px; max-height: 150px; overflow-y: auto;';
errorList.innerHTML = errors.map(err => `❌ ${err}`).join('<br>');
timerElement.appendChild(errorList);
}
if (finalProgress.status === 'completed') {
timerElement.style.background = '#e8f5e8';
timerElement.style.borderColor = '#4caf50';
} else if (finalProgress.status === 'cancelled') {
timerElement.style.background = '#fff3cd';
timerElement.style.borderColor = '#ffc107';
} else if (finalProgress.status === 'error') {
timerElement.style.background = '#ffeaea';
timerElement.style.borderColor = '#f44336';
}
}
} catch (error) {
clearInterval(timerInterval);
timerElement.innerHTML = `<strong>❌ Repair Error</strong><br>${error.message}`;
timerElement.style.background = '#ffeaea';
timerElement.style.borderColor = '#f44336';
console.error('Repair error:', error);
} finally {
clearInterval(timerInterval);
currentRepairJobId = null;
repairCancelRequested = false;
if (currentRepairController) {
currentRepairController.abort();
currentRepairController = null;
}
repairBtn.textContent = '🔧 Repair Schema Violations';
repairBtn.style.background = '';
repairBtn.onclick = repairSchemas;
repairBtn.disabled = false;
}
}