dwh-audit
Version:
Modular CLI tool for auditing data warehouses - extract, analyze, and report on schemas, data quality, and analytics readiness
1,039 lines (978 loc) • 146 kB
JavaScript
// HTML template tag for syntax highlighting
const html = (strings, ...values) => {
return strings.reduce((result, string, i) => {
return result + string + (values[i] || '');
}, '');
};
function generateHtmlReport(data) {
return html`
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Data Warehouse Audit Report</title>
<link rel="icon" type="image/x-icon" href="favicon.ico">
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/d3@7"></script>
<style>
:root {
/* Mixpanel-inspired Dark Mode Colors */
--bg-primary: #0D1116;
--bg-secondary: #161B22;
--bg-tertiary: #21262D;
--bg-card: #161B22;
--bg-elevated: #1C2128;
--text-primary: #F0F6FC;
--text-secondary: #8B949E;
--text-muted: #6E7681;
--text-accent: #FFFFFF;
--border-primary: #30363D;
--border-secondary: #21262D;
--border-focus: #8B5CF6;
/* Mixpanel Core Purple */
--accent-purple: #8B5CF6;
--accent-purple-hover: #7C3AED;
--accent-purple-light: #A78BFA;
--accent-purple-dark: #6D28D9;
/* Mixpanel Brand Colors */
--mixpanel-purple: #7856FF;
--mixpanel-purple-light: #9B7EFF;
--mixpanel-purple-dark: #5B3FD6;
/* Status Colors */
--success: #238636;
--success-light: #2DA44E;
--warning: #D1742F;
--warning-light: #F85149;
--error: #DA3633;
--error-light: #FF6B6B;
/* Chart Colors - Mixpanel Inspired */
--chart-color-1: #8B5CF6;
--chart-color-2: #06B6D4;
--chart-color-3: #10B981;
--chart-color-4: #F59E0B;
--chart-color-5: #EF4444;
--chart-color-6: #8B5A2B;
--chart-color-7: #EC4899;
--chart-color-8: #84CC16;
/* Effects */
--shadow: 0 8px 24px rgba(0, 0, 0, 0.4);
--shadow-sm: 0 2px 8px rgba(0, 0, 0, 0.3);
--shadow-card: 0 1px 3px rgba(0, 0, 0, 0.6);
--glow-purple: 0 0 0 3px rgba(139, 92, 246, 0.1);
--font-mono: 'SF Mono', 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
--font-sans: "Apercu Pro", "Helvetica Neue", Helvetica, Tahoma, Geneva, Arial, sans-serif;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: var(--font-sans);
background: var(--bg-primary);
color: var(--text-primary);
line-height: 1.5;
min-height: 100vh;
}
/* Header Styles */
.container { max-width: 1440px; margin: 0 auto; padding: 32px 24px; }
header {
text-align: center;
margin-bottom: 48px;
padding: 40px 0;
background: linear-gradient(135deg, var(--bg-secondary) 0%, var(--bg-tertiary) 100%);
border-radius: 16px;
border: 1px solid var(--border-primary);
box-shadow: var(--shadow-card);
}
h1 {
font-size: 3rem;
font-weight: 700;
background: linear-gradient(135deg, var(--mixpanel-purple), var(--accent-purple-light));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
background-clip: text;
margin-bottom: 12px;
letter-spacing: -0.02em;
}
.subtitle {
color: var(--text-secondary);
font-size: 1.125rem;
font-weight: 400;
max-width: 600px;
margin: 0 auto;
}
.subtitle code {
background: var(--bg-tertiary);
padding: 4px 8px;
border-radius: 6px;
font-weight: 500;
color: var(--accent-purple-light);
font-family: var(--font-mono);
font-size: 0.9em;
border: 1px solid var(--border-secondary);
}
/* Summary Cards */
.summary-cards {
display: grid;
grid-template-columns: repeat(4, 1fr);
gap: 24px;
margin-bottom: 48px;
}
@media (max-width: 1200px) {
.summary-cards {
grid-template-columns: repeat(2, 1fr);
}
}
@media (max-width: 600px) {
.summary-cards {
grid-template-columns: 1fr;
}
}
.card {
background: var(--bg-card);
border: 1px solid var(--border-primary);
border-radius: 12px;
padding: 32px 24px;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
position: relative;
overflow: hidden;
}
.card::before {
content: '';
position: absolute;
top: 0;
left: 0;
right: 0;
height: 3px;
background: linear-gradient(90deg, var(--mixpanel-purple), var(--accent-purple-light));
opacity: 0;
transition: opacity 0.2s ease;
}
.card:hover {
transform: translateY(-2px);
box-shadow: var(--shadow);
border-color: var(--border-focus);
}
.card:hover::before { opacity: 1; }
.card h3 {
color: var(--text-secondary);
font-size: 0.875rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
margin-bottom: 16px;
}
.card .number {
font-size: 2.5rem;
font-weight: 700;
color: var(--text-primary);
line-height: 1;
margin-bottom: 8px;
}
.card.error .number { color: var(--error-light); }
.card.error::before { background: linear-gradient(90deg, var(--error), var(--error-light)); }
/* Search and Controls */
.search-box {
margin-bottom: 32px;
display: flex;
gap: 16px;
align-items: center;
flex-wrap: wrap;
}
.search-box input {
flex: 1;
min-width: 320px;
padding: 16px 20px;
font-size: 1rem;
background: var(--bg-card);
border: 1px solid var(--border-primary);
border-radius: 8px;
color: var(--text-primary);
transition: all 0.2s ease;
font-family: var(--font-sans);
}
.search-box input::placeholder {
color: var(--text-muted);
}
.search-box input:focus {
outline: none;
border-color: var(--border-focus);
box-shadow: var(--glow-purple);
background: var(--bg-elevated);
}
.expand-collapse-btn {
padding: 14px 24px;
background: var(--mixpanel-purple);
color: var(--text-accent);
border: none;
border-radius: 8px;
cursor: pointer;
font-weight: 600;
font-family: var(--font-sans);
transition: all 0.2s ease;
white-space: nowrap;
}
.expand-collapse-btn:hover {
background: var(--mixpanel-purple-dark);
transform: translateY(-1px);
}
.expand-collapse-btn:active {
transform: translateY(0);
}
/* Table Items */
.table-item {
background: var(--bg-card);
border: 1px solid var(--border-primary);
border-radius: 12px;
margin-bottom: 20px;
overflow: hidden;
transition: all 0.2s ease;
box-shadow: var(--shadow-card);
}
.table-item:hover {
border-color: var(--border-focus);
box-shadow: var(--shadow-sm);
}
.table-header {
padding: 24px;
cursor: pointer;
display: flex;
justify-content: space-between;
align-items: center;
border-bottom: 1px solid transparent;
transition: all 0.2s ease;
}
.table-header:hover {
background: var(--bg-elevated);
}
.table-item.expanded .table-header {
border-bottom-color: var(--border-secondary);
}
.table-name {
font-size: 1.25rem;
font-weight: 600;
color: var(--mixpanel-purple-light);
display: flex;
align-items: center;
font-family: var(--font-mono);
}
.expand-icon {
transition: transform 0.2s ease;
margin-right: 16px;
color: var(--text-secondary);
font-size: 0.875rem;
}
.table-item.expanded .expand-icon { transform: rotate(90deg); }
/* Badges */
.badges { display: flex; gap: 12px; flex-wrap: wrap; }
.badge {
padding: 6px 12px;
border-radius: 6px;
font-size: 0.75rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.025em;
border: 1px solid transparent;
}
.badge.table {
background: rgba(139, 92, 246, 0.15);
color: var(--accent-purple-light);
border-color: rgba(139, 92, 246, 0.25);
}
.badge.view {
background: rgba(16, 185, 129, 0.15);
color: var(--chart-color-3);
border-color: rgba(16, 185, 129, 0.25);
}
.badge.error {
background: rgba(239, 68, 68, 0.15);
color: var(--error-light);
border-color: rgba(239, 68, 68, 0.25);
}
/* Table Details */
.details {
display: none;
padding: 24px;
background: var(--bg-tertiary);
border-top: 1px solid var(--border-secondary);
}
.table-item.expanded .details { display: block; }
.details-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 24px;
margin-bottom: 24px;
}
.detail-item strong {
display: block;
color: var(--text-secondary);
font-size: 0.75rem;
margin-bottom: 6px;
text-transform: uppercase;
font-weight: 600;
letter-spacing: 0.05em;
}
.detail-item span {
font-weight: 500;
color: var(--text-primary);
font-size: 0.9rem;
}
h4 {
color: var(--accent-purple-light);
font-weight: 600;
margin: 24px 0 16px;
font-size: 1.1rem;
}
/* Tables and Code */
pre, table {
background: var(--bg-primary);
border: 1px solid var(--border-primary);
border-radius: 8px;
padding: 16px;
font-size: 0.875rem;
color: var(--text-secondary);
max-height: 400px;
overflow: auto;
font-family: var(--font-mono);
}
table {
width: 100%;
border-collapse: collapse;
padding: 0;
}
th, td {
padding: 12px 16px;
text-align: left;
border-bottom: 1px solid var(--border-secondary);
font-size: 0.875rem;
}
th {
color: var(--text-primary);
font-weight: 600;
background: var(--bg-secondary);
position: sticky;
top: 0;
z-index: 1;
}
tr:hover {
background: var(--bg-elevated);
}
tr:hover td {
color: var(--text-primary);
}
/* Special States */
.error-message {
background: rgba(239, 68, 68, 0.1);
border: 1px solid var(--error-light);
color: var(--error-light);
padding: 12px;
border-radius: 8px;
margin-top: 16px;
font-family: var(--font-mono);
font-size: 0.875rem;
}
.join-key-row {
background: rgba(139, 92, 246, 0.08);
border-left: 3px solid var(--accent-purple);
}
.join-key-cell {
color: var(--accent-purple-light);
font-weight: 600;
}
.complex-field-row {
background: rgba(245, 158, 11, 0.08);
border-left: 3px solid var(--chart-color-4);
}
.complex-field-cell {
color: var(--chart-color-4);
font-weight: 600;
}
.pii-field-row {
background: rgba(239, 68, 68, 0.12);
border-left: 3px solid var(--error-light);
}
.pii-field-cell {
color: var(--error-light);
font-weight: 600;
}
.nested-field-indicator {
display: inline-block;
margin-left: 8px;
padding: 3px 8px;
background: rgba(245, 158, 11, 0.15);
color: var(--chart-color-4);
font-size: 0.7rem;
border-radius: 4px;
font-weight: 600;
border: 1px solid rgba(245, 158, 11, 0.25);
}
/* Sample Data */
.sample-json {
background: var(--bg-primary);
border: 1px solid var(--border-primary);
border-radius: 8px;
padding: 16px;
font-family: var(--font-mono);
font-size: 0.875rem;
color: var(--text-secondary);
max-height: 400px;
overflow: auto;
white-space: pre-wrap;
}
.sample-section { margin-top: 24px; }
.sample-toggle {
color: var(--accent-purple-light);
font-weight: 600;
margin: 24px 0 12px;
transition: color 0.2s ease;
cursor: pointer;
}
.sample-toggle:hover { color: var(--text-primary); }
.sample-expand-icon { color: var(--accent-purple-light); }
/* Charts */
.analytics-section { margin-bottom: 48px; }
.charts-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
gap: 24px;
margin-bottom: 32px;
}
.chart-container.full-width {
grid-column: 1 / -1;
}
.chart-container {
background: var(--bg-card);
border: 1px solid var(--border-primary);
border-radius: 12px;
padding: 24px;
position: relative;
max-height: 600px;
overflow-y: auto;
overflow-x: hidden;
box-shadow: var(--shadow-card);
transition: all 0.2s ease;
}
.chart-container:hover {
border-color: var(--border-focus);
box-shadow: var(--shadow-sm);
}
.chart-container h3 {
color: var(--text-primary);
font-size: 1.125rem;
font-weight: 600;
margin-bottom: 20px;
padding-bottom: 12px;
border-bottom: 1px solid var(--border-secondary);
}
.d3-chart {
width: 100%;
min-height: 300px;
}
.d3-bar {
transition: opacity 0.2s ease;
}
.d3-bar:hover {
opacity: 0.8;
}
.d3-axis-text {
fill: var(--text-secondary);
font-family: 'Inter', sans-serif;
font-size: 11px;
font-weight: 400;
}
.d3-axis-label {
fill: var(--text-secondary);
font-family: 'Inter', sans-serif;
font-size: 12px;
font-weight: 500;
}
.d3-grid-line {
stroke: rgba(48, 54, 61, 0.3);
stroke-width: 1;
}
.d3-tooltip {
position: absolute;
background: rgba(33, 38, 45, 0.95);
color: var(--text-primary);
border: 1px solid var(--border-primary);
border-radius: 8px;
padding: 8px 12px;
font-size: 12px;
font-family: 'Inter', sans-serif;
pointer-events: none;
z-index: 1000;
opacity: 0;
transition: opacity 0.2s ease;
}
.d3-pie-slice {
transition: transform 0.2s ease;
}
.d3-pie-slice:hover {
transform: scale(1.05);
}
.d3-legend {
font-family: 'Inter', sans-serif;
font-size: 12px;
font-weight: 500;
fill: var(--text-secondary);
}
.chart-title {
color: var(--text-primary);
font-size: 1.1rem;
font-weight: 600;
margin-bottom: 15px;
text-align: center;
}
.lineage-section { margin-bottom: 40px; }
.lineage-container {
background: var(--bg-card);
border: 1px solid var(--border-primary);
border-radius: 12px;
padding: 20px;
min-height: 500px;
position: relative;
}
.lineage-controls {
margin-bottom: 15px;
display: flex;
gap: 10px;
align-items: center;
}
.lineage-controls button {
padding: 8px 16px;
background: var(--accent-purple);
color: white;
border: none;
border-radius: 6px;
cursor: pointer;
font-size: 0.85rem;
transition: background 0.2s ease;
}
.lineage-controls button:hover {
background: rgba(139, 92, 246, 0.8);
}
.lineage-svg {
width: 100%;
height: 100%;
border-radius: 8px;
}
.node {
cursor: pointer;
transition: all 0.3s ease;
}
.node:hover {
stroke-width: 3px;
}
.node-table {
fill: var(--chart-color-1);
stroke: var(--accent-purple);
}
.node-view {
fill: var(--chart-color-3);
stroke: var(--success);
}
.node-text {
fill: var(--text-primary);
font-size: 12px;
font-weight: 600;
text-anchor: middle;
pointer-events: none;
}
.link {
fill: none;
}
.link-view-dependency {
stroke: rgba(245, 158, 11, 0.8);
stroke-width: 3;
marker-end: url(#arrowhead-view);
}
.link-join-key {
stroke: rgba(16, 185, 129, 0.6);
stroke-width: 2;
stroke-dasharray: 5,5;
}
.link:hover {
stroke-width: 4;
}
.edge-label {
fill: var(--text-secondary);
font-size: 10px;
text-anchor: middle;
pointer-events: none;
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>BigQuery Dataset Audit</h1>
<p class="subtitle">Project: <code id="headerProject"></code> • Dataset: <code id="headerDataset"></code> • <span id="headerErrors" style="color: var(--error-light);"></span></p>
</header>
<section class="summary-cards">
<div class="card"><h3>Total Objects</h3><div class="number" id="summaryTotalObjects">0</div></div>
<div class="card">
<h3>Tables & Views</h3>
<div style="display: flex; align-items: baseline; gap: 16px; justify-content: center;">
<div style="text-align: center;">
<div class="number" id="summaryTables">0</div>
<div style="font-size: 0.75rem; color: var(--text-muted); margin-top: 4px;">Tables</div>
</div>
<div style="color: var(--text-muted); font-size: 1.5rem;">+</div>
<div style="text-align: center;">
<div class="number" id="summaryViews">0</div>
<div style="font-size: 0.75rem; color: var(--text-muted); margin-top: 4px;">Views</div>
</div>
</div>
</div>
<div class="card"><h3>Total Rows</h3><div class="number" id="summaryTotalRows">0</div></div>
<div class="card"><h3>Total Size</h3><div class="number" id="summaryTotalSize">0 B</div></div>
</section>
<!-- Analytics Insights Section -->
<section class="analytics-insights-section">
<h2 style="color: var(--accent-purple-light); font-weight: 600; margin-bottom: 20px; font-size: 1.8rem;">🎯 Analytics & Data Quality Insights</h2>
<!-- Mixpanel Compatibility Cards -->
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 15px; margin-bottom: 25px;">
<div class="card analytics-card" data-category="mixpanel_ready" style="background: linear-gradient(135deg, rgba(139, 92, 246, 0.1), rgba(139, 92, 246, 0.05)); border-color: rgba(139, 92, 246, 0.3); cursor: pointer; transition: all 0.2s ease;">
<h3 style="color: var(--accent-purple);">🚀 Mixpanel Ready</h3>
<div class="number" style="color: var(--accent-purple);" id="mixpanelReadyCount">0</div>
<div style="font-size: 0.8rem; color: var(--text-secondary); margin-top: 5px;">Tables with timestamp + user ID</div>
<div id="mixpanelReadyList" class="table-list" style="display: none; margin-top: 10px; font-size: 0.8rem;"></div>
</div>
<div class="card analytics-card" data-category="event_tables" style="background: linear-gradient(135deg, rgba(16, 185, 129, 0.1), rgba(16, 185, 129, 0.05)); border-color: rgba(16, 185, 129, 0.3); cursor: pointer; transition: all 0.2s ease;">
<h3 style="color: var(--chart-color-3);">📅 Event Tables</h3>
<div class="number" style="color: var(--chart-color-3);" id="eventTablesCount">0</div>
<div style="font-size: 0.8rem; color: var(--text-secondary); margin-top: 5px;">Tables with timestamp fields</div>
<div id="eventTablesList" class="table-list" style="display: none; margin-top: 10px; font-size: 0.8rem;"></div>
</div>
<div class="card analytics-card" data-category="user_tables" style="background: linear-gradient(135deg, rgba(245, 158, 11, 0.1), rgba(245, 158, 11, 0.05)); border-color: rgba(245, 158, 11, 0.3); cursor: pointer; transition: all 0.2s ease;">
<h3 style="color: var(--chart-color-4);">👤 User Tables</h3>
<div class="number" style="color: var(--chart-color-4);" id="userTablesCount">0</div>
<div style="font-size: 0.8rem; color: var(--text-secondary); margin-top: 5px;">Tables with user identifiers</div>
<div id="userTablesList" class="table-list" style="display: none; margin-top: 10px; font-size: 0.8rem;"></div>
</div>
<div class="card analytics-card" data-category="pii_detected" style="background: linear-gradient(135deg, rgba(239, 68, 68, 0.1), rgba(239, 68, 68, 0.05)); border-color: rgba(239, 68, 68, 0.3); cursor: pointer; transition: all 0.2s ease;">
<h3 style="color: var(--error-light);">⚠️ PII Detected</h3>
<div class="number" style="color: var(--error-light);" id="piiTablesCount">0</div>
<div style="font-size: 0.8rem; color: var(--text-secondary); margin-top: 5px;">Tables with potential PII</div>
<div id="piiTablesList" class="table-list" style="display: none; margin-top: 10px; font-size: 0.8rem;"></div>
</div>
</div>
<!-- Field Pattern Analysis -->
<div style="display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 20px; margin-bottom: 25px;">
<div style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 12px; padding: 20px;">
<h3 style="color: var(--accent-purple-light); font-size: 1.1rem; margin-bottom: 15px;">🏷️ Common Field Patterns</h3>
<div id="fieldPatternsAnalysis" style="display: grid; gap: 10px;"></div>
</div>
<div style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 12px; padding: 20px;">
<h3 style="color: var(--accent-purple-light); font-size: 1.1rem; margin-bottom: 15px;">📊 Data Quality Overview</h3>
<div id="dataQualityOverview" style="display: grid; gap: 8px;"></div>
</div>
<div style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 12px; padding: 20px;">
<h3 style="color: var(--error-light); font-size: 1.1rem; margin-bottom: 15px;">🏗️ Schema Complexity</h3>
<div id="schemaComplexityOverview" style="display: grid; gap: 8px;"></div>
</div>
</div>
</section>
<section class="analytics-section">
<h2 style="color: var(--accent-purple-light); font-weight: 600; margin-bottom: 20px; font-size: 1.8rem;">📊 Table Analytics</h2>
<div class="charts-grid">
<div class="chart-container full-width">
<div class="chart-title">Row Count Distribution (Top 20)</div>
<svg id="rowCountChart" class="d3-chart"></svg>
</div>
<div class="chart-container">
<div class="chart-title">Table Types</div>
<svg id="tableTypeChart" class="d3-chart"></svg>
</div>
<div class="chart-container">
<div class="chart-title">Partitioned vs Non-Partitioned</div>
<svg id="partitionChart" class="d3-chart"></svg>
</div>
<div class="chart-container">
<div class="chart-title">Data Freshness Distribution</div>
<svg id="freshnessChart" class="d3-chart"></svg>
</div>
<div class="chart-container">
<div class="chart-title">Analytics Readiness Score</div>
<svg id="analyticsScoreChart" class="d3-chart"></svg>
</div>
</div>
</section>
<section class="lineage-section">
<h2 style="color: var(--accent-purple-light); font-weight: 600; margin-bottom: 20px; font-size: 1.8rem;">🔗 Table Relationships & Join Key Analysis</h2>
<!-- Relationship Summary Cards -->
<div class="relationship-summary" style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 15px; margin-bottom: 25px;">
<div class="relationship-card" style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 8px; padding: 15px; text-align: center;">
<div style="font-size: 1.5rem; font-weight: 700; color: var(--accent-purple);" id="totalJoinKeys">0</div>
<div style="font-size: 0.8rem; color: var(--text-secondary); text-transform: uppercase;">Potential Join Keys</div>
</div>
<div class="relationship-card" style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 8px; padding: 15px; text-align: center;">
<div style="font-size: 1.5rem; font-weight: 700; color: var(--chart-color-3);" id="connectedTables">0</div>
<div style="font-size: 0.8rem; color: var(--text-secondary); text-transform: uppercase;">Connected Tables</div>
</div>
<div class="relationship-card" style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 8px; padding: 15px; text-align: center;">
<div style="font-size: 1.5rem; font-weight: 700; color: var(--chart-color-4);" id="viewDependencies">0</div>
<div style="font-size: 0.8rem; color: var(--text-secondary); text-transform: uppercase;">View Dependencies</div>
</div>
</div>
<!-- ERD Table Selection -->
<div style="background: var(--bg-elevated); border: 1px solid var(--border-primary); border-radius: 8px; padding: 15px; margin-bottom: 20px;">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 12px;">
<h4 style="color: var(--accent-purple-light); font-size: 0.95rem; margin: 0;">📊 Select Tables for Diagram</h4>
<div style="display: flex; gap: 8px;">
<button id="selectAllTables" style="padding: 4px 8px; background: var(--chart-color-3); color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 0.8rem;">Select All</button>
<button id="clearAllTables" style="padding: 4px 8px; background: var(--text-secondary); color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 0.8rem;">Clear All</button>
</div>
</div>
<input type="text" id="tableSearchInput" placeholder="Search tables..." style="width: 100%; padding: 8px 12px; margin-bottom: 12px; font-size: 0.85rem; background: var(--bg-primary); border: 1px solid var(--border-primary); border-radius: 4px; color: var(--text-primary);">
<div id="tableCheckboxes" style="max-height: 150px; overflow-y: auto; display: grid; grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); gap: 8px; padding: 8px; background: var(--bg-primary); border-radius: 6px;"></div>
</div>
<!-- Interactive Network Diagram -->
<div style="display: grid; grid-template-columns: 2fr 1fr; gap: 20px; margin-bottom: 20px;">
<div style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 12px; padding: 20px; position: relative;">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 15px;">
<h3 style="color: var(--accent-purple-light); font-size: 1.1rem; margin: 0;">🌐 Interactive Relationship Diagram</h3>
<div style="display: flex; gap: 8px; align-items: center;">
<button id="resetDiagram" style="padding: 4px 8px; background: var(--accent-purple); color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 0.8rem;">Reset</button>
<label style="display: flex; align-items: center; font-size: 0.8rem; color: var(--text-secondary);"><input type="checkbox" id="showLabels" checked style="margin-right: 4px;"> Labels</label>
</div>
</div>
<div style="height: 500px; border-radius: 8px; border: 1px solid var(--border-primary); position: relative; overflow: hidden;">
<svg id="networkDiagram" style="width: 100%; height: 100%;"></svg>
<div id="diagramTooltip" style="position: absolute; background: rgba(33, 38, 45, 0.95); color: var(--text-primary); border: 1px solid var(--border-primary); border-radius: 6px; padding: 8px 12px; font-size: 0.85rem; pointer-events: none; opacity: 0; transition: opacity 0.2s; z-index: 1000;"></div>
</div>
<div style="margin-top: 10px; font-size: 0.8rem; color: var(--text-secondary); display: flex; gap: 20px;">
<span><span style="display:inline-block;width:12px;height:12px;background:var(--accent-purple);border-radius:50%;margin-right:4px;"></span>Tables</span>
<span><span style="display:inline-block;width:12px;height:12px;background:var(--chart-color-3);border-radius:50%;margin-right:4px;"></span>Views</span>
<span><span style="display:inline-block;width:15px;height:2px;background:var(--chart-color-3);margin-right:4px;"></span>Join Keys</span>
<span><span style="display:inline-block;width:15px;height:2px;background:var(--chart-color-4);margin-right:4px;"></span>Dependencies</span>
</div>
</div>
<!-- Analysis Panel -->
<div style="background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 12px; padding: 20px;">
<h3 style="color: var(--accent-purple-light); font-size: 1.1rem; margin-bottom: 15px;">🔍 Analysis</h3>
<div id="selectedNodeInfo" style="margin-bottom: 20px; padding: 12px; background: var(--bg-primary); border-radius: 6px; min-height: 60px; display: flex; align-items: center; justify-content: center; color: var(--text-secondary); font-style: italic; font-size: 0.9rem;">Click a table or view to see details</div>
<h4 style="color: var(--chart-color-3); font-size: 0.95rem; margin-bottom: 10px;">🔑 Join Keys</h4>
<div id="joinKeysAnalysis" style="max-height: 200px; overflow-y: auto;"></div>
<h4 style="color: var(--chart-color-4); font-size: 0.95rem; margin: 15px 0 10px 0;">👁️ Dependencies</h4>
<div id="viewDependenciesAnalysis" style="max-height: 150px; overflow-y: auto;"></div>
</div>
</div>
</section>
<main>
<div style="margin-bottom: 25px;">
<h2 style="color: var(--accent-purple-light); font-weight: 600; margin-bottom: 15px; font-size: 1.8rem;">🔍 Table Explorer</h2>
<!-- Enhanced Search and Filters -->
<div style="display: grid; grid-template-columns: 2fr 1fr auto; gap: 15px; margin-bottom: 15px;">
<input type="text" id="searchInput" placeholder="Search by table name, field name, or data type..." style="padding: 16px; font-size: 1rem; background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 8px; color: var(--text-primary);">
<select id="analyticsFilter" style="padding: 16px; font-size: 1rem; background: var(--bg-card); border: 1px solid var(--border-primary); border-radius: 8px; color: var(--text-primary);">
<option value="">Show All Tables</option>
<option value="mixpanel_ready">🚀 Mixpanel Ready</option>
<option value="event_tables">📅 Event Tables</option>
<option value="user_tables">👤 User Tables</option>
<option value="has_pii">⚠️ Contains PII</option>
<option value="high_quality">✅ High Quality</option>
<option value="partitioned">🗂️ Partitioned</option>
<option value="fresh_data">🟢 Fresh Data (≤7 days)</option>
<option value="stale_data">🟠 Stale Data (>30 days)</option>
<option value="complex_schema">🏗️ Complex Schema</option>
<option value="simple_schema">✅ Simple Schema</option>
</select>
<button class="expand-collapse-btn" id="expandCollapseBtn">Expand All</button>
</div>
<!-- Quick Stats for Filtered Results -->
<div id="filterStats" style="display: none; background: var(--bg-elevated); border: 1px solid var(--border-primary); border-radius: 8px; padding: 12px; margin-bottom: 15px; font-size: 0.9rem; color: var(--text-secondary);"></div>
</div>
<div id="tablesContainer"></div>
</main>
</div>
<script id="auditData" type="application/json">
${JSON.stringify(data, null, 2)}
</script>
<script>
document.addEventListener('DOMContentLoaded', () => {
const data = JSON.parse(document.getElementById('auditData').textContent);
const tablesContainer = document.getElementById('tablesContainer');
const searchInput = document.getElementById('searchInput');
// Populate header with project/dataset info
const projectElement = document.getElementById('headerProject');
const datasetElement = document.getElementById('headerDataset');
const errorsElement = document.getElementById('headerErrors');
if (data.audit_metadata) {
projectElement.textContent = data.audit_metadata.project_id || 'Unknown';
datasetElement.textContent = data.audit_metadata.dataset_id || 'Unknown';
} else if (data.extraction_metadata) {
projectElement.textContent = data.extraction_metadata.project_id || 'Unknown';
datasetElement.textContent = data.extraction_metadata.dataset_id || 'Unknown';
}
const errorCount = data.tables ? data.tables.filter(t => t.has_permission_error).length : 0;
errorsElement.textContent = errorCount > 0 ? errorCount + ' Errors' : 'No Errors';
// Utility function for human-readable bytes
const bytesHuman = function (bytes, dp = 2, si = true) {
//https://stackoverflow.com/a/14919494
const thresh = si ? 1000 : 1024;
if (Math.abs(bytes) < thresh) {
return bytes + ' B';
}
const units = si ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
let u = -1;
const r = 10 ** dp;
do {
bytes /= thresh;
++u;
} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
return bytes.toFixed(dp) + ' ' + units[u];
};
function renderSummary() {
document.getElementById('headerProject').textContent = data.audit_metadata.project_id;
document.getElementById('headerDataset').textContent = data.audit_metadata.dataset_id;
// Header errors already handled above
document.getElementById('summaryTotalObjects').textContent = data.summary.total_objects.toLocaleString();
document.getElementById('summaryTables').textContent = data.summary.total_tables.toLocaleString();
document.getElementById('summaryViews').textContent = data.summary.total_views.toLocaleString();
// Calculate total rows and size
let totalRows = 0;
let totalBytes = 0;
data.tables.forEach(table => {
if (table.row_count && typeof table.row_count === 'number') {
totalRows += table.row_count;
}
if (table.size_bytes && typeof table.size_bytes === 'number') {
totalBytes += table.size_bytes; // Already in bytes
}
});
document.getElementById('summaryTotalRows').textContent = totalRows.toLocaleString();
document.getElementById('summaryTotalSize').textContent = bytesHuman(totalBytes);
}
function renderAnalyticsInsights() {
if (!data.analytics) {
console.log('No analytics data found');
// Hide analytics section if no data
const analyticsSection = document.querySelector('.analytics-insights-section');
if (analyticsSection) {
analyticsSection.style.display = 'none';
}
return;
}
// Update analytics cards
const mixpanelReady = data.analytics.data_quality?.filter(t => t.mixpanel_compatibility >= 4) || [];
const eventTables = data.analytics.event_tables || [];
const userTables = data.analytics.user_tables || [];
const piiTables = data.analytics.data_quality?.filter(t => t.data_quality?.potential_pii?.length > 0) || [];
document.getElementById('mixpanelReadyCount').textContent = mixpanelReady.length;
document.getElementById('eventTablesCount').textContent = eventTables.length;
document.getElementById('userTablesCount').textContent = userTables.length;
document.getElementById('piiTablesCount').textContent = piiTables.length;
// Function to create table list
function createTableList(tables, containerId) {
const container = document.getElementById(containerId);
if (tables.length === 0) {
container.innerHTML = '<div style="color: var(--text-secondary); font-style: italic;">No tables in this category</div>';
return;
}
const tableLinks = tables.map((table, index) => {
const scoreSpan = table.mixpanel_compatibility ?
'<span style="float: right; font-size: 0.7rem; opacity: 0.8;">Score: ' + table.mixpanel_compatibility + '</span>' : '';
const linkId = 'tableLink_' + index;
return '<div id="' + linkId + '" style="margin: 2px 0; padding: 4px 8px; background: rgba(255,255,255,0.1); border-radius: 4px; cursor: pointer;" ' +
'data-table-name="' + table.table_name.replace(/"/g, '"') + '" ' +
'onmouseover="this.style.background="rgba(255,255,255,0.2)"" ' +
'onmouseout="this.style.background="rgba(255,255,255,0.1)"">' +
'📊 ' + table.table_name +
scoreSpan +
'</div>';
}).join('');
container.innerHTML = tableLinks;
// Add event delegation for table links
container.addEventListener('click', function(e) {
const clickedElement = e.target.closest('[data-table-name]');
if (clickedElement) {
const tableName = clickedElement.getAttribute('data-table-name');
scrollToTable(tableName);
}
});
}
// Populate table lists
createTableList(mixpanelReady, 'mixpanelReadyList');
createTableList(eventTables, 'eventTablesList');
createTableList(userTables, 'userTablesList');
createTableList(piiTables, 'piiTablesList');
// Add click handlers for analytics cards
document.querySelectorAll('.analytics-card').forEach(card => {
card.addEventListener('click', function() {
const category = this.dataset.category;
const tableList = this.querySelector('.table-list');
// Toggle visibility
if (tableList.style.display === 'none') {
// Hide all other lists first
document.querySelectorAll('.table-list').forEach(list => list.style.display = 'none');
tableList.style.display = 'block';
this.style.transform = 'scale(1.02)';
} else {
tableList.style.display = 'none';
this.style.transform = 'scale(1)';
}
});
// Add hover effects
card.addEventListener('mouseenter', function() {
if (this.querySelector('.table-list').style.display !== 'block') {
this.style.transform = 'scale(1.01)';
}
});
card.addEventListener('mouseleave', function() {
if (this.querySelector('.table-list').style.display !== 'block') {
this.style.transform = 'scale(1)';
}
});
});
// Function to scroll to a specific table
window.scrollToTable = function(tableName) {
const tableElement = document.getElementById('table-' + tableName);
if (tableElement) {
tableElement.scrollIntoView({ behavior: 'smooth', block: 'start' });
// Highlight the table briefly
tableElement.style.background = 'rgba(139, 92, 246, 0.1)';
setTimeout(() => {
tableElement.style.background = '';
}, 2000);
}
};
// Render field patterns
renderFieldPatterns();
renderDataQualityOverview();
renderSchemaComplexityOverview();
}
function renderFieldPatterns() {
const container = document.getElementById('fieldPatternsAnalysis');
if (!data.analytics || !data.analytics.field_patterns) {
container.innerHTML = '<div style="color: var(--text-secondary); font-style: italic; font-size: 0.9rem;">Analytics data not available - run a new audit to see field patterns</div>';
return;
}
const patterns = data.analytics.field_patterns;
let html = '';
if (patterns.timestamp_fields && patterns.timestamp_fields.length > 0) {
const truncated = patterns.timestamp_fields.slice(0, 3).join(', ') + (patterns.timestamp_fields.length > 3 ? ', ...' : '');
const full = patterns.timestamp_fields.join(', ');
html += \`<div style="background: var(--bg-primary); border-radius: 6px; padding: 10px; border-left: 3px solid var(--chart-color-3);">
<div style="font-weight: 600; color: var(--chart-color-3); font-size: 0.9rem; margin-bottom: 3px; cursor: pointer; display: flex; justify-content: space-between; align-items: center;" onclick="toggleFieldPattern('timestamp')">
<span>⏰ Timestamp Fields (\${patterns.timestamp_fields.length})</span>
<span id="timestamp-arrow" style="color: var(--chart-color-3); font-size: 0.8rem;">▶</span>
</div>
<div style="font-size: 0.8rem; color: var(--text-secondary);">
<div id="timestamp-truncated">\${truncated}</div>
<div id="timestamp-full" style="display: none;">\${full}</div>
</div>
</div>\`;
}
if (patterns.user_id_fields && patterns.user_id_fields.length > 0) {
const truncated = patterns.user_id_fields.slice(0, 3).join(', ') + (patterns.user_id_fields.length > 3 ? ', ...' : '');
const full = patterns.user_id_fields.join(', ');
html += \`<div style="background: var(--bg-primary); border-radius: 6px; padding: 10px; border-left: 3px solid var(--accent-purple);">
<div style="font-weight: 600; color: var(--accent-purple); font-size: 0.9rem; margin-bottom: 3px; cursor: pointer; display: flex; justify-content: space-between; align-items: center;" onclick="toggleFieldPattern('userid')">
<span>👤 User ID Fields (\${patterns.user_id_fields.length})</span>
<span id="userid-arrow" style="color: var(--accent-purple); font-size: 0.8rem;">▶</span>
</div>
<div style="font-size: 0.8rem; color: var(--text-secondary);">
<div id="userid-truncated">\${truncated}</div>
<div id="userid-full" style="display: none;">\${full}</div>
</div>
</div>\`;
}