nestjs-reverse-engineering
Version:
A powerful TypeScript/NestJS library for database reverse engineering, entity generation, and CRUD operations
484 lines (483 loc) • 19 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.DataExporter = void 0;
/* eslint-disable prettier/prettier */
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const database_types_1 = require("../types/database.types");
class DataExporter {
constructor(dataSource, options = {}) {
this.dataSource = dataSource;
this.options = {
dialect: database_types_1.DatabaseDialect.POSTGRES,
batchSize: 1000,
outputPath: './generated/sql/data',
prettyPrint: true,
alignValues: true,
nullHandling: 'NULL',
includeHeaders: true,
includeTableComments: true,
dataMasking: {
enabled: false,
sensitiveFields: ['password', 'email', 'phone', 'mobile', 'ssn', 'credit_card'],
maskingPatterns: {
email: { type: 'email', replacement: 'user{n}@example.com' },
password: { type: 'custom', replacement: 'MASKED_PASSWORD' },
phone: { type: 'phone', pattern: 'XXX-XXX-XXXX' },
name: { type: 'name', replacement: 'User {n}' }
}
},
...options
};
}
/**
* Export all table data as INSERT scripts
*/
async exportAllTables() {
console.log('🔍 Discovering tables...');
const tables = await this.discoverTables();
const filteredTables = this.filterTables(tables);
console.log(`📊 Found ${tables.length} tables, exporting ${filteredTables.length} tables`);
return await this.exportTables(filteredTables);
}
/**
* Export specific tables
*/
async exportTables(tableNames) {
const results = {
sql: '',
tableCount: 0,
totalRows: 0,
fileCount: 0,
outputPaths: [],
statistics: {}
};
// Ensure output directory exists
if (!fs.existsSync(this.options.outputPath)) {
fs.mkdirSync(this.options.outputPath, { recursive: true });
}
for (const tableName of tableNames) {
console.log(`📄 Exporting table: ${tableName}`);
try {
const tableData = await this.extractTableData(tableName);
const insertScripts = await this.generateInsertScripts(tableData);
// Write to file(s)
const filePaths = await this.writeInsertScripts(tableName, insertScripts, tableData.totalRows);
results.tableCount++;
results.totalRows += tableData.totalRows;
results.fileCount += filePaths.length;
results.outputPaths.push(...filePaths);
results.statistics[tableName] = {
rows: tableData.totalRows,
batches: insertScripts.length
};
console.log(` ✅ ${tableData.totalRows} rows exported in ${insertScripts.length} batches`);
}
catch (error) {
console.error(` ❌ Failed to export ${tableName}:`, error);
}
}
// Generate summary file
const summaryPath = await this.generateSummaryFile(results);
results.outputPaths.push(summaryPath);
results.fileCount++;
return results;
}
/**
* Discover all tables in the database
*/
async discoverTables() {
let query;
switch (this.options.dialect) {
case database_types_1.DatabaseDialect.POSTGRES:
query = `
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_type = 'BASE TABLE'
ORDER BY table_name
`;
break;
case database_types_1.DatabaseDialect.MYSQL:
query = `
SELECT table_name
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_type = 'BASE TABLE'
ORDER BY table_name
`;
break;
default:
throw new Error(`Unsupported dialect: ${this.options.dialect}`);
}
const result = await this.dataSource.query(query);
return result.map((row) => row.table_name);
}
/**
* Filter tables based on options
*/
filterTables(tables) {
let filtered = tables;
if (this.options.tables && this.options.tables.length > 0) {
filtered = filtered.filter(table => this.options.tables.includes(table));
}
if (this.options.excludeTables && this.options.excludeTables.length > 0) {
filtered = filtered.filter(table => !this.options.excludeTables.includes(table));
}
return filtered;
}
/**
* Extract data from a specific table
*/
async extractTableData(tableName) {
// Get column information
const columns = await this.getTableColumns(tableName);
// Build query
let query = `SELECT * FROM ${this.escapeIdentifier(tableName)}`;
// Add WHERE condition if specified
if (this.options.whereConditions && this.options.whereConditions[tableName]) {
query += ` WHERE ${this.options.whereConditions[tableName]}`;
}
// Add ORDER BY if specified
if (this.options.orderBy && this.options.orderBy[tableName]) {
query += ` ORDER BY ${this.options.orderBy[tableName]}`;
}
// Execute query
const rawRows = await this.dataSource.query(query);
// Convert rows to array format and apply data masking
const rows = rawRows.map((row) => columns.map(column => this.applyDataMasking(column, row[column], tableName)));
return {
tableName,
columns,
rows,
totalRows: rows.length
};
}
/**
* Get column names for a table
*/
async getTableColumns(tableName) {
let query;
switch (this.options.dialect) {
case database_types_1.DatabaseDialect.POSTGRES:
query = `
SELECT column_name
FROM information_schema.columns
WHERE table_name = $1
AND table_schema = 'public'
ORDER BY ordinal_position
`;
break;
case database_types_1.DatabaseDialect.MYSQL:
query = `
SELECT column_name
FROM information_schema.columns
WHERE table_name = ?
AND table_schema = DATABASE()
ORDER BY ordinal_position
`;
break;
default:
throw new Error(`Unsupported dialect: ${this.options.dialect}`);
}
const result = await this.dataSource.query(query, [tableName]);
return result.map((row) => row.column_name);
}
/**
* Generate INSERT scripts for table data
*/
async generateInsertScripts(tableData) {
const scripts = [];
const { tableName, columns, rows } = tableData;
// Process data in batches
for (let i = 0; i < rows.length; i += this.options.batchSize) {
const batchRows = rows.slice(i, i + this.options.batchSize);
const script = this.generateBatchInsert(tableName, columns, batchRows, i);
scripts.push(script);
}
return scripts;
}
/**
* Generate a single batch INSERT script
*/
generateBatchInsert(tableName, columns, rows, batchIndex) {
const lines = [];
// Add header comment
if (this.options.includeHeaders) {
lines.push(`-- Table: ${tableName}`);
lines.push(`-- Batch: ${Math.floor(batchIndex / this.options.batchSize) + 1}`);
lines.push(`-- Rows: ${rows.length}`);
lines.push('');
}
if (rows.length === 0) {
lines.push(`-- No data for table ${tableName}`);
return lines.join('\n');
}
// Build column list
const columnList = columns.map(col => this.escapeIdentifier(col)).join(', ');
// Start INSERT statement
lines.push(`INSERT INTO ${this.escapeIdentifier(tableName)} (${columnList})`);
lines.push('VALUES');
// Add data rows
const valueLines = [];
if (this.options.prettyPrint && this.options.alignValues) {
// Calculate column widths for alignment
const columnWidths = this.calculateColumnWidths(columns, rows);
rows.forEach((row, index) => {
const formattedValues = row.map((value, colIndex) => this.formatValue(value, columnWidths[colIndex]));
const comma = index < rows.length - 1 ? ',' : '';
valueLines.push(` (${formattedValues.join(', ')})${comma}`);
});
}
else {
// Simple formatting
rows.forEach((row, index) => {
const formattedValues = row.map(value => this.formatValue(value));
const comma = index < rows.length - 1 ? ',' : '';
valueLines.push(` (${formattedValues.join(', ')})${comma}`);
});
}
lines.push(...valueLines);
lines.push(';');
lines.push('');
return lines.join('\n');
}
/**
* Calculate column widths for alignment
*/
calculateColumnWidths(columns, rows) {
const widths = columns.map(col => col.length + 2); // Start with column name length
rows.forEach(row => {
row.forEach((value, index) => {
const formattedValue = this.formatValue(value);
widths[index] = Math.max(widths[index], formattedValue.length);
});
});
return widths;
}
/**
* Format a single value for SQL
*/
formatValue(value, width) {
let formatted;
if (value === null || value === undefined) {
switch (this.options.nullHandling) {
case 'NULL':
formatted = 'NULL';
break;
case 'DEFAULT':
formatted = 'DEFAULT';
break;
case 'SKIP':
formatted = '\'\''; // Empty string
break;
default:
formatted = 'NULL';
}
}
else if (typeof value === 'string') {
// Escape single quotes and wrap in quotes
formatted = `'${value.replace(/'/g, "''")}'`;
}
else if (typeof value === 'boolean') {
if (this.options.dialect === database_types_1.DatabaseDialect.POSTGRES) {
formatted = value ? 'true' : 'false';
}
else {
formatted = value ? '1' : '0';
}
}
else if (value instanceof Date) {
formatted = `'${value.toISOString()}'`;
}
else if (typeof value === 'object') {
// JSON objects
formatted = `'${JSON.stringify(value).replace(/'/g, "''")}'`;
}
else {
// Numbers and other types
formatted = String(value);
}
// Apply width padding if specified
if (width && this.options.alignValues) {
formatted = formatted.padEnd(width);
}
return formatted;
}
/**
* Apply data masking to sensitive fields
*/
applyDataMasking(columnName, value, tableName) {
if (!this.options.dataMasking.enabled || value === null || value === undefined) {
return value;
}
const lowerColumnName = columnName.toLowerCase();
const isSensitive = this.options.dataMasking.sensitiveFields.some(field => lowerColumnName.includes(field.toLowerCase()));
if (!isSensitive) {
return value;
}
// Apply masking based on patterns
for (const [pattern, rule] of Object.entries(this.options.dataMasking.maskingPatterns)) {
if (lowerColumnName.includes(pattern.toLowerCase())) {
return this.applyMaskingRule(value, rule, tableName);
}
}
// Default masking
return '***MASKED***';
}
/**
* Apply a specific masking rule
*/
applyMaskingRule(value, rule, tableName) {
const stringValue = String(value);
switch (rule.type) {
case 'email':
return rule.replacement?.replace('{n}', Math.floor(Math.random() * 1000).toString()) || 'user@example.com';
case 'phone':
return rule.pattern?.replace(/X/g, () => Math.floor(Math.random() * 10).toString()) || 'XXX-XXX-XXXX';
case 'name':
return rule.replacement?.replace('{n}', Math.floor(Math.random() * 1000).toString()) || 'Anonymous User';
case 'custom':
return rule.replacement || '***MASKED***';
default:
if (rule.preserveLength) {
return '*'.repeat(stringValue.length);
}
return '***MASKED***';
}
}
/**
* Write INSERT scripts to files
*/
async writeInsertScripts(tableName, scripts, totalRows) {
const filePaths = [];
if (scripts.length === 1) {
// Single file
const fileName = `insert_${tableName}_${Date.now()}.sql`;
const filePath = path.join(this.options.outputPath, fileName);
const content = this.buildFileHeader(tableName, totalRows, 1, 1) + scripts[0];
fs.writeFileSync(filePath, content, 'utf8');
filePaths.push(filePath);
}
else {
// Multiple files for large tables
scripts.forEach((script, index) => {
const fileName = `insert_${tableName}_part${index + 1}_${Date.now()}.sql`;
const filePath = path.join(this.options.outputPath, fileName);
const content = this.buildFileHeader(tableName, totalRows, index + 1, scripts.length) + script;
fs.writeFileSync(filePath, content, 'utf8');
filePaths.push(filePath);
});
}
return filePaths;
}
/**
* Build file header
*/
buildFileHeader(tableName, totalRows, partNumber, totalParts) {
const lines = [];
lines.push('-- Generated INSERT statements');
lines.push(`-- Table: ${tableName}`);
lines.push(`-- Total rows: ${totalRows}`);
lines.push(`-- Part: ${partNumber} of ${totalParts}`);
lines.push(`-- Generated at: ${new Date().toISOString()}`);
lines.push(`-- Dialect: ${this.options.dialect.toUpperCase()}`);
lines.push(`-- Batch size: ${this.options.batchSize}`);
if (this.options.dataMasking.enabled) {
lines.push('-- Data masking: ENABLED');
}
lines.push('');
// Add dialect-specific settings
if (this.options.dialect === database_types_1.DatabaseDialect.MYSQL) {
lines.push('-- MySQL specific settings');
lines.push('SET FOREIGN_KEY_CHECKS = 0;');
lines.push('SET AUTOCOMMIT = 0;');
lines.push('START TRANSACTION;');
lines.push('');
}
else if (this.options.dialect === database_types_1.DatabaseDialect.POSTGRES) {
lines.push('-- PostgreSQL specific settings');
lines.push('SET session_replication_role = replica;');
lines.push('');
}
return lines.join('\n');
}
/**
* Generate summary file
*/
async generateSummaryFile(results) {
const summaryPath = path.join(this.options.outputPath, `export_summary_${Date.now()}.md`);
const lines = [];
lines.push('# Data Export Summary');
lines.push('');
lines.push(`**Export Date:** ${new Date().toISOString()}`);
lines.push(`**Dialect:** ${this.options.dialect.toUpperCase()}`);
lines.push(`**Total Tables:** ${results.tableCount}`);
lines.push(`**Total Rows:** ${results.totalRows.toLocaleString()}`);
lines.push(`**Total Files:** ${results.fileCount}`);
lines.push(`**Batch Size:** ${this.options.batchSize}`);
lines.push(`**Data Masking:** ${this.options.dataMasking.enabled ? 'ENABLED' : 'DISABLED'}`);
lines.push('');
lines.push('## Table Statistics');
lines.push('');
lines.push('| Table Name | Rows | Batches |');
lines.push('|------------|------|---------|');
Object.entries(results.statistics).forEach(([tableName, stats]) => {
lines.push(`| ${tableName} | ${stats.rows.toLocaleString()} | ${stats.batches} |`);
});
lines.push('');
lines.push('## Generated Files');
lines.push('');
results.outputPaths.forEach(filePath => {
lines.push(`- \`${path.basename(filePath)}\``);
});
fs.writeFileSync(summaryPath, lines.join('\n'), 'utf8');
return summaryPath;
}
/**
* Escape SQL identifiers
*/
escapeIdentifier(identifier) {
if (this.options.dialect === database_types_1.DatabaseDialect.POSTGRES) {
return `"${identifier}"`;
}
else if (this.options.dialect === database_types_1.DatabaseDialect.MYSQL) {
return `\`${identifier}\``;
}
return identifier;
}
}
exports.DataExporter = DataExporter;