sql-to-json-converter
Version:
Powerful SQL to JSON converter with support for large files and multiple output formats. Converts SQL database dumps to structured JSON files.
654 lines (651 loc) ā¢ 24.6 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.processLargeSQL = exports.sqlToJsonFiles = exports.sqlToJson = exports.SQLToJSONConverter = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const readline = __importStar(require("readline"));
/**
* SQL to JSON Converter class with full TypeScript support
*/
class SQLToJSONConverter {
constructor(options = {}) {
this.processedStatements = 0;
this.tables = {};
this.currentTable = null;
this.insideTransaction = false;
this.batchSize = options.batchSize || 500;
this.showMemory = options.showMemory || false;
this.limit = options.limit || null;
this.skipUnparsable = options.skipUnparsable || false;
this.outputMode = options.outputMode || 'combined';
this.outputDir = options.outputDir || 'json-output';
}
/**
* Log current memory usage if enabled
*/
logMemoryUsage() {
if (this.showMemory) {
const used = process.memoryUsage();
const rss = Math.round(used.rss / 1024 / 1024 * 100) / 100;
const heap = Math.round(used.heapUsed / 1024 / 1024 * 100) / 100;
console.log(`Memory: RSS ${rss} MB, Heap ${heap} MB`);
}
}
/**
* Parse CREATE TABLE statement
*/
parseCreateTable(sql) {
try {
// Extract table name - support both with and without backticks
const tableMatch = sql.match(/CREATE TABLE\s+`?([^`\s(]+)`?\s*\(/i);
if (!tableMatch)
return null;
const tableName = tableMatch[1].replace(/^SERVMASK_PREFIX_/, '');
// Extract columns from CREATE TABLE statement
const columns = [];
// Find the content between the first ( and the last )
const startParen = sql.indexOf('(');
const endParen = sql.lastIndexOf(')');
if (startParen === -1 || endParen === -1)
return null;
const columnsPart = sql.substring(startParen + 1, endParen);
// Split by comma but handle commas within parentheses (e.g., DECIMAL(10,2))
const columnDefs = [];
let current = '';
let parenDepth = 0;
for (let i = 0; i < columnsPart.length; i++) {
const char = columnsPart[i];
if (char === '(') {
parenDepth++;
current += char;
}
else if (char === ')') {
parenDepth--;
current += char;
}
else if (char === ',' && parenDepth === 0) {
// This is a column separator comma, not a comma within a type
columnDefs.push(current.trim());
current = '';
}
else {
current += char;
}
}
// Add the last column definition
if (current.trim()) {
columnDefs.push(current.trim());
}
for (const colDef of columnDefs) {
const trimmed = colDef.trim();
// Skip empty definitions, comments, PRIMARY KEY, KEY definitions, and ENGINE
if (!trimmed ||
trimmed.startsWith('--') ||
trimmed.toUpperCase().startsWith('PRIMARY KEY') ||
trimmed.toUpperCase().startsWith('KEY ') ||
trimmed.toUpperCase().startsWith('UNIQUE KEY') ||
trimmed.toUpperCase().startsWith('FOREIGN KEY') ||
trimmed.toUpperCase().includes('ENGINE=')) {
continue;
}
// Look for column definitions - support both with and without backticks
// Match: `column_name` TYPE or column_name TYPE
const columnMatch = trimmed.match(/^`?([^`\s]+)`?\s+(.+)$/);
if (columnMatch) {
const columnName = columnMatch[1];
const columnType = columnMatch[2].trim();
columns.push({
name: columnName,
type: columnType
});
}
}
return {
tableName,
columns,
data: []
};
}
catch (error) {
if (!this.skipUnparsable) {
console.error(`Error parsing CREATE TABLE: ${error.message}`);
}
return null;
}
}
/**
* Parse INSERT INTO statement
*/
parseInsertInto(sql) {
try {
// Extract table name - support both with and without backticks
const tableMatch = sql.match(/INSERT INTO\s+`?([^`\s(]+)`?\s*(?:\([^)]+\))?\s*VALUES/i);
if (!tableMatch)
return null;
const tableName = tableMatch[1].replace(/^SERVMASK_PREFIX_/, '');
// Extract values - handle multiple value sets
const valuesMatch = sql.match(/VALUES\s*(.*)/is);
if (!valuesMatch)
return null;
const valuesString = valuesMatch[1];
const records = [];
// Split by ),( to handle multiple records
const valueGroups = valuesString.split(/\),\s*\(/);
for (let i = 0; i < valueGroups.length; i++) {
let valueGroup = valueGroups[i];
// Clean up the value group
valueGroup = valueGroup.replace(/^\(/, '').replace(/\);?\s*$/, '');
// Parse individual values
const values = this.parseValues(valueGroup);
if (values) {
records.push(values);
}
}
return {
tableName,
records
};
}
catch (error) {
if (!this.skipUnparsable) {
console.error(`Error parsing INSERT INTO: ${error.message}`);
}
return null;
}
}
/**
* Parse individual values from VALUES clause
*/
parseValues(valueString) {
const values = [];
let current = '';
let inString = false;
let stringChar = '';
let depth = 0;
for (let i = 0; i < valueString.length; i++) {
const char = valueString[i];
if (!inString) {
if (char === "'" || char === '"') {
inString = true;
stringChar = char;
current += char;
}
else if (char === '(' || char === '{') {
depth++;
current += char;
}
else if (char === ')' || char === '}') {
depth--;
current += char;
}
else if (char === ',' && depth === 0) {
values.push(this.cleanValue(current.trim()));
current = '';
}
else {
current += char;
}
}
else {
current += char;
if (char === stringChar && valueString[i - 1] !== '\\') {
inString = false;
}
}
}
if (current.trim()) {
values.push(this.cleanValue(current.trim()));
}
return values;
}
/**
* Clean and convert SQL values to proper JavaScript types
*/
cleanValue(value) {
value = value.trim();
// Handle NULL
if (value.toUpperCase() === 'NULL') {
return null;
}
// Handle quoted strings
if ((value.startsWith("'") && value.endsWith("'")) ||
(value.startsWith('"') && value.endsWith('"'))) {
return value.slice(1, -1).replace(/\\'/g, "'").replace(/\\"/g, '"');
}
// Handle numbers
if (/^-?\d+$/.test(value)) {
return parseInt(value, 10);
}
if (/^-?\d+\.\d+$/.test(value)) {
return parseFloat(value);
}
return value;
}
/**
* Process a single SQL statement
*/
processStatement(sql) {
sql = sql.trim();
if (!sql)
return true;
this.processedStatements++;
if (this.limit && this.processedStatements > this.limit) {
return false; // Stop processing
}
if (sql.startsWith('START TRANSACTION')) {
this.insideTransaction = true;
return true;
}
if (sql.startsWith('COMMIT')) {
this.insideTransaction = false;
return true;
}
if (sql.toUpperCase().startsWith('DROP TABLE')) {
return true; // Skip DROP statements
}
if (sql.toUpperCase().startsWith('CREATE TABLE')) {
const tableInfo = this.parseCreateTable(sql);
if (tableInfo) {
this.tables[tableInfo.tableName] = tableInfo;
this.currentTable = tableInfo.tableName;
}
return true;
}
if (sql.toUpperCase().startsWith('INSERT INTO')) {
const insertInfo = this.parseInsertInto(sql);
if (insertInfo && this.tables[insertInfo.tableName]) {
const table = this.tables[insertInfo.tableName];
// Convert records to objects using column names
for (const record of insertInfo.records) {
const obj = {};
table.columns.forEach((col, index) => {
if (index < record.length) {
obj[col.name] = record[index];
}
});
table.data.push(obj);
}
}
return true;
}
return true;
}
/**
* Process large SQL files with streaming
*/
async processLargeSQL(inputFile, outputFile) {
console.log('š Starting SQL to JSON conversion...');
console.log(`š Input: ${inputFile}`);
if (this.outputMode === 'separate') {
console.log(`š Output directory: ${this.outputDir}/`);
this.ensureOutputDirectory();
}
else {
console.log(`š Output: ${outputFile || 'stdout'}`);
}
console.log(`āļø Batch size: ${this.batchSize}`);
if (this.limit)
console.log(`š¢ Limit: ${this.limit} statements`);
const fileStream = fs.createReadStream(inputFile);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
let currentStatement = '';
let lineCount = 0;
const startTime = Date.now();
for await (const line of rl) {
lineCount++;
if (lineCount % 1000 === 0) {
console.log(`š Processed ${lineCount} lines, ${this.processedStatements} statements, ${Object.keys(this.tables).length} tables`);
this.logMemoryUsage();
}
// Skip comments and empty lines
const trimmedLine = line.trim();
if (!trimmedLine || trimmedLine.startsWith('--')) {
continue;
}
currentStatement += ' ' + line;
// Check if statement is complete (ends with semicolon)
if (trimmedLine.endsWith(';')) {
const shouldContinue = this.processStatement(currentStatement.trim());
currentStatement = '';
if (!shouldContinue) {
console.log('š Reached processing limit');
break;
}
}
}
// Process any remaining statement
if (currentStatement.trim()) {
this.processStatement(currentStatement.trim());
}
const endTime = Date.now();
const duration = (endTime - startTime) / 1000;
console.log(`ā
Conversion completed in ${duration.toFixed(2)}s`);
console.log(`š Final stats:`);
console.log(` - Lines processed: ${lineCount}`);
console.log(` - Statements processed: ${this.processedStatements}`);
console.log(` - Tables found: ${Object.keys(this.tables).length}`);
Object.keys(this.tables).forEach(tableName => {
console.log(` - ${tableName}: ${this.tables[tableName].data.length} records`);
});
this.logMemoryUsage();
// Generate output based on mode
if (this.outputMode === 'separate') {
this.writeSeparateJSONFiles();
}
else {
this.writeCombinedJSON(outputFile);
}
}
/**
* Ensure output directory exists
*/
ensureOutputDirectory() {
if (!fs.existsSync(this.outputDir)) {
fs.mkdirSync(this.outputDir, { recursive: true });
console.log(`š Created output directory: ${this.outputDir}`);
}
}
/**
* Write separate JSON files for each table
*/
writeSeparateJSONFiles() {
this.ensureOutputDirectory();
let filesWritten = 0;
for (const [tableName, table] of Object.entries(this.tables)) {
const fileName = `${tableName}.json`;
const filePath = path.join(this.outputDir, fileName);
const tableData = {
tableName,
columns: table.columns,
recordCount: table.data.length,
generatedAt: new Date().toISOString(),
data: table.data
};
fs.writeFileSync(filePath, JSON.stringify(tableData, null, 2), 'utf-8');
console.log(`ā
Wrote ${filePath} (${table.data.length} records)`);
filesWritten++;
}
// Create summary file
const summary = {
generatedAt: new Date().toISOString(),
totalTables: Object.keys(this.tables).length,
totalRecords: Object.values(this.tables).reduce((sum, table) => sum + table.data.length, 0),
tables: Object.keys(this.tables).map(tableName => ({
name: tableName,
recordCount: this.tables[tableName].data.length,
fileName: `${tableName}.json`
}))
};
const summaryPath = path.join(this.outputDir, '_summary.json');
fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');
console.log(`ā
Wrote summary file: ${summaryPath}`);
console.log(`š Successfully wrote ${filesWritten} table files + 1 summary file`);
}
/**
* Write combined JSON file
*/
writeCombinedJSON(outputFile) {
const result = {
metadata: {
generatedAt: new Date().toISOString(),
totalTables: Object.keys(this.tables).length,
totalRecords: Object.values(this.tables).reduce((sum, table) => sum + table.data.length, 0)
},
tables: this.tables
};
const json = JSON.stringify(result, null, 2);
if (outputFile) {
fs.writeFileSync(outputFile, json);
console.log(`š¾ Output written to ${outputFile}`);
}
else {
console.log(json);
}
}
/**
* Convert SQL content to JSON (in-memory processing)
*/
sqlToJson(content) {
// Remove BOM if present
content = content.replace(/^\uFEFF/, '');
const statements = content
.split(';')
.map(s => s.trim())
.filter(s => s && !s.startsWith('--'));
for (const statement of statements) {
if (statement.length > 0) {
const shouldContinue = this.processStatement(statement);
if (!shouldContinue)
break;
}
}
return {
metadata: {
generatedAt: new Date().toISOString(),
totalTables: Object.keys(this.tables).length,
totalRecords: Object.values(this.tables).reduce((sum, table) => sum + table.data.length, 0)
},
tables: this.tables
};
}
/**
* Convert SQL content to separate JSON files
*/
sqlToJsonFiles(content, outputDir = 'json-output') {
// Remove BOM if present
content = content.replace(/^\uFEFF/, '');
// Split by semicolon and newline patterns, then clean up
const statements = content
.split(';')
.map(s => s.trim())
.filter(s => s && !s.startsWith('--')); // Remove empty and comment-only statements
for (const statement of statements) {
if (statement.length > 0) {
const shouldContinue = this.processStatement(statement);
if (!shouldContinue)
break;
}
}
this.writeSeparateJSONFiles();
return {
metadata: {
generatedAt: new Date().toISOString(),
totalTables: Object.keys(this.tables).length,
totalRecords: Object.values(this.tables).reduce((sum, table) => sum + table.data.length, 0),
outputDirectory: this.outputDir
},
tables: Object.keys(this.tables)
};
}
}
exports.SQLToJSONConverter = SQLToJSONConverter;
/**
* Show CLI help information
*/
function showHelp() {
console.log(`
š SQL to JSON Converter
Powerful SQL to JSON converter with efficient processing
Usage:
npx sql-to-json-converter [input.sql] [options]
sql-to-json [input.sql] [options]
Options:
--help, -h Show help
--version, -v Show version
--output [file] Output file for combined mode
--separate Export separate files (default)
--combined Export combined file
--output-dir [dir] Output directory (default: json-output)
--memory, -m Show memory usage
--batch-size [num] Batch size (default: 500)
--limit [num] Limit number of statements
--skip-unparsable Skip unparsable statements
Examples:
npx sql-to-json-converter database.sql
npx sql-to-json-converter database.sql --output-dir my-json-files
npx sql-to-json-converter database.sql --combined --output result.json
sql-to-json database.sql --memory --batch-size 1000
`);
}
/**
* Show version information
*/
function showVersion() {
const pkg = require('../package.json');
console.log(pkg.version);
}
/**
* Parse CLI arguments
*/
function parseArgs(args) {
const result = {
inputFile: args[0] || '',
outputDir: 'json-output',
showMemory: false,
skipUnparsable: false,
isCombined: false,
batchSize: 500,
limit: null
};
// Parse flags
result.showMemory = args.includes('--memory') || args.includes('-m');
result.skipUnparsable = args.includes('--skip-unparsable');
result.isCombined = args.includes('--combined');
// Parse options with values
if (args.includes('--output')) {
const outputIndex = args.indexOf('--output');
result.outputFile = args[outputIndex + 1];
}
if (args.includes('--output-dir')) {
const dirIndex = args.indexOf('--output-dir');
result.outputDir = args[dirIndex + 1] || 'json-output';
}
if (args.includes('--batch-size')) {
const batchIndex = args.indexOf('--batch-size');
result.batchSize = parseInt(args[batchIndex + 1]) || 500;
}
if (args.includes('--limit')) {
const limitIndex = args.indexOf('--limit');
result.limit = parseInt(args[limitIndex + 1]) || null;
}
return result;
}
/**
* Main CLI function
*/
async function main() {
const args = process.argv.slice(2);
if (args.includes('--help') || args.includes('-h')) {
return showHelp();
}
if (args.includes('--version') || args.includes('-v')) {
return showVersion();
}
if (args.length === 0) {
console.error('ā Error: Please provide SQL file path');
showHelp();
process.exit(1);
}
const parsedArgs = parseArgs(args);
if (!fs.existsSync(parsedArgs.inputFile)) {
console.error(`ā File not found: ${parsedArgs.inputFile}`);
process.exit(1);
}
const stats = fs.statSync(parsedArgs.inputFile);
const sizeMB = stats.size / (1024 * 1024);
console.log(`š File size: ${sizeMB.toFixed(2)} MB`);
try {
const converter = new SQLToJSONConverter({
batchSize: parsedArgs.batchSize,
showMemory: parsedArgs.showMemory,
limit: parsedArgs.limit,
skipUnparsable: parsedArgs.skipUnparsable,
outputMode: parsedArgs.isCombined ? 'combined' : 'separate',
outputDir: parsedArgs.outputDir
});
if (sizeMB > 10 || parsedArgs.showMemory || parsedArgs.limit) {
console.log('š¦ Large file detected. Using stream processing...');
await converter.processLargeSQL(parsedArgs.inputFile, parsedArgs.outputFile);
}
else {
console.log('š¦ Small file detected. Using in-memory parsing...');
const content = fs.readFileSync(parsedArgs.inputFile, 'utf8');
if (parsedArgs.isCombined) {
const result = converter.sqlToJson(content);
const json = JSON.stringify(result, null, 2);
if (parsedArgs.outputFile) {
fs.writeFileSync(parsedArgs.outputFile, json);
console.log(`ā
Result written to ${parsedArgs.outputFile}`);
}
else {
console.log(json);
}
}
else {
converter.sqlToJsonFiles(content, parsedArgs.outputDir);
}
}
}
catch (err) {
console.error(`ā Error: ${err.message}`);
console.error(err.stack);
process.exit(1);
}
}
// Export functions for use as library
const sqlToJson = (content, options = {}) => {
const converter = new SQLToJSONConverter(options);
return converter.sqlToJson(content);
};
exports.sqlToJson = sqlToJson;
const sqlToJsonFiles = (content, outputDir = 'json-output', options = {}) => {
const converter = new SQLToJSONConverter({ ...options, outputDir, outputMode: 'separate' });
return converter.sqlToJsonFiles(content, outputDir);
};
exports.sqlToJsonFiles = sqlToJsonFiles;
const processLargeSQL = async (inputFile, outputFile, options = {}) => {
const converter = new SQLToJSONConverter(options);
return converter.processLargeSQL(inputFile, outputFile);
};
exports.processLargeSQL = processLargeSQL;
if (require.main === module) {
main().catch(console.error);
}
//# sourceMappingURL=cli.js.map