longcelot-sheet-db
Version:
Google Sheets-backed staging database adapter for Node.js with schema-first design
382 lines • 20.7 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateMigrateDataScript = generateMigrateDataScript;
exports.resolveRunDriver = resolveRunDriver;
exports.migrateDataCommand = migrateDataCommand;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const chalk_1 = __importDefault(require("chalk"));
const actorConfig_1 = require("../../utils/actorConfig");
const cliFiles_1 = require("../../utils/cliFiles");
const adminAdapter_1 = require("../lib/adminAdapter");
const postgresAdapter_1 = require("../../adapter/sql/postgresAdapter");
const mysqlAdapter_1 = require("../../adapter/sql/mysqlAdapter");
const migrate_1 = require("./migrate");
function loadSchemas(config) {
const schemas = [];
const schemasRoot = config.schemasDir
? path_1.default.resolve(process.cwd(), config.schemasDir)
: path_1.default.join(process.cwd(), 'schemas');
for (const actorEntry of config.actors) {
const actor = (0, actorConfig_1.resolveActorName)(actorEntry);
const actorDir = path_1.default.join(schemasRoot, actor);
if (!fs_1.default.existsSync(actorDir))
continue;
const files = fs_1.default.readdirSync(actorDir).filter((f) => f.endsWith('.ts') || f.endsWith('.js'));
for (const file of files) {
try {
const schema = require(path_1.default.join(actorDir, file)).default;
if (schema?.columns)
schemas.push(schema);
}
catch {
// skip unloadable files
}
}
}
return schemas;
}
// Serialises a schema to a JSON literal safe for embedding in generated JS.
// RegExp patterns are intentionally dropped — not needed for read-only export.
function schemaToLiteral(schema) {
const safe = {
name: schema.name,
actor: schema.actor,
timestamps: schema.timestamps,
softDelete: schema.softDelete,
pkColumn: schema.pkColumn,
columns: Object.fromEntries(Object.entries(schema.columns).map(([k, v]) => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { pattern: _pattern, ...rest } = v;
return [k, rest];
})),
};
return JSON.stringify(safe);
}
function generateMigrateDataScript(schemas, allUsers = false) {
const adminSchemas = schemas.filter((s) => s.actor === 'admin');
const userSchemas = schemas.filter((s) => s.actor !== 'admin');
// Build a role → table-names map for the --all-users section
const roleTablesMap = {};
for (const s of userSchemas) {
if (!roleTablesMap[s.actor])
roleTablesMap[s.actor] = [];
roleTablesMap[s.actor].push(s.name);
}
const commandName = allUsers ? 'lsdb migrate-data --all-users' : 'lsdb migrate-data';
const scriptDesc = allUsers
? 'Exports admin sheet data AND all registered user sheets'
: 'Exports admin sheet data only';
const lines = [
'/**',
` * longcelot-sheet-db → Production DB Data Export Script`,
` * Generated by: ${commandName}`,
` * ${scriptDesc}`,
' *',
' * Steps:',
' * 1. Install your production DB adapter (e.g. @prisma/client)',
' * 2. Replace the stub insertRow() with your real DB write',
' * 3. Run: node migrate-data.js',
' */',
'',
"const { createSheetAdapter } = require('longcelot-sheet-db');",
"require('dotenv').config();",
'',
'// Schemas embedded at generation time (pattern validators omitted — read-only export)',
'const schemas = [',
...schemas.map((s) => ` ${schemaToLiteral(s)},`),
'];',
'',
'async function migrateData() {',
' const adapter = createSheetAdapter({',
' adminSheetId: process.env.ADMIN_SHEET_ID,',
' credentials: {',
' clientId: process.env.GOOGLE_CLIENT_ID,',
' clientSecret: process.env.GOOGLE_CLIENT_SECRET,',
' redirectUri: process.env.GOOGLE_REDIRECT_URI,',
' },',
` tokens: require('./${cliFiles_1.TOKENS_FILENAME}'),`,
' });',
'',
' adapter.registerSchemas(schemas);',
'',
' // ---------- Replace this stub with your real DB client ----------',
' async function insertRow(tableName, row, userId) {',
" console.log(` INSERT INTO ${tableName}:`, JSON.stringify(row));",
' // e.g. await prisma[tableName].create({ data: row });',
' // userId is provided for user-sheet rows so you can associate them with the correct user FK',
' }',
' // ----------------------------------------------------------------',
'',
];
// Admin tables section
if (adminSchemas.length > 0) {
lines.push(" // ─── Admin tables ───────────────────────────────────────────────────────");
lines.push(" const adminCtx = adapter.withContext({ userId: 'migrate-cli', actor: 'admin', actorSheetId: process.env.ADMIN_SHEET_ID });");
lines.push('');
for (const schema of adminSchemas) {
lines.push(` // ─── ${schema.name} ───`);
lines.push(` console.log('\\nExporting table: ${schema.name}...');`);
lines.push(` try {`);
lines.push(` const rows = await adminCtx.table('${schema.name}').findMany({ includeDeleted: true });`);
lines.push(` console.log(\` Found \${rows.length} row(s)\`);`);
lines.push(` for (const row of rows) { await insertRow('${schema.name}', row); }`);
lines.push(` console.log(\` ✓ ${schema.name}: \${rows.length} rows exported\`);`);
lines.push(` } catch (err) {`);
lines.push(` console.error(\` ✖ ${schema.name} failed:\`, err.message);`);
lines.push(` }`);
lines.push('');
}
}
// User sheets section (only when --all-users)
if (allUsers && userSchemas.length > 0) {
lines.push(" // ─── User sheets (--all-users) ──────────────────────────────────────────");
lines.push(` const ROLE_TABLES = ${JSON.stringify(roleTablesMap, null, 4).replace(/\n/g, '\n ')};`);
lines.push('');
lines.push(" console.log('\\nFetching registered users for per-user export...');");
lines.push(' try {');
lines.push(" const adminCtxForUsers = adapter.withContext({ userId: 'migrate-cli', actor: 'admin', actorSheetId: process.env.ADMIN_SHEET_ID });");
lines.push(" const users = await adminCtxForUsers.table('users').findMany({ includeDeleted: true });");
lines.push(' console.log(` Found ${users.length} user(s)\\n`);');
lines.push('');
lines.push(' for (const user of users) {');
lines.push(' if (!user.actor_sheet_id) continue;');
lines.push(' const roleTables = ROLE_TABLES[user.role] || [];');
lines.push(' if (!roleTables.length) continue;');
lines.push('');
lines.push(' console.log(`\\n── User: ${user.user_id} (${user.role}) → ${user.actor_sheet_id}`);');
lines.push(' const userCtx = adapter.withContext({ userId: user.user_id, actor: user.role, actorSheetId: user.actor_sheet_id });');
lines.push('');
lines.push(' for (const tableName of roleTables) {');
lines.push(' console.log(` Exporting ${tableName}...`);');
lines.push(' try {');
lines.push(' const rows = await userCtx.table(tableName).findMany({ includeDeleted: true });');
lines.push(' console.log(` Found ${rows.length} row(s)`);');
lines.push(' for (const row of rows) { await insertRow(tableName, row, user.user_id); }');
lines.push(' console.log(` ✓ ${tableName}: ${rows.length} rows exported`);');
lines.push(' } catch (err) {');
lines.push(' console.error(` ✖ ${tableName} failed:`, err.message);');
lines.push(' }');
lines.push(' }');
lines.push(' }');
lines.push(' } catch (err) {');
lines.push(" console.error('\\n✖ Failed to fetch users for --all-users export:', err.message);");
lines.push(' }');
lines.push('');
}
else if (!allUsers && userSchemas.length > 0) {
// Hint that --all-users exists
lines.push(" // Tip: run `lsdb migrate-data --all-users` to also export all registered user sheets");
lines.push('');
}
lines.push(" console.log('\\nData export complete.');");
lines.push('}');
lines.push('');
lines.push('migrateData().catch(console.error);');
lines.push('');
return lines.join('\n');
}
/**
* Validates --run's own required flags before touching the filesystem for lsdb.config.ts, so a
* misconfigured --run fails fast with the relevant error instead of an unrelated config error.
*/
function resolveRunDriver(options) {
const connectionString = options.connectionString ?? process.env.DATABASE_URL;
if (!connectionString) {
console.error(chalk_1.default.red('❌ --connection-string or $DATABASE_URL is required for --run'));
process.exit(1);
}
const driver = options.driver ?? (0, migrate_1.inferDriverFromConnectionString)(connectionString);
if (driver !== 'postgres' && driver !== 'mysql') {
console.error(chalk_1.default.red("❌ --run only supports --driver postgres|mysql (not prisma — this CLI process can't " +
'construct a consumer PrismaClient; see FAQ.md #13). Pass --driver explicitly if it could not be inferred.'));
process.exit(1);
}
return { connectionString, driver };
}
/**
* `--run`: executes the same admin-then-per-user traversal generateMigrateDataScript() encodes
* as a generated script, but in-process against a real SQL adapter as the write target instead
* of emitting a stub insertRow() for a human to fill in (Phase 16.7). Always upserts by `_id` —
* unconditionally idempotent, no separate opt-in flag, since unattended CI reruns need this safe
* by default (matches `seed --upsert`'s existing conflict-handling convention). `--driver prisma`
* isn't supported: there's no way for this CLI process to construct a consumer's typed
* PrismaClient — Prisma-track consumers use `--run --driver postgres`/`mysql` for the one-time
* cutover regardless of their app's long-term client, since the data doesn't care which client
* inserted it (see FAQ.md #13).
*/
async function runMigrateData(schemas, connectionString, driver, options) {
var _a;
const config = (0, adminAdapter_1.loadCLIConfig)();
const { adapter: sourceAdapter, adminCtx } = await (0, adminAdapter_1.buildAdminAdapter)({ config, schemas, tokenFile: options.tokenFile });
const targetAdapter = driver === 'postgres' ? (0, postgresAdapter_1.createPostgresAdapter)({ connectionString }) : (0, mysqlAdapter_1.createMySQLAdapter)({ connectionString });
targetAdapter.registerSchemas(schemas);
const adminSchemas = schemas.filter((s) => s.actor === 'admin');
const userSchemas = schemas.filter((s) => s.actor !== 'admin');
let totalRows = 0;
console.log(chalk_1.default.blue.bold(`\n📦 ${options.dryRun ? '[DRY RUN] ' : ''}Running data cutover to ${driver}...\n`));
for (const schema of adminSchemas) {
// includeDeleted: a soft-deleted row is still a real historical fact other rows can
// legitimately reference by FK (e.g. an order assigned to a since-removed cleaner) — silently
// excluding it here (findMany()'s default) would migrate the referencing row but not the row
// it points at, failing with a real FK violation on the target database. The target's own
// findMany() still filters _deleted_at by default afterward, so this only affects what gets
// migrated, not what the application sees post-cutover (see FAQ.md §13).
const rows = await adminCtx.table(schema.name).findMany({ includeDeleted: true });
console.log(chalk_1.default.cyan(`${schema.name}: ${rows.length} row(s)`));
if (!options.dryRun) {
const targetCtx = targetAdapter.withContext({ userId: 'migrate-data-cli', actor: 'admin' });
for (const row of rows) {
await targetCtx.table(schema.name).upsert({ where: { _id: row._id }, data: row, skipFKValidation: true });
}
}
totalRows += rows.length;
}
if (options.allUsers && userSchemas.length > 0) {
const roleSchemasMap = {};
for (const s of userSchemas) {
(roleSchemasMap[_a = s.actor] ?? (roleSchemasMap[_a] = [])).push(s);
}
// See the matching comment above — a soft-deleted (deactivated) user's historical data
// shouldn't vanish from the cutover just because their account was later deactivated.
const users = await adminCtx.table('users').findMany({ includeDeleted: true });
console.log(chalk_1.default.cyan(`\nFound ${users.length} user(s) for per-user cutover`));
for (const user of users) {
const actorSheetId = user.actor_sheet_id;
const role = user.role;
if (!actorSheetId || !role)
continue;
const roleSchemas = roleSchemasMap[role] ?? [];
if (roleSchemas.length === 0)
continue;
const sourceUserCtx = sourceAdapter.withContext({ userId: user.user_id, actor: role, actorSheetId });
for (const schema of roleSchemas) {
const rows = await sourceUserCtx.table(schema.name).findMany({ includeDeleted: true });
console.log(` ${role}:${user.user_id} → ${schema.name}: ${rows.length} row(s)`);
if (!options.dryRun) {
// Tenant key for the SQL target reuses actorSheetId as an opaque tenant value — the
// same generalization resolveNonAdminTenantKey() documents (see FAQ.md #13).
const targetUserCtx = targetAdapter.withContext({ userId: user.user_id, actor: role, actorSheetId });
for (const row of rows) {
await targetUserCtx.table(schema.name).upsert({ where: { _id: row._id }, data: row, skipFKValidation: true });
}
}
totalRows += rows.length;
}
}
}
else if (!options.allUsers && userSchemas.length > 0) {
console.log(chalk_1.default.gray('\nTip: pass --all-users to also migrate all registered user sheets.'));
}
console.log(chalk_1.default.green(`\n✅ ${options.dryRun ? 'Would migrate' : 'Migrated'} ${totalRows} row(s) total.`));
}
async function migrateDataCommand(options) {
if (options.run) {
const { connectionString, driver } = resolveRunDriver(options);
require('dotenv').config();
const config = (0, adminAdapter_1.loadCLIConfig)();
// Row inserts hit real FOREIGN KEY constraints, same as migrate --sql --apply's inline FKs
// (see sortSchemasByDependency()'s doc comment / FAQ.md §13) — this command has its own
// loadSchemas() (duplicated from migrate.ts, not shared) that never got the same fix, so a
// referencing table's rows could be upserted before the table it references had any rows at
// all, failing with a genuine FK violation instead of a DDL-time ordering error.
let schemas = (0, migrate_1.sortSchemasByDependency)(loadSchemas(config));
if (options.table) {
schemas = schemas.filter((s) => s.name === options.table);
if (schemas.length === 0) {
console.error(chalk_1.default.red(`❌ No schema found for table: ${options.table}`));
process.exit(1);
}
}
if (schemas.length === 0) {
console.log(chalk_1.default.yellow('⚠️ No schemas found. Nothing to migrate.'));
return;
}
await runMigrateData(schemas, connectionString, driver, options);
return;
}
console.log(chalk_1.default.blue.bold('📦 Generating data export script...\n'));
require('dotenv').config();
let config;
try {
config = require((0, cliFiles_1.resolveConfigPath)()).default;
}
catch {
console.error(chalk_1.default.red('❌ lsdb.config.ts not found. Run: lsdb init'));
process.exit(1);
}
// Same ordering fix as the --run path above — keeps the generated stub script's table order
// dependency-safe too, in case whoever fills in insertRow() runs it top-to-bottom.
let schemas = (0, migrate_1.sortSchemasByDependency)(loadSchemas(config));
if (options.table) {
schemas = schemas.filter((s) => s.name === options.table);
if (schemas.length === 0) {
console.error(chalk_1.default.red(`❌ No schema found for table: ${options.table}`));
process.exit(1);
}
}
if (schemas.length === 0) {
console.log(chalk_1.default.yellow('⚠️ No schemas found. Nothing to export.'));
return;
}
const adminSchemas = schemas.filter((s) => s.actor === 'admin');
const userSchemas = schemas.filter((s) => s.actor !== 'admin');
console.log(chalk_1.default.cyan(`Found ${schemas.length} table(s):`));
for (const s of schemas) {
console.log(` ${chalk_1.default.white(s.name)} ${chalk_1.default.gray(`(${s.actor})`)}`);
}
console.log();
if (options.dryRun) {
console.log(chalk_1.default.yellow('[DRY RUN] Script would be written — no file created.\n'));
console.log(chalk_1.default.bold('Export plan:'));
console.log(` Admin tables: ${adminSchemas.map((s) => s.name).join(', ') || '(none)'}`);
if (options.allUsers && userSchemas.length > 0) {
const roles = [...new Set(userSchemas.map((s) => s.actor))];
for (const role of roles) {
const tables = userSchemas.filter((s) => s.actor === role).map((s) => s.name);
console.log(` ${role} tables: ${tables.join(', ')}`);
}
console.log(chalk_1.default.gray('\n Note: --all-users will loop over all rows in admin users table at runtime.'));
}
else {
for (const s of userSchemas) {
console.log(` ${s.actor}.${s.name}: ${Object.keys(s.columns).length} column(s) — not included (omit --all-users to skip user sheets)`);
}
}
return;
}
const allUsers = options.allUsers ?? false;
const script = generateMigrateDataScript(schemas, allUsers);
const outDir = options.output ?? process.cwd();
if (!fs_1.default.existsSync(outDir)) {
fs_1.default.mkdirSync(outDir, { recursive: true });
}
const outPath = path_1.default.join(outDir, 'migrate-data.js');
fs_1.default.writeFileSync(outPath, script, 'utf-8');
console.log(chalk_1.default.green(`✅ Data export script written to ${outPath}`));
console.log();
console.log(chalk_1.default.cyan('Next steps:'));
console.log(' 1. ' + chalk_1.default.white('Review the generated migrate-data.js'));
console.log(' 2. ' + chalk_1.default.white('Replace the insertRow() stub with your real DB client'));
console.log(' 3. ' + chalk_1.default.white('Run: node migrate-data.js'));
if (!allUsers && userSchemas.length > 0) {
console.log();
console.log(chalk_1.default.gray(`Tip: Run with --all-users to also export all registered user sheets.`));
}
console.log();
console.log(chalk_1.default.gray('Tip: Run `lsdb migrate --prisma` first to generate your Prisma schema.'));
const missingEnv = [];
for (const envVar of ['GOOGLE_CLIENT_ID', 'GOOGLE_CLIENT_SECRET', 'ADMIN_SHEET_ID']) {
if (!process.env[envVar])
missingEnv.push(envVar);
}
if (missingEnv.length > 0) {
console.log();
console.log(chalk_1.default.yellow(`⚠️ Missing env vars for script execution: ${missingEnv.join(', ')}`));
}
}
exports.default = migrateDataCommand;
//# sourceMappingURL=migrate-data.js.map