dataweave
Version:
AI-assisted CLI for modern data pipelines with DBT, Dagster, and Supabase integration
255 lines (252 loc) • 10.6 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DbtManager = void 0;
const promises_1 = require("fs/promises");
const path_1 = require("path");
const fs_1 = require("fs");
const child_process_1 = require("child_process");
const chalk_1 = __importDefault(require("chalk"));
class DbtManager {
constructor(config) {
this.config = config;
}
async generateModel(options) {
const { name, sql, description, materializedAs = 'view', columns = [], tests = [], tags = [] } = options;
console.log(chalk_1.default.blue(`📊 Generating DBT model: ${name}`));
const modelDir = this.getModelDirectory(name);
const modelPath = (0, path_1.join)(this.config.modelsDir, modelDir);
if (!(0, fs_1.existsSync)(modelPath)) {
await (0, promises_1.mkdir)(modelPath, { recursive: true });
}
const sqlContent = sql || this.generateDefaultSql(name);
const sqlWithConfig = this.addModelConfig(sqlContent, materializedAs, tags);
await (0, promises_1.writeFile)((0, path_1.join)(modelPath, `${name}.sql`), sqlWithConfig);
await this.updateSchema(modelDir, name, description || '', columns, tests);
console.log(chalk_1.default.green(`✓ Created model ${name} in ${modelDir}/`));
}
async runModel(modelName) {
const command = modelName ? `dbt run --select ${modelName}` : 'dbt run';
console.log(chalk_1.default.blue(`🚀 Running: ${command}`));
await this.executeDbtCommand(['run'].concat(modelName ? ['--select', modelName] : []));
}
async testModel(modelName) {
const command = modelName ? `dbt test --select ${modelName}` : 'dbt test';
console.log(chalk_1.default.blue(`🧪 Testing: ${command}`));
await this.executeDbtCommand(['test'].concat(modelName ? ['--select', modelName] : []));
}
async compileModel(modelName) {
const command = modelName ? `dbt compile --select ${modelName}` : 'dbt compile';
console.log(chalk_1.default.blue(`⚙️ Compiling: ${command}`));
await this.executeDbtCommand(['compile'].concat(modelName ? ['--select', modelName] : []));
}
async generateDocs() {
console.log(chalk_1.default.blue('📚 Generating DBT documentation...'));
await this.executeDbtCommand(['docs', 'generate']);
await this.executeDbtCommand(['docs', 'serve', '--port', '8001']);
console.log(chalk_1.default.green('✓ Documentation available at http://localhost:8001'));
}
async introspectDatabase() {
console.log(chalk_1.default.blue('🔍 Introspecting database schema...'));
try {
await this.executeDbtCommand(['debug']);
const sourcesPath = (0, path_1.join)(this.config.modelsDir, 'sources.yml');
if ((0, fs_1.existsSync)(sourcesPath)) {
const sourcesContent = await (0, promises_1.readFile)(sourcesPath, 'utf-8');
console.log(chalk_1.default.green('✓ Found existing sources configuration'));
return sourcesContent;
}
else {
console.log(chalk_1.default.yellow('⚠️ No sources.yml found. Consider running dbt source to introspect your database.'));
return null;
}
}
catch (error) {
console.error(chalk_1.default.red('❌ Database introspection failed:'), error instanceof Error ? error.message : String(error));
throw error;
}
}
getModelDirectory(modelName) {
if (modelName.startsWith('stg_')) {
return 'staging';
}
else if (modelName.startsWith('int_')) {
return 'intermediate';
}
else if (modelName.startsWith('fct_') || modelName.startsWith('dim_')) {
return 'marts';
}
else {
return 'staging';
}
}
generateDefaultSql(modelName) {
return `-- Generated model: ${modelName}
-- TODO: Replace this with your actual SQL logic
{{ config(materialized='view') }}
select
1 as id,
'sample' as name,
current_timestamp as created_at
-- TODO: Add your transformations here
-- Uncomment and modify the following example:
-- from {{ ref('source_table') }}
-- where condition = 'value'
`;
}
addModelConfig(sql, materializedAs, tags) {
const configOptions = [`materialized='${materializedAs}'`];
if (tags.length > 0) {
configOptions.push(`tags=['${tags.join("', '")}']`);
}
const configLine = `{{ config(${configOptions.join(', ')}) }}`;
if (sql.includes('{{ config(')) {
return sql.replace(/\{\{ config\([^}]+\) \}\}/, configLine);
}
else {
const lines = sql.split('\n');
const firstNonCommentLine = lines.findIndex(line => line.trim() && !line.trim().startsWith('--'));
if (firstNonCommentLine === -1) {
return `${configLine}\n\n${sql}`;
}
else {
lines.splice(firstNonCommentLine, 0, configLine, '');
return lines.join('\n');
}
}
}
async updateSchema(modelDir, modelName, description, columns, tests) {
const schemaPath = (0, path_1.join)(this.config.modelsDir, modelDir, 'schema.yml');
let schemaData = { version: 2, models: [] };
if ((0, fs_1.existsSync)(schemaPath)) {
try {
const existingContent = await (0, promises_1.readFile)(schemaPath, 'utf-8');
schemaData = this.parseBasicYaml(existingContent);
}
catch (error) {
console.warn(chalk_1.default.yellow(`⚠️ Failed to parse existing schema.yml: ${error instanceof Error ? error.message : String(error)}`));
}
}
let modelEntry = schemaData.models.find((m) => m.name === modelName);
if (!modelEntry) {
modelEntry = { name: modelName };
schemaData.models.push(modelEntry);
}
if (description) {
modelEntry.description = description;
}
if (tests.length > 0) {
modelEntry.tests = tests;
}
if (columns.length > 0) {
modelEntry.columns = columns.map(col => ({
name: col.name,
description: col.description,
tests: col.tests || []
}));
}
const yamlContent = this.generateBasicYaml(schemaData);
await (0, promises_1.writeFile)(schemaPath, yamlContent);
}
parseBasicYaml(content) {
try {
const lines = content.split('\n');
const result = { version: 2, models: [] };
let currentModel = null;
let currentColumns = [];
let inColumns = false;
for (const line of lines) {
const trimmed = line.trim();
if (trimmed.startsWith('- name:') && !inColumns) {
if (currentModel) {
if (currentColumns.length > 0) {
currentModel.columns = currentColumns;
}
result.models.push(currentModel);
}
currentModel = { name: trimmed.split(':')[1].trim() };
currentColumns = [];
inColumns = false;
}
else if (trimmed.startsWith('description:') && currentModel) {
currentModel.description = trimmed.split(':').slice(1).join(':').trim();
}
else if (trimmed === 'columns:') {
inColumns = true;
}
else if (trimmed.startsWith('- name:') && inColumns) {
currentColumns.push({ name: trimmed.split(':')[1].trim() });
}
}
if (currentModel) {
if (currentColumns.length > 0) {
currentModel.columns = currentColumns;
}
result.models.push(currentModel);
}
return result;
}
catch (error) {
return { version: 2, models: [] };
}
}
generateBasicYaml(data) {
let yaml = `version: ${data.version}\n\nmodels:\n`;
for (const model of data.models) {
yaml += ` - name: ${model.name}\n`;
if (model.description) {
yaml += ` description: ${model.description}\n`;
}
if (model.tests && model.tests.length > 0) {
yaml += ' tests:\n';
for (const test of model.tests) {
yaml += ` - ${test}\n`;
}
}
if (model.columns && model.columns.length > 0) {
yaml += ' columns:\n';
for (const column of model.columns) {
yaml += ` - name: ${column.name}\n`;
if (column.description) {
yaml += ` description: ${column.description}\n`;
}
if (column.tests && column.tests.length > 0) {
yaml += ' tests:\n';
for (const test of column.tests) {
yaml += ` - ${test}\n`;
}
}
}
}
yaml += '\n';
}
return yaml;
}
async executeDbtCommand(args) {
return new Promise((resolve, reject) => {
const dbt = (0, child_process_1.spawn)('dbt', args, {
cwd: this.config.projectPath,
env: {
...process.env,
DBT_PROFILES_DIR: this.config.profilesDir,
},
stdio: 'inherit',
});
dbt.on('close', (code) => {
if (code === 0) {
resolve();
}
else {
reject(new Error(`dbt command failed with exit code ${code}`));
}
});
dbt.on('error', (error) => {
reject(new Error(`Failed to execute dbt command: ${error instanceof Error ? error.message : String(error)}`));
});
});
}
}
exports.DbtManager = DbtManager;
//# sourceMappingURL=index.js.map