@gftdcojp/gftd-cli
Version:
GFTD Schema Management CLI - Drizzle-like schema management for Kafka and ksqlDB
351 lines • 15 kB
JavaScript
;
/**
* GFTD Schema Manager - Core schema management with organization support
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SchemaManager = void 0;
const axios_1 = __importDefault(require("axios"));
const fs = __importStar(require("fs"));
const types_1 = require("../types");
const YAML = require('yaml');
class SchemaManager {
constructor(config) {
this.config = config;
this.naming = new types_1.NamingConvention();
this.credentials = Buffer.from(`${config.ksqlDB?.apiKey}:${config.ksqlDB?.apiSecret}`).toString('base64');
this.kafkaUrl = config.kafka?.endpoint || '';
this.ksqlUrl = `${config.ksqlDB?.endpoint}/ksql`;
}
/**
* スキーマ定義ファイルを読み込み
*/
loadSchema(schemaPath) {
const configPath = schemaPath || this.config.schemaPath || './gftd.schema.yml';
if (!fs.existsSync(configPath)) {
throw new Error(`Schema file not found: ${configPath}`);
}
const content = fs.readFileSync(configPath, 'utf8');
const schema = YAML.parse(content);
// orgId とプロジェクト名の一致確認
if (schema.orgId !== this.config.orgId) {
throw new Error(`Schema orgId (${schema.orgId}) does not match config orgId (${this.config.orgId})`);
}
if (schema.project !== this.config.project) {
throw new Error(`Schema project (${schema.project}) does not match config project (${this.config.project})`);
}
return schema;
}
/**
* スキーマの検証
*/
validateSchema(schema) {
const errors = [];
const warnings = [];
// 必須フィールドの確認
if (!schema.orgId)
errors.push('orgId is required');
if (!schema.project)
errors.push('project is required');
if (!schema.version)
errors.push('version is required');
// トピック名の重複チェック
const topicNames = Object.keys(schema.topics);
const duplicateTopics = topicNames.filter((name, index) => topicNames.indexOf(name) !== index);
if (duplicateTopics.length > 0) {
errors.push(`Duplicate topic names: ${duplicateTopics.join(', ')}`);
}
// ストリーム名の重複チェック
const streamNames = Object.keys(schema.streams);
const duplicateStreams = streamNames.filter((name, index) => streamNames.indexOf(name) !== index);
if (duplicateStreams.length > 0) {
errors.push(`Duplicate stream names: ${duplicateStreams.join(', ')}`);
}
// テーブル名の重複チェック
const tableNames = Object.keys(schema.tables);
const duplicateTables = tableNames.filter((name, index) => tableNames.indexOf(name) !== index);
if (duplicateTables.length > 0) {
errors.push(`Duplicate table names: ${duplicateTables.join(', ')}`);
}
// ストリームとトピックの関連チェック
Object.entries(schema.streams).forEach(([streamKey, streamConfig]) => {
const referencedTopic = streamKey; // ストリームキーと同名のトピックを参照
if (!schema.topics[referencedTopic]) {
errors.push(`Stream '${streamKey}' references non-existent topic '${referencedTopic}'`);
}
});
// テーブルとストリームの関連チェック
Object.entries(schema.tables).forEach(([tableKey, tableConfig]) => {
const referencedStream = tableKey; // テーブルキーと同名のストリームを参照
if (!schema.streams[referencedStream]) {
errors.push(`Table '${tableKey}' references non-existent stream '${referencedStream}'`);
}
});
return {
isValid: errors.length === 0,
errors,
warnings
};
}
/**
* マイグレーションの生成
*/
generateMigrations(schema) {
const migrations = [];
const timestamp = Date.now();
// 1. トピック作成マイグレーション
Object.entries(schema.topics).forEach(([topicKey, topicConfig], index) => {
const fullTopicName = this.naming.generateTopicName(schema.orgId, topicKey);
const fullConfig = {
name: fullTopicName,
...topicConfig
};
migrations.push({
id: `${timestamp}_${index.toString().padStart(3, '0')}_create_topic_${topicKey}`,
name: `Create topic ${topicKey}`,
timestamp: timestamp + index,
applied: false,
sql: this.generateCreateTopicSQL(fullConfig),
type: 'TOPIC',
orgId: schema.orgId
});
});
// 2. ストリーム作成マイグレーション
Object.entries(schema.streams).forEach(([streamKey, streamConfig], index) => {
const migrationIndex = Object.keys(schema.topics).length + index;
const fullStreamName = this.naming.generateStreamName(schema.orgId, streamKey);
const fullTopicName = this.naming.generateTopicName(schema.orgId, streamKey);
const fullConfig = {
name: fullStreamName,
kafkaTopic: fullTopicName,
...streamConfig
};
migrations.push({
id: `${timestamp}_${migrationIndex.toString().padStart(3, '0')}_create_stream_${streamKey}`,
name: `Create stream ${streamKey}`,
timestamp: timestamp + migrationIndex,
applied: false,
sql: this.generateCreateStreamSQL(fullConfig),
type: 'STREAM',
orgId: schema.orgId
});
});
// 3. テーブル作成マイグレーション
Object.entries(schema.tables).forEach(([tableKey, tableConfig], index) => {
const migrationIndex = Object.keys(schema.topics).length +
Object.keys(schema.streams).length +
index;
const fullTableName = this.naming.generateTableName(schema.orgId, tableKey);
const fullStreamName = this.naming.generateStreamName(schema.orgId, tableKey);
const fullConfig = {
name: fullTableName,
sourceStream: fullStreamName,
...tableConfig
};
migrations.push({
id: `${timestamp}_${migrationIndex.toString().padStart(3, '0')}_create_table_${tableKey}`,
name: `Create table ${tableKey}`,
timestamp: timestamp + migrationIndex,
applied: false,
sql: this.generateCreateTableSQL(fullConfig),
type: 'TABLE',
orgId: schema.orgId
});
});
return migrations;
}
/**
* マイグレーションの実行
*/
async runMigrations(options = {}) {
const schema = this.loadSchema();
const validation = this.validateSchema(schema);
if (!validation.isValid) {
throw new Error(`Schema validation failed:\n${validation.errors.join('\n')}`);
}
const migrations = this.generateMigrations(schema);
if (options.dryRun) {
console.log('🔍 Dry run - SQL statements that would be executed:');
migrations.forEach(migration => {
console.log(`\n--- ${migration.name} ---`);
console.log(migration.sql);
});
return;
}
console.log(`🚀 Running ${migrations.length} migrations for ${schema.orgId}...`);
for (const migration of migrations) {
const success = await this.executeMigration(migration);
if (!success && !options.force) {
throw new Error(`Migration failed: ${migration.name}`);
}
// 実行間隔を空ける
await new Promise(resolve => setTimeout(resolve, 1000));
}
console.log('✅ All migrations completed successfully');
}
/**
* 単一マイグレーションの実行
*/
async executeMigration(migration) {
try {
console.log(`⚡ Executing: ${migration.name}`);
const response = await axios_1.default.post(this.ksqlUrl, { ksql: migration.sql }, {
headers: {
'Accept': 'application/vnd.ksql.v1+json',
'Content-Type': 'application/vnd.ksql.v1+json',
'Authorization': `Basic ${this.credentials}`
}
});
if (response.status === 200) {
console.log(`✅ Completed: ${migration.name}`);
return true;
}
else {
console.error(`❌ Failed: ${migration.name}`, response.data);
return false;
}
}
catch (error) {
console.error(`❌ Error in ${migration.name}:`, error.message);
return false;
}
}
/**
* SQL生成メソッド群
*/
generateCreateTopicSQL(topic) {
// Note: Kafka topicの作成はksqlDBではサポートされていないため、
// 実際の実装ではKafka Admin APIを使用する必要があります
return `-- Topic creation is handled via Kafka Admin API\n-- Topic: ${topic.name}`;
}
generateCreateStreamSQL(stream) {
const columns = stream.columns
.map(col => {
const keyPart = col.key ? ' KEY' : '';
return `${col.name} ${col.type}${keyPart}`;
})
.join(',\n ');
return `
CREATE STREAM ${stream.name} (
${columns}
) WITH (
KAFKA_TOPIC='${stream.kafkaTopic}',
VALUE_FORMAT='${stream.valueFormat}'${stream.keyFormat ? `,\n KEY_FORMAT='${stream.keyFormat}'` : ''}
);
`.trim();
}
generateCreateTableSQL(table) {
if (table.sourceStream && table.aggregation) {
return `
CREATE TABLE ${table.name} AS
${table.aggregation.replace(/\{sourceStream\}/g, table.sourceStream)};
`.trim();
}
if (table.kafkaTopic && table.columns) {
const columns = table.columns
.map(col => {
const pkPart = col.primaryKey ? ' PRIMARY KEY' : '';
return `${col.name} ${col.type}${pkPart}`;
})
.join(',\n ');
return `
CREATE TABLE ${table.name} (
${columns}
) WITH (
KAFKA_TOPIC='${table.kafkaTopic}',
VALUE_FORMAT='${table.valueFormat || 'JSON'}'${table.keyFormat ? `,\n KEY_FORMAT='${table.keyFormat}'` : ''}
);
`.trim();
}
throw new Error(`Invalid table configuration for ${table.name}`);
}
/**
* 現在の状態確認
*/
async status() {
try {
console.log(`🔍 Checking schema status for org: ${this.config.orgId}...`);
// ストリーム一覧を確認
const streamsResponse = await axios_1.default.post(this.ksqlUrl, { ksql: 'SHOW STREAMS;' }, {
headers: {
'Accept': 'application/vnd.ksql.v1+json',
'Content-Type': 'application/vnd.ksql.v1+json',
'Authorization': `Basic ${this.credentials}`
}
});
console.log('📊 Current streams:');
if (streamsResponse.data?.[0]?.streams) {
const orgStreams = streamsResponse.data[0].streams.filter((stream) => stream.name.startsWith(`${this.config.orgId?.toUpperCase() || 'ORG'}_`));
if (orgStreams.length > 0) {
orgStreams.forEach((stream) => {
console.log(` - ${stream.name} (topic: ${stream.topic})`);
});
}
else {
console.log(' No streams found for this organization');
}
}
// テーブル一覧を確認
const tablesResponse = await axios_1.default.post(this.ksqlUrl, { ksql: 'SHOW TABLES;' }, {
headers: {
'Accept': 'application/vnd.ksql.v1+json',
'Content-Type': 'application/vnd.ksql.v1+json',
'Authorization': `Basic ${this.credentials}`
}
});
console.log('📋 Current tables:');
if (tablesResponse.data?.[0]?.tables) {
const orgTables = tablesResponse.data[0].tables.filter((table) => table.name.startsWith(`${this.config.orgId?.toUpperCase() || 'ORG'}_`));
if (orgTables.length > 0) {
orgTables.forEach((table) => {
console.log(` - ${table.name} (topic: ${table.topic})`);
});
}
else {
console.log(' No tables found for this organization');
}
}
}
catch (error) {
console.error('❌ Error checking status:', error.message);
throw error;
}
}
}
exports.SchemaManager = SchemaManager;
//# sourceMappingURL=schema-manager.js.map