UNPKG

@gftdcojp/ksqldb-orm

Version:

ksqldb-orm - Server-Side TypeScript ORM for ksqlDB with enterprise security extensions

753 lines 30.5 kB
"use strict"; /** * ksqlDB コネクション - REST API 経由でクエリを実行、DDL/DML を発行 */ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); exports.KsqlDbClient = void 0; exports.initializeKsqlDbClient = initializeKsqlDbClient; exports.transformArrayRowsToObjects = transformArrayRowsToObjects; exports.executeQuery = executeQuery; exports.executePullQuery = executePullQuery; exports.executePushQuery = executePushQuery; exports.executeAnyQuery = executeAnyQuery; exports.executeDDL = executeDDL; exports.executeDDLWithOptions = executeDDLWithOptions; exports.createStreamWithInfinityRetention = createStreamWithInfinityRetention; exports.createTableWithInfinityRetention = createTableWithInfinityRetention; exports.listStreams = listStreams; exports.listTables = listTables; exports.listTopics = listTopics; exports.describeStream = describeStream; exports.describeTable = describeTable; exports.getClientConfig = getClientConfig; exports.isConnected = isConnected; exports.closeClient = closeClient; exports.executePullQueryEnhanced = executePullQueryEnhanced; const axios_1 = __importDefault(require("axios")); const logger_1 = require("./utils/logger"); const column_name_transform_1 = require("./utils/column-name-transform"); let ksqlClient = null; let config = null; /** * ksqlDB クライアントを初期化 */ function initializeKsqlDbClient(ksqlConfig) { config = ksqlConfig; const headers = { 'Content-Type': 'application/vnd.ksql.v1+json; charset=utf-8', ...ksqlConfig.headers, }; // 認証情報がある場合は Authorization ヘッダーを追加 if (ksqlConfig.apiKey && ksqlConfig.apiSecret) { const credentials = btoa(`${ksqlConfig.apiKey}:${ksqlConfig.apiSecret}`); headers['Authorization'] = `Basic ${credentials}`; } ksqlClient = axios_1.default.create({ baseURL: ksqlConfig.url, headers, timeout: 30000, // 30秒のタイムアウト }); } /** * 配列レスポンスをオブジェクトに変換する関数(強化版) * @param rows - データ行の配列 * @param columnNames - カラム名の配列 * @param transformConfig - 列名変換設定(オプション、指定されない場合はグローバル設定を使用) * @returns オブジェクトの配列 */ function transformArrayRowsToObjects(rows, columnNames, transformConfig) { if (!rows || !Array.isArray(rows) || rows.length === 0) { return []; } if (!columnNames || !Array.isArray(columnNames) || columnNames.length === 0) { logger_1.log.warn('transformArrayRowsToObjects: columnNames is empty, returning original rows'); return rows; } // 列名変換設定を決定(優先順位: 引数 > グローバル設定 > デフォルト) const finalTransformConfig = transformConfig || config?.columnNameTransform || { strategy: 'camelCase' }; // 列名を適切な形式に変換 const transformedColumnNames = (0, column_name_transform_1.transformColumnNamesFromKsqlDb)(columnNames, finalTransformConfig); logger_1.log.debug('Column name transformation:', { original: columnNames, transformed: transformedColumnNames, config: finalTransformConfig }); return rows.map((row) => { const obj = {}; transformedColumnNames.forEach((columnName, index) => { obj[columnName] = row[index]; }); return obj; }); } /** * DDL/DML文を実行(CREATE, INSERT, UPDATE, DELETE, DROP など) * /ksql エンドポイントを使用 */ async function executeQuery(sql) { if (!ksqlClient) { throw new Error('ksqlDB client is not initialized. Call initializeKsqlDbClient() first.'); } try { const response = await ksqlClient.post('/ksql', { ksql: sql, streamsProperties: {}, }); return response.data; } catch (error) { if (axios_1.default.isAxiosError(error)) { throw new Error(`ksqlDB query failed: ${error.response?.data?.message || error.message}`); } throw error; } } /** * Pull Query を実行(一度だけ結果を取得するSELECT文) * /query-stream エンドポイントを使用 * @param sql - 実行するSQL文 * @param options - Pull Queryのオプション */ async function executePullQuery(sql, options = {}) { if (!ksqlClient) { throw new Error('ksqlDB client is not initialized. Call initializeKsqlDbClient() first.'); } const { format = 'object' } = options; logger_1.log.debug(`executePullQuery - SQL: ${sql}`); logger_1.log.debug(`executePullQuery - Options:`, options); logger_1.log.debug(`executePullQuery - Config:`, config); try { const response = await ksqlClient.post('/query-stream', { sql, properties: {}, }, { headers: { 'Content-Type': 'application/vnd.ksql.v1+json', 'Accept': 'application/vnd.ksqlapi.delimited.v1', }, }); logger_1.log.debug(`executePullQuery - Response status: ${response.status}`); logger_1.log.debug(`executePullQuery - Response headers:`, response.headers); // レスポンスを解析 if (typeof response.data === 'string') { const lines = response.data.split('\n').filter((line) => line.trim()); const results = []; let header = null; logger_1.log.debug(`executePullQuery - Response lines count: ${lines.length}`); for (const line of lines) { try { const parsed = JSON.parse(line); if (!header && parsed.columnNames) { header = parsed; logger_1.log.debug(`executePullQuery - Header:`, header); } else if (Array.isArray(parsed)) { results.push(parsed); } } catch (parseError) { logger_1.log.warn(`executePullQuery - Failed to parse line: ${line}`, parseError); } } logger_1.log.debug(`executePullQuery - Results count: ${results.length}`); // 結果の形式を選択 if (format === 'object' && header?.columnNames) { const transformedData = transformArrayRowsToObjects(results, header.columnNames); return { header, data: transformedData, columnNames: header.columnNames, queryId: header.queryId }; } else { return { header, data: results, columnNames: header?.columnNames, queryId: header?.queryId }; } } logger_1.log.debug(`executePullQuery - Raw response data:`, response.data); return response.data; } catch (error) { logger_1.log.error(`executePullQuery failed:`, error); if (axios_1.default.isAxiosError(error)) { const errorDetails = { status: error.response?.status, statusText: error.response?.statusText, data: error.response?.data, headers: error.response?.headers, config: { url: error.config?.url, method: error.config?.method, headers: error.config?.headers, }, sql: sql, ksqlDbConfig: config, }; logger_1.log.error(`executePullQuery - Axios error details:`, errorDetails); let errorMessage = 'Unknown error'; if (error.response?.data) { if (typeof error.response.data === 'string') { errorMessage = error.response.data; } else if (error.response.data.message) { errorMessage = error.response.data.message; } else if (error.response.data.error_code) { errorMessage = `${error.response.data.error_code}: ${error.response.data.message || 'Unknown error'}`; } else { errorMessage = JSON.stringify(error.response.data); } } else if (error.message) { errorMessage = error.message; } // ストリーム vs テーブルの問題を特定しやすくする if (errorMessage.includes('stream') || errorMessage.includes('table')) { logger_1.log.error(`executePullQuery - Possible stream/table issue. SQL: ${sql}`); logger_1.log.error(`executePullQuery - Remember: Pull queries work only on TABLES, not STREAMS`); } throw new Error(`ksqlDB pull query failed (${error.response?.status || 'unknown'}): ${errorMessage}`); } logger_1.log.error(`executePullQuery - Non-axios error:`, error); throw error; } } /** * Push Query を実行(継続的にデータを受信するSELECT文) * /query-stream エンドポイントでストリーミング */ async function executePushQuery(sql, onData, onError, onComplete) { if (!ksqlClient) { throw new Error('ksqlDB client is not initialized. Call initializeKsqlDbClient() first.'); } let queryId = null; let terminated = false; try { // ksqlDB専用のaxiosインスタンスを作成(ストリーミング用) const streamClient = axios_1.default.create({ baseURL: config?.url, headers: { 'Content-Type': 'application/vnd.ksql.v1+json', 'Accept': 'application/vnd.ksqlapi.delimited.v1', ...(config?.headers || {}), }, timeout: 0, // ストリーミングの場合はタイムアウトなし responseType: 'stream', }); // 認証情報を追加 if (config?.apiKey && config?.apiSecret) { const credentials = btoa(`${config.apiKey}:${config.apiSecret}`); streamClient.defaults.headers['Authorization'] = `Basic ${credentials}`; } const response = await streamClient.post('/query-stream', { sql, properties: {}, }); let header = null; let buffer = ''; response.data.on('data', (chunk) => { if (terminated) return; try { buffer += chunk.toString(); const lines = buffer.split('\n'); // 最後の行が不完全な可能性があるので、一つ残しておく buffer = lines.pop() || ''; for (const line of lines) { if (!line.trim()) continue; try { const data = JSON.parse(line); // ヘッダー情報を保存 if (!header && data.columnNames) { header = data; queryId = data.queryId; } else if (Array.isArray(data)) { // データ行を処理 onData({ header, row: data }); } } catch (parseError) { // 個別行のパースエラーは無視 logger_1.log.warn('Failed to parse line:', { line, parseError }); } } } catch (error) { if (onError && !terminated) { onError(new Error(`Failed to process streaming data: ${error}`)); } } }); response.data.on('error', (error) => { if (onError && !terminated) { onError(error); } }); response.data.on('end', () => { if (onComplete && !terminated) { onComplete(); } }); // クエリ終了関数を返す return { terminate: async () => { if (terminated) return; terminated = true; // ストリームを閉じる if (response.data && response.data.destroy) { response.data.destroy(); } // ksqlDBにクエリ終了を通知 if (queryId) { try { await ksqlClient.post('/close-query', { queryId, }); } catch (error) { // クエリが既に終了している場合は正常なケースとして処理 if (axios_1.default.isAxiosError(error) && error.response?.status === 400) { // 400エラーは通常、クエリが既に存在しないことを意味するため、警告レベルで処理 logger_1.log.debug('Query already terminated or completed:', { queryId }); } else { logger_1.log.warn('Failed to terminate query:', error); } } } } }; } catch (error) { if (axios_1.default.isAxiosError(error)) { // 詳細なエラー情報を取得 const errorMessage = error.response?.data?.message || error.response?.statusText || error.message; const statusCode = error.response?.status || 'unknown'; const ksqlError = new Error(`ksqlDB push query failed (${statusCode}): ${errorMessage}`); if (onError) { onError(ksqlError); } else { throw ksqlError; } } else { if (onError) { onError(error); } else { throw error; } } return { terminate: () => { } }; } } /** * SELECT文かどうかを判定 */ function isSelectStatement(sql) { const trimmed = sql.trim().toUpperCase(); return trimmed.startsWith('SELECT') || trimmed.startsWith('PRINT'); } /** * 汎用クエリ実行(自動でエンドポイントを選択) */ async function executeAnyQuery(sql) { const trimmed = sql.trim().toUpperCase(); // SELECT文の場合は /query-stream を使用 if (isSelectStatement(sql)) { // EMIT CHANGES が含まれている場合はプッシュクエリ if (trimmed.includes('EMIT CHANGES')) { return new Promise((resolve, reject) => { const results = []; let header = null; executePushQuery(sql, (data) => { if (!header) header = data.header; results.push(data.row); }, reject, () => resolve({ header, data: results })); // 5秒後に自動終了(デモ用) setTimeout(() => { resolve({ header, data: results }); }, 5000); }); } else { // プルクエリ return executePullQuery(sql); } } else { // DDL/DML文の場合は /ksql を使用 return executeQuery(sql); } } /** * DDL文を解析してretention設定を自動追加する機能 * CREATE STREAM/TABLE文にretention.ms=-1(infinity)を自動的に追加 */ function enhanceDDLWithInfinityRetention(ddl) { // DDL文を正規化(複数行、余分な空白を除去) const normalizedDDL = ddl.replace(/\s+/g, ' ').trim(); // CREATE STREAM または CREATE TABLE の判定 const isCreateStatement = /^CREATE\s+(STREAM|TABLE)/i.test(normalizedDDL); if (!isCreateStatement) { return ddl; // CREATE文でない場合はそのまま返す } // 既にretention.msが設定されているかチェック if (/['"]?retention\.ms['"]?\s*=/i.test(normalizedDDL)) { logger_1.log.debug('DDL already contains retention.ms setting, skipping auto-enhancement'); return ddl; // 既に設定されている場合はそのまま返す } // WITH句の存在チェック const withClauseMatch = normalizedDDL.match(/\s+WITH\s*\(/i); if (withClauseMatch) { // WITH句が存在する場合、閉じ括弧の前にretention設定を追加 const beforeWith = ddl.substring(0, ddl.toLowerCase().indexOf('with (') + 'with ('.length); const afterWithStart = ddl.toLowerCase().indexOf('with (') + 'with ('.length; const afterWith = ddl.substring(afterWithStart); // 既存の設定があるかチェック(閉じ括弧まで) const closingBraceIndex = afterWith.lastIndexOf(')'); if (closingBraceIndex === -1) { logger_1.log.warn('Invalid WITH clause syntax in DDL, skipping retention enhancement'); return ddl; } const withContent = afterWith.substring(0, closingBraceIndex).trim(); const afterClosingBrace = afterWith.substring(closingBraceIndex); // retention.ms設定を追加 let enhancedWithContent = withContent; if (withContent && !withContent.endsWith(',')) { enhancedWithContent += ','; } enhancedWithContent += "\n 'retention.ms'='-1'"; // infinity設定 const enhancedDDL = beforeWith + enhancedWithContent + afterClosingBrace; logger_1.log.info('Enhanced DDL with infinite retention setting'); return enhancedDDL; } else { // WITH句が存在しない場合、末尾に追加 // セミコロンの直前に追加 const hasSemicolon = ddl.trim().endsWith(';'); const baseDDL = hasSemicolon ? ddl.trim().slice(0, -1) : ddl.trim(); const enhancedDDL = baseDDL + `\n WITH (\n 'retention.ms'='-1'\n )` + (hasSemicolon ? ';' : ''); logger_1.log.info('Enhanced DDL with infinite retention setting (new WITH clause)'); return enhancedDDL; } } /** * DDL文を実行(CREATE STREAM/TABLE など) * 自動的にretention.ms=-1(infinity)を追加する機能付き */ async function executeDDL(ddl) { if (!ksqlClient) { throw new Error('ksqlDB client is not initialized. Call initializeKsqlDbClient() first.'); } try { const enhancedDDL = enhanceDDLWithInfinityRetention(ddl); const response = await ksqlClient.post('/ksql', { ksql: enhancedDDL, streamsProperties: {}, }); if (response.data && response.data[0] && response.data[0].errorMessage) { throw new Error(`DDL execution failed: ${response.data[0].errorMessage.message}`); } return response.data; } catch (error) { if (axios_1.default.isAxiosError(error)) { throw new Error(`ksqlDB DDL failed: ${error.response?.data?.message || error.message}`); } throw error; } } /** * オプション付きDDL文実行 * @param ddl - 実行するDDL文 * @param options - 実行オプション */ async function executeDDLWithOptions(ddl, options = {}) { if (!ksqlClient) { throw new Error('ksqlDB client is not initialized. Call initializeKsqlDbClient() first.'); } try { let enhancedDDL = ddl; // retention設定の処理 if (!options.disableAutoInfinityRetention) { if (options.customRetentionMs !== undefined) { // カスタムretention設定を使用 enhancedDDL = enhanceDDLWithCustomRetention(ddl, options.customRetentionMs); } else { // デフォルトのinfinity設定を使用 enhancedDDL = enhanceDDLWithInfinityRetention(ddl); } } // 追加のWITH設定があれば適用 if (options.additionalWithSettings && Object.keys(options.additionalWithSettings).length > 0) { enhancedDDL = enhanceDDLWithAdditionalSettings(enhancedDDL, options.additionalWithSettings); } const response = await ksqlClient.post('/ksql', { ksql: enhancedDDL, streamsProperties: {}, }); if (response.data && response.data[0] && response.data[0].errorMessage) { throw new Error(`DDL execution failed: ${response.data[0].errorMessage.message}`); } return response.data; } catch (error) { if (axios_1.default.isAxiosError(error)) { throw new Error(`ksqlDB DDL failed: ${error.response?.data?.message || error.message}`); } throw error; } } /** * カスタムretention設定でDDLを拡張 */ function enhanceDDLWithCustomRetention(ddl, retentionMs) { const normalizedDDL = ddl.replace(/\s+/g, ' ').trim(); const isCreateStatement = /^CREATE\s+(STREAM|TABLE)/i.test(normalizedDDL); if (!isCreateStatement) { return ddl; } // 既にretention.msが設定されているかチェック if (/['"]?retention\.ms['"]?\s*=/i.test(normalizedDDL)) { logger_1.log.debug('DDL already contains retention.ms setting, skipping custom retention'); return ddl; } const retentionValue = retentionMs === -1 ? "'-1'" : `'${retentionMs}'`; // WITH句の処理(enhanceDDLWithInfinityRetentionと同様のロジック) const withClauseMatch = normalizedDDL.match(/\s+WITH\s*\(/i); if (withClauseMatch) { const beforeWith = ddl.substring(0, ddl.toLowerCase().indexOf('with (') + 'with ('.length); const afterWithStart = ddl.toLowerCase().indexOf('with (') + 'with ('.length; const afterWith = ddl.substring(afterWithStart); const closingBraceIndex = afterWith.lastIndexOf(')'); if (closingBraceIndex === -1) { logger_1.log.warn('Invalid WITH clause syntax in DDL, skipping retention enhancement'); return ddl; } const withContent = afterWith.substring(0, closingBraceIndex).trim(); const afterClosingBrace = afterWith.substring(closingBraceIndex); let enhancedWithContent = withContent; if (withContent && !withContent.endsWith(',')) { enhancedWithContent += ','; } enhancedWithContent += `\n 'retention.ms'=${retentionValue}`; const enhancedDDL = beforeWith + enhancedWithContent + afterClosingBrace; logger_1.log.info(`Enhanced DDL with custom retention setting: ${retentionMs}ms`); return enhancedDDL; } else { const hasSemicolon = ddl.trim().endsWith(';'); const baseDDL = hasSemicolon ? ddl.trim().slice(0, -1) : ddl.trim(); const enhancedDDL = baseDDL + `\n WITH (\n 'retention.ms'=${retentionValue}\n )` + (hasSemicolon ? ';' : ''); logger_1.log.info(`Enhanced DDL with custom retention setting: ${retentionMs}ms (new WITH clause)`); return enhancedDDL; } } /** * 追加のWITH設定でDDLを拡張 */ function enhanceDDLWithAdditionalSettings(ddl, settings) { const normalizedDDL = ddl.replace(/\s+/g, ' ').trim(); const isCreateStatement = /^CREATE\s+(STREAM|TABLE)/i.test(normalizedDDL); if (!isCreateStatement) { return ddl; } const withClauseMatch = normalizedDDL.match(/\s+WITH\s*\(/i); if (withClauseMatch) { const beforeWith = ddl.substring(0, ddl.toLowerCase().indexOf('with (') + 'with ('.length); const afterWithStart = ddl.toLowerCase().indexOf('with (') + 'with ('.length; const afterWith = ddl.substring(afterWithStart); const closingBraceIndex = afterWith.lastIndexOf(')'); if (closingBraceIndex === -1) { logger_1.log.warn('Invalid WITH clause syntax in DDL, skipping additional settings'); return ddl; } const withContent = afterWith.substring(0, closingBraceIndex).trim(); const afterClosingBrace = afterWith.substring(closingBraceIndex); let enhancedWithContent = withContent; // 追加設定を挿入 Object.entries(settings).forEach(([key, value]) => { // 既に設定されていないかチェック const keyRegex = new RegExp(`['"]?${key.replace('.', '\\.')}['"]?\\s*=`, 'i'); if (!keyRegex.test(withContent)) { if (enhancedWithContent && !enhancedWithContent.endsWith(',')) { enhancedWithContent += ','; } enhancedWithContent += `\n '${key}'='${value}'`; } }); const enhancedDDL = beforeWith + enhancedWithContent + afterClosingBrace; logger_1.log.info('Enhanced DDL with additional WITH settings'); return enhancedDDL; } else { const hasSemicolon = ddl.trim().endsWith(';'); const baseDDL = hasSemicolon ? ddl.trim().slice(0, -1) : ddl.trim(); const settingsStr = Object.entries(settings) .map(([key, value]) => ` '${key}'='${value}'`) .join(',\n'); const enhancedDDL = baseDDL + `\n WITH (\n${settingsStr}\n )` + (hasSemicolon ? ';' : ''); logger_1.log.info('Enhanced DDL with additional WITH settings (new WITH clause)'); return enhancedDDL; } } /** * ストリーム作成ヘルパー - infinite retention設定済み */ async function createStreamWithInfinityRetention(streamName, fields, options = {}) { const { topic = streamName.toLowerCase(), valueFormat = 'JSON', keyField, partitions = 3, replicas = 1, additionalSettings = {} } = options; // フィールド定義を作成 const fieldDefinitions = Object.entries(fields) .map(([name, type]) => `${name} ${type}`) .join(',\n '); // WITH句の設定を作成 const withSettings = { 'kafka_topic': topic, 'value_format': valueFormat, 'partitions': partitions.toString(), 'replicas': replicas.toString(), 'retention.ms': '-1', // infinity ...additionalSettings }; if (keyField) { withSettings['key'] = keyField; } const withClause = Object.entries(withSettings) .map(([key, value]) => ` '${key}'='${value}'`) .join(',\n'); const ddl = ` CREATE STREAM ${streamName} ( ${fieldDefinitions} ) WITH ( ${withClause} );`; logger_1.log.info(`Creating stream ${streamName} with infinite retention`); // disableAutoInfinityRetentionをtrueにして重複を避ける return executeDDLWithOptions(ddl, { disableAutoInfinityRetention: true }); } /** * テーブル作成ヘルパー - infinite retention設定済み */ async function createTableWithInfinityRetention(tableName, sourceStream, selectClause, groupByClause, options = {}) { const { additionalSettings = {} } = options; const defaultSelectClause = selectClause || '*'; const groupByPart = groupByClause ? `GROUP BY ${groupByClause}` : ''; const ddl = ` CREATE TABLE ${tableName} WITH ( 'retention.ms'='-1'${Object.keys(additionalSettings).length > 0 ? ',' : ''} ${Object.entries(additionalSettings).map(([key, value]) => ` '${key}'='${value}'`).join(',\n')} ) AS SELECT ${defaultSelectClause} FROM ${sourceStream} ${groupByPart} EMIT CHANGES;`; logger_1.log.info(`Creating table ${tableName} with infinite retention`); return executeDDLWithOptions(ddl, { disableAutoInfinityRetention: true }); } /** * ストリーム/テーブル一覧を取得 */ async function listStreams() { return executeQuery('LIST STREAMS;'); } async function listTables() { return executeQuery('LIST TABLES;'); } /** * トピック一覧を取得 */ async function listTopics() { return executeQuery('LIST TOPICS;'); } /** * スキーマ情報を取得 */ async function describeStream(streamName) { return executeQuery(`DESCRIBE ${streamName};`); } async function describeTable(tableName) { return executeQuery(`DESCRIBE ${tableName};`); } /** * クライアント設定を取得 */ function getClientConfig() { return config; } /** * 接続状態を確認 */ function isConnected() { return ksqlClient !== null && config !== null; } /** * クライアントを閉じる */ function closeClient() { ksqlClient = null; config = null; } /** * Enhanced Pull Query実行(リジリエンス機能対応) * * 下位互換性を保ちながら、拡張オプションをサポート * 通常のexecutePullQueryと同じ動作だが、将来的にリジリエンス機能を統合 * * @param sql - 実行するSQL文 * @param options - 拡張されたPull Queryオプション */ async function executePullQueryEnhanced(sql, options = {}) { // 拡張オプションを標準オプションに変換 const { retries, backoffStrategy, fallbackToHttp, timeout, ...standardOptions } = options; // 現時点では標準のexecutePullQueryを使用(将来的にリジリエンス機能を統合) if (retries || backoffStrategy || fallbackToHttp || timeout) { logger_1.log.info('Enhanced options detected, but resilience features not yet integrated', { retries, backoffStrategy, fallbackToHttp, timeout }); } return executePullQuery(sql, standardOptions); } exports.KsqlDbClient = { initialize: initializeKsqlDbClient, executeQuery, executePullQuery, executePushQuery, executeDDL, executeDDLWithOptions, createStreamWithInfinityRetention, createTableWithInfinityRetention, listStreams, listTables, listTopics, describeStream, describeTable, isConnected, getConfig: getClientConfig, close: closeClient, executeAnyQuery, executePullQueryEnhanced, transformArrayRowsToObjects, }; //# sourceMappingURL=ksqldb-client.js.map