rcc-configuration
Version:
RCC Configuration Module - Comprehensive configuration management for modular TypeScript applications
1,577 lines (1,569 loc) • 209 kB
JavaScript
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
var rccBasemodule = require('rcc-basemodule');
var fs = require('fs/promises');
var path = require('path');
function _interopNamespaceDefault(e) {
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () { return e[k]; }
});
}
});
}
n.default = e;
return Object.freeze(n);
}
var fs__namespace = /*#__PURE__*/_interopNamespaceDefault(fs);
var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
/**
* Configuration Module Constants
*
* This file contains all constants used by the Configuration module.
* Following RCC anti-hardcoding policy, all configuration values
* are centralized here.
*/
/**
* Configuration System Constants
*/
const CONFIGURATION_SYSTEM_CONSTANTS = {
// Module Information
MODULE_TYPE: 'configuration-system',
MODULE_VERSION: '1.0.0',
MODULE_NAME: 'ConfigurationSystem',
MODULE_DESCRIPTION: 'Central configuration management system',
// Default Configuration
DEFAULT_CONFIG_FILE: 'config.json',
DEFAULT_CONFIG_DIRECTORY: './config',
DEFAULT_BACKUP_DIRECTORY: './config/backups',
DEFAULT_SCHEMA_FILE: 'config-schema.json',
// Validation Constants
MAX_CONFIG_SIZE_MB: 10,
MAX_NESTING_DEPTH: 10,
MAX_ARRAY_LENGTH: 1000,
MAX_STRING_LENGTH: 10000,
// Timeouts and Intervals
DEFAULT_TIMEOUT_MS: 30000,
CONFIG_WATCH_DEBOUNCE_MS: 1000,
BACKUP_INTERVAL_MS: 3600000, // 1 hour
VALIDATION_TIMEOUT_MS: 5000,
// Encryption Constants
DEFAULT_ENCRYPTION_ALGORITHM: 'aes-256-gcm',
KEY_DERIVATION_ITERATIONS: 100000,
IV_LENGTH: 16,
TAG_LENGTH: 16,
// Message Types
MESSAGE_TYPES: {
CONFIG_LOADED: 'config:loaded',
CONFIG_SAVED: 'config:saved',
CONFIG_VALIDATED: 'config:validated',
CONFIG_CHANGED: 'config:changed',
CONFIG_ERROR: 'config:error',
VALIDATION_FAILED: 'config:validation-failed',
BACKUP_CREATED: 'config:backup-created',
ENCRYPTION_ENABLED: 'config:encryption-enabled'
},
// Error Codes
ERROR_CODES: {
INVALID_CONFIG_FORMAT: 'CFG_E001',
VALIDATION_FAILED: 'CFG_E002',
FILE_NOT_FOUND: 'CFG_E003',
PERMISSION_DENIED: 'CFG_E004',
ENCRYPTION_FAILED: 'CFG_E005',
DECRYPTION_FAILED: 'CFG_E006',
BACKUP_FAILED: 'CFG_E007',
SCHEMA_INVALID: 'CFG_E008',
SIZE_LIMIT_EXCEEDED: 'CFG_E009',
TIMEOUT_EXCEEDED: 'CFG_E010',
INITIALIZATION_FAILED: 'CFG_E011',
CONFIG_NOT_LOADED: 'CFG_E012',
CONFIG_UPDATE_FAILED: 'CFG_E013',
WATCH_SETUP_FAILED: 'CFG_E014',
UNSUPPORTED_FORMAT: 'CFG_E015',
EXPORT_FAILED: 'CFG_E016',
IMPORT_FAILED: 'CFG_E017',
DESTRUCTION_FAILED: 'CFG_E018'
}
};
/**
* Config Loader Module Constants
*/
const CONFIG_LOADER_CONSTANTS = {
// Module Information
MODULE_TYPE: 'config-loader',
MODULE_NAME: 'ConfigLoaderModule',
MODULE_DESCRIPTION: 'Loads configuration from various sources',
// Supported Formats
SUPPORTED_FORMATS: ['json', 'yaml', 'yml', 'toml', 'ini', 'env'],
// Source Types
SOURCE_TYPES: {
FILE: 'file',
ENVIRONMENT: 'environment',
REMOTE: 'remote',
DATABASE: 'database',
MEMORY: 'memory'
},
// Default Settings
DEFAULT_ENCODING: 'utf8',
DEFAULT_ENV_PREFIX: 'APP_',
DEFAULT_MERGE_STRATEGY: 'deep',
// HTTP Client Settings
HTTP_TIMEOUT_MS: 15000,
HTTP_RETRY_COUNT: 3,
HTTP_RETRY_DELAY_MS: 1000,
// Cache Settings
CACHE_TTL_MS: 300000, // 5 minutes
MAX_CACHE_SIZE: 50,
// File Watching
WATCH_DEBOUNCE_MS: 500,
WATCH_PERSISTENCE_MS: 1000
};
/**
* Config UI Module Constants
*/
const CONFIG_UI_CONSTANTS = {
// Module Information
MODULE_TYPE: 'config-ui',
MODULE_NAME: 'ConfigUIModule',
MODULE_DESCRIPTION: 'Provides user interface for configuration management',
// UI Components
COMPONENT_TYPES: {
EDITOR: 'editor',
VIEWER: 'viewer',
WIZARD: 'wizard',
VALIDATOR: 'validator',
DIFF: 'diff'
},
// Editor Settings
EDITOR_THEMES: ['light', 'dark', 'auto'],
EDITOR_LANGUAGES: ['json', 'yaml', 'toml'],
DEFAULT_EDITOR_THEME: 'auto',
DEFAULT_EDITOR_LANGUAGE: 'json',
// UI Layout
DEFAULT_SIDEBAR_WIDTH: 300,
DEFAULT_PANEL_HEIGHT: 400,
MIN_EDITOR_WIDTH: 400,
MIN_EDITOR_HEIGHT: 200,
// Auto-save Settings
AUTO_SAVE_DELAY_MS: 2000,
AUTO_BACKUP_INTERVAL_MS: 60000, // 1 minute
// UI Messages
UI_MESSAGES: {
LOADING: 'Loading configuration...',
SAVING: 'Saving configuration...',
VALIDATING: 'Validating configuration...',
SAVED: 'Configuration saved successfully',
VALIDATION_ERROR: 'Configuration validation failed',
LOAD_ERROR: 'Failed to load configuration',
SAVE_ERROR: 'Failed to save configuration'
}
};
/**
* Config Persistence Module Constants
*/
const CONFIG_PERSISTENCE_CONSTANTS = {
// Module Information
MODULE_TYPE: 'config-persistence',
MODULE_NAME: 'ConfigPersistenceModule',
MODULE_DESCRIPTION: 'Manages configuration data persistence',
// Storage Types
STORAGE_TYPES: {
FILE_SYSTEM: 'filesystem',
DATABASE: 'database',
MEMORY: 'memory',
CLOUD: 'cloud',
ENCRYPTED: 'encrypted'
},
// File Operations
FILE_PERMISSIONS: 0o600, // Read/write for owner only
BACKUP_SUFFIX: '.backup',
TEMP_SUFFIX: '.tmp',
LOCK_SUFFIX: '.lock',
// Backup Settings
MAX_BACKUP_COUNT: 10,
BACKUP_COMPRESSION: true,
BACKUP_NAMING_PATTERN: 'YYYY-MM-DD_HH-mm-ss',
// Database Settings
CONNECTION_POOL_SIZE: 5,
QUERY_TIMEOUT_MS: 10000,
TRANSACTION_TIMEOUT_MS: 30000,
// Cloud Storage
CLOUD_TIMEOUT_MS: 30000,
CLOUD_RETRY_COUNT: 3,
CLOUD_CHUNK_SIZE: 1024 * 1024, // 1MB
// Locking
LOCK_TIMEOUT_MS: 5000,
LOCK_RETRY_INTERVAL_MS: 100,
LOCK_MAX_RETRIES: 50
};
/**
* Config Validator Module Constants
*/
const CONFIG_VALIDATOR_CONSTANTS = {
// Module Information
MODULE_TYPE: 'config-validator',
MODULE_NAME: 'ConfigValidatorModule',
MODULE_DESCRIPTION: 'Validates configuration data integrity',
// Validation Types
VALIDATION_TYPES: {
SCHEMA: 'schema',
BUSINESS_RULES: 'business-rules',
DEPENDENCIES: 'dependencies',
SECURITY: 'security',
PERFORMANCE: 'performance'
},
// Schema Formats
SCHEMA_FORMATS: ['json-schema', 'ajv', 'joi', 'yup'],
DEFAULT_SCHEMA_FORMAT: 'json-schema',
// Validation Levels
VALIDATION_LEVELS: {
STRICT: 'strict',
NORMAL: 'normal',
LOOSE: 'loose'
},
// Security Validation
SECURITY_PATTERNS: {
SQL_INJECTION: /(\b(SELECT|INSERT|UPDATE|DELETE|DROP|CREATE|ALTER|EXEC|UNION)\b)/i,
XSS_PATTERN: /<script[^>]*>.*?<\/script>/gi,
PATH_TRAVERSAL: /\.\.[\\/]/,
COMMAND_INJECTION: /[;&|`$(){}[\]]/
},
// Performance Limits
MAX_VALIDATION_TIME_MS: 5000,
MAX_RECURSIVE_DEPTH: 20,
MAX_PROPERTY_COUNT: 10000,
// Default Rules
DEFAULT_RULES: {
REQUIRED_FIELDS: ['name', 'version'],
FORBIDDEN_FIELDS: ['__proto__', 'constructor', 'prototype'],
MAX_OBJECT_DEPTH: 10,
MAX_ARRAY_SIZE: 1000
}
};
/**
* Common Configuration Constants
*/
const COMMON_CONSTANTS = {
// Logging Levels
LOG_LEVELS: {
TRACE: 'trace',
DEBUG: 'debug',
INFO: 'info',
WARN: 'warn',
ERROR: 'error'
},
// Event Types
EVENT_TYPES: {
MODULE_INITIALIZED: 'module:initialized',
MODULE_CONFIGURED: 'module:configured',
MODULE_DESTROYED: 'module:destroyed',
CONNECTION_ESTABLISHED: 'connection:established',
CONNECTION_CLOSED: 'connection:closed',
DATA_RECEIVED: 'data:received',
DATA_SENT: 'data:sent',
ERROR_OCCURRED: 'error:occurred'
},
// Status Codes
STATUS_CODES: {
SUCCESS: 200,
CREATED: 201,
ACCEPTED: 202,
NO_CONTENT: 204,
BAD_REQUEST: 400,
UNAUTHORIZED: 401,
FORBIDDEN: 403,
NOT_FOUND: 404,
CONFLICT: 409,
INTERNAL_ERROR: 500,
NOT_IMPLEMENTED: 501,
SERVICE_UNAVAILABLE: 503
},
// Default Values
DEFAULT_TIMEOUT: 30000,
DEFAULT_RETRY_COUNT: 3,
DEFAULT_BATCH_SIZE: 100,
DEFAULT_BUFFER_SIZE: 8192
};
/**
* 配置加载器
*
* 负责配置文件的加载和保存
*/
/**
* 配置加载器类
*/
class ConfigLoader {
constructor() {
this.initialized = false;
}
/**
* 初始化加载器
*/
async initialize() {
if (this.initialized) {
return;
}
this.initialized = true;
console.log('ConfigLoader initialized successfully');
}
/**
* 加载配置文件
*/
async loadConfig(configPath) {
try {
// 检查文件是否存在
await fs__namespace.access(configPath);
// 读取文件内容
const content = await fs__namespace.readFile(configPath, 'utf-8');
// 解析JSON
const config = JSON.parse(content);
console.log(`Configuration loaded from ${configPath}`);
return config;
}
catch (error) {
const nodeError = error;
if (nodeError.code === 'ENOENT') {
console.log(`Configuration file not found at ${configPath}, creating empty config`);
return this.createEmptyConfig();
}
else if (nodeError instanceof SyntaxError) {
// 如果是JSON解析错误，重新抛出错误
console.error(`Invalid JSON in configuration file ${configPath}:`, error);
throw new Error(`Invalid JSON in configuration file ${configPath}: ${error.message}`);
}
console.error(`Failed to load configuration from ${configPath}:`, error);
throw error;
}
}
/**
* 保存配置文件
*/
async saveConfig(config, configPath) {
try {
// 确保目录存在
const dir = path__namespace.dirname(configPath);
await fs__namespace.mkdir(dir, { recursive: true });
// 格式化并保存配置
const content = JSON.stringify(config, null, 2);
await fs__namespace.writeFile(configPath, content, 'utf-8');
console.log(`Configuration saved to ${configPath}`);
}
catch (error) {
console.error(`Failed to save configuration to ${configPath}:`, error);
throw error;
}
}
/**
* 创建空配置
*/
createEmptyConfig() {
const now = new Date().toISOString();
return {
version: '1.0.0',
providers: {},
virtualModels: {},
createdAt: now,
updatedAt: now
};
}
/**
* 检查配置文件是否存在
*/
async configFileExists(configPath) {
try {
await fs__namespace.access(configPath);
return true;
}
catch {
return false;
}
}
/**
* 获取配置文件信息
*/
async getConfigFileInfo(configPath) {
try {
const stats = await fs__namespace.stat(configPath);
return {
exists: true,
size: stats.size,
modified: stats.mtime
};
}
catch {
return { exists: false };
}
}
/**
* 销毁加载器
*/
async destroy() {
this.initialized = false;
console.log('ConfigLoader destroyed successfully');
}
}
/**
* 配置解析器
*
* 负责将原始配置数据解析为标准化的ConfigData结构
*/
/**
* 配置解析器类
*/
class ConfigParser {
constructor() {
this.initialized = false;
}
/**
* 初始化解析器
*/
async initialize() {
if (this.initialized) {
return;
}
this.initialized = true;
console.log('ConfigParser initialized successfully');
}
/**
* 解析配置数据
*/
async parseConfig(rawData) {
try {
// 解析基本配置信息
const config = {
version: rawData.version || '1.0.0',
providers: {},
virtualModels: {},
createdAt: rawData.createdAt || new Date().toISOString(),
updatedAt: rawData.updatedAt || new Date().toISOString()
};
// 解析供应商配置
if (rawData.providers) {
config.providers = this.parseProviders(rawData.providers);
}
// 解析虚拟模型配置
if (rawData.virtualModels) {
config.virtualModels = this.parseVirtualModels(rawData.virtualModels);
}
// 更新时间戳
config.updatedAt = new Date().toISOString();
console.log('Configuration parsed successfully');
return config;
}
catch (error) {
console.error('Failed to parse configuration:', error);
throw error;
}
}
/**
* 解析供应商配置
*/
parseProviders(rawProviders) {
const providers = {};
for (const [providerId, rawProvider] of Object.entries(rawProviders)) {
if (typeof rawProvider !== 'object' || rawProvider === null) {
continue;
}
const provider = {
id: providerId,
name: rawProvider.name || providerId,
type: rawProvider.type || 'unknown',
endpoint: rawProvider.endpoint,
models: {},
auth: {
type: rawProvider.auth?.type || 'api-key',
keys: this.extractApiKeys(rawProvider)
}
};
// 解析模型配置
if (rawProvider.models) {
provider.models = this.parseModels(rawProvider.models);
}
providers[providerId] = provider;
}
return providers;
}
/**
* 提取API密钥
*
* @param rawProvider 原始供应商配置
* @returns API密钥数组
*/
extractApiKeys(rawProvider) {
// 首先检查auth.keys字段
if (Array.isArray(rawProvider.auth?.keys)) {
return rawProvider.auth.keys;
}
// 然后检查api_key字段（兼容旧格式）
if (Array.isArray(rawProvider.api_key)) {
return rawProvider.api_key;
}
// 检查单个api_key字段
if (typeof rawProvider.api_key === 'string') {
return [rawProvider.api_key];
}
// 返回空数组
return [];
}
/**
* 解析模型配置
*/
parseModels(rawModels) {
const models = {};
for (const [modelId, rawModel] of Object.entries(rawModels)) {
if (typeof rawModel !== 'object' || rawModel === null) {
continue;
}
const model = {
id: modelId,
name: rawModel.name || modelId,
contextLength: rawModel.contextLength,
supportsFunctions: rawModel.supportsFunctions
};
models[modelId] = model;
}
return models;
}
/**
* 解析虚拟模型配置
*/
parseVirtualModels(rawVirtualModels) {
const virtualModels = {};
for (const [vmId, rawVm] of Object.entries(rawVirtualModels)) {
if (typeof rawVm !== 'object' || rawVm === null) {
continue;
}
// 处理targets数组
let targets = [];
if (Array.isArray(rawVm.targets)) {
targets = rawVm.targets.map((target) => ({
providerId: target.providerId || '',
modelId: target.modelId || '',
keyIndex: target.keyIndex || 0
}));
}
else if (rawVm.targetProvider && rawVm.targetModel) {
// 兼容旧格式，转换为新格式
targets = [{
providerId: rawVm.targetProvider || '',
modelId: rawVm.targetModel || '',
keyIndex: rawVm.keyIndex || 0
}];
}
else {
// 默认空目标
targets = [{
providerId: '',
modelId: '',
keyIndex: 0
}];
}
const virtualModel = {
id: vmId,
targets: targets,
enabled: rawVm.enabled !== false,
priority: rawVm.priority || 1
};
virtualModels[vmId] = virtualModel;
}
return virtualModels;
}
/**
* 从文件解析配置
*
* @param configPath 配置文件路径
* @param options 预处理选项
* @returns 解析后的配置数据
*/
async parseConfigFromFile(configPath, options) {
try {
// 设置默认选项
const opts = {
substituteEnvVars: true,
processTemplates: true,
validateData: true,
enableCaching: true,
...options
};
// 步骤1: 读取文件
let rawData = await this.readFile(configPath);
// 步骤2: 预处理数据
rawData = await this.preprocessConfig(rawData, opts);
// 步骤3: 解析配置（使用现有逻辑）
const config = await this.parseConfig(rawData);
console.log(`Configuration parsed successfully from ${configPath}`);
return config;
}
catch (error) {
console.error(`Failed to parse configuration from ${configPath}:`, error);
throw error;
}
}
/**
* 预处理配置数据
*
* @param rawData 原始配置数据
* @param options 预处理选项
* @returns 预处理后的数据
*/
async preprocessConfig(rawData, options) {
const opts = {
substituteEnvVars: true,
processTemplates: true,
validateData: true,
...options
};
let processedData = rawData;
// 步骤1: 环境变量替换
if (opts.substituteEnvVars) {
processedData = this.substituteEnvVars(processedData);
}
// 步骤2: 模板处理
if (opts.processTemplates) {
processedData = this.processTemplates(processedData);
}
// 步骤3: 自定义处理器
if (opts.customProcessors && opts.customProcessors.length > 0) {
processedData = this.applyCustomProcessors(processedData, opts.customProcessors);
}
// 步骤4: 验证
if (opts.validateData) {
this.validatePreprocessedData(processedData);
}
return processedData;
}
/**
* 翻译配置
*
* @param config 配置数据
* @param locale 语言环境
* @returns 翻译后的配置数据
*/
async translateConfig(config, locale) {
// 基本翻译支持
if (locale) {
console.log(`Translation to locale ${locale} requested but not implemented`);
// 在实际实现中，这里会进行键值翻译
// 例如：将配置中的英文键翻译为其他语言
// 这可以基于翻译资源文件或外部翻译服务
}
return config;
}
/**
* 读取配置文件
*
* @param configPath 配置文件路径
* @returns 解析后的文件内容
*/
async readFile(configPath) {
try {
// 检查文件是否存在
await fs__namespace.access(configPath);
// 读取文件内容
const content = await fs__namespace.readFile(configPath, 'utf-8');
// 根据文件扩展名解析
if (configPath.endsWith('.json')) {
return JSON.parse(content);
}
else if (configPath.endsWith('.yaml') || configPath.endsWith('.yml')) {
// YAML支持需要额外的包
throw new Error('YAML support not implemented');
}
else {
// 默认为JSON
return JSON.parse(content);
}
}
catch (error) {
if (error.code === 'ENOENT') {
throw new Error(`Configuration file not found: ${configPath}`);
}
throw new Error(`Failed to read configuration file ${configPath}: ${error}`);
}
}
/**
* 环境变量替换
*
* @param data 配置数据
* @returns 替换环境变量后的数据
*/
substituteEnvVars(data) {
if (typeof data === 'string') {
return data.replace(/\$\{([^}]+)\}/g, (match, envVar) => {
return process.env[envVar] || match;
});
}
else if (Array.isArray(data)) {
return data.map(item => this.substituteEnvVars(item));
}
else if (typeof data === 'object' && data !== null) {
const result = {};
for (const [key, value] of Object.entries(data)) {
result[key] = this.substituteEnvVars(value);
}
return result;
}
return data;
}
/**
* 模板处理
*
* @param data 配置数据
* @returns 处理模板后的数据
*/
processTemplates(data) {
// 基本模板处理
// 支持简单的变量插值: {{variable}}
if (typeof data === 'string') {
return data.replace(/\{\{([^}]+)\}\}/g, (match, variable) => {
// 简单的变量查找，可以从环境变量或预定义变量中获取
return process.env[variable.trim()] || match;
});
}
else if (Array.isArray(data)) {
return data.map(item => this.processTemplates(item));
}
else if (typeof data === 'object' && data !== null) {
const result = {};
for (const [key, value] of Object.entries(data)) {
result[key] = this.processTemplates(value);
}
return result;
}
return data;
}
/**
* 验证预处理的数据
*
* @param data 预处理后的数据
* @returns 验证是否通过
*/
validatePreprocessedData(data) {
// 基本验证检查
if (!data || typeof data !== 'object') {
throw new Error('Configuration data must be an object');
}
// 检查必需字段
if (data.providers !== undefined && typeof data.providers !== 'object') {
throw new Error('Configuration providers must be an object');
}
if (data.virtualModels !== undefined && typeof data.virtualModels !== 'object') {
throw new Error('Configuration virtualModels must be an object');
}
// 验证供应商配置结构
if (data.providers) {
for (const [providerId, provider] of Object.entries(data.providers)) {
if (typeof provider !== 'object' || provider === null) {
throw new Error(`Provider ${providerId} must be an object`);
}
// 检查必需字段
if (!provider.name) {
console.warn(`Provider ${providerId} missing name field`);
}
if (!provider.type) {
console.warn(`Provider ${providerId} missing type field`);
}
// 验证模型结构
if (provider.models !== undefined && typeof provider.models !== 'object') {
throw new Error(`Provider ${providerId} models must be an object`);
}
if (provider.models) {
for (const [modelId, model] of Object.entries(provider.models)) {
if (typeof model !== 'object' || model === null) {
throw new Error(`Model ${modelId} in provider ${providerId} must be an object`);
}
if (!model.name) {
console.warn(`Model ${modelId} in provider ${providerId} missing name field`);
}
}
}
}
}
// 验证虚拟模型配置结构
if (data.virtualModels) {
for (const [vmId, vm] of Object.entries(data.virtualModels)) {
if (typeof vm !== 'object' || vm === null) {
throw new Error(`Virtual model ${vmId} must be an object`);
}
// 检查targets数组
if (vm.targets !== undefined && !Array.isArray(vm.targets)) {
throw new Error(`Virtual model ${vmId} targets must be an array`);
}
if (vm.targets) {
for (let i = 0; i < vm.targets.length; i++) {
const target = vm.targets[i];
if (typeof target !== 'object' || target === null) {
throw new Error(`Virtual model ${vmId} target ${i} must be an object`);
}
if (!target.providerId) {
console.warn(`Virtual model ${vmId} target ${i} missing providerId`);
}
if (!target.modelId) {
console.warn(`Virtual model ${vmId} target ${i} missing modelId`);
}
}
}
}
}
return true;
}
/**
* 应用自定义处理器
*
* @param data 配置数据
* @param processors 自定义处理器函数数组
* @returns 处理后的数据
*/
applyCustomProcessors(data, processors) {
let processedData = data;
for (const processor of processors) {
if (typeof processor === 'function') {
processedData = processor(processedData);
}
}
return processedData;
}
/**
* 销毁解析器
*/
async destroy() {
this.initialized = false;
console.log('ConfigParser destroyed successfully');
}
}
/**
* 流水线配置生成器 - 增强版
*
* 负责根据配置数据生成完整的流水线配置，包括装配表和调度器配置
*/
/**
* 增强的流水线配置生成器类
*/
class EnhancedPipelineConfigGenerator {
constructor(fixedVirtualModels) {
this.initialized = false;
this.fixedVirtualModels = fixedVirtualModels || [
'default',
'longcontext',
'thinking',
'background',
'websearch',
'vision',
'coding'
];
// 初始化默认模块注册表
this.defaultModules = this.initializeDefaultModules();
// 初始化默认组装策略
this.defaultAssemblyStrategies = this.initializeDefaultAssemblyStrategies();
}
/**
* 初始化生成器
*/
async initialize() {
if (this.initialized) {
return;
}
this.initialized = true;
console.log('PipelineTableGenerator initialized successfully');
}
/**
* 生成完整的流水线配置
*/
async generateCompletePipelineConfig(config) {
try {
const startTime = Date.now();
// 生成装配表配置
const assemblyConfig = await this.generateAssemblyConfig(config);
// 生成调度器配置
const schedulerConfig = await this.generateSchedulerConfig(config, assemblyConfig);
const metadata = {
generatedAt: new Date().toISOString(),
configVersion: config.version || '1.0.0',
virtualModelCount: Object.keys(config.virtualModels || {}).length,
pipelineTemplateCount: assemblyConfig.pipelineTemplates.length,
moduleRegistryCount: assemblyConfig.moduleRegistry.length
};
console.log('Complete pipeline configuration generated', {
assemblyTime: Date.now() - startTime,
virtualModelCount: metadata.virtualModelCount,
pipelineTemplateCount: metadata.pipelineTemplateCount
});
return {
assemblyConfig,
schedulerConfig,
metadata
};
}
catch (error) {
console.error('Failed to generate complete pipeline configuration:', error);
throw error;
}
}
/**
* 生成装配表配置
*/
async generateAssemblyConfig(config) {
try {
const routingRules = await this.generateRoutingRules(config);
const pipelineTemplates = await this.generatePipelineTemplates(config);
const moduleRegistry = await this.generateModuleRegistry(config);
const assemblyStrategies = this.defaultAssemblyStrategies;
const assemblyConfig = {
version: '1.0.0',
metadata: {
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
description: 'Generated pipeline assembly table from configuration',
author: 'RCC Configuration System'
},
routingRules,
pipelineTemplates,
moduleRegistry,
assemblyStrategies
};
console.log('Assembly configuration generated', {
routingRulesCount: routingRules.length,
pipelineTemplatesCount: pipelineTemplates.length,
moduleRegistryCount: moduleRegistry.length
});
return assemblyConfig;
}
catch (error) {
console.error('Failed to generate assembly configuration:', error);
throw error;
}
}
/**
* 生成调度器配置
*/
async generateSchedulerConfig(config, assemblyConfig) {
try {
// 基于虚拟模型权重生成负载均衡配置
const weights = this.generatePipelineWeights(config);
const schedulerConfig = {
basic: this.generateBasicConfig(),
loadBalancing: this.generateLoadBalancingConfig(weights),
healthCheck: this.generateHealthCheckConfig(),
errorHandling: this.generateErrorHandlingConfig(),
performance: this.generatePerformanceConfig(),
monitoring: this.generateMonitoringConfig(),
security: this.generateSecurityConfig()
};
console.log('Scheduler configuration generated');
return schedulerConfig;
}
catch (error) {
console.error('Failed to generate scheduler configuration:', error);
throw error;
}
}
/**
* 生成基本配置
* @private
*/
generateBasicConfig() {
return {
schedulerId: 'rcc-pipeline-scheduler',
name: 'RCC Pipeline Scheduler',
version: '1.0.0',
description: 'Auto-generated scheduler configuration from RCC configuration'
};
}
/**
* 生成负载均衡配置
* @private
*/
generateLoadBalancingConfig(weights) {
return {
strategy: 'weighted',
strategyConfig: {
weighted: {
weights,
enableDynamicWeightAdjustment: true,
weightAdjustmentInterval: 60000
}
},
failover: {
enabled: true,
maxRetries: 3,
retryDelay: 1000,
backoffMultiplier: 2,
enableCircuitBreaker: true
}
};
}
/**
* 生成健康检查配置
* @private
*/
generateHealthCheckConfig() {
return {
strategy: 'hybrid',
intervals: {
activeCheckInterval: 30000,
passiveCheckInterval: 10000,
fullCheckInterval: 300000
},
checks: {
basic: {
enabled: true,
timeout: 5000
},
detailed: {
enabled: true,
timeout: 10000,
includeMetrics: true,
includeDependencies: false
},
custom: {
enabled: false,
checkFunction: '',
parameters: {}
}
},
thresholds: {
healthyThreshold: 2,
unhealthyThreshold: 3,
degradationThreshold: 1
},
recovery: {
autoRecovery: true,
recoveryStrategies: [],
maxRecoveryAttempts: 3
}
};
}
/**
* 生成错误处理配置
* @private
*/
generateErrorHandlingConfig() {
return {
errorClassification: {
enableAutomaticClassification: true,
customClassifiers: []
},
strategies: {
unrecoverableErrors: {
action: 'destroy_pipeline',
notificationEnabled: true,
logLevel: 'error'
},
recoverableErrors: {
action: 'blacklist_temporary',
maxRetryAttempts: 3,
blacklistDuration: 60000,
exponentialBackoff: true
},
authenticationErrors: {
action: 'enter_maintenance',
maintenanceDuration: 300000
},
networkErrors: {
action: 'retry_with_backoff',
maxRetryAttempts: 3,
backoffMultiplier: 2,
bufferSize: 100
}
},
blacklist: {
enabled: true,
maxEntries: 1000,
defaultDuration: 60000,
maxDuration: 3600000,
cleanupInterval: 300000,
autoExpiry: true
},
reporting: {
enableDetailedReporting: true,
reportInterval: 60000,
includeStackTraces: false,
includeContext: true,
customReporters: []
}
};
}
/**
* 生成性能配置
* @private
*/
generatePerformanceConfig() {
return {
concurrency: {
maxConcurrentRequests: 1000,
maxConcurrentRequestsPerPipeline: 100,
queueSize: 5000,
enablePriorityQueue: true
},
timeouts: {
defaultTimeout: 30000,
executionTimeout: 60000,
idleTimeout: 300000,
startupTimeout: 60000,
shutdownTimeout: 30000
},
caching: {
enabled: false,
strategy: 'lru',
maxSize: 1000,
ttl: 3600000
},
rateLimiting: {
enabled: false,
strategy: 'token_bucket',
requestsPerSecond: 100,
burstSize: 200
}
};
}
/**
* 生成监控配置
* @private
*/
generateMonitoringConfig() {
return {
metrics: {
enabled: true,
collectionInterval: 10000,
metrics: [
{
name: 'request_count',
type: 'counter',
description: 'Total number of requests',
labels: {
pipeline: '.*',
status: '.*'
}
},
{
name: 'response_time',
type: 'histogram',
description: 'Response time distribution',
labels: {
pipeline: '.*'
},
buckets: [100, 500, 1000, 5000, 10000, 30000]
}
],
aggregation: {
enabled: true,
interval: 60000,
functions: ['avg', 'sum', 'count', 'max', 'min']
}
},
logging: {
level: 'info',
format: 'json',
outputs: [
{
type: 'console',
config: {},
level: 'info'
}
],
sampling: {
enabled: false,
rate: 1.0
}
},
tracing: {
enabled: false,
samplingRate: 0.01,
includePayloads: false,
customSpans: []
},
alerts: {
enabled: false,
rules: [],
channels: []
}
};
}
/**
* 生成安全配置
* @private
*/
generateSecurityConfig() {
return {
authentication: {
enabled: false,
method: 'jwt',
config: {}
},
authorization: {
enabled: false,
roles: [],
permissions: {}
},
encryption: {
enabled: false,
algorithm: 'aes-256-gcm',
keyRotationInterval: 86400000
},
rateLimiting: {
enabled: false,
requestsPerMinute: 1000,
burstSize: 100
}
};
}
/**
* 生成路由规则
*/
async generateRoutingRules(config) {
const routingRules = [];
for (const [vmId, virtualModel] of Object.entries(config.virtualModels)) {
if (!this.fixedVirtualModels.includes(vmId)) {
continue;
}
// 为每个虚拟模型创建路由规则
const rule = {
ruleId: `routing-${vmId}`,
name: `Virtual Model ${vmId} Routing`,
priority: virtualModel.priority || 1,
enabled: virtualModel.enabled !== false,
conditions: [
{
field: 'request.model',
operator: 'equals',
value: vmId
}
],
pipelineSelection: {
strategy: 'weighted',
targetPipelineIds: [`pipeline-${vmId}`],
weights: this.generateWeightsForVirtualModel(virtualModel)
},
moduleFilters: [],
dynamicConfig: {
enableAdaptiveRouting: true,
performanceThresholds: {
maxResponseTime: 30000,
minSuccessRate: 0.9,
maxErrorRate: 0.1
}
}
};
routingRules.push(rule);
}
return routingRules;
}
/**
* 生成流水线模板
*/
async generatePipelineTemplates(config) {
const templates = [];
for (const [vmId, virtualModel] of Object.entries(config.virtualModels)) {
if (!this.fixedVirtualModels.includes(vmId)) {
continue;
}
// 获取第一个目标作为主要配置
const firstTarget = virtualModel.targets[0];
if (!firstTarget)
continue;
const template = {
templateId: `pipeline-${vmId}`,
name: `Pipeline for ${vmId}`,
description: `Pipeline for virtual model ${vmId} routing to ${firstTarget.providerId}/${firstTarget.modelId}`,
version: '1.0.0',
baseConfig: {
timeout: 60000,
maxConcurrentRequests: 100,
priority: virtualModel.priority || 1,
enabled: virtualModel.enabled !== false
},
moduleAssembly: {
moduleInstances: this.generateModuleInstances(vmId, firstTarget),
connections: this.generateModuleConnections(vmId),
dataMappings: this.generateDataMappings(vmId),
conditions: []
},
executionStrategy: {
mode: 'sequential',
timeout: 60000,
retryPolicy: {
maxRetries: 3,
baseDelay: 1000,
maxDelay: 10000,
backoffMultiplier: 2,
jitter: true
}
},
dataFlow: {
inputSchema: {
type: 'object',
properties: {
model: { type: 'string' },
messages: { type: 'array' }
},
required: ['model', 'messages']
},
outputSchema: {
type: 'object',
properties: {
id: { type: 'string' },
choices: { type: 'array' }
}
},
validation: {
enabled: true,
strict: true
}
}
};
templates.push(template);
}
return templates;
}
/**
* 生成模块注册表
*/
async generateModuleRegistry(config) {
// 合并默认模块和配置中定义的模块
const registry = [...this.defaultModules];
// 添加配置特定的模块（如果有）
for (const [providerId, provider] of Object.entries(config.providers)) {
for (const [modelId, model] of Object.entries(provider.models)) {
// 为每个提供商-模型组合创建特定的提供者模块
const providerModule = {
moduleId: `provider-${providerId}-${modelId}`,
name: `Provider ${providerId} - ${modelId}`,
version: '1.0.0',
type: 'provider',
description: `Provider module for ${providerId}/${modelId}`,
capabilities: ['api-calls', 'model-execution'],
dependencies: ['compatibility-module'],
configSchema: {
type: 'object',
properties: {
providerId: { type: 'string' },
modelId: { type: 'string' },
keyIndex: { type: 'number' }
},
required: ['providerId', 'modelId']
},
initializationConfig: {
setupFunction: 'initializeProvider',
validationFunction: 'validateProviderConfig'
},
tags: ['provider', providerId, modelId]
};
registry.push(providerModule);
}
}
return registry;
}
/**
* 生成模块实例配置
*/
generateModuleInstances(vmId, target) {
return [
{
instanceId: `${vmId}-compatibility`,
moduleId: 'compatibility-module',
name: 'Compatibility Module',
initialization: {
config: {
targetProvider: target.providerId,
targetModel: target.modelId
},
dependencies: [],
startupOrder: 1,
required: true
},
execution: {
timeout: 10000,
retryPolicy: {
maxRetries: 2,
baseDelay: 1000,
maxDelay: 5000,
backoffMultiplier: 2,
jitter: true
},
circuitBreaker: {
failureThreshold: 5,
recoveryTime: 60000,
requestVolumeThreshold: 10,
timeout: 5000
},
healthCheck: {
enabled: true,
interval: 30000,
timeout: 5000
}
},
conditions: {
enableConditions: [],
skipConditions: []
}
},
{
instanceId: `${vmId}-provider`,
moduleId: `provider-${target.providerId}-${target.modelId}`,
name: 'Provider Module',
initialization: {
config: {
providerId: target.providerId,
modelId: target.modelId,
keyIndex: target.keyIndex || 0
},
dependencies: [`${vmId}-compatibility`],
startupOrder: 2,
required: true
},
execution: {
timeout: 30000,
retryPolicy: {
maxRetries: 3,
baseDelay: 1000,
maxDelay: 10000,
backoffMultiplier: 2,
jitter: true
}
},
conditions: {
enableConditions: [],
skipConditions: []
}
}
];
}
/**
* 生成模块连接
*/
generateModuleConnections(vmId) {
return [
{
id: `${vmId}-compatibility-to-provider`,
from: `${vmId}-compatibility`,
to: `${vmId}-provider`,
type: 'success',
dataMapping: {
sourcePath: 'compatibility.output',
targetPath: 'provider.input',
required: true
}
}
];
}
/**
* 生成数据映射
*/
generateDataMappings(vmId) {
return [
{
sourcePath: 'request.body',
targetPath: `${vmId}-compatibility.input`,
required: true
}
];
}
/**
* 生成流水线权重
*/
generatePipelineWeights(config) {
const weights = {};
let totalPriority = 0;
// 计算总优先级
for (const [vmId, virtualModel] of Object.entries(config.virtualModels)) {
if (this.fixedVirtualModels.includes(vmId)) {
totalPriority += virtualModel.priority || 1;
}
}
// 根据优先级分配权重
for (const [vmId, virtualModel] of Object.entries(config.virtualModels)) {
if (this.fixedVirtualModels.includes(vmId)) {
const priority = virtualModel.priority || 1;
weights[`pipeline-${vmId}`] = Math.round((priority / totalPriority) * 100);
}
}
return weights;
}
/**
* 为虚拟模型生成权重
*/
generateWeightsForVirtualModel(virtualModel) {
const weights = {};
// 为每个目标分配权重
virtualModel.targets.forEach((target, index) => {
const pipelineId = `pipeline-${virtualModel.id}_target_${index}`;
weights[pipelineId] = 100 / virtualModel.targets.length;
});
return weights;
}
/**
* 初始化默认模块注册表
*/
initializeDefaultModules() {
return [
{
moduleId: 'compatibility-module',
name: 'Compatibility Module',
version: '1.0.0',
type: 'compatibility',
description: 'Handles request/response format compatibility',
capabilities: ['format-conversion', 'field-mapping', 'validation'],
dependencies: [],
configSchema: {
type: 'object',
properties: {
targetProvider: { type: 'string' },
targetModel: { type: 'string' }
},
required: ['targetProvider', 'targetModel']
},
initializationConfig: {
setupFunction: 'initializeCompatibility',
validationFunction: 'validateCompatibilityConfig'
},
tags: ['compatibility', 'adapter']
}
];
}
/**
* 初始化默认组装策略
*/
initializeDefaultAssemblyStrategies() {
return [
{
strategyId: 'performance-based-assembly',
name: 'Performance Based Assembly',
description: 'Assemble pipelines based on performance metrics',
algorithm: 'dynamic',
config: {
performanceMetrics: ['latency', 'errorRate', 'throughput'],
weighting: {
latency: 0.5,
errorRate: 0.3,
throughput: 0.2
}
},
selectionCriteria: {
performance: true,
cost: false,
reliability: true
}
}
];
}
/**
* 生成流水线表
*/
async generatePipelineTable(config) {
try {
// 生成装配表配置
const assemblyConfig = await this.generateAssemblyConfig(config);
// 创建流水线表
const pipelineTable = new Map();
// 为每个虚拟模型创建流水线条目
for (const [vmId, virtualModel] of Object.entries(config.virtualModels || {})) {
if (!this.fixedVirtualModels.includes(vmId)) {
continue;
}
// 为每个目标创建条目
for (let i = 0; i < virtualModel.targets