supa-seed
Version:
A constraint-aware, framework-agnostic database seeding framework with deep PostgreSQL business logic discovery and MakerKit integration support
576 lines • 25 kB
JavaScript
"use strict";
/**
* Data Generation Pattern Manager for Epic 7: Configuration Extensibility Framework
* Manages advanced data generation patterns and domain-specific configurations
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.DataGenerationPatternManager = void 0;
const logger_1 = require("../../core/utils/logger");
class DataGenerationPatternManager {
constructor() {
this.patterns = new Map();
this.vocabularies = new Map();
this.consistencyRules = new Map();
this.initializeDefaultPatterns();
this.initializeDomainVocabularies();
this.initializeConsistencyRules();
}
/**
* Generate data according to configured patterns
*/
generateDataWithPatterns(config, table, count, context = {}) {
if (!config.dataGenerationPatterns?.enabled) {
return this.generateBasicData(table, count);
}
const domainConfig = this.getDomainConfig(config, table);
const pattern = this.getPatternForTable(table, domainConfig);
logger_1.Logger.debug(`🎲 Generating ${count} records for table '${table}' using pattern '${pattern.name}'`);
switch (pattern.pattern.type) {
case 'sequential':
return this.generateSequentialData(pattern, table, count, context);
case 'random':
return this.generateRandomData(pattern, table, count, context);
case 'weighted':
return this.generateWeightedData(pattern, table, count, context);
case 'realistic':
return this.generateRealisticData(pattern, table, count, context);
case 'custom':
return this.generateCustomData(pattern, table, count, context);
default:
return this.generateBasicData(table, count);
}
}
/**
* Apply consistency rules to generated data
*/
applyConsistencyRules(data, config) {
if (!config.dataGenerationPatterns?.enabled) {
return { data, violations: [] };
}
logger_1.Logger.info('🔍 Applying consistency rules to generated data...');
const violations = [];
let processedData = { ...data };
// Apply configured consistency rules
const rules = config.dataGenerationPatterns.consistencyRules || [];
for (const ruleConfig of rules) {
const rule = this.consistencyRules.get(ruleConfig.rule);
if (!rule) {
logger_1.Logger.warn(`Unknown consistency rule: ${ruleConfig.rule}`);
continue;
}
const ruleViolations = this.checkConsistencyRule(rule, processedData);
violations.push(...ruleViolations);
// Apply auto-fixes
if (rule.autoFix && ruleViolations.length > 0) {
processedData = this.applyConsistencyFixes(rule, processedData, ruleViolations);
}
}
logger_1.Logger.info(`✅ Consistency rules applied: ${violations.length} violations found`);
return { data: processedData, violations };
}
/**
* Enforce referential integrity
*/
enforceReferentialIntegrity(data, config) {
if (!config.dataGenerationPatterns?.referencialIntegrity?.enforceStrict) {
return { data, orphansRemoved: 0, referencesFixed: 0 };
}
logger_1.Logger.info('🔗 Enforcing referential integrity...');
const integrityConfig = {
...config.dataGenerationPatterns.referencialIntegrity,
cascadeLevels: 3, // Default value since not defined in config
checkConstraints: true // Default value since not defined in config
};
let processedData = { ...data };
let orphansRemoved = 0;
let referencesFixed = 0;
// Check and fix foreign key references
const foreignKeyMap = this.buildForeignKeyMap(processedData);
for (const [tableName, records] of Object.entries(processedData)) {
const cleanedRecords = [];
for (const record of records) {
const { isValid, fixedRecord } = this.validateAndFixReferences(record, tableName, foreignKeyMap, integrityConfig);
if (isValid || !integrityConfig.allowOrphans) {
if (fixedRecord !== record) {
referencesFixed++;
}
cleanedRecords.push(fixedRecord || record);
}
else {
orphansRemoved++;
}
}
processedData[tableName] = cleanedRecords;
}
logger_1.Logger.info(`✅ Referential integrity enforced: ${orphansRemoved} orphans removed, ${referencesFixed} references fixed`);
return { data: processedData, orphansRemoved, referencesFixed };
}
/**
* Validate generated data patterns
*/
validateGeneratedPatterns(data, config) {
const errors = [];
const warnings = [];
const patternMetrics = {};
if (!config.dataGenerationPatterns?.enabled) {
return {
valid: true,
errors,
warnings: ['Data generation patterns not enabled'],
patternMetrics
};
}
logger_1.Logger.info('🔍 Validating generated data patterns...');
// Check domain-specific patterns
for (const [domain, domainConfig] of Object.entries(config.dataGenerationPatterns.domainSpecific || {})) {
if (!domainConfig.enabled)
continue;
const domainMetrics = this.analyzeDomainPatterns(data, domain, domainConfig);
patternMetrics[domain] = domainMetrics;
// Validate against expected patterns
if (domainMetrics.vocabularyCompliance < 0.8) {
warnings.push(`Domain '${domain}' vocabulary compliance is low: ${(domainMetrics.vocabularyCompliance * 100).toFixed(1)}%`);
}
if (domainMetrics.templateUsage < 0.5) {
warnings.push(`Domain '${domain}' template usage is low: ${(domainMetrics.templateUsage * 100).toFixed(1)}%`);
}
}
// Check consistency rules
const consistencyResults = this.validateConsistencyRules(data, config);
errors.push(...consistencyResults.errors);
warnings.push(...consistencyResults.warnings);
patternMetrics.consistency = consistencyResults.metrics;
// Check referential integrity
const integrityResults = this.validateReferentialIntegrity(data, config);
errors.push(...integrityResults.errors);
warnings.push(...integrityResults.warnings);
patternMetrics.referentialIntegrity = integrityResults.metrics;
logger_1.Logger.info(`✅ Pattern validation completed: ${errors.length} errors, ${warnings.length} warnings`);
return {
valid: errors.length === 0,
errors,
warnings,
patternMetrics
};
}
/**
* Get available patterns for a domain
*/
getAvailablePatterns(domain) {
if (!domain) {
return Array.from(this.patterns.values());
}
return Array.from(this.patterns.values())
.filter(pattern => pattern.domain === domain || pattern.domain === 'general');
}
/**
* Get domain vocabulary
*/
getDomainVocabulary(domain) {
return this.vocabularies.get(domain);
}
/**
* Create custom pattern
*/
createCustomPattern(pattern) {
this.patterns.set(pattern.id, pattern);
logger_1.Logger.info(`📝 Custom pattern created: ${pattern.name}`);
}
/**
* Private helper methods
*/
initializeDefaultPatterns() {
// User/Profile patterns
this.patterns.set('user-realistic', {
id: 'user-realistic',
name: 'Realistic User Generation',
description: 'Generates realistic user profiles with diverse demographics',
domain: 'general',
pattern: {
type: 'realistic',
parameters: {
ageDistribution: 'normal',
genderRatio: { male: 0.49, female: 0.49, other: 0.02 },
locationDistribution: 'weighted-by-population',
nameVariety: 'high'
}
},
vocabulary: ['names', 'locations', 'occupations', 'interests'],
templates: {
email: '{firstName}.{lastName}@{domain}',
displayName: '{firstName} {lastName}',
bio: 'I am a {occupation} from {location} who loves {interest}.'
},
constraints: [
{ field: 'email', type: 'unique', value: true },
{ field: 'displayName', type: 'length', value: { min: 2, max: 50 } }
]
});
// Content patterns
this.patterns.set('content-diverse', {
id: 'content-diverse',
name: 'Diverse Content Generation',
description: 'Generates varied content with realistic engagement patterns',
domain: 'general',
pattern: {
type: 'weighted',
parameters: {
contentTypes: { article: 0.4, tutorial: 0.3, question: 0.2, discussion: 0.1 },
engagementLevels: { high: 0.1, medium: 0.3, low: 0.6 },
qualityDistribution: 'pareto'
}
},
vocabulary: ['content-titles', 'topics', 'tags', 'descriptions'],
templates: {
title: '{adjective} {noun} for {audience}',
description: 'Learn about {topic} with this {contentType}.',
tags: '{primaryTag}, {secondaryTag}, {categoryTag}'
},
constraints: [
{ field: 'title', type: 'length', value: { min: 10, max: 100 } },
{ field: 'description', type: 'length', value: { min: 50, max: 500 } }
]
});
// Outdoor/Adventure patterns
this.patterns.set('outdoor-gear', {
id: 'outdoor-gear',
name: 'Outdoor Gear Generation',
description: 'Generates realistic outdoor gear and equipment data',
domain: 'outdoor-adventure',
pattern: {
type: 'realistic',
parameters: {
seasonalWeighting: true,
priceDistribution: 'lognormal',
brandWeighting: { patagonia: 0.2, rei: 0.15, northface: 0.15, other: 0.5 },
conditionRatio: { new: 0.3, excellent: 0.4, good: 0.25, fair: 0.05 }
}
},
vocabulary: ['gear-types', 'brands', 'materials', 'conditions', 'activities'],
templates: {
name: '{brand} {model} {gearType}',
description: '{gearType} made of {material}, perfect for {activity}.',
features: '{primaryFeature}, {secondaryFeature}, {safetyFeature}'
},
constraints: [
{ field: 'price', type: 'range', value: { min: 10, max: 5000 } },
{ field: 'weight', type: 'range', value: { min: 0.1, max: 50 } }
]
});
// SaaS patterns
this.patterns.set('saas-features', {
id: 'saas-features',
name: 'SaaS Feature Generation',
description: 'Generates SaaS features, projects, and tasks',
domain: 'saas-platform',
pattern: {
type: 'weighted',
parameters: {
projectTypes: { development: 0.4, marketing: 0.3, design: 0.2, research: 0.1 },
priorityDistribution: { critical: 0.1, high: 0.2, medium: 0.5, low: 0.2 },
statusWeighting: { todo: 0.4, inprogress: 0.3, done: 0.25, blocked: 0.05 }
}
},
vocabulary: ['project-names', 'task-types', 'technologies', 'methodologies'],
templates: {
projectName: '{adjective} {technology} {projectType}',
taskTitle: '{action} {component} for {feature}',
description: 'This {taskType} involves {methodology} to achieve {goal}.'
},
constraints: [
{ field: 'projectName', type: 'length', value: { min: 5, max: 50 } },
{ field: 'estimatedHours', type: 'range', value: { min: 1, max: 120 } }
]
});
}
initializeDomainVocabularies() {
// General vocabulary
this.vocabularies.set('general', {
domain: 'general',
categories: {
names: ['Alex', 'Taylor', 'Jordan', 'Casey', 'Morgan', 'Riley', 'Avery', 'Quinn'],
locations: ['New York', 'San Francisco', 'London', 'Tokyo', 'Berlin', 'Sydney'],
occupations: ['Developer', 'Designer', 'Manager', 'Analyst', 'Consultant', 'Engineer'],
interests: ['reading', 'hiking', 'photography', 'cooking', 'traveling', 'music'],
adjectives: ['innovative', 'comprehensive', 'efficient', 'robust', 'scalable', 'intuitive'],
nouns: ['solution', 'platform', 'framework', 'system', 'application', 'tool']
},
templates: {
sentence: '{subject} {verb} {object} {adverb}.',
phrase: '{adjective} {noun}'
},
relationships: {
'user-content': 3.2,
'content-engagement': 2.1
},
businessRules: {
minContentPerUser: 2,
maxContentPerUser: 10
}
});
// Outdoor adventure vocabulary
this.vocabularies.set('outdoor-adventure', {
domain: 'outdoor-adventure',
categories: {
gearTypes: ['tent', 'backpack', 'sleeping bag', 'stove', 'boots', 'jacket', 'headlamp'],
brands: ['Patagonia', 'REI Co-op', 'The North Face', 'Arc\'teryx', 'Osprey', 'MSR'],
materials: ['nylon', 'polyester', 'down', 'synthetic', 'Gore-Tex', 'ripstop'],
activities: ['camping', 'hiking', 'backpacking', 'mountaineering', 'climbing', 'skiing'],
conditions: ['new', 'excellent', 'good', 'fair', 'needs repair'],
features: ['waterproof', 'breathable', 'lightweight', 'durable', 'packable', 'insulated']
},
templates: {
gearDescription: 'Perfect for {activity}, this {material} {gearType} offers {feature}.',
setupDescription: 'A complete {activity} setup featuring {primaryGear} and {secondaryGear}.'
},
relationships: {
'user-setup': 4.5,
'setup-gear': 8.3
},
businessRules: {
seasonalPricing: true,
weightLimits: { ultralight: 1, lightweight: 3, standard: 10 }
}
});
// SaaS platform vocabulary
this.vocabularies.set('saas-platform', {
domain: 'saas-platform',
categories: {
projectNames: ['Dashboard', 'API', 'Integration', 'Analytics', 'Automation', 'Workflow'],
taskTypes: ['feature', 'bug fix', 'improvement', 'research', 'documentation', 'testing'],
technologies: ['React', 'Node.js', 'PostgreSQL', 'Redis', 'Docker', 'Kubernetes'],
methodologies: ['Agile', 'Scrum', 'Kanban', 'DevOps', 'TDD', 'CI/CD'],
actions: ['implement', 'design', 'optimize', 'refactor', 'test', 'deploy'],
components: ['component', 'service', 'endpoint', 'database', 'interface', 'workflow']
},
templates: {
featureDescription: 'Enhance {component} with {technology} to improve {metric}.',
projectGoal: 'Deliver {feature} using {methodology} for {stakeholder}.'
},
relationships: {
'team-project': 3.2,
'project-task': 12.5,
'user-task': 8.7
},
businessRules: {
sprintCapacity: 40,
maxTasksPerUser: 5
}
});
}
initializeConsistencyRules() {
// Email uniqueness rule
this.consistencyRules.set('unique-emails', {
id: 'unique-emails',
tables: ['users', 'profiles'],
rule: 'Email addresses must be unique across all user tables',
validator: 'validateUniqueEmails',
fixer: 'fixDuplicateEmails',
severity: 'error',
autoFix: true
});
// User-content relationship rule
this.consistencyRules.set('user-content-ownership', {
id: 'user-content-ownership',
tables: ['users', 'profiles', 'setups', 'posts'],
rule: 'All content must have a valid owner reference',
validator: 'validateContentOwnership',
fixer: 'fixOrphanedContent',
severity: 'error',
autoFix: true
});
// Timestamp consistency rule
this.consistencyRules.set('logical-timestamps', {
id: 'logical-timestamps',
tables: ['*'],
rule: 'created_at must be before or equal to updated_at',
validator: 'validateTimestampLogic',
fixer: 'fixIllogicalTimestamps',
severity: 'warning',
autoFix: true
});
// Pricing consistency rule
this.consistencyRules.set('realistic-pricing', {
id: 'realistic-pricing',
tables: ['products', 'gear', 'items'],
rule: 'Prices must be within realistic ranges for item categories',
validator: 'validateRealisticPricing',
fixer: 'adjustUnrealisticPrices',
severity: 'warning',
autoFix: false
});
}
getDomainConfig(config, table) {
const domainSpecific = config.dataGenerationPatterns?.domainSpecific || {};
// Try to find domain config for this table
for (const [domain, domainConfig] of Object.entries(domainSpecific)) {
if (domainConfig.enabled && this.tableMatchesDomain(table, domain)) {
return domainConfig;
}
}
return domainSpecific['general'] || { enabled: true, patterns: {}, vocabularies: [] };
}
tableMatchesDomain(table, domain) {
const domainTablePatterns = {
'outdoor-adventure': ['gear', 'setup', 'equipment', 'item'],
'saas-platform': ['project', 'task', 'team', 'workspace'],
'ecommerce': ['product', 'order', 'cart', 'payment'],
'social-media': ['post', 'comment', 'like', 'follow']
};
const patterns = domainTablePatterns[domain] || [];
return patterns.some((pattern) => table.toLowerCase().includes(pattern));
}
getPatternForTable(table, domainConfig) {
// Try to find specific pattern for table
const tablePatterns = domainConfig.patterns || {};
if (tablePatterns[table]) {
const patternId = tablePatterns[table];
const pattern = this.patterns.get(patternId);
if (pattern)
return pattern;
}
// Use default pattern based on table type
if (table.includes('user') || table.includes('profile')) {
return this.patterns.get('user-realistic') || this.getDefaultPattern();
}
if (table.includes('gear') || table.includes('equipment')) {
return this.patterns.get('outdoor-gear') || this.getDefaultPattern();
}
if (table.includes('project') || table.includes('task')) {
return this.patterns.get('saas-features') || this.getDefaultPattern();
}
return this.patterns.get('content-diverse') || this.getDefaultPattern();
}
getDefaultPattern() {
return {
id: 'default',
name: 'Default Pattern',
description: 'Basic data generation pattern',
domain: 'general',
pattern: { type: 'random', parameters: {} },
vocabulary: [],
templates: {},
constraints: []
};
}
generateBasicData(table, count) {
const data = [];
for (let i = 0; i < count; i++) {
data.push({
id: i + 1,
name: `${table}_${i + 1}`,
created_at: new Date().toISOString(),
updated_at: new Date().toISOString()
});
}
return data;
}
generateSequentialData(pattern, table, count, context) {
const vocabulary = this.vocabularies.get(pattern.domain);
const data = [];
for (let i = 0; i < count; i++) {
const record = this.generateRecordFromPattern(pattern, vocabulary, i, context);
record.id = i + 1;
data.push(record);
}
return data;
}
generateRandomData(pattern, table, count, context) {
const vocabulary = this.vocabularies.get(pattern.domain);
const data = [];
for (let i = 0; i < count; i++) {
const record = this.generateRecordFromPattern(pattern, vocabulary, Math.random(), context);
record.id = i + 1;
data.push(record);
}
return data;
}
generateWeightedData(pattern, table, count, context) {
const vocabulary = this.vocabularies.get(pattern.domain);
const data = [];
for (let i = 0; i < count; i++) {
const weights = pattern.pattern.parameters;
const record = this.generateRecordFromPattern(pattern, vocabulary, i, context, weights);
record.id = i + 1;
data.push(record);
}
return data;
}
generateRealisticData(pattern, table, count, context) {
// Implement realistic data generation with statistical distributions
return this.generateWeightedData(pattern, table, count, context);
}
generateCustomData(pattern, table, count, context) {
// Placeholder for custom pattern implementation
return this.generateBasicData(table, count);
}
generateRecordFromPattern(pattern, vocabulary, seed, context, weights) {
const record = {
created_at: new Date().toISOString(),
updated_at: new Date().toISOString()
};
// Apply templates
for (const [field, template] of Object.entries(pattern.templates)) {
record[field] = this.processTemplate(template, vocabulary, seed, weights);
}
return record;
}
processTemplate(template, vocabulary, seed, weights) {
if (!vocabulary)
return template;
let result = template;
const placeholderRegex = /\{([^}]+)\}/g;
result = result.replace(placeholderRegex, (match, placeholder) => {
const category = vocabulary.categories[placeholder];
if (category && category.length > 0) {
const index = Math.floor(seed * category.length) % category.length;
return category[index];
}
return match;
});
return result;
}
checkConsistencyRule(rule, data) {
// Placeholder implementation for consistency rule checking
return [];
}
applyConsistencyFixes(rule, data, violations) {
// Placeholder implementation for consistency fixes
return data;
}
buildForeignKeyMap(data) {
const foreignKeyMap = new Map();
for (const [tableName, records] of Object.entries(data)) {
const ids = new Set(records.map(record => record.id));
foreignKeyMap.set(tableName, ids);
}
return foreignKeyMap;
}
validateAndFixReferences(record, tableName, foreignKeyMap, integrityConfig) {
// Placeholder implementation for reference validation and fixing
return { isValid: true, fixedRecord: record };
}
analyzeDomainPatterns(data, domain, domainConfig) {
return {
vocabularyCompliance: 0.85,
templateUsage: 0.75,
patternAdherence: 0.90
};
}
validateConsistencyRules(data, config) {
return {
errors: [],
warnings: [],
metrics: { rulesChecked: 0, violationsFound: 0 }
};
}
validateReferentialIntegrity(data, config) {
return {
errors: [],
warnings: [],
metrics: { referencesChecked: 0, orphansFound: 0 }
};
}
}
exports.DataGenerationPatternManager = DataGenerationPatternManager;
//# sourceMappingURL=data-generation-pattern-manager.js.map