supa-seed
Version:
A constraint-aware, framework-agnostic database seeding framework with deep PostgreSQL business logic discovery and MakerKit integration support
443 lines • 23.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SetupSeeder = void 0;
const types_1 = require("../../../core/types/types");
const domains_1 = require("../../../domains");
const batch_processor_1 = require("../../../core/batch-processor");
const logger_1 = require("../../../core/utils/logger");
const memory_manager_1 = __importDefault(require("../../../core/utils/memory-manager"));
const query_translator_1 = require("../../../schema/query-translator");
class SetupSeeder extends types_1.SeedModule {
constructor() {
super(...arguments);
// Legacy outdoor setup types for backward compatibility
this.outdoorSetupTypes = {
'Vehicle': [
'Weekend Car Camping',
'Extended Overland Trip',
'Daily Driver + Adventure',
'Base Camp Setup',
'Family Road Trip',
'Solo Adventure Rig',
'Photography Expedition',
'Hunting/Fishing Mobile Base'
],
'Backpack': [
'Day Hiking Essentials',
'Overnight Backpacking',
'Multi-day Wilderness Trek',
'Ultralight Minimalist',
'Photography Hiking Kit',
'Peak Bagging Setup',
'Winter Backpacking',
'Desert Adventure Pack'
]
};
}
getSetupTypes() {
const domainConfig = (0, domains_1.getDomainConfig)(this.context.config.domain);
// Fallback to legacy outdoor setup types if domain doesn't have the needed types
const domainSetupTypes = domainConfig.setupTypes;
const hasVehicleTypes = domainSetupTypes['Vehicle']?.length > 0;
const hasBackpackTypes = domainSetupTypes['Backpack']?.length > 0;
if (!hasVehicleTypes || !hasBackpackTypes) {
// Merge domain types with legacy outdoor types for backward compatibility
return {
...domainSetupTypes,
...this.outdoorSetupTypes
};
}
return domainSetupTypes;
}
/**
* Validate configured categories against database enum constraints
*/
async validateCategoriesAgainstEnum(categories) {
const { client } = this.context;
const errors = [];
try {
// Query PostgreSQL to get enum values for outdoor_category type
const { data: enumData, error } = await client
.from('information_schema.enum_range')
.select('enumlabel')
.ilike('enumtypid', '%outdoor_category%')
.limit(100);
if (error) {
logger_1.Logger.debug('Could not query enum values, attempting alternative approach');
// Alternative: Try to get enum values from pg_enum
const { data: pgEnumData, error: pgEnumError } = await client.rpc('get_enum_values', {
enum_name: 'outdoor_category'
});
if (pgEnumError) {
logger_1.Logger.warn('Could not validate categories against database enum - validation skipped');
return { valid: true, errors: [] }; // Allow to proceed without validation
}
const validEnumValues = Array.isArray(pgEnumData) ? pgEnumData.map((row) => row.enumlabel) : [];
const invalidCategories = categories.filter(cat => !validEnumValues.includes(cat));
if (invalidCategories.length > 0) {
errors.push(`Invalid enum values: ${invalidCategories.join(', ')}. Valid values: ${validEnumValues.join(', ')}`);
}
return { valid: errors.length === 0, errors };
}
const validEnumValues = Array.isArray(enumData) ? enumData.map((row) => row.enumlabel) : [];
const invalidCategories = categories.filter(cat => !validEnumValues.includes(cat));
if (invalidCategories.length > 0) {
errors.push(`Invalid enum values: ${invalidCategories.join(', ')}. Valid values: ${validEnumValues.join(', ')}`);
}
return { valid: errors.length === 0, errors };
}
catch (validationError) {
logger_1.Logger.warn('Enum validation failed, proceeding without validation:', validationError);
return { valid: true, errors: [] }; // Allow to proceed
}
}
async seed() {
console.log('🎒 Seeding setups with memory-efficient batch processing...');
// Validate and log category configuration
const configuredCategories = this.context.config.tables?.setups?.categories;
if (configuredCategories && configuredCategories.length > 0) {
console.log(`🎯 Using configured categories: ${configuredCategories.join(', ')}`);
logger_1.Logger.info(`Configuration-based category selection: ${configuredCategories.length} categories provided`);
// Validate categories against database enum
const validation = await this.validateCategoriesAgainstEnum(configuredCategories);
if (!validation.valid) {
console.error('❌ Configuration validation failed:');
for (const error of validation.errors) {
console.error(` ${error}`);
}
logger_1.Logger.error('Invalid categories in configuration', { errors: validation.errors });
throw new Error(`Invalid category configuration: ${validation.errors.join('; ')}`);
}
else {
console.log('✅ All configured categories validated against database schema');
logger_1.Logger.info('Category validation passed - all categories are valid enum values');
}
}
else {
console.log('⚠️ No categories configured, will use domain-based setup types');
logger_1.Logger.warn('No configured categories found in tables.setups.categories - falling back to hardcoded types');
}
// Validate and log count configuration
const totalSetupsConfig = this.context.config.tables?.setups?.count;
if (totalSetupsConfig && totalSetupsConfig > 0) {
console.log(`🎯 Using total count distribution: ${totalSetupsConfig} setups distributed across all users`);
logger_1.Logger.info(`Total count mode: will generate exactly ${totalSetupsConfig} setups total`);
// Add safety validation
const MAX_REASONABLE_SETUPS = 1000; // Configurable safety limit
if (totalSetupsConfig > MAX_REASONABLE_SETUPS) {
const errorMsg = `Setup count ${totalSetupsConfig} exceeds reasonable limit of ${MAX_REASONABLE_SETUPS}. Use smaller count or add explicit override.`;
console.error(`❌ ${errorMsg}`);
logger_1.Logger.error('Setup count safety limit exceeded', {
requested: totalSetupsConfig,
limit: MAX_REASONABLE_SETUPS
});
throw new Error(errorMsg);
}
if (totalSetupsConfig > 100) {
console.warn(`⚠️ Large dataset configured: ${totalSetupsConfig} setups may take significant time`);
logger_1.Logger.warn(`Large dataset warning: ${totalSetupsConfig} setups configured`);
}
}
else {
console.log(`⚠️ Using legacy per-user count: max ${this.context.config.setupsPerUser} setups per user`);
logger_1.Logger.warn(`Legacy mode: using setupsPerUser=${this.context.config.setupsPerUser} (may generate unexpected amounts)`);
}
const users = this.context.cache.get('users');
const templates = this.context.cache.get('baseTemplates');
const schemaAdapter = this.context.cache.get('schemaAdapter');
const noUsersCreated = this.context.cache.get('noUsersCreated');
if (!users?.length || noUsersCreated) {
const reason = noUsersCreated ?
'User creation failed - check schema configuration and column mappings' :
'No users found in cache';
console.log(`⚠️ Skipping setup seeding: ${reason}`);
return;
}
// Check if setups table exists
if (!(await this.checkTableExists('setups'))) {
console.log('⚠️ Setups table not found, skipping setup seeding');
return;
}
if (!templates?.length) {
console.log('⚠️ No base templates found in cache, creating default setups instead');
}
// Use streaming batch processing to prevent OOM
await this.processSetupsInBatches(users, templates, schemaAdapter);
}
async createSetup(user, templates, schemaAdapter) {
const { client, faker } = this.context;
// Get categories from configuration first, fallback to domain-based categories
const configuredCategories = this.context.config.tables?.setups?.categories;
let setupCategory;
let template;
if (configuredCategories && configuredCategories.length > 0) {
// Use configured categories exclusively
logger_1.Logger.debug(`🎯 Using configured categories: ${configuredCategories.join(', ')}`);
setupCategory = faker.helpers.arrayElement(configuredCategories);
// Use template if available, otherwise create generic
if (templates && templates.length > 0) {
template = faker.helpers.arrayElement(templates);
}
else {
template = {
id: '',
type: 'General',
make: '',
model: '',
};
}
}
else {
// Fallback to legacy domain-based setup types
logger_1.Logger.debug('⚠️ No categories configured, falling back to domain-based setup types');
if (templates && templates.length > 0) {
template = faker.helpers.arrayElement(templates);
const domainSetupTypes = this.getSetupTypes();
const setupTypes = domainSetupTypes[template.type] || [];
if (setupTypes.length === 0) {
console.warn(`⚠️ No setup types found for template type: ${template.type}`);
return null;
}
setupCategory = faker.helpers.arrayElement(setupTypes);
}
else {
// Create a generic setup without templates
const domainSetupTypes = this.getSetupTypes();
const allSetupTypes = Object.values(domainSetupTypes).flat();
template = {
id: '',
type: 'General',
make: '',
model: '',
};
setupCategory = faker.helpers.arrayElement(allSetupTypes.length > 0 ? allSetupTypes : ['Adventure Kit', 'Travel Setup', 'Outdoor Essentials', 'Exploration Gear']);
}
}
// Generate contextual title and description
const title = this.generateSetupTitle(template, setupCategory);
const description = this.generateSetupDescription(template, setupCategory);
// Use schema adapter to determine the correct foreign key
const userForeignKey = schemaAdapter ? schemaAdapter.getUserForeignKey() : 'account_id';
const setupData = {
[userForeignKey]: user.id,
title,
description,
category: setupCategory,
is_public: faker.datatype.boolean(0.8), // 80% public
created_at: faker.date.between({
from: new Date('2024-01-01'),
to: new Date()
}).toISOString(),
};
// Only add base_template_id if we have a real template
if (template.id) {
setupData.base_template_id = template.id;
}
try {
// Use query translator for dynamic table mapping
const framework = this.context.config.schema?.framework || 'makerkit';
const translator = (0, query_translator_1.createQueryTranslator)(client, {
framework,
enableValidation: true,
enableCaching: true
});
const { data, error } = await (await translator.from('setups'))
.insert(setupData)
.select('id')
.single();
if (error) {
console.error(`❌ Failed to create setup for user ${user.username}:`, error);
return null;
}
return {
id: data.id,
userId: user.id,
title,
category: setupCategory,
baseTemplateId: template.id,
};
}
catch (error) {
console.error(`❌ Error creating setup:`, error);
return null;
}
}
generateSetupTitle(template, category) {
const { faker } = this.context;
const domainConfig = (0, domains_1.getDomainConfig)(this.context.config.domain);
// Use domain-specific descriptors or fallback to generic ones
const descriptors = domainConfig.titleDescriptors || {
'Vehicle': [
'Custom', 'Professional', 'Complete', 'Premium', 'Essential',
'Advanced', 'Standard', 'Optimized', 'Versatile', 'Reliable'
],
'Backpack': [
'Essential', 'Complete', 'Tested', 'Proven', 'Reliable',
'Compact', 'Versatile', 'Optimized', 'Professional', 'Premium'
],
'General': [
'Custom', 'Professional', 'Complete', 'Essential', 'Advanced',
'Standard', 'Optimized', 'Versatile', 'Reliable', 'Premium'
]
};
const templateDescriptors = descriptors[template.type] || descriptors['General'] || ['Custom'];
const desc = faker.helpers.arrayElement(templateDescriptors);
const year = template.year || '';
if (template.type === 'Vehicle' && template.make && template.model) {
return `${desc} ${template.make} ${template.model} ${year} Build`.trim();
}
else {
return `${desc} ${category} Kit`;
}
}
generateSetupDescription(template, category) {
const { faker } = this.context;
const domainConfig = (0, domains_1.getDomainConfig)(this.context.config.domain);
// Use domain-specific descriptions or fallback to generic ones
const experiences = domainConfig.experiences || [
'after extensive testing',
'refined through regular use',
'proven through experience',
'optimized for reliability',
'developed through practice',
'tested in various conditions'
];
const contexts = domainConfig.contexts || [
'different environments', 'various situations', 'multiple contexts',
'diverse conditions', 'real-world scenarios', 'practical applications'
];
const experience = faker.helpers.arrayElement(experiences);
const context = faker.helpers.arrayElement(contexts);
if (template.type === 'Vehicle' && template.make && template.model) {
return `My ${template.make} ${template.model} setup for ${category.toLowerCase()}, ${experience}. Perfect for ${context} and handling whatever comes your way. This build focuses on reliability, comfort, and versatility.`;
}
else {
return `A carefully curated ${category.toLowerCase()} kit ${experience}. This setup has served me well across ${context}. Every item has earned its place through real-world use.`;
}
}
async processSetupsInBatches(users, templates, schemaAdapter) {
const createdSetups = [];
// Configure batch processor for memory efficiency
const batchProcessor = new batch_processor_1.StreamingBatchProcessor({
defaultBatchSize: 25, // Smaller batches for setup generation
memoryThresholdMB: 512, // Lower threshold for setup processing
minBatchSize: 5,
maxBatchSize: 50,
forceGCBetweenBatches: true,
batchDelayMs: 50 // Brief pause between batches
});
// Create a processor function for each batch of users
const userBatchProcessor = async (userBatch) => {
const batchResults = [];
for (const userItem of userBatch) {
const batchStartTime = Date.now();
const memoryBefore = process.memoryUsage().heapUsed / 1024 / 1024;
try {
const user = userItem.data;
const userSetups = [];
// Calculate setup count - use new total count distribution or fall back to per-user
let setupCount;
const totalSetupsConfig = this.context.config.tables?.setups?.count;
if (totalSetupsConfig && totalSetupsConfig > 0) {
// New behavior: distribute total count across all users
const totalUsers = users.length;
const baseSetupsPerUser = Math.floor(totalSetupsConfig / totalUsers);
const remainderSetups = totalSetupsConfig % totalUsers;
// Give base count to all users, plus 1 extra to some users for remainder
const userIndex = users.findIndex(u => u.id === user.id);
setupCount = baseSetupsPerUser + (userIndex < remainderSetups ? 1 : 0);
// Ensure at least 0 setups (edge case where total < users)
setupCount = Math.max(0, setupCount);
logger_1.Logger.debug(`🎯 Using total count distribution: ${setupCount} setups for user ${user.username} (${userIndex + 1}/${totalUsers}, total: ${totalSetupsConfig})`);
}
else {
// Legacy behavior: random setups per user
setupCount = this.context.faker.number.int({
min: 1,
max: this.context.config.setupsPerUser
});
logger_1.Logger.debug(`⚠️ Using legacy setupsPerUser: ${setupCount} setups for user ${user.username}`);
}
logger_1.Logger.debug(`🎒 Creating ${setupCount} setups for user: ${user.username}`);
// Create setups for this user
for (let i = 0; i < setupCount; i++) {
const setup = await this.createSetup(user, templates, schemaAdapter);
if (setup) {
userSetups.push(setup);
this.context.stats.setupsCreated++;
}
}
const processingTime = Date.now() - batchStartTime;
const memoryAfter = process.memoryUsage().heapUsed / 1024 / 1024;
batchResults.push({
success: true,
result: userSetups,
processingTimeMs: processingTime,
memoryUsageMB: memoryAfter
});
}
catch (error) {
logger_1.Logger.error(`❌ Failed to create setups for user ${userItem.data.username}:`, error);
batchResults.push({
success: false,
error: error,
processingTimeMs: Date.now() - batchStartTime,
memoryUsageMB: process.memoryUsage().heapUsed / 1024 / 1024
});
}
}
return batchResults;
};
logger_1.Logger.info(`🔄 Processing ${users.length} users in memory-efficient batches...`);
// Process all users through the batch processor
const { results, stats, recommendations } = await batchProcessor.processAll(users, userBatchProcessor);
// Collect all successful results
let totalSetups = 0;
for (const result of results) {
if (result.success && result.result) {
createdSetups.push(...result.result);
totalSetups += result.result.length;
}
}
// Cache the results
this.context.cache.set('setups', createdSetups);
console.log(`📦 Cached ${createdSetups.length} setups for dependent seeders`);
// Log completion with memory statistics
logger_1.Logger.info(`✅ Setup seeding completed:`, {
totalUsers: users.length,
totalSetups: totalSetups,
batchesProcessed: stats.batchesCompleted,
peakMemoryMB: stats.peakMemoryUsageMB.toFixed(1),
avgMemoryMB: stats.averageMemoryUsageMB.toFixed(1),
totalTimeMs: stats.totalProcessingTimeMs
});
console.log(`✅ Created ${totalSetups} setups for ${users.length} users using streaming batch processing`);
// Validate total count matches configuration
const expectedTotal = this.context.config.tables?.setups?.count;
if (expectedTotal && expectedTotal > 0 && totalSetups !== expectedTotal) {
console.warn(`⚠️ Generated ${totalSetups} setups but configured for ${expectedTotal} - distribution may be uneven`);
logger_1.Logger.warn('Setup count mismatch', { expected: expectedTotal, actual: totalSetups });
}
else if (expectedTotal && expectedTotal > 0) {
console.log(`🎯 Successfully generated exactly ${totalSetups} setups as configured`);
logger_1.Logger.info('Setup count validation passed - exact count achieved');
}
// Log recommendations if any
if (recommendations.length > 0) {
logger_1.Logger.info('💡 Performance recommendations:', recommendations);
}
// Force final cleanup and log memory usage
const finalMemoryStats = memory_manager_1.default.getMemoryStats();
logger_1.Logger.info('📊 Final memory usage after setup seeding:', {
heapUsedMB: finalMemoryStats.usage.heapUsedMB.toFixed(1),
heapTotalMB: finalMemoryStats.usage.heapTotalMB.toFixed(1),
percentageUsed: finalMemoryStats.percentageUsed.toFixed(1),
recommendations: finalMemoryStats.recommendations
});
}
}
exports.SetupSeeder = SetupSeeder;
//# sourceMappingURL=setup-seeder.js.map