supa-seed
Version:
A constraint-aware, framework-agnostic database seeding framework with deep PostgreSQL business logic discovery and MakerKit integration support
464 lines • 17.4 kB
JavaScript
;
/**
* Basic File System Asset Loading
* Phase 1, Checkpoint A3 - Foundation for hybrid seeding
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AssetLoader = void 0;
const glob_1 = require("glob");
const promises_1 = require("fs/promises");
const yaml_1 = require("yaml");
const path_1 = __importDefault(require("path"));
const logger_1 = require("../../../core/utils/logger");
class AssetLoader {
constructor() {
this.defaultOptions = {
includeContent: true,
validateSchema: true,
maxFileSize: 10 * 1024 * 1024, // 10MB
allowedExtensions: ['.md', '.markdown', '.json', '.csv', '.jpg', '.jpeg', '.png', '.gif', '.webp']
};
}
/**
* Load assets from file system using glob patterns
*/
async loadAssets(options) {
const opts = { ...this.defaultOptions, ...options };
const baseDir = opts.baseDirectory || process.cwd();
const errors = [];
const assets = [];
logger_1.Logger.info(`📁 Loading assets from ${baseDir} with patterns: ${opts.patterns.join(', ')}`);
try {
// Find all files matching patterns
const allFiles = [];
for (const pattern of opts.patterns) {
const fullPattern = path_1.default.resolve(baseDir, pattern);
const matchingFiles = await (0, glob_1.glob)(fullPattern, { nodir: true });
allFiles.push(...matchingFiles);
}
// Remove duplicates
const uniqueFiles = [...new Set(allFiles)];
logger_1.Logger.info(`🔍 Found ${uniqueFiles.length} potential asset files`);
// Process each file
for (const filePath of uniqueFiles) {
try {
// Check file extension
const ext = path_1.default.extname(filePath).toLowerCase();
if (opts.allowedExtensions && !opts.allowedExtensions.includes(ext)) {
errors.push({ filePath, error: `Unsupported file extension: ${ext}` });
continue;
}
// Check file size
const stats = await (0, promises_1.stat)(filePath);
if (opts.maxFileSize && stats.size > opts.maxFileSize) {
errors.push({
filePath,
error: `File too large: ${stats.size} bytes > ${opts.maxFileSize} bytes`
});
continue;
}
// Load asset based on type
const asset = await this.loadSingleAsset(filePath, stats, opts);
if (asset) {
assets.push(asset);
}
}
catch (error) {
errors.push({ filePath, error: error.message });
logger_1.Logger.warn(`⚠️ Failed to load asset ${filePath}: ${error.message}`);
}
}
logger_1.Logger.info(`✅ Loaded ${assets.length}/${uniqueFiles.length} assets successfully`);
return {
assets,
errors,
totalFiles: uniqueFiles.length,
successCount: assets.length,
errorCount: errors.length
};
}
catch (error) {
logger_1.Logger.error('Asset loading failed:', error);
throw new Error(`Asset loading failed: ${error.message}`);
}
}
/**
* Load a single asset file
*/
async loadSingleAsset(filePath, stats, options) {
const ext = path_1.default.extname(filePath).toLowerCase();
const filename = path_1.default.basename(filePath, ext);
const id = this.generateAssetId(filePath);
let type;
let content;
let metadata = {};
let isValid = true;
let validationErrors = [];
// Determine asset type
if (['.md', '.markdown'].includes(ext)) {
type = 'markdown';
}
else if (ext === '.json') {
type = 'json';
}
else if (ext === '.csv') {
type = 'csv';
}
else if (['.jpg', '.jpeg', '.png', '.gif', '.webp'].includes(ext)) {
type = 'image';
}
else {
return null; // Skip unsupported files
}
try {
let rawContent;
// Always read content for metadata parsing (except images)
if (type !== 'image') {
rawContent = await (0, promises_1.readFile)(filePath, 'utf-8');
}
// Parse metadata based on type
switch (type) {
case 'markdown':
const markdownResult = this.parseMarkdownWithFrontmatter(rawContent || '');
metadata = markdownResult.metadata;
// Only include parsed content if requested
content = options.includeContent ? markdownResult.content : undefined;
if (options.validateSchema) {
const validation = this.validateMarkdownAsset(metadata, markdownResult.content);
isValid = validation.isValid;
validationErrors = validation.errors;
}
break;
case 'json':
try {
const jsonData = JSON.parse(rawContent || '{}');
metadata = this.extractJsonMetadata(jsonData);
// Only include raw content if requested
content = options.includeContent ? rawContent : undefined;
if (options.validateSchema) {
const validation = this.validateJsonAsset(jsonData);
isValid = validation.isValid;
validationErrors = validation.errors;
}
}
catch (error) {
isValid = false;
validationErrors.push(`Invalid JSON: ${error.message}`);
}
break;
case 'csv':
try {
const csvData = this.parseCsvData(rawContent || '');
metadata = this.extractCsvMetadata(csvData, rawContent || '');
// Only include raw content if requested
content = options.includeContent ? rawContent : undefined;
if (options.validateSchema) {
const validation = this.validateCsvAsset(csvData, rawContent || '');
isValid = validation.isValid;
validationErrors = validation.errors;
}
}
catch (error) {
isValid = false;
validationErrors.push(`Invalid CSV: ${error.message}`);
}
break;
case 'image':
metadata = await this.extractImageMetadata(filePath, stats);
content = undefined; // Images never have text content
if (options.validateSchema) {
const validation = this.validateImageAsset(metadata);
isValid = validation.isValid;
validationErrors = validation.errors;
}
break;
}
// Add common metadata
metadata.filename = filename;
metadata.extension = ext;
metadata.path = filePath;
return {
id,
filePath,
type,
content,
metadata,
fileSize: stats.size,
lastModified: new Date(stats.mtime),
isValid,
validationErrors: validationErrors.length > 0 ? validationErrors : undefined
};
}
catch (error) {
logger_1.Logger.warn(`Failed to process ${type} asset ${filePath}: ${error.message}`);
return null;
}
}
/**
* Parse markdown file with frontmatter
*/
parseMarkdownWithFrontmatter(content) {
const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
const match = content.match(frontmatterRegex);
if (match) {
try {
const frontmatter = (0, yaml_1.parse)(match[1]) || {};
return {
metadata: frontmatter,
content: match[2].trim()
};
}
catch (error) {
logger_1.Logger.warn(`Failed to parse frontmatter: ${error.message}`);
return { metadata: {}, content };
}
}
return { metadata: {}, content };
}
/**
* Extract metadata from JSON data
*/
extractJsonMetadata(data) {
const metadata = {};
// Common metadata fields
const metadataFields = ['title', 'description', 'tags', 'category', 'author', 'date', 'featured'];
for (const field of metadataFields) {
if (data[field] !== undefined) {
metadata[field] = data[field];
}
}
// If data has a metadata object, merge it
if (data.metadata && typeof data.metadata === 'object') {
Object.assign(metadata, data.metadata);
}
return metadata;
}
/**
* Extract metadata from image files
*/
async extractImageMetadata(filePath, stats) {
const filename = path_1.default.basename(filePath);
const ext = path_1.default.extname(filePath);
return {
filename: path_1.default.basename(filePath, ext),
extension: ext,
mimeType: this.getMimeType(ext),
width: undefined, // Would need image processing library for actual dimensions
height: undefined,
aspectRatio: undefined,
fileSize: stats.size,
lastModified: stats.mtime
};
}
/**
* Validate markdown asset
*/
validateMarkdownAsset(metadata, content) {
const errors = [];
// Basic validation rules
if (!content || content.trim().length === 0) {
errors.push('Markdown content is empty');
}
if (content.length < 10) {
errors.push('Markdown content is too short (minimum 10 characters)');
}
if (metadata.tags && !Array.isArray(metadata.tags)) {
errors.push('Tags must be an array');
}
if (metadata.date && isNaN(Date.parse(metadata.date))) {
errors.push('Invalid date format');
}
return {
isValid: errors.length === 0,
errors
};
}
/**
* Validate JSON asset
*/
validateJsonAsset(data) {
const errors = [];
if (data === null || data === undefined) {
errors.push('JSON data is null or undefined');
}
if (typeof data !== 'object') {
errors.push('JSON root must be an object');
}
// Additional JSON-specific validation can be added here
return {
isValid: errors.length === 0,
errors
};
}
/**
* Validate image asset
*/
validateImageAsset(metadata) {
const errors = [];
if (!metadata.mimeType) {
errors.push('Missing MIME type');
}
if (metadata.fileSize && metadata.fileSize < 50) {
errors.push('Image file too small (possible corruption)');
}
return {
isValid: errors.length === 0,
errors
};
}
/**
* Generate unique asset ID from file path
*/
generateAssetId(filePath) {
return Buffer.from(filePath).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 16);
}
/**
* Get MIME type from file extension
*/
getMimeType(ext) {
const mimeTypes = {
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.gif': 'image/gif',
'.webp': 'image/webp',
'.md': 'text/markdown',
'.markdown': 'text/markdown',
'.json': 'application/json',
'.csv': 'text/csv'
};
return mimeTypes[ext.toLowerCase()] || 'application/octet-stream';
}
/**
* Parse CSV data into structured format
*/
parseCsvData(content) {
const lines = content.split('\n').filter(line => line.trim());
if (lines.length === 0) {
return [];
}
// Simple CSV parser - assumes comma separation and no nested quotes
const headers = this.parseCsvLine(lines[0]);
const rows = [];
for (let i = 1; i < lines.length; i++) {
const values = this.parseCsvLine(lines[i]);
if (values.length === headers.length) {
const row = {};
headers.forEach((header, index) => {
row[header.trim()] = values[index]?.trim() || '';
});
rows.push(row);
}
}
return rows;
}
/**
* Parse a single CSV line
*/
parseCsvLine(line) {
const result = [];
let current = '';
let inQuotes = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
if (char === '"') {
inQuotes = !inQuotes;
}
else if (char === ',' && !inQuotes) {
result.push(current);
current = '';
}
else {
current += char;
}
}
result.push(current);
return result;
}
/**
* Extract metadata from CSV data
*/
extractCsvMetadata(data, rawContent) {
const metadata = {};
// Basic CSV statistics
metadata.rowCount = data.length;
metadata.columns = data.length > 0 ? Object.keys(data[0]) : [];
metadata.columnCount = metadata.columns.length;
// Try to detect content type from column names
const columnNames = metadata.columns.map(col => col.toLowerCase());
if (columnNames.includes('title') || columnNames.includes('name')) {
metadata.contentType = 'structured_data';
}
if (columnNames.includes('email') || columnNames.includes('user')) {
metadata.dataType = 'users';
}
else if (columnNames.some(col => col.includes('product')) || columnNames.includes('item')) {
metadata.dataType = 'products';
}
else if (columnNames.includes('post') || columnNames.includes('article')) {
metadata.dataType = 'content';
}
else {
metadata.dataType = 'generic';
}
// Sample first few rows for preview
metadata.sampleRows = data.slice(0, 3);
return metadata;
}
/**
* Validate CSV asset
*/
validateCsvAsset(data, rawContent) {
const errors = [];
if (!rawContent || rawContent.trim().length === 0) {
errors.push('CSV content is empty');
}
if (data.length === 0) {
errors.push('No valid CSV rows found');
}
// Check for consistent column count
if (data.length > 0) {
const expectedColumns = Object.keys(data[0]).length;
for (let i = 0; i < data.length; i++) {
if (Object.keys(data[i]).length !== expectedColumns) {
errors.push(`Row ${i + 1} has inconsistent column count`);
break; // Only report first inconsistency
}
}
}
// Check for minimum rows
if (data.length < 1) {
errors.push('CSV must have at least 1 data row');
}
return {
isValid: errors.length === 0,
errors
};
}
/**
* Get asset loading statistics
*/
static getLoadingStats(result) {
const successRate = result.totalFiles > 0 ? (result.successCount / result.totalFiles) * 100 : 0;
const avgFileSize = result.assets.length > 0
? result.assets.reduce((sum, asset) => sum + asset.fileSize, 0) / result.assets.length
: 0;
const typeDistribution = {};
result.assets.forEach(asset => {
typeDistribution[asset.type] = (typeDistribution[asset.type] || 0) + 1;
});
const validAssets = result.assets.filter(asset => asset.isValid).length;
const validationRate = result.assets.length > 0 ? (validAssets / result.assets.length) * 100 : 0;
return {
successRate,
avgFileSize,
typeDistribution,
validationRate
};
}
}
exports.AssetLoader = AssetLoader;
//# sourceMappingURL=asset-loader.js.map