claude-flow-novice
Version:
Claude Flow Novice - Advanced orchestration platform for multi-agent AI workflows with CFN Loop architecture Includes Local RuVector Accelerator and all CFN skills for complete functionality.
695 lines (694 loc) • 23.6 kB
JavaScript
/**
* Schema Transform Library
*
* Provides bidirectional transformations between SQLite and Redis schemas.
* Ensures data consistency and prevents data loss during transformations.
*
* Task: Integration Standardization Plan - Task 2.2
* Version: 1.0.0
*/ import { getGlobalLogger } from './logging.js';
import { StandardError } from './errors.js';
const logger = getGlobalLogger();
// ============================================================================
// Type Converters
// ============================================================================
export const TYPE_CONVERTERS = {
/**
* DECIMAL to REAL (PostgreSQL → SQLite)
*/ 'DECIMAL_TO_REAL': (val)=>{
if (val === null || val === undefined) return null;
const num = typeof val === 'string' ? parseFloat(val) : val;
return isNaN(num) ? null : num;
},
/**
* INTEGER conversion (handles string/number inputs)
*/ 'INTEGER': (val)=>{
if (val === null || val === undefined) return null;
const num = typeof val === 'string' ? parseInt(val, 10) : val;
return isNaN(num) ? null : num;
},
/**
* REAL conversion (handles string/number inputs)
*/ 'REAL': (val)=>{
if (val === null || val === undefined) return null;
const num = typeof val === 'string' ? parseFloat(val) : val;
return isNaN(num) ? null : num;
},
/**
* Number to string (for Redis storage)
*/ 'NUMBER_TO_STRING': (val)=>{
if (val === null || val === undefined) return null;
if (isNaN(val)) return null;
return val.toString();
},
/**
* String to number (from Redis)
*/ 'STRING_TO_NUMBER': (val)=>{
if (val === null || val === undefined || val === '') return null;
const num = parseFloat(val);
return isNaN(num) ? null : num;
},
/**
* ENUM to TEXT (PostgreSQL → SQLite)
*/ 'ENUM_TO_TEXT': (val)=>{
if (val === null || val === undefined) return null;
return val.toString();
},
/**
* DATETIME string to Unix timestamp (SQLite → Redis)
*/ 'DATETIME_TO_TIMESTAMP': (val)=>{
if (val === null || val === undefined) return null;
const timestamp = new Date(val).getTime();
return isNaN(timestamp) ? null : timestamp;
},
/**
* Unix timestamp to DATETIME string (Redis → SQLite)
*/ 'TIMESTAMP_TO_DATETIME': (val)=>{
if (val === null || val === undefined) return null;
const num = typeof val === 'string' ? parseInt(val, 10) : val;
if (isNaN(num)) return null;
return new Date(num).toISOString();
},
/**
* PostgreSQL TIMESTAMP to SQLite DATETIME
*/ 'TIMESTAMP_TO_ISO': (val)=>{
if (val === null || val === undefined) return null;
try {
if (val instanceof Date) return val.toISOString();
if (typeof val === 'number') return new Date(val * 1000).toISOString();
return new Date(val).toISOString();
} catch {
return null;
}
},
/**
* Parse JSON string to object
*/ 'TEXT_TO_JSON': (val)=>{
if (val === null || val === undefined || val === '') return null;
try {
return JSON.parse(val);
} catch (err) {
logger.warn('Failed to parse JSON', {
value: val,
error: err.message
});
return null;
}
},
/**
* Stringify object to JSON
*/ 'JSON_TO_TEXT': (val)=>{
if (val === null || val === undefined) return null;
try {
return JSON.stringify(val);
} catch (err) {
logger.warn('Failed to stringify JSON', {
value: val,
error: err.message
});
return null;
}
},
/**
* Boolean to INTEGER (0/1 for SQLite)
*/ 'BOOLEAN_TO_INTEGER': (val)=>{
if (val === null || val === undefined) return null;
if (typeof val === 'number') return val === 1 ? 1 : 0;
return val ? 1 : 0;
},
/**
* INTEGER to Boolean (SQLite → JavaScript)
*/ 'INTEGER_TO_BOOLEAN': (val)=>{
if (val === null || val === undefined) return null;
const num = typeof val === 'string' ? parseInt(val, 10) : val;
return num === 1;
},
/**
* Identity transform (no conversion needed)
*/ 'IDENTITY': (val)=>val
};
// ============================================================================
// Schema Mappings
// ============================================================================
export const SCHEMA_MAPPINGS = {
/**
* Agent Executions: Redis ↔ SQLite
*/ agent_executions: {
version: '1.0.0',
schema: 'agent_executions',
primaryKey: 'agent_id',
ttl: 86400,
fields: [
{
source: 'id',
destination: 'agent_id',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'agent_id',
destination: 'agent_id',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'task_id',
destination: 'task_id',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'type',
destination: 'type',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'status',
destination: 'status',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'confidence',
destination: 'confidence',
sourceType: 'REAL',
destinationType: 'string',
transform: null,
required: false
},
{
source: 'spawned_at',
destination: 'spawned_at',
sourceType: 'DATETIME',
destinationType: 'number',
transform: null,
required: true
},
{
source: 'completed_at',
destination: 'completed_at',
sourceType: 'DATETIME',
destinationType: 'number',
transform: null,
required: false
},
{
source: 'metadata',
destination: 'metadata',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: false
}
]
},
/**
* Skill Executions: PostgreSQL → SQLite (one-way)
*/ skill_executions: {
version: '1.0.0',
schema: 'skill_executions',
primaryKey: 'id',
fields: [
{
source: 'id',
destination: 'id',
sourceType: 'SERIAL',
destinationType: 'INTEGER',
transform: TYPE_CONVERTERS.INTEGER,
required: true
},
{
source: 'skill_id',
destination: 'skill_id',
sourceType: 'INTEGER',
destinationType: 'INTEGER',
transform: TYPE_CONVERTERS.INTEGER,
required: true
},
{
source: 'execution_time_ms',
destination: 'execution_time_ms',
sourceType: 'INTEGER',
destinationType: 'INTEGER',
transform: TYPE_CONVERTERS.INTEGER,
required: true
},
{
source: 'cost_usd',
destination: 'cost_usd',
sourceType: 'DECIMAL',
destinationType: 'REAL',
transform: TYPE_CONVERTERS.DECIMAL_TO_REAL,
required: false
},
{
source: 'tokens_avoided',
destination: 'tokens_avoided',
sourceType: 'INTEGER',
destinationType: 'INTEGER',
transform: TYPE_CONVERTERS.INTEGER,
required: false
},
{
source: 'status',
destination: 'status',
sourceType: 'ENUM',
destinationType: 'TEXT',
transform: TYPE_CONVERTERS.ENUM_TO_TEXT,
required: true
},
{
source: 'executed_at',
destination: 'executed_at',
sourceType: 'TIMESTAMP',
destinationType: 'DATETIME',
transform: TYPE_CONVERTERS.TIMESTAMP_TO_ISO,
required: true
},
{
source: 'metadata',
destination: 'metadata',
sourceType: 'JSONB',
destinationType: 'TEXT',
transform: TYPE_CONVERTERS.JSON_TO_TEXT,
required: false
}
]
},
/**
* Artifacts: SQLite ↔ Redis (cache layer)
*/ artifacts: {
version: '1.0.0',
schema: 'artifacts',
primaryKey: 'id',
ttl: 2592000,
fields: [
{
source: 'id',
destination: 'artifact_id',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'name',
destination: 'name',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'type',
destination: 'type',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'content',
destination: 'content',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: false
},
{
source: 'content_hash',
destination: 'content_hash',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: false
},
{
source: 'size_bytes',
destination: 'size_bytes',
sourceType: 'INTEGER',
destinationType: 'string',
transform: null,
required: false
},
{
source: 'metadata',
destination: 'metadata',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: false
},
{
source: 'created_at',
destination: 'created_at',
sourceType: 'DATETIME',
destinationType: 'number',
transform: null,
required: true
},
{
source: 'expires_at',
destination: 'expires_at',
sourceType: 'DATETIME',
destinationType: 'number',
transform: null,
required: false
},
{
source: 'status',
destination: 'status',
sourceType: 'TEXT',
destinationType: 'string',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
}
]
},
/**
* Coordination Events: Redis → SQLite (archival)
*/ coordination_events: {
version: '1.0.0',
schema: 'coordination_events',
primaryKey: 'event_id',
ttl: 3600,
fields: [
{
source: 'event_id',
destination: 'id',
sourceType: 'string',
destinationType: 'TEXT',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'task_id',
destination: 'task_id',
sourceType: 'string',
destinationType: 'TEXT',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'agent_id',
destination: 'agent_id',
sourceType: 'string',
destinationType: 'TEXT',
transform: TYPE_CONVERTERS.IDENTITY,
required: false
},
{
source: 'event_type',
destination: 'event_type',
sourceType: 'string',
destinationType: 'TEXT',
transform: TYPE_CONVERTERS.IDENTITY,
required: true
},
{
source: 'timestamp',
destination: 'timestamp',
sourceType: 'number',
destinationType: 'DATETIME',
transform: null,
required: true
},
{
source: 'payload',
destination: 'payload',
sourceType: 'string',
destinationType: 'TEXT',
transform: TYPE_CONVERTERS.IDENTITY,
required: false
}
]
}
};
// ============================================================================
// Transform Functions
// ============================================================================
/**
* Transform SQLite row to Redis data
*/ export function sqliteToRedis(schema, row) {
const errors = [];
const warnings = [];
const result = {};
const mapping = SCHEMA_MAPPINGS[schema];
if (!mapping) {
throw new StandardError(`Unknown schema: ${schema}`, 'SCHEMA_TRANSFORM_ERROR');
}
for (const field of mapping.fields){
const value = row[field.source];
// Handle required fields
if (field.required && (value === null || value === undefined)) {
errors.push(`Required field '${field.source}' is missing`);
continue;
}
// Skip null/undefined for optional fields
if (value === null || value === undefined) {
continue;
}
// Apply transformation
let transformed;
if (field.transform) {
transformed = field.transform(value);
} else {
// Direction-specific transforms
if (field.sourceType === 'REAL' && field.destinationType === 'string') {
transformed = TYPE_CONVERTERS.NUMBER_TO_STRING(value);
} else if (field.sourceType === 'INTEGER' && field.destinationType === 'string') {
transformed = TYPE_CONVERTERS.NUMBER_TO_STRING(value);
} else if (field.sourceType === 'DATETIME' && field.destinationType === 'number') {
transformed = TYPE_CONVERTERS.DATETIME_TO_TIMESTAMP(value);
} else {
transformed = value;
}
}
// Validate transformation
if (transformed === null && field.required) {
warnings.push(`Transformation of required field '${field.source}' resulted in null`);
}
// Only add non-null values to Redis (Redis doesn't have NULL)
if (transformed !== null) {
result[field.destination] = transformed;
}
}
if (errors.length > 0) {
return {
success: false,
errors,
warnings
};
}
logger.debug('SQLite → Redis transformation completed', {
schema,
fieldCount: Object.keys(result).length,
warnings: warnings.length
});
return {
success: true,
data: result,
warnings: warnings.length > 0 ? warnings : undefined
};
}
/**
* Transform Redis data to SQLite row
*/ export function redisToSqlite(schema, data) {
const errors = [];
const warnings = [];
const result = {};
const mapping = SCHEMA_MAPPINGS[schema];
if (!mapping) {
throw new StandardError(`Unknown schema: ${schema}`, 'SCHEMA_TRANSFORM_ERROR');
}
for (const field of mapping.fields){
const value = data[field.destination];
// Handle required fields
if (field.required && (value === null || value === undefined)) {
errors.push(`Required field '${field.destination}' is missing`);
continue;
}
// Set null for missing optional fields
if (value === null || value === undefined) {
result[field.source] = null;
continue;
}
// Apply transformation
let transformed;
if (field.transform) {
transformed = field.transform(value);
} else {
// Direction-specific transforms
if (field.destinationType === 'string' && field.sourceType === 'REAL') {
transformed = TYPE_CONVERTERS.STRING_TO_NUMBER(value);
} else if (field.destinationType === 'string' && field.sourceType === 'INTEGER') {
transformed = TYPE_CONVERTERS.STRING_TO_NUMBER(value);
} else if (field.destinationType === 'number' && field.sourceType === 'DATETIME') {
transformed = TYPE_CONVERTERS.TIMESTAMP_TO_DATETIME(value);
} else {
transformed = value;
}
}
// Validate transformation
if (transformed === null && field.required) {
warnings.push(`Transformation of required field '${field.destination}' resulted in null`);
}
result[field.source] = transformed;
}
if (errors.length > 0) {
return {
success: false,
errors,
warnings
};
}
logger.debug('Redis → SQLite transformation completed', {
schema,
fieldCount: Object.keys(result).length,
warnings: warnings.length
});
return {
success: true,
data: result,
warnings: warnings.length > 0 ? warnings : undefined
};
}
/**
* Transform PostgreSQL row to SQLite row (one-way)
*/ export function postgrestoSqlite(schema, row) {
if (schema !== 'skill_executions') {
throw new StandardError(`PostgreSQL → SQLite transform only supports 'skill_executions', got: ${schema}`, 'SCHEMA_TRANSFORM_ERROR');
}
const errors = [];
const warnings = [];
const result = {};
const mapping = SCHEMA_MAPPINGS[schema];
for (const field of mapping.fields){
const value = row[field.source];
// Handle required fields
if (field.required && (value === null || value === undefined)) {
errors.push(`Required field '${field.source}' is missing`);
continue;
}
// Set null for missing optional fields
if (value === null || value === undefined) {
result[field.destination] = null;
continue;
}
// Apply transformation
const transformed = field.transform ? field.transform(value) : value;
// Validate transformation
if (transformed === null && field.required) {
warnings.push(`Transformation of required field '${field.source}' resulted in null`);
}
result[field.destination] = transformed;
}
if (errors.length > 0) {
return {
success: false,
errors,
warnings
};
}
logger.debug('PostgreSQL → SQLite transformation completed', {
schema,
fieldCount: Object.keys(result).length,
warnings: warnings.length
});
return {
success: true,
data: result,
warnings: warnings.length > 0 ? warnings : undefined
};
}
/**
* Transform batch of records
*/ export function transformBatch(schema, data, direction) {
const results = [];
const allErrors = [];
const allWarnings = [];
for(let i = 0; i < data.length; i++){
const item = data[i];
let transformResult;
try {
switch(direction){
case 'sqlite-to-redis':
transformResult = sqliteToRedis(schema, item);
break;
case 'redis-to-sqlite':
transformResult = redisToSqlite(schema, item);
break;
case 'postgres-to-sqlite':
transformResult = postgrestoSqlite(schema, item);
break;
default:
throw new StandardError(`Unknown transform direction: ${direction}`, 'SCHEMA_TRANSFORM_ERROR');
}
if (transformResult.success && transformResult.data) {
results.push(transformResult.data);
} else {
allErrors.push(`Record ${i}: ${transformResult.errors?.join(', ')}`);
}
if (transformResult.warnings) {
allWarnings.push(...transformResult.warnings.map((w)=>`Record ${i}: ${w}`));
}
} catch (err) {
allErrors.push(`Record ${i}: ${err.message}`);
}
}
logger.info('Batch transformation completed', {
schema,
direction,
total: data.length,
succeeded: results.length,
failed: allErrors.length,
warnings: allWarnings.length
});
if (allErrors.length > 0) {
return {
success: false,
data: results,
errors: allErrors,
warnings: allWarnings.length > 0 ? allWarnings : undefined
};
}
return {
success: true,
data: results,
warnings: allWarnings.length > 0 ? allWarnings : undefined
};
}
/**
* Get schema mapping for a given schema
*/ export function getSchemaMapping(schema) {
const mapping = SCHEMA_MAPPINGS[schema];
if (!mapping) {
throw new StandardError(`Unknown schema: ${schema}`, 'SCHEMA_TRANSFORM_ERROR');
}
return mapping;
}
/**
* Check if transformation is supported
*/ export function isTransformSupported(schema, direction) {
if (!SCHEMA_MAPPINGS[schema]) {
return false;
}
// skill_executions only supports postgres-to-sqlite
if (schema === 'skill_executions') {
return direction === 'postgres-to-sqlite';
}
// Other schemas support bidirectional
return direction === 'sqlite-to-redis' || direction === 'redis-to-sqlite';
}
//# sourceMappingURL=schema-transform.js.map