@yihuangdb/storage-object
Version:
A Node.js storage object layer library using Redis OM
737 lines • 24.1 kB
JavaScript
"use strict";
/**
* @module storage-schema
* @description Enhanced Object-Oriented schema management for StorageObject.
* Provides comprehensive schema definition, validation, migration, and transformation capabilities.
*
* @since 0.1.0
* @author StorageObject Team
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.StorageSchema = void 0;
const redis_om_1 = require("redis-om");
const schema_versioning_1 = require("./schema-versioning");
const redis_om_version_validator_1 = require("./redis-om-version-validator");
/**
* StorageSchema - Enhanced schema management with validation and migration capabilities.
*
* This class provides a comprehensive API for defining, validating, and transforming storage schemas.
* It supports schema evolution through extension, merging, and migration planning.
*
* @class StorageSchema
* @template T - Type of the entities described by this schema
* @since 0.1.0
*
* @example Basic Schema Definition
* ```typescript
* // Define a schema using the static factory method
* const userSchema = StorageSchema.define({
* name: { type: 'string', indexed: true },
* email: { type: 'string', indexed: true, required: true },
* age: { type: 'number', indexed: true, validate: (v) => v >= 0 || 'Age must be positive' }
* });
*
* // Create storage with the schema fields
* const storage = await StorageSystem.create('users', userSchema.getFields());
* ```
*
* @example Schema Validation
* ```typescript
* // Validate data against schema
* const result = userSchema.validate({
* name: 'John Doe',
* email: 'john@example.com',
* age: 25
* });
*
* if (!result.valid) {
* console.error('Validation errors:', result.errors);
* }
* ```
*
* @example Schema Evolution
* ```typescript
* // Extend existing schema
* const extendedSchema = userSchema.extend({
* role: { type: 'string', indexed: true, default: 'user' },
* tags: { type: 'string[]', separator: ',', indexed: true }
* });
*
* // Merge two schemas
* const mergedSchema = userSchema.merge(profileSchema);
*
* // Plan migration between schemas
* const migrationPlan = userSchema.migrateTo(newSchema);
* console.log('Migration steps:', migrationPlan.steps);
* ```
*/
class StorageSchema {
schema;
fields;
dataStructure;
entityName;
versionManager;
versionValidator;
/**
* Create a new StorageSchema instance
*
* @param entityName - Name of the entity
* @param fields - Schema field configuration
* @param useJSON - Whether to use JSON or HASH structure (default: true)
*/
constructor(entityName, fields, useJSON = true) {
this.entityName = entityName;
this.fields = this.normalizeFields(fields);
this.dataStructure = useJSON ? 'JSON' : 'HASH';
// Initialize Redis OM version validator
this.versionValidator = (0, redis_om_version_validator_1.getRedisOMVersionValidator)();
const schemaDefinition = this.buildSchemaDefinition(this.fields);
this.schema = new redis_om_1.Schema(entityName, schemaDefinition, {
dataStructure: this.dataStructure,
});
}
/**
* Define a new schema using a static factory method
*
* @param config - Schema configuration
* @param options - Schema options
* @returns New StorageSchema instance
*
* @example
* ```typescript
* const schema = StorageSchema.define({
* name: 'string',
* age: 'number',
* email: { type: 'string', indexed: true }
* });
* ```
*/
static define(config, options) {
const entityName = options?.entityName || 'Entity';
const useJSON = options?.useJSON !== false;
return new StorageSchema(entityName, config, useJSON);
}
/**
* Create a schema from a JSON string or object
*
* @param json - JSON string or parsed object
* @param options - Schema options
* @returns New StorageSchema instance
*
* @example
* ```typescript
* const schema = StorageSchema.from(jsonString);
* ```
*/
static from(json, options) {
let parsed;
if (typeof json === 'string') {
try {
parsed = JSON.parse(json);
}
catch (error) {
throw new Error(`Invalid JSON for schema: ${error instanceof Error ? error.message : String(error)}`);
}
}
else {
parsed = json;
}
if (!parsed.fields && !parsed.schema) {
throw new Error('Invalid schema format: missing fields or schema property');
}
const fields = parsed.fields || parsed.schema || parsed;
const entityName = options?.entityName || parsed.entityName || 'Entity';
const useJSON = options?.useJSON !== false && parsed.dataStructure !== 'HASH';
return new StorageSchema(entityName, fields, useJSON);
}
/**
* Validate data against the schema
*
* @param data - Data to validate
* @returns Validation result
*
* @example
* ```typescript
* const result = schema.validate({ name: 'John', age: 30 });
* if (!result.valid) {
* console.error('Validation errors:', result.errors);
* }
* ```
*/
validate(data) {
const result = {
valid: true,
errors: [],
warnings: [],
};
if (!data || typeof data !== 'object') {
result.valid = false;
result.errors.push({
field: '_root',
message: 'Data must be an object',
value: data,
});
return result;
}
// Check required fields
for (const [fieldName, fieldConfig] of Object.entries(this.fields)) {
const config = this.getFieldConfig(fieldConfig);
// Check if required field is missing
if (config.required && !(fieldName in data)) {
result.valid = false;
result.errors.push({
field: fieldName,
message: `Required field '${fieldName}' is missing`,
});
continue;
}
// Skip validation if field is not present and not required
if (!(fieldName in data)) {
continue;
}
const value = data[fieldName];
// Type validation
if (!this.validateFieldType(value, config.type)) {
result.valid = false;
result.errors.push({
field: fieldName,
message: `Field '${fieldName}' must be of type ${config.type}`,
value,
});
}
// Custom validation
if (config.validate) {
const validationResult = config.validate(value);
if (validationResult !== true) {
result.valid = false;
result.errors.push({
field: fieldName,
message: typeof validationResult === 'string'
? validationResult
: `Field '${fieldName}' failed validation`,
value,
});
}
}
}
// Check for unknown fields
for (const fieldName of Object.keys(data)) {
if (!(fieldName in this.fields) && fieldName !== 'entityId') {
result.warnings.push({
field: fieldName,
message: `Unknown field '${fieldName}' in data`,
});
}
}
return result;
}
/**
* Get all fields in the schema
*
* @returns Schema field configuration
*/
getFields() {
return { ...this.fields };
}
/**
* Get indexed fields in the schema
*
* @returns Array of indexed field names
*
* @example
* ```typescript
* const indexedFields = schema.getIndexedFields();
* console.log('Indexed fields:', indexedFields);
* ```
*/
getIndexedFields() {
const indexed = [];
for (const [fieldName, fieldConfig] of Object.entries(this.fields)) {
const config = this.getFieldConfig(fieldConfig);
if (config.indexed) {
indexed.push(fieldName);
}
}
return indexed;
}
/**
* Extend the schema with additional fields
*
* @param additionalFields - Fields to add to the schema
* @returns New extended StorageSchema instance
*
* @example
* ```typescript
* const extendedSchema = schema.extend({
* createdAt: { type: 'date', indexed: true },
* updatedAt: { type: 'date', indexed: true }
* });
* ```
*/
extend(additionalFields) {
const mergedFields = {
...this.fields,
...additionalFields,
};
return new StorageSchema(this.entityName, mergedFields, this.isJSON());
}
/**
* Merge this schema with another schema
*
* @param otherSchema - Schema to merge with
* @param strategy - Merge strategy ('override' or 'preserve')
* @returns New merged StorageSchema instance
*
* @example
* ```typescript
* const mergedSchema = schema1.merge(schema2, 'override');
* ```
*/
merge(otherSchema, strategy = 'override') {
const otherFields = otherSchema.getFields();
let mergedFields;
if (strategy === 'override') {
mergedFields = {
...this.fields,
...otherFields,
};
}
else {
mergedFields = {
...otherFields,
...this.fields,
};
}
return new StorageSchema(this.entityName, mergedFields, this.isJSON() && otherSchema.isJSON());
}
/**
* Convert the schema to a JSON representation
*
* @returns JSON object representing the schema
*
* @example
* ```typescript
* const json = schema.toJSON();
* console.log(JSON.stringify(json, null, 2));
* ```
*/
toJSON() {
return {
entityName: this.entityName,
dataStructure: this.dataStructure,
fields: this.fields,
};
}
/**
* Create a deep copy of the schema
*
* @returns New cloned StorageSchema instance
*
* @example
* ```typescript
* const clonedSchema = schema.clone();
* ```
*/
clone() {
// Deep clone the fields
const clonedFields = JSON.parse(JSON.stringify(this.fields));
// Restore any functions that were lost in JSON serialization
for (const [fieldName, fieldConfig] of Object.entries(this.fields)) {
if (typeof fieldConfig === 'object' && fieldConfig.validate) {
clonedFields[fieldName].validate = fieldConfig.validate;
}
}
return new StorageSchema(this.entityName, clonedFields, this.isJSON());
}
/**
* Check if this schema is equal to another schema
*
* @param otherSchema - Schema to compare with
* @returns True if schemas are equal, false otherwise
*
* @example
* ```typescript
* if (schema1.equals(schema2)) {
* console.log('Schemas are identical');
* }
* ```
*/
equals(otherSchema) {
if (this.entityName !== otherSchema.entityName) {
return false;
}
if (this.dataStructure !== otherSchema.dataStructure) {
return false;
}
const thisFields = this.getFields();
const otherFields = otherSchema.getFields();
const thisKeys = Object.keys(thisFields).sort();
const otherKeys = Object.keys(otherFields).sort();
if (thisKeys.length !== otherKeys.length) {
return false;
}
for (let i = 0; i < thisKeys.length; i++) {
if (thisKeys[i] !== otherKeys[i]) {
return false;
}
const thisField = thisFields[thisKeys[i]];
const otherField = otherFields[otherKeys[i]];
if (!this.fieldsEqual(thisField, otherField)) {
return false;
}
}
return true;
}
/**
* Find differences between this schema and another schema
*
* @param otherSchema - Schema to compare with
* @returns Difference report
*
* @example
* ```typescript
* const diff = schema1.diff(schema2);
* console.log('Added fields:', diff.added);
* console.log('Removed fields:', diff.removed);
* console.log('Modified fields:', diff.modified);
* ```
*/
diff(otherSchema) {
const thisFields = this.getFields();
const otherFields = otherSchema.getFields();
const thisKeys = new Set(Object.keys(thisFields));
const otherKeys = new Set(Object.keys(otherFields));
const added = [];
const removed = [];
const modified = [];
// Find added fields
for (const key of otherKeys) {
if (!thisKeys.has(key)) {
added.push(key);
}
}
// Find removed fields
for (const key of thisKeys) {
if (!otherKeys.has(key)) {
removed.push(key);
}
}
// Find modified fields
for (const key of thisKeys) {
if (otherKeys.has(key)) {
const thisField = thisFields[key];
const otherField = otherFields[key];
if (!this.fieldsEqual(thisField, otherField)) {
modified.push({
field: key,
oldConfig: thisField,
newConfig: otherField,
});
}
}
}
return {
added,
removed,
modified,
identical: added.length === 0 && removed.length === 0 && modified.length === 0,
};
}
/**
* Create a migration plan to transform from this schema to another schema
*
* @param newSchema - Target schema to migrate to
* @param options - Migration options
* @returns Migration plan
*
* @example
* ```typescript
* const plan = oldSchema.migrateTo(newSchema, {
* transformers: {
* fullName: (data) => `${data.firstName} ${data.lastName}`
* }
* });
*
* if (plan.safe) {
* console.log('Migration is safe, no data loss');
* } else {
* console.warn('Migration warnings:', plan.warnings);
* }
* ```
*/
migrateTo(newSchema, options) {
const diff = this.diff(newSchema);
const steps = [];
const warnings = [];
let safe = true;
// Handle removed fields
for (const field of diff.removed) {
if (!options?.allowDataLoss) {
safe = false;
warnings.push(`Field '${field}' will be removed, potential data loss`);
}
steps.push({
action: 'remove',
field,
description: `Remove field '${field}'`,
});
}
// Handle added fields
for (const field of diff.added) {
const fieldConfig = this.getFieldConfig(newSchema.getFields()[field]);
if (fieldConfig.required && !fieldConfig.default && !options?.transformers?.[field]) {
safe = false;
warnings.push(`New required field '${field}' needs a default value or transformer`);
}
steps.push({
action: 'add',
field,
description: `Add field '${field}' of type ${fieldConfig.type}`,
transform: options?.transformers?.[field],
});
}
// Handle modified fields
for (const modification of diff.modified) {
const oldConfig = this.getFieldConfig(modification.oldConfig);
const newConfig = this.getFieldConfig(modification.newConfig);
if (oldConfig.type !== newConfig.type) {
steps.push({
action: 'transform',
field: modification.field,
description: `Transform field '${modification.field}' from ${oldConfig.type} to ${newConfig.type}`,
transform: options?.transformers?.[modification.field],
});
if (!options?.transformers?.[modification.field]) {
safe = false;
warnings.push(`Field '${modification.field}' type change requires a transformer`);
}
}
else {
steps.push({
action: 'modify',
field: modification.field,
description: `Modify field '${modification.field}' configuration`,
});
}
}
return {
steps,
safe,
warnings,
};
}
/**
* Get the underlying Redis-OM schema
*
* @returns Redis-OM Schema instance
*/
getSchema() {
return this.schema;
}
/**
* Check if the schema uses JSON structure
*
* @returns True if using JSON, false if using HASH
*/
isJSON() {
return this.dataStructure === 'JSON';
}
/**
* Get the entity name
*
* @returns Entity name
*/
getEntityName() {
return this.entityName;
}
// Private helper methods
normalizeFields(fields) {
const normalized = {};
for (const [key, value] of Object.entries(fields)) {
if (typeof value === 'string') {
normalized[key] = { type: value };
}
else {
normalized[key] = value;
}
}
return normalized;
}
getFieldConfig(field) {
if (typeof field === 'string') {
return { type: field };
}
return field;
}
buildSchemaDefinition(fields) {
const definition = {};
for (const [fieldName, fieldConfig] of Object.entries(fields)) {
const config = this.getFieldConfig(fieldConfig);
definition[fieldName] = this.createFieldDefinition(config.type, config);
}
return definition;
}
createFieldDefinition(type, options = {}) {
const baseDefinition = { type: this.mapFieldType(type) };
if (options.indexed !== undefined) {
baseDefinition.indexed = options.indexed;
}
if (type === 'text' && options.sortable) {
baseDefinition.sortable = options.sortable;
}
if (type === 'text' && options.normalized) {
baseDefinition.normalized = options.normalized;
}
if (type === 'string[]' && options.separator) {
baseDefinition.separator = options.separator;
}
return baseDefinition;
}
mapFieldType(type) {
const typeMap = {
'string': 'string',
'number': 'number',
'boolean': 'boolean',
'date': 'date',
'point': 'point',
'string[]': 'string[]',
'number[]': 'number[]',
'text': 'text',
};
return typeMap[type] || 'string';
}
validateFieldType(value, type) {
if (value === null || value === undefined) {
return true; // Allow null/undefined unless required
}
switch (type) {
case 'string':
case 'text':
return typeof value === 'string';
case 'number':
return typeof value === 'number' && !isNaN(value);
case 'boolean':
return typeof value === 'boolean';
case 'date':
return value instanceof Date || !isNaN(Date.parse(value));
case 'point':
return typeof value === 'object' &&
typeof value.latitude === 'number' &&
typeof value.longitude === 'number';
case 'string[]':
return Array.isArray(value) && value.every(v => typeof v === 'string');
case 'number[]':
return Array.isArray(value) && value.every(v => typeof v === 'number');
default:
return false;
}
}
fieldsEqual(field1, field2) {
const config1 = this.getFieldConfig(field1);
const config2 = this.getFieldConfig(field2);
// Compare all properties except functions
const props1 = { ...config1 };
const props2 = { ...config2 };
delete props1.validate;
delete props2.validate;
return JSON.stringify(props1) === JSON.stringify(props2);
}
/**
* Set the Redis client for version management
*
* @param client - Redis client instance
*/
async setRedisClient(client) {
// Initialize version validator with Redis client
await this.versionValidator.initialize(client);
// Validate Redis OM version before proceeding
await this.versionValidator.assertValidVersion();
if (!this.versionManager) {
this.versionManager = new schema_versioning_1.SchemaVersionManager(this.entityName, client);
}
}
/**
* Initialize version tracking for this schema
*
* @param client - Redis client instance
*/
async initializeVersioning(client) {
await this.setRedisClient(client);
if (this.versionManager) {
// Initialize schema metadata
const indexedFields = this.getIndexedFields();
await this.versionManager.initialize(this.fields, indexedFields, this.dataStructure);
}
}
/**
* Track entity creation for versioning
*
* @param entityId - ID of the created entity
*/
async trackEntityCreation(entityId) {
if (this.versionManager) {
await this.versionManager.trackEntity(entityId);
}
}
/**
* Update schema version if structure changed
*
* @param newFields - New schema fields
* @returns True if version was incremented
*/
async updateVersion(newFields) {
if (!this.versionManager) {
return false;
}
const metadata = await this.versionManager.getMetadata();
if (!metadata) {
return false;
}
const oldFieldsStr = metadata.fields;
const newFieldsStr = JSON.stringify(newFields);
// Check for structural changes
if (this.versionManager.isStructuralChange(oldFieldsStr, newFieldsStr)) {
await this.versionManager.incrementVersion('Schema structure changed');
await this.versionManager.saveMetadata({
fields: newFieldsStr
});
return true;
}
return false;
}
/**
* Get version metadata
*
* @returns Schema version metadata
*/
async getVersionMetadata() {
if (!this.versionManager) {
return null;
}
return this.versionManager.getMetadata();
}
/**
* Get entity count from HyperLogLog
*
* @returns Approximate entity count
*/
async getEntityCount() {
if (!this.versionManager) {
return 0;
}
return this.versionManager.getEntityCount();
}
/**
* Get version history
*
* @returns Array of version history entries
*/
async getVersionHistory() {
if (!this.versionManager) {
return [];
}
return this.versionManager.getVersionHistory();
}
}
exports.StorageSchema = StorageSchema;
//# sourceMappingURL=storage-schema.js.map