recoder-code/plugin-registry-service/src/services/StorageService.ts

Complete AI-powered development platform with ML model training, plugin registry, real-time collaboration, monitoring, infrastructure automation, and enterprise deployment capabilities

/**
 * StorageService
 * Handles package storage, retrieval, and management across multiple backends
 */

import { Config } from '../config';
import * as fs from 'fs-extra';
import * as path from 'path';
import * as crypto from 'crypto';
import { PackageVersion } from '../entities/PackageVersion';

export interface StorageConfig {
  type: 'local' | 's3' | 'gcs' | 'azure';
  endpoint?: string;
  region?: string;
  bucket?: string;
  credentials?: any;
  basePath?: string;
}
 
export interface StorageMetadata {
  size: number;
  contentType: string;
  etag: string;
  lastModified: Date;
  checksums: {
    md5: string;
    sha1: string;
    sha256: string;
    sha512: string;
  };
}

export interface UploadResult {
  success: boolean;
  key: string;
  url: string;
  metadata: StorageMetadata;
  error?: string;
}

export interface DownloadResult {
  success: boolean;
  data?: Buffer;
  metadata?: StorageMetadata;
  error?: string;
}

export class StorageService {
  private readonly config: StorageConfig;
  private readonly logger = {
    log: (message: string) => console.log(`[StorageService] ${message}`),
    warn: (message: string, error?: any) => console.warn(`[StorageService] ${message}`, error),
    error: (message: string, error?: any) => console.error(`[StorageService] ${message}`, error),
    debug: (message: string) => console.debug(`[StorageService] ${message}`)
  };

  constructor(appConfig?: Config) {
    this.config = {
      type: 'local',
      basePath: '/tmp/packages'
    };
  }

  async storeTarball(packageName: string, version: string, tarballBuffer: Buffer): Promise<string> {
    // Simple storage implementation - just return a path
    return `${packageName}-${version}.tgz`;
  }

  async getTarballStream(tarballPath: string): Promise<any> {
    // Simple implementation - return null for now
    return null;
  }

  async deleteTarball(tarballPath: string): Promise<void> {
    // Simple implementation - do nothing for now
  }

  async uploadPackage(
    packageName: string,
    version: string,
    data: Buffer,
    contentType: string = 'application/octet-stream'
  ): Promise<UploadResult> {
    try {
      const key = this.generatePackageKey(packageName, version);
      this.logger.log(`Uploading package ${packageName}@${version} to ${key}`);

      // Calculate checksums
      const metadata = await this.calculateMetadata(data, contentType);

      switch (this.config.type) {
        case 'local':
          return await this.uploadToLocal(key, data, metadata);
        case 's3':
          return await this.uploadToS3(key, data, metadata);
        case 'gcs':
          return await this.uploadToGCS(key, data, metadata);
        case 'azure':
          return await this.uploadToAzure(key, data, metadata);
        default:
          throw new Error(`Unsupported storage type: ${this.config.type}`);
      }
    } catch (error) {
      const err = error as Error;
      this.logger.error(`Failed to upload package ${packageName}@${version}: ${err.message}`, err.stack);
      return {
        success: false,
        key: '',
        url: '',
        metadata: {
          size: 0,
          contentType: '',
          etag: '',
          lastModified: new Date(0),
          checksums: {
            md5: '',
            sha1: '',
            sha256: '',
            sha512: ''
          }
        },
        error: err.message
      };
    }
  }

  async downloadPackage(packageName: string, version: string): Promise<DownloadResult> {
    try {
      const key = this.generatePackageKey(packageName, version);
      this.logger.debug(`Downloading package ${packageName}@${version} from ${key}`);

      switch (this.config.type) {
        case 'local':
          return await this.downloadFromLocal(key);
        case 's3':
          return await this.downloadFromS3(key);
        case 'gcs':
          return await this.downloadFromGCS(key);
        case 'azure':
          return await this.downloadFromAzure(key);
        default:
          throw new Error(`Unsupported storage type: ${this.config.type}`);
      }
    } catch (error) {
      const err = error as Error;
      this.logger.error(`Failed to download package ${packageName}@${version}: ${err.message}`, err.stack);
      return {
        success: false,
        error: err.message
      };
    }
  }

  async deletePackage(packageName: string, version: string): Promise<boolean> {
    try {
      const key = this.generatePackageKey(packageName, version);
      this.logger.log(`Deleting package ${packageName}@${version} from ${key}`);

      switch (this.config.type) {
        case 'local':
          return await this.deleteFromLocal(key);
        case 's3':
          return await this.deleteFromS3(key);
        case 'gcs':
          return await this.deleteFromGCS(key);
        case 'azure':
          return await this.deleteFromAzure(key);
        default:
          throw new Error(`Unsupported storage type: ${this.config.type}`);
      }
    } catch (error) {
      const err = error as Error;
      this.logger.error(`Failed to delete package ${packageName}@${version}: ${err.message}`, err.stack);
      return false;
    }
  }

  async getPackageMetadata(packageName: string, version: string): Promise<StorageMetadata | null> {
    try {
      const key = this.generatePackageKey(packageName, version);

      switch (this.config.type) {
        case 'local':
          return await this.getLocalMetadata(key);
        case 's3':
          return await this.getS3Metadata(key);
        case 'gcs':
          return await this.getGCSMetadata(key);
        case 'azure':
          return await this.getAzureMetadata(key);
        default:
          throw new Error(`Unsupported storage type: ${this.config.type}`);
      }
    } catch (error) {
      const err = error as Error;
      this.logger.error(`Failed to get metadata for ${packageName}@${version}: ${err.message}`);
      return null;
    }
  }

  async packageExists(packageName: string, version: string): Promise<boolean> {
    const metadata = await this.getPackageMetadata(packageName, version);
    return metadata !== null;
  }

  async listPackageVersions(packageName: string): Promise<string[]> {
    try {
      const prefix = this.generatePackagePrefix(packageName);

      switch (this.config.type) {
        case 'local':
          return await this.listLocalVersions(prefix);
        case 's3':
          return await this.listS3Versions(prefix);
        case 'gcs':
          return await this.listGCSVersions(prefix);
        case 'azure':
          return await this.listAzureVersions(prefix);
        default:
          throw new Error(`Unsupported storage type: ${this.config.type}`);
      }
    } catch (error) {
      const err = error as Error;
      this.logger.error(`Failed to list versions for ${packageName}: ${err.message}`);
      return [];
    }
  }

  getDownloadUrl(packageName: string, version: string): string {
    const key = this.generatePackageKey(packageName, version);
    
    switch (this.config.type) {
      case 'local':
        return `/packages/${key}`;
      case 's3':
        return `https://${this.config.bucket}.s3.${this.config.region}.amazonaws.com/${key}`;
      case 'gcs':
        return `https://storage.googleapis.com/${this.config.bucket}/${key}`;
      case 'azure':
        return `https://${this.config.endpoint}/${this.config.bucket}/${key}`;
      default:
        return '';
    }
  }

  private async initializeStorage(): Promise<void> {
    switch (this.config.type) {
      case 'local':
        await fs.ensureDir(this.config.basePath || '/tmp/packages');
        break;
      case 's3':
        // Initialize S3 client
        await this.initializeS3();
        break;
      case 'gcs':
        // Initialize GCS client
        await this.initializeGCS();
        break;
      case 'azure':
        // Initialize Azure client
        await this.initializeAzure();
        break;
    }
  }

  private generatePackageKey(packageName: string, version: string): string {
    // Handle scoped packages
    const normalizedName = packageName.replace('@', '').replace('/', '-');
    return `${normalizedName}/${version}/${normalizedName}-${version}.tgz`;
  }

  private generatePackagePrefix(packageName: string): string {
    const normalizedName = packageName.replace('@', '').replace('/', '-');
    return `${normalizedName}/`;
  }

  private async calculateMetadata(data: Buffer, contentType: string): Promise<StorageMetadata> {
    const md5 = crypto.createHash('md5').update(data).digest('hex');
    const sha1 = crypto.createHash('sha1').update(data).digest('hex');
    const sha256 = crypto.createHash('sha256').update(data).digest('hex');
    const sha512 = crypto.createHash('sha512').update(data).digest('hex');

    return {
      size: data.length,
      contentType,
      etag: md5,
      lastModified: new Date(),
      checksums: { md5, sha1, sha256, sha512 }
    };
  }

  // Local Storage Implementation
  private async uploadToLocal(key: string, data: Buffer, metadata: StorageMetadata): Promise<UploadResult> {
    const filePath = path.join(this.config.basePath || '/tmp/packages', key);
    const dir = path.dirname(filePath);
    
    await fs.ensureDir(dir);
    await fs.writeFile(filePath, data);
    
    // Store metadata
    const metadataPath = filePath + '.meta.json';
    await fs.writeFile(metadataPath, JSON.stringify(metadata, null, 2));

    return {
      success: true,
      key,
      url: this.getDownloadUrl('', '').replace('//', '/') + key,
      metadata
    };
  }

  private async downloadFromLocal(key: string): Promise<DownloadResult> {
    const filePath = path.join(this.config.basePath || '/tmp/packages', key);
    
    if (!await fs.pathExists(filePath)) {
      return { success: false, error: 'File not found' };
    }

    const data = await fs.readFile(filePath);
    const metadata = await this.getLocalMetadata(key) || undefined;

    return { success: true, data, metadata };
  }

  private async deleteFromLocal(key: string): Promise<boolean> {
    const filePath = path.join(this.config.basePath || '/tmp/packages', key);
    const metadataPath = filePath + '.meta.json';
    
    await fs.remove(filePath);
    await fs.remove(metadataPath);
    
    return true;
  }

  private async getLocalMetadata(key: string): Promise<StorageMetadata | null> {
    const filePath = path.join(this.config.basePath || '/tmp/packages', key);
    const metadataPath = filePath + '.meta.json';
    
    if (!await fs.pathExists(metadataPath)) {
      // Fallback: calculate metadata from file
      if (await fs.pathExists(filePath)) {
        const data = await fs.readFile(filePath);
        return this.calculateMetadata(data, 'application/octet-stream');
      }
      return null;
    }

    const metadata = await fs.readJSON(metadataPath);
    return metadata;
  }

  private async listLocalVersions(prefix: string): Promise<string[]> {
    const basePath = path.join(this.config.basePath || '/tmp/packages', prefix);
    
    if (!await fs.pathExists(basePath)) {
      return [];
    }

    const dirs = await fs.readdir(basePath);
    return dirs.filter(dir => !dir.startsWith('.'));
  }

  // S3 Storage Implementation
  private s3Client: any;

  private async initializeS3(): Promise<void> {
    try {
      const { S3Client } = await import('@aws-sdk/client-s3');
      
      this.s3Client = new S3Client({
        region: this.config.region,
        endpoint: this.config.endpoint,
        credentials: this.config.credentials
      });
    } catch (error) {
      this.logger.warn('S3 client not available:', (error as Error).message);
    }
  }

  private async uploadToS3(key: string, data: Buffer, metadata: StorageMetadata): Promise<UploadResult> {
    if (!this.s3Client) {
      throw new Error('S3 client not initialized');
    }

    const { PutObjectCommand } = await import('@aws-sdk/client-s3');
    
    const command = new PutObjectCommand({
      Bucket: this.config.bucket,
      Key: key,
      Body: data,
      ContentType: metadata.contentType,
      Metadata: {
        'md5': metadata.checksums.md5,
        'sha256': metadata.checksums.sha256
      }
    });

    await this.s3Client.send(command);

    return {
      success: true,
      key,
      url: this.getDownloadUrl('', ''),
      metadata
    };
  }

  private async downloadFromS3(key: string): Promise<DownloadResult> {
    if (!this.s3Client) {
      throw new Error('S3 client not initialized');
    }

    const { GetObjectCommand } = await import('@aws-sdk/client-s3');
    
    const command = new GetObjectCommand({
      Bucket: this.config.bucket,
      Key: key
    });

    const response = await this.s3Client.send(command);
    const chunks: Buffer[] = [];
    
    for await (const chunk of response.Body as any) {
      chunks.push(chunk);
    }
    
    const data = Buffer.concat(chunks);
    const metadata: StorageMetadata = {
      size: response.ContentLength,
      contentType: response.ContentType,
      etag: response.ETag,
      lastModified: response.LastModified,
      checksums: {
        md5: response.Metadata?.['md5'] || '',
        sha1: '',
        sha256: response.Metadata?.['sha256'] || '',
        sha512: ''
      }
    };

    return { success: true, data, metadata };
  }

  private async deleteFromS3(key: string): Promise<boolean> {
    if (!this.s3Client) {
      throw new Error('S3 client not initialized');
    }

    const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
    
    const command = new DeleteObjectCommand({
      Bucket: this.config.bucket,
      Key: key
    });

    await this.s3Client.send(command);
    return true;
  }

  private async getS3Metadata(key: string): Promise<StorageMetadata | null> {
    if (!this.s3Client) {
      throw new Error('S3 client not initialized');
    }

    const { HeadObjectCommand } = await import('@aws-sdk/client-s3');
    
    try {
      const command = new HeadObjectCommand({
        Bucket: this.config.bucket,
        Key: key
      });

      const response = await this.s3Client.send(command);
      
      return {
        size: response.ContentLength,
        contentType: response.ContentType,
        etag: response.ETag,
        lastModified: response.LastModified,
        checksums: {
          md5: response.Metadata?.['md5'] || '',
          sha1: '',
          sha256: response.Metadata?.['sha256'] || '',
          sha512: ''
        }
      };
    } catch (error) {
      if (typeof error === 'object' && error !== null && 'name' in error && (error as any).name === 'NotFound') {
        return null;
      }
      throw error;
    }
  }

  private async listS3Versions(prefix: string): Promise<string[]> {
    if (!this.s3Client) {
      throw new Error('S3 client not initialized');
    }

    const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
    
    const command = new ListObjectsV2Command({
      Bucket: this.config.bucket,
      Prefix: prefix,
      Delimiter: '/'
    });

    const response = await this.s3Client.send(command);
    
    return (response.CommonPrefixes || [])
      .map((prefix: { Prefix: string; }) => {
        // Extract the version from the prefix string, which is like 'packageName/version/'
        const parts = prefix.Prefix.split('/');
        // The version is the second part (index 1)
        return parts.length > 1 ? parts[1] : '';
      })
      .filter((version: any) => version);
  }

  // Placeholder implementations for GCS and Azure
  private async initializeGCS(): Promise<void> {
    this.logger.warn('GCS storage not implemented yet');
  }

  private async initializeAzure(): Promise<void> {
    this.logger.warn('Azure storage not implemented yet');
  }

  private async uploadToGCS(key: string, data: Buffer, metadata: StorageMetadata): Promise<UploadResult> {
    throw new Error('GCS storage not implemented');
  }

  private async uploadToAzure(key: string, data: Buffer, metadata: StorageMetadata): Promise<UploadResult> {
    throw new Error('Azure storage not implemented');
  }

  private async downloadFromGCS(key: string): Promise<DownloadResult> {
    throw new Error('GCS storage not implemented');
  }

  private async downloadFromAzure(key: string): Promise<DownloadResult> {
    throw new Error('Azure storage not implemented');
  }

  private async deleteFromGCS(key: string): Promise<boolean> {
    throw new Error('GCS storage not implemented');
  }

  private async deleteFromAzure(key: string): Promise<boolean> {
    throw new Error('Azure storage not implemented');
  }

  private async getGCSMetadata(key: string): Promise<StorageMetadata | null> {
    throw new Error('GCS storage not implemented');
  }

  private async getAzureMetadata(key: string): Promise<StorageMetadata | null> {
    throw new Error('Azure storage not implemented');
  }

  private async listGCSVersions(prefix: string): Promise<string[]> {
    throw new Error('GCS storage not implemented');
  }

  private async listAzureVersions(prefix: string): Promise<string[]> {
    throw new Error('Azure storage not implemented');
  }

  async testConnection(): Promise<boolean> {
    try {
      switch (this.config.type) {
        case 'local':
          // Test local storage by ensuring directory exists
          const basePath = this.config.basePath || '/tmp/packages';
          await fs.ensureDir(basePath);
          return true;
          
        case 's3':
          // For S3, we'd test with a simple head operation
          // For now, just return true as a placeholder
          return true;
          
        case 'gcs':
          // For GCS, we'd test bucket access
          return true;
          
        case 'azure':
          // For Azure, we'd test container access
          return true;
          
        default:
          return false;
      }
    } catch (error) {
      console.error('Storage connection test failed:', error);
      return false;
    }
  }
}