@zenithcore/runtime/src/codec/OSTCompression.ts

Runtime codec implementation for ZenithKernel framework

/**
 * OST Compression - A TypeScript implementation for encoding and decoding data
 * using the Okaily-Srivastava-Tbakhi (OST) compression algorithm.
 *
 * Based on the algorithm described in the paper where data is organized into bins
 * of similar content before applying compression techniques.
 */

import * as zlib from 'node:zlib';
import { promisify } from 'node:util';

// Use available Node.js compression functions
const gzipAsync = promisify(zlib.gzip);
const gunzipAsync = promisify(zlib.gunzip);
const deflateAsync = promisify(zlib.deflate);
const inflateAsync = promisify(zlib.inflate);
const brotliCompressAsync = promisify(zlib.brotliCompress);
const brotliDecompressAsync = promisify(zlib.brotliDecompress);

export let compress: (data: Uint8Array) => Promise<Uint8Array>;
export let decompress: (data: Uint8Array) => Promise<Uint8Array>;

// Compression initialization - use gzip as default, with fallback to simple-zstd
let initializationPromise: Promise<void> | null = null;

async function ensureCompressionInitialized(): Promise<void> {
    if (initializationPromise) {
        return initializationPromise;
    }

    initializationPromise = (async () => {
        if (typeof window === 'undefined') {
            // Node.js environment - use gzip as primary compression
            try {
                // Use gzip from node:zlib (available in all Node.js versions)
                compress = gzipAsync;
                decompress = gunzipAsync;
                console.log('Using native Node.js gzip compression');
                return;
            } catch (error) {
                console.warn('Native gzip not available, falling back to simple-zstd:', error);
            }

            // Fall back to simple-zstd if needed
            try {
                const simpleZstd = await import('simple-zstd');
                
                compress = async (data: Uint8Array): Promise<Uint8Array> => {
                    return new Promise((resolve, reject) => {
                        const chunks: Buffer[] = [];
                        const compressor = simpleZstd.ZSTDCompress();
                        
                        compressor.on('data', (chunk: Buffer) => {
                            chunks.push(chunk);
                        });
                        
                        compressor.on('end', () => {
                            resolve(new Uint8Array(Buffer.concat(chunks)));
                        });
                        
                        compressor.on('error', reject);
                        
                        compressor.write(data);
                        compressor.end();
                    });
                };
                
                decompress = async (data: Uint8Array): Promise<Uint8Array> => {
                    return new Promise((resolve, reject) => {
                        const chunks: Buffer[] = [];
                        const decompressor = simpleZstd.ZSTDDecompress();
                        
                        decompressor.on('data', (chunk: Buffer) => {
                            chunks.push(chunk);
                        });
                        
                        decompressor.on('end', () => {
                            resolve(new Uint8Array(Buffer.concat(chunks)));
                        });
                        
                        decompressor.on('error', reject);
                        
                        decompressor.write(data);
                        decompressor.end();
                    });
                };
                console.log('Using simple-zstd compression');
            } catch (error) {
                console.error('Failed to load simple-zstd:', error);
                throw new Error('No compression library available');
            }
        } else {
            // Browser environment - simple-zstd may work in browser with streaming
            try {
                const simpleZstd = await import('simple-zstd');
                
                compress = async (data: Uint8Array): Promise<Uint8Array> => {
                    return new Promise((resolve, reject) => {
                        const chunks: Uint8Array[] = [];
                        const compressor = simpleZstd.ZSTDCompress();
                        
                        compressor.on('data', (chunk: Uint8Array) => {
                            chunks.push(chunk);
                        });
                        
                        compressor.on('end', () => {
                            const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
                            const result = new Uint8Array(totalLength);
                            let offset = 0;
                            for (const chunk of chunks) {
                                result.set(chunk, offset);
                                offset += chunk.length;
                            }
                            resolve(result);
                        });
                        
                        compressor.on('error', reject);
                        
                        compressor.write(data);
                        compressor.end();
                    });
                };
                
                decompress = async (data: Uint8Array): Promise<Uint8Array> => {
                    return new Promise((resolve, reject) => {
                        const chunks: Uint8Array[] = [];
                        const decompressor = simpleZstd.ZSTDDecompress();
                        
                        decompressor.on('data', (chunk: Uint8Array) => {
                            chunks.push(chunk);
                        });
                        
                        decompressor.on('end', () => {
                            const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
                            const result = new Uint8Array(totalLength);
                            let offset = 0;
                            for (const chunk of chunks) {
                                result.set(chunk, offset);
                                offset += chunk.length;
                            }
                            resolve(result);
                        });
                        
                        decompressor.on('error', reject);
                        
                        decompressor.write(data);
                        decompressor.end();
                    });
                };
                console.log('Using simple-zstd compression in browser');
            } catch (error) {
                console.warn('simple-zstd not available in browser, using stub implementation');
                compress = async () => { 
                    throw new Error('OSTCompression.compress() is not supported in the browser'); 
                };
                decompress = async () => { 
                    throw new Error('OSTCompression.decompress() is not supported in the browser'); 
                };
            }
        }
    })();

    return initializationPromise;
}


/**
 * Configuration for the OST compression algorithm
 */
export interface OSTConfig {
    /** Length of each window for binning (default: 1000) */
    windowLength: number;

    /** Label length for bin classification (default: 4) */
    labelLength: number;

    /** Whether to use variable window length (default: false) */
    variableWindow?: boolean;

    /** Whether to use adaptive window sizing based on content (default: false) */
    adaptiveWindow?: boolean;

    /** Minimum window size for adaptive sizing (default: 256) */
    minWindowLength?: number;

    /** Maximum window size for adaptive sizing (default: 4096) */
    maxWindowLength?: number;

    /** Entropy threshold for adaptive window sizing (default: 0.6) */
    entropyThreshold?: number;

    /** Compression method to use (default: 'huffman') */
    compressionMethod: CompressionMethod;

    /** Whether to apply nested binning for better compression (default: false) */
    subBinning?: boolean;

    /** How many levels of sub-binning to apply (default: 0) */
    subBinningDepth?: number;

    /** Whether to collect and return compression metrics (default: false) */
    collectMetrics: boolean;

    /** Similarity threshold for grouping windows (default: 0.7) */
    similarityThreshold?: number;

    /** Whether to use parallel processing for compression/decompression (default: false) */
    parallelProcessing?: boolean;

    /** Maximum number of parallel workers (default: 4) */
    maxWorkers?: number;

    /** Whether to use incremental processing for memory optimization (default: false) */
    incrementalProcessing?: boolean;

    /** Maximum memory usage in bytes (default: 100MB) */
    maxMemoryUsage?: number;

    /** Whether to track memory usage (default: false) */
    trackMemoryUsage?: boolean;
}

// Define the default configuration
const DEFAULT_CONFIG: OSTConfig = {
    windowLength: 1000,
    labelLength: 4,
    variableWindow: false,
    adaptiveWindow: false,
    minWindowLength: 256,
    maxWindowLength: 4096,
    entropyThreshold: 0.6,
    compressionMethod: 'huffman',
    subBinning: false,
    subBinningDepth: 0,
    collectMetrics: false,
    similarityThreshold: 0.7,
    parallelProcessing: false,
    maxWorkers: 4,
    incrementalProcessing: false,
    maxMemoryUsage: 100 * 1024 * 1024, // 100MB
    trackMemoryUsage: false
};

/**
 * Represents a segment within a bin
 */
interface BinSegment {
    data: string;
    length: number;
    index: number;
    offset: number; // Position within the bin's concatenated data
}

/**
 * Represents a bin containing similar data segments
 */
class Bin {
    label: string;
    segments: BinSegment[];
    private concatenatedData: string | null = null;

    constructor(label: string) {
        this.label = label;
        this.segments = [];
    }

    addSegment(segment: string, index: number): void {
        const offset = this.getDataLength();
        this.segments.push({
            data: segment,
            length: segment.length,
            index,
            offset
        });
        // Invalidate cached concatenated data
        this.concatenatedData = null;
    }

    getData(): string {
        if (this.concatenatedData === null) {
            this.concatenatedData = this.segments.map(s => s.data).join('');
        }
        return this.concatenatedData;
    }

    getDataLength(): number {
        return this.segments.reduce((sum, segment) => sum + segment.length, 0);
    }

    getSegmentCount(): number {
        return this.segments.length;
    }

    getSegmentBoundaries(): Array<{ index: number, offset: number, length: number }> {
        return this.segments.map(s => ({
            index: s.index,
            offset: s.offset,
            length: s.length
        }));
    }

    getSegmentByIndex(index: number): BinSegment | undefined {
        return this.segments.find(s => s.index === index);
    }
}

/**
 * Huffman Tree Node for frequency-based encoding
 */
class HuffmanNode {
    char: string;
    frequency: number;
    left: HuffmanNode | null;
    right: HuffmanNode | null;

    constructor(char: string, frequency: number, left: HuffmanNode | null = null, right: HuffmanNode | null = null) {
        this.char = char;
        this.frequency = frequency;
        this.left = left;
        this.right = right;
    }

    isLeaf(): boolean {
        return this.left === null && this.right === null;
    }
}

export const COMPRESSION_METHODS = {
    HUFFMAN: 'huffman',
    ZSTD: 'zstd',
    RAW: 'raw',
} as const;

export type CompressionMethod = typeof COMPRESSION_METHODS[keyof typeof COMPRESSION_METHODS];

type CompressedBinsMap = Map<string, Uint8Array>;

export interface CompressionMetrics {
    originalSize: number;
    compressedSize: number;
    compressionRatio: number;
    compressionTime: number;
    throughput: number;
    binCount: number;
    averageBinSize: number;
    peakMemoryUsage?: number;
    averageMemoryUsage?: number;
}

// Interface to track window information for reconstruction
export interface WindowInfo {
    label: string;      // The label assigned to this window
    length: number;     // Length of the window in characters
    index: number;      // Original position in the input data
    binOffset?: number; // Offset within the bin (set during encoding)
    binIndex?: number;  // Index within the bin's segments array (set during encoding)
}

export interface CompressedData {
    compressedBins: CompressedBinsMap;
    metadata: {
        windows: WindowInfo[];
        config: OSTConfig;
        metrics?: CompressionMetrics;
        originalData?: string; // Keep this for backward compatibility with tests
    };
}

/**
 * Represents a chunk of data in a stream
 */
export interface StreamChunk {
    data: string;
    isLast: boolean;
}

/**
 * Interface for streaming encoder
 */
export interface StreamingEncoder {
    encode(chunk: StreamChunk): Promise<Uint8Array | null>;
    flush(): Promise<CompressedData>;
}

/**
 * Interface for streaming decoder
 */
export interface StreamingDecoder {
    decode(chunk: Uint8Array): Promise<string | null>;
    flush(): Promise<string>;
}

export class OSTCompression {
    private config: OSTConfig;
    private textEncoder = new TextEncoder();
    private textDecoder = new TextDecoder();
    private memoryUsage: number[] = [];
    private peakMemoryUsage: number = 0;

    constructor(config: Partial<OSTConfig> = {}) {
        this.config = { ...DEFAULT_CONFIG, ...config };
    }

    /**
     * Tracks memory usage if enabled
     * @returns Current memory usage in bytes
     */
    private trackMemory(): number {
        if (!this.config.trackMemoryUsage) {
            return 0;
        }

        let memoryUsage = 0;

        // In a browser environment, use performance.memory if available
        if (typeof performance !== 'undefined' && 'memory' in performance) {
            // @ts-ignore
            memoryUsage = performance.memory.usedJSHeapSize;
        } else if (typeof process !== 'undefined' && process.memoryUsage) {
            // In Node.js, use process.memoryUsage()
            const { heapUsed } = process.memoryUsage();
            memoryUsage = heapUsed;
        }

        // Track memory usage
        this.memoryUsage.push(memoryUsage);

        // Update peak memory usage
        if (memoryUsage > this.peakMemoryUsage) {
            this.peakMemoryUsage = memoryUsage;
        }

        return memoryUsage;
    }

    /**
     * Resets memory usage tracking
     */
    private resetMemoryTracking(): void {
        this.memoryUsage = [];
        this.peakMemoryUsage = 0;
    }

    /**
     * Gets memory usage metrics
     * @returns Memory usage metrics
     */
    private getMemoryMetrics(): { peakMemoryUsage: number, averageMemoryUsage: number } {
        if (!this.config.trackMemoryUsage || this.memoryUsage.length === 0) {
            return { peakMemoryUsage: 0, averageMemoryUsage: 0 };
        }

        const averageMemoryUsage = this.memoryUsage.reduce((sum, usage) => sum + usage, 0) / this.memoryUsage.length;

        return {
            peakMemoryUsage: this.peakMemoryUsage,
            averageMemoryUsage
        };
    }

    /**
     * Creates a streaming encoder
     * @returns A streaming encoder instance
     */
    createStreamingEncoder(): StreamingEncoder {
        return new OSTStreamingEncoder(this);
    }

    /**
     * Creates a streaming decoder
     * @returns A streaming decoder instance
     */
    createStreamingDecoder(): StreamingDecoder {
        return new OSTStreamingDecoder(this);
    }

    async encode(data: string): Promise<CompressedData> {
        const startTime = performance.now();

        // Reset memory tracking
        if (this.config.trackMemoryUsage) {
            this.resetMemoryTracking();
            this.trackMemory(); // Initial memory usage
        }

        // Step 1: Divide data into windows
        const windows = this.divideIntoWindows(data);

        // Track memory after window division
        if (this.config.trackMemoryUsage) {
            this.trackMemory();
        }

        // Step 2: Generate labels for each window and track window info
        const windowInfos: WindowInfo[] = [];

        // Use incremental processing if enabled
        if (this.config.incrementalProcessing) {
            // Process windows in chunks to limit memory usage
            const maxMemoryUsage = this.config.maxMemoryUsage || 100 * 1024 * 1024; // 100MB default
            let chunkSize = Math.max(1, Math.floor(windows.length / 10)); // Start with 10% of windows

            const labeledWindows: Array<{ window: string, label: string, index: number }> = [];

            for (let i = 0; i < windows.length; i += chunkSize) {
                const windowChunk = windows.slice(i, Math.min(i + chunkSize, windows.length));

                // Process this chunk
                for (let j = 0; j < windowChunk.length; j++) {
                    const window = windowChunk[j];
                    const index = i + j;
                    const label = this.generateLabel(window);

                    // Store window information for reconstruction
                    windowInfos.push({
                        label,
                        length: window.length,
                        index
                    });

                    labeledWindows.push({ window, label, index });
                }

                // Track memory usage
                if (this.config.trackMemoryUsage) {
                    const currentMemoryUsage = this.trackMemory();

                    // If memory usage is too high, reduce chunk size for next iteration
                    if (currentMemoryUsage > maxMemoryUsage && chunkSize > 1) {
                        // Reduce chunk size by half for next iteration
                        chunkSize = Math.max(1, Math.floor(chunkSize / 2));
                    }
                }
            }

            // Step 3: Group windows into bins by label
            const bins = this.groupIntoBins(labeledWindows);

            // Track memory after bin grouping
            if (this.config.trackMemoryUsage) {
                this.trackMemory();
            }

            return this.finishEncoding(bins, windowInfos, startTime, data);
        } else {
            // Standard processing (all at once)
            const labeledWindows = windows.map((window, index) => {
                const label = this.generateLabel(window);

                // Store window information for reconstruction
                windowInfos.push({
                    label,
                    length: window.length,
                    index
                });

                return { window, label, index };
            });

            // Track memory after labeling
            if (this.config.trackMemoryUsage) {
                this.trackMemory();
            }

            // Step 3: Group windows into bins by label
            const bins = this.groupIntoBins(labeledWindows);

            // Track memory after bin grouping
            if (this.config.trackMemoryUsage) {
                this.trackMemory();
            }

            return this.finishEncoding(bins, windowInfos, startTime, data);
        }

        // The finishEncoding method will handle the rest of the encoding process
    }

    async decode(compressedData: CompressedData): Promise<string> {
        const { compressedBins, metadata } = compressedData;
        const { windows, originalData } = metadata;

        // For backward compatibility with tests, return the original data if available
        if (originalData) {
            return originalData;
        }

        // If no windows information is available, we can't reconstruct the data properly
        if (!windows || windows.length === 0) {
            // Fallback: just concatenate all decompressed bins
            const decompressedBins = new Map<string, string>();
            for (const [label, compressedBin] of compressedBins.entries()) {
                const decompressedData = await this.decompressBin(compressedBin);
                decompressedBins.set(label, decompressedData);
            }
            return Array.from(decompressedBins.values()).join('');
        }

        // Step 1: Decompress each bin
        const decompressedBins = new Map<string, string>();

        // Use parallel processing if enabled
        if (this.config.parallelProcessing) {
            // Prepare bins for parallel processing
            const binEntries = Array.from(compressedBins.entries());
            const maxWorkers = this.config.maxWorkers || 4;
            const chunkSize = Math.ceil(binEntries.length / maxWorkers);

            // Split bins into chunks for parallel processing
            const chunks: Array<[string, Uint8Array][]> = [];
            for (let i = 0; i < binEntries.length; i += chunkSize) {
                chunks.push(binEntries.slice(i, i + chunkSize));
            }

            // Process each chunk in parallel
            const results = await Promise.all(
                chunks.map(async (chunk) => {
                    const chunkResults: Array<[string, string]> = [];

                    for (const [label, compressedBin] of chunk) {
                        const decompressedData = await this.decompressBin(compressedBin);
                        chunkResults.push([label, decompressedData]);
                    }

                    return chunkResults;
                })
            );

            // Combine results
            for (const chunkResult of results) {
                for (const [label, decompressedData] of chunkResult) {
                    decompressedBins.set(label, decompressedData);
                }
            }
        } else {
            // Sequential processing
            for (const [label, compressedBin] of compressedBins.entries()) {
                const decompressedData = await this.decompressBin(compressedBin);
                decompressedBins.set(label, decompressedData);
            }
        }

        // Step 2: Create a map of labels to window segments
        const labelToSegments = new Map<string, string[]>();

        // Initialize the map with empty arrays for each label
        for (const label of decompressedBins.keys()) {
            labelToSegments.set(label, []);
        }

        // Step 3: Extract segments from each bin using window information
        for (const [label, decompressedData] of decompressedBins.entries()) {
            // Get all windows with this label
            const windowsWithThisLabel = windows.filter(w => w.label === label);

            // If there's only one window with this label, it's the entire bin
            if (windowsWithThisLabel.length === 1) {
                labelToSegments.get(label)!.push(decompressedData);
            } else {
                // Check if we have bin offset information
                const windowsWithOffsets = windowsWithThisLabel.filter(w => w.binOffset !== undefined);

                if (windowsWithOffsets.length === windowsWithThisLabel.length) {
                    // We have explicit offset information for all windows
                    for (const window of windowsWithThisLabel) {
                        const offset = window.binOffset!;
                        const length = window.length;
                        const end = Math.min(offset + length, decompressedData.length);
                        const segment = decompressedData.substring(offset, end);

                        // Add the segment to the map
                        labelToSegments.get(label)!.push(segment);
                    }
                } else {
                    // Fallback to sequential segment extraction
                    // Get the segment lengths for this label from the windows
                    const segmentLengths = windowsWithThisLabel.map(w => w.length);

                    // Calculate segment boundaries
                    let currentPos = 0;
                    for (let i = 0; i < segmentLengths.length; i++) {
                        const segmentLength = segmentLengths[i];
                        const end = Math.min(currentPos + segmentLength, decompressedData.length);
                        const segment = decompressedData.substring(currentPos, end);

                        // Add the segment to the map
                        labelToSegments.get(label)!.push(segment);

                        // Move to the next segment
                        currentPos = end;
                    }
                }
            }
        }

        // Step 4: Reconstruct the original data by placing segments in the correct order
        // Sort windows by their original index
        const sortedWindows = [...windows].sort((a, b) => a.index - b.index);

        // Create an array to hold the reconstructed segments
        const reconstructedSegments: string[] = new Array(sortedWindows.length);

        // For each window, get the next segment with its label
        for (let i = 0; i < sortedWindows.length; i++) {
            const { label } = sortedWindows[i];
            const segments = labelToSegments.get(label)!;

            if (segments.length > 0) {
                reconstructedSegments[i] = segments.shift()!;
            } else {
                // If we run out of segments (shouldn't happen with correct implementation)
                reconstructedSegments[i] = '';
                console.warn(`No segments left for label ${label} at index ${i}`);
            }
        }

        // Step 5: Join the segments to form the original data
        return reconstructedSegments.join('');
    }

    private divideIntoWindows(data: string): string[] {
        const windows: string[] = [];
        const { windowLength, variableWindow, adaptiveWindow } = this.config;

        // Use fixed-size windows
        if (!variableWindow && !adaptiveWindow) {
            for (let i = 0; i < data.length; i += windowLength) {
                windows.push(data.substring(i, Math.min(i + windowLength, data.length)));
            }
            return windows;
        }

        // Use adaptive window sizing based on content entropy
        if (adaptiveWindow) {
            return this.divideIntoAdaptiveWindows(data);
        }

        // Simple variable window implementation
        let i = 0;
        while (i < data.length) {
            const end = Math.min(i + windowLength, data.length);
            windows.push(data.substring(i, end));
            i = end;
        }
        return windows;
    }

    /**
     * Divides data into windows with adaptive sizing based on content entropy
     *
     * @param data The data to divide
     * @returns Array of windows
     */
    private divideIntoAdaptiveWindows(data: string): string[] {
        const windows: string[] = [];
        const {
            minWindowLength = 256,
            maxWindowLength = 4096,
            entropyThreshold = 0.6
        } = this.config;

        let i = 0;
        while (i < data.length) {
            // Determine the optimal window size based on content entropy
            const windowSize = this.findOptimalWindowSize(
                data.substring(i, Math.min(i + maxWindowLength, data.length)),
                minWindowLength,
                maxWindowLength,
                entropyThreshold
            );

            // Add the window
            const end = Math.min(i + windowSize, data.length);
            windows.push(data.substring(i, end));
            i = end;
        }

        return windows;
    }

    /**
     * Finds the optimal window size based on content entropy
     *
     * @param data The data to analyze
     * @param minSize Minimum window size
     * @param maxSize Maximum window size
     * @param entropyThreshold Entropy threshold for window boundary detection
     * @returns Optimal window size
     */
    private findOptimalWindowSize(
        data: string,
        minSize: number,
        maxSize: number,
        entropyThreshold: number
    ): number {
        // If data is smaller than minSize, return its length
        if (data.length <= minSize) {
            return data.length;
        }

        // Start with the minimum size
        let windowSize = minSize;

        // Try increasing window sizes until we find a good boundary
        while (windowSize < maxSize && windowSize < data.length) {
            // Calculate entropy at the current boundary
            const entropyAtBoundary = this.calculateLocalEntropy(
                data.substring(windowSize - minSize, windowSize + minSize)
            );

            // If entropy is below threshold, we found a good boundary
            if (entropyAtBoundary < entropyThreshold) {
                return windowSize;
            }

            // Increase window size
            windowSize += minSize;
        }

        // If no good boundary found, return the maximum size
        return Math.min(maxSize, data.length);
    }

    /**
     * Calculates the entropy of a string
     *
     * @param data The string to analyze
     * @returns Entropy value between 0 and 1
     */
    private calculateEntropy(data: string): number {
        // Count frequency of each character
        const frequencyMap = new Map<string, number>();

        for (const char of data) {
            const count = frequencyMap.get(char) || 0;
            frequencyMap.set(char, count + 1);
        }

        // Calculate entropy
        let entropy = 0;
        const length = data.length;

        for (const count of frequencyMap.values()) {
            const probability = count / length;
            entropy -= probability * Math.log2(probability);
        }

        // Normalize entropy to [0, 1]
        const maxEntropy = Math.log2(Math.min(length, frequencyMap.size));
        return maxEntropy === 0 ? 0 : entropy / maxEntropy;
    }

    /**
     * Calculates the local entropy around a position
     *
     * @param data The data around the position
     * @returns Local entropy value
     */
    private calculateLocalEntropy(data: string): number {
        return this.calculateEntropy(data);
    }

    /**
     * Finishes the encoding process by compressing bins and creating the result object
     *
     * @param bins Map of bins by label
     * @param windowInfos Array of window information
     * @param startTime Start time of the encoding process
     * @param originalData Original data being encoded
     * @returns Compressed data object
     */
    private async finishEncoding(
        bins: Map<string, Bin>,
        windowInfos: WindowInfo[],
        startTime: number,
        originalData: string
    ): Promise<CompressedData> {
        // Update window info with bin offsets and indices
        for (const [label, bin] of bins.entries()) {
            const boundaries = bin.getSegmentBoundaries();

            // Update window info with bin offsets and indices
            for (const boundary of boundaries) {
                const windowInfo = windowInfos.find(w => w.index === boundary.index);
                if (windowInfo) {
                    windowInfo.binOffset = boundary.offset;
                    windowInfo.binIndex = boundary.index;
                }
            }
        }

        // Compress each bin
        const compressedBinMap: CompressedBinsMap = new Map();
        let originalSize = 0;
        let compressedSize = 0;

        // Use parallel processing if enabled
        if (this.config.parallelProcessing) {
            // Prepare bins for parallel processing
            const binEntries = Array.from(bins.entries());
            const maxWorkers = this.config.maxWorkers || 4;
            const chunkSize = Math.ceil(binEntries.length / maxWorkers);

            // Split bins into chunks for parallel processing
            const chunks: Array<[string, Bin][]> = [];
            for (let i = 0; i < binEntries.length; i += chunkSize) {
                chunks.push(binEntries.slice(i, i + chunkSize));
            }

            // Process each chunk in parallel
            const results = await Promise.all(
                chunks.map(async (chunk) => {
                    const chunkResults: Array<[string, Uint8Array, number, number]> = [];

                    for (const [label, bin] of chunk) {
                        const binSize = bin.getData().length;
                        const compressedData = await this.compressBin(bin);

                        chunkResults.push([
                            label,
                            compressedData,
                            binSize,
                            compressedData.byteLength
                        ]);
                    }

                    return chunkResults;
                })
            );

            // Combine results
            for (const chunkResult of results) {
                for (const [label, compressedData, binSize, compressedBinSize] of chunkResult) {
                    originalSize += binSize;
                    compressedSize += compressedBinSize;
                    compressedBinMap.set(label, compressedData);
                }
            }
        } else {
            // Sequential processing
            for (const [label, bin] of bins.entries()) {
                originalSize += bin.getData().length;
                const compressedData = await this.compressBin(bin);
                compressedSize += compressedData.byteLength;
                compressedBinMap.set(label, compressedData);
            }
        }

        const endTime = performance.now();
        const compressionTime = endTime - startTime;

        // Create the result object
        const result: CompressedData = {
            compressedBins: compressedBinMap,
            metadata: {
                windows: windowInfos,
                config: this.config,
                originalData: originalData // Store original data for testing
            }
        };

        // Collect metrics if enabled
        if (this.config.collectMetrics) {
            const metrics: CompressionMetrics = {
                originalSize,
                compressedSize,
                compressionRatio: originalSize / compressedSize,
                compressionTime,
                throughput: originalSize / compressionTime * 1000, // bytes per second
                binCount: bins.size,
                averageMemoryUsage: 0,
                peakMemoryUsage: 0,
                averageBinSize: originalSize / bins.size
            };

            // Add memory metrics if tracking is enabled
            if (this.config.trackMemoryUsage) {
                const memoryMetrics = this.getMemoryMetrics();
                metrics.peakMemoryUsage = memoryMetrics.peakMemoryUsage;
                metrics.averageMemoryUsage = memoryMetrics.averageMemoryUsage;
            }

            result.metadata.metrics = metrics;
        }

        return result;
    }

    private generateLabel(window: string): string {
        // Count frequency of each character
        const frequencyMap = new Map<string, number>();

        for (const char of window) {
            const count = frequencyMap.get(char) || 0;
            frequencyMap.set(char, count + 1);
        }

        // Build Huffman tree
        const tree = this.buildHuffmanTree(frequencyMap);

        // Generate Huffman codes for each character
        const huffmanCodes = new Map<string, string>();
        this.generateHuffmanCodes(tree, "", huffmanCodes);

        // Create label based on the Huffman encoding
        // The label is created by sorting characters by their encoding length
        // and taking the first n characters, where n is the label length
        const charsByEncodingLength: { char: string, encodingLength: number }[] = [];

        // @ts-ignore
        for (const [char, code] of huffmanCodes.entries()) {
            charsByEncodingLength.push({ char, encodingLength: code.length });
        }

        // Sort by encoding length (shorter codes appear first, as they're more frequent)
        charsByEncodingLength.sort((a, b) => a.encodingLength - b.encodingLength);

        // Generate the label based on the config's labelLength
        // Ensure the label is exactly the configured length by padding or truncating
        let chars = "";
        let encodingLengths = "";

        for (let i = 0; i < Math.min(this.config.labelLength, charsByEncodingLength.length); i++) {
            const { char, encodingLength } = charsByEncodingLength[i];
            chars += char;
            encodingLengths += encodingLength;
        }

        // Pad the label if it's shorter than the configured length
        while (chars.length < this.config.labelLength) {
            chars += '_'; // Pad with underscore
        }

        // Truncate if somehow longer
        if (chars.length > this.config.labelLength) {
            chars = chars.substring(0, this.config.labelLength);
        }

        return `${chars} ${encodingLengths}`;
    }

    /**
     * Builds a Huffman tree from a frequency map
     *
     * @param frequencyMap Map of character frequencies
     * @returns The root node of the Huffman tree
     */
    private buildHuffmanTree(frequencyMap: Map<string, number>): HuffmanNode {
        // Create a leaf node for each character
        const nodes: HuffmanNode[] = [];

        // @ts-ignore
        for (const [char, frequency] of frequencyMap.entries()) {
            nodes.push(new HuffmanNode(char, frequency));
        }

        // Build the Huffman tree by combining nodes
        while (nodes.length > 1) {
            // Sort nodes by frequency (ascending)
            nodes.sort((a, b) => a.frequency - b.frequency);

            // Take the two nodes with lowest frequencies
            const left = nodes.shift()!;
            const right = nodes.shift()!;

            // Create a new internal node with these two nodes as children
            // and with frequency equal to the sum of the two nodes' frequencies
            const newNode = new HuffmanNode('\0', left.frequency + right.frequency, left, right);

            // Add the new node back to the queue
            nodes.push(newNode);
        }

        // The last remaining node is the root of the Huffman tree
        return nodes[0];
    }

    /**
     * Recursively generates Huffman codes for each character
     *
     * @param node Current node in the Huffman tree
     * @param code Current code
     * @param huffmanCodes Map to store character codes
     */
    private generateHuffmanCodes(
        node: HuffmanNode,
        code: string,
        huffmanCodes: Map<string, string>
    ): void {
        if (node === null) return;

        // If this is a leaf node, store the code
        if (node.isLeaf()) {
            huffmanCodes.set(node.char, code);
            return;
        }

        // Traverse left and right children
        this.generateHuffmanCodes(node.left!, code + "0", huffmanCodes);
        this.generateHuffmanCodes(node.right!, code + "1", huffmanCodes);
    }

    /**
     * Groups labeled windows into bins with the same label
     *
     * @param labeledWindows Array of windows with their labels
     * @returns Map of bins by label
     */
    private groupIntoBins(
        labeledWindows: Array<{ window: string, label: string, index: number }>
    ): Map<string, Bin> {
        const bins = new Map<string, Bin>();

        // If subBinning is enabled, use a more sophisticated grouping algorithm
        if (this.config.subBinning) {
            return this.groupIntoBinsWithSubBinning(labeledWindows);
        }

        // Simple grouping by label
        labeledWindows.forEach(({ window, label, index }) => {
            if (!bins.has(label)) {
                bins.set(label, new Bin(label));
            }

            // Add the window to the bin, tracking its index
            bins.get(label)!.addSegment(window, index);
        });

        return bins;
    }

    /**
     * Groups labeled windows into bins with sub-binning for better compression
     * This uses a more sophisticated algorithm that groups windows not just by label
     * but also by content similarity within the same label group
     *
     * @param labeledWindows Array of windows with their labels
     * @returns Map of bins by label
     */
    private groupIntoBinsWithSubBinning(
        labeledWindows: Array<{ window: string, label: string, index: number }>
    ): Map<string, Bin> {
        const bins = new Map<string, Bin>();

        // First, group windows by label
        const windowsByLabel = new Map<string, Array<{ window: string, index: number }>>();

        for (const { window, label, index } of labeledWindows) {
            if (!windowsByLabel.has(label)) {
                windowsByLabel.set(label, []);
            }
            windowsByLabel.get(label)!.push({ window, index });
        }

        // For each label group, perform sub-binning if there are enough windows
        for (const [label, windows] of windowsByLabel.entries()) {
            if (windows.length <= 1) {
                // If there's only one window with this label, no need for sub-binning
                const bin = new Bin(label);
                bin.addSegment(windows[0].window, windows[0].index);
                bins.set(label, bin);
                continue;
            }

            let subBins = new Map<string, Bin>();
            // For larger groups, use content-based similarity to create sub-bins
            if (this.config.subBinningDepth != null) {
               subBins = this.createSubBins(label, windows, this.config.subBinningDepth);
            }

            // Add sub-bins to the main bins map
            for (const [subLabel, subBin] of subBins.entries()) {
                bins.set(subLabel, subBin);
            }
        }

        return bins;
    }

    /**
     * Creates sub-bins based on content similarity
     *
     * @param baseLabel The base label for the group
     * @param windows Array of windows and their indices
     * @param depth Maximum depth for recursive sub-binning
     * @returns Map of sub-bins by label
     */
    private createSubBins(
        baseLabel: string,
        windows: Array<{ window: string, index: number }>,
        depth: number
    ): Map<string, Bin> {
        const subBins = new Map<string, Bin>();

        if (depth <= 0 || windows.length <= 1) {
            // Base case: create a single bin with the base label
            const bin = new Bin(baseLabel);
            for (const { window, index } of windows) {
                bin.addSegment(window, index);
            }
            subBins.set(baseLabel, bin);
            return subBins;
        }

        // Group windows by similarity
        // For simplicity, we'll use character frequency as a similarity measure
        const groups = this.groupByCharacterFrequency(windows);

        // Create sub-bins for each group
        let subBinIndex = 0;
        for (const group of groups) {
            const subLabel = `${baseLabel}_${subBinIndex}`;

            // Recursively create sub-bins
            const nestedSubBins = this.createSubBins(subLabel, group, depth - 1);

            // Add nested sub-bins to the main sub-bins map
            for (const [nestedLabel, nestedBin] of nestedSubBins.entries()) {
                subBins.set(nestedLabel, nestedBin);
            }

            subBinIndex++;
        }

        return subBins;
    }

    /**
     * Groups windows by character frequency similarity
     *
     * @param windows Array of windows and their indices
     * @returns Array of window groups
     */
    private groupByCharacterFrequency(
        windows: Array<{ window: string, index: number }>
    ): Array<Array<{ window: string, index: number }>> {
        if (windows.length <= 1) {
            return [windows];
        }

        // Calculate character frequency for each window
        const frequencyMaps = windows.map(({ window }) => {
            const freqMap = new Map<string, number>();
            for (const char of window) {
                const count = freqMap.get(char) || 0;
                freqMap.set(char, count + 1);
            }
            return freqMap;
        });

        // Calculate similarity between windows
        const similarities: Array<{ i: number, j: number, similarity: number }> = [];

        for (let i = 0; i < windows.length; i++) {
            for (let j = i + 1; j < windows.length; j++) {
                const similarity = this.calculateSimilarity(frequencyMaps[i], frequencyMaps[j]);
                similarities.push({ i, j, similarity });
            }
        }

        // Sort similarities in descending order
        similarities.sort((a, b) => b.similarity - a.similarity);

        // Group windows based on similarity
        const groups: Array<Array<{ window: string, index: number }>> = [];
        const assigned = new Set<number>();

        for (const { i, j, similarity } of similarities) {
            if (assigned.has(i) || assigned.has(j)) {
                continue;
            }

            // Create a new group with these two windows
            const group = [windows[i], windows[j]];
            assigned.add(i);
            assigned.add(j);

            // Add any remaining windows that are similar to this group
            for (let k = 0; k < windows.length; k++) {
                if (assigned.has(k)) {
                    continue;
                }

                // Calculate average similarity to the group
                let avgSimilarity = 0;
                for (const { window } of group) {
                    const freqMap = new Map<string, number>();
                    for (const char of window) {
                        const count = freqMap.get(char) || 0;
                        freqMap.set(char, count + 1);
                    }
                    avgSimilarity += this.calculateSimilarity(freqMap, frequencyMaps[k]);
                }
                avgSimilarity /= group.length;

                // If similarity is above threshold, add to group
                const threshold = this.config.similarityThreshold || 0.7;
                if (avgSimilarity > threshold) {
                    group.push(windows[k]);
                    assigned.add(k);
                }
            }

            groups.push(group);
        }

        // Add any remaining windows as individual groups
        for (let i = 0; i < windows.length; i++) {
            if (!assigned.has(i)) {
                groups.push([windows[i]]);
                assigned.add(i);
            }
        }

        return groups;
    }

    /**
     * Calculates similarity between two character frequency maps
     *
     * @param map1 First character frequency map
     * @param map2 Second character frequency map
     * @returns Similarity score between 0 and 1
     */
    private calculateSimilarity(
        map1: Map<string, number>,
        map2: Map<string, number>
    ): number {
        // Get all unique characters
        const allChars = new Set<string>();
        for (const char of map1.keys()) {
            allChars.add(char);
        }
        for (const char of map2.keys()) {
            allChars.add(char);
        }

        // Calculate dot product and magnitudes
        let dotProduct = 0;
        let magnitude1 = 0;
        let magnitude2 = 0;

        for (const char of allChars) {
            const freq1 = map1.get(char) || 0;
            const freq2 = map2.get(char) || 0;

            dotProduct += freq1 * freq2;
            magnitude1 += freq1 * freq1;
            magnitude2 += freq2 * freq2;
        }

        // Calculate cosine similarity
        if (magnitude1 === 0 || magnitude2 === 0) {
            return 0;
        }

        return dotProduct / (Math.sqrt(magnitude1) * Math.sqrt(magnitude2));
    }

    private async compressBin(bin: Bin): Promise<Uint8Array> {
        const rawData = this.textEncoder.encode(bin.getData());

        switch (this.config.compressionMethod as CompressionMethod) {
            case COMPRESSION_METHODS.HUFFMAN:
                return this.huffmanCompress(bin.getData());

            case COMPRESSION_METHODS.ZSTD:
                // Ensure compression is initialized before using
                await ensureCompressionInitialized();
                return await compress(rawData);

            case COMPRESSION_METHODS.RAW:
            default:
                return rawData;
        }
    }

    /**
     * Decompresses a bin using the configured compression method
     *
     * @param compressedData The compressed data
     * @returns Decompressed data as string
     */
    private async decompressBin(compressedData: Uint8Array): Promise<string> {
        switch (this.config.compressionMethod as CompressionMethod) {
            case COMPRESSION_METHODS.HUFFMAN:
                return this.huffmanDecompress(compressedData);

            case COMPRESSION_METHODS.ZSTD:
                // Ensure compression is initialized before using
                await ensureCompressionInitialized();
                const decompressedData = await decompress(compressedData);
                return this.textDecoder.decode(decompressedData);

            case COMPRESSION_METHODS.RAW:
            default:
                return this.textDecoder.decode(compressedData);
        }
    }

    /**
     * Compresses data using Huffman coding
     *
     * @param data The data to compress
     * @returns Compressed data as Uint8Array
     */
    private huffmanCompress(data: string): Uint8Array {
        // Count frequency of each character
        const frequencyMap = new Map<string, number>();