semantic-ds-toolkit/dist/src/optimization/index.js

Performance-first semantic layer for modern data stacks - Stable Column Anchors & intelligent inference

"use strict";
// Performance Optimization Module
// High-performance components for Semantic Data Science Toolkit
Object.defineProperty(exports, "__esModule", { value: true });
exports.OPTIMIZATION_CONFIGS = exports.PERFORMANCE_TARGETS = exports.globalBenchmarkSuite = exports.runPerformanceBenchmark = exports.OptimizationBenchmarkSuite = exports.createGlobalPools = exports.createOptimizedPool = exports.WarehouseConnectionPool = exports.ConnectionPool = exports.globalAnchorHNSW = exports.globalHNSW = exports.ColumnAnchorHNSW = exports.HierarchicalNavigableSmallWorld = exports.globalFingerprintCompressor = exports.globalProductQuantizer = exports.ColumnFingerprintCompressor = exports.ProductQuantizer = exports.BatchWorkerPool = exports.BatchRingBuffer = exports.BatchObjectPool = exports.optimizedBatchProcessor = exports.OptimizedBatchProcessorV2 = exports.globalBatchProcessor = exports.HighPerformanceBatchProcessor = exports.globalLazyEvaluator = exports.LazyComputation = exports.LazyEvaluator = exports.globalCacheManager = exports.MultiTierCacheManager = exports.globalProfiler = exports.PerformanceProfiler = void 0;
exports.getOptimizationConfig = getOptimizationConfig;
exports.validatePerformanceTargets = validatePerformanceTargets;
// Core performance components
var performance_profiler_1 = require("./performance-profiler");
Object.defineProperty(exports, "PerformanceProfiler", { enumerable: true, get: function () { return performance_profiler_1.PerformanceProfiler; } });
Object.defineProperty(exports, "globalProfiler", { enumerable: true, get: function () { return performance_profiler_1.globalProfiler; } });
var cache_manager_1 = require("./cache-manager");
Object.defineProperty(exports, "MultiTierCacheManager", { enumerable: true, get: function () { return cache_manager_1.MultiTierCacheManager; } });
Object.defineProperty(exports, "globalCacheManager", { enumerable: true, get: function () { return cache_manager_1.globalCacheManager; } });
var lazy_evaluator_1 = require("./lazy-evaluator");
Object.defineProperty(exports, "LazyEvaluator", { enumerable: true, get: function () { return lazy_evaluator_1.LazyEvaluator; } });
Object.defineProperty(exports, "LazyComputation", { enumerable: true, get: function () { return lazy_evaluator_1.LazyComputation; } });
Object.defineProperty(exports, "globalLazyEvaluator", { enumerable: true, get: function () { return lazy_evaluator_1.globalLazyEvaluator; } });
var batch_processor_1 = require("./batch-processor");
Object.defineProperty(exports, "HighPerformanceBatchProcessor", { enumerable: true, get: function () { return batch_processor_1.HighPerformanceBatchProcessor; } });
Object.defineProperty(exports, "globalBatchProcessor", { enumerable: true, get: function () { return batch_processor_1.globalBatchProcessor; } });
var batch_processor_v2_1 = require("./batch-processor-v2");
Object.defineProperty(exports, "OptimizedBatchProcessorV2", { enumerable: true, get: function () { return batch_processor_v2_1.OptimizedBatchProcessorV2; } });
Object.defineProperty(exports, "optimizedBatchProcessor", { enumerable: true, get: function () { return batch_processor_v2_1.optimizedBatchProcessor; } });
Object.defineProperty(exports, "BatchObjectPool", { enumerable: true, get: function () { return batch_processor_v2_1.ObjectPool; } });
Object.defineProperty(exports, "BatchRingBuffer", { enumerable: true, get: function () { return batch_processor_v2_1.RingBuffer; } });
Object.defineProperty(exports, "BatchWorkerPool", { enumerable: true, get: function () { return batch_processor_v2_1.WorkerPool; } });
// Vector operations and compression
var vector_compression_1 = require("./vector-compression");
Object.defineProperty(exports, "ProductQuantizer", { enumerable: true, get: function () { return vector_compression_1.ProductQuantizer; } });
Object.defineProperty(exports, "ColumnFingerprintCompressor", { enumerable: true, get: function () { return vector_compression_1.ColumnFingerprintCompressor; } });
Object.defineProperty(exports, "globalProductQuantizer", { enumerable: true, get: function () { return vector_compression_1.globalProductQuantizer; } });
Object.defineProperty(exports, "globalFingerprintCompressor", { enumerable: true, get: function () { return vector_compression_1.globalFingerprintCompressor; } });
// Similarity search and indexing
var hnsw_index_1 = require("./hnsw-index");
Object.defineProperty(exports, "HierarchicalNavigableSmallWorld", { enumerable: true, get: function () { return hnsw_index_1.HierarchicalNavigableSmallWorld; } });
Object.defineProperty(exports, "ColumnAnchorHNSW", { enumerable: true, get: function () { return hnsw_index_1.ColumnAnchorHNSW; } });
Object.defineProperty(exports, "globalHNSW", { enumerable: true, get: function () { return hnsw_index_1.globalHNSW; } });
Object.defineProperty(exports, "globalAnchorHNSW", { enumerable: true, get: function () { return hnsw_index_1.globalAnchorHNSW; } });
// Database connection management
var connection_pool_1 = require("./connection-pool");
Object.defineProperty(exports, "ConnectionPool", { enumerable: true, get: function () { return connection_pool_1.ConnectionPool; } });
Object.defineProperty(exports, "WarehouseConnectionPool", { enumerable: true, get: function () { return connection_pool_1.WarehouseConnectionPool; } });
Object.defineProperty(exports, "createOptimizedPool", { enumerable: true, get: function () { return connection_pool_1.createOptimizedPool; } });
Object.defineProperty(exports, "createGlobalPools", { enumerable: true, get: function () { return connection_pool_1.createGlobalPools; } });
// Benchmarking and testing
var benchmark_suite_1 = require("./benchmark-suite");
Object.defineProperty(exports, "OptimizationBenchmarkSuite", { enumerable: true, get: function () { return benchmark_suite_1.OptimizationBenchmarkSuite; } });
Object.defineProperty(exports, "runPerformanceBenchmark", { enumerable: true, get: function () { return benchmark_suite_1.runPerformanceBenchmark; } });
Object.defineProperty(exports, "globalBenchmarkSuite", { enumerable: true, get: function () { return benchmark_suite_1.globalBenchmarkSuite; } });
// Performance targets and constants
exports.PERFORMANCE_TARGETS = {
    // From strategic analysis
    SCA_PROCESSING: "1M+ rows/second",
    INFERENCE_LATENCY: "<100ms for 1M rows",
    CACHE_HIT_RATE: ">90%",
    MEMORY_FOOTPRINT: "<10MB per million rows",
    CONCURRENT_OPERATIONS: "100+ parallel scans",
    // Competitive benchmarks
    METADATA_SCALE: "100K+ SCAs like DataHub",
    QUERY_RESPONSE: "<200ms like Elasticsearch catalogs",
    TIME_TO_VALUE: "<15 minutes (market expects)",
    // Technical benchmarks
    HASH_THROUGHPUT: "13.2 GB/s (xxHash64)",
    VECTOR_COMPRESSION: "4x+ compression ratio",
    SEARCH_PERFORMANCE: "1K+ searches/sec"
};
// Optimization configuration presets
exports.OPTIMIZATION_CONFIGS = {
    // Development environment
    DEVELOPMENT: {
        profiler: { maxConcurrency: 4 },
        cache: { l1MaxSize: 1000, l2MaxSize: 10000, l3MaxSize: 100000 },
        batchProcessor: { maxWorkers: 2 },
        connectionPool: { minConnections: 2, maxConnections: 10 }
    },
    // Production environment
    PRODUCTION: {
        profiler: { maxConcurrency: 20 },
        cache: { l1MaxSize: 5000, l2MaxSize: 50000, l3MaxSize: 500000 },
        batchProcessor: { maxWorkers: 16 },
        connectionPool: { minConnections: 10, maxConnections: 100 }
    },
    // Enterprise environment
    ENTERPRISE: {
        profiler: { maxConcurrency: 50 },
        cache: { l1MaxSize: 10000, l2MaxSize: 100000, l3MaxSize: 1000000 },
        batchProcessor: { maxWorkers: 32 },
        connectionPool: { minConnections: 20, maxConnections: 200 }
    }
};
// Utility functions
function getOptimizationConfig(environment) {
    return exports.OPTIMIZATION_CONFIGS[environment];
}
function validatePerformanceTargets(metrics) {
    const results = [];
    let passed = 0;
    // Hash operations
    if (metrics.hashThroughput) {
        const hashPassed = metrics.hashThroughput > 1000000; // 1M ops/sec
        results.push({
            metric: 'Hash Throughput',
            value: `${metrics.hashThroughput.toLocaleString()} ops/sec`,
            target: '1M+ ops/sec',
            passed: hashPassed
        });
        if (hashPassed)
            passed++;
    }
    // Cache hit rate
    if (metrics.cacheHitRate !== undefined) {
        const cachePassed = metrics.cacheHitRate >= 0.9;
        results.push({
            metric: 'Cache Hit Rate',
            value: `${(metrics.cacheHitRate * 100).toFixed(1)}%`,
            target: '90%+',
            passed: cachePassed
        });
        if (cachePassed)
            passed++;
    }
    // Batch processing
    if (metrics.batchThroughput) {
        const batchPassed = metrics.batchThroughput > 1000000; // 1M rows/sec
        results.push({
            metric: 'Batch Throughput',
            value: `${metrics.batchThroughput.toLocaleString()} rows/sec`,
            target: '1M+ rows/sec',
            passed: batchPassed
        });
        if (batchPassed)
            passed++;
    }
    // Vector compression
    if (metrics.compressionRatio) {
        const compressionPassed = metrics.compressionRatio >= 4;
        results.push({
            metric: 'Compression Ratio',
            value: `${metrics.compressionRatio.toFixed(1)}x`,
            target: '4x+',
            passed: compressionPassed
        });
        if (compressionPassed)
            passed++;
    }
    // Search performance
    if (metrics.searchThroughput) {
        const searchPassed = metrics.searchThroughput > 1000; // 1K searches/sec
        results.push({
            metric: 'Search Throughput',
            value: `${metrics.searchThroughput.toLocaleString()} searches/sec`,
            target: '1K+ searches/sec',
            passed: searchPassed
        });
        if (searchPassed)
            passed++;
    }
    return {
        passed,
        total: results.length,
        details: results
    };
}
//# sourceMappingURL=index.js.map