@gravityai-dev/pinecone/dist/pinecone/operations/upsert.js

Pinecone vector database nodes for GravityWorkflow - knowledge management and vector operations

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.upsertVectorsInBatches = upsertVectorsInBatches;
/**
 * Pinecone upsert operations
 */
const logger_1 = require("../../../logger");
const DEFAULT_RETRY_CONFIG = {
    maxRetries: 3,
    baseDelayMs: 1000, // Start with 1 second
    maxDelayMs: 30000, // Max 30 seconds
    backoffMultiplier: 2,
};
/**
 * Check if an error is retryable (timeouts, rate limits, temporary failures)
 */
function isRetryableError(error) {
    if (!error)
        return false;
    const errorMessage = error.message?.toLowerCase() || '';
    const errorCode = error.code || error.status;
    // Check for common retryable conditions
    const retryableConditions = [
        // Timeout errors
        errorMessage.includes('timeout'),
        errorMessage.includes('504'),
        errorMessage.includes('gateway timeout'),
        errorMessage.includes('request timeout'),
        // Rate limiting
        errorMessage.includes('rate limit'),
        errorMessage.includes('too many requests'),
        errorCode === 429,
        // Temporary server errors
        errorMessage.includes('502'),
        errorMessage.includes('503'),
        errorMessage.includes('bad gateway'),
        errorMessage.includes('service unavailable'),
        errorCode >= 500 && errorCode < 600,
        // Network errors
        errorMessage.includes('network'),
        errorMessage.includes('connection'),
        errorMessage.includes('econnreset'),
        errorMessage.includes('enotfound'),
    ];
    return retryableConditions.some(condition => condition);
}
/**
 * Sleep for specified milliseconds
 */
function sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
}
/**
 * Calculate delay with exponential backoff and jitter
 */
function calculateDelay(attempt, config) {
    const exponentialDelay = config.baseDelayMs * Math.pow(config.backoffMultiplier, attempt);
    const cappedDelay = Math.min(exponentialDelay, config.maxDelayMs);
    // Add jitter (±25% randomization) to prevent thundering herd
    const jitter = cappedDelay * 0.25 * (Math.random() - 0.5);
    return Math.max(cappedDelay + jitter, config.baseDelayMs);
}
/**
 * Upsert vectors to Pinecone in batches with retry logic
 */
async function upsertVectorsInBatches(indexOperation, vectors, batchSize = 100, workflowId = "", runId = "", retryConfig = {}) {
    const config = { ...DEFAULT_RETRY_CONFIG, ...retryConfig };
    let totalUpserted = 0;
    for (let i = 0; i < vectors.length; i += batchSize) {
        const batch = vectors.slice(i, i + batchSize);
        const batchNumber = Math.floor(i / batchSize) + 1;
        const totalBatches = Math.ceil(vectors.length / batchSize);
        let lastError;
        let success = false;
        // Retry logic for each batch
        for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
            try {
                if (attempt > 0) {
                    const delay = calculateDelay(attempt - 1, config);
                    logger_1.logger.info(`Retrying batch ${batchNumber}/${totalBatches} (attempt ${attempt + 1}/${config.maxRetries + 1}) after ${Math.round(delay)}ms delay`, "upsertVectorsInBatches");
                    await sleep(delay);
                }
                await indexOperation.upsert(batch);
                totalUpserted += batch.length;
                success = true;
                const logMessage = attempt > 0
                    ? `Successfully upserted batch ${batchNumber}/${totalBatches} of ${batch.length} vectors after ${attempt} retries`
                    : `Upserted batch ${batchNumber}/${totalBatches} of ${batch.length} vectors`;
                logger_1.logger.info(logMessage, "upsertVectorsInBatches");
                break; // Success, exit retry loop
            }
            catch (error) {
                lastError = error;
                const isRetryable = isRetryableError(error);
                const isLastAttempt = attempt === config.maxRetries;
                logger_1.logger.warn(`Failed to upsert batch ${batchNumber}/${totalBatches} starting at index ${i} (attempt ${attempt + 1}/${config.maxRetries + 1})`, "upsertVectorsInBatches", {
                    error: error.message,
                    isRetryable,
                    isLastAttempt,
                    batchSize: batch.length,
                });
                // If not retryable or last attempt, break out of retry loop
                if (!isRetryable || isLastAttempt) {
                    break;
                }
            }
        }
        // If we didn't succeed after all retries, throw the last error
        if (!success) {
            logger_1.logger.error(`Failed to upsert batch ${batchNumber}/${totalBatches} after ${config.maxRetries + 1} attempts`, "upsertVectorsInBatches", {
                error: lastError?.message,
                batchStartIndex: i,
                batchSize: batch.length,
                totalVectors: vectors.length,
            });
            throw lastError;
        }
    }
    logger_1.logger.info(`Successfully upserted all ${totalUpserted} vectors in ${Math.ceil(vectors.length / batchSize)} batches`, "upsertVectorsInBatches");
    return totalUpserted;
}
//# sourceMappingURL=upsert.js.map