@cumulus/ingest/lock.js

Ingest utilities

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.proceed = exports.removeLock = exports.countLock = exports.checkOldLocks = void 0;
const S3_1 = require("@cumulus/aws-client/S3");
const common_1 = require("@cumulus/common");
const log = __importStar(require("@cumulus/common/log"));
const lockPrefix = 'lock';
/**
 * Checks all locks and removes those older than five minutes. Returns a count
 * of locks that are not older than configured retention period or 5 minutes.
 *
 * @param {object} bucket - The AWS S3 bucket with the locks to check
 * @param {Array} locks - The list of locks in the bucket
 * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
 * @returns {integer} - Number of locks remaining in bucket
 */
async function checkOldLocks(bucket, locks = [], retentionTimeInSeconds = 300) {
    const expirationTimestamp = Date.now() - (retentionTimeInSeconds * 1000);
    const expiredLocks = locks.filter((lock) => {
        if (!lock.LastModified) {
            throw new TypeError(`Could not find LastModified on ${JSON.stringify(lock)}`);
        }
        return lock.LastModified.getTime() < expirationTimestamp;
    });
    await Promise.all(expiredLocks.map((lock) => {
        if (!lock.Key) {
            throw new TypeError(`Could not find Key on ${JSON.stringify(lock)}`);
        }
        log.debug(`Removing expired lock ${JSON.stringify(lock)}`);
        return (0, S3_1.deleteS3Object)(bucket, lock.Key);
    }));
    return locks.length - expiredLocks.length;
}
exports.checkOldLocks = checkOldLocks;
/**
 * Counts the number of locks in a bucket.
 *
 * @param {object} bucket - The AWS S3 bucket to check
 * @param {string} providerName - The provider name
 * @param {number} retentionTimeInSeconds - lock retention time in seconds, default is 300
 * @returns {integer} - Number of current locks in the bucket
 */
async function countLock(bucket, providerName, retentionTimeInSeconds) {
    const locks = await (0, S3_1.listS3ObjectsV2)({
        Bucket: bucket,
        Prefix: `${lockPrefix}/${providerName}`,
    });
    return checkOldLocks(bucket, locks, retentionTimeInSeconds);
}
exports.countLock = countLock;
async function addLock(bucket, providerName, granuleId) {
    const key = `${lockPrefix}/${providerName}/${granuleId}`;
    await (0, S3_1.s3PutObject)({
        Bucket: bucket,
        Key: key,
        Body: '',
    });
}
async function removeLock(bucket, providerName, granuleId) {
    await (0, S3_1.deleteS3Object)(bucket, `${lockPrefix}/${providerName}/${granuleId}`);
}
exports.removeLock = removeLock;
/**
 *
 * @param {string} bucket - system bucket to place the lock files
 * @param {object} provider - provider object
 * @param {string} provider.id - provider id
 * @param {number} provider.globalConnectionLimit - provider globalConnectionLimit
 * @param {number} provider.maxDownloadTime - provider maxDownloadTime for a granule
 * @param {string} granuleId - id of downloading granule
 * @param {number} counter - retry counter
 * @returns {Promise<boolean>}
 */
async function proceed(bucket, provider, granuleId, counter = 0) {
    const { globalConnectionLimit, maxDownloadTime } = provider;
    if (globalConnectionLimit === undefined) {
        return true;
    }
    // Fail if lock is not removed after 270 tries.
    if (counter > 270) {
        log.debug(`The "${provider.id}" provider has no lock available after ${counter} retries`);
        return false;
    }
    const count = await countLock(bucket, provider.id, maxDownloadTime);
    if (count >= globalConnectionLimit) {
        log.debug(`The "${provider.id}" provider's globalConnectionLimit of "${provider.globalConnectionLimit}" has been reached.`);
        // wait for 5 second and try again
        await (0, common_1.sleep)(5000);
        return proceed(bucket, provider, granuleId, counter + 1);
    }
    // add the lock
    await addLock(bucket, provider.id, granuleId);
    return true;
}
exports.proceed = proceed;
//# sourceMappingURL=lock.js.map