s3-bucket-stream/build/local_generators.js

Readable stream of the Body of every object in an S3 bucket.

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const awslib_1 = require("./bws/awslib");
const awsbatch_1 = require("./bws/awsbatch");
const fs_1 = __importDefault(require("fs"));
let keysPath = "./keys";
async function* generateKeys(Bucket, MaxKeys = 1000) {
    let StartAfter = undefined;
    if (keysPath && fs_1.default.existsSync(keysPath)) {
        // User can define a filepath to read from
        // containing the current progress in the form
        // of a list of Keys.
        let keyfile = fs_1.default.readFileSync(keysPath).toString();
        let keys = keyfile.split("\n").sort();
        let progress = undefined;
        for (let i = keys.length - 1; i >= 0; i--) {
            progress = keys[i];
            if (progress !== undefined && progress.length > 0) {
                break;
            }
        }
        if (progress !== undefined && progress.length > 0) {
            StartAfter = progress;
            console.log(`Found existing progress, continuing from ${StartAfter}`);
        }
    }
    let Keys = null;
    try {
        Keys = await awslib_1.list(Bucket, StartAfter, MaxKeys);
        StartAfter = Keys.slice(-1)[0];
        yield { Bucket, Keys };
    }
    catch {
        return;
    }
    while (Keys.length === MaxKeys) {
        try {
            Keys = await awslib_1.list(Bucket, StartAfter, MaxKeys);
        }
        catch {
            continue;
        }
        StartAfter = Keys.slice(-1)[0];
        if (Keys.length > 0) {
            yield { Bucket, Keys };
        }
    }
    StartAfter = undefined;
}
exports.generateKeys = generateKeys;
async function* generateBatch(Bucket, MaxKeys = 1000) {
    for await (let keygen of generateKeys(Bucket, MaxKeys)) {
        try {
            let batch = await awsbatch_1.getBatch(keygen.Bucket, keygen.Keys);
            yield batch;
        }
        catch {
            continue;
        }
    }
}
exports.generateBatch = generateBatch;