crawfishcloud/build/es/utils.js

A Streaming S3 Bucket Glob Crawler

import { URL } from 'url';
export const s3ConfigToUrl = (s3cfg) => {
    const { Bucket, Key, ...others } = s3cfg;
    const s3urlQ = Object.entries(others).map(([key, val]) => {
        return `${encodeURI(key)}=${encodeURI(val)}`;
    }).join('&');
    return `s3://${Bucket}/${Key}${s3urlQ.length > 0 ? `?${s3urlQ}` : ''}`;
};
export const s3urlToConfig = (s3url) => {
    const s3Cfg = new URL(s3url);
    const qs = [...s3Cfg.searchParams.entries()]
        .reduce((acc, [key, val]) => ({ ...acc, [key]: val }), {});
    return {
        Bucket: s3Cfg.hostname,
        Key: s3Cfg.pathname.slice(1),
        ...qs
    };
};
export const loadObjectList = (s3c, Bucket, ...keyList) => Promise.all(keyList.map(async (o) => {
    var _a;
    return ({
        ...(await s3c.getObject({
            Bucket,
            Key: typeof o === 'string'
                ? o
                : (_a = o.Key) !== null && _a !== void 0 ? _a : ''
        }).promise()),
        ...o,
    });
}));
export const s3urlToConfigWfilters = (filter, i, a) => {
    const s3Cfg = s3urlToConfig(filter);
    const prefix = s3Cfg.Key.includes('*') ? s3Cfg.Key.split('*')[0] : s3Cfg.Key;
    const suffix = s3Cfg.Key.includes('*') ? s3Cfg.Key.slice(prefix.length) : '';
    return { ...s3Cfg, prefix, suffix };
};