crawfishcloud/build/es/exporters.js

A Streaming S3 Bucket Glob Crawler

import vfile from 'vfile';
import Vinyl from 'vinyl';
export const drainReadable = (init, r, tail) => {
    const acc = [init];
    return new Promise((resolve, reject) => {
        r
            .on('data', (chunk) => { acc.push(chunk.toString()); })
            .on('end', () => resolve(`${acc.join('')}${tail}`))
            .on('error', (er) => reject(er));
    });
};
export const asS3 = async (o, i) => o;
export const asVfile = async (o, i) => {
    const Body = o.Body;
    if (Buffer.isBuffer(Body) || typeof Body === 'string') {
        return vfile({
            path: o.Key,
            contents: Buffer.from(Body),
            Bucket: o.Bucket
        });
    }
    else {
        return vfile({
            path: o.Key,
            contents: await drainReadable('', Body, ''),
            Bucket: o.Bucket
        });
    }
};
export const asVinyl = async (o, i) => {
    const Body = o.Body;
    if (Buffer.isBuffer(Body) || typeof Body === 'string') {
        return new Vinyl({
            path: o.Key,
            contents: Buffer.from(Body),
            Bucket: o.Bucket
        });
    }
    else {
        return new Vinyl({
            path: o.Key,
            contents: Buffer.from(await drainReadable('', Body, '')),
            Bucket: o.Bucket
        });
    }
};