crawfishcloud
Version:
A Streaming S3 Bucket Glob Crawler
70 lines (69 loc) • 2.32 kB
TypeScript
/// <reference types="node" />
import type { Readable } from 'stream';
import type { S3 } from 'aws-sdk';
import type { VFile } from 'vfile';
import type vinyl from 'vinyl';
export type { S3 } from 'aws-sdk';
export type { VFile } from 'vfile';
export declare type Vinyl = vinyl;
export interface S3Config {
Bucket: string;
Key: string;
[others: string]: string;
}
export interface S3BucketPrefix {
Bucket: string;
Key: string;
prefix: string;
suffix: string;
[others: string]: string;
}
export declare type S3NodeBody = Buffer | string | Readable;
export declare type FullS3Object = S3.Object & S3.GetObjectOutput & {
Bucket: string;
};
export declare type S3Item = FullS3Object & {
Body: S3NodeBody;
};
export declare type Dict<T> = {
[key: string]: T;
};
export declare type UsingFunc<T> = (s3Item: S3Item, i: number) => Promise<T>;
export declare type VfileWithBucket = VFile & {
Bucket: string;
};
export declare type VinylWithBucket = Vinyl & {
Bucket: string;
};
export interface CrawfishCloudReturnNoProto {
iter<T>(inp: {
body: boolean;
using: UsingFunc<T>;
NextContinuationToken?: string;
}, ...filters: string[]): AsyncGenerator<T, void, undefined>;
stream<T>(inp: {
body: boolean;
using: UsingFunc<T>;
}, ...filters: string[]): Readable;
all<T>(inp: {
body: boolean;
using: UsingFunc<T>;
}, ...filters: string[]): Promise<T[]>;
reduce<T, U>(init: T, mapper: UsingFunc<U>, reducer: (prior: T, current: U, i: number) => T, ...filters: string[]): Promise<T>;
vfileStream(...filters: string[]): Readable;
vinylStream(...filters: string[]): Readable;
s3Stream(...filters: string[]): Readable;
vfileIter(...filters: string[]): AsyncGenerator<VFile, void, undefined>;
vinylIter(...filters: string[]): AsyncGenerator<Vinyl, void, undefined>;
s3Iter(...filters: string[]): AsyncGenerator<S3Item, void, undefined>;
vfileArray(...filters: string[]): Promise<VFile[]>;
vinylArray(...filters: string[]): Promise<Vinyl[]>;
s3Array(...filters: string[]): Promise<S3Item[]>;
}
export interface CrawfishCloud {
(input: {
s3c: S3;
body?: boolean;
maxkeys?: number;
}, ...filters: string[]): CrawfishCloudReturnNoProto;
}