crawfishcloud
Version:
A Streaming S3 Bucket Glob Crawler
66 lines (65 loc) • 2.6 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.asVinyl = exports.asVfile = exports.asS3 = exports.drainReadable = void 0;
const vfile_1 = __importDefault(require("vfile"));
const vinyl_1 = __importDefault(require("vinyl"));
const drainReadable = (init, r, tail) => {
const acc = [init];
return new Promise((resolve, reject) => {
r
.on('data', (chunk) => { acc.push(chunk.toString()); })
.on('end', () => resolve(`${acc.join('')}${tail}`))
.on('error', (er) => reject(er));
});
};
exports.drainReadable = drainReadable;
const asS3 = (o, i) => __awaiter(void 0, void 0, void 0, function* () { return o; });
exports.asS3 = asS3;
const asVfile = (o, i) => __awaiter(void 0, void 0, void 0, function* () {
const Body = o.Body;
if (Buffer.isBuffer(Body) || typeof Body === 'string') {
return vfile_1.default({
path: o.Key,
contents: Buffer.from(Body),
Bucket: o.Bucket
});
}
else {
return vfile_1.default({
path: o.Key,
contents: yield exports.drainReadable('', Body, ''),
Bucket: o.Bucket
});
}
});
exports.asVfile = asVfile;
const asVinyl = (o, i) => __awaiter(void 0, void 0, void 0, function* () {
const Body = o.Body;
if (Buffer.isBuffer(Body) || typeof Body === 'string') {
return new vinyl_1.default({
path: o.Key,
contents: Buffer.from(Body),
Bucket: o.Bucket
});
}
else {
return new vinyl_1.default({
path: o.Key,
contents: Buffer.from(yield exports.drainReadable('', Body, '')),
Bucket: o.Bucket
});
}
});
exports.asVinyl = asVinyl;