crawfishcloud
Version:
A Streaming S3 Bucket Glob Crawler
57 lines (56 loc) • 2.92 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __rest = (this && this.__rest) || function (s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.s3urlToConfigWfilters = exports.loadObjectList = exports.s3urlToConfig = exports.s3ConfigToUrl = void 0;
const url_1 = require("url");
const s3ConfigToUrl = (s3cfg) => {
const { Bucket, Key } = s3cfg, others = __rest(s3cfg, ["Bucket", "Key"]);
const s3urlQ = Object.entries(others).map(([key, val]) => {
return `${encodeURI(key)}=${encodeURI(val)}`;
}).join('&');
return `s3://${Bucket}/${Key}${s3urlQ.length > 0 ? `?${s3urlQ}` : ''}`;
};
exports.s3ConfigToUrl = s3ConfigToUrl;
const s3urlToConfig = (s3url) => {
const s3Cfg = new url_1.URL(s3url);
const qs = [...s3Cfg.searchParams.entries()]
.reduce((acc, [key, val]) => (Object.assign(Object.assign({}, acc), { [key]: val })), {});
return Object.assign({ Bucket: s3Cfg.hostname, Key: s3Cfg.pathname.slice(1) }, qs);
};
exports.s3urlToConfig = s3urlToConfig;
const loadObjectList = (s3c, Bucket, ...keyList) => Promise.all(keyList.map((o) => __awaiter(void 0, void 0, void 0, function* () {
var _a;
return (Object.assign(Object.assign({}, (yield s3c.getObject({
Bucket,
Key: typeof o === 'string'
? o
: (_a = o.Key) !== null && _a !== void 0 ? _a : ''
}).promise())), o));
})));
exports.loadObjectList = loadObjectList;
const s3urlToConfigWfilters = (filter, i, a) => {
const s3Cfg = exports.s3urlToConfig(filter);
const prefix = s3Cfg.Key.includes('*') ? s3Cfg.Key.split('*')[0] : s3Cfg.Key;
const suffix = s3Cfg.Key.includes('*') ? s3Cfg.Key.slice(prefix.length) : '';
return Object.assign(Object.assign({}, s3Cfg), { prefix, suffix });
};
exports.s3urlToConfigWfilters = s3urlToConfigWfilters;