@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
294 lines (293 loc) • 15.5 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.S3SourceDriver = exports.S3DestinationDriver = void 0;
const client_s3_1 = require("@aws-sdk/client-s3");
const Affirm_1 = __importDefault(require("../core/Affirm"));
const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
const readline_1 = __importDefault(require("readline"));
const Algo_1 = __importDefault(require("../core/Algo"));
const xlsx_1 = __importDefault(require("xlsx"));
const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
class S3DestinationDriver {
constructor() {
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
this._bucketName = source.authentication['bucket'];
const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
const config = {
region: source.authentication['region'],
credentials: {
accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
sessionToken: sessionToken ? sessionToken : undefined
}
};
this._client = new client_s3_1.S3Client(config);
// TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
return this;
});
this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(options, `Invalid upload options`);
const { content, name } = options;
const commandParams = {
Bucket: this._bucketName,
Key: name,
Body: content
};
const command = new client_s3_1.PutObjectCommand(commandParams);
const res = yield this._client.send(command);
(0, Affirm_1.default)(res.$metadata.httpStatusCode === 200, `Failed to upload the file "${name}" to the bucket "${this._bucketName}": status code ${res.$metadata.httpStatusCode}`);
return { res: true, key: name, bucket: this._bucketName };
});
this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(options, `Invalid upload options`);
(0, Affirm_1.default)(options.contents && options.contents.length > 0, 'No contents provided for multipart upload');
(0, Affirm_1.default)(options.name, 'No filename provided for multipart upload');
try {
// Create the multipart upload
const createMultipartUploadRes = yield this._client.send(new client_s3_1.CreateMultipartUploadCommand({
Bucket: this._bucketName,
Key: options.name
}));
const uploadId = createMultipartUploadRes.UploadId;
(0, Affirm_1.default)(uploadId, 'Failed to initiate multipart upload');
// Upload each part
const uploadPromises = options.contents.map((content, index) => __awaiter(this, void 0, void 0, function* () {
const partNumber = index + 1;
const uploadPartRes = yield this._client.send(new client_s3_1.UploadPartCommand({
Bucket: this._bucketName,
Key: options.name,
UploadId: uploadId,
PartNumber: partNumber,
Body: Buffer.from(content)
}));
return {
PartNumber: partNumber,
ETag: uploadPartRes.ETag
};
}));
const uploadedParts = yield Promise.all(uploadPromises);
// Complete the multipart upload
const completeRes = yield this._client.send(new client_s3_1.CompleteMultipartUploadCommand({
Bucket: this._bucketName,
Key: options.name,
UploadId: uploadId,
MultipartUpload: {
Parts: uploadedParts
}
}));
(0, Affirm_1.default)(completeRes.$metadata.httpStatusCode === 200, `Failed to complete multipart upload for "${options.name}": status code ${completeRes.$metadata.httpStatusCode}`);
return { res: true, key: options.name, bucket: this._bucketName };
}
catch (error) {
// If anything fails, make sure to abort the multipart upload
if (error.UploadId) {
yield this._client.send(new client_s3_1.AbortMultipartUploadCommand({
Bucket: this._bucketName,
Key: options.name,
UploadId: error.UploadId
}));
}
throw error;
}
});
}
}
exports.S3DestinationDriver = S3DestinationDriver;
class S3SourceDriver {
constructor() {
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
this._bucketName = source.authentication['bucket'];
const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
const config = {
region: source.authentication['region'],
credentials: {
accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
sessionToken: sessionToken ? sessionToken : undefined
}
};
this._client = new client_s3_1.S3Client(config);
// TODO: is there a way to test if the connection was successful? like a query or scan that I can do?
return this;
});
this.download = (request) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
(0, Affirm_1.default)(request, `Invalid download request`);
(0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
const { fileKey, fileType, options } = request;
const bucket = this._bucketName;
const response = yield this._client.send(new client_s3_1.GetObjectCommand({
Bucket: bucket,
Key: fileKey
}));
(0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
const stream = response.Body;
switch (fileType) {
case 'CSV':
case 'JSON':
case 'JSONL':
case 'TXT':
return yield this._readLines(stream);
case 'XLS':
case 'XLSX':
return yield this._readExcelLines(stream, options === null || options === void 0 ? void 0 : options.sheetName);
case 'XML':
return yield this._readXmlLines(stream);
}
});
this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
(0, Affirm_1.default)(request, 'Invalid read request');
(0, Affirm_1.default)(request.options, 'Invalid read request options');
const { fileKey, fileType, options: { sheetName, lineFrom, lineTo } } = request;
const bucket = this._bucketName;
const response = yield this._client.send(new client_s3_1.GetObjectCommand({
Bucket: bucket,
Key: fileKey
}));
(0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
const stream = response.Body;
switch (fileType) {
case 'CSV':
case 'JSON':
case 'JSONL':
case 'TXT':
return yield this._readLines(stream, lineFrom, lineTo);
case 'XLS':
case 'XLSX':
return yield this._readExcelLines(stream, sheetName, lineFrom, lineTo);
case 'XML':
return yield this._readXmlLines(stream, lineFrom, lineTo);
}
});
this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
var _a;
(0, Affirm_1.default)(this._client, 'S3 client not yet initialized, call "connect()" first');
(0, Affirm_1.default)(producer, 'Invalid read producer');
const bucket = this._bucketName;
const fileKey = producer.settings.fileKey;
(0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
try {
yield this._client.send(new client_s3_1.HeadObjectCommand({ Bucket: bucket, Key: fileKey }));
return true;
}
catch (error) {
if (((_a = error.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode) === 404 || error.name === 'NotFound')
return false;
throw error;
}
});
this._readLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
var _a, e_1, _b, _c;
const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
const lines = [];
let lineCounter = 0;
try {
for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
_c = reader_1_1.value;
_d = false;
const line = _c;
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
if (lineCounter >= lineFrom && lineCounter < lineTo) {
lines.push(line);
}
lineCounter++;
if (lineCounter >= lineTo)
break;
}
else {
lines.push(line);
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
}
finally { if (e_1) throw e_1.error; }
}
reader.close();
return lines;
});
this._readExcelLines = (stream, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
var _a, stream_1, stream_1_1;
var _b, e_2, _c, _d;
(0, Affirm_1.default)(sheetName, `Invalid sheetname`);
const chunks = [];
try {
for (_a = true, stream_1 = __asyncValues(stream); stream_1_1 = yield stream_1.next(), _b = stream_1_1.done, !_b; _a = true) {
_d = stream_1_1.value;
_a = false;
const chunk = _d;
chunks.push(chunk);
}
}
catch (e_2_1) { e_2 = { error: e_2_1 }; }
finally {
try {
if (!_a && !_b && (_c = stream_1.return)) yield _c.call(stream_1);
}
finally { if (e_2) throw e_2.error; }
}
const buffer = Buffer.concat(chunks);
const excel = xlsx_1.default.read(buffer, { type: 'buffer' });
(0, Affirm_1.default)(excel.SheetNames.includes(sheetName), `The sheet "${sheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
const sheet = excel.Sheets[sheetName];
const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
const lines = csv.split('\n');
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
return lines.slice(lineFrom, lineTo + 1);
else
return lines;
});
this._readXmlLines = (stream, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
var _a, stream_2, stream_2_1;
var _b, e_3, _c, _d;
const chunks = [];
try {
for (_a = true, stream_2 = __asyncValues(stream); stream_2_1 = yield stream_2.next(), _b = stream_2_1.done, !_b; _a = true) {
_d = stream_2_1.value;
_a = false;
const chunk = _d;
chunks.push(chunk);
}
}
catch (e_3_1) { e_3 = { error: e_3_1 }; }
finally {
try {
if (!_a && !_b && (_c = stream_2.return)) yield _c.call(stream_2);
}
finally { if (e_3) throw e_3.error; }
}
const buffer = Buffer.concat(chunks);
const jsonData = XMLParser_1.default.xmlToJson(buffer);
// Convert JSON data to string lines. This might need adjustment based on XML structure.
let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
lines = lines.slice(lineFrom, lineTo + 1);
}
return lines;
});
}
}
exports.S3SourceDriver = S3SourceDriver;