@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
258 lines (257 loc) • 13.7 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.LocalDestinationDriver = exports.LocalSourceDriver = void 0;
const fs = __importStar(require("fs"));
const path_1 = __importDefault(require("path"));
const readline_1 = __importDefault(require("readline"));
const Affirm_1 = __importDefault(require("../core/Affirm"));
const Algo_1 = __importDefault(require("../core/Algo"));
const xlsx_1 = __importDefault(require("xlsx"));
const XMLParser_1 = __importDefault(require("../engines/parsing/XMLParser")); // Added XMLParser import
class LocalSourceDriver {
constructor() {
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
const fileURL = source.authentication['path'];
(0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
const exist = fs.existsSync(fileURL);
(0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
this._path = source.authentication['path'];
return this;
});
this.download = (request) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(this._path, `Invalid path`);
(0, Affirm_1.default)(request, `Invalid download request`);
(0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
(0, Affirm_1.default)(request.fileType, `Invalid file type for download request`);
const { fileKey, options } = request;
const fileUrl = path_1.default.join(this._path, fileKey);
switch (request.fileType) {
case 'CSV':
case 'JSON':
case 'JSONL':
case 'TXT':
return yield this._readLines(fileUrl);
case 'XLS':
case 'XLSX':
return yield this._readExcelLines(fileUrl, options === null || options === void 0 ? void 0 : options.sheetName);
case 'XML':
return yield this._readXmlLines(fileUrl);
}
});
this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(this._path, `Invalid path`);
(0, Affirm_1.default)(request, 'Invalid read options');
(0, Affirm_1.default)(request.fileKey, 'Invalid file key');
(0, Affirm_1.default)(request.fileType, `Invalid file type`);
(0, Affirm_1.default)(request.options, `Invalid request options`);
Affirm_1.default.hasValue(request.options.lineFrom, `Invalid request options line from`);
Affirm_1.default.hasValue(request.options.lineTo, `Invalid request options line to`);
const { fileKey, fileType, options: { lineFrom, lineTo, sheetName } } = request;
const fileUrl = path_1.default.join(this._path, fileKey);
switch (fileType) {
case 'CSV':
case 'JSON':
case 'JSONL':
case 'TXT':
return yield this._readLines(fileUrl, lineFrom, lineTo);
case 'XLS':
case 'XLSX':
return yield this._readExcelLines(fileUrl, sheetName, lineFrom, lineTo);
case 'XML':
return yield this._readXmlLines(fileUrl, lineFrom, lineTo);
}
});
this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(this._path, `Invalid path`);
(0, Affirm_1.default)(producer, `Invalid producer`);
const fileKey = producer.settings.fileKey;
(0, Affirm_1.default)(fileKey, `Invalid file key for download request`);
const fileUrl = path_1.default.join(this._path, fileKey);
return fs.existsSync(fileUrl);
});
this._readLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
var _a, e_1, _b, _c;
const stream = fs.createReadStream(fileUri);
const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
const lines = [];
let lineCounter = 0;
try {
for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
_c = reader_1_1.value;
_d = false;
const line = _c;
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
if (lineCounter >= lineFrom && lineCounter < lineTo) {
lines.push(line);
}
lineCounter++;
if (lineCounter >= lineTo)
break;
}
else {
lines.push(line);
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
}
finally { if (e_1) throw e_1.error; }
}
reader.close();
stream.close();
return lines;
});
this._readExcelLines = (fileUri, sheetName, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
const excel = xlsx_1.default.readFile(fileUri);
let targetSheetName = sheetName;
if (!targetSheetName) {
(0, Affirm_1.default)(excel.SheetNames.length > 0, 'The Excel file has no sheets.');
targetSheetName = excel.SheetNames[0];
}
else {
(0, Affirm_1.default)(excel.SheetNames.includes(targetSheetName), `The sheet "${targetSheetName}" doesn't exist in the excel (available: ${excel.SheetNames.join(', ')})`);
}
const sheet = excel.Sheets[targetSheetName];
const csv = xlsx_1.default.utils.sheet_to_csv(sheet);
const lines = csv.split('\n');
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo))
return lines.slice(lineFrom, lineTo + 1);
else
return lines;
});
this._readXmlLines = (fileUri, lineFrom, lineTo) => __awaiter(this, void 0, void 0, function* () {
const fileContent = fs.readFileSync(fileUri, 'utf-8');
const jsonData = XMLParser_1.default.xmlToJson(fileContent);
// Convert JSON data to string lines. This might need adjustment based on XML structure.
// Assuming jsonData is an array of objects, where each object is a record.
let lines = Array.isArray(jsonData) ? jsonData.map(item => JSON.stringify(item)) : [JSON.stringify(jsonData)];
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
lines = lines.slice(lineFrom, lineTo + 1);
}
return lines;
});
}
}
exports.LocalSourceDriver = LocalSourceDriver;
class LocalDestinationDriver {
constructor() {
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(source, `Invalid source`);
const fileURL = source.authentication['path'];
(0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
const exist = fs.existsSync(fileURL);
(0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
this._path = source.authentication['path'];
return this;
});
this.uploadFile = (options) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(this._path, 'Path not initialized');
(0, Affirm_1.default)(options, 'Invalid upload options');
(0, Affirm_1.default)(options.name, 'File name is required');
(0, Affirm_1.default)(options.content != null, 'File content is required');
const folder = this._path;
try {
if (!fs.existsSync(folder))
fs.mkdirSync(folder, { recursive: true });
const filePath = path_1.default.join(folder, options.name);
fs.writeFileSync(filePath, options.content);
return { bucket: folder, key: filePath, res: true };
}
catch (error) {
throw new Error(`Failed to upload local file "${options.name}": ${error.message}`);
}
});
this.multipartUpload = (options) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(this._path, 'Path not initialized');
(0, Affirm_1.default)(options, 'Invalid upload options');
(0, Affirm_1.default)(options.name, 'File name is required');
(0, Affirm_1.default)(options.contents && Array.isArray(options.contents), 'Contents must be an array');
(0, Affirm_1.default)(options.contents.length > 0, 'Contents array cannot be empty');
const folder = this._path;
try {
if (!fs.existsSync(folder)) {
fs.mkdirSync(folder, { recursive: true });
}
const filePath = path_1.default.join(folder, options.name);
// Create or truncate the file first
fs.writeFileSync(filePath, '');
// Append each chunk
for (const chunk of options.contents) {
(0, Affirm_1.default)(typeof chunk === 'string', 'Each chunk must be a string');
fs.appendFileSync(filePath, chunk);
}
return { bucket: folder, key: filePath, res: true };
}
catch (error) {
// Clean up the partial file if it exists
const filePath = path_1.default.join(folder, options.name);
if (fs.existsSync(filePath)) {
try {
fs.unlinkSync(filePath);
}
catch (cleanupError) {
console.error(`Failed to clean up partial file after error: ${cleanupError.message}`);
throw cleanupError;
}
}
throw new Error(`Failed to complete local multipart upload for "${options.name}": ${error.message}`);
}
});
}
}
exports.LocalDestinationDriver = LocalDestinationDriver;