@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
209 lines (208 loc) • 11.4 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HttpApiSourceDriver = void 0;
const Affirm_1 = __importDefault(require("../core/Affirm"));
const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
const Algo_1 = __importDefault(require("../core/Algo"));
const Logger_1 = __importDefault(require("../helper/Logger"));
const DriverHelper_1 = __importDefault(require("./DriverHelper"));
class HttpApiSourceDriver {
constructor() {
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(source, 'Invalid source');
(0, Affirm_1.default)(source.authentication, 'Invalid authentication');
(0, Affirm_1.default)(source.authentication.url, 'HTTP API source requires a URL in authentication.url');
this._source = source;
this._baseUrl = SecretManager_1.default.replaceSecret(source.authentication.url);
this._httpMethod = source.authentication.httpMethod || 'GET';
this._timeout = source.authentication.timeout || 30000; // 30 seconds default
this._headers = source.authentication.headers ? Object.assign({}, source.authentication.headers) : {};
this._queryParams = source.authentication.queryParams ? Object.assign({}, source.authentication.queryParams) : {};
// Handle different authentication methods
switch (source.authentication.method) {
case 'bearer-token': {
(0, Affirm_1.default)(source.authentication.bearerToken, 'Bearer token authentication requires bearerToken');
this._headers['Authorization'] = `Bearer ${SecretManager_1.default.replaceSecret(source.authentication.bearerToken)}`;
break;
}
case 'api-key': {
(0, Affirm_1.default)(source.authentication.apiKey, 'API key authentication requires apiKey');
const apiKeyHeader = source.authentication.apiKeyHeader || 'X-API-Key';
this._headers[apiKeyHeader] = SecretManager_1.default.replaceSecret(source.authentication.apiKey);
break;
}
case 'username-password': {
(0, Affirm_1.default)(source.authentication.user && source.authentication.password, 'Username-password authentication requires user and password');
const credentials = Buffer.from(`${SecretManager_1.default.replaceSecret(source.authentication.user)}:${SecretManager_1.default.replaceSecret(source.authentication.password)}`).toString('base64');
this._headers['Authorization'] = `Basic ${credentials}`;
break;
}
case 'none':
// No authentication required
break;
default:
throw new Error(`Authentication method "${source.authentication.method}" is not supported for HTTP API sources`);
}
// Test connection
try {
yield this._makeRequest(this._baseUrl);
Logger_1.default.log(`HTTP API connection to ${this._baseUrl} successful`);
}
catch (error) {
throw new Error(`Failed to connect to HTTP API at ${this._baseUrl}: ${error.message}`);
}
return this;
});
this._makeRequest = (url, options) => __awaiter(this, void 0, void 0, function* () {
const method = (options === null || options === void 0 ? void 0 : options.method) || this._httpMethod;
const headers = Object.assign(Object.assign({}, this._headers), options === null || options === void 0 ? void 0 : options.additionalHeaders);
const queryParams = Object.assign(Object.assign({}, this._queryParams), options === null || options === void 0 ? void 0 : options.additionalQueryParams);
// Build URL with query parameters
const urlWithParams = new URL(url);
Object.entries(queryParams).forEach(([key, value]) => {
urlWithParams.searchParams.append(key, value);
});
const fetchOptions = {
method,
headers,
signal: AbortSignal.timeout(this._timeout)
};
if ((options === null || options === void 0 ? void 0 : options.body) && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
fetchOptions.body = typeof options.body === 'string'
? options.body
: JSON.stringify(options.body);
if (!headers['Content-Type']) {
headers['Content-Type'] = 'application/json';
}
}
const response = yield fetch(urlWithParams.toString(), fetchOptions);
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const contentType = response.headers.get('content-type');
if (contentType === null || contentType === void 0 ? void 0 : contentType.includes('application/json')) {
return yield response.json();
}
else {
return yield response.text();
}
});
this.execute = (_sql) => __awaiter(this, void 0, void 0, function* () {
void _sql;
throw new Error('SQL execution is not supported for HTTP API sources. Use query() or readAll() instead.');
});
this.query = (_sql, _values) => __awaiter(this, void 0, void 0, function* () {
void _sql;
void _values;
throw new Error('SQL queries are not supported for HTTP API sources. Use readAll() to fetch data from the API.');
});
this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
try {
const endpoint = producer.settings.fileKey || '';
const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
yield this._makeRequest(url, { method: 'HEAD' });
return true;
}
catch (error) {
if (error.message.includes('404')) {
return false;
}
throw error;
}
});
this.readAll = (request, values) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(request, 'Invalid read request');
(0, Affirm_1.default)(request.fileKey, 'Invalid file key (endpoint path)');
const endpoint = request.fileKey;
const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
// Convert IQueryParameter[] to query params if provided
const additionalQueryParams = {};
if (values && values.length > 0) {
values.forEach(param => {
additionalQueryParams[param.name] = param.value;
});
}
const data = yield this._makeRequest(url, { additionalQueryParams });
// Convert response to string array (lines)
return this._extractObjectsFromResponse(data, request.httpApi).map(x => JSON.stringify(x));
});
this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(request, 'Invalid read request');
(0, Affirm_1.default)(request.options, 'Invalid read request options');
const allLines = yield this.readAll(request);
const { lineFrom, lineTo } = request.options;
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
return allLines.slice(lineFrom, lineTo);
}
return allLines;
});
this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(dataset, 'Invalid dataset');
const file = dataset.getFile();
(0, Affirm_1.default)(file, 'Invalid dataset file');
(0, Affirm_1.default)(file.fileKey, 'Invalid file key (endpoint path)');
const endpoint = file.fileKey;
const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
const data = yield this._makeRequest(url);
const apiObjects = this._extractObjectsFromResponse(data, file.httpApi);
dataset.setFirstLine(JSON.stringify(apiObjects[0]));
const totalLineCount = yield DriverHelper_1.default.appendObjectsToUnifiedFile({
append: true,
delimiter: dataset.getDelimiter(),
destinationPath: dataset.getPath(),
objects: apiObjects
});
dataset.setCount(totalLineCount);
return dataset;
});
this._extractObjectsFromResponse = (data, httpApi) => {
let itemsData = [];
if (httpApi && httpApi.dataProperty && httpApi.dataProperty.length > 0) {
itemsData = data[httpApi.dataProperty];
}
else {
if (typeof data === 'string') {
itemsData = data.split('\n').filter(line => line.trim().length > 0);
}
else if (Array.isArray(data)) {
itemsData = data;
}
else if (typeof data === 'object' && data !== null) {
const dataObj = data;
if (dataObj.data && Array.isArray(dataObj.data)) {
itemsData = dataObj.data;
}
else if (dataObj.results && Array.isArray(dataObj.results)) {
itemsData = dataObj.results;
}
else if (dataObj.items && Array.isArray(dataObj.items)) {
itemsData = dataObj.items;
}
else {
// Single object, return as single line
itemsData = [data];
}
}
}
return itemsData;
};
this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
void request;
throw new Error('Not implemented yet');
});
}
}
exports.HttpApiSourceDriver = HttpApiSourceDriver;
exports.default = HttpApiSourceDriver;