@tableau/taco-toolkit
Version:
Tableau Connector Toolkit
216 lines • 16.9 kB
JavaScript
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createTacoFileParser = exports.FileBasedPipeline = void 0;
var taco_file_parser_1 = require("../../../shared/enums/taco-file-parser");
var eps_logger_1 = require("../../logging/eps-logger");
var handler_resolver_1 = require("./handler-resolver");
var sandbox_pipeline_1 = require("./sandbox-pipeline");
var system_utils_1 = require("../../utils/system-utils");
var csv_parser_1 = require("../handlers/taco-parser/csv-parser");
var excel_parser_1 = require("../handlers/taco-parser/excel-parser");
var fetch_utils_1 = require("../../sandbox/sdk/fetcher-utils/fetch-utils");
var FileBasedPipeline = /** @class */ (function (_super) {
__extends(FileBasedPipeline, _super);
function FileBasedPipeline(handlerInput, extractorId, extractorOptions, tempDataManager, networkAdapter) {
var _this = _super.call(this, extractorOptions, extractorId, tempDataManager, networkAdapter) || this;
_this.handlerInput = handlerInput;
_this.handlerConfig = _this.resolveHandlers();
new fetch_utils_1.FetchUtilsPlugin({
extractorId: extractorId,
handlerInput: handlerInput,
permission: extractorOptions.permission,
}, networkAdapter, tempDataManager).registerClass(_this.sandboxVM);
return _this;
}
FileBasedPipeline.prototype.run = function () {
return __awaiter(this, void 0, void 0, function () {
var authResult;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.loadHandlers()];
case 1:
_a.sent();
return [4 /*yield*/, this.runAuthPhase(this.handlerConfig, this.handlerInput)];
case 2:
authResult = _a.sent();
return [4 /*yield*/, this.runDataGatheringPhase(authResult)];
case 3:
_a.sent();
return [2 /*return*/, this.processFileBasedParser()];
}
});
});
};
FileBasedPipeline.prototype.runDataGatheringPhase = function (authResult) {
return __awaiter(this, void 0, void 0, function () {
var fetcher, fetcherClassName, codeText;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
fetcher = this.handlerConfig.fetcher;
fetcherClassName = fetcher.className;
codeText = getCodeText(fetcherClassName);
return [4 /*yield*/, this.executePipelineCode(codeText, [
this.handlerInput,
{
authResult: authResult,
secrets: this.extractorOptions.secrets,
},
])];
case 1:
_a.sent();
eps_logger_1.Logger.info("Returned from pipeline after data ingestion.");
return [2 /*return*/];
}
});
});
};
FileBasedPipeline.prototype.resolveHandlers = function () {
var handlerConfig = {};
var handlersDir = this.extractorOptions.handlersDir;
var _a = this.handlerInput, fetcherName = _a.fetcher, parserName = _a.parser, authName = _a.auth;
handlerConfig.auth = (0, handler_resolver_1.resolveHandlerFile)({
required: false,
handlerName: authName,
handlerType: 'auth',
handlersDirPath: handlersDir,
});
handlerConfig.fetcher = (0, handler_resolver_1.resolveHandlerFile)({
required: true,
handlerName: fetcherName,
handlerType: 'fetcher',
handlersDirPath: handlersDir,
});
handlerConfig.parser = parserName;
return handlerConfig;
};
/**
* Register parser for filed-based handler input
* @param handlerInputs
*/
FileBasedPipeline.prototype.processFileBasedParser = function () {
var _a = this.handlerInput, parserName = _a.parser, name = _a.name;
// Excel is currently unique that it is one file to many tables
if (parserName === taco_file_parser_1.TacoFileParser.EXCEL) {
var excelTacoParser = new excel_parser_1.ExcelTacoParser(this.tempDataManager, this.extractorId, name);
return {
type: 'excel',
parser: excelTacoParser,
};
}
var parser = createTacoFileParser(name, parserName, this.extractorId, this.tempDataManager);
return {
type: 'single-table',
tableName: name,
parser: parser,
};
};
FileBasedPipeline.prototype.loadHandlers = function () {
return __awaiter(this, void 0, void 0, function () {
var _a, auth, fetcher;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
_a = this.handlerConfig, auth = _a.auth, fetcher = _a.fetcher;
if (!auth) return [3 /*break*/, 2];
return [4 /*yield*/, this.sandboxVM.importModule(auth.path)];
case 1:
_b.sent();
_b.label = 2;
case 2: return [4 /*yield*/, this.sandboxVM.importModule(fetcher.path)];
case 3:
_b.sent();
return [2 /*return*/];
}
});
});
};
return FileBasedPipeline;
}(sandbox_pipeline_1.SandboxPipeline));
exports.FileBasedPipeline = FileBasedPipeline;
function createTacoFileParser(tableName, parser, extractorId, tempDataManager) {
switch (parser) {
case taco_file_parser_1.TacoFileParser.CSV:
case taco_file_parser_1.TacoFileParser.DATA: {
if ((0, system_utils_1.isCentOS)()) {
return new csv_parser_1.CsvTacoParser(tempDataManager, extractorId, tableName);
}
else {
var CsvDuckdbTacoParser =
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('../handlers/taco-parser/csv-duckdb-parser').CsvDuckdbTacoParser;
return new CsvDuckdbTacoParser(tempDataManager, extractorId, tableName);
}
}
case taco_file_parser_1.TacoFileParser.PARQUET: {
if ((0, system_utils_1.isCentOS)()) {
throw new Error("".concat(parser, " is not supported on Cent OS."));
}
var ParquetTacoParser =
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('../handlers/taco-parser/parquet-parser').ParquetTacoParser;
return new ParquetTacoParser(tempDataManager, extractorId, tableName);
}
default: {
throw new Error("Unknown parse name: ".concat(parser));
}
}
}
exports.createTacoFileParser = createTacoFileParser;
function getCodeText(fetcherClassName) {
// If mode: large-data and has a built-in-parser than skip running the parser step
// in the sandbox pipeline
return "\n const fetcher = new globalRefs[\"".concat(fetcherClassName, "\"]();\n const handlerInput = $0.copySync();\n const context = $1.copySync();\n\n return (async () => {\n try {\n const { authResult, secrets } = context;\n\n const fetchIter = fetcher.fetch({ authResult, handlerInput, secrets });\n if (typeof fetchIter?.next !== 'function') {\n throw new Error(\"").concat(fetcherClassName, ".fetch is not a generator function.\");\n }\n\n while (true) {\n let fetchResult;\n try {\n fetchResult = await fetchIter.next();\n } catch (error) {\n error.message = '").concat(fetcherClassName, ".fetch failed. ' + error.message;\n throw error;\n }\n\n const { value, done } = fetchResult;\n if (done) {\n break;\n }\n }\n }\n catch (error) {\n error.message = 'Failure in sandbox pipeline. ' + error.message;\n throw error;\n }\n })();\n ");
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmlsZS1iYXNlZC1waXBlbGluZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3NyYy9lcHMvZXh0cmFjdG9yL3BpcGVsaW5lcy9maWxlLWJhc2VkLXBpcGVsaW5lLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztBQUFBLDJFQUEyRjtBQUczRix1REFBaUQ7QUFJakQsdURBQXdFO0FBQ3hFLHVEQUFvRDtBQUVwRCx5REFBbUQ7QUFFbkQsaUVBQWtFO0FBRWxFLHFFQUFzRTtBQUN0RSwyRUFBOEU7QUFtQjlFO0lBQXVDLHFDQUFlO0lBR3BELDJCQUNtQixZQUE0QyxFQUM3RCxXQUFtQixFQUNuQixnQkFBa0MsRUFDbEMsZUFBZ0MsRUFDaEMsY0FBOEI7UUFMaEMsWUFPRSxrQkFBTSxnQkFBZ0IsRUFBRSxXQUFXLEVBQUUsZUFBZSxFQUFFLGNBQWMsQ0FBQyxTQWF0RTtRQW5Ca0Isa0JBQVksR0FBWixZQUFZLENBQWdDO1FBUTdELEtBQUksQ0FBQyxhQUFhLEdBQUcsS0FBSSxDQUFDLGVBQWUsRUFBRSxDQUFBO1FBRTNDLElBQUksOEJBQWdCLENBQ2xCO1lBQ0UsV0FBVyxhQUFBO1lBQ1gsWUFBWSxjQUFBO1lBQ1osVUFBVSxFQUFFLGdCQUFnQixDQUFDLFVBQVU7U0FDeEMsRUFDRCxjQUFjLEVBQ2QsZUFBZSxDQUNoQixDQUFDLGFBQWEsQ0FBQyxLQUFJLENBQUMsU0FBUyxDQUFDLENBQUE7O0lBQ2pDLENBQUM7SUFFSywrQkFBRyxHQUFUOzs7Ozs0QkFDRSxxQkFBTSxJQUFJLENBQUMsWUFBWSxFQUFFLEVBQUE7O3dCQUF6QixTQUF5QixDQUFBO3dCQUNOLHFCQUFNLElBQUksQ0FBQyxZQUFZLENBQUMsSUFBSSxDQUFDLGFBQWEsRUFBRSxJQUFJLENBQUMsWUFBWSxDQUFDLEVBQUE7O3dCQUEzRSxVQUFVLEdBQUcsU0FBOEQ7d0JBQ2pGLHFCQUFNLElBQUksQ0FBQyxxQkFBcUIsQ0FBQyxVQUFVLENBQUMsRUFBQTs7d0JBQTVDLFNBQTRDLENBQUE7d0JBRTVDLHNCQUFPLElBQUksQ0FBQyxzQkFBc0IsRUFBRSxFQUFBOzs7O0tBQ3JDO0lBRWUsaURBQXFCLEdBQXJDLFVBQXNDLFVBQXNCOzs7Ozs7d0JBQ2xELE9BQU8sR0FBSyxJQUFJLENBQUMsYUFBYSxRQUF2QixDQUF1Qjt3QkFDaEMsZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLFNBQVMsQ0FBQTt3QkFFcEMsUUFBUSxHQUFHLFdBQVcsQ0FBQyxnQkFBZ0IsQ0FBQyxDQUFBO3dCQUM5QyxxQkFBTSxJQUFJLENBQUMsbUJBQW1CLENBQUMsUUFBUSxFQUFFO2dDQUN2QyxJQUFJLENBQUMsWUFBWTtnQ0FDakI7b0NBQ0UsVUFBVSxZQUFBO29DQUNWLE9BQU8sRUFBRSxJQUFJLENBQUMsZ0JBQWdCLENBQUMsT0FBTztpQ0FDdkM7NkJBQ0YsQ0FBQyxFQUFBOzt3QkFORixTQU1FLENBQUE7d0JBRUYsbUJBQU0sQ0FBQyxJQUFJLENBQUMsOENBQThDLENBQUMsQ0FBQTs7Ozs7S0FDNUQ7SUFFUywyQ0FBZSxHQUF6QjtRQUNFLElBQU0sYUFBYSxHQUFvQyxFQUFFLENBQUE7UUFDakQsSUFBQSxXQUFXLEdBQUssSUFBSSxDQUFDLGdCQUFnQixZQUExQixDQUEwQjtRQUV2QyxJQUFBLEtBQStELElBQUksQ0FBQyxZQUFZLEVBQXJFLFdBQVcsYUFBQSxFQUFVLFVBQVUsWUFBQSxFQUFRLFFBQVEsVUFBc0IsQ0FBQTtRQUV0RixhQUFhLENBQUMsSUFBSSxHQUFHLElBQUEscUNBQWtCLEVBQUM7WUFDdEMsUUFBUSxFQUFFLEtBQUs7WUFDZixXQUFXLEVBQUUsUUFBUTtZQUNyQixXQUFXLEVBQUUsTUFBTTtZQUNuQixlQUFlLEVBQUUsV0FBVztTQUM3QixDQUFDLENBQUE7UUFFRixhQUFhLENBQUMsT0FBTyxHQUFHLElBQUEscUNBQWtCLEVBQUM7WUFDekMsUUFBUSxFQUFFLElBQUk7WUFDZCxXQUFXLEVBQUUsV0FBVztZQUN4QixXQUFXLEVBQUUsU0FBUztZQUN0QixlQUFlLEVBQUUsV0FBVztTQUM3QixDQUFDLENBQUE7UUFFRixhQUFhLENBQUMsTUFBTSxHQUFHLFVBQVUsQ0FBQTtRQUVqQyxPQUFPLGFBQXVDLENBQUE7SUFDaEQsQ0FBQztJQUVEOzs7T0FHRztJQUNLLGtEQUFzQixHQUE5QjtRQUNRLElBQUEsS0FBK0IsSUFBSSxDQUFDLFlBQVksRUFBdEMsVUFBVSxZQUFBLEVBQUUsSUFBSSxVQUFzQixDQUFBO1FBRXRELCtEQUErRDtRQUMvRCxJQUFJLFVBQVUsS0FBSyxpQ0FBYyxDQUFDLEtBQUssRUFBRTtZQUN2QyxJQUFNLGVBQWUsR0FBRyxJQUFJLDhCQUFlLENBQUMsSUFBSSxDQUFDLGVBQWUsRUFBRSxJQUFJLENBQUMsV0FBVyxFQUFFLElBQUksQ0FBQyxDQUFBO1lBRXpGLE9BQU87Z0JBQ0wsSUFBSSxFQUFFLE9BQU87Z0JBQ2IsTUFBTSxFQUFFLGVBQWU7YUFDeEIsQ0FBQTtTQUNGO1FBRUQsSUFBTSxNQUFNLEdBQUcsb0JBQW9CLENBQUMsSUFBSSxFQUFFLFVBQVUsRUFBRSxJQUFJLENBQUMsV0FBVyxFQUFFLElBQUksQ0FBQyxlQUFlLENBQUMsQ0FBQTtRQUU3RixPQUFPO1lBQ0wsSUFBSSxFQUFFLGNBQWM7WUFDcEIsU0FBUyxFQUFFLElBQUk7WUFDZixNQUFNLFFBQUE7U0FDUCxDQUFBO0lBQ0gsQ0FBQztJQUVlLHdDQUFZLEdBQTVCOzs7Ozs7d0JBQ1EsS0FBb0IsSUFBSSxDQUFDLGFBQWEsRUFBcEMsSUFBSSxVQUFBLEVBQUUsT0FBTyxhQUFBLENBQXVCOzZCQUV4QyxJQUFJLEVBQUosd0JBQUk7d0JBQ04scUJBQU0sSUFBSSxDQUFDLFNBQVMsQ0FBQyxZQUFZLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFBOzt3QkFBNUMsU0FBNEMsQ0FBQTs7NEJBRTlDLHFCQUFNLElBQUksQ0FBQyxTQUFTLENBQUMsWUFBWSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsRUFBQTs7d0JBQS9DLFNBQStDLENBQUE7Ozs7O0tBQ2hEO0lBQ0gsd0JBQUM7QUFBRCxDQUFDLEFBNUdELENBQXVDLGtDQUFlLEdBNEdyRDtBQTVHWSw4Q0FBaUI7QUE4RzlCLFNBQWdCLG9CQUFvQixDQUNsQyxTQUFpQixFQUNqQixNQUFjLEVBQ2QsV0FBbUIsRUFDbkIsZUFBZ0M7SUFFaEMsUUFBUSxNQUFNLEVBQUU7UUFDZCxLQUFLLGlDQUFjLENBQUMsR0FBRyxDQUFDO1FBQ3hCLEtBQUssaUNBQWMsQ0FBQyxJQUFJLENBQUMsQ0FBQztZQUN4QixJQUFJLElBQUEsdUJBQVEsR0FBRSxFQUFFO2dCQUNkLE9BQU8sSUFBSSwwQkFBYSxDQUFDLGVBQWUsRUFBRSxXQUFXLEVBQUUsU0FBUyxDQUFDLENBQUE7YUFDbEU7aUJBQU07Z0JBQ0wsSUFBTSxtQkFBbUI7Z0JBQ3ZCLDhEQUE4RDtnQkFDOUQsT0FBTyxDQUFDLDJDQUEyQyxDQUFDLENBQUMsbUJBQW1CLENBQUE7Z0JBQzFFLE9BQU8sSUFBSSxtQkFBbUIsQ0FBQyxlQUFlLEVBQUUsV0FBVyxFQUFFLFNBQVMsQ0FBQyxDQUFBO2FBQ3hFO1NBQ0Y7UUFDRCxLQUFLLGlDQUFjLENBQUMsT0FBTyxDQUFDLENBQUM7WUFDM0IsSUFBSSxJQUFBLHVCQUFRLEdBQUUsRUFBRTtnQkFDZCxNQUFNLElBQUksS0FBSyxDQUFDLFVBQUcsTUFBTSxrQ0FBK0IsQ0FBQyxDQUFBO2FBQzFEO1lBQ0QsSUFBTSxpQkFBaUI7WUFDckIsOERBQThEO1lBQzlELE9BQU8sQ0FBQyx3Q0FBd0MsQ0FBQyxDQUFDLGlCQUFpQixDQUFBO1lBQ3JFLE9BQU8sSUFBSSxpQkFBaUIsQ0FBQyxlQUFlLEVBQUUsV0FBVyxFQUFFLFNBQVMsQ0FBQyxDQUFBO1NBQ3RFO1FBQ0QsT0FBTyxDQUFDLENBQUM7WUFDUCxNQUFNLElBQUksS0FBSyxDQUFDLDhCQUF1QixNQUFNLENBQUUsQ0FBQyxDQUFBO1NBQ2pEO0tBQ0Y7QUFDSCxDQUFDO0FBL0JELG9EQStCQztBQUVELFNBQVMsV0FBVyxDQUFDLGdCQUF3QjtJQUMzQyxrRkFBa0Y7SUFDbEYsMEJBQTBCO0lBQzFCLE9BQU8scURBQ2lDLGdCQUFnQixtWUFVekIsZ0JBQWdCLGlSQVFkLGdCQUFnQixvWkFlMUMsQ0FBQTtBQUNULENBQUMifQ==