@tableau/taco-toolkit
Version:
Tableau Connector Toolkit
167 lines • 14.7 kB
JavaScript
;
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryTablePipeline = void 0;
var eps_logger_1 = require("../../logging/eps-logger");
var fetch_utils_1 = require("../../sandbox/sdk/fetcher-utils/fetch-utils");
var handler_resolver_1 = require("./handler-resolver");
var sandbox_pipeline_1 = require("./sandbox-pipeline");
var MemoryTablePipeline = /** @class */ (function (_super) {
__extends(MemoryTablePipeline, _super);
function MemoryTablePipeline(handlerInput, extractorId, extractorOptions, tempDataManager, networkAdapter) {
var _this = _super.call(this, extractorOptions, extractorId, tempDataManager, networkAdapter) || this;
_this.handlerInput = handlerInput;
_this.handlerConfig = _this.resolveHandlers();
new fetch_utils_1.FetchUtilsPlugin({
extractorId: extractorId,
handlerInput: handlerInput,
permission: extractorOptions.permission,
}, networkAdapter, tempDataManager).registerClass(_this.sandboxVM);
return _this;
}
MemoryTablePipeline.prototype.run = function () {
return __awaiter(this, void 0, void 0, function () {
var authResult;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.loadHandlers()];
case 1:
_a.sent();
return [4 /*yield*/, this.runAuthPhase(this.handlerConfig, this.handlerInput)];
case 2:
authResult = _a.sent();
return [4 /*yield*/, this.runDataGatheringPhase(authResult)];
case 3: return [2 /*return*/, _a.sent()];
}
});
});
};
MemoryTablePipeline.prototype.runDataGatheringPhase = function (authResult) {
return __awaiter(this, void 0, void 0, function () {
var _a, parser, fetcher, fetcherClassName, parserClassName, codeText, container;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
_a = this.handlerConfig, parser = _a.parser, fetcher = _a.fetcher;
fetcherClassName = fetcher.className;
parserClassName = parser.className;
codeText = getInMemoryCodeText(fetcherClassName, parserClassName);
return [4 /*yield*/, this.executePipelineCode(codeText, [
this.handlerInput,
{
authResult: authResult,
secrets: this.extractorOptions.secrets,
},
])];
case 1:
container = _b.sent();
eps_logger_1.Logger.info("Returned from pipeline with ".concat(container.tables.length, " tables"));
return [2 /*return*/, container];
}
});
});
};
MemoryTablePipeline.prototype.resolveHandlers = function () {
var handlerConfig = {};
var handlersDir = this.extractorOptions.handlersDir;
var _a = this.handlerInput, fetcherName = _a.fetcher, parserName = _a.parser, authName = _a.auth;
handlerConfig.auth = (0, handler_resolver_1.resolveHandlerFile)({
required: false,
handlerName: authName,
handlerType: 'auth',
handlersDirPath: handlersDir,
});
handlerConfig.fetcher = (0, handler_resolver_1.resolveHandlerFile)({
required: true,
handlerName: fetcherName,
handlerType: 'fetcher',
handlersDirPath: handlersDir,
});
handlerConfig.parser = (0, handler_resolver_1.resolveHandlerFile)({
required: true,
handlerName: parserName,
handlerType: 'parser',
handlersDirPath: handlersDir,
});
return handlerConfig;
};
MemoryTablePipeline.prototype.loadHandlers = function () {
return __awaiter(this, void 0, void 0, function () {
var _a, auth, fetcher, parser;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
_a = this.handlerConfig, auth = _a.auth, fetcher = _a.fetcher, parser = _a.parser;
if (!auth) return [3 /*break*/, 2];
return [4 /*yield*/, this.sandboxVM.importModule(auth.path)];
case 1:
_b.sent();
_b.label = 2;
case 2: return [4 /*yield*/, this.sandboxVM.importModule(fetcher.path)];
case 3:
_b.sent();
return [4 /*yield*/, this.sandboxVM.importModule(parser.path)];
case 4:
_b.sent();
return [2 /*return*/];
}
});
});
};
return MemoryTablePipeline;
}(sandbox_pipeline_1.SandboxPipeline));
exports.MemoryTablePipeline = MemoryTablePipeline;
function getInMemoryCodeText(fetcherClassName, parserClassName) {
return "\n const fetcher = new globalRefs[\"".concat(fetcherClassName, "\"]();\n const parser = new globalRefs[\"").concat(parserClassName, "\"]();\n\n const handlerInput = $0.copySync();\n const context = $1.copySync();\n\n return (async () => {\n try {\n const { authResult, secrets } = context;\n\n const fetchIter = fetcher.fetch({ authResult, handlerInput, secrets });\n if (typeof fetchIter?.next !== 'function') {\n throw new Error(\"").concat(fetcherClassName, ".fetch is not a generator function.\");\n }\n\n let dataContainer = createDataContainer();\n\n while (true) {\n let fetchResult;\n try {\n fetchResult = await fetchIter.next();\n } catch (error) {\n error.message = '").concat(fetcherClassName, ".fetch failed. ' + error.message;\n throw error;\n }\n\n const { value, done } = fetchResult;\n if (done) {\n break;\n }\n\n try {\n const parseOptions = {\n dataContainer,\n handlerInput,\n }\n\n if (parser instanceof AsyncParser) {\n dataContainer = await parser.parse(value, parseOptions);\n } else {\n dataContainer = parser.parse(value, parseOptions);\n }\n\n if (!dataContainer || !Array.isArray(dataContainer.tables)) {\n throw new Error(\"A proper dataContainer was not returned.\");\n }\n } catch (error) {\n error.message = '").concat(parserClassName, ".parse failed. ' + error.message;\n throw error;\n }\n\n log('pipeline table count after parser.run is ' + dataContainer.tables.length);\n\n // Next step: flush container's table's rows to disk.\n // store.run(handlerInput, context);\n // TODO: we should spike on how to achieve this ^\n\n }\n return dataContainer;\n }\n catch (error) {\n error.message = 'Failure in sandbox pipeline. ' + error.message;\n throw error;\n }\n })();\n ");
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWVtb3J5LXRhYmxlLXBpcGVsaW5lLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vLi4vc3JjL2Vwcy9leHRyYWN0b3IvcGlwZWxpbmVzL21lbW9yeS10YWJsZS1waXBlbGluZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7QUFHQSx1REFBaUQ7QUFHakQsMkVBQThFO0FBRTlFLHVEQUF3RTtBQUN4RSx1REFBb0Q7QUFRcEQ7SUFBeUMsdUNBQWU7SUFHdEQsNkJBQ21CLFlBQXdDLEVBQ3pELFdBQW1CLEVBQ25CLGdCQUFrQyxFQUNsQyxlQUFnQyxFQUNoQyxjQUE4QjtRQUxoQyxZQU9FLGtCQUFNLGdCQUFnQixFQUFFLFdBQVcsRUFBRSxlQUFlLEVBQUUsY0FBYyxDQUFDLFNBWXRFO1FBbEJrQixrQkFBWSxHQUFaLFlBQVksQ0FBNEI7UUFRekQsS0FBSSxDQUFDLGFBQWEsR0FBRyxLQUFJLENBQUMsZUFBZSxFQUFFLENBQUE7UUFDM0MsSUFBSSw4QkFBZ0IsQ0FDbEI7WUFDRSxXQUFXLGFBQUE7WUFDWCxZQUFZLGNBQUE7WUFDWixVQUFVLEVBQUUsZ0JBQWdCLENBQUMsVUFBVTtTQUN4QyxFQUNELGNBQWMsRUFDZCxlQUFlLENBQ2hCLENBQUMsYUFBYSxDQUFDLEtBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQTs7SUFDakMsQ0FBQztJQUVLLGlDQUFHLEdBQVQ7Ozs7OzRCQUNFLHFCQUFNLElBQUksQ0FBQyxZQUFZLEVBQUUsRUFBQTs7d0JBQXpCLFNBQXlCLENBQUE7d0JBQ04scUJBQU0sSUFBSSxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsYUFBYSxFQUFFLElBQUksQ0FBQyxZQUFZLENBQUMsRUFBQTs7d0JBQTNFLFVBQVUsR0FBRyxTQUE4RDt3QkFDMUUscUJBQU0sSUFBSSxDQUFDLHFCQUFxQixDQUFDLFVBQVUsQ0FBQyxFQUFBOzRCQUFuRCxzQkFBTyxTQUE0QyxFQUFBOzs7O0tBQ3BEO0lBRWEsbURBQXFCLEdBQW5DLFVBQW9DLFVBQXNCOzs7Ozs7d0JBQ2xELEtBQXNCLElBQUksQ0FBQyxhQUFhLEVBQXRDLE1BQU0sWUFBQSxFQUFFLE9BQU8sYUFBQSxDQUF1Qjt3QkFDeEMsZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLFNBQVMsQ0FBQTt3QkFFcEMsZUFBZSxHQUFJLE1BQTBCLENBQUMsU0FBUyxDQUFBO3dCQUV2RCxRQUFRLEdBQUcsbUJBQW1CLENBQUMsZ0JBQWdCLEVBQUUsZUFBZSxDQUFDLENBQUE7d0JBQ3RDLHFCQUFNLElBQUksQ0FBQyxtQkFBbUIsQ0FBQyxRQUFRLEVBQUU7Z0NBQ3hFLElBQUksQ0FBQyxZQUFZO2dDQUNqQjtvQ0FDRSxVQUFVLFlBQUE7b0NBQ1YsT0FBTyxFQUFFLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxPQUFPO2lDQUN2Qzs2QkFDRixDQUFDLEVBQUE7O3dCQU5JLFNBQVMsR0FBa0IsU0FNL0I7d0JBRUYsbUJBQU0sQ0FBQyxJQUFJLENBQUMsc0NBQStCLFNBQVMsQ0FBQyxNQUFNLENBQUMsTUFBTSxZQUFTLENBQUMsQ0FBQTt3QkFFNUUsc0JBQU8sU0FBUyxFQUFBOzs7O0tBQ2pCO0lBRU8sNkNBQWUsR0FBdkI7UUFDRSxJQUFNLGFBQWEsR0FBc0MsRUFBRSxDQUFBO1FBQ25ELElBQUEsV0FBVyxHQUFLLElBQUksQ0FBQyxnQkFBZ0IsWUFBMUIsQ0FBMEI7UUFFdkMsSUFBQSxLQUErRCxJQUFJLENBQUMsWUFBWSxFQUFyRSxXQUFXLGFBQUEsRUFBVSxVQUFVLFlBQUEsRUFBUSxRQUFRLFVBQXNCLENBQUE7UUFFdEYsYUFBYSxDQUFDLElBQUksR0FBRyxJQUFBLHFDQUFrQixFQUFDO1lBQ3RDLFFBQVEsRUFBRSxLQUFLO1lBQ2YsV0FBVyxFQUFFLFFBQVE7WUFDckIsV0FBVyxFQUFFLE1BQU07WUFDbkIsZUFBZSxFQUFFLFdBQVc7U0FDN0IsQ0FBQyxDQUFBO1FBRUYsYUFBYSxDQUFDLE9BQU8sR0FBRyxJQUFBLHFDQUFrQixFQUFDO1lBQ3pDLFFBQVEsRUFBRSxJQUFJO1lBQ2QsV0FBVyxFQUFFLFdBQVc7WUFDeEIsV0FBVyxFQUFFLFNBQVM7WUFDdEIsZUFBZSxFQUFFLFdBQVc7U0FDN0IsQ0FBQyxDQUFBO1FBRUYsYUFBYSxDQUFDLE1BQU0sR0FBRyxJQUFBLHFDQUFrQixFQUFDO1lBQ3hDLFFBQVEsRUFBRSxJQUFJO1lBQ2QsV0FBVyxFQUFFLFVBQVU7WUFDdkIsV0FBVyxFQUFFLFFBQVE7WUFDckIsZUFBZSxFQUFFLFdBQVc7U0FDN0IsQ0FBQyxDQUFBO1FBRUYsT0FBTyxhQUF5QyxDQUFBO0lBQ2xELENBQUM7SUFFYSwwQ0FBWSxHQUExQjs7Ozs7O3dCQUNRLEtBQTRCLElBQUksQ0FBQyxhQUFhLEVBQTVDLElBQUksVUFBQSxFQUFFLE9BQU8sYUFBQSxFQUFFLE1BQU0sWUFBQSxDQUF1Qjs2QkFFaEQsSUFBSSxFQUFKLHdCQUFJO3dCQUNOLHFCQUFNLElBQUksQ0FBQyxTQUFTLENBQUMsWUFBWSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsRUFBQTs7d0JBQTVDLFNBQTRDLENBQUE7OzRCQUc5QyxxQkFBTSxJQUFJLENBQUMsU0FBUyxDQUFDLFlBQVksQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLEVBQUE7O3dCQUEvQyxTQUErQyxDQUFBO3dCQUMvQyxxQkFBTSxJQUFJLENBQUMsU0FBUyxDQUFDLFlBQVksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLEVBQUE7O3dCQUE5QyxTQUE4QyxDQUFBOzs7OztLQUMvQztJQUNILDBCQUFDO0FBQUQsQ0FBQyxBQTFGRCxDQUF5QyxrQ0FBZSxHQTBGdkQ7QUExRlksa0RBQW1CO0FBNEZoQyxTQUFTLG1CQUFtQixDQUFDLGdCQUF3QixFQUFFLGVBQXVCO0lBQzVFLE9BQU8scURBQ2lDLGdCQUFnQiw2REFDakIsZUFBZSxxWUFXdkIsZ0JBQWdCLDJVQVVkLGdCQUFnQiw4MUJBeUJoQixlQUFlLDRsQkFrQnpDLENBQUE7QUFDVCxDQUFDIn0=