@versatiledatakit/data-pipelines
Version:
Data Pipelines help Data Engineers develop, deploy, run, and manage data processing workloads (called 'Data Job')
71 lines • 12.3 kB
JavaScript
/*
* Copyright 2023-2025 Broadcom
* SPDX-License-Identifier: Apache-2.0
*/
import { CollectionsUtil } from '@versatiledatakit/shared';
import { DataJobExecutionStatus, DataJobExecutionStatusDeprecated } from '../../model';
/**
* ** Utils for Data Job.
*/
export class DataJobUtil {
/**
* ** Find if some Job is running in provided Executions.
*/
static isJobRunning(jobExecutions) {
// eslint-disable-next-line @typescript-eslint/unbound-method
return jobExecutions.findIndex(DataJobUtil.isJobRunningPredicate) !== -1;
}
static convertFromExecutionDetailsToExecutionState(jobExecutionDetails) {
if (CollectionsUtil.isNil(jobExecutionDetails)) {
return {
id: null
};
}
const execution = {
id: jobExecutionDetails.id,
jobName: jobExecutionDetails.job_name,
opId: jobExecutionDetails.op_id,
status: jobExecutionDetails.status.toUpperCase(),
startedBy: jobExecutionDetails.started_by,
startTime: jobExecutionDetails.start_time,
endTime: jobExecutionDetails.end_time,
message: jobExecutionDetails.message,
type: jobExecutionDetails.type.toUpperCase(),
logsUrl: jobExecutionDetails.logs_url,
deployment: {
schedule: {},
resources: {}
}
};
if (CollectionsUtil.isLiteralObject(jobExecutionDetails.deployment)) {
execution.deployment.id = jobExecutionDetails.deployment.id;
execution.deployment.enabled = jobExecutionDetails.deployment.enabled;
execution.deployment.jobVersion = jobExecutionDetails.deployment.job_version;
execution.deployment.vdkVersion = jobExecutionDetails.deployment.vdk_version;
execution.deployment.mode = jobExecutionDetails.deployment.mode;
execution.deployment.deployedDate = jobExecutionDetails.deployment.deployed_date;
execution.deployment.deployedBy = jobExecutionDetails.deployment.deployed_by;
execution.deployment.jobPythonVersion = jobExecutionDetails.deployment.python_version;
if (CollectionsUtil.isLiteralObject(jobExecutionDetails.deployment.schedule)) {
execution.deployment.schedule.scheduleCron = jobExecutionDetails.deployment.schedule.schedule_cron;
}
if (CollectionsUtil.isLiteralObject(jobExecutionDetails.deployment.resources)) {
execution.deployment.resources.cpuRequest = jobExecutionDetails.deployment.resources.cpu_request;
execution.deployment.resources.cpuLimit = jobExecutionDetails.deployment.resources.cpu_limit;
execution.deployment.resources.memoryRequest = jobExecutionDetails.deployment.resources.memory_request;
execution.deployment.resources.memoryLimit = jobExecutionDetails.deployment.resources.memory_limit;
}
}
return execution;
}
}
/**
* ** Predicate for Job Running.
*/
DataJobUtil.isJobRunningPredicate = (jobExecution) => {
return (jobExecution.status === DataJobExecutionStatus.RUNNING ||
jobExecution.status === DataJobExecutionStatus.SUBMITTED ||
jobExecution.status === DataJobExecutionStatusDeprecated.RUNNING ||
jobExecution.status === DataJobExecutionStatusDeprecated.SUBMITTED);
};
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZGF0YS1qb2IudXRpbC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uLy4uL3Byb2plY3RzL2RhdGEtcGlwZWxpbmVzL3NyYy9saWIvc2hhcmVkL3V0aWxzL2RhdGEtam9iLnV0aWwudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUE7OztHQUdHO0FBRUgsT0FBTyxFQUFFLGVBQWUsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBRTNELE9BQU8sRUFJSCxzQkFBc0IsRUFDdEIsZ0NBQWdDLEVBRW5DLE1BQU0sYUFBYSxDQUFDO0FBRXJCOztHQUVHO0FBQ0gsTUFBTSxPQUFPLFdBQVc7SUFhcEI7O09BRUc7SUFDSCxNQUFNLENBQUMsWUFBWSxDQUFDLGFBQTZEO1FBQzdFLDZEQUE2RDtRQUM3RCxPQUFPLGFBQWEsQ0FBQyxTQUFTLENBQUMsV0FBVyxDQUFDLHFCQUFxQixDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUM7SUFDN0UsQ0FBQztJQUVELE1BQU0sQ0FBQywyQ0FBMkMsQ0FBQyxtQkFBNEM7UUFDM0YsSUFBSSxlQUFlLENBQUMsS0FBSyxDQUFDLG1CQUFtQixDQUFDLEVBQUU7WUFDNUMsT0FBTztnQkFDSCxFQUFFLEVBQUUsSUFBSTthQUNYLENBQUM7U0FDTDtRQUVELE1BQU0sU0FBUyxHQUFxQjtZQUNoQyxFQUFFLEVBQUUsbUJBQW1CLENBQUMsRUFBRTtZQUMxQixPQUFPLEVBQUUsbUJBQW1CLENBQUMsUUFBUTtZQUNyQyxJQUFJLEVBQUUsbUJBQW1CLENBQUMsS0FBSztZQUMvQixNQUFNLEVBQUUsbUJBQW1CLENBQUMsTUFBTSxDQUFDLFdBQVcsRUFBNEI7WUFDMUUsU0FBUyxFQUFFLG1CQUFtQixDQUFDLFVBQVU7WUFDekMsU0FBUyxFQUFFLG1CQUFtQixDQUFDLFVBQVU7WUFDekMsT0FBTyxFQUFFLG1CQUFtQixDQUFDLFFBQVE7WUFDckMsT0FBTyxFQUFFLG1CQUFtQixDQUFDLE9BQU87WUFDcEMsSUFBSSxFQUFFLG1CQUFtQixDQUFDLElBQUksQ0FBQyxXQUFXLEVBQTBCO1lBQ3BFLE9BQU8sRUFBRSxtQkFBbUIsQ0FBQyxRQUFRO1lBQ3JDLFVBQVUsRUFBRTtnQkFDUixRQUFRLEVBQUUsRUFBRTtnQkFDWixTQUFTLEVBQUUsRUFBRTthQUNLO1NBQ3pCLENBQUM7UUFFRixJQUFJLGVBQWUsQ0FBQyxlQUFlLENBQUMsbUJBQW1CLENBQUMsVUFBVSxDQUFDLEVBQUU7WUFDakUsU0FBUyxDQUFDLFVBQVUsQ0FBQyxFQUFFLEdBQUcsbUJBQW1CLENBQUMsVUFBVSxDQUFDLEVBQUUsQ0FBQztZQUM1RCxTQUFTLENBQUMsVUFBVSxDQUFDLE9BQU8sR0FBRyxtQkFBbUIsQ0FBQyxVQUFVLENBQUMsT0FBTyxDQUFDO1lBQ3RFLFNBQVMsQ0FBQyxVQUFVLENBQUMsVUFBVSxHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxXQUFXLENBQUM7WUFDN0UsU0FBUyxDQUFDLFVBQVUsQ0FBQyxVQUFVLEdBQUcsbUJBQW1CLENBQUMsVUFBVSxDQUFDLFdBQVcsQ0FBQztZQUM3RSxTQUFTLENBQUMsVUFBVSxDQUFDLElBQUksR0FBRyxtQkFBbUIsQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDO1lBQ2hFLFNBQVMsQ0FBQyxVQUFVLENBQUMsWUFBWSxHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxhQUFhLENBQUM7WUFDakYsU0FBUyxDQUFDLFVBQVUsQ0FBQyxVQUFVLEdBQUcsbUJBQW1CLENBQUMsVUFBVSxDQUFDLFdBQVcsQ0FBQztZQUM3RSxTQUFTLENBQUMsVUFBVSxDQUFDLGdCQUFnQixHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxjQUFjLENBQUM7WUFFdEYsSUFBSSxlQUFlLENBQUMsZUFBZSxDQUFDLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsRUFBRTtnQkFDMUUsU0FBUyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsWUFBWSxHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsYUFBYSxDQUFDO2FBQ3RHO1lBRUQsSUFBSSxlQUFlLENBQUMsZUFBZSxDQUFDLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsRUFBRTtnQkFDM0UsU0FBUyxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsVUFBVSxHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsV0FBVyxDQUFDO2dCQUNqRyxTQUFTLENBQUMsVUFBVSxDQUFDLFNBQVMsQ0FBQyxRQUFRLEdBQUcsbUJBQW1CLENBQUMsVUFBVSxDQUFDLFNBQVMsQ0FBQyxTQUFTLENBQUM7Z0JBQzdGLFNBQVMsQ0FBQyxVQUFVLENBQUMsU0FBUyxDQUFDLGFBQWEsR0FBRyxtQkFBbUIsQ0FBQyxVQUFVLENBQUMsU0FBUyxDQUFDLGNBQWMsQ0FBQztnQkFDdkcsU0FBUyxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsV0FBVyxHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsWUFBWSxDQUFDO2FBQ3RHO1NBQ0o7UUFFRCxPQUFPLFNBQVMsQ0FBQztJQUNyQixDQUFDOztBQW5FRDs7R0FFRztBQUNJLGlDQUFxQixHQUFHLENBQUMsWUFBd0QsRUFBVyxFQUFFO0lBQ2pHLE9BQU8sQ0FDRixZQUFpQyxDQUFDLE1BQU0sS0FBSyxzQkFBc0IsQ0FBQyxPQUFPO1FBQzNFLFlBQWlDLENBQUMsTUFBTSxLQUFLLHNCQUFzQixDQUFDLFNBQVM7UUFDN0UsWUFBd0MsQ0FBQyxNQUFNLEtBQUssZ0NBQWdDLENBQUMsT0FBTztRQUM1RixZQUF3QyxDQUFDLE1BQU0sS0FBSyxnQ0FBZ0MsQ0FBQyxTQUFTLENBQ2xHLENBQUM7QUFDTixDQUFDLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKlxuICogQ29weXJpZ2h0IDIwMjMtMjAyNSBCcm9hZGNvbVxuICogU1BEWC1MaWNlbnNlLUlkZW50aWZpZXI6IEFwYWNoZS0yLjBcbiAqL1xuXG5pbXBvcnQgeyBDb2xsZWN0aW9uc1V0aWwgfSBmcm9tICdAdmVyc2F0aWxlZGF0YWtpdC9zaGFyZWQnO1xuXG5pbXBvcnQge1xuICAgIERhdGFKb2JEZXBsb3ltZW50LFxuICAgIERhdGFKb2JFeGVjdXRpb24sXG4gICAgRGF0YUpvYkV4ZWN1dGlvbkRldGFpbHMsXG4gICAgRGF0YUpvYkV4ZWN1dGlvblN0YXR1cyxcbiAgICBEYXRhSm9iRXhlY3V0aW9uU3RhdHVzRGVwcmVjYXRlZCxcbiAgICBEYXRhSm9iRXhlY3V0aW9uVHlwZVxufSBmcm9tICcuLi8uLi9tb2RlbCc7XG5cbi8qKlxuICogKiogVXRpbHMgZm9yIERhdGEgSm9iLlxuICovXG5leHBvcnQgY2xhc3MgRGF0YUpvYlV0aWwge1xuICAgIC8qKlxuICAgICAqICoqIFByZWRpY2F0ZSBmb3IgSm9iIFJ1bm5pbmcuXG4gICAgICovXG4gICAgc3RhdGljIGlzSm9iUnVubmluZ1ByZWRpY2F0ZSA9IChqb2JFeGVjdXRpb246IERhdGFKb2JFeGVjdXRpb24gfCBEYXRhSm9iRXhlY3V0aW9uRGV0YWlscyk6IGJvb2xlYW4gPT4ge1xuICAgICAgICByZXR1cm4gKFxuICAgICAgICAgICAgKGpvYkV4ZWN1dGlvbiBhcyBEYXRhSm9iRXhlY3V0aW9uKS5zdGF0dXMgPT09IERhdGFKb2JFeGVjdXRpb25TdGF0dXMuUlVOTklORyB8fFxuICAgICAgICAgICAgKGpvYkV4ZWN1dGlvbiBhcyBEYXRhSm9iRXhlY3V0aW9uKS5zdGF0dXMgPT09IERhdGFKb2JFeGVjdXRpb25TdGF0dXMuU1VCTUlUVEVEIHx8XG4gICAgICAgICAgICAoam9iRXhlY3V0aW9uIGFzIERhdGFKb2JFeGVjdXRpb25EZXRhaWxzKS5zdGF0dXMgPT09IERhdGFKb2JFeGVjdXRpb25TdGF0dXNEZXByZWNhdGVkLlJVTk5JTkcgfHxcbiAgICAgICAgICAgIChqb2JFeGVjdXRpb24gYXMgRGF0YUpvYkV4ZWN1dGlvbkRldGFpbHMpLnN0YXR1cyA9PT0gRGF0YUpvYkV4ZWN1dGlvblN0YXR1c0RlcHJlY2F0ZWQuU1VCTUlUVEVEXG4gICAgICAgICk7XG4gICAgfTtcblxuICAgIC8qKlxuICAgICAqICoqIEZpbmQgaWYgc29tZSBKb2IgaXMgcnVubmluZyBpbiBwcm92aWRlZCBFeGVjdXRpb25zLlxuICAgICAqL1xuICAgIHN0YXRpYyBpc0pvYlJ1bm5pbmcoam9iRXhlY3V0aW9uczogRGF0YUpvYkV4ZWN1dGlvbltdIHwgRGF0YUpvYkV4ZWN1dGlvbkRldGFpbHNbXSk6IGJvb2xlYW4ge1xuICAgICAgICAvLyBlc2xpbnQtZGlzYWJsZS1uZXh0LWxpbmUgQHR5cGVzY3JpcHQtZXNsaW50L3VuYm91bmQtbWV0aG9kXG4gICAgICAgIHJldHVybiBqb2JFeGVjdXRpb25zLmZpbmRJbmRleChEYXRhSm9iVXRpbC5pc0pvYlJ1bm5pbmdQcmVkaWNhdGUpICE9PSAtMTtcbiAgICB9XG5cbiAgICBzdGF0aWMgY29udmVydEZyb21FeGVjdXRpb25EZXRhaWxzVG9FeGVjdXRpb25TdGF0ZShqb2JFeGVjdXRpb25EZXRhaWxzOiBEYXRhSm9iRXhlY3V0aW9uRGV0YWlscyk6IERhdGFKb2JFeGVjdXRpb24ge1xuICAgICAgICBpZiAoQ29sbGVjdGlvbnNVdGlsLmlzTmlsKGpvYkV4ZWN1dGlvbkRldGFpbHMpKSB7XG4gICAgICAgICAgICByZXR1cm4ge1xuICAgICAgICAgICAgICAgIGlkOiBudWxsXG4gICAgICAgICAgICB9O1xuICAgICAgICB9XG5cbiAgICAgICAgY29uc3QgZXhlY3V0aW9uOiBEYXRhSm9iRXhlY3V0aW9uID0ge1xuICAgICAgICAgICAgaWQ6IGpvYkV4ZWN1dGlvbkRldGFpbHMuaWQsXG4gICAgICAgICAgICBqb2JOYW1lOiBqb2JFeGVjdXRpb25EZXRhaWxzLmpvYl9uYW1lLFxuICAgICAgICAgICAgb3BJZDogam9iRXhlY3V0aW9uRGV0YWlscy5vcF9pZCxcbiAgICAgICAgICAgIHN0YXR1czogam9iRXhlY3V0aW9uRGV0YWlscy5zdGF0dXMudG9VcHBlckNhc2UoKSBhcyBEYXRhSm9iRXhlY3V0aW9uU3RhdHVzLFxuICAgICAgICAgICAgc3RhcnRlZEJ5OiBqb2JFeGVjdXRpb25EZXRhaWxzLnN0YXJ0ZWRfYnksXG4gICAgICAgICAgICBzdGFydFRpbWU6IGpvYkV4ZWN1dGlvbkRldGFpbHMuc3RhcnRfdGltZSxcbiAgICAgICAgICAgIGVuZFRpbWU6IGpvYkV4ZWN1dGlvbkRldGFpbHMuZW5kX3RpbWUsXG4gICAgICAgICAgICBtZXNzYWdlOiBqb2JFeGVjdXRpb25EZXRhaWxzLm1lc3NhZ2UsXG4gICAgICAgICAgICB0eXBlOiBqb2JFeGVjdXRpb25EZXRhaWxzLnR5cGUudG9VcHBlckNhc2UoKSBhcyBEYXRhSm9iRXhlY3V0aW9uVHlwZSxcbiAgICAgICAgICAgIGxvZ3NVcmw6IGpvYkV4ZWN1dGlvbkRldGFpbHMubG9nc191cmwsXG4gICAgICAgICAgICBkZXBsb3ltZW50OiB7XG4gICAgICAgICAgICAgICAgc2NoZWR1bGU6IHt9LFxuICAgICAgICAgICAgICAgIHJlc291cmNlczoge31cbiAgICAgICAgICAgIH0gYXMgRGF0YUpvYkRlcGxveW1lbnRcbiAgICAgICAgfTtcblxuICAgICAgICBpZiAoQ29sbGVjdGlvbnNVdGlsLmlzTGl0ZXJhbE9iamVjdChqb2JFeGVjdXRpb25EZXRhaWxzLmRlcGxveW1lbnQpKSB7XG4gICAgICAgICAgICBleGVjdXRpb24uZGVwbG95bWVudC5pZCA9IGpvYkV4ZWN1dGlvbkRldGFpbHMuZGVwbG95bWVudC5pZDtcbiAgICAgICAgICAgIGV4ZWN1dGlvbi5kZXBsb3ltZW50LmVuYWJsZWQgPSBqb2JFeGVjdXRpb25EZXRhaWxzLmRlcGxveW1lbnQuZW5hYmxlZDtcbiAgICAgICAgICAgIGV4ZWN1dGlvbi5kZXBsb3ltZW50LmpvYlZlcnNpb24gPSBqb2JFeGVjdXRpb25EZXRhaWxzLmRlcGxveW1lbnQuam9iX3ZlcnNpb247XG4gICAgICAgICAgICBleGVjdXRpb24uZGVwbG95bWVudC52ZGtWZXJzaW9uID0gam9iRXhlY3V0aW9uRGV0YWlscy5kZXBsb3ltZW50LnZka192ZXJzaW9uO1xuICAgICAgICAgICAgZXhlY3V0aW9uLmRlcGxveW1lbnQubW9kZSA9IGpvYkV4ZWN1dGlvbkRldGFpbHMuZGVwbG95bWVudC5tb2RlO1xuICAgICAgICAgICAgZXhlY3V0aW9uLmRlcGxveW1lbnQuZGVwbG95ZWREYXRlID0gam9iRXhlY3V0aW9uRGV0YWlscy5kZXBsb3ltZW50LmRlcGxveWVkX2RhdGU7XG4gICAgICAgICAgICBleGVjdXRpb24uZGVwbG95bWVudC5kZXBsb3llZEJ5ID0gam9iRXhlY3V0aW9uRGV0YWlscy5kZXBsb3ltZW50LmRlcGxveWVkX2J5O1xuICAgICAgICAgICAgZXhlY3V0aW9uLmRlcGxveW1lbnQuam9iUHl0aG9uVmVyc2lvbiA9IGpvYkV4ZWN1dGlvbkRldGFpbHMuZGVwbG95bWVudC5weXRob25fdmVyc2lvbjtcblxuICAgICAgICAgICAgaWYgKENvbGxlY3Rpb25zVXRpbC5pc0xpdGVyYWxPYmplY3Qoam9iRXhlY3V0aW9uRGV0YWlscy5kZXBsb3ltZW50LnNjaGVkdWxlKSkge1xuICAgICAgICAgICAgICAgIGV4ZWN1dGlvbi5kZXBsb3ltZW50LnNjaGVkdWxlLnNjaGVkdWxlQ3JvbiA9IGpvYkV4ZWN1dGlvbkRldGFpbHMuZGVwbG95bWVudC5zY2hlZHVsZS5zY2hlZHVsZV9jcm9uO1xuICAgICAgICAgICAgfVxuXG4gICAgICAgICAgICBpZiAoQ29sbGVjdGlvbnNVdGlsLmlzTGl0ZXJhbE9iamVjdChqb2JFeGVjdXRpb25EZXRhaWxzLmRlcGxveW1lbnQucmVzb3VyY2VzKSkge1xuICAgICAgICAgICAgICAgIGV4ZWN1dGlvbi5kZXBsb3ltZW50LnJlc291cmNlcy5jcHVSZXF1ZXN0ID0gam9iRXhlY3V0aW9uRGV0YWlscy5kZXBsb3ltZW50LnJlc291cmNlcy5jcHVfcmVxdWVzdDtcbiAgICAgICAgICAgICAgICBleGVjdXRpb24uZGVwbG95bWVudC5yZXNvdXJjZXMuY3B1TGltaXQgPSBqb2JFeGVjdXRpb25EZXRhaWxzLmRlcGxveW1lbnQucmVzb3VyY2VzLmNwdV9saW1pdDtcbiAgICAgICAgICAgICAgICBleGVjdXRpb24uZGVwbG95bWVudC5yZXNvdXJjZXMubWVtb3J5UmVxdWVzdCA9IGpvYkV4ZWN1dGlvbkRldGFpbHMuZGVwbG95bWVudC5yZXNvdXJjZXMubWVtb3J5X3JlcXVlc3Q7XG4gICAgICAgICAgICAgICAgZXhlY3V0aW9uLmRlcGxveW1lbnQucmVzb3VyY2VzLm1lbW9yeUxpbWl0ID0gam9iRXhlY3V0aW9uRGV0YWlscy5kZXBsb3ltZW50LnJlc291cmNlcy5tZW1vcnlfbGltaXQ7XG4gICAgICAgICAgICB9XG4gICAgICAgIH1cblxuICAgICAgICByZXR1cm4gZXhlY3V0aW9uO1xuICAgIH1cbn1cbiJdfQ==