@finos/legend-extension-dsl-data-quality
Version:
Legend extension for Data Quality
547 lines • 26.6 kB
JavaScript
/**
* Copyright (c) 2020-present, Goldman Sachs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ElementEditorState, } from '@finos/legend-application-studio';
import { DataQualityRelationValidation, DataQualityRelationValidationConfiguration, RelationValidationType, } from '../../graph/metamodel/pure/packageableElements/data-quality/DataQualityValidationConfiguration.js';
import { assertErrorThrown, assertType, guaranteeNonNullable, guaranteeType, hashArray, LogEvent, filterByType, getContentTypeFileExtension, ActionState, StopWatch, } from '@finos/legend-shared';
import { buildSourceInformationSourceId, GRAPH_MANAGER_EVENT, isStubbed_PackageableElement, isStubbed_RawLambda, ParserError, RawLambda, buildLambdaVariableExpressions, observe_ValueSpecification, VariableExpression, V1_DELEGATED_EXPORT_HEADER, RelationTypeMetadata, observe_RelationTypeMetadata, } from '@finos/legend-graph';
import { action, computed, flow, flowResult, makeObservable, observable, } from 'mobx';
import { buildExecutionParameterValues, ExecutionPlanState, LambdaEditorState, LambdaParametersState, LambdaParameterState, PARAMETER_SUBMIT_ACTION, } from '@finos/legend-query-builder';
import { DataQualityRelationValidationState } from './DataQualityRelationValidationState.js';
import { DataQualityRelationResultState } from './DataQualityRelationResultState.js';
import { DATA_QUALITY_HASH_STRUCTURE } from '../../graph/metamodel/DSL_DataQuality_HashUtils.js';
import {} from '@finos/legend-art';
import { getDataQualityPureGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataQuality_PureGraphManagerExtension.js';
import { downloadStream } from '@finos/legend-application';
import { SuggestedValidationsState, SuggestionType, } from './DataQualityRelationValidationSuggestedValidationState.js';
import { dataQualityRelationValidation_addValidation, dataQualityRelationValidation_setAssertion, } from '../../graph-manager/DSL_DataQuality_GraphModifierHelper.js';
export var DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB;
(function (DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB) {
DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB["DEFINITION"] = "Definition";
DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB["VALIDATIONS"] = "Validations";
DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB["TRIAL_RUN"] = "Trial Run";
})(DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB || (DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB = {}));
export const DEFAULT_QUERY_LIMIT = 1000;
export var EXECUTION_TYPE;
(function (EXECUTION_TYPE) {
EXECUTION_TYPE["EXECUTION"] = "EXECUTION";
EXECUTION_TYPE["PROFILING"] = "PROFILING";
})(EXECUTION_TYPE || (EXECUTION_TYPE = {}));
export class RelationFunctionDefinitionEditorState extends LambdaEditorState {
editorStore;
relationValidationElement;
configurationState;
isConvertingFunctionBodyToString = false;
constructor(relationValidationElement, editorStore, configurationState) {
super('', '|');
makeObservable(this, {
relationValidationElement: observable,
isConvertingFunctionBodyToString: observable,
});
this.relationValidationElement = relationValidationElement;
this.editorStore = editorStore;
this.configurationState = configurationState;
}
get lambdaId() {
return buildSourceInformationSourceId([
this.relationValidationElement.path,
]);
}
*convertLambdaGrammarStringToObject() {
if (this.lambdaString) {
try {
const lambda = (yield this.editorStore.graphManagerState.graphManager.pureCodeToLambda(this.fullLambdaString, this.lambdaId));
this.setParserError(undefined);
this.relationValidationElement.query.body = lambda.body;
// Refresh relation columns after successful query update
yield flowResult(this.configurationState.setupValidationStatesWithColumns());
}
catch (error) {
assertErrorThrown(error);
if (error instanceof ParserError) {
this.setParserError(error);
}
this.editorStore.applicationStore.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.PARSING_FAILURE), error);
}
}
else {
this.clearErrors();
this.relationValidationElement.query.body = new RawLambda(undefined, undefined).body;
this.relationValidationElement.query.parameters = [];
}
}
*convertLambdaObjectToGrammarString(options) {
if (!isStubbed_PackageableElement(this.relationValidationElement)) {
this.isConvertingFunctionBodyToString = true;
try {
const lambdas = new Map();
const functionLamba = new RawLambda([], this.relationValidationElement.query.body);
lambdas.set(this.lambdaId, functionLamba);
const isolatedLambdas = (yield this.editorStore.graphManagerState.graphManager.lambdasToPureCode(lambdas, options?.pretty));
const grammarText = isolatedLambdas.get(this.lambdaId);
if (grammarText) {
this.setLambdaString(this.extractLambdaString(grammarText));
}
else {
this.setLambdaString('');
}
// `firstLoad` flag is used in the first rendering of the function editor (in a `useEffect`)
// This flag helps block editing while the JSON is converting to text and to avoid reseting parser/compiler error in reveal error
if (!options?.firstLoad) {
this.clearErrors({
preserveCompilationError: options?.preserveCompilationError,
});
}
this.isConvertingFunctionBodyToString = false;
}
catch (error) {
assertErrorThrown(error);
this.editorStore.applicationStore.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.PARSING_FAILURE), error);
this.isConvertingFunctionBodyToString = false;
}
}
else {
this.clearErrors();
this.setLambdaString('');
}
}
get hashCode() {
return hashArray([
DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_RELATION_FUNCTION_DEFINITION,
this.lambdaString,
]);
}
}
export class RelationDefinitionParameterState extends LambdaParametersState {
relationValidationConfigurationState;
constructor(relationValidationConfigurationState) {
super();
makeObservable(this, {
parameterValuesEditorState: observable,
parameterStates: observable,
addParameter: action,
removeParameter: action,
openModal: action,
build: action,
setParameters: action,
});
this.relationValidationConfigurationState =
relationValidationConfigurationState;
}
openModal(lambda, onSubmit) {
this.parameterStates = this.build(lambda);
this.parameterValuesEditorState.open(() => flowResult(onSubmit()).catch(this.relationValidationConfigurationState.editorStore.applicationStore
.alertUnhandledError), PARAMETER_SUBMIT_ACTION.RUN);
}
build(lambda) {
const parameters = buildLambdaVariableExpressions(lambda, this.relationValidationConfigurationState.editorStore.graphManagerState)
.map((parameter) => observe_ValueSpecification(parameter, this.relationValidationConfigurationState.editorStore
.changeDetectionState.observerContext))
.filter(filterByType(VariableExpression));
const states = parameters.map((variable) => {
const parmeterState = new LambdaParameterState(variable, this.relationValidationConfigurationState.editorStore.changeDetectionState.observerContext, this.relationValidationConfigurationState.editorStore.graphManagerState.graph);
parmeterState.mockParameterValue();
return parmeterState;
});
return states;
}
}
export class DataQualityRelationValidationConfigurationState extends ElementEditorState {
relationFunctionDefinitionEditorState;
exportState = ActionState.create();
selectedTab;
lastExecutionType = undefined;
currentExecutionType = undefined;
isGeneratingPlan = false;
runPromise = undefined;
executionResult;
executionPlanState;
validationStates = [];
suggestedValidationsState;
parametersState;
isConvertingValidationLambdaObjects = false;
resultState;
executionDuration;
latestRunHashCode;
isSuggestionPanelOpen = false;
relationTypeMetadata = new RelationTypeMetadata();
lastRelationColumnsQueryHash = undefined;
queryLimit = DEFAULT_QUERY_LIMIT;
constructor(editorStore, element) {
super(editorStore, element);
makeObservable(this, {
selectedTab: observable,
currentExecutionType: observable,
runPromise: observable,
executionResult: observable,
resultState: observable,
executionDuration: observable,
latestRunHashCode: observable,
lastExecutionType: observable,
isSuggestionPanelOpen: observable,
validationStates: observable,
queryLimit: observable,
setSelectedTab: action,
setRunPromise: action,
setExecutionResult: action,
addValidationState: action,
resetResultState: action,
setExecutionDuration: action,
applyOrModifySuggestion: action,
applySuggestion: action,
modifyExistingSuggestion: action,
deleteValidationState: action,
setQueryLimit: action,
validationElement: computed,
relationValidationOptions: computed,
checkForStaleResults: computed,
isRunning: computed,
run: flow,
handleRun: flow,
exportData: flow,
getRelationColumns: flow,
relationTypeMetadata: observable,
convertValidationLambdaObjects: flow,
cancelRun: flow,
generatePlan: flow,
setupValidationStatesWithColumns: flow,
});
assertType(element, DataQualityRelationValidationConfiguration, 'Element inside data quality relation validation editor state must be a data quality relation validation element');
this.relationFunctionDefinitionEditorState =
new RelationFunctionDefinitionEditorState(element, this.editorStore, this);
this.selectedTab = DATA_QUALITY_RELATION_VALIDATION_EDITOR_TAB.DEFINITION;
this.relationTypeMetadata = observe_RelationTypeMetadata(this.relationTypeMetadata);
this.validationElement.validations.forEach((validation) => {
this.validationStates.push(new DataQualityRelationValidationState(validation, editorStore));
});
this.executionPlanState = new ExecutionPlanState(this.editorStore.applicationStore, this.editorStore.graphManagerState);
this.parametersState = new RelationDefinitionParameterState(this);
this.resultState = new DataQualityRelationResultState(this);
this.suggestedValidationsState = new SuggestedValidationsState(this);
flowResult(this.setupValidationStatesWithColumns()).catch(this.editorStore.applicationStore.alertUnhandledError);
}
*setupValidationStatesWithColumns() {
yield flowResult(this.getRelationColumns());
this.validationStates.forEach((validationState) => {
validationState.initializeWithColumns(this.relationTypeMetadata.columns);
});
}
reprocess(newElement, editorStore) {
return new DataQualityRelationValidationConfigurationState(editorStore, newElement);
}
get validationElement() {
return guaranteeType(this.element, DataQualityRelationValidationConfiguration, 'Element inside data quality relation validation state must be a data quality relation validation configuration element');
}
get validationOptions() {
return this.validationElement.validations.map((validation) => {
return {
label: validation.name,
value: validation,
};
});
}
getNullableValidationState = (relationValidation) => this.validationStates.find((validationState) => validationState.relationValidation === relationValidation);
getValidationState = (validation) => guaranteeNonNullable(this.getNullableValidationState(validation), `Can't find validation state for validation ${validation}`);
get relationValidationOptions() {
return Object.values(RelationValidationType).map((type) => ({
label: type,
value: type,
}));
}
get checkForStaleResults() {
if (this.latestRunHashCode !== this.hashCode) {
return true;
}
return false;
}
get isRunning() {
return this.currentExecutionType !== undefined;
}
setExecutionDuration(val) {
this.executionDuration = val;
}
resetResultState() {
this.resultState = new DataQualityRelationResultState(this);
}
addValidationState(validation) {
if (!this.validationStates.find((validationState) => validationState.relationValidation === validation)) {
const validationState = new DataQualityRelationValidationState(validation, this.editorStore);
validationState.initializeWithColumns(this.relationTypeMetadata.columns);
this.validationStates.push(validationState);
}
}
deleteValidationState(validation) {
const idx = this.validationStates.findIndex((validationState) => validationState.relationValidation === validation);
if (idx !== -1) {
this.validationStates.splice(idx, 1);
}
}
setQueryLimit(queryLimit) {
this.queryLimit = Math.max(1, queryLimit);
}
setRunPromise = (promise) => {
this.runPromise = promise;
};
setSelectedTab(tab) {
this.selectedTab = tab;
}
setExecutionResult = (executionResult, type) => {
this.lastExecutionType = type;
this.executionResult = executionResult;
};
*handleRun(type) {
if (this.isRunning) {
return;
}
const queryLambda = this.bodyExpressionSequence;
const parameters = (queryLambda.parameters ?? []);
if (parameters.length) {
this.parametersState.openModal(queryLambda, () => this.run(type));
}
else {
flowResult(this.run(type)).catch(this.editorStore.applicationStore.alertUnhandledError);
}
}
*convertValidationLambdaObjects() {
const lambdas = new Map();
const index = new Map();
this.validationStates.forEach((validationState) => {
if (!isStubbed_RawLambda(validationState.relationValidation.assertion)) {
lambdas.set(validationState.lambdaId, validationState.relationValidation.assertion);
index.set(validationState.lambdaId, validationState);
}
});
if (lambdas.size) {
this.isConvertingValidationLambdaObjects = true;
try {
const isolatedLambdas = (yield this.editorStore.graphManagerState.graphManager.lambdasToPureCode(lambdas));
isolatedLambdas.forEach((grammarText, key) => {
const validationState = index.get(key);
validationState?.setLambdaString(validationState.extractLambdaString(grammarText));
});
}
catch (error) {
assertErrorThrown(error);
this.editorStore.applicationStore.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.PARSING_FAILURE), error);
}
finally {
this.isConvertingValidationLambdaObjects = false;
}
}
}
get bodyExpressionSequence() {
return new RawLambda(this.validationElement.query.parameters.map((parameter) => this.editorStore.graphManagerState.graphManager.serializeRawValueSpecification(parameter)), this.validationElement.query.body);
}
*run(type) {
let promise = undefined;
const stopWatch = new StopWatch();
try {
this.currentExecutionType = type;
const currentHashCode = this.hashCode;
const packagePath = this.validationElement.path;
const model = this.editorStore.graphManagerState.graph;
const extension = getDataQualityPureGraphManagerExtension(this.editorStore.graphManagerState.graphManager);
const options = {
lambdaParameterValues: buildExecutionParameterValues(this.parametersState.parameterStates, this.editorStore.graphManagerState),
};
promise =
type === EXECUTION_TYPE.PROFILING
? extension.runDataProfiling(model, packagePath, options)
: extension.execute(model, packagePath, {
...options,
runQuery: true,
queryLimit: this.queryLimit,
});
this.setRunPromise(promise);
const result = (yield promise);
if (this.runPromise === promise) {
this.setExecutionResult(result, type);
this.latestRunHashCode = currentHashCode;
this.setExecutionDuration(stopWatch.elapsed);
}
}
catch (error) {
if (this.runPromise === promise) {
assertErrorThrown(error);
this.setExecutionResult(undefined, type);
this.editorStore.applicationStore.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE), error);
this.editorStore.applicationStore.notificationService.notifyError(error);
}
}
finally {
this.currentExecutionType = undefined;
}
}
*cancelRun() {
this.currentExecutionType = undefined;
this.setRunPromise(undefined);
try {
yield this.editorStore.graphManagerState.graphManager.cancelUserExecutions(true);
}
catch (error) {
// Don't notify users about success or failure
this.editorStore.applicationStore.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE), error);
}
}
*generatePlan(debug) {
const packagePath = this.validationElement.path;
const model = this.editorStore.graphManagerState.graph;
try {
this.isGeneratingPlan = true;
let rawPlan;
if (debug) {
const debugResult = (yield getDataQualityPureGraphManagerExtension(this.editorStore.graphManagerState.graphManager).debugExecutionPlanGeneration(model, packagePath, {
runQuery: true,
}));
rawPlan = debugResult.plan;
this.executionPlanState.setDebugText(debugResult.debug);
}
else {
rawPlan = (yield getDataQualityPureGraphManagerExtension(this.editorStore.graphManagerState.graphManager).generatePlan(model, packagePath, {
runQuery: true,
}));
}
try {
this.executionPlanState.setRawPlan(rawPlan);
const plan = this.editorStore.graphManagerState.graphManager.buildExecutionPlan(rawPlan, this.editorStore.graphManagerState.graph);
this.executionPlanState.initialize(plan);
}
catch {
//do nothing
}
}
catch (error) {
assertErrorThrown(error);
this.editorStore.applicationStore.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.EXECUTION_FAILURE), error);
this.editorStore.applicationStore.notificationService.notifyError(error);
}
finally {
this.isGeneratingPlan = false;
}
}
*exportData(format) {
try {
this.exportState.inProgress();
const type = this.lastExecutionType;
const packagePath = this.validationElement.path;
const model = this.editorStore.graphManagerState.graph;
this.editorStore.applicationStore.notificationService.notifySuccess(`Export ${format} will run in background`);
const exportData = this.resultState.getExportDataInfo(format);
const contentType = exportData.contentType;
const serializationFormat = exportData.serializationFormat;
const extension = getDataQualityPureGraphManagerExtension(this.editorStore.graphManagerState.graphManager);
const options = {
serializationFormat,
lambdaParameterValues: buildExecutionParameterValues(this.parametersState.parameterStates, this.editorStore.graphManagerState),
};
const result = type === EXECUTION_TYPE.PROFILING
? (yield extension.exportDataProfiling(model, packagePath, options))
: (yield extension.exportData(model, packagePath, {
...options,
runQuery: true,
}));
if (result.headers.get(V1_DELEGATED_EXPORT_HEADER) === 'true') {
if (result.status === 200) {
this.exportState.pass();
}
else {
this.exportState.fail();
}
return;
}
downloadStream(result, `result.${getContentTypeFileExtension(contentType)}`, exportData.contentType)
.then(() => {
this.exportState.pass();
})
.catch((error) => {
assertErrorThrown(error);
});
}
catch (error) {
this.exportState.fail();
assertErrorThrown(error);
this.editorStore.applicationStore.notificationService.notifyError(error);
this.exportState.complete();
}
}
*getRelationColumns() {
// skip if the query body is not defined
const { body, parameters } = this.relationFunctionDefinitionEditorState.relationValidationElement
.query;
if (!body || (Array.isArray(body) && body.length === 0)) {
return;
}
const lambda = new RawLambda(parameters, body);
// this is to avoid unecessary calls, we only care if the actual lambda has changed, otherwise we don't want to updated column metadata
const currentQueryHash = this.relationFunctionDefinitionEditorState.hashCode;
if (currentQueryHash === this.lastRelationColumnsQueryHash) {
return;
}
try {
this.relationTypeMetadata = observe_RelationTypeMetadata(yield this.editorStore.graphManagerState.graphManager.getLambdaRelationType(lambda, this.editorStore.graphManagerState.graph));
this.lastRelationColumnsQueryHash = currentQueryHash;
}
catch (error) {
assertErrorThrown(error);
this.editorStore.applicationStore.notificationService.notifyError(`Error getting relation type columns: ${error.message}`);
}
}
applySuggestion(validationState) {
const relationValidation = validationState.relationValidation;
// Create a NEW validation instance
const newValidation = new DataQualityRelationValidation(relationValidation.name, new RawLambda(relationValidation.assertion.parameters, relationValidation.assertion.body));
if (relationValidation.type) {
newValidation.type = relationValidation.type;
}
newValidation.description = relationValidation.description;
// Add to model (this modifies the graph)
dataQualityRelationValidation_addValidation(this.validationElement, newValidation);
this.addValidationState(newValidation);
const newValidationState = this.getValidationState(newValidation);
newValidationState.setLambdaString(validationState.lambdaString);
// Force proper GUI editor initialization if needed
if (newValidationState.isGUIEditor) {
newValidationState.initializeWithColumns(this.relationTypeMetadata.columns);
}
}
modifyExistingSuggestion(validation) {
const existingValidation = this.validationElement.validations.find((v) => v.name === validation.relationValidation.name);
if (existingValidation) {
dataQualityRelationValidation_setAssertion(existingValidation, new RawLambda(validation.relationValidation.assertion.parameters, validation.relationValidation.assertion.body));
const existingValidationState = this.getValidationState(existingValidation);
existingValidationState.setLambdaString(validation.lambdaString);
if (existingValidationState.isGUIEditor) {
existingValidationState.initializeWithColumns(this.relationTypeMetadata.columns);
}
}
}
applyOrModifySuggestion(validationState) {
const suggestionType = this.suggestedValidationsState.getSuggestionType(validationState);
if (suggestionType === SuggestionType.NEW) {
this.applySuggestion(validationState);
}
else if (suggestionType === SuggestionType.EDIT) {
this.modifyExistingSuggestion(validationState);
}
}
get hashCode() {
return hashArray([
DATA_QUALITY_HASH_STRUCTURE.DATA_QUALITY_RELATION_VALIDATION,
this.relationFunctionDefinitionEditorState,
hashArray(this.validationStates),
]);
}
}
//# sourceMappingURL=DataQualityRelationValidationConfigurationState.js.map