@finos/legend-graph
Version:
Legend graph and graph manager
755 lines (754 loc) • 130 kB
JavaScript
/**
* Copyright (c) 2020-present, Goldman Sachs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GRAPH_MANAGER_EVENT } from '../../../../__lib__/GraphManagerEvent.js';
import { CORE_PURE_PATH, PackageableElementPointerType, } from '../../../../graph/MetaModelConst.js';
import { ActionState, TracerService, LogEvent, getClass, guaranteeNonNullable, UnsupportedOperationError, assertErrorThrown, promisify, StopWatch, isNonNullable, filterByType, isString, assertNonEmptyString, uniq, guaranteeType, guaranteeNonEmptyString, uuid, } from '@finos/legend-shared';
import { AbstractPureGraphManager, } from '../../../../graph-manager/AbstractPureGraphManager.js';
import { PureModel, } from '../../../../graph/PureModel.js';
import { RawLambda } from '../../../../graph/metamodel/pure/rawValueSpecification/RawLambda.js';
import { ServiceRegistrationSuccess, ServiceRegistrationFail, } from '../../../../graph-manager/action/service/ServiceRegistrationResult.js';
import { ServiceExecutionMode } from '../../../../graph-manager/action/service/ServiceExecutionMode.js';
import { PureMultiExecution, PureSingleExecution, } from '../../../../graph/metamodel/pure/packageableElements/service/ServiceExecution.js';
import { V1_deserializeRawValueSpecification, V1_serializeRawValueSpecification, } from './transformation/pureProtocol/serializationHelpers/V1_RawValueSpecificationSerializationHelper.js';
import { V1_serializeValueSpecification, V1_deserializeValueSpecification, } from './transformation/pureProtocol/serializationHelpers/V1_ValueSpecificationSerializer.js';
import V1_CORE_SYSTEM_MODELS from './V1_Core_SystemModels.json' with { type: 'json' };
import { V1_serializePackageableElement } from './transformation/pureProtocol/V1_PackageableElementSerialization.js';
import { V1_entitiesToPureModelContextData, V1_serializePureModelContext, V1_deserializePureModelContextData, V1_setupPureModelContextDataSerialization, } from './transformation/pureProtocol/V1_PureProtocolSerialization.js';
import { V1_PureModelContextData } from './model/context/V1_PureModelContextData.js';
import { V1_PackageableElementPointer, } from './model/packageableElements/V1_PackageableElement.js';
import { V1_ElementFirstPassBuilder } from './transformation/pureGraph/to/V1_ElementFirstPassBuilder.js';
import { V1_ElementSecondPassBuilder } from './transformation/pureGraph/to/V1_ElementSecondPassBuilder.js';
import { V1_ElementThirdPassBuilder } from './transformation/pureGraph/to/V1_ElementThirdPassBuilder.js';
import { V1_ElementFourthPassBuilder } from './transformation/pureGraph/to/V1_ElementFourthPassBuilder.js';
import { V1_ElementFifthPassBuilder } from './transformation/pureGraph/to/V1_ElementFifthPassBuilder.js';
import { V1_RawValueSpecificationBuilder } from './transformation/pureGraph/to/V1_RawValueSpecificationBuilder.js';
import { V1_RawBaseExecutionContext } from './model/rawValueSpecification/V1_RawExecutionContext.js';
import { V1_GraphBuilderContextBuilder, } from './transformation/pureGraph/to/V1_GraphBuilderContext.js';
import { V1_PureModelContextPointer } from './model/context/V1_PureModelContextPointer.js';
import { V1_RemoteEngine } from './engine/V1_RemoteEngine.js';
import { V1_transformPackageableElement } from './transformation/pureGraph/from/V1_PackageableElementTransformer.js';
import { V1_transformRawLambda, V1_RawValueSpecificationTransformer, } from './transformation/pureGraph/from/V1_RawValueSpecificationTransformer.js';
import { V1_transformRuntime } from './transformation/pureGraph/from/V1_RuntimeTransformer.js';
import { V1_RawLambda } from './model/rawValueSpecification/V1_RawLambda.js';
import { V1_ExecuteInput, V1_TestDataGenerationExecutionInput, V1_TestDataGenerationExecutionWithSeedInput, } from './engine/execution/V1_ExecuteInput.js';
import { V1_buildValueSpecification } from './transformation/pureGraph/to/helpers/V1_ValueSpecificationBuilderHelper.js';
import { V1_transformRootValueSpecification } from './transformation/pureGraph/from/V1_ValueSpecificationTransformer.js';
import { V1_Profile } from './model/packageableElements/domain/V1_Profile.js';
import { V1_Class } from './model/packageableElements/domain/V1_Class.js';
import { V1_Enumeration } from './model/packageableElements/domain/V1_Enumeration.js';
import { V1_Association } from './model/packageableElements/domain/V1_Association.js';
import { V1_Measure } from './model/packageableElements/domain/V1_Measure.js';
import { V1_Store } from './model/packageableElements/store/V1_Store.js';
import { V1_Service } from './model/packageableElements/service/V1_Service.js';
import { V1_PackageableRuntime } from './model/packageableElements/runtime/V1_PackageableRuntime.js';
import { V1_PackageableConnection } from './model/packageableElements/connection/V1_PackageableConnection.js';
import { V1_FileGenerationSpecification } from './model/packageableElements/fileGeneration/V1_FileGenerationSpecification.js';
import { V1_SectionIndex } from './model/packageableElements/section/V1_SectionIndex.js';
import { V1_GenerationSpecification } from './model/packageableElements/generationSpecification/V1_GenerationSpecification.js';
import { V1_Mapping } from './model/packageableElements/mapping/V1_Mapping.js';
import { V1_ConcreteFunctionDefinition } from './model/packageableElements/function/V1_ConcreteFunctionDefinition.js';
import { V1_PureModelContextComposite } from './model/context/V1_PureModelContextComposite.js';
import { V1_LegendSDLC } from './model/context/V1_SDLC.js';
import { V1_Protocol } from './model/V1_Protocol.js';
import { V1_GraphBuilderExtensions } from './transformation/pureGraph/to/V1_GraphBuilderExtensions.js';
import { V1_DatabaseBuilderConfig, V1_DatabaseBuilderInput, V1_DatabasePattern, V1_TargetDatabase, } from './engine/generation/V1_DatabaseBuilderInput.js';
import { V1_transformRelationalDatabaseConnection } from './transformation/pureGraph/from/V1_ConnectionTransformer.js';
import { V1_FlatData } from './model/packageableElements/store/flatData/model/V1_FlatData.js';
import { V1_Database } from './model/packageableElements/store/relational/model/V1_Database.js';
import { V1_setupDatabaseSerialization } from './transformation/pureProtocol/serializationHelpers/V1_DatabaseSerializationHelper.js';
import { V1_setupEngineRuntimeSerialization, V1_setupLegacyRuntimeSerialization, } from './transformation/pureProtocol/serializationHelpers/V1_RuntimeSerializationHelper.js';
import { V1_GraphTransformerContextBuilder } from './transformation/pureGraph/from/V1_GraphTransformerContext.js';
import { V1_transformExecutionNode, V1_transformExecutionPlan, } from './transformation/pureGraph/from/executionPlan/V1_ExecutionPlanTransformer.js';
import { V1_deserializeExecutionPlan, V1_serializeExecutionNode, V1_serializeExecutionPlan, } from './transformation/pureProtocol/serializationHelpers/executionPlan/V1_ExecutionPlanSerializationHelper.js';
import { V1_buildExecutionPlan } from './transformation/pureGraph/to/V1_ExecutionPlanBuilder.js';
import { QueryExplicitExecutionContextInfo, QueryTaggedValue, } from '../../../../graph-manager/action/query/Query.js';
import { V1_buildQuery, V1_buildServiceRegistrationSuccess, V1_transformQuery, V1_buildGenerationOutput, V1_buildLightQuery, V1_transformQuerySearchSpecification, V1_buildSourceInformation, V1_buildExecutionContextInfo, } from './engine/V1_EngineHelper.js';
import { V1_buildExecutionResult } from './engine/execution/V1_ExecutionHelper.js';
import { ENTITY_PATH_DELIMITER, } from '@finos/legend-storage';
import { DependencyGraphBuilderError, GraphBuilderError, PureClientVersion, SystemGraphBuilderError, } from '../../../../graph-manager/GraphManagerUtils.js';
import { PackageableElementReference } from '../../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js';
import { V1_ExternalFormatModelGenerationInput } from './engine/externalFormat/V1_ExternalFormatModelGeneration.js';
import { V1_GenerateSchemaInput } from './engine/externalFormat/V1_GenerateSchemaInput.js';
import { createGraphBuilderReport, createGraphManagerOperationReport, } from '../../../GraphManagerStatistics.js';
import { V1_DataElement } from './model/packageableElements/data/V1_DataElement.js';
import { V1_RunTestsInput, V1_RunTestsTestableInput, } from './engine/test/V1_RunTestsInput.js';
import { V1_UniqueTestId } from './model/test/V1_UniqueTestId.js';
import { V1_buildTestsResult } from './engine/test/V1_RunTestsResult.js';
import {} from '../../../../graph/metamodel/pure/test/result/TestResult.js';
import { getNullableIDFromTestable, getNullableTestable, } from '../../../helpers/DSL_Data_GraphManagerHelper.js';
import { extractElementNameFromPath, extractPackagePathFromPath, pruneSourceInformation, } from '../../../../graph/MetaModelUtils.js';
import { V1_buildModelCoverageAnalysisResult, V1_MappingModelCoverageAnalysisInput, V1_MappingModelCoverageAnalysisResult, } from './engine/analytics/V1_MappingModelCoverageAnalysis.js';
import { deserialize } from 'serializr';
import { SchemaSet } from '../../../../graph/metamodel/pure/packageableElements/externalFormat/schemaSet/DSL_ExternalFormat_SchemaSet.js';
import { CompilationWarning } from '../../../action/compilation/CompilationWarning.js';
import { V1_transformParameterValue } from './transformation/pureGraph/from/V1_ServiceTransformer.js';
import { V1_transformModelUnit } from './transformation/pureGraph/from/V1_DSL_ExternalFormat_Transformer.js';
import { V1_LambdaReturnTypeInput } from './engine/compilation/V1_LambdaReturnType.js';
import { V1_ExecutionEnvironmentInstance } from './model/packageableElements/service/V1_ExecutionEnvironmentInstance.js';
import { V1_EntitlementReportAnalyticsInput, V1_StoreEntitlementAnalysisInput, V1_buildDatasetEntitlementReport, V1_buildDatasetSpecification, V1_transformDatasetSpecification, } from './engine/analytics/V1_StoreEntitlementAnalysis.js';
import { LegendSDLC, GraphEntities, } from '../../../../graph/GraphDataOrigin.js';
import { InMemoryGraphData, GraphDataWithOrigin, } from '../../../GraphData.js';
import { DEPRECATED__validate_MappingTest } from '../../../action/validation/DSL_Mapping_ValidationHelper.js';
import { V1_INTERNAL__UnknownPackageableElement } from './model/packageableElements/V1_INTERNAL__UnknownPackageableElement.js';
import { FunctionActivatorConfiguration } from '../../../action/functionActivator/FunctionActivatorConfiguration.js';
import { V1_FunctionActivatorInput } from './engine/functionActivator/V1_FunctionActivatorInput.js';
import { V1_FunctionActivator } from './model/packageableElements/function/V1_FunctionActivator.js';
import { V1_INTERNAL__UnknownFunctionActivator } from './model/packageableElements/function/V1_INTERNAL__UnknownFunctionActivator.js';
import { V1_DatabaseToModelGenerationInput } from './engine/relational/V1_DatabaseToModelGenerationInput.js';
import {} from '../../../../STO_Relational_Exports.js';
import { V1_RawSQLExecuteInput } from './engine/execution/V1_RawSQLExecuteInput.js';
import { V1_INTERNAL__UnknownStore } from './model/packageableElements/store/V1_INTERNAL__UnknownStore.js';
import { V1_ArtifactGenerationExtensionInput, V1_buildArtifactsByExtensionElement, } from './engine/generation/V1_ArtifactGenerationExtensionApi.js';
import { V1_TestDataGenerationInput } from './engine/service/V1_TestDataGenerationInput.js';
import { V1_buildTestDataGenerationResult } from './engine/service/V1_TestDataGenerationResult.js';
import { RelationalDatabaseTypeConfiguration } from '../../../action/relational/RelationalDatabaseTypeConfiguration.js';
import { V1_ColumnValuePair, V1_RowIdentifier, V1_TableRowIdentifiers, } from './engine/service/V1_TableRowIdentifiers.js';
import { V1_transformTablePointer } from './transformation/pureGraph/from/V1_DatabaseTransformer.js';
import { EngineError } from '../../../action/EngineError.js';
import { V1_SnowflakeApp } from './model/packageableElements/function/V1_SnowflakeApp.js';
import { V1_INTERNAL__UnknownElement } from './model/packageableElements/V1_INTERNAL__UnknownElement.js';
import { V1_HostedService } from './model/packageableElements/function/V1_HostedService.js';
import { V1_UserListOwnership } from './model/packageableElements/service/V1_ServiceOwnership.js';
import { V1_PureSingleExecution } from './model/packageableElements/service/V1_ServiceExecution.js';
import { V1_RuntimePointer, } from './model/packageableElements/runtime/V1_Runtime.js';
import { V1_buildDebugTestsResult } from './engine/test/V1_DebugTestsResult.js';
import { V1_CompleteCodeInput } from './engine/compilation/V1_CompleteCodeInput.js';
import { V1_QueryParameterValue } from './engine/query/V1_Query.js';
import { V1_Multiplicity } from './model/packageableElements/domain/V1_Multiplicity.js';
import { V1_buildFunctionSignature, V1_createGenericTypeWithElementPath, } from './helpers/V1_DomainHelper.js';
import { V1_DataProduct } from './model/packageableElements/dataProduct/V1_DataProduct.js';
class V1_PureModelContextDataIndex {
elements = [];
nativeElements = [];
associations = [];
classes = [];
enumerations = [];
functions = [];
functionActivators = [];
profiles = [];
measures = [];
stores = [];
mappings = [];
connections = [];
runtimes = [];
sectionIndices = [];
fileGenerations = [];
generationSpecifications = [];
dataElements = [];
services = [];
executionEnvironments = [];
products = [];
INTERNAL__UnknownElement = [];
INTERNAL__unknownElements = [];
otherElementsByBuilder = new Map();
}
const mergePureModelContextData = (...data) => {
const mergedData = new V1_PureModelContextData();
for (const _data of data) {
mergedData.elements = mergedData.elements.concat(_data.elements);
const rawDependencyEntities = (mergedData.INTERNAL__rawDependencyEntities ?? []).concat(_data.INTERNAL__rawDependencyEntities ?? []);
mergedData.INTERNAL__rawDependencyEntities = rawDependencyEntities.length
? rawDependencyEntities
: undefined;
mergedData.serializer = _data.serializer ?? mergedData.serializer;
mergedData.origin = _data.origin ?? mergedData.origin;
}
return mergedData;
};
export const V1_indexPureModelContextData = (report, data, extensions) => {
const index = new V1_PureModelContextDataIndex();
index.elements = data.elements;
const otherElementsByClass = new Map();
data.elements.forEach((el) => {
let isIndexedAsOtherElement = false;
if (el instanceof V1_INTERNAL__UnknownElement) {
index.INTERNAL__UnknownElement.push(el);
}
else if (el instanceof V1_INTERNAL__UnknownPackageableElement) {
index.INTERNAL__unknownElements.push(el);
}
else if (el instanceof V1_Association) {
index.associations.push(el);
}
else if (el instanceof V1_Class) {
index.classes.push(el);
}
else if (el instanceof V1_Enumeration) {
index.enumerations.push(el);
}
else if (el instanceof V1_ConcreteFunctionDefinition) {
index.functions.push(el);
}
else if (el instanceof V1_FunctionActivator) {
index.functionActivators.push(el);
}
else if (el instanceof V1_Profile) {
index.profiles.push(el);
}
else if (el instanceof V1_Measure) {
index.measures.push(el);
}
else if (el instanceof V1_Mapping) {
index.mappings.push(el);
}
else if (el instanceof V1_PackageableConnection) {
index.connections.push(el);
}
else if (el instanceof V1_PackageableRuntime) {
index.runtimes.push(el);
}
else if (el instanceof V1_Store) {
index.stores.push(el);
}
else if (el instanceof V1_SectionIndex) {
index.sectionIndices.push(el);
}
else if (el instanceof V1_Service) {
index.services.push(el);
}
else if (el instanceof V1_FileGenerationSpecification) {
index.fileGenerations.push(el);
}
else if (el instanceof V1_GenerationSpecification) {
index.generationSpecifications.push(el);
}
else if (el instanceof V1_DataElement) {
index.dataElements.push(el);
}
else if (el instanceof V1_ExecutionEnvironmentInstance) {
index.executionEnvironments.push(el);
}
else if (el instanceof V1_DataProduct) {
index.products.push(el);
}
else {
const clazz = getClass(el);
if (otherElementsByClass.has(clazz)) {
otherElementsByClass.get(clazz)?.push(el);
}
else {
otherElementsByClass.set(clazz, [el]);
}
isIndexedAsOtherElement = true;
}
// we index everything else as native
if (!isIndexedAsOtherElement) {
index.nativeElements.push(el);
}
});
otherElementsByClass.forEach((elements, _class) => {
const builder = extensions.getExtraBuilderForProtocolClassOrThrow(_class);
index.otherElementsByBuilder.set(builder, (index.otherElementsByBuilder.get(builder) ?? []).concat(elements));
});
// report
report.elementCount.total =
(report.elementCount.total ?? 0) + index.elements.length;
report.elementCount.other =
(report.elementCount.other ?? 0) +
otherElementsByClass.size +
index.fileGenerations.length +
index.generationSpecifications.length;
report.elementCount.sectionIndex =
(report.elementCount.sectionIndex ?? 0) + index.sectionIndices.length;
report.elementCount.association =
(report.elementCount.association ?? 0) + index.associations.length;
report.elementCount.class =
(report.elementCount.class ?? 0) + index.classes.length;
report.elementCount.enumeration =
(report.elementCount.enumeration ?? 0) + index.enumerations.length;
report.elementCount.function =
(report.elementCount.function ?? 0) + index.functions.length;
report.elementCount.functionActivators =
(report.elementCount.functionActivators ?? 0) +
index.functionActivators.length;
report.elementCount.profile =
(report.elementCount.profile ?? 0) + index.profiles.length;
report.elementCount.measure =
(report.elementCount.measure ?? 0) + index.measures.length;
report.elementCount.dataElement =
(report.elementCount.dataElement ?? 0) + index.dataElements.length;
report.elementCount.store =
(report.elementCount.store ?? 0) + index.stores.length;
report.elementCount.mapping =
(report.elementCount.mapping ?? 0) + index.mappings.length;
report.elementCount.connection =
(report.elementCount.connection ?? 0) + index.connections.length;
report.elementCount.runtime =
(report.elementCount.runtime ?? 0) + index.runtimes.length;
report.elementCount.service =
(report.elementCount.service ?? 0) + index.services.length;
report.elementCount.executionEnvironment =
(report.elementCount.executionEnvironment ?? 0) +
index.executionEnvironments.length;
report.elementCount.unknown =
(report.elementCount.unknown ?? 0) + index.INTERNAL__unknownElements.length;
return index;
};
export class V1_PureGraphManager extends AbstractPureGraphManager {
elementClassifierPathMap = new Map();
subtypeInfo = {
storeSubtypes: [],
functionActivatorSubtypes: [],
};
// Pure Client Version represent the version of the pure protocol.
// Most Engine APIs will interrupt an undefined pure client version to mean
// use the latest production version of the protocol i.e V20_0_0, while version
// `VX_X_X` represents the version in development and used for testing
static PURE_PROTOCOL_NAME = 'pure';
static DEV_PROTOCOL_VERSION = PureClientVersion.VX_X_X;
static PROD_PROTOCOL_VERSION = undefined;
engine;
graphBuilderExtensions;
constructor(pluginManager, logService, engine) {
super(pluginManager, logService);
this.engine = engine ?? new V1_RemoteEngine({}, logService);
// setup plugins
this.graphBuilderExtensions = new V1_GraphBuilderExtensions(this.pluginManager.getPureProtocolProcessorPlugins());
}
TEMPORARY__getEngineConfig() {
return this.engine.config;
}
async initialize(config, options) {
this.engine =
options?.engine ??
new V1_RemoteEngine(config.clientConfig, this.logService);
// TODO: improve abstraction so that we do not need to access the engine server client directly
if (this.engine instanceof V1_RemoteEngine) {
this.engine
.getEngineServerClient()
.setTracerService(options?.tracerService ?? new TracerService());
}
if (!options?.disableGraphConfiguration) {
// TODO: should probably be moved into each store's own initialize method
await Promise.all([
this.engine.setup(config),
this.configureElementClassifierPathMap(config),
this.configureSubtypeInfoMap(config),
]);
// setup serialization plugins
V1_setupPureModelContextDataSerialization(this.pluginManager.getPureProtocolProcessorPlugins(), this.subtypeInfo, this.elementClassifierPathMap);
V1_setupDatabaseSerialization(this.pluginManager.getPureProtocolProcessorPlugins());
V1_setupEngineRuntimeSerialization(this.pluginManager.getPureProtocolProcessorPlugins());
V1_setupLegacyRuntimeSerialization(this.pluginManager.getPureProtocolProcessorPlugins());
}
}
async configureElementClassifierPathMap(config) {
const classifierPathMapEntries = config.TEMPORARY__classifierPathMapping ??
(await this.engine.getClassifierPathMapping());
classifierPathMapEntries.forEach((entry) => {
this.elementClassifierPathMap.set(entry.type, entry.classifierPath);
});
}
async configureSubtypeInfoMap(config) {
const subtypeInfo = config.TEMPORARY__subtypeInfo ?? (await this.engine.getSubtypeInfo());
this.subtypeInfo.storeSubtypes = subtypeInfo.storeSubtypes;
this.subtypeInfo.functionActivatorSubtypes =
subtypeInfo.functionActivatorSubtypes;
}
// --------------------------------------------- Generic ---------------------------------------------
getSupportedProtocolVersion() {
return PureClientVersion.V1_0_0;
}
getElementEntities(entities) {
return entities.filter((entity) => entity.classifierPath !== CORE_PURE_PATH.SECTION_INDEX);
}
// --------------------------------------------- Graph Builder ---------------------------------------------
async buildSystem(coreModel, systemModel, buildState, options, _report) {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
// Create a dummy graph for system processing. This is to ensure system model does not depend on the main graph
const graph = new PureModel(coreModel, systemModel, this.pluginManager.getPureGraphPlugins());
try {
// deserialize
buildState.setMessage(`Collecting and deserializing elements...`);
const systemData = mergePureModelContextData(V1_deserializePureModelContextData(V1_CORE_SYSTEM_MODELS), ...this.pluginManager
.getPureProtocolProcessorPlugins()
.flatMap((plugin) => plugin.V1_getExtraSystemModels?.() ?? [])
.map((modelContextData) => V1_deserializePureModelContextData(modelContextData)));
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS);
// prepare build inputs
const buildInputs = [
{
model: systemModel,
data: V1_indexPureModelContextData(report, systemData, this.graphBuilderExtensions),
},
];
// build
await this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, buildState, options);
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
}
catch (error) {
assertErrorThrown(error);
buildState.fail();
this.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), error);
throw new SystemGraphBuilderError(error);
}
finally {
buildState.setMessage(undefined);
}
}
async buildDependencies(coreModel, systemModel, dependencyManager, dependencyEntitiesIndex, buildState, options, _report) {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
// Create a dummy graph for system processing. This is to ensure dependency models do not depend on the main graph
const graph = new PureModel(coreModel, systemModel, this.pluginManager.getPureGraphPlugins());
graph.dependencyManager = dependencyManager;
try {
dependencyManager.initialize(dependencyEntitiesIndex);
// deserialize
buildState.setMessage(`Partitioning and deserializing elements...`);
const dependencyGraphDataIndex = new Map();
await Promise.all(Array.from(dependencyEntitiesIndex.entries()).map(([dependencyKey, entitiesWithOrigin]) => {
const projectModelData = new V1_PureModelContextData();
dependencyGraphDataIndex.set(dependencyKey, projectModelData);
return V1_entitiesToPureModelContextData(entitiesWithOrigin.entities, projectModelData, this.pluginManager.getPureProtocolProcessorPlugins(), this.subtypeInfo, this.elementClassifierPathMap);
}));
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS);
// prepare build inputs
const buildInputs = Array.from(dependencyGraphDataIndex.entries()).map(([dependencyKey, dependencyData]) => ({
model: graph.dependencyManager.getModel(dependencyKey),
data: V1_indexPureModelContextData(report, dependencyData, this.graphBuilderExtensions),
}));
// build
await this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, buildState, options);
// set dependency manager graph origin to entities
if (dependencyManager.origin === undefined) {
dependencyManager.setOrigin(new GraphEntities(Array.from(dependencyEntitiesIndex.values())
.map((e) => e.entities)
.flat()));
}
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
}
catch (error) {
assertErrorThrown(error);
this.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), error);
buildState.fail();
throw new DependencyGraphBuilderError(error);
}
finally {
buildState.setMessage(undefined);
}
}
async buildGraph(graph, entities, buildState, options, _report) {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
try {
// deserialize
buildState.setMessage(`Deserializing elements...`);
const data = new V1_PureModelContextData();
await V1_entitiesToPureModelContextData(entities, data, this.pluginManager.getPureProtocolProcessorPlugins(), this.subtypeInfo, this.elementClassifierPathMap);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS);
// prepare build inputs
const buildInputs = [
{
model: graph,
data: V1_indexPureModelContextData(report, data, this.graphBuilderExtensions),
},
];
// build
await this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, buildState, options);
/**
* For now, we delete the section index. We are able to read both resolved and unresolved element paths
* but when we write (serialize) we write only resolved paths. In the future once the issue with dependency is solved we will
* perserve the element path both resolved and unresolved
*/
if (!options?.TEMPORARY__preserveSectionIndex) {
graph.TEMPORARY__deleteOwnSectionIndex();
}
if (options?.origin) {
graph.setOrigin(options.origin);
}
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
}
catch (error) {
assertErrorThrown(error);
this.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), error);
buildState.fail();
/**
* Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
* But we might want to rethink this decision in the future and throw appropriate type of error
*/
throw error instanceof GraphBuilderError
? error
: new GraphBuilderError(error);
}
finally {
buildState.setMessage(undefined);
}
}
async buildLightGraph(graph, entities, buildState, options, _report) {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
try {
// deserialize
buildState.setMessage(`Deserializing elements...`);
const data = new V1_PureModelContextData();
await V1_entitiesToPureModelContextData(entities, data, this.pluginManager.getPureProtocolProcessorPlugins(), this.subtypeInfo, this.elementClassifierPathMap);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS);
// prepare build inputs
const buildInputs = [
{
model: graph,
data: V1_indexPureModelContextData(report, data, this.graphBuilderExtensions),
},
];
// build
await this.buildLightGraphFromInputs(graph, buildInputs, report, stopWatch, buildState, options);
/**
* For now, we delete the section index. We are able to read both resolved and unresolved element paths
* but when we write (serialize) we write only resolved paths. In the future once the issue with dependency is solved we will
* perserve the element path both resolved and unresolved
*/
if (!options?.TEMPORARY__preserveSectionIndex) {
graph.TEMPORARY__deleteOwnSectionIndex();
}
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
}
catch (error) {
assertErrorThrown(error);
this.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), error);
buildState.fail();
/**
* Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
* But we might want to rethink this decision in the future and throw appropriate type of error
*/
throw error instanceof GraphBuilderError
? error
: new GraphBuilderError(error);
}
finally {
buildState.setMessage(undefined);
}
}
async buildGenerations(graph, generatedEntities, buildState, options, _report) {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
const generatedModel = graph.generationModel;
buildState.reset();
try {
// deserialize
buildState.setMessage(`Deserializing elements...`);
const generationGraphDataIndex = new Map();
await Promise.all(Array.from(generatedEntities.entries()).map(([generationParentPath, entities]) => {
const generatedData = new V1_PureModelContextData();
generationGraphDataIndex.set(generationParentPath, generatedData);
return V1_entitiesToPureModelContextData(entities, generatedData, this.pluginManager.getPureProtocolProcessorPlugins(), this.subtypeInfo, this.elementClassifierPathMap);
}));
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS);
// prepare build inputs
const buildInputs = Array.from(generationGraphDataIndex.entries()).map(([generationParentPath, generatedData]) => ({
model: generatedModel,
data: V1_indexPureModelContextData(report, generatedData, this.graphBuilderExtensions),
}));
// build
await this.buildGraphFromInputs(graph, buildInputs, report, stopWatch, buildState, options);
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
}
catch (error) {
assertErrorThrown(error);
this.logService.error(LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), error);
buildState.fail();
/**
* Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
* But we might want to rethink this decision in the future and throw appropriate type of error
*/
throw error instanceof GraphBuilderError
? error
: new GraphBuilderError(error);
}
finally {
buildState.setMessage(undefined);
}
}
async buildGraphFromInputs(graph, inputs, report, stopWatch, graphBuilderState, options) {
// index
graphBuilderState.setMessage(`Indexing ${report.elementCount.total} elements...`);
await this.initializeAndIndexElements(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS);
// build section index
graphBuilderState.setMessage(`Building section indices...`);
await this.buildSectionIndices(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_SECTION_INDICES__SUCCESS);
// build types
graphBuilderState.setMessage(`Building domain models...`);
await this.buildTypes(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS);
// build stores
graphBuilderState.setMessage(`Building stores...`);
await this.buildStores(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_STORES__SUCCESS);
// build mappings
graphBuilderState.setMessage(`Building mappings...`);
await this.buildMappings(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_MAPPINGS__SUCCESS);
// build connections and runtimes
graphBuilderState.setMessage(`Building connections and runtimes...`);
await this.buildConnectionsAndRuntimes(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_CONNECTIONS_AND_RUNTIMES__SUCCESS);
// build function activators
graphBuilderState.setMessage(`Building function activators...`);
await this.buildFunctionActivators(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS);
// build services
graphBuilderState.setMessage(`Building services and execution environments...`);
await this.buildServices(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_SERVICES__SUCCESS);
// build data elements
graphBuilderState.setMessage(`Building data elements...`);
await this.buildDataElements(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DATA_ELEMENTS__SUCCESS);
// build data products
graphBuilderState.setMessage(`Building data products...`);
await this.buildDataProducts(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DATA_PRODUCTS__SUCCESS);
// build other elements
graphBuilderState.setMessage(`Building other elements...`);
await this.buildFileGenerations(graph, inputs, options);
await this.buildGenerationSpecifications(graph, inputs, options);
await this.buildOtherElements(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS);
}
async buildLightGraphFromInputs(graph, inputs, report, stopWatch, graphBuilderState, options) {
// index
graphBuilderState.setMessage(`Indexing ${report.elementCount.total} elements...`);
await this.initializeAndIndexElements(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS);
}
getBuilderContext(graph, currentSubGraph, element, options) {
return new V1_GraphBuilderContextBuilder(graph, currentSubGraph, this.graphBuilderExtensions, this.logService, options)
.withElement(element)
.build();
}
/**
* This will run the first pass builder for all elements and index them.
* This process is needed so other core processes such as building the section indices
* or building processes that relies on the `existence` of other elements to refer to them,
* but not necessarily use them.
*
* NOTE: We aim to not do anything more than running the first pass and indexing the first pass.
*/
async initializeAndIndexElements(graph, inputs, options) {
// create the element path cache for faster duplication check
// NOTE: We base on the assumption here that our graph building is cascading, i.e.
// it first builds core, system, dependencies, graph, and generation.
// this way, as we build the a graph, we know the next step's duplication check
// has path cache consisting of all element from its base graphs
const elementPathCache = new Set(graph.allElements.map((el) => el.path));
await Promise.all(inputs.flatMap(async (input) => {
// create the package cache
const packageCache = new Map();
await Promise.all(input.data.nativeElements.map((element) => {
return this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFirstPassBuilder(this.getBuilderContext(graph, input.model, element, options), packageCache, elementPathCache));
}));
await Promise.all(this.graphBuilderExtensions.sortedExtraElementBuilders.flatMap((builder) => (input.data.otherElementsByBuilder.get(builder) ?? []).map((element) => {
return this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFirstPassBuilder(this.getBuilderContext(graph, input.model, element, options), packageCache, elementPathCache));
})));
}));
}
async buildTypes(graph, inputs, options) {
// Second pass
await Promise.all(inputs.flatMap((input) => input.data.profiles.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.classes.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.enumerations.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.measures.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.functions.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
// Third pass
await Promise.all(inputs.flatMap((input) => input.data.classes.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementThirdPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.associations.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementThirdPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
// Fourth Pass
await Promise.all(inputs.flatMap((input) => input.data.classes.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFourthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.associations.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFourthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
// Fifth pass
await Promise.all(inputs.flatMap((input) => input.data.classes.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFifthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildFunctionActivators(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.functionActivators.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildStores(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.stores.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.stores.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementThirdPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.stores.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFourthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.stores.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFifthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildMappings(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.mappings.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.mappings.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementThirdPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.mappings.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFourthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildConnectionsAndRuntimes(graph, inputs, options) {
// NOTE: connections must be built before runtimes
await Promise.all(inputs.flatMap((input) => input.data.connections.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.runtimes.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildServices(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.services.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => input.data.executionEnvironments.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildDataElements(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.dataElements.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildDataProducts(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.products.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildFileGenerations(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.fileGenerations.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildGenerationSpecifications(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.generationSpecifications.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildSectionIndices(graph, inputs, options) {
await Promise.all(inputs.flatMap((input) => input.data.sectionIndices.map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
}
async buildOtherElements(graph, inputs, options) {
await Promise.all(this.graphBuilderExtensions.sortedExtraElementBuilders.map(async (builder) => {
await Promise.all(inputs.flatMap((input) => (input.data.otherElementsByBuilder.get(builder) ?? []).map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementSecondPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => (input.data.otherElementsByBuilder.get(builder) ?? []).map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementThirdPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => (input.data.otherElementsByBuilder.get(builder) ?? []).map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFourthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));
await Promise.all(inputs.flatMap((input) => (input.data.otherElementsByBuilder.get(builder) ?? []).map((element) => this.visitWithGraphBuilderErrorHandling(element, new V1_ElementFifthPassBuilder(this.getBuilderContext(graph, input.model, element, options))))));