@finos/legend-graph
Version:
Legend graph and graph manager
1,256 lines (1,171 loc) • 158 kB
text/typescript
/**
* Copyright (c) 2020-present, Goldman Sachs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GRAPH_MANAGER_EVENT } from '../../../../__lib__/GraphManagerEvent.js';
import {
CORE_PURE_PATH,
PackageableElementPointerType,
} from '../../../../graph/MetaModelConst.js';
import {
type Clazz,
type ContentType,
type LogService,
type PlainObject,
type ServerClientConfig,
ActionState,
TracerService,
LogEvent,
getClass,
guaranteeNonNullable,
UnsupportedOperationError,
assertErrorThrown,
promisify,
StopWatch,
isNonNullable,
filterByType,
isString,
assertNonEmptyString,
uniq,
guaranteeType,
guaranteeNonEmptyString,
uuid,
} from '@finos/legend-shared';
import type { TEMPORARY__AbstractEngineConfig } from '../../../../graph-manager/action/TEMPORARY__AbstractEngineConfig.js';
import {
AbstractPureGraphManager,
type TEMPORARY__EngineSetupConfig,
type GraphBuilderOptions,
type ExecutionOptions,
type ServiceRegistrationOptions,
} from '../../../../graph-manager/AbstractPureGraphManager.js';
import type { Mapping } from '../../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js';
import type { Runtime } from '../../../../graph/metamodel/pure/packageableElements/runtime/Runtime.js';
import type { PackageableElement } from '../../../../graph/metamodel/pure/packageableElements/PackageableElement.js';
import {
type SystemModel,
type CoreModel,
PureModel,
type GraphTextInputOption,
} from '../../../../graph/PureModel.js';
import type { BasicModel } from '../../../../graph/BasicModel.js';
import type { DependencyManager } from '../../../../graph/DependencyManager.js';
import type { Class } from '../../../../graph/metamodel/pure/packageableElements/domain/Class.js';
import { RawLambda } from '../../../../graph/metamodel/pure/rawValueSpecification/RawLambda.js';
import type { RawValueSpecification } from '../../../../graph/metamodel/pure/rawValueSpecification/RawValueSpecification.js';
import type { FileGenerationSpecification } from '../../../../graph/metamodel/pure/packageableElements/fileGeneration/FileGenerationSpecification.js';
import type {
GenerationConfigurationDescription,
GenerationMode,
} from '../../../../graph-manager/action/generation/GenerationConfigurationDescription.js';
import {
type ServiceRegistrationResult,
ServiceRegistrationSuccess,
ServiceRegistrationFail,
} from '../../../../graph-manager/action/service/ServiceRegistrationResult.js';
import type { GenerationOutput } from '../../../../graph-manager/action/generation/GenerationOutput.js';
import type { ValueSpecification } from '../../../../graph/metamodel/pure/valueSpecification/ValueSpecification.js';
import { ServiceExecutionMode } from '../../../../graph-manager/action/service/ServiceExecutionMode.js';
import {
PureMultiExecution,
PureSingleExecution,
} from '../../../../graph/metamodel/pure/packageableElements/service/ServiceExecution.js';
import {
V1_deserializeRawValueSpecification,
V1_serializeRawValueSpecification,
} from './transformation/pureProtocol/serializationHelpers/V1_RawValueSpecificationSerializationHelper.js';
import {
V1_serializeValueSpecification,
V1_deserializeValueSpecification,
} from './transformation/pureProtocol/serializationHelpers/V1_ValueSpecificationSerializer.js';
import V1_CORE_SYSTEM_MODELS from './V1_Core_SystemModels.json' with { type: 'json' };
import { V1_serializePackageableElement } from './transformation/pureProtocol/V1_PackageableElementSerialization.js';
import {
V1_entitiesToPureModelContextData,
V1_serializePureModelContext,
V1_deserializePureModelContextData,
V1_setupPureModelContextDataSerialization,
} from './transformation/pureProtocol/V1_PureProtocolSerialization.js';
import { V1_PureModelContextData } from './model/context/V1_PureModelContextData.js';
import {
type V1_PackageableElement,
type V1_PackageableElementVisitor,
V1_PackageableElementPointer,
} from './model/packageableElements/V1_PackageableElement.js';
import { V1_ElementFirstPassBuilder } from './transformation/pureGraph/to/V1_ElementFirstPassBuilder.js';
import { V1_ElementSecondPassBuilder } from './transformation/pureGraph/to/V1_ElementSecondPassBuilder.js';
import { V1_ElementThirdPassBuilder } from './transformation/pureGraph/to/V1_ElementThirdPassBuilder.js';
import { V1_ElementFourthPassBuilder } from './transformation/pureGraph/to/V1_ElementFourthPassBuilder.js';
import { V1_ElementFifthPassBuilder } from './transformation/pureGraph/to/V1_ElementFifthPassBuilder.js';
import { V1_RawValueSpecificationBuilder } from './transformation/pureGraph/to/V1_RawValueSpecificationBuilder.js';
import { V1_RawBaseExecutionContext } from './model/rawValueSpecification/V1_RawExecutionContext.js';
import {
type V1_GraphBuilderContext,
V1_GraphBuilderContextBuilder,
} from './transformation/pureGraph/to/V1_GraphBuilderContext.js';
import { V1_PureModelContextPointer } from './model/context/V1_PureModelContextPointer.js';
import { V1_RemoteEngine } from './engine/V1_RemoteEngine.js';
import { V1_transformPackageableElement } from './transformation/pureGraph/from/V1_PackageableElementTransformer.js';
import {
V1_transformRawLambda,
V1_RawValueSpecificationTransformer,
} from './transformation/pureGraph/from/V1_RawValueSpecificationTransformer.js';
import { V1_transformRuntime } from './transformation/pureGraph/from/V1_RuntimeTransformer.js';
import { V1_RawLambda } from './model/rawValueSpecification/V1_RawLambda.js';
import {
V1_ExecuteInput,
V1_TestDataGenerationExecutionInput,
V1_TestDataGenerationExecutionWithSeedInput,
} from './engine/execution/V1_ExecuteInput.js';
import type { V1_PureModelContextGenerationInput } from './engine/import/V1_PureModelContextGenerationInput.js';
import { V1_buildValueSpecification } from './transformation/pureGraph/to/helpers/V1_ValueSpecificationBuilderHelper.js';
import { V1_transformRootValueSpecification } from './transformation/pureGraph/from/V1_ValueSpecificationTransformer.js';
import { V1_Profile } from './model/packageableElements/domain/V1_Profile.js';
import { V1_Class } from './model/packageableElements/domain/V1_Class.js';
import { V1_Enumeration } from './model/packageableElements/domain/V1_Enumeration.js';
import { V1_Association } from './model/packageableElements/domain/V1_Association.js';
import { V1_Measure } from './model/packageableElements/domain/V1_Measure.js';
import { V1_Store } from './model/packageableElements/store/V1_Store.js';
import { V1_Service } from './model/packageableElements/service/V1_Service.js';
import { V1_PackageableRuntime } from './model/packageableElements/runtime/V1_PackageableRuntime.js';
import { V1_PackageableConnection } from './model/packageableElements/connection/V1_PackageableConnection.js';
import { V1_FileGenerationSpecification } from './model/packageableElements/fileGeneration/V1_FileGenerationSpecification.js';
import { V1_SectionIndex } from './model/packageableElements/section/V1_SectionIndex.js';
import { V1_GenerationSpecification } from './model/packageableElements/generationSpecification/V1_GenerationSpecification.js';
import { V1_Mapping } from './model/packageableElements/mapping/V1_Mapping.js';
import { V1_ConcreteFunctionDefinition } from './model/packageableElements/function/V1_ConcreteFunctionDefinition.js';
import { V1_PureModelContextComposite } from './model/context/V1_PureModelContextComposite.js';
import { V1_LegendSDLC } from './model/context/V1_SDLC.js';
import { V1_Protocol } from './model/V1_Protocol.js';
import type { V1_PureModelContext } from './model/context/V1_PureModelContext.js';
import type { V1_ElementBuilder } from './transformation/pureGraph/to/V1_ElementBuilder.js';
import { V1_GraphBuilderExtensions } from './transformation/pureGraph/to/V1_GraphBuilderExtensions.js';
import type {
DatabaseBuilderInput,
DatabasePattern,
} from '../../../../graph-manager/action/generation/DatabaseBuilderInput.js';
import {
V1_DatabaseBuilderConfig,
V1_DatabaseBuilderInput,
V1_DatabasePattern,
V1_TargetDatabase,
} from './engine/generation/V1_DatabaseBuilderInput.js';
import { V1_transformRelationalDatabaseConnection } from './transformation/pureGraph/from/V1_ConnectionTransformer.js';
import { V1_FlatData } from './model/packageableElements/store/flatData/model/V1_FlatData.js';
import { V1_Database } from './model/packageableElements/store/relational/model/V1_Database.js';
import { V1_setupDatabaseSerialization } from './transformation/pureProtocol/serializationHelpers/V1_DatabaseSerializationHelper.js';
import {
V1_setupEngineRuntimeSerialization,
V1_setupLegacyRuntimeSerialization,
} from './transformation/pureProtocol/serializationHelpers/V1_RuntimeSerializationHelper.js';
import type { DSL_Generation_PureProtocolProcessorPlugin_Extension } from '../extensions/DSL_Generation_PureProtocolProcessorPlugin_Extension.js';
import type { RawRelationalOperationElement } from '../../../../graph/metamodel/pure/packageableElements/store/relational/model/RawRelationalOperationElement.js';
import { V1_GraphTransformerContextBuilder } from './transformation/pureGraph/from/V1_GraphTransformerContext.js';
import type {
ExecutionPlan,
RawExecutionPlan,
} from '../../../../graph/metamodel/pure/executionPlan/ExecutionPlan.js';
import type { V1_ExecutionNode } from './model/executionPlan/nodes/V1_ExecutionNode.js';
import type { ExecutionNode } from '../../../../graph/metamodel/pure/executionPlan/nodes/ExecutionNode.js';
import type { V1_ExecutionPlan } from './model/executionPlan/V1_ExecutionPlan.js';
import {
V1_transformExecutionNode,
V1_transformExecutionPlan,
} from './transformation/pureGraph/from/executionPlan/V1_ExecutionPlanTransformer.js';
import {
V1_deserializeExecutionPlan,
V1_serializeExecutionNode,
V1_serializeExecutionPlan,
} from './transformation/pureProtocol/serializationHelpers/executionPlan/V1_ExecutionPlanSerializationHelper.js';
import { V1_buildExecutionPlan } from './transformation/pureGraph/to/V1_ExecutionPlanBuilder.js';
import {
type LightQuery,
type Query,
QueryExplicitExecutionContextInfo,
type QueryInfo,
QueryTaggedValue,
} from '../../../../graph-manager/action/query/Query.js';
import {
V1_buildQuery,
V1_buildServiceRegistrationSuccess,
V1_transformQuery,
V1_buildGenerationOutput,
V1_buildLightQuery,
V1_transformQuerySearchSpecification,
V1_buildSourceInformation,
V1_buildExecutionContextInfo,
} from './engine/V1_EngineHelper.js';
import { V1_buildExecutionResult } from './engine/execution/V1_ExecutionHelper.js';
import {
type Entity,
type EntitiesWithOrigin,
ENTITY_PATH_DELIMITER,
} from '@finos/legend-storage';
import {
DependencyGraphBuilderError,
GraphBuilderError,
PureClientVersion,
SystemGraphBuilderError,
} from '../../../../graph-manager/GraphManagerUtils.js';
import { PackageableElementReference } from '../../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js';
import type { GraphManagerPluginManager } from '../../../GraphManagerPluginManager.js';
import type { QuerySearchSpecification } from '../../../../graph-manager/action/query/QuerySearchSpecification.js';
import type { ExternalFormatDescription } from '../../../../graph-manager/action/externalFormat/ExternalFormatDescription.js';
import type { ConfigurationProperty } from '../../../../graph/metamodel/pure/packageableElements/fileGeneration/ConfigurationProperty.js';
import { V1_ExternalFormatModelGenerationInput } from './engine/externalFormat/V1_ExternalFormatModelGeneration.js';
import { V1_GenerateSchemaInput } from './engine/externalFormat/V1_GenerateSchemaInput.js';
import {
createGraphBuilderReport,
createGraphManagerOperationReport,
type GraphManagerOperationReport,
} from '../../../GraphManagerStatistics.js';
import type { Package } from '../../../../graph/metamodel/pure/packageableElements/domain/Package.js';
import { V1_DataElement } from './model/packageableElements/data/V1_DataElement.js';
import {
V1_RunTestsInput,
V1_RunTestsTestableInput,
} from './engine/test/V1_RunTestsInput.js';
import { V1_UniqueTestId } from './model/test/V1_UniqueTestId.js';
import type { RunTestsTestableInput } from '../../../../graph/metamodel/pure/test/result/RunTestsTestableInput.js';
import { V1_buildTestsResult } from './engine/test/V1_RunTestsResult.js';
import { type TestResult } from '../../../../graph/metamodel/pure/test/result/TestResult.js';
import type { Testable } from '../../../../graph/metamodel/pure/test/Testable.js';
import {
getNullableIDFromTestable,
getNullableTestable,
} from '../../../helpers/DSL_Data_GraphManagerHelper.js';
import {
extractElementNameFromPath,
extractPackagePathFromPath,
pruneSourceInformation,
} from '../../../../graph/MetaModelUtils.js';
import {
V1_buildModelCoverageAnalysisResult,
V1_MappingModelCoverageAnalysisInput,
V1_MappingModelCoverageAnalysisResult,
} from './engine/analytics/V1_MappingModelCoverageAnalysis.js';
import type {
MappingModelCoverageAnalysisResult,
RawMappingModelCoverageAnalysisResult,
} from '../../../../graph-manager/action/analytics/MappingModelCoverageAnalysis.js';
import { deserialize } from 'serializr';
import { SchemaSet } from '../../../../graph/metamodel/pure/packageableElements/externalFormat/schemaSet/DSL_ExternalFormat_SchemaSet.js';
import type {
CompilationResult,
TextCompilationResult,
} from '../../../action/compilation/CompilationResult.js';
import { CompilationWarning } from '../../../action/compilation/CompilationWarning.js';
import { V1_transformParameterValue } from './transformation/pureGraph/from/V1_ServiceTransformer.js';
import { V1_transformModelUnit } from './transformation/pureGraph/from/V1_DSL_ExternalFormat_Transformer.js';
import type { ModelUnit } from '../../../../graph/metamodel/pure/packageableElements/externalFormat/store/DSL_ExternalFormat_ModelUnit.js';
import { V1_LambdaReturnTypeInput } from './engine/compilation/V1_LambdaReturnType.js';
import type { ParameterValue } from '../../../../graph/metamodel/pure/packageableElements/service/ParameterValue.js';
import type { Service } from '../../../../graph/metamodel/pure/packageableElements/service/Service.js';
import { V1_ExecutionEnvironmentInstance } from './model/packageableElements/service/V1_ExecutionEnvironmentInstance.js';
import {
V1_EntitlementReportAnalyticsInput,
V1_StoreEntitlementAnalysisInput,
V1_buildDatasetEntitlementReport,
V1_buildDatasetSpecification,
V1_transformDatasetSpecification,
} from './engine/analytics/V1_StoreEntitlementAnalysis.js';
import type {
DatasetEntitlementReport,
DatasetSpecification,
} from '../../../action/analytics/StoreEntitlementAnalysis.js';
import {
LegendSDLC,
type GraphDataOrigin,
GraphEntities,
} from '../../../../graph/GraphDataOrigin.js';
import {
InMemoryGraphData,
type GraphData,
GraphDataWithOrigin,
} from '../../../GraphData.js';
import type { DEPRECATED__MappingTest } from '../../../../graph/metamodel/pure/packageableElements/mapping/DEPRECATED__MappingTest.js';
import { DEPRECATED__validate_MappingTest } from '../../../action/validation/DSL_Mapping_ValidationHelper.js';
import { V1_INTERNAL__UnknownPackageableElement } from './model/packageableElements/V1_INTERNAL__UnknownPackageableElement.js';
import type { SourceInformation } from '../../../action/SourceInformation.js';
import type { V1_SourceInformation } from './model/V1_SourceInformation.js';
import type { FunctionActivator } from '../../../../graph/metamodel/pure/packageableElements/function/FunctionActivator.js';
import { FunctionActivatorConfiguration } from '../../../action/functionActivator/FunctionActivatorConfiguration.js';
import { V1_FunctionActivatorInput } from './engine/functionActivator/V1_FunctionActivatorInput.js';
import { V1_FunctionActivator } from './model/packageableElements/function/V1_FunctionActivator.js';
import { V1_INTERNAL__UnknownFunctionActivator } from './model/packageableElements/function/V1_INTERNAL__UnknownFunctionActivator.js';
import { V1_DatabaseToModelGenerationInput } from './engine/relational/V1_DatabaseToModelGenerationInput.js';
import { type RelationalDatabaseConnection } from '../../../../STO_Relational_Exports.js';
import { V1_RawSQLExecuteInput } from './engine/execution/V1_RawSQLExecuteInput.js';
import type { SubtypeInfo } from '../../../action/protocol/ProtocolInfo.js';
import { V1_INTERNAL__UnknownStore } from './model/packageableElements/store/V1_INTERNAL__UnknownStore.js';
import type { V1_ValueSpecification } from './model/valueSpecification/V1_ValueSpecification.js';
import type { V1_GrammarParserBatchInputEntry } from './engine/V1_EngineServerClient.js';
import type { ArtifactGenerationExtensionResult } from '../../../action/generation/ArtifactGenerationExtensionResult.js';
import {
V1_ArtifactGenerationExtensionInput,
V1_buildArtifactsByExtensionElement,
} from './engine/generation/V1_ArtifactGenerationExtensionApi.js';
import type { V1_RawValueSpecification } from './model/rawValueSpecification/V1_RawValueSpecification.js';
import { V1_TestDataGenerationInput } from './engine/service/V1_TestDataGenerationInput.js';
import type { TestDataGenerationResult } from '../../../../graph/metamodel/pure/packageableElements/service/TestGenerationResult.js';
import { V1_buildTestDataGenerationResult } from './engine/service/V1_TestDataGenerationResult.js';
import { RelationalDatabaseTypeConfiguration } from '../../../action/relational/RelationalDatabaseTypeConfiguration.js';
import type { TableRowIdentifiers } from '../../../../graph/metamodel/pure/packageableElements/service/TableRowIdentifiers.js';
import {
V1_ColumnValuePair,
V1_RowIdentifier,
V1_TableRowIdentifiers,
} from './engine/service/V1_TableRowIdentifiers.js';
import { V1_transformTablePointer } from './transformation/pureGraph/from/V1_DatabaseTransformer.js';
import { EngineError } from '../../../action/EngineError.js';
import { V1_SnowflakeApp } from './model/packageableElements/function/V1_SnowflakeApp.js';
import type {
ExecutionResult,
ExecutionResultWithMetadata,
} from '../../../action/execution/ExecutionResult.js';
import { V1_INTERNAL__UnknownElement } from './model/packageableElements/V1_INTERNAL__UnknownElement.js';
import { V1_HostedService } from './model/packageableElements/function/V1_HostedService.js';
import type { PostValidationAssertionResult } from '../../../../DSL_Service_Exports.js';
import { V1_UserListOwnership } from './model/packageableElements/service/V1_ServiceOwnership.js';
import { V1_PureSingleExecution } from './model/packageableElements/service/V1_ServiceExecution.js';
import {
type V1_Runtime,
V1_RuntimePointer,
} from './model/packageableElements/runtime/V1_Runtime.js';
import type { TestDebug } from '../../../../graph/metamodel/pure/test/result/DebugTestsResult.js';
import { V1_buildDebugTestsResult } from './engine/test/V1_DebugTestsResult.js';
import type { V1_GraphManagerEngine } from './engine/V1_GraphManagerEngine.js';
import type { RelationTypeMetadata } from '../../../action/relation/RelationTypeMetadata.js';
import type { CodeCompletionResult } from '../../../action/compilation/Completion.js';
import { V1_CompleteCodeInput } from './engine/compilation/V1_CompleteCodeInput.js';
import type { DeploymentResult } from '../../../action/DeploymentResult.js';
import type {
LightPersistentDataCube,
PersistentDataCube,
} from '../../../action/query/PersistentDataCube.js';
import { V1_QueryParameterValue } from './engine/query/V1_Query.js';
import { V1_Multiplicity } from './model/packageableElements/domain/V1_Multiplicity.js';
import {
V1_buildFunctionSignature,
V1_createGenericTypeWithElementPath,
} from './helpers/V1_DomainHelper.js';
import { V1_DataProduct } from './model/packageableElements/dataProduct/V1_DataProduct.js';
class V1_PureModelContextDataIndex {
elements: V1_PackageableElement[] = [];
nativeElements: V1_PackageableElement[] = [];
associations: V1_Association[] = [];
classes: V1_Class[] = [];
enumerations: V1_Enumeration[] = [];
functions: V1_ConcreteFunctionDefinition[] = [];
functionActivators: V1_FunctionActivator[] = [];
profiles: V1_Profile[] = [];
measures: V1_Measure[] = [];
stores: V1_Store[] = [];
mappings: V1_Mapping[] = [];
connections: V1_PackageableConnection[] = [];
runtimes: V1_PackageableRuntime[] = [];
sectionIndices: V1_SectionIndex[] = [];
fileGenerations: V1_FileGenerationSpecification[] = [];
generationSpecifications: V1_GenerationSpecification[] = [];
dataElements: V1_DataElement[] = [];
services: V1_Service[] = [];
executionEnvironments: V1_ExecutionEnvironmentInstance[] = [];
products: V1_DataProduct[] = [];
INTERNAL__UnknownElement: V1_INTERNAL__UnknownElement[] = [];
INTERNAL__unknownElements: V1_INTERNAL__UnknownPackageableElement[] = [];
otherElementsByBuilder: Map<
V1_ElementBuilder<V1_PackageableElement>,
V1_PackageableElement[]
> = new Map<
V1_ElementBuilder<V1_PackageableElement>,
V1_PackageableElement[]
>();
}
const mergePureModelContextData = (
...data: V1_PureModelContextData[]
): V1_PureModelContextData => {
const mergedData = new V1_PureModelContextData();
for (const _data of data) {
mergedData.elements = mergedData.elements.concat(_data.elements);
const rawDependencyEntities = (
mergedData.INTERNAL__rawDependencyEntities ?? []
).concat(_data.INTERNAL__rawDependencyEntities ?? []);
mergedData.INTERNAL__rawDependencyEntities = rawDependencyEntities.length
? rawDependencyEntities
: undefined;
mergedData.serializer = _data.serializer ?? mergedData.serializer;
mergedData.origin = _data.origin ?? mergedData.origin;
}
return mergedData;
};
export const V1_indexPureModelContextData = (
report: GraphManagerOperationReport,
data: V1_PureModelContextData,
extensions: V1_GraphBuilderExtensions,
): V1_PureModelContextDataIndex => {
const index = new V1_PureModelContextDataIndex();
index.elements = data.elements;
const otherElementsByClass = new Map<
Clazz<V1_PackageableElement>,
V1_PackageableElement[]
>();
data.elements.forEach((el) => {
let isIndexedAsOtherElement = false;
if (el instanceof V1_INTERNAL__UnknownElement) {
index.INTERNAL__UnknownElement.push(el);
} else if (el instanceof V1_INTERNAL__UnknownPackageableElement) {
index.INTERNAL__unknownElements.push(el);
} else if (el instanceof V1_Association) {
index.associations.push(el);
} else if (el instanceof V1_Class) {
index.classes.push(el);
} else if (el instanceof V1_Enumeration) {
index.enumerations.push(el);
} else if (el instanceof V1_ConcreteFunctionDefinition) {
index.functions.push(el);
} else if (el instanceof V1_FunctionActivator) {
index.functionActivators.push(el);
} else if (el instanceof V1_Profile) {
index.profiles.push(el);
} else if (el instanceof V1_Measure) {
index.measures.push(el);
} else if (el instanceof V1_Mapping) {
index.mappings.push(el);
} else if (el instanceof V1_PackageableConnection) {
index.connections.push(el);
} else if (el instanceof V1_PackageableRuntime) {
index.runtimes.push(el);
} else if (el instanceof V1_Store) {
index.stores.push(el);
} else if (el instanceof V1_SectionIndex) {
index.sectionIndices.push(el);
} else if (el instanceof V1_Service) {
index.services.push(el);
} else if (el instanceof V1_FileGenerationSpecification) {
index.fileGenerations.push(el);
} else if (el instanceof V1_GenerationSpecification) {
index.generationSpecifications.push(el);
} else if (el instanceof V1_DataElement) {
index.dataElements.push(el);
} else if (el instanceof V1_ExecutionEnvironmentInstance) {
index.executionEnvironments.push(el);
} else if (el instanceof V1_DataProduct) {
index.products.push(el);
} else {
const clazz = getClass<V1_PackageableElement>(el);
if (otherElementsByClass.has(clazz)) {
otherElementsByClass.get(clazz)?.push(el);
} else {
otherElementsByClass.set(clazz, [el]);
}
isIndexedAsOtherElement = true;
}
// we index everything else as native
if (!isIndexedAsOtherElement) {
index.nativeElements.push(el);
}
});
otherElementsByClass.forEach((elements, _class) => {
const builder = extensions.getExtraBuilderForProtocolClassOrThrow(_class);
index.otherElementsByBuilder.set(
builder,
(index.otherElementsByBuilder.get(builder) ?? []).concat(elements),
);
});
// report
report.elementCount.total =
(report.elementCount.total ?? 0) + index.elements.length;
report.elementCount.other =
(report.elementCount.other ?? 0) +
otherElementsByClass.size +
index.fileGenerations.length +
index.generationSpecifications.length;
report.elementCount.sectionIndex =
(report.elementCount.sectionIndex ?? 0) + index.sectionIndices.length;
report.elementCount.association =
(report.elementCount.association ?? 0) + index.associations.length;
report.elementCount.class =
(report.elementCount.class ?? 0) + index.classes.length;
report.elementCount.enumeration =
(report.elementCount.enumeration ?? 0) + index.enumerations.length;
report.elementCount.function =
(report.elementCount.function ?? 0) + index.functions.length;
report.elementCount.functionActivators =
(report.elementCount.functionActivators ?? 0) +
index.functionActivators.length;
report.elementCount.profile =
(report.elementCount.profile ?? 0) + index.profiles.length;
report.elementCount.measure =
(report.elementCount.measure ?? 0) + index.measures.length;
report.elementCount.dataElement =
(report.elementCount.dataElement ?? 0) + index.dataElements.length;
report.elementCount.store =
(report.elementCount.store ?? 0) + index.stores.length;
report.elementCount.mapping =
(report.elementCount.mapping ?? 0) + index.mappings.length;
report.elementCount.connection =
(report.elementCount.connection ?? 0) + index.connections.length;
report.elementCount.runtime =
(report.elementCount.runtime ?? 0) + index.runtimes.length;
report.elementCount.service =
(report.elementCount.service ?? 0) + index.services.length;
report.elementCount.executionEnvironment =
(report.elementCount.executionEnvironment ?? 0) +
index.executionEnvironments.length;
report.elementCount.unknown =
(report.elementCount.unknown ?? 0) + index.INTERNAL__unknownElements.length;
return index;
};
// NOTE: this interface is somewhat naive since `model` is of type `BasicModel`,
// so this can only be used for pre-processing/indexing
// we might need to change model to PureModel in the future when we support other use case
interface V1_PureGraphBuilderInput {
model: BasicModel;
data: V1_PureModelContextDataIndex;
origin?: GraphDataOrigin | undefined;
}
export interface V1_EngineSetupConfig {
env: string;
tabSize: number;
clientConfig: ServerClientConfig;
}
interface ServiceRegistrationInput {
service: Service;
context: V1_PureModelContext;
}
export class V1_PureGraphManager extends AbstractPureGraphManager {
private readonly elementClassifierPathMap = new Map<string, string>();
private readonly subtypeInfo: SubtypeInfo = {
storeSubtypes: [],
functionActivatorSubtypes: [],
};
// Pure Client Version represent the version of the pure protocol.
// Most Engine APIs will interrupt an undefined pure client version to mean
// use the latest production version of the protocol i.e V20_0_0, while version
// `VX_X_X` represents the version in development and used for testing
static readonly PURE_PROTOCOL_NAME = 'pure';
static readonly DEV_PROTOCOL_VERSION = PureClientVersion.VX_X_X;
static readonly PROD_PROTOCOL_VERSION = undefined;
engine: V1_GraphManagerEngine;
readonly graphBuilderExtensions: V1_GraphBuilderExtensions;
constructor(
pluginManager: GraphManagerPluginManager,
logService: LogService,
engine?: V1_GraphManagerEngine,
) {
super(pluginManager, logService);
this.engine = engine ?? new V1_RemoteEngine({}, logService);
// setup plugins
this.graphBuilderExtensions = new V1_GraphBuilderExtensions(
this.pluginManager.getPureProtocolProcessorPlugins(),
);
}
TEMPORARY__getEngineConfig(): TEMPORARY__AbstractEngineConfig {
return this.engine.config;
}
async initialize(
config: TEMPORARY__EngineSetupConfig,
options?: {
tracerService?: TracerService | undefined;
disableGraphConfiguration?: boolean | undefined;
engine?: V1_GraphManagerEngine;
},
): Promise<void> {
this.engine =
options?.engine ??
new V1_RemoteEngine(config.clientConfig, this.logService);
// TODO: improve abstraction so that we do not need to access the engine server client directly
if (this.engine instanceof V1_RemoteEngine) {
this.engine
.getEngineServerClient()
.setTracerService(options?.tracerService ?? new TracerService());
}
if (!options?.disableGraphConfiguration) {
// TODO: should probably be moved into each store's own initialize method
await Promise.all([
this.engine.setup(config),
this.configureElementClassifierPathMap(config),
this.configureSubtypeInfoMap(config),
]);
// setup serialization plugins
V1_setupPureModelContextDataSerialization(
this.pluginManager.getPureProtocolProcessorPlugins(),
this.subtypeInfo,
this.elementClassifierPathMap,
);
V1_setupDatabaseSerialization(
this.pluginManager.getPureProtocolProcessorPlugins(),
);
V1_setupEngineRuntimeSerialization(
this.pluginManager.getPureProtocolProcessorPlugins(),
);
V1_setupLegacyRuntimeSerialization(
this.pluginManager.getPureProtocolProcessorPlugins(),
);
}
}
private async configureElementClassifierPathMap(
config: TEMPORARY__EngineSetupConfig,
): Promise<void> {
const classifierPathMapEntries =
config.TEMPORARY__classifierPathMapping ??
(await this.engine.getClassifierPathMapping());
classifierPathMapEntries.forEach((entry) => {
this.elementClassifierPathMap.set(entry.type, entry.classifierPath);
});
}
private async configureSubtypeInfoMap(
config: TEMPORARY__EngineSetupConfig,
): Promise<void> {
const subtypeInfo =
config.TEMPORARY__subtypeInfo ?? (await this.engine.getSubtypeInfo());
this.subtypeInfo.storeSubtypes = subtypeInfo.storeSubtypes;
this.subtypeInfo.functionActivatorSubtypes =
subtypeInfo.functionActivatorSubtypes;
}
// --------------------------------------------- Generic ---------------------------------------------
getSupportedProtocolVersion(): string {
return PureClientVersion.V1_0_0;
}
getElementEntities(entities: Entity[]): Entity[] {
return entities.filter(
(entity) => entity.classifierPath !== CORE_PURE_PATH.SECTION_INDEX,
);
}
// --------------------------------------------- Graph Builder ---------------------------------------------
async buildSystem(
coreModel: CoreModel,
systemModel: SystemModel,
buildState: ActionState,
options?: GraphBuilderOptions,
_report?: GraphManagerOperationReport,
): Promise<void> {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
// Create a dummy graph for system processing. This is to ensure system model does not depend on the main graph
const graph = new PureModel(
coreModel,
systemModel,
this.pluginManager.getPureGraphPlugins(),
);
try {
// deserialize
buildState.setMessage(`Collecting and deserializing elements...`);
const systemData = mergePureModelContextData(
V1_deserializePureModelContextData(V1_CORE_SYSTEM_MODELS),
...this.pluginManager
.getPureProtocolProcessorPlugins()
.flatMap((plugin) => plugin.V1_getExtraSystemModels?.() ?? [])
.map((modelContextData) =>
V1_deserializePureModelContextData(modelContextData),
),
);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS,
);
// prepare build inputs
const buildInputs = [
{
model: systemModel,
data: V1_indexPureModelContextData(
report,
systemData,
this.graphBuilderExtensions,
),
},
];
// build
await this.buildGraphFromInputs(
graph,
buildInputs,
report,
stopWatch,
buildState,
options,
);
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
} catch (error) {
assertErrorThrown(error);
buildState.fail();
this.logService.error(
LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE),
error,
);
throw new SystemGraphBuilderError(error);
} finally {
buildState.setMessage(undefined);
}
}
async buildDependencies(
coreModel: CoreModel,
systemModel: SystemModel,
dependencyManager: DependencyManager,
dependencyEntitiesIndex: Map<string, EntitiesWithOrigin>,
buildState: ActionState,
options?: GraphBuilderOptions,
_report?: GraphManagerOperationReport,
): Promise<void> {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
// Create a dummy graph for system processing. This is to ensure dependency models do not depend on the main graph
const graph = new PureModel(
coreModel,
systemModel,
this.pluginManager.getPureGraphPlugins(),
);
graph.dependencyManager = dependencyManager;
try {
dependencyManager.initialize(dependencyEntitiesIndex);
// deserialize
buildState.setMessage(`Partitioning and deserializing elements...`);
const dependencyGraphDataIndex = new Map<
string,
V1_PureModelContextData
>();
await Promise.all(
Array.from(dependencyEntitiesIndex.entries()).map(
([dependencyKey, entitiesWithOrigin]) => {
const projectModelData = new V1_PureModelContextData();
dependencyGraphDataIndex.set(dependencyKey, projectModelData);
return V1_entitiesToPureModelContextData(
entitiesWithOrigin.entities,
projectModelData,
this.pluginManager.getPureProtocolProcessorPlugins(),
this.subtypeInfo,
this.elementClassifierPathMap,
);
},
),
);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS,
);
// prepare build inputs
const buildInputs: V1_PureGraphBuilderInput[] = Array.from(
dependencyGraphDataIndex.entries(),
).map(([dependencyKey, dependencyData]) => ({
model: graph.dependencyManager.getModel(dependencyKey),
data: V1_indexPureModelContextData(
report,
dependencyData,
this.graphBuilderExtensions,
),
}));
// build
await this.buildGraphFromInputs(
graph,
buildInputs,
report,
stopWatch,
buildState,
options,
);
// set dependency manager graph origin to entities
if (dependencyManager.origin === undefined) {
dependencyManager.setOrigin(
new GraphEntities(
Array.from(dependencyEntitiesIndex.values())
.map((e) => e.entities)
.flat(),
),
);
}
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
} catch (error) {
assertErrorThrown(error);
this.logService.error(
LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE),
error,
);
buildState.fail();
throw new DependencyGraphBuilderError(error);
} finally {
buildState.setMessage(undefined);
}
}
async buildGraph(
graph: PureModel,
entities: Entity[],
buildState: ActionState,
options?: GraphBuilderOptions,
_report?: GraphManagerOperationReport,
): Promise<void> {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
try {
// deserialize
buildState.setMessage(`Deserializing elements...`);
const data = new V1_PureModelContextData();
await V1_entitiesToPureModelContextData(
entities,
data,
this.pluginManager.getPureProtocolProcessorPlugins(),
this.subtypeInfo,
this.elementClassifierPathMap,
);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS,
);
// prepare build inputs
const buildInputs: V1_PureGraphBuilderInput[] = [
{
model: graph,
data: V1_indexPureModelContextData(
report,
data,
this.graphBuilderExtensions,
),
},
];
// build
await this.buildGraphFromInputs(
graph,
buildInputs,
report,
stopWatch,
buildState,
options,
);
/**
* For now, we delete the section index. We are able to read both resolved and unresolved element paths
* but when we write (serialize) we write only resolved paths. In the future once the issue with dependency is solved we will
* perserve the element path both resolved and unresolved
*/
if (!options?.TEMPORARY__preserveSectionIndex) {
graph.TEMPORARY__deleteOwnSectionIndex();
}
if (options?.origin) {
graph.setOrigin(options.origin);
}
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
} catch (error) {
assertErrorThrown(error);
this.logService.error(
LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE),
error,
);
buildState.fail();
/**
* Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
* But we might want to rethink this decision in the future and throw appropriate type of error
*/
throw error instanceof GraphBuilderError
? error
: new GraphBuilderError(error);
} finally {
buildState.setMessage(undefined);
}
}
async buildLightGraph(
graph: PureModel,
entities: Entity[],
buildState: ActionState,
options?: GraphBuilderOptions,
_report?: GraphManagerOperationReport,
): Promise<void> {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
buildState.reset();
try {
// deserialize
buildState.setMessage(`Deserializing elements...`);
const data = new V1_PureModelContextData();
await V1_entitiesToPureModelContextData(
entities,
data,
this.pluginManager.getPureProtocolProcessorPlugins(),
this.subtypeInfo,
this.elementClassifierPathMap,
);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS,
);
// prepare build inputs
const buildInputs: V1_PureGraphBuilderInput[] = [
{
model: graph,
data: V1_indexPureModelContextData(
report,
data,
this.graphBuilderExtensions,
),
},
];
// build
await this.buildLightGraphFromInputs(
graph,
buildInputs,
report,
stopWatch,
buildState,
options,
);
/**
* For now, we delete the section index. We are able to read both resolved and unresolved element paths
* but when we write (serialize) we write only resolved paths. In the future once the issue with dependency is solved we will
* perserve the element path both resolved and unresolved
*/
if (!options?.TEMPORARY__preserveSectionIndex) {
graph.TEMPORARY__deleteOwnSectionIndex();
}
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
} catch (error) {
assertErrorThrown(error);
this.logService.error(
LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE),
error,
);
buildState.fail();
/**
* Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
* But we might want to rethink this decision in the future and throw appropriate type of error
*/
throw error instanceof GraphBuilderError
? error
: new GraphBuilderError(error);
} finally {
buildState.setMessage(undefined);
}
}
async buildGenerations(
graph: PureModel,
generatedEntities: Map<string, Entity[]>,
buildState: ActionState,
options?: GraphBuilderOptions,
_report?: GraphManagerOperationReport,
): Promise<void> {
const stopWatch = new StopWatch();
const report = _report ?? createGraphBuilderReport();
const generatedModel = graph.generationModel;
buildState.reset();
try {
// deserialize
buildState.setMessage(`Deserializing elements...`);
const generationGraphDataIndex = new Map<
string,
V1_PureModelContextData
>();
await Promise.all(
Array.from(generatedEntities.entries()).map(
([generationParentPath, entities]) => {
const generatedData = new V1_PureModelContextData();
generationGraphDataIndex.set(generationParentPath, generatedData);
return V1_entitiesToPureModelContextData(
entities,
generatedData,
this.pluginManager.getPureProtocolProcessorPlugins(),
this.subtypeInfo,
this.elementClassifierPathMap,
);
},
),
);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS,
);
// prepare build inputs
const buildInputs: V1_PureGraphBuilderInput[] = Array.from(
generationGraphDataIndex.entries(),
).map(([generationParentPath, generatedData]) => ({
model: generatedModel,
data: V1_indexPureModelContextData(
report,
generatedData,
this.graphBuilderExtensions,
),
}));
// build
await this.buildGraphFromInputs(
graph,
buildInputs,
report,
stopWatch,
buildState,
options,
);
buildState.pass();
const totalTime = stopWatch.elapsed;
report.timings = {
...Object.fromEntries(stopWatch.records),
[GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime,
total: totalTime,
};
} catch (error) {
assertErrorThrown(error);
this.logService.error(
LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE),
error,
);
buildState.fail();
/**
* Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder
* But we might want to rethink this decision in the future and throw appropriate type of error
*/
throw error instanceof GraphBuilderError
? error
: new GraphBuilderError(error);
} finally {
buildState.setMessage(undefined);
}
}
private async buildGraphFromInputs(
graph: PureModel,
inputs: V1_PureGraphBuilderInput[],
report: GraphManagerOperationReport,
stopWatch: StopWatch,
graphBuilderState: ActionState,
options?: GraphBuilderOptions,
): Promise<void> {
// index
graphBuilderState.setMessage(
`Indexing ${report.elementCount.total} elements...`,
);
await this.initializeAndIndexElements(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS);
// build section index
graphBuilderState.setMessage(`Building section indices...`);
await this.buildSectionIndices(graph, inputs, options);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_SECTION_INDICES__SUCCESS,
);
// build types
graphBuilderState.setMessage(`Building domain models...`);
await this.buildTypes(graph, inputs, options);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS,
);
// build stores
graphBuilderState.setMessage(`Building stores...`);
await this.buildStores(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_STORES__SUCCESS);
// build mappings
graphBuilderState.setMessage(`Building mappings...`);
await this.buildMappings(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_MAPPINGS__SUCCESS);
// build connections and runtimes
graphBuilderState.setMessage(`Building connections and runtimes...`);
await this.buildConnectionsAndRuntimes(graph, inputs, options);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_CONNECTIONS_AND_RUNTIMES__SUCCESS,
);
// build function activators
graphBuilderState.setMessage(`Building function activators...`);
await this.buildFunctionActivators(graph, inputs, options);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS,
);
// build services
graphBuilderState.setMessage(
`Building services and execution environments...`,
);
await this.buildServices(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_SERVICES__SUCCESS);
// build data elements
graphBuilderState.setMessage(`Building data elements...`);
await this.buildDataElements(graph, inputs, options);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DATA_ELEMENTS__SUCCESS,
);
// build data products
graphBuilderState.setMessage(`Building data products...`);
await this.buildDataProducts(graph, inputs, options);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DATA_PRODUCTS__SUCCESS,
);
// build other elements
graphBuilderState.setMessage(`Building other elements...`);
await this.buildFileGenerations(graph, inputs, options);
await this.buildGenerationSpecifications(graph, inputs, options);
await this.buildOtherElements(graph, inputs, options);
stopWatch.record(
GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS,
);
}
private async buildLightGraphFromInputs(
graph: PureModel,
inputs: V1_PureGraphBuilderInput[],
report: GraphManagerOperationReport,
stopWatch: StopWatch,
graphBuilderState: ActionState,
options?: GraphBuilderOptions,
): Promise<void> {
// index
graphBuilderState.setMessage(
`Indexing ${report.elementCount.total} elements...`,
);
await this.initializeAndIndexElements(graph, inputs, options);
stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS);
}
private getBuilderContext(
graph: PureModel,
currentSubGraph: BasicModel,
element: V1_PackageableElement,
options?: GraphBuilderOptions,
): V1_GraphBuilderContext {
return new V1_GraphBuilderContextBuilder(
graph,
currentSubGraph,
this.graphBuilderExtensions,
this.logService,
options,
)
.withElement(element)
.build();
}
/**
* This will run the first pass builder for all elements and index them.
* This process is needed so other core processes such as building the section indices
* or building processes that relies on the `existence` of other elements to refer to them,
* but not necessarily use them.
*
* NOTE: We aim to not do anything more than running the first pass and indexing the first pass.
*/
private async initializeAndIndexElements(
graph: PureModel,
inputs: V1_PureGraphBuilderInput[],
options?: GraphBuilderOptions,
): Promise<void> {
// create the element path cache for faster duplication check
// NOTE: We base on the assumption here that our graph building is cascading, i.e.
// it first builds core, system, dependencies, graph, and generation.
// this way, as we build the a graph, we know the next step's duplication check
// has path cache consisting of all element from its base graphs
const element