@finos/legend-application-marketplace/src/stores/LegendMarketplaceBaseStore.ts

Legend Marketplace application core

/**
 * Copyright (c) 2020-present, Goldman Sachs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import {
  type GeneratorFn,
  ActionState,
  assertErrorThrown,
  isNonNullable,
  LogEvent,
  UserSearchService,
} from '@finos/legend-shared';
import {
  type ApplicationStore,
  LegendApplicationTelemetryHelper,
  APPLICATION_EVENT,
  DEFAULT_TAB_SIZE,
} from '@finos/legend-application';
import { action, flow, makeObservable, observable } from 'mobx';
import {
  DepotServerClient,
  StoreProjectData,
} from '@finos/legend-server-depot';
import {
  MarketplaceServerClient,
  RegistryServerClient,
  TerminalAccessServerClient,
} from '@finos/legend-server-marketplace';
import {
  type V1_EngineServerClient,
  getCurrentUserIDFromEngineServer,
  V1_entitlementsDataProductDetailsResponseToDataProductDetails,
  V1_PureGraphManager,
  V1_RemoteEngine,
} from '@finos/legend-graph';
import type { LegendMarketplaceApplicationConfig } from '../application/LegendMarketplaceApplicationConfig.js';
import type { LegendMarketplacePluginManager } from '../application/LegendMarketplacePluginManager.js';
import { LegendMarketplaceEventHelper } from '../__lib__/LegendMarketplaceEventHelper.js';
import {
  LakehouseContractServerClient,
  LakehouseIngestServerClient,
  LakehousePlatformServerClient,
  LakehouseWorkflowServerClient,
  PermitWorkflowServerClient,
} from '@finos/legend-server-lakehouse';
import { CartStore } from './cart/CartStore.js';
import { PendingTasksCache } from './lakehouse/PendingTasksCache.js';
import { parseGAVCoordinates, type Entity } from '@finos/legend-storage';
import { V1_deserializeDataSpace } from '@finos/legend-extension-dsl-data-space/graph';
import {
  DevelopmentLegendMarketplaceEnvState,
  LegendMarketplaceEnv,
  ProdLegendMarketplaceEnvState,
  ProdParallelLegendMarketplaceEnvState,
  type LegendMarketplaceEnvState,
} from './LegendMarketplaceEnvState.js';
import { ProductCardState } from './lakehouse/dataProducts/ProductCardState.js';
import {
  convertEntitlementsDataProductDetailsToSearchResult,
  convertLegacyDataProductToSearchResult,
  convertTrendingEntryToSearchResult,
} from '../utils/SearchUtils.js';
import { LakehouseDataProductService } from './lakehouse/LakehouseDataProductService.js';

export type LegendMarketplaceApplicationStore = ApplicationStore<
  LegendMarketplaceApplicationConfig,
  LegendMarketplacePluginManager
>;

export class LegendMarketplaceBaseStore {
  readonly applicationStore: LegendMarketplaceApplicationStore;
  readonly envState: LegendMarketplaceEnvState;
  readonly adjacentEnvState: LegendMarketplaceEnvState | undefined;
  readonly marketplaceServerClient: MarketplaceServerClient;
  readonly depotServerClient: DepotServerClient;
  readonly lakehouseContractServerClient: LakehouseContractServerClient;
  readonly lakehousePlatformServerClient: LakehousePlatformServerClient;
  readonly lakehouseIngestServerClient: LakehouseIngestServerClient;
  readonly engineServerClient: V1_EngineServerClient;
  readonly registryServerClient: RegistryServerClient | undefined;
  readonly pluginManager: LegendMarketplacePluginManager;
  readonly remoteEngine: V1_RemoteEngine;
  readonly userSearchService: UserSearchService | undefined;
  readonly lakehouseWorkflowServerClient: LakehouseWorkflowServerClient;
  readonly permitWorkflowServerClient: PermitWorkflowServerClient | undefined;
  readonly lakehouseDataProductService: LakehouseDataProductService;
  readonly cartStore: CartStore;
  readonly terminalAccessServerClient: TerminalAccessServerClient;
  readonly pendingTasksCache: PendingTasksCache;

  readonly initState = ActionState.create();

  showDemoModal = false;

  constructor(applicationStore: LegendMarketplaceApplicationStore) {
    makeObservable<LegendMarketplaceBaseStore>(this, {
      showDemoModal: observable,
      setDemoModal: action,
      initialize: flow,
    });

    this.applicationStore = applicationStore;
    this.pluginManager = applicationStore.pluginManager;

    // marketplace
    this.envState =
      applicationStore.config.dataProductEnv === LegendMarketplaceEnv.PRODUCTION
        ? new ProdLegendMarketplaceEnvState()
        : applicationStore.config.dataProductEnv ===
            LegendMarketplaceEnv.PRODUCTION_PARALLEL
          ? new ProdParallelLegendMarketplaceEnvState()
          : new DevelopmentLegendMarketplaceEnvState();
    this.adjacentEnvState = this.buildAdjacentEnvState();
    this.marketplaceServerClient = new MarketplaceServerClient({
      serverUrl: this.applicationStore.config.marketplaceServerUrl,
      subscriptionUrl: this.applicationStore.config.marketplaceSubscriptionUrl,
    });
    this.marketplaceServerClient.setTracerService(
      this.applicationStore.tracerService,
    );

    // registry
    if (this.applicationStore.config.registryUrl) {
      this.registryServerClient = new RegistryServerClient({
        baseUrl: this.applicationStore.config.registryUrl,
      });
      this.registryServerClient.setTracerService(
        this.applicationStore.tracerService,
      );
    }

    // depot
    this.depotServerClient = new DepotServerClient({
      serverUrl: this.applicationStore.config.depotServerUrl,
    });
    this.depotServerClient.setTracerService(
      this.applicationStore.tracerService,
    );

    // lakehouse contract
    this.lakehouseContractServerClient = new LakehouseContractServerClient({
      baseUrl: this.applicationStore.config.lakehouseServerUrl,
    });
    this.lakehouseContractServerClient.setTracerService(
      this.applicationStore.tracerService,
    );

    // terminal
    this.terminalAccessServerClient = new TerminalAccessServerClient({
      baseUrl: this.applicationStore.config.terminalServerUrl,
    });
    this.terminalAccessServerClient.setTracerService(
      this.applicationStore.tracerService,
    );

    // lakehouse platform
    this.lakehousePlatformServerClient = new LakehousePlatformServerClient(
      this.applicationStore.config.lakehousePlatformUrl,
    );
    this.lakehousePlatformServerClient.setTracerService(
      this.applicationStore.tracerService,
    );

    // lakehouse workflow
    this.lakehouseWorkflowServerClient = new LakehouseWorkflowServerClient({
      baseUrl: this.applicationStore.config.lakehouseWorkflowServerUrl,
    });
    this.lakehouseWorkflowServerClient.setTracerService(
      this.applicationStore.tracerService,
    );

    // permit + eTask workflow
    if (this.applicationStore.config.lakehousePermitWorkflowServerUrl) {
      this.permitWorkflowServerClient = new PermitWorkflowServerClient({
        authBaseUrl: this.applicationStore.config.lakehouseServerUrl,
        workflowBaseUrl:
          this.applicationStore.config.lakehousePermitWorkflowServerUrl,
      });
      this.permitWorkflowServerClient.setTracerService(
        this.applicationStore.tracerService,
      );
    } else {
      this.permitWorkflowServerClient = undefined;
    }

    // lakehouse ingest
    this.lakehouseIngestServerClient = new LakehouseIngestServerClient(
      undefined,
    );
    this.lakehouseIngestServerClient.setTracerService(
      this.applicationStore.tracerService,
    );

    this.remoteEngine = new V1_RemoteEngine(
      {
        baseUrl: this.applicationStore.config.engineServerUrl,
      },
      applicationStore.logService,
    );
    this.engineServerClient = this.remoteEngine.getEngineServerClient();
    this.engineServerClient.setTracerService(applicationStore.tracerService);

    // User search
    if (this.pluginManager.getUserPlugins().length > 0) {
      this.pluginManager
        .getUserPlugins()
        .forEach((plugin) =>
          plugin.setup(this.applicationStore.config.marketplaceUserSearchUrl),
        );
      this.userSearchService = new UserSearchService({
        userProfileImageUrl:
          this.applicationStore.config.marketplaceUserProfileImageUrl,
        applicationDirectoryUrl:
          this.applicationStore.config.lakehouseEntitlementsConfig
            ?.applicationDirectoryUrl,
      });
      this.userSearchService.registerPlugins(
        this.pluginManager.getUserPlugins(),
      );
    }

    // Data Product service
    this.lakehouseDataProductService = new LakehouseDataProductService(
      this,
      this.lakehousePlatformServerClient,
      this.lakehouseContractServerClient,
    );

    // Initialize cart store
    this.cartStore = new CartStore(this);

    // Shared cache + in-flight dedupe for /datacontracts/tasks/pending
    this.pendingTasksCache = new PendingTasksCache(
      this.lakehouseContractServerClient,
    );
  }

  buildAdjacentEnvState(): LegendMarketplaceEnvState | undefined {
    const adjacentEnv = this.envState.adjacentEnv;
    if (adjacentEnv) {
      return adjacentEnv === LegendMarketplaceEnv.PRODUCTION
        ? new ProdLegendMarketplaceEnvState()
        : new ProdParallelLegendMarketplaceEnvState();
    }
    return undefined;
  }

  private buildVendorImageMap(): Map<string, string> {
    const vendorImageMap = new Map<string, string>();
    const assetsBaseUrl = this.applicationStore.config.assetsBaseUrl;
    for (const [vendorName, filename] of Object.entries(
      this.applicationStore.config.assetsProductImageMap,
    )) {
      vendorImageMap.set(vendorName, `${assetsBaseUrl}/${filename}`);
    }
    return vendorImageMap;
  }

  async createInitializedGraphManager(): Promise<V1_PureGraphManager> {
    const graphManager = new V1_PureGraphManager(
      this.applicationStore.pluginManager,
      this.applicationStore.logService,
      this.remoteEngine,
    );
    await graphManager.initialize(
      {
        env: this.applicationStore.config.env,
        tabSize: DEFAULT_TAB_SIZE,
        clientConfig: {
          baseUrl: this.applicationStore.config.engineServerUrl,
        },
      },
      { engine: this.remoteEngine },
    );
    return graphManager;
  }

  private parseDataProductEntries(
    entriesString: string,
  ): (
    | { dataProductId: string; deploymentId: number; gav?: undefined }
    | { dataProductId: string; gav: string; deploymentId?: undefined }
  )[] {
    return entriesString
      .split(',')
      .map((entry) => {
        const vals = entry.split('/');
        if (vals[0] === undefined || vals[1] === undefined) {
          return undefined;
        }
        const id = vals[0];
        const secondPart = vals[1];
        if (Number.isInteger(Number(secondPart))) {
          return {
            dataProductId: id,
            deploymentId: parseInt(secondPart),
          };
        } else {
          return { dataProductId: id, gav: secondPart };
        }
      })
      .filter(isNonNullable);
  }

  async initHighlightedDataProducts(
    token: string | undefined,
  ): Promise<Record<string, ProductCardState[]> | undefined> {
    const highlightedConfig =
      this.applicationStore.config.options.highlightedDataProducts;
    if (!highlightedConfig) {
      return undefined;
    }

    const sectionEntries = Object.entries(highlightedConfig);
    if (sectionEntries.length === 0) {
      return undefined;
    }

    const vendorImageMap = this.buildVendorImageMap();

    const getDataProductState = async (
      dataProductId: string,
      deploymentId: number,
      graphManager: V1_PureGraphManager,
    ) => {
      const rawResponse =
        await this.lakehouseContractServerClient.getDataProductByIdAndDID(
          dataProductId,
          deploymentId,
          token,
        );
      const dataProductDetail =
        V1_entitlementsDataProductDetailsResponseToDataProductDetails(
          rawResponse,
        )[0];

      if (dataProductDetail) {
        const searchResult =
          convertEntitlementsDataProductDetailsToSearchResult(
            dataProductDetail,
          );
        return new ProductCardState(
          this,
          searchResult,
          graphManager,
          vendorImageMap,
        );
      } else {
        return undefined;
      }
    };

    const getLegacyDataProductState = async (
      dataProductId: string,
      gav: string,
      graphManager: V1_PureGraphManager,
    ) => {
      const coordinates = parseGAVCoordinates(gav);
      const storeProject = new StoreProjectData();
      storeProject.groupId = coordinates.groupId;
      storeProject.artifactId = coordinates.artifactId;
      const legacyDataProuct = await this.depotServerClient.getEntity(
        storeProject,
        coordinates.versionId,
        dataProductId,
      );
      const dataSpace = V1_deserializeDataSpace(
        (legacyDataProuct as unknown as Entity).content,
      );
      const searchResult = convertLegacyDataProductToSearchResult(
        dataSpace,
        coordinates.groupId,
        coordinates.artifactId,
        coordinates.versionId,
      );
      return new ProductCardState(
        this,
        searchResult,
        graphManager,
        vendorImageMap,
      );
    };

    const graphManager = await this.createInitializedGraphManager();

    const result: Record<string, ProductCardState[]> = {};
    await Promise.all(
      sectionEntries.map(async ([sectionName, entriesString]) => {
        const entries = this.parseDataProductEntries(entriesString);
        const states = (
          await Promise.all(
            entries.map(async (dataProduct) =>
              dataProduct.deploymentId !== undefined
                ? getDataProductState(
                    dataProduct.dataProductId,
                    dataProduct.deploymentId,
                    graphManager,
                  )
                : getLegacyDataProductState(
                    dataProduct.dataProductId,
                    dataProduct.gav,
                    graphManager,
                  ),
            ),
          )
        ).filter(isNonNullable);
        states.forEach((state) => state.init(token));
        result[sectionName] = states;
      }),
    );

    return result;
  }

  async fetchTrendingDataProducts(
    token: string | undefined,
  ): Promise<Record<string, ProductCardState[]> | undefined> {
    const vendorImageMap = this.buildVendorImageMap();
    const graphManager = await this.createInitializedGraphManager();

    const trendingEntries =
      await this.marketplaceServerClient.getTrendingDataProducts(
        this.envState.lakehouseEnvironment,
      );

    const states = trendingEntries.slice(0, 4).map((entry) => {
      const searchResult = convertTrendingEntryToSearchResult(entry);
      return new ProductCardState(
        this,
        searchResult,
        graphManager,
        vendorImageMap,
      );
    });

    states.forEach((state) => state.init(token));

    return { Trending: states };
  }

  setDemoModal(val: boolean): void {
    this.showDemoModal = val;
  }

  *initialize(): GeneratorFn<void> {
    if (!this.initState.isInInitialState) {
      this.applicationStore.notificationService.notifyIllegalState(
        'Base store is re-initialized',
      );
      return;
    }
    this.initState.inProgress();

    // retrieved the user identity is not already configured
    if (this.applicationStore.identityService.isAnonymous) {
      try {
        this.applicationStore.identityService.setCurrentUser(
          (yield getCurrentUserIDFromEngineServer(
            this.applicationStore.config.engineServerUrl,
          )) as string,
        );
      } catch (error) {
        assertErrorThrown(error);
        this.applicationStore.logService.error(
          LogEvent.create(APPLICATION_EVENT.IDENTITY_AUTO_FETCH__FAILURE),
          error,
        );
        this.applicationStore.notificationService.notifyWarning(error.message);
      }
    }

    // setup telemetry service
    this.applicationStore.telemetryService.setup();

    LegendApplicationTelemetryHelper.logEvent_ApplicationInitializationSucceeded(
      this.applicationStore.telemetryService,
      this.applicationStore,
    );

    LegendMarketplaceEventHelper.notify_ApplicationLoadSucceeded(
      this.applicationStore.eventService,
    );

    // Initialize cart store to load existing items
    try {
      yield* this.cartStore.initialize();
    } catch (error) {
      assertErrorThrown(error);
      this.applicationStore.logService.warn(
        LogEvent.create(APPLICATION_EVENT.IDENTITY_AUTO_FETCH__FAILURE),
        'Failed to initialize cart store',
        error,
      );
      // Don't show notification as cart initialization failure shouldn't block app startup
    }

    this.initState.complete();
  }
}