@finos/legend-application-marketplace
Version:
Legend Marketplace application core
565 lines • 25.7 kB
JavaScript
/**
* Copyright (c) 2020-present, Goldman Sachs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { action, computed, flow, makeObservable, observable } from 'mobx';
import { ActionState, assertErrorThrown, isNonNullable, LogEvent, } from '@finos/legend-shared';
import { DataProductSearchResult, DataProductSearchResultDetailsType, DataProductSearchResponse, ErrorDataProductSearchResultDetails, LakehouseAdHocDataProductSearchResultOrigin, LakehouseDataProductSearchResultDetails, LakehouseDataProductSearchResultOriginType, LakehouseSDLCDataProductSearchResultOrigin, } from '@finos/legend-server-marketplace';
import { ProductCardState } from './dataProducts/ProductCardState.js';
import { DEFAULT_TAB_SIZE } from '@finos/legend-application';
import { DATA_SPACE_ELEMENT_CLASSIFIER_PATH, V1_deserializeDataSpace, } from '@finos/legend-extension-dsl-data-space/graph';
import { V1_entitlementsDataProductLiteResponseToDataProductLite, V1_PureGraphManager, extractPackagePathFromPath, extractElementNameFromPath, V1_SdlcDeploymentDataProductOrigin, } from '@finos/legend-graph';
import { DepotScope, } from '@finos/legend-server-depot';
import { LEGEND_MARKETPLACE_APP_EVENT } from '../../__lib__/LegendMarketplaceAppEvent.js';
export var DataProductSort;
(function (DataProductSort) {
DataProductSort["DEFAULT"] = "Default";
DataProductSort["NAME_ALPHABETICAL"] = "Name A-Z";
DataProductSort["NAME_REVERSE_ALPHABETICAL"] = "Name Z-A";
})(DataProductSort || (DataProductSort = {}));
export var DataProductTypeFilter;
(function (DataProductTypeFilter) {
DataProductTypeFilter["LAKEHOUSE"] = "lakehouse";
DataProductTypeFilter["LEGACY"] = "legacy";
})(DataProductTypeFilter || (DataProductTypeFilter = {}));
export var DataProductSourceFilter;
(function (DataProductSourceFilter) {
DataProductSourceFilter["EXTERNAL"] = "External";
DataProductSourceFilter["INTERNAL"] = "Internal";
})(DataProductSourceFilter || (DataProductSourceFilter = {}));
export var SearchResultsViewMode;
(function (SearchResultsViewMode) {
SearchResultsViewMode["TILE"] = "tile";
SearchResultsViewMode["LIST"] = "list";
})(SearchResultsViewMode || (SearchResultsViewMode = {}));
export var SearchResultViewOption;
(function (SearchResultViewOption) {
SearchResultViewOption["DATA_PRODUCTS"] = "Data Products";
SearchResultViewOption["DATA_FIELDS"] = "Data Fields";
})(SearchResultViewOption || (SearchResultViewOption = {}));
const LEGEND_MARKETPLACE_SETTING_KEY_VIEW_MODE = 'marketplace.search-results.viewMode';
export class LegendMarketplaceSearchResultsStore {
marketplaceBaseStore;
marketplaceServerClient;
searchQuery = undefined;
_lastTaxonomyQuery = undefined;
useProducerSearch = undefined;
semanticSearchProductCardStates = [];
producerSearchDataProductCardStates = [];
producerSearchLegacyDataProductCardStates = [];
sort = DataProductSort.DEFAULT;
viewMode;
taxonomyTree = [];
selectedTaxonomyNodeIds = new Set();
selectedDataProductTypes = new Set();
selectedSources = new Set();
filterCounts = {
lakehouse_count: 0,
legacy_count: 0,
external_source_count: 0,
};
page = 1;
itemsPerPage = 12;
totalItems = 0;
showAllProducts = false;
hasFilteredDataProducts = false;
executingSemanticSearchState = ActionState.create();
fetchingProducerSearchDataProductsState = ActionState.create();
fetchingProducerSearchLegacyDataProductsState = ActionState.create();
constructor(marketplaceBaseStore) {
this.marketplaceBaseStore = marketplaceBaseStore;
this.marketplaceServerClient = marketplaceBaseStore.marketplaceServerClient;
const persistedViewMode = this.marketplaceBaseStore.applicationStore.settingService.getStringValue(LEGEND_MARKETPLACE_SETTING_KEY_VIEW_MODE);
this.viewMode =
persistedViewMode === SearchResultsViewMode.LIST
? SearchResultsViewMode.LIST
: SearchResultsViewMode.TILE;
makeObservable(this, {
searchQuery: observable,
useProducerSearch: observable,
semanticSearchProductCardStates: observable,
producerSearchDataProductCardStates: observable,
producerSearchLegacyDataProductCardStates: observable,
sort: observable,
viewMode: observable,
taxonomyTree: observable,
selectedTaxonomyNodeIds: observable,
selectedDataProductTypes: observable,
selectedSources: observable,
filterCounts: observable,
_lastTaxonomyQuery: false,
setSearchQuery: action,
setUseProducerSearch: action,
page: observable,
itemsPerPage: observable,
totalItems: observable,
showAllProducts: observable,
hasFilteredDataProducts: observable,
setSemanticSearchProductCardStates: action,
setProducerSearchDataProductCardStates: action,
setProducerSearchLegacyDataProductCardStates: action,
setSort: action,
setViewMode: action,
setPage: action,
setItemsPerPage: action,
setTotalItems: action,
setShowAllProducts: action,
setHasFilteredDataProducts: action,
setTaxonomyTree: action,
setFilterCounts: action,
setSelectedTaxonomyNodeIds: action,
toggleTaxonomyNode: action,
simpleToggleTaxonomyNode: action,
toggleDataProductType: action,
toggleSource: action,
clearAllFilters: action,
filterSortProducts: computed,
isLoading: computed,
isOnLastPage: computed,
hasActiveFilters: computed,
executeSearch: flow,
});
}
setSearchQuery(query) {
this.searchQuery = query;
}
setUseProducerSearch(value) {
this.useProducerSearch = value;
}
get isOnLastPage() {
if (this.totalItems === 0) {
return false;
}
const totalPages = Math.ceil(this.totalItems / this.itemsPerPage);
return this.page >= totalPages;
}
get filterSortProducts() {
const productCardStates = this.useProducerSearch
? [
...this.producerSearchDataProductCardStates,
...this.producerSearchLegacyDataProductCardStates,
].sort((a, b) => a.title.localeCompare(b.title))
: this.semanticSearchProductCardStates;
let filtered = productCardStates.filter((productCardState) => this.marketplaceBaseStore.envState.filterDataProduct(productCardState));
if (this.useProducerSearch && this.selectedTaxonomyNodeIds.size > 0) {
filtered = filtered.filter((productCardState) => {
const productTaxonomyPaths = productCardState.searchResult.tags2.flatMap((tag) => tag.split(',').map((t) => t.trim()));
return productTaxonomyPaths.some((path) => Array.from(this.selectedTaxonomyNodeIds).some((selectedId) => path === selectedId || path.startsWith(`${selectedId}::`)));
});
}
return filtered.sort((a, b) => {
switch (this.sort) {
case DataProductSort.DEFAULT:
return 0;
case DataProductSort.NAME_ALPHABETICAL:
return a.title.localeCompare(b.title);
case DataProductSort.NAME_REVERSE_ALPHABETICAL:
return b.title.localeCompare(a.title);
default:
return 0;
}
});
}
get isLoading() {
return this.useProducerSearch
? this.fetchingProducerSearchDataProductsState.isInProgress ||
this.fetchingProducerSearchDataProductsState.isInInitialState ||
this.fetchingProducerSearchLegacyDataProductsState.isInProgress
: this.executingSemanticSearchState.isInProgress ||
this.executingSemanticSearchState.isInInitialState;
}
setPage(value) {
this.page = value;
}
setItemsPerPage(value) {
this.itemsPerPage = value;
this.page = 1;
}
setTotalItems(value) {
this.totalItems = value;
}
setShowAllProducts(value) {
this.showAllProducts = value;
}
setHasFilteredDataProducts(value) {
this.hasFilteredDataProducts = value;
}
setSemanticSearchProductCardStates(dataProductCardStates) {
this.semanticSearchProductCardStates = dataProductCardStates;
}
setProducerSearchDataProductCardStates(dataProductCardStates) {
this.producerSearchDataProductCardStates = dataProductCardStates;
}
setProducerSearchLegacyDataProductCardStates(dataProductCardStates) {
this.producerSearchLegacyDataProductCardStates = dataProductCardStates;
}
setSort(sort) {
this.sort = sort;
}
setViewMode(viewMode) {
this.viewMode = viewMode;
this.marketplaceBaseStore.applicationStore.settingService.persistValue(LEGEND_MARKETPLACE_SETTING_KEY_VIEW_MODE, viewMode);
}
setTaxonomyTree(tree) {
this.taxonomyTree = tree;
}
setFilterCounts(counts) {
this.filterCounts = counts;
}
setSelectedTaxonomyNodeIds(ids) {
this.selectedTaxonomyNodeIds = new Set(ids);
}
collectAllNodeIds(node) {
const ids = [node.id];
for (const child of node.children) {
ids.push(...this.collectAllNodeIds(child));
}
return ids;
}
findNode(nodes, nodeId) {
for (const node of nodes) {
if (node.id === nodeId) {
return node;
}
const found = this.findNode(node.children, nodeId);
if (found) {
return found;
}
}
return undefined;
}
findAncestorPath(nodes, nodeId, currentPath = []) {
for (const node of nodes) {
if (node.id === nodeId) {
return [...currentPath];
}
const found = this.findAncestorPath(node.children, nodeId, [
...currentPath,
node,
]);
if (found) {
return found;
}
}
return undefined;
}
toggleTaxonomyNode(nodeId) {
if (this.selectedTaxonomyNodeIds.has(nodeId)) {
this.deselectTaxonomyNode(nodeId);
}
else {
this.selectTaxonomyNode(nodeId);
}
}
deselectTaxonomyNode(nodeId) {
const node = this.findNode(this.taxonomyTree, nodeId);
if (node) {
const idsToRemove = this.collectAllNodeIds(node);
for (const id of idsToRemove) {
this.selectedTaxonomyNodeIds.delete(id);
}
}
else {
this.selectedTaxonomyNodeIds.delete(nodeId);
}
}
selectTaxonomyNode(nodeId) {
const node = this.findNode(this.taxonomyTree, nodeId);
if (node) {
const idsToAdd = this.collectAllNodeIds(node);
for (const id of idsToAdd) {
this.selectedTaxonomyNodeIds.add(id);
}
}
else {
this.selectedTaxonomyNodeIds.add(nodeId);
}
const ancestors = this.findAncestorPath(this.taxonomyTree, nodeId, []);
if (ancestors) {
for (const ancestor of ancestors) {
this.selectedTaxonomyNodeIds.add(ancestor.id);
}
}
}
simpleToggleTaxonomyNode(nodeId) {
if (this.selectedTaxonomyNodeIds.has(nodeId)) {
this.selectedTaxonomyNodeIds.delete(nodeId);
}
else {
this.selectedTaxonomyNodeIds.add(nodeId);
}
}
toggleDataProductType(value) {
if (this.selectedDataProductTypes.has(value)) {
this.selectedDataProductTypes.delete(value);
}
else {
this.selectedDataProductTypes.add(value);
}
}
toggleSource(value) {
if (this.selectedSources.has(value)) {
this.selectedSources.delete(value);
}
else {
this.selectedSources.add(value);
}
}
clearAllFilters() {
this.selectedDataProductTypes.clear();
this.selectedSources.clear();
this.selectedTaxonomyNodeIds.clear();
}
get hasActiveFilters() {
return (this.selectedDataProductTypes.size > 0 ||
this.selectedSources.size > 0 ||
this.selectedTaxonomyNodeIds.size > 0);
}
computeFilterNodeIds() {
const selectedIds = this.selectedTaxonomyNodeIds;
if (selectedIds.size === 0) {
return [];
}
const filterIds = [];
const visitedIds = new Set();
const processNode = (node) => {
visitedIds.add(node.id);
if (!selectedIds.has(node.id)) {
node.children.forEach((child) => {
processNode(child);
});
return;
}
const selectedChildCount = node.children.filter((c) => selectedIds.has(c.id)).length;
if (node.children.length === 0 ||
selectedChildCount === 0 ||
selectedChildCount === node.children.length) {
filterIds.push(node.id);
const markVisited = (n) => {
visitedIds.add(n.id);
n.children.forEach(markVisited);
};
node.children.forEach(markVisited);
}
else {
node.children.forEach((child) => {
processNode(child);
});
}
};
this.taxonomyTree.forEach((rootNode) => {
processNode(rootNode);
});
for (const id of selectedIds) {
if (!visitedIds.has(id)) {
filterIds.push(id);
}
}
return filterIds;
}
buildSearchFilters() {
const filters = [];
if (this.selectedDataProductTypes.size > 0) {
filters.push(`data_product_type=${Array.from(this.selectedDataProductTypes).join(',')}`);
}
if (this.selectedSources.size > 0) {
filters.push(`data_product_source=${Array.from(this.selectedSources).join(',')}`);
}
const taxonomyFilterIds = this.computeFilterNodeIds();
if (taxonomyFilterIds.length > 0) {
filters.push(`taxonomy=${taxonomyFilterIds.join(',')}`);
}
return filters;
}
*executeSearch(query, useProducerSearch, token) {
try {
this.setSemanticSearchProductCardStates([]);
this.setProducerSearchDataProductCardStates([]);
this.setProducerSearchLegacyDataProductCardStates([]);
const searchFilters = this.buildSearchFilters();
// Create graph manager for parsing ad-hoc deployed data products
const graphManager = new V1_PureGraphManager(this.marketplaceBaseStore.applicationStore.pluginManager, this.marketplaceBaseStore.applicationStore.logService, this.marketplaceBaseStore.remoteEngine);
yield graphManager.initialize({
env: this.marketplaceBaseStore.applicationStore.config.env,
tabSize: DEFAULT_TAB_SIZE,
clientConfig: {
baseUrl: this.marketplaceBaseStore.applicationStore.config.engineServerUrl,
},
}, { engine: this.marketplaceBaseStore.remoteEngine });
if (useProducerSearch) {
yield this.executeProducerSearch(query, graphManager, token);
}
else {
yield this.executeSemanticSearch(query, graphManager, token, searchFilters);
}
}
catch (error) {
assertErrorThrown(error);
if (this.marketplaceBaseStore.applicationStore.config.options
.showDevFeatures) {
this.marketplaceBaseStore.applicationStore.notificationService.notifyError(error, `Error executing search: ${error.name}\n${error.message}\n${error.cause}\n${error.stack}`);
}
else {
this.marketplaceBaseStore.applicationStore.notificationService.notifyError(`Error executing search: ${error.message}`);
}
}
}
processRawSearchResults(rawResults, graphManager, token) {
const response = DataProductSearchResponse.serialization.fromJson(rawResults);
const validResults = response.results.filter((result) => !(result.dataProductDetails instanceof
ErrorDataProductSearchResultDetails) &&
!(result.dataProductDetails instanceof
LakehouseDataProductSearchResultDetails &&
result.dataProductDetails.origin === null));
const usedImages = new Set();
const productCardStates = validResults.map((result) => new ProductCardState(this.marketplaceBaseStore, result, graphManager, new Map(), usedImages));
productCardStates.forEach((dataProductState) => dataProductState.init(token));
return { productCardStates, response };
}
async executeSemanticSearch(query, graphManager, token, filters = []) {
this.executingSemanticSearchState.inProgress();
try {
const rawResults = await this.marketplaceServerClient.dataProductSearch(query, this.marketplaceBaseStore.envState.lakehouseEnvironment, 'hybrid', filters, this.itemsPerPage, this.page, this.showAllProducts);
const { productCardStates, response } = this.processRawSearchResults(rawResults, graphManager, token);
this.setTotalItems(response.metadata.total_count);
this.setHasFilteredDataProducts(response.metadata.has_filtered_products ?? false);
this.setSemanticSearchProductCardStates(productCardStates);
const isNewQuery = query !== this._lastTaxonomyQuery;
if (response.filters_metadata && isNewQuery) {
this.setTaxonomyTree(response.filters_metadata.taxonomy_tree);
this._lastTaxonomyQuery = query;
}
this.setFilterCounts({
lakehouse_count: response.metadata.lakehouse_count ?? 0,
legacy_count: response.metadata.legacy_count ?? 0,
external_source_count: response.metadata.external_source_count ?? 0,
});
}
finally {
this.executingSemanticSearchState.complete();
}
}
async executeProducerSearch(query, graphManager, token) {
await Promise.all([
this.DEPRECATED_fetchDataProducts(query, graphManager, token),
this.fetchLegacyDataProducts(query, graphManager, token),
]);
this.setTotalItems(this.producerSearchDataProductCardStates.length +
this.producerSearchLegacyDataProductCardStates.length);
}
async DEPRECATED_fetchDataProducts(query, graphManager, token) {
this.fetchingProducerSearchDataProductsState.inProgress();
try {
const rawResponse = await this.marketplaceBaseStore.lakehouseContractServerClient.getAllLiteDataProducts(this.marketplaceBaseStore.envState.lakehouseEnvironment, undefined, token);
const dataProductLiteDetails = V1_entitlementsDataProductLiteResponseToDataProductLite(rawResponse);
const usedImages = new Set();
const productCardStates = dataProductLiteDetails
.map((detail) => {
try {
const origin = detail.origin instanceof V1_SdlcDeploymentDataProductOrigin
? LakehouseSDLCDataProductSearchResultOrigin.serialization.fromJson({
_type: LakehouseDataProductSearchResultOriginType.SDLC,
groupId: detail.origin.group,
artifactId: detail.origin.artifact,
versionId: detail.origin.version,
path: detail.fullPath,
})
: LakehouseAdHocDataProductSearchResultOrigin.serialization.fromJson({
_type: LakehouseDataProductSearchResultOriginType.AD_HOC,
});
const searchResult = DataProductSearchResult.serialization.fromJson({
dataProductTitle: detail.title ?? detail.id,
dataProductDescription: detail.description,
tags1: [],
tags2: [],
tag_score: 0,
similarity: 0,
dataProductDetails: {
_type: DataProductSearchResultDetailsType.LAKEHOUSE,
dataProductId: detail.id,
deploymentId: detail.deploymentId,
producerEnvironmentName: detail.lakehouseEnvironment?.producerEnvironmentName,
producerEnvironmentType: detail.lakehouseEnvironment?.type,
origin,
},
});
return new ProductCardState(this.marketplaceBaseStore, searchResult, graphManager, new Map(), usedImages);
}
catch (error) {
this.marketplaceBaseStore.applicationStore.logService.error(LogEvent.create(LEGEND_MARKETPLACE_APP_EVENT.DESERIALIZE_DATA_PRODUCT_SEARCH_RESULT_FAILURE), `Can't deserialize data product search result: ${error}`);
return undefined;
}
})
.filter(isNonNullable);
const filteredProductCardStates = productCardStates.filter((productCardState) => productCardState.title.toLowerCase().includes(query.toLowerCase()));
filteredProductCardStates.forEach((dataProductState) => dataProductState.init(token));
this.setProducerSearchDataProductCardStates(filteredProductCardStates);
}
finally {
this.fetchingProducerSearchDataProductsState.complete();
}
}
async fetchLegacyDataProducts(query, graphManager, token) {
if (!this.marketplaceBaseStore.envState.supportsLegacyDataProducts()) {
return;
}
this.fetchingProducerSearchLegacyDataProductsState.inProgress();
try {
const dataSpaceEntitySummaries = (await this.marketplaceBaseStore.depotServerClient.getEntitiesSummaryByClassifier(DATA_SPACE_ELEMENT_CLASSIFIER_PATH, {
scope: DepotScope.RELEASES,
summary: true,
}));
const usedImages = new Set();
const productCardStates = dataSpaceEntitySummaries
.map((entity) => {
try {
const dataSpace = V1_deserializeDataSpace({
executionContexts: [],
defaultExecutionContext: '',
package: extractPackagePathFromPath(entity.path) ?? entity.path,
name: extractElementNameFromPath(entity.path),
});
const searchResult = DataProductSearchResult.serialization.fromJson({
dataProductTitle: dataSpace.title ?? dataSpace.name,
dataProductDescription: dataSpace.description,
tags1: [],
tags2: [],
tag_score: 0,
similarity: 0,
dataProductDetails: {
_type: DataProductSearchResultDetailsType.LEGACY,
groupId: entity.groupId,
artifactId: entity.artifactId,
versionId: entity.versionId,
path: entity.path,
},
});
return new ProductCardState(this.marketplaceBaseStore, searchResult, graphManager, new Map(), usedImages);
}
catch (error) {
this.marketplaceBaseStore.applicationStore.logService.error(LogEvent.create(LEGEND_MARKETPLACE_APP_EVENT.DESERIALIZE_DATA_PRODUCT_SEARCH_RESULT_FAILURE), `Can't deserialize data product search result: ${error}`);
return undefined;
}
})
.filter(isNonNullable);
const filteredProductCardStates = productCardStates.filter((productCardState) => productCardState.title.toLowerCase().includes(query.toLowerCase()));
filteredProductCardStates.forEach((dataProductState) => dataProductState.init(token));
this.setProducerSearchLegacyDataProductCardStates(filteredProductCardStates);
}
finally {
this.fetchingProducerSearchLegacyDataProductsState.complete();
}
}
}
//# sourceMappingURL=LegendMarketplaceSearchResultsStore.js.map