@finos/legend-data-cube
Version:
365 lines (348 loc) • 12.4 kB
text/typescript
/**
* Copyright (c) 2020-present, Goldman Sachs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
PRIMITIVE_TYPE,
V1_GenericTypeInstance,
type V1_ColSpec,
type V1_AppliedFunction,
V1_GenericType,
V1_RelationType,
V1_getGenericTypeFullPath,
V1_createRelationTypeColumn,
} from '@finos/legend-graph';
import { type DataCubeSnapshot } from '../../core/DataCubeSnapshot.js';
import {
DataCubeQueryFilterGroupOperator,
DataCubeQueryFilterOperator,
DataCubeFunction,
type DataCubeQueryFunctionMap,
isPivotResultColumnName,
PIVOT_COLUMN_NAME_VALUE_SEPARATOR,
DataCubeQuerySortDirection,
DataCubeOperationAdvancedValueType,
} from '../../core/DataCubeQueryEngine.js';
import {
guaranteeNonNullable,
guaranteeType,
isNullable,
uniq,
} from '@finos/legend-shared';
import {
_groupByAggCols,
_colSpec,
_cols,
_filter,
_function,
_lambda,
_var,
_functionCompositionUnProcessor,
_property,
_collection,
_col,
} from '../../core/DataCubeQueryBuilderUtils.js';
import {
INTERNAL__GRID_CLIENT_MISSING_VALUE,
INTERNAL__GRID_CLIENT_ROOT_AGGREGATION_COLUMN_ID,
INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID,
type DataCubeGridClientDataFetchRequest,
} from './DataCubeGridClientEngine.js';
import { DataCubeConfiguration } from '../../core/model/DataCubeConfiguration.js';
import { _colSpecArrayParam } from '../../core/DataCubeSnapshotBuilderUtils.js';
import {
buildDimensionalExecutableQuery,
buildExecutableQuery,
} from '../../core/DataCubeQueryBuilder.js';
import type { DataCubeSource } from '../../core/model/DataCubeSource.js';
import type { DataCubeEngine } from '../../core/DataCubeEngine.js';
import type { DataCubeDimensionalNode } from './DataCubeGridDimensionalTree.js';
/*****************************************************************************
* [GRID]
*
* These are utilities used to build adhoc executable queries for fetching
* data for the grid. The grid comes with features that requires building
* slight modifications of the persistent query that would be generated from
* the snapshot, such as in the case of pagination, drilldown (group by).
*****************************************************************************/
// --------------------------------- BUILDING BLOCKS ---------------------------------
export function _aggCountCol() {
const variable = _var();
return _colSpec(
INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID,
_lambda([variable], [variable]),
_lambda([variable], [_function(DataCubeFunction.COUNT, [variable])]),
);
}
function _aggCountCastCol(colName: string) {
const variable = _var();
return _colSpec(
colName,
_lambda([variable], [_property(colName)]),
_lambda([variable], [_function(DataCubeFunction.SUM, [variable])]),
);
}
// if pivot is present, modify pivot and pivot cast expressions to include
// count aggregation columns.
function _addCountAggColumnToPivot(
funcMap: DataCubeQueryFunctionMap,
countAggColumns: V1_ColSpec[],
): void {
if (funcMap.pivot && funcMap.pivotCast) {
_colSpecArrayParam(funcMap.pivot, 1).colSpecs.push(_aggCountCol());
const relationType = guaranteeType(
guaranteeType(
guaranteeType(funcMap.pivotCast.parameters[0], V1_GenericTypeInstance)
.genericType.typeArguments[0],
V1_GenericType,
).rawType,
V1_RelationType,
);
const castColumns = relationType.columns.map((column) =>
_colSpec(
column.name,
undefined,
undefined,
V1_getGenericTypeFullPath(column.genericType),
),
);
uniq(
castColumns
.filter((col) => isPivotResultColumnName(col.name))
.map((col) =>
col.name.substring(
0,
col.name.lastIndexOf(PIVOT_COLUMN_NAME_VALUE_SEPARATOR),
),
),
)
.map(
(col) =>
col +
PIVOT_COLUMN_NAME_VALUE_SEPARATOR +
INTERNAL__GRID_CLIENT_ROW_GROUPING_COUNT_AGG_COLUMN_ID,
)
.forEach((col) => {
// Here we directly modify the list of cast columns in the relation type
relationType.columns.push(
V1_createRelationTypeColumn(col, PRIMITIVE_TYPE.INTEGER),
);
countAggColumns.push(
_colSpec(col, undefined, undefined, PRIMITIVE_TYPE.INTEGER),
);
});
}
}
// --------------------------------- MAIN ---------------------------------
export function buildGridDataFetchExecutableQuery(
request: DataCubeGridClientDataFetchRequest,
snapshot: DataCubeSnapshot,
source: DataCubeSource,
engine: DataCubeEngine,
enablePagination: boolean,
) {
const processedSnapshot = snapshot.clone();
const configuration = DataCubeConfiguration.serialization.fromJson(
processedSnapshot.data.configuration,
);
if (configuration.showRootAggregation) {
processedSnapshot.data.groupBy = {
columns: [
{
name: INTERNAL__GRID_CLIENT_ROOT_AGGREGATION_COLUMN_ID,
type: PRIMITIVE_TYPE.STRING,
},
...(processedSnapshot.data.groupBy?.columns ?? []),
],
};
}
return buildExecutableQuery(processedSnapshot, source, engine, {
postProcessor: generateGridDataFetchExecutableQueryPostProcessor(request),
rootAggregation: {
columnName: INTERNAL__GRID_CLIENT_ROOT_AGGREGATION_COLUMN_ID,
},
pagination:
enablePagination &&
request.startRow !== undefined &&
request.endRow !== undefined
? {
start: request.startRow,
end: request.endRow,
}
: undefined,
});
}
export function buildDimensionalGridDataFetchExecutableQuery(
snapshot: DataCubeSnapshot,
source: DataCubeSource,
engine: DataCubeEngine,
nodes: DataCubeDimensionalNode[],
) {
//TODO: Try to remove this cloning
const processedSnapshot = snapshot.clone();
//TODO: check if we need to show root aggregation in this mode
return buildDimensionalExecutableQuery(
processedSnapshot,
source,
engine,
nodes,
);
}
function generateGridDataFetchExecutableQueryPostProcessor(
request: DataCubeGridClientDataFetchRequest,
) {
return (
snapshot: DataCubeSnapshot,
sequence: V1_AppliedFunction[],
funcMap: DataCubeQueryFunctionMap,
configuration: DataCubeConfiguration,
engine: DataCubeEngine,
) => {
const _unprocess = _functionCompositionUnProcessor(sequence, funcMap);
const data = snapshot.data;
if (funcMap.groupBy) {
// --------------------------------- GROUP BY ---------------------------------
const groupBy = guaranteeNonNullable(data.groupBy);
// if any level of drilldown has been done
// add a pre-filter to groupBy()
if (request.groupKeys.length) {
sequence.splice(
sequence.indexOf(funcMap.groupBy),
0,
_function(DataCubeFunction.FILTER, [
_lambda(
[_var()],
[
_filter(
{
conditions: request.groupKeys.map((value, i) => {
const groupByColumn = guaranteeNonNullable(
groupBy.columns[i],
);
if (
isNullable(value) ||
value === INTERNAL__GRID_CLIENT_MISSING_VALUE
) {
return {
...groupByColumn,
operator: DataCubeQueryFilterOperator.IS_NULL,
value: {
type: DataCubeOperationAdvancedValueType.VOID,
},
};
}
const condition = {
...groupByColumn,
operator: DataCubeQueryFilterOperator.EQUAL,
};
switch (groupByColumn.type) {
case PRIMITIVE_TYPE.BOOLEAN:
return {
...condition,
value: {
type: PRIMITIVE_TYPE.BOOLEAN,
value: value === 'true',
},
};
case PRIMITIVE_TYPE.INTEGER:
return {
...condition,
value: {
type: groupByColumn.type,
value: parseInt(value, 10),
},
};
case PRIMITIVE_TYPE.NUMBER:
case PRIMITIVE_TYPE.DECIMAL:
case PRIMITIVE_TYPE.FLOAT:
return {
...condition,
value: {
type: groupByColumn.type,
value: parseFloat(value),
},
};
case PRIMITIVE_TYPE.STRING:
case PRIMITIVE_TYPE.DATE:
case PRIMITIVE_TYPE.DATETIME:
case PRIMITIVE_TYPE.STRICTDATE:
case PRIMITIVE_TYPE.STRICTTIME:
default:
return {
...condition,
value: { type: groupByColumn.type, value },
};
}
}),
groupOperator: DataCubeQueryFilterGroupOperator.AND,
},
engine.filterOperations,
),
],
),
]),
);
}
// if pivot is present, populate the count aggregation columns
// by modifying pivot and pivot cast function expressions
const countAggColumns: V1_ColSpec[] = [];
_addCountAggColumnToPivot(funcMap, countAggColumns);
// modify groupBy() based off the current drilldown level
if (request.groupKeys.length < groupBy.columns.length) {
const groupByIdx = sequence.indexOf(funcMap.groupBy);
const groupByColumns = groupBy.columns.slice(
0,
request.groupKeys.length + 1,
);
const groupByFunc = _function(DataCubeFunction.GROUP_BY, [
_cols(groupByColumns.map((col) => _colSpec(col.name))),
_cols([
..._groupByAggCols(
groupByColumns,
snapshot,
configuration,
engine.aggregateOperations,
),
// if pivot is present, add sum aggregation columns for each
// count aggregation column from pivot aggregation
// else, simply add a count aggregation column
...(countAggColumns.length
? countAggColumns.map((col) => _aggCountCastCol(col.name))
: [_aggCountCol()]),
]),
]);
sequence[groupByIdx] = groupByFunc;
funcMap.groupBy = groupByFunc;
// update sort accordingly
const groupBySort = guaranteeNonNullable(funcMap.groupBySort);
groupBySort.parameters[0] = _collection(
groupByColumns.map((col) =>
_function(
configuration.treeColumnSortDirection ===
DataCubeQuerySortDirection.ASCENDING
? DataCubeFunction.ASCENDING
: DataCubeFunction.DESCENDING,
[_col(col.name)],
),
),
);
} else {
// when maximum level of drilldown is reached, we simply just need to
// filter the data to match drilldown values, no groupBy() is needed.
_unprocess('groupBy');
_unprocess('groupBySort');
}
}
};
}