flexmonster-mongo-connector
Version:
MongoDB connector for Flexmonster Pivot Table and Charts
318 lines • 15.3 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AggregationApiRequest = void 0;
const MongoResponseParser_1 = require("../../../parsers/MongoResponseParser");
const AbstractApiRequest_1 = require("./AbstractApiRequest");
const LoggingManager_1 = require("../../../logging/LoggingManager");
class AggregationApiRequest extends AbstractApiRequest_1.AbstractApiRequest {
constructor(requestArgument) {
super(requestArgument);
this.GROUPING_LIMITATION = 100000;
this._isPaginationEnabled = false;
this._templateQuery = null;
this._isFinished = false;
this._currentTimer = undefined;
this.parseQueryResult = (queryResult, date = null, preFilteredQueries) => MongoResponseParser_1.MongoResponseParser.getInstance().parseCalculationsFromCursor(queryResult, preFilteredQueries, this.CHUNK_SIZE, date, this);
this._loggingTemplate = "aggregations";
this._templateQuery = this._splitedQueries.shift();
this._isPaginationEnabled = this.isPaginationNecessary(this._splitedQueries);
if (this._isPaginationEnabled) {
this._splitedQueries.sort((first, second) => {
return first.queryStats.expectedNumberOfRecords - second.queryStats.expectedNumberOfRecords;
});
}
}
getData(queryBuilder, queryExecutor) {
return __awaiter(this, void 0, void 0, function* () {
const dataObject = yield this._getData(queryBuilder, queryExecutor);
this.loadDataAsync(queryBuilder, queryExecutor, dataObject);
return dataObject;
});
}
_getData(queryBuilder, queryExecutor) {
return __awaiter(this, void 0, void 0, function* () {
if (this._isPaginationEnabled)
this.applyPaginationStrategie(this._splitedQueries);
const preFilteredQueries = this._isPaginationEnabled
? this.preFilterQueries(this._splitedQueries)
: this._splitedQueries;
const mongoQuery = this.buildMongoQuery(queryBuilder, this._schema, preFilteredQueries);
LoggingManager_1.LoggingManager.log(`Getting ${this.loggingTemplate} data`);
LoggingManager_1.LoggingManager.log(`Generated pipeline query to MongoDB ${JSON.stringify(mongoQuery)}`);
const startDate = new Date();
const queryResultCursor = this.executeQuery(queryExecutor, mongoQuery);
return this.parseQueryResult(queryResultCursor, startDate, preFilteredQueries);
});
}
loadDataAsync(queryBuilder, queryExecutor, dataObject) {
if (this._isPaginationEnabled) {
this._isFinished = this.isEverythingLoaded(this._splitedQueries);
if (this._currentTimer !== undefined)
clearTimeout(this._currentTimer);
if (!this._isFinished) {
dataObject.isCompleted = false;
this._currentTimer = setTimeout(() => __awaiter(this, void 0, void 0, function* () {
const data = yield this.getData(queryBuilder, queryExecutor);
dataObject.push(data);
this.loadDataAsync(queryBuilder, queryExecutor, dataObject);
}), 100);
}
else {
this._isFinished = true;
dataObject.isCompleted = true;
}
}
else {
this._isFinished = true;
dataObject.isCompleted = true;
}
return;
}
buildMongoQuery(queryBuilder, schema, preFilteredQueries) {
if (queryBuilder == null)
throw new Error("Illegal argument exception");
if (preFilteredQueries === undefined)
preFilteredQueries = this._splitedQueries;
const mongoQuery = queryBuilder.buildAggregationPipelineFacet(preFilteredQueries, schema, this._templateQuery, this._isPaginationEnabled);
return mongoQuery;
}
preFilterQueries(queries) {
if (!this._isPaginationEnabled)
return queries;
const paginatedQueries = [];
for (let i = 0; i < queries.length; i++) {
if (!queries[i].queryStats.isAllQueryDataLoaded && queries[i].queryStats.chunkToLoad > 0) {
paginatedQueries.push(queries[i]);
}
}
return paginatedQueries;
}
updateLoadingStatus(data) {
if (data === undefined)
return;
for (let i = 0; i < this._splitedQueries.length; i++) {
const dataChunk = data[this._splitedQueries[i].definition];
if (dataChunk !== undefined) {
this._splitedQueries[i].queryStats.loadedNumberOfRecords = dataChunk.length;
this.updateQueryStats(this._splitedQueries[i]);
}
}
}
toJSON(response, nextPageToken) {
const jsonResponse = {
"aggs": response
};
if (nextPageToken != null)
jsonResponse["nextPageToken"] = nextPageToken;
return jsonResponse;
}
isAllQueryDataLoaded(query) {
return query.queryStats.loadedNumberOfRecords < query.queryStats.chunkToLoad ||
query.queryStats.sumOfLoadedRecords + query.queryStats.loadedNumberOfRecords >= query.queryStats.expectedNumberOfRecords;
}
updateQueryStats(query) {
if (this.isAllQueryDataLoaded(query)) {
query.queryStats.isAllQueryDataLoaded = true;
}
else {
query.queryStats.sumOfLoadedRecords += query.queryStats.loadedNumberOfRecords;
query.queryStats.chunkToLoad = 0;
query.queryStats.loadedNumberOfRecords = 0;
}
}
isEverythingLoaded(quries) {
for (let i = 0; i < quries.length; i++) {
if (!quries[i].queryStats.isAllQueryDataLoaded) {
return false;
}
}
return true;
}
_splitQuery(query) {
if (query == null)
throw new Error("Illegal argument exception");
const aggregationQueries = new Map();
const expectedNumberOfRecords = this.areSubTotalsAvailable(query) ?
this.getExpectedNumberOfRecords(query["aggs"]["by"]["rows"], query["aggs"]["values"]) +
this.getExpectedNumberOfRecords(query["aggs"]["by"]["cols"], query["aggs"]["values"]) :
0;
aggregationQueries.set("intersection", {
definition: "intersection",
clientQuery: query,
queryStats: {
expectedNumberOfRecords: expectedNumberOfRecords,
chunkToLoad: 0,
isAllQueryDataLoaded: false,
loadedNumberOfRecords: 0,
sumOfLoadedRecords: 0
}
});
this._splitIntersectionQuery(query, aggregationQueries);
if (this.areValuesAvailable(query)) {
this._splitSubTotalQuery(query, aggregationQueries);
this._splitGrandTotalQuery(query, aggregationQueries);
}
const uniqueAggregationQueries = Array.from(aggregationQueries.values());
return uniqueAggregationQueries;
}
applyPaginationStrategie(queries) {
let currentSizeChunk = 0;
const coeficient = 0.25;
while (currentSizeChunk < this.GROUPING_LIMITATION) {
currentSizeChunk = this.balanceGroupingLimitation(queries, currentSizeChunk, coeficient);
}
}
balanceGroupingLimitation(queries, currentSizeChunk, coeficient) {
let queryNumber = 0;
while (currentSizeChunk < this.GROUPING_LIMITATION && queryNumber < queries.length) {
if (queries[queryNumber].queryStats.isAllQueryDataLoaded ||
queries[queryNumber].queryStats.chunkToLoad === queries[queryNumber].queryStats.expectedNumberOfRecords) {
queryNumber++;
continue;
}
const query = queries[queryNumber];
let expectedChunkToLoad = (this.GROUPING_LIMITATION * coeficient > query.queryStats.expectedNumberOfRecords - query.queryStats.sumOfLoadedRecords) ?
query.queryStats.expectedNumberOfRecords - query.queryStats.sumOfLoadedRecords :
this.GROUPING_LIMITATION * coeficient;
if (currentSizeChunk + expectedChunkToLoad <= this.GROUPING_LIMITATION) {
query.queryStats.chunkToLoad += expectedChunkToLoad;
currentSizeChunk += expectedChunkToLoad;
}
else {
expectedChunkToLoad = this.GROUPING_LIMITATION - currentSizeChunk;
currentSizeChunk += expectedChunkToLoad;
}
queryNumber++;
}
return currentSizeChunk;
}
isPaginationNecessary(queryList) {
let sumOfExpectedRecords = 0;
for (let i = 0; i < queryList.length; i++) {
if (queryList[i].queryStats === undefined) {
continue;
}
sumOfExpectedRecords += queryList[i].queryStats.expectedNumberOfRecords;
}
return this.GROUPING_LIMITATION < sumOfExpectedRecords;
}
_splitIntersectionQuery(query, aggregationQueries) {
if (typeof query["aggs"] === "undefined" || typeof query["aggs"]["by"] === "undefined" ||
typeof query["aggs"]["by"]["rows"] === "undefined" || typeof query["aggs"]["by"]["cols"] === "undefined")
return;
const rowsList = query["aggs"]["by"]["rows"];
const colsList = query["aggs"]["by"]["cols"];
let rowsItems = [];
let colsItems = [];
let intersectionQuery = JSON.parse(JSON.stringify(query));
delete intersectionQuery["aggs"]["by"]["rows"];
delete intersectionQuery["aggs"]["by"]["cols"];
for (let i = 0; i < rowsList.length; i++) {
rowsItems.push(rowsList[i]);
colsItems = rowsItems.slice(0);
for (let j = 0; j < colsList.length; j++) {
colsItems = colsItems.slice(0);
colsItems.push(colsList[j]);
intersectionQuery = JSON.parse(JSON.stringify(intersectionQuery));
intersectionQuery["aggs"]["by"] = {
"cols": colsItems
};
aggregationQueries.set(JSON.stringify(colsItems), {
definition: "intersection" + i + j,
clientQuery: intersectionQuery,
queryStats: {
expectedNumberOfRecords: this.getExpectedNumberOfRecords(colsItems, intersectionQuery["aggs"]["values"]),
chunkToLoad: 0,
isAllQueryDataLoaded: false,
loadedNumberOfRecords: 0,
sumOfLoadedRecords: 0
}
});
}
}
return;
}
getExpectedNumberOfRecords(colsItems, valuesList) {
if (colsItems === undefined)
return 0;
let expectedNumberOfRecords = this._schema.fields.get(colsItems[0]["uniqueName"]).fieldStats.distinctMembersNumber;
let numberOfMeasures = valuesList === undefined ? 1 : valuesList.length;
for (let i = 1; i < colsItems.length; i++) {
expectedNumberOfRecords *= this._schema.fields.get(colsItems[i]["uniqueName"]).fieldStats.distinctMembersNumber;
}
expectedNumberOfRecords *= numberOfMeasures;
return expectedNumberOfRecords;
}
_splitSubTotalQuery(query, aggregationQueries) {
if (!this.areSubTotalsAvailable(query))
return;
const rowByQuery = JSON.parse(JSON.stringify(query));
delete rowByQuery["aggs"]["by"]["cols"];
const colsByQuery = JSON.parse(JSON.stringify(query));
delete colsByQuery["aggs"]["by"]["rows"];
if (rowByQuery["aggs"]["by"]["rows"] != null) {
this._generateAllSubtotalsCombinations(rowByQuery, aggregationQueries, "rows", "totalRows");
}
if (colsByQuery["aggs"]["by"]["cols"] != null) {
this._generateAllSubtotalsCombinations(colsByQuery, aggregationQueries, "cols", "totalColumns");
}
return;
}
_generateAllSubtotalsCombinations(query, aggregationQueries, axisName, definitionLabel) {
const rowsColumnsList = query["aggs"]["by"][axisName];
let axisItemsList = [];
let subTotalQuery = null;
for (let i = 0; i < rowsColumnsList.length; i++) {
subTotalQuery = JSON.parse(JSON.stringify(query));
axisItemsList = axisItemsList.slice(0);
axisItemsList.push(rowsColumnsList[i]);
subTotalQuery["aggs"]["by"][axisName] = axisItemsList;
aggregationQueries.set(JSON.stringify(axisItemsList), {
definition: definitionLabel + i,
clientQuery: subTotalQuery,
queryStats: {
expectedNumberOfRecords: this.getExpectedNumberOfRecords(axisItemsList, subTotalQuery["aggs"]["values"]),
chunkToLoad: 0,
isAllQueryDataLoaded: false,
loadedNumberOfRecords: 0,
sumOfLoadedRecords: 0
}
});
}
return;
}
areSubTotalsAvailable(query) {
return (query["aggs"] != null && query["aggs"]["by"] != null);
}
_splitGrandTotalQuery(query, aggregationQueries) {
const grandTotalQuery = JSON.parse(JSON.stringify(query));
delete grandTotalQuery["aggs"]["by"];
const definitionLabel = "grandTotal";
aggregationQueries.set(definitionLabel, {
definition: definitionLabel,
clientQuery: grandTotalQuery,
queryStats: {
expectedNumberOfRecords: grandTotalQuery["aggs"]["values"].length,
chunkToLoad: 0,
isAllQueryDataLoaded: false,
loadedNumberOfRecords: 0,
sumOfLoadedRecords: 0
}
});
return;
}
areValuesAvailable(query) {
return (query["aggs"] != null && query["aggs"]["values"] != null && query["aggs"]["values"].length != 0);
}
}
exports.AggregationApiRequest = AggregationApiRequest;
//# sourceMappingURL=AggregationApiRequest.js.map