flexmonster-mongo-connector
Version:
Custom data source API implementation for MongoDB
314 lines • 15.9 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MongoResponseParser = void 0;
const SchemaBuilder_1 = require("../schema/SchemaBuilder");
const Delimeters_1 = require("../utils/consts/Delimeters");
const MongoFieldType_1 = require("../utils/consts/MongoFieldType");
const ArrayDataObject_1 = require("../cache/dataObject/impl/ArrayDataObject");
const FlatRequestDataObject_1 = require("../cache/dataObject/impl/FlatRequestDataObject");
const MemoryCalculator_1 = require("../utils/MemoryCalculator");
class MongoResponseParser {
constructor() {
this.dotsDelimeterRegExp = new RegExp(Delimeters_1.Delimeter.DOTS_DELIMETER, "g");
if (MongoResponseParser._responseParserInstance != null) {
throw new Error("Initialization failed: " +
"use Singleton.getInstance() instead of new.");
}
MongoResponseParser._responseParserInstance = this;
}
static getInstance() {
if (MongoResponseParser._responseParserInstance == null) {
MongoResponseParser._responseParserInstance = new MongoResponseParser();
}
return MongoResponseParser._responseParserInstance;
}
parseShemaFromDocument(document) {
return SchemaBuilder_1.SchemaBuilder.getInstance().createShemaFromDocument(document);
}
_parseValues(values, storage) {
let measure = null;
let measureKey = null;
let valueParsed = null;
let dataMemorySize = MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
for (let key in values) {
if (key == "_id")
continue;
measure = key.toString().split(Delimeters_1.Delimeter.FIELD_DELIMETER);
measureKey = measure[0].replace(this.dotsDelimeterRegExp, '.');
if (typeof storage[measureKey] === "undefined") {
storage[measureKey] = {};
dataMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE + MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
}
valueParsed = this.parseValueFromComplexType(values[key]);
storage[measureKey][measure[1]] = valueParsed.value;
dataMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE + valueParsed.memorySize;
}
return {
"value": storage,
"memorySize": MemoryCalculator_1.MemoryCalculator.roundTo8bytes(dataMemorySize)
};
}
parseCalculationsFromCursor(cursor, query, dataChunkSize, startDate = null, aggregationApiRequest) {
return new Promise((resolve, reject) => {
cursor.then((documents) => __awaiter(this, void 0, void 0, function* () {
const parsingResult = yield this.parseAggregations(documents, query, dataChunkSize, startDate, aggregationApiRequest);
resolve(new ArrayDataObject_1.ArrayDataObject(parsingResult.result, parsingResult.startDate, parsingResult.memorySize));
}));
});
}
parseAggregations(documents, queries, dataChunkSize, startDate = null, aggregationApiRequest) {
return __awaiter(this, void 0, void 0, function* () {
let parsedData = [];
let dataChunk = [];
let keysParsed = null;
let valuesParsed = null;
let dataMemorySize = 0;
let parseStart = null;
yield documents.forEach((data) => {
parseStart = new Date();
aggregationApiRequest.updateLoadingStatus(data);
let queryDefinition = null;
for (let i = 0; i < queries.length; i++) {
queryDefinition = queries[i].definition;
for (let j = 0; j < data[queryDefinition].length; j++) {
keysParsed = this.parseDotsFromKeys(data[queryDefinition][j]["_id"]);
valuesParsed = this._parseValues(data[queryDefinition][j], {});
if (dataChunkSize <= dataChunk.length) {
parsedData.push(dataChunk);
dataChunk = [];
dataMemorySize += 2 * MemoryCalculator_1.MemoryConstants.REFERENCE;
}
dataChunk.push({
"keys": keysParsed["value"],
"values": valuesParsed["value"]
});
dataMemorySize += 2 * MemoryCalculator_1.MemoryConstants.REFERENCE + 2 * MemoryCalculator_1.MemoryConstants.REFERENCE + keysParsed.memorySize + valuesParsed.memorySize;
}
data[queryDefinition] = null;
}
parsedData.push(dataChunk);
dataMemorySize += 4 * MemoryCalculator_1.MemoryConstants.REFERENCE;
});
return {
startDate: startDate,
result: parsedData,
memorySize: dataMemorySize
};
});
}
parseMembersFromCursor(cursor, fieldObject, dataChunkSize, startDate) {
return new Promise((resolve, reject) => {
const fieldUniqueName = fieldObject.uniqueName;
let membersMemorySize = 0;
cursor.then((documents) => {
let result = [];
let membersChunk = [];
let keyWithoutDots = fieldUniqueName.replace(/\./g, Delimeters_1.Delimeter.DOTS_DELIMETER);
let parsedResult = null;
documents.forEach((data) => {
let value = data["_id"][keyWithoutDots];
if (membersChunk.length >= dataChunkSize) {
result.push(membersChunk);
membersChunk = [];
membersMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
}
parsedResult = this.parseValueFromComplexType(value);
const membersObject = {
"value": parsedResult.value
};
membersMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE * 2 + MemoryCalculator_1.MemoryConstants.REFERENCE + parsedResult.memorySize;
membersChunk.push(membersObject);
}, () => {
result.push(membersChunk);
membersMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE * 2 + MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
resolve(new ArrayDataObject_1.ArrayDataObject(result, startDate, membersMemorySize));
});
});
});
}
parseFlatFromCursor(cursor, fields, queries, dataChunkSize, startDate) {
return new Promise((resolve, reject) => {
const result = {
"fields": [],
"hits": [],
"aggs": []
};
let hitsChunk = [];
let keys = null;
let values = null;
let flatDataMemorySize = 0;
let parsedData = null;
cursor.then((documents) => __awaiter(this, void 0, void 0, function* () {
yield documents.forEach((data) => {
let queryDefinition = null;
for (let i = 0; i < queries.length; i++) {
queryDefinition = queries[i].definition;
if (queryDefinition === "dataRecords") {
const fieldTypes = this.defineFieldTypesFromData(data[queryDefinition][0]);
for (let j = 0; j < data[queryDefinition].length; j++) {
if (hitsChunk.length >= dataChunkSize) {
result.hits.push(hitsChunk);
flatDataMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
hitsChunk = [];
}
parsedData = this.parseDrillThroughHit(data[queryDefinition][j], fields, fieldTypes);
flatDataMemorySize += parsedData.hitMemorySize;
hitsChunk.push(parsedData.hit);
}
result.hits.push(hitsChunk);
flatDataMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE * 4;
}
else {
for (let j = 0; j < data[queryDefinition].length; j++) {
keys = this.parseDotsFromKeys(data[queryDefinition][j]["_id"]).value;
values = this._parseValues(data[queryDefinition][j], {});
result.aggs.push({
"keys": keys,
"values": values.value
});
}
}
}
});
result["fields"] = this.parseDrillThroughFields(fields);
flatDataMemorySize += MemoryCalculator_1.MemoryCalculator.calculateFlatDataSize(result);
resolve(new FlatRequestDataObject_1.FlatResultDataObject(result, flatDataMemorySize, startDate));
}));
});
}
defineFieldTypesFromData(dataRow) {
const fieldTypes = [];
for (let i = 0; i < dataRow.length; i++) {
if (typeof dataRow[i] === "number") {
fieldTypes.push(MongoFieldType_1.MongoFieldType.NUMBER);
}
else {
fieldTypes.push(MongoFieldType_1.MongoFieldType.STRING);
}
}
return fieldTypes;
}
parseDrillThroughFromCursor(cursor, fields, dataChunkSize, startDate) {
return new Promise((resolve, reject) => {
cursor.then((documents) => __awaiter(this, void 0, void 0, function* () {
const parsingResult = yield this.parseDrillThroughData(documents, fields, dataChunkSize);
resolve(new FlatRequestDataObject_1.FlatResultDataObject(parsingResult.data, parsingResult.dataMemorySize, startDate));
}));
});
}
parseDrillThroughData(documents, fields, dataChunkSize) {
return __awaiter(this, void 0, void 0, function* () {
const result = {
"fields": [],
"hits": []
};
let document = null;
let hitsChunk = [];
let fieldTypes = null;
let drillThroughMemoryDataSize = 0;
let parsingResult = null;
yield documents.forEach((data) => {
document = data;
fieldTypes = fieldTypes === null ? this.defineFieldTypesFromData(document) : fieldTypes;
if (hitsChunk.length >= dataChunkSize) {
result.hits.push(hitsChunk);
drillThroughMemoryDataSize += MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
hitsChunk = [];
}
parsingResult = this.parseDrillThroughHit(document, fields, fieldTypes);
drillThroughMemoryDataSize += parsingResult.hitMemorySize;
hitsChunk.push(parsingResult.hit);
});
result.hits.push(hitsChunk);
result["fields"] = this.parseDrillThroughFields(fields);
drillThroughMemoryDataSize += MemoryCalculator_1.MemoryCalculator.calculateFlatDataSize(result);
drillThroughMemoryDataSize += MemoryCalculator_1.MemoryConstants.REFERENCE * 4;
return {
data: result,
dataMemorySize: drillThroughMemoryDataSize
};
});
}
parseDrillThroughFields(fieldsFromQuery) {
const fields = [];
for (let i = 0; i < fieldsFromQuery.length; i++) {
fields.push({
"uniqueName": fieldsFromQuery[i].uniqueName
});
}
return fields;
}
parseDrillThroughHit(document, fieldsFromQuery, fieldsTypes) {
const hit = [];
let parsedValue = null;
let hitMemorySize = MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
for (let i = 0; i < fieldsFromQuery.length; i++) {
const fieldKey = fieldsFromQuery[i].uniqueName;
parsedValue = this._getNestedObjectValue(fieldKey, document);
hitMemorySize += parsedValue.memorySize;
hit.push(parsedValue.value);
}
return {
hit: hit,
hitMemorySize: hitMemorySize
};
}
_getNestedObjectValue(key, data) {
let keyItems = key.split('.');
let value = null;
let i = 0;
while (i < keyItems.length) {
data = data[keyItems[i]];
i++;
}
value = data;
return this.parseValueFromComplexType(value);
}
parseValueFromComplexType(value) {
let resultValue = value;
let memorySize = 0;
if (value != null && value["_bsontype"] != null && value["_bsontype"] == MongoFieldType_1.MongoFieldType.DECIMAL128) {
resultValue = Number(value);
memorySize += MemoryCalculator_1.MemoryConstants.NUMBER;
}
else if (value != null && value["_bsontype"] != null) {
resultValue = value.toString();
memorySize += resultValue == null ? MemoryCalculator_1.MemoryConstants.REFERENCE : resultValue.length * MemoryCalculator_1.MemoryConstants.CHARACTER;
}
else {
memorySize += resultValue == null ? MemoryCalculator_1.MemoryConstants.REFERENCE : (resultValue.length === undefined ? MemoryCalculator_1.MemoryConstants.NUMBER : resultValue.length * MemoryCalculator_1.MemoryConstants.CHARACTER);
}
return {
"value": resultValue,
"memorySize": MemoryCalculator_1.MemoryCalculator.roundTo8bytes(memorySize)
};
}
parseDotsFromKeys(fieldsKeys) {
let result = {};
let objectMemorySize = MemoryCalculator_1.MemoryConstants.REFERENCE * 2;
let keyValue = null;
let parsedValue = null;
for (let key in fieldsKeys) {
keyValue = key.replace(this.dotsDelimeterRegExp, '.');
parsedValue = this.parseValueFromComplexType(fieldsKeys[key]);
result[keyValue] = parsedValue.value;
objectMemorySize += MemoryCalculator_1.MemoryConstants.REFERENCE + parsedValue.memorySize;
}
return {
"value": result,
"memorySize": MemoryCalculator_1.MemoryCalculator.roundTo8bytes(objectMemorySize)
};
}
}
exports.MongoResponseParser = MongoResponseParser;
MongoResponseParser._responseParserInstance = null;
//# sourceMappingURL=MongoResponseParser.js.map