mingo-exp
Version:
extend mingo with any custom function and support async code in aggregations
298 lines (296 loc) • 15.2 kB
JavaScript
;
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AsyncAggregator = exports.$collect = exports.$runOnce = exports.generateCustomOperator = exports.customParseExpression = void 0;
var unkink_polygon_1 = __importDefault(require("@turf/unkink-polygon"));
var async_lock_1 = __importDefault(require("async-lock"));
var dayjs_1 = __importDefault(require("dayjs"));
var deasync_obj_1 = require("deasync-obj");
var lodash_1 = __importDefault(require("lodash"));
var memory_cache_1 = __importDefault(require("memory-cache"));
var mingo_1 = require("mingo");
var core_1 = require("mingo/core");
var pipeline_1 = require("mingo/operators/pipeline");
var object_hash_1 = __importDefault(require("object-hash"));
var uuid_1 = require("uuid");
var wkx_1 = require("wkx");
var lock = new async_lock_1.default();
var defaultCustomFunctions = {
date: function (val) { return (0, dayjs_1.default)(val).toISOString(); },
number: function (val) { return lodash_1.default.toNumber(val); },
text: function (val) { return lodash_1.default.toString(val).trim(); },
boolean: function (val) {
if (lodash_1.default.isBoolean(val))
return val;
if (!lodash_1.default.isString(val))
return undefined;
if (val.toLowerCase() === 'true')
return true;
if (val.toLowerCase() === 'false')
return false;
return undefined;
},
point: function (longitude, latitude) { return new wkx_1.Point(longitude, latitude).toWkt(); },
wkt: function (wktString) {
var geometry = wkx_1.Geometry.parse(wktString);
var _a = geometry.toGeoJSON(), type = _a.type, coordinates = _a.coordinates;
if (['Polygon', 'MultiPolygon', 'Feature', 'FeatureCollection'].includes(type)) {
var features = (0, unkink_polygon_1.default)({ type: type, coordinates: coordinates }).features;
if (features.length > 1) {
var geometries = features.map(function (_a) {
var geometry = _a.geometry;
return wkx_1.Geometry.parseGeoJSON(geometry);
});
return new wkx_1.GeometryCollection(geometries).toWkt();
}
}
return geometry.toWkt();
},
geoJson: function (obj) { return wkx_1.Geometry.parseGeoJSON(obj).toWkt(); },
orDefault: function (val, defaultVal) { return val !== null && val !== void 0 ? val : defaultVal; },
concat: function (separator) {
var args = [];
for (var _i = 1; _i < arguments.length; _i++) {
args[_i - 1] = arguments[_i];
}
return lodash_1.default.compact(args).join(separator);
}
};
/**
* alternative for generateCustomOperator, generate single operator that can execute custom functions by type and arguments
* @augments additionalCustomFunctions object with parsing functions that can be accessed by their type(key in this object) and args, you can pass any function you want to use in aggregations, async functions are supported using AsyncAggregator
* also gives useful default parsing functions
* @returns default type-> date: (val) => string // ISOString using dayjs
* @returns default type-> number: (val) => number
* @returns default type-> text: (val) => string // trimmed string
* @returns default type-> boolean: (val: string | boolean) => boolean // support for boolean or string representations: 'TRUE' 'False' 'true' false
* @returns default type-> point: (longitude: number, latitude: number) => string // wkt string
* @returns default type-> wkt: (wktString: string) => string // valid wkt string (without self intersections using '@turf/unkink-polygon')
* @returns default type-> geoJson: (obj) => string // wkt string
* @returns default type-> orDefault: (val, defaultVal) => unknown
* @returns default type-> concat: (separator: string, ...args: string[]) => string
* @example const { $customParse } = customParseExpression({
multAsync: async (num1: number, num2: number) => {
await delay(1000);
return num1*num2;
},
lodashGet: _.get
});
* @example useOperators(OperatorType.EXPRESSION, { $customParse }); //must do to be able to use in aggregations
* @example {
$project: {
id: '$a.b',
response: { type: 'multAsync', args: ['$a.b', 5] } // this will run $multAsync(doc.a.b, 5), arguments are passed by order in an array
}
},...
*
*/
function customParseExpression(additionalCustomFunctions) {
if (additionalCustomFunctions === void 0) { additionalCustomFunctions = {}; }
var customFunctions = __assign(__assign({}, defaultCustomFunctions), additionalCustomFunctions);
var $customParse = function (obj, expr, options) {
var _a;
var computedValue = (0, core_1.computeValue)(obj, expr, undefined, options);
return (_a = customFunctions)[computedValue.type].apply(_a, (computedValue.args));
};
return {
customFunctions: customFunctions,
$customParse: $customParse
};
}
exports.customParseExpression = customParseExpression;
/**
* simplified operation custom generation
* @augments customOperators object with keys that are names for custom operators and values are any function you want to use in aggregation, async functions are supported using AsyncAggregator
* @example const { $multAsync, $lodashGet } = generateCustomOperator({
$multAsync: async (num1: number, num2: number) => {
await delay(1000);
return num1*num2;
},
$lodashGet: _.get
});
* @example useOperators(OperatorType.EXPRESSION, { $multAsync, $lodashGet }); //must do to be able to use in aggregations
* @example {
$project: {
id: '$a.b',
response: { $multAsync: ['$a.b', 5] } // this will run $multAsync(doc.a.b, 5), arguments are passed by order in an array
}
},...
*
*/
function generateCustomOperator(customOperators) {
var generatedCustomOperators = lodash_1.default.clone(customOperators);
Object.keys(generatedCustomOperators).forEach(function (key) {
generatedCustomOperators[key] = function (obj, args, options) {
var computedArgs = (0, core_1.computeValue)(obj, args, undefined, options);
return customOperators[key].apply(customOperators, (computedArgs));
};
});
return generatedCustomOperators;
}
exports.generateCustomOperator = generateCustomOperator;
var MEMORY_CACHE_TIMEOUT = 600000;
/**
* custom AsyncAggregator expression, caches request based on operation and its arguments, run it only once and returns the same response
* no need to call useOperators(OperatorType.EXPRESSION, { $runOnce }) because AsyncAggregator calls it already
* @example {
$project: {
id: '$a.b',
response: { $runOnce: { $someAsyncCustomOperator: ['$ids'] } } // this will run someAsyncCustomOperator(ids) only once and use the response for every doc that called someAsyncCustomOperator(ids) with the same ids
}
},...
*
*/
function $runOnce(obj, args, options) {
return __awaiter(this, void 0, void 0, function () {
var pipelineId, argsHash, cacheKey;
return __generator(this, function (_a) {
pipelineId = (0, core_1.computeValue)({ pipelineId: '$pipelineId' }, '$pipelineId', undefined, options);
argsHash = (0, object_hash_1.default)(args);
cacheKey = "runOnce_".concat(pipelineId).concat(argsHash);
return [2 /*return*/, lock.acquire(cacheKey, function () {
var existingResponse = memory_cache_1.default.get(cacheKey);
if (lodash_1.default.isNil(existingResponse)) {
existingResponse = (0, core_1.computeValue)(obj, args, undefined, options);
memory_cache_1.default.put(cacheKey, existingResponse, MEMORY_CACHE_TIMEOUT);
}
return existingResponse;
})];
});
});
}
exports.$runOnce = $runOnce;
/**
* custom AsyncAggregator expression, collects values to an array
* no need to call useOperators(OperatorType.EXPRESSION, { $collect }) because AsyncAggregator calls it already
* @example {
$project: {
id: '$a.b',
idsForBatch: { $collect: '$a.b' } // this will put in every doc an array named idsForBatch with every value of $a.b
}
},...
*
*/
function $collect(obj, args, options) {
return __awaiter(this, void 0, void 0, function () {
var pipelineId, argsHash, cacheKey;
return __generator(this, function (_a) {
pipelineId = (0, core_1.computeValue)({ pipelineId: '$pipelineId' }, '$pipelineId', undefined, options);
argsHash = (0, object_hash_1.default)(args);
cacheKey = "collect_".concat(pipelineId).concat(argsHash);
return [2 /*return*/, lock.acquire(cacheKey, function () {
var existingResponse = memory_cache_1.default.get(cacheKey);
if (lodash_1.default.isNil(existingResponse)) {
existingResponse = [(0, core_1.computeValue)(obj, args, undefined, options)];
memory_cache_1.default.put(cacheKey, existingResponse, MEMORY_CACHE_TIMEOUT);
}
else {
existingResponse.push((0, core_1.computeValue)(obj, args, undefined, options));
}
return existingResponse;
})];
});
});
}
exports.$collect = $collect;
/**
* Provides functionality for the mongoDB aggregation pipeline
* similar to Aggregator but with async support (awaits all promises after each step in the aggregation)
* uses $collect and $runOnce custom expression operators by default
*
* @param pipeline an Array of pipeline operators
* @param options An optional Options to pass the aggregator
* @constructor
*/
var AsyncAggregator = /** @class */ (function () {
function AsyncAggregator(pipeline, options) {
this.pipeline = pipeline;
this.options = options;
(0, core_1.useOperators)(core_1.OperatorType.PIPELINE, { $addFields: pipeline_1.$addFields, $unset: pipeline_1.$unset });
(0, core_1.useOperators)(core_1.OperatorType.EXPRESSION, { $runOnce: $runOnce, $collect: $collect });
}
AsyncAggregator.prototype.addPipelineIdPipe = function () {
var pipelineId = (0, uuid_1.v4)();
return { $addFields: { pipelineId: pipelineId } };
};
AsyncAggregator.prototype.unsetPipelineIdPipe = function () {
return { $unset: 'pipelineId' };
};
AsyncAggregator.prototype.run = function (collection) {
return __awaiter(this, void 0, void 0, function () {
var aggregators, _i, aggregators_1, agg;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
aggregators = this.pipeline.map(function (pipe) { return new mingo_1.Aggregator([_this.addPipelineIdPipe(), pipe, _this.unsetPipelineIdPipe()], _this.options); });
_i = 0, aggregators_1 = aggregators;
_a.label = 1;
case 1:
if (!(_i < aggregators_1.length)) return [3 /*break*/, 4];
agg = aggregators_1[_i];
collection = agg.run(collection);
return [4 /*yield*/, (0, deasync_obj_1.deasyncObj)(collection)];
case 2:
_a.sent();
_a.label = 3;
case 3:
_i++;
return [3 /*break*/, 1];
case 4: return [2 /*return*/, collection];
}
});
});
};
return AsyncAggregator;
}());
exports.AsyncAggregator = AsyncAggregator;