dce-mango
Version:
Harvard DCE's Non-relational DB Wrapper.
555 lines • 23.9 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
// Import for testing
const console_1 = require("console");
// Import UUID
const uuid_1 = require("uuid");
// Import state
const dbState_1 = __importDefault(require("../dbState"));
// Import helpers
const initCollection_1 = __importDefault(require("../helpers/initCollection"));
const processFindResult_1 = __importDefault(require("../helpers/processFindResult"));
const getLockCollectionName_1 = __importDefault(require("../helpers/getLockCollectionName"));
const getLockCollectionOpts_1 = __importDefault(require("../helpers/getLockCollectionOpts"));
const pollResource_1 = __importStar(require("../helpers/pollResource"));
// Import constants
const LOCK_POLL_INTERVAL_MS_1 = __importDefault(require("../constants/LOCK_POLL_INTERVAL_MS"));
const DEFAULT_LOCK_TTL_MS_1 = __importDefault(require("../constants/DEFAULT_LOCK_TTL_MS"));
/*------------------------------------------------------------------------*/
/* Collection */
/*------------------------------------------------------------------------*/
class Collection {
/**
* Create a new Collection
* @author Gabe Abrams
* @param collectionName the collection name
* @param options the options for the collection (used when
* creating it)
* @param [options.uniqueIndexKey] the name of the unique index
* (only created if included)
* @param [options.expireAfterSeconds=no expiry] if unique index is
* created, this is the number of seconds before items on each key expire
* @param [options.indexKeys] the names of keys to build
* secondary indexes on
*/
constructor(collectionName, options = {}) {
var _a;
// Whether the collection supports concurrency
this.supportConcurrency = false;
// Save collection name
this.collectionName = collectionName;
// Remember unique index
this.uniqueKey = options.uniqueIndexKey;
// Make sure init has been called first
if (!dbState_1.default.schemaVersionTag
|| !dbState_1.default.initDB) {
// Init hasn't been called yet
// eslint-disable-next-line no-console
(0, console_1.log)('DCE-MANGO: Collection class instance created before initMango was called. Fatal error. Exiting.');
process.exit(1);
}
// Initialize and save the promise so other functions can wait for the
// initialization to complete
this.collection = (0, initCollection_1.default)(collectionName, options);
if (options.supportConcurrency) {
this.supportConcurrency = true;
if (!this.uniqueKey) {
throw new Error('A collection without a `uniqueIndexKey` cannot support concurrency.');
}
const lockCollectionName = (0, getLockCollectionName_1.default)(collectionName);
const lockCollectionOpts = (0, getLockCollectionOpts_1.default)(options.atomicUpdateTimeoutMs);
this.serverId = (0, uuid_1.v4)();
this.lockCollection = (0, initCollection_1.default)(lockCollectionName, lockCollectionOpts);
this.lockTimeToLiveMS = (_a = options.atomicUpdateTimeoutMs) !== null && _a !== void 0 ? _a : DEFAULT_LOCK_TTL_MS_1.default;
}
}
/*------------------------------------------------------------------------*/
/* Getter Functions */
/*------------------------------------------------------------------------*/
/**
* Get whether the collection supports concurrency or not.
* @author Benedikt Arnarsson
* @returns boolean indicating whether the collection supports concurrency.
*/
getSupportConcurrency() {
return this.supportConcurrency;
}
/*------------------------------------------------------------------------*/
/* Public Functions */
/*------------------------------------------------------------------------*/
/**
* Run a query
* @author Gabe Abrams
* @param query the query to run
* @param [includeMongoTimestamp] if true, include the timestamp
* in the mongo objects
* @returns documents
*/
find(query, includeMongoTimestamp) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Get the list of matching items
const items = yield collection.find(query).toArray();
// Filter out internal ids and add timestamps
const processedItems = items.map((item) => {
return (0, processFindResult_1.default)(item, includeMongoTimestamp);
});
return (items
? processedItems
: []);
});
}
/**
* Run a query with pagination
* @author Yuen Ler Chow
* @param query the query to run
* @param perPage the number of items per page
* @param pageNumber the page number to return, 1-indexed
* @param [includeMongoTimestamp] if true, include the timestamp
* in the mongo objects
* @param [sortKey] the key to sort by, or _id if not provided
* @param [sortDescending] if true, sort descending
* @returns documents
*/
findPaged(opts) {
return __awaiter(this, void 0, void 0, function* () {
const { query, perPage = 10, pageNumber = 1, includeMongoTimestamp, sortKey = '_id', sortDescending, } = opts;
if (pageNumber < 1) {
throw new Error('Page number must be at least 1');
}
if (perPage < 1) {
throw new Error('Per page must be at least 1');
}
// Wait for initialization to complete
const collection = yield this.collection;
// Get the list of matching items for the specified page
const items = yield (collection
.find(query)
.sort({ [sortKey]: sortDescending ? -1 : 1 })
.skip(perPage * (pageNumber - 1))
// Get 1 extra item to check if there is another page
.limit(perPage + 1)
.toArray());
const hasAnotherPage = items.length > perPage;
// Remove the extra item if there is another page
if (hasAnotherPage) {
items.pop();
}
// Filter out internal ids and add timestamps
const processedItems = items.map((item) => {
return (0, processFindResult_1.default)(item, includeMongoTimestamp);
});
return {
items: (items
? processedItems
: []),
currentPageNumber: pageNumber,
perPage,
hasAnotherPage,
};
});
}
/**
* Find elements then only return the values for one specific property from
* each item
* @author Gabe Abrams
* @param query the query to run
* @param prop the name of the property to extract
* @param [excludeFalsy] if true, exclude falsy values
* @returns array of values of the property
*/
findAndExtractProp(query, prop, excludeFalsy) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Get the list of matching items
const items = yield collection.find(query, {
projection: { [prop]: 1 },
}).toArray();
// Only return the value of the prop, filter falsy values if requested
return (items
.map((item) => {
return item[prop];
})
.filter((item) => {
return (!excludeFalsy || item);
}));
});
}
/**
* Count the number of matching elements
* @author Gabe Abrams
* @param query the query to run
* @returns number of documents that match
*/
count(query) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Count
const count = yield collection.countDocuments(query);
return count;
});
}
/**
* List distinct values for a property in a collection
* @author Gabe Abrams
* @param prop the property to list distinct values for
* @param [query] the query to run. If excluded, all distinct
* values are included
* @returns array of distinct values
*/
distinct(prop, query) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Get distinct values
const distinctValues = yield collection.distinct(prop, query !== null && query !== void 0 ? query : {});
return distinctValues;
});
}
/**
* Increment value of an integer for an object
* @author Gabe Abrams
* @param id id of the object to increment
* @param prop property to increment
*/
increment(id, prop) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Increment
return collection.findOneAndUpdate({
id,
}, {
$inc: {
[prop]: 1,
},
});
});
}
/**
* Increment value of an integer for an object, found by a query
* @author Gabe Abrams
* @param query query to find the object to increment
* @param prop property to increment
*/
incrementByQuery(query, prop) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Increment
return collection.findOneAndUpdate(query, {
$inc: {
[prop]: 1,
},
});
});
}
/**
* Add/update object values in an entry in the collection. The entry must
* already exist
* @author Gabe Abrams
* @param query query to apply to find the object to update
* @param updates map of updates: { prop => value } where prop is
* the potentially nested name of the property
* (e.g. "age" or "profile.age")
*/
updatePropValues(query, updates) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Perform update
return collection.findOneAndUpdate(query, {
$set: updates,
});
});
}
/**
* Add an object to an array in an object
* @author Gabe Abrams
* @param id the id of the object to modify
* @param arrayProp the name of the array to insert into
* @param obj the object to insert into the array
*/
push(id, arrayProp, obj) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Push
return collection.findOneAndUpdate({
id,
}, {
$push: {
[arrayProp]: obj,
},
});
});
}
/**
* Filter an array of objects or primitives in an entry.
* @author Gabe Abrams
* @param opts object containing all args
* @param opts.id the id of the object to modify
* @param opts.arrayProp the name of the array to filter
* @param [opts.compareProp] the name of the array entry prop to compare.
* @param opts.compareValue the value of the array entry prop to filter
* out
*/
filterOut(opts) {
return __awaiter(this, void 0, void 0, function* () {
const { id, arrayProp, compareProp, compareValue, } = opts;
// Wait for initialization to complete
const collection = yield this.collection;
// Perform update
if (!compareProp) {
return collection.findOneAndUpdate({
id,
}, {
$pull: {
[arrayProp]: compareValue,
}, // Cast is required because mongodb lib uses an improper type
});
}
return collection.findOneAndUpdate({
id,
}, {
$pull: {
[arrayProp]: {
[compareProp]: compareValue,
},
}, // Cast is required because mongodb lib uses an improper type
});
});
}
/**
* Write a record to the collection
* @author Gabe Abrams
* @param obj the object to insert
*/
insert(obj) {
return __awaiter(this, void 0, void 0, function* () {
// Remove stuff from object
const updatedObj = obj;
delete updatedObj.mongoTimestamp;
// Wait for initialization to complete
const collection = yield this.collection;
// Check if we are inserting uniquely
if (this.uniqueKey) {
// Unique! Use replacement
const query = {
// Cast obj to any to force allowing this operation
[this.uniqueKey]: updatedObj[this.uniqueKey],
};
yield collection.updateOne(query, // Query to find the option to replace
{ $set: updatedObj }, // Type of update is a "set" operation
{ upsert: true });
}
else {
// Not unique. Just insert
yield collection.insertOne(updatedObj);
}
});
}
/**
* Delete the first document that matches the query in the collection
* @author Gabe Abrams
* @param query the query that will match the item
*/
delete(query) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Delete the object
yield collection.deleteOne(query);
});
}
/**
* Delete all documents that match the query in the collection
* @author Gabe Abrams
* @param query the query that will match the items to delete
*/
deleteAll(query) {
return __awaiter(this, void 0, void 0, function* () {
// Wait for initialization to complete
const collection = yield this.collection;
// Delete all matches
yield collection.deleteMany(query);
});
}
/*------------------------------------------------------------------------*/
/* Concurrent Methods */
/*------------------------------------------------------------------------*/
/**
* Creates a lock in the lock collection, to make sure modifications aren't made to the document with id=id.
* @author Benedikt Arnarsson
* @param id the 'id' of the document we want to lock.
*/
lock(id) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
if (!this.supportConcurrency) {
throw new Error('Cannot call lock on a Collection without concurrency support!');
}
const lockCollection = yield this.lockCollection;
const { serverId } = this;
// Creating the lock with the document Id and serverId
const lock = {
id,
serverId,
};
/**
* Process of acquiring the lock from the lock-collection, used by pollResource.
* @author Benedikt Arnarsson
* @returns the Lock which is needed for concurrent process, must be of type Promise<MongoDB.UpdateResult>
*/
const callResource = () => __awaiter(this, void 0, void 0, function* () {
// Using updateOne w/ setOnInsert & upsert will make it only insert when there is no lock with same document Id
return lockCollection.updateOne(
// TODO: add time of insertion
{ id }, { $setOnInsert: lock }, { upsert: true });
});
/**
* Process of validating that we acquired the lock from the lock-collection, used by pollResource.
* @author Benedikt Arnarsson
* @param res the output of acquiring the resource, in this case the lock.
* @returns boolean indicating whether we successfully acquired the lock.
*/
const validateResult = (res) => {
return res.upsertedCount > 0;
};
// Initiate pollResource
const result = yield (0, pollResource_1.default)({
callResource,
validateResult,
waitForMS: LOCK_POLL_INTERVAL_MS_1.default,
// FIXME: when multiple servers are working
// Add some randomness here so other servers can acquire lock
timeOut: 2 * this.lockTimeToLiveMS,
});
// Logging
if (((_a = process.env.MANGO_LOG_LEVEL) !== null && _a !== void 0 ? _a : '').toLowerCase() === 'info') {
(0, console_1.log)(`Locking on ${serverId} (${this.collectionName}):\n\t- upserted: ${result.upsertedCount}\n\t- matched: ${result.matchedCount}\n\t- modified: ${result.modifiedCount}`);
}
});
}
/**
* Inverse of lock operation. Unlock a document which you have locked.
* @author Benedikt Arnarsson
* @param id the 'id' of the document that is being unlocked.
*/
unlock(id) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
if (!this.supportConcurrency) {
throw new Error('Cannot call unlock on a Collection without concurrency support!');
}
const lockCollection = yield this.lockCollection;
const { serverId } = this;
// Only deletes a lock with the same id *and* serverId
yield lockCollection.deleteOne({ id, serverId });
// Logging
if (((_a = process.env.MANGO_LOG_LEVEL) !== null && _a !== void 0 ? _a : '').toLowerCase() === 'info') {
(0, console_1.log)(`Unlocked document ${id}, with ${serverId} (${this.collectionName})`);
}
});
}
/**
* Given a function representing a set of operations on the collection and a set of ids to lock,
* will run the function such that other collections cannot make modifications while the function runs.
* For use in situations with multiple concurrent servers using the same database.
* @author Benedikt Arnarsson
* @param opts object containing all parameters
* @param opts.idOrIdsToLock the one Id or list of Ids to lock for the procedure provided.
* @param opts.procedure the procedure that is being wrapped in the lock-unlock calls
* @returns the result of opts.procedure
*/
runAtomicProcedure(opts) {
return __awaiter(this, void 0, void 0, function* () {
if (!this.supportConcurrency) {
throw new Error('Cannot call runAtomicProcedure on a Collection without concurrency support!');
}
// Destructure
const { idOrIdsToLock, procedure, } = opts;
const collection = yield this.collection;
const ids = (Array.isArray(idOrIdsToLock)
? idOrIdsToLock
: [idOrIdsToLock]);
const query = Object.fromEntries([
[this.uniqueKey, { $in: ids }],
]);
// Get the list of matching items
const items = yield collection.find(query).toArray();
// Filter out internal ids
const uniqueIndexValues = (items
? items.map((item) => {
return item[this.uniqueKey];
})
: []);
let result;
try {
// Lock all items
yield Promise.all(uniqueIndexValues.map((uniqueIndexValue) => __awaiter(this, void 0, void 0, function* () {
yield this.lock(uniqueIndexValue);
})));
// Execute the procedure
result = yield procedure(this);
}
catch (err) {
if (err instanceof pollResource_1.PollingError) {
throw new pollResource_1.PollingError('Exceeded timeout when attempting to lock for atomic procedure.');
}
else {
const errMsg = 'Failed to complete atomic procedure:';
err.message = `${errMsg} ${err.message}`;
throw err;
}
}
finally {
// Unlock all items
yield Promise.all(uniqueIndexValues.map((uniqueIndexValue) => __awaiter(this, void 0, void 0, function* () {
yield this.unlock(uniqueIndexValue);
})));
}
return result;
});
}
}
exports.default = Collection;
//# sourceMappingURL=Collection.js.map