UNPKG

adaptorex

Version:

Connect all your live interactive storytelling devices and software

1,159 lines (1,082 loc) 32.2 kB
/** * Structures access to mongo database * * setup mongo DB: * $ sh mongo.sh setup * * start mongo DB: * $ sh mongo.sh start * * stop mongo DB: * $ sh mongo.sh start * * @requires mongodb * @requires file * * @module mongo * @copyright Lasse Marburg 2021 * @license MIT */ const file = require("./file.js") const { exec } = require("child_process") const mongodb = require("mongodb") const client = mongodb.MongoClient //var url = "mongodb://localhost:27017,localhost:27018,localhost:27019?replicaSet=mongo-repl" /** mongodb server url to connect to * @default */ var url = "mongodb://localhost:27017/?replicaSet=adaptor-repl" /** mongodb client connection once returned by client connect */ var connection /** * connect to mongodb server based on url variable. * adjust url variable if config contains a mongo url * * @todo fetch connection information from `mongo.sh setup`. Having to feed * the config file with the mongo url is redundant */ function start(config) { return new Promise((resolve, reject) => { if (config.url) { url = config.url } log.info("MongoDB", "connecting to " + url) client .connect(url) // , { useNewUrlParser: true, useUnifiedTopology: true, poolSize: 100 } .then((conn) => { connection = conn return resolve("started") }) .catch((error) => { return reject(error) }) }) } /** * * @todo stop mongo instance from running */ function stop() {} /** * Allows access to a mongo Database. * * @param {Object} config * @param {String} config.name - database ID * @param {number} [config.url] - host and port of mongoDB and replikas. Optional if start() function was called * */ class Database { constructor(config) { Object.assign(this, config) this.type = "mongodb" } /** * try connecting to existing database or create and connect if it * doesn't exist. * * add all existing collections as Collection Objects to Database Object * * increased pool size following this issue conversation: * https://jira.mongodb.org/browse/SERVER-32946 * * @returns {Promise} resolves with "connected" if db existed, "created" if db was missing */ connect() { return new Promise((resolve, reject) => { log.info(this.name, "Accessing Game Database") this.db = connection.db(this.name) this.db .listCollections() .toArray() .then((collections) => { if (collections.length > 0) { for (let i = 0; i < collections.length; i++) { var col = collections[i].name log.debug(this.name, "Created collection Object of: " + col) this[col] = new Collection(col, this.db.collection(col)) } log.info(this.name, "connected") resolve("connected") } else { log.info(this.name, "created") resolve("created") } }) .catch((err) => { if (err instanceof mongodb.MongoNetworkError) { log.error(0, "There was an error connecting to the mongo DB:") log.error(0, err) } else { reject(err) } }) }) } /** * Create a copy of an existing mongodb database * * uses child process that runs mongodump to standart output and pipes into mongorestore * Details: https://docs.mongodb.com/database-tools/mongodump/#std-label-mongodump-example-copy-clone-database * * @param {string} copy_name - name of database copy * @returns {Promise<undefined>} */ clone(copy_name) { return new Promise((resolve, reject) => { if (url.match(/(?<!\/)\/(?!\/)/)) { var mongodump_url = url.replace(/(?<!\/)\/\w*(?!\/)/, `/${this.name}`) } else { var mongodump_url = url + "/" + this.name } exec( `mongodump --archive --uri="${mongodump_url}" | mongorestore --archive --uri="${url}" --drop --nsFrom='${this.name}.*' --nsTo='${copy_name}.*'`, (err, stdout, stderr) => { if (err) { return reject(err) } log.debug(this.name, `stdout: ${stdout}`) log.debug(this.name, `stderr: ${stderr}`) return resolve() } ) // Exec string with mongodump --db option: // `mongodump --archive --uri="${mongodump_url}" --db='${this.name}' | mongorestore --archive --uri="${url}" --drop --nsFrom='${this.name}.*' --nsTo='${copy_name}.*'` }) } /** * replace or create whole database from a properly formatted json file * @todo empty */ import(filename, callback) {} /** * return all collections in this database as a list of strings * * @returns {Promise} array of Collection names (string). */ listCollections() { return new Promise((resolve, reject) => { this.db .listCollections() .toArray() .then( function (result) { let colls = [] for (let i = 0; i < result.length; i++) { colls.push(result[i].name) } return resolve(colls) }.bind(this) ) .catch( function (error) { log.error(this.name, error) return reject(error) }.bind(this) ) }) } /** * adds collection to mongo Database and to Database object * * @param {String} name - new collections name * * @returns {Promise} forwards response when done */ addCollection(name) { return new Promise( function (resolve, reject) { this.db .createCollection(name) .then( function (res) { this[name] = new Collection(name, this.db.collection(name)) log.debug(0, "Added Collection " + name + " to " + this.name) resolve(res) }.bind(this) ) .catch( function (err) { log.error(this.name, err) }.bind(this) ) }.bind(this) ) } /** * drop the collection and delete the reference * * @param {String} name - to be dropped collections name * * @returns {Promise} forwards response when done */ dropCollection(name) { return new Promise( function (resolve, reject) { this[name].c .drop() .then( function (res) { if (res) { delete this[name] } resolve(res) }.bind(this) ) .catch( function (err) { log.error(this.name, err) }.bind(this) ) }.bind(this) ) } /** * Drop Database */ drop() { let name = this.name this.db .dropDatabase(this.name) .then(function (result) { log.info(0, "Database " + name + " deleted") }) .catch(function (error) { log.error(0, error) }) } /** * Convert an int, number or string to int64 "Long" type * * @param {number|string} value - value to be converted * @returns {mongodb.Long} An Object of type mongodb.Long that will be stored as int64 in the database */ toLong(value) { if (typeof value === "number") { let long_value = mongodb.Long.fromNumber(value) return long_value } else if (typeof value === "string") { let long_value = mongodb.Long.fromString(value) return long_value } else if (typeof value === "bigint") { let long_value = new mongodb.Long( Number(value & 0xffffffffn), Number((value >> 32n) & 0xffffffffn) ) return long_value } throw new Error( "Can only convert number, string or bigint to mongo 64bit 'Long' integer not " + typeof value ) } /** * check if string is a valid mongodb ObjectId of 12 or 24 alphanumeric characters * * @param {string} id - value to be tested * @returns {boolean} true if id is a valid mongo ObjectId */ validId(id) { return mongodb.ObjectId.isValid(id) } /** * quit connection to database */ quit() {} } /** * Collection Object that inherits a mongo collection and adds query functions * * @param {String} name - name of collection. Same as object instance name in Database Object * @param {Object} col - Collection Object created from db connection */ class Collection { constructor(name, col) { this.name = name /** @type {mongodb.Collection} */ this.c = col } /** * @param {Object} query - json to identify Document * * @returns {Document} easier access to the first document in this collection the query returns * @throws {adaptor.NotFoundError} if document is not found */ async document(query) { let doc = new Document(query, this) await doc.setup() return doc } /** * start a change stream * * @param {Object} [options] - see https://docs.mongodb.com/manual/changeStreams/#modify-change-stream-output for possible modification options * * @returns {Object} a mongodb Change stream (https://docs.mongodb.com/manual/changeStreams/) */ changes(options) { if (options) { var stream = this.c.watch(options) } else { var stream = this.c.watch() } return stream } /** * compare two _ids and return if they are equal * * @param {mongodb.ObjectId|String} idA - id to be checked against idB * @param {mongodb.ObjectId|String} idB - id to be checked against idA * * @returns {boolean} true if ids are equal */ compareIDs(idA, idB) { idA = this.makeID(idA) idB = this.makeID(idB) if (idA.equals(idB)) { return true } else { return false } } /** * @param {string} id - id to be transformed * @returns {mongodb.ObjectId} string id changed into mongo ObjectId */ makeID(id) { if (typeof id === "string") { return new mongodb.ObjectId(id) } else { return id } } /** * check if string is a valid mongodb ObjectId of 12 or 24 alphanumeric characters * * @param {string} id - value to be tested * @returns {boolean} true if id is a valid mongo ObjectId */ validId(id) { return mongodb.ObjectId.isValid(id) } /** * convert _id string property in top level and 2nd level data objects to mongo ObjectId. * Bypass if no _id exists or needs to be converted * * @param {Object} data - data that might contain _id property * @returns {Object} data object with _id property converted to mongo ObjectId if applicable */ normalizeID(data) { if (data.hasOwnProperty("_id")) { data._id = this.makeID(data._id) } for (let d in data) { if (Array.isArray(data[d])) { for (let dat of data[d]) { if (typeof dat === "object") { if (dat.hasOwnProperty("_id")) { dat._id = this.makeID(dat._id) } } } } if (typeof data[d] === "object") { if (data[d].hasOwnProperty("_id")) { data[d]._id = this.makeID(data[d]._id) } } } return data } /** * Recursive function that iterates over query object and converts mongo db specific key/value pairs. * * - Make sure value behind `_id` keys is a valid mongo ObjectId. Try to convert if required. * - Convert value behind '$date' key to Date Object * - Convert string that matches ISO Date standart to Date Object * * @param {*} obj - The query object * @returns {*} Modified or original query object. */ extendQuery(obj) { var key, val for (key in obj) { if (!obj.hasOwnProperty(key)) continue val = obj[key] switch (key) { case "$regex": continue case "_id": obj[key] = this.makeID(val) continue case "$date": return new Date(val) } if (typeof val === "object") { obj[key] = this.extendQuery(val) } else if (typeof val === "string") { if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.?\d{0,3}Z?$/.test(val)) { obj[key] = new Date(val) } } } return obj } /** * create a new indexed field * * @param {string} field - property that shall become an index * @param {boolean} unique - set true if index is supposed to be restricted to unique values */ createIndex(field, unique) { let index = {} index[field] = 1 this.c.createIndex(index, { unique: unique }) } /** * change existing document. Only objects mentioned in query are updated. * If you don't want to rewrite the object as a whole, use string dot notation * to specify the nested object * * prepends $set to updated data * * @param {Object} query - query what documents to update * @param {data} data - json of what is to be updated * */ set(query, data) { return new Promise((resolve, reject) => { this.update(query, { $set: data }) .then((res) => { return resolve(res) }) .catch((err) => { return reject(err) }) }) } /** * Update 1 or more documents * * make a mongo document update using any of the update operators: * https://docs.mongodb.com/manual/reference/operator/update/ * * @param {Object} query - query what documents to update * @param {data} data - json of what is to be updated * @param {object} [options] - update options * @param {boolean} [options.multiple] - Set `true` to allow to update many. Set `false` to update one document only */ update(query, data, { multiple = true } = { multiple: true }) { return new Promise((resolve, reject) => { let operation = "updateMany" if (!multiple) { operation = "updateOne" } this.c[operation](this.extendQuery(query), this.encode(data)) .then((res) => { this.feedback(res) return resolve(res) }) .catch((err) => { return reject(err) }) }) } /** * Update one document and return the original or the new document. * * make a mongo document update using any of the update operators: * https://docs.mongodb.com/manual/reference/operator/update/ * * Returns original document by default. Use `{returnDocument: "after"}` option to return the resulting document. * * @param {Object} query - query what documents to update * @param {Object} data - update operations data * @param {Object} [options] - additional options for operation. See https://mongodb.github.io/node-mongodb-native/4.1/interfaces/FindOneAndUpdateOptions.html * @param {boolean} [options.returnDocument="before"] - before: include original document in return value. after: include updated document in return value * * @returns {Promise<object>} original or updated document */ findOneAndUpdate(query, data, options) { return new Promise((resolve, reject) => { this.c .findOneAndUpdate(this.extendQuery(query), this.encode(data), options) .then((res) => { return resolve(res) }) .catch((err) => { return reject(err) }) }) } /** * make a mongo document replace. Replace 1st Document returned by query. * * @param {Object} query - query what documents to replace * @param {data} data - json of how the replaced document looks like * @param {data} options - see replaceOne options: https://docs.mongodb.com/manual/reference/method/db.collection.replaceOne/ */ replace(query, data, options) { return new Promise((resolve, reject) => { this.c .replaceOne(this.extendQuery(query), this.encode(data)) .then((res) => { this.feedback(res) return resolve(res) }) .catch((err) => { log.error(this.name, new Error(err)) return reject(err) }) }) } /** * add new document to collection * * @returns {Promise} data about the inserted document including _id property */ insert(data) { return new Promise((resolve, reject) => { this.c .insertOne(this.encode(data)) .then((res) => { this.feedback(res) return resolve(res) }) .catch((err) => { return reject(err) }) }) } /** * Some special characters in json schemes conflict with mongodb ($ and .) So they are replaced before update and after find */ setSchema(query, data) { return new Promise((resolve, reject) => { let c_data = this.encode(data) //log.info("SCHEMA",c_data.schema.cue.switch.properties.if.items.oneOf[2].properties) this.set(query, c_data) .then((result) => { return resolve(result) }) .catch((error) => { log.error(this.name, error) return reject(error) }) }) } /** * middle function before any update, replace or insert function. * * replace $ with similar looking char $ and . with . in schema, package and control properties. * * search and replace in schema property on first and second level. * * @todo maybe disclaim storing the schemes to the database. Afaik they are anyhow always updated based on external variables * * idea from: * https://grokbase.com/t/gg/json-schema/133n7jfp8m/question-about-in-key-names-like-schema-and-ref#20130322rvyoejes76tyv5teeqsc6yma4e * * for readability it now uses stringification of object and string replace function. * might be quite performance heavy with big json objects and should not be used for time critical tasks. */ encode(obj) { var key, val for (key in obj) { if (!obj.hasOwnProperty(key)) continue val = obj[key] switch (key) { case "control": case "schema": case "package": obj[key] = this.replaceEncode(obj[key]) continue case "$date": return new Date(val) case "_id": obj[key] = this.makeID(obj[key]) continue } if (typeof val === "object") { obj[key] = this.encode(val) } else if (typeof val === "string") { if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.?\d{0,3}Z?$/.test(val)) { obj[key] = new Date(val) } } } return obj } replaceEncode(obj) { let obj_str = JSON.stringify(obj) obj_str = obj_str .replace(/\$ref/gm, "$ref") .replace(/\$schema/gm, "$schema") .replace(/\./gm, ".") // .replace(/\./gm, ".") return JSON.parse(obj_str) } /** * middle function before any return of find or distinct function * * replace $ with similar looking char $ and . with . in schema, package and control properties. */ decode(obj) { var key, val for (key in obj) { if (!obj.hasOwnProperty(key)) continue val = obj[key] switch (key) { case "control": case "schema": case "package": obj[key] = this.replaceDecode(obj[key]) continue } if (typeof val === "object") { obj[key] = this.decode(val) } } return obj } replaceDecode(obj) { let obj_str = JSON.stringify(obj) obj_str = obj_str.replace(/\$/gm, "$").replace(/\./gm, ".") return JSON.parse(obj_str) } /** * get list of Documents based on query * * @param {Object} query - check documents against query * @param {Object} options.project - specify what fields to return with mongo projection object * @param {Object} options.sort - sort documents by fields in direction (1 or -1) * @param {Object} options.limit - set maximum numbers of documents to return * @param {Object} options.skp - return from nth document only * * @returns {Promise} forwards an array of all found documents. Returns no rejection! */ find(query, { project, limit = 0, skip = 0, sort = {} } = {}) { if (!query) { query = {} } if (typeof query !== "object") { throw new adaptor.InvalidError( `MongoDB failed to run find operation. Find query property is ${query} and of type ${typeof query}.` ) } let options = this.queryOptions(query, { project, limit, skip, sort }) if (!sort._id) { sort["_id"] = 1 } return new Promise((resolve, reject) => { this.c .find(this.extendQuery(query), { projection: options.project }) .skip(options.skip) .limit(options.limit) .sort(options.sort) .toArray() .then((result) => { return resolve(this.decode(result)) }) .catch((error) => { log.error(this.name, error) }) }) } queryOptions(query, options) { for (let option of ["sort", "limit", "skip", "project"]) { let $option = "$" + option if (query.hasOwnProperty($option)) { options[option] = query[$option] delete query[$option] } } return options } /** * get specific element of Documents based on query. * * @param {Object} key - define key in documents you want to have returned * @param {Object} query - check documents against query * * @returns {Promise} forwards an array of all found document keys. Returns no rejection! */ distinct(key, query) { return new Promise((resolve, reject) => { if (typeof query !== "object") { throw new adaptor.InvalidError( `MongoDB failed to run find operation. Find query property is ${query} and of type ${typeof query}.` ) } this.c .distinct(key, query) .then((result) => { resolve(this.decode(result)) }) .catch((error) => { log.error(this.name, error) }) }) } /** * return document if it contains a certain object * * @param {string} path - '.' notation path to object */ contains(path) { return new Promise( function (resolve, reject) { let query = {} query[path] = { $exists: true } this.c .find(query) .toArray() .then(function (result) { resolve(this.decode(result)) }) .catch(function (error) { log.error(this.name, error) }) }.bind(this) ) } /** * Counts documents based on query. * * @param {Object} query - check documents against query * * @returns {Promise} forwards the number of all found documents. */ count(query) { return new Promise((resolve, reject) => { this.c .countDocuments(this.extendQuery(query)) .then((result) => { return resolve(result) }) .catch((error) => { log.error(this.name, error) }) }) } /** * delete documents * * @param {Object} query - documents to be deleted * @param {object} [options] - update options * @param {boolean} [options.multiple] - Set `true` to allow to delete many. Set `false` to delete one document only */ delete(query, { multiple = true } = { multiple: true }) { return new Promise((resolve, reject) => { let operation = "deleteMany" if (!multiple) { operation = "deleteOne" } this.c[operation](this.extendQuery(query)) .then((result) => { return resolve(result.deletedCount) }) .catch((error) => { log.error(this.name, error) }) }) } /** * delete documents based on query and return them * * @param {Object} query - what entries to cut out * * @returns {Promise} deleted documents as array */ slice(query) { return new Promise((resolve, reject) => { let docs = [] this.find(query) .then((result) => { docs = result return this.c.deleteMany(query) }) .then((result) => { return resolve(this.decode(docs)) }) .catch((error) => { return reject(error) }) }) } /** * use $unset to remove object from document * * this is not in use * * @param {Object} query - query to find documents where you want to delete objects * @param {Object} obj - objects to to delete * * @returns {Promise} result */ unset(query, obj) { return new Promise((resolve, reject) => { this.c .updateMany(query, { $unset: obj }) .then((res) => { resolve(res) }) .catch((error) => { log.error(this.name, new Error(error)) }) }) } /** * append item to array in object. Don't allow duplicates * * @param {Object} data - Array name/path and values to append. Use string with dot notation for nested array (e.g.: {'object.array':{key:value}}) * * @returns {Promise} amount of modified elements. 0 if none (e.g. duplicate) */ push(query, data, options) { return new Promise((resolve, reject) => { this.update(query, { $addToSet: data }, options) .then((res) => { //this.feedback(res) return resolve(res) // resolve(res.modifiedCount) }) .catch((err) => { return reject(err) }) }) } /** * remove item from Array */ remove(query, pull_query) { return new Promise((resolve, reject) => { this.update(query, { $pull: pull_query }) .then((result) => { this.feedback(result) return resolve(result) }) .catch((error) => { // log.error(this.name, query) // log.error(this.name, pull_query) // log.error(this.name, error) return reject(error) }) }) } /** * remove item from array and return it. Works only with 1. level array so far * * @todo rebuild without findOneAndUpdate. Too many restrictions. Also it works different than * pick wich is confusing and, I guess, not necessary * * @param {Object} query - define what item to pull (e.g. {arrayname: {_id:"someid"}}) * * @returns {Promise} returns removed document if successfull */ pull(query, pull_query) { return new Promise( function (resolve, reject) { let proj_doc = {} let array_key = "" for (var key in pull_query) { proj_doc[key] = {} proj_doc[key]["$elemMatch"] = pull_query[key] array_key = key } log.debug("Elem Match Projection: ") log.debug(0, proj_doc) this.c .findOneAndUpdate( query, { $pull: pull_query }, { returnDocument: "before", projection: proj_doc } ) .then(function (result) { if (result.value.hasOwnProperty(array_key)) { if (result.value[array_key].length) { resolve(this.decode(result.value[array_key][0])) } else { reject("No element in " + array_key) } } else { reject("Query didn't return " + array_key) } }) .catch( function (error) { log.error(0, error) }.bind(this) ) }.bind(this) ) } /** * return only specific element in array. * based on $ projection operator: * https://docs.mongodb.com/manual/reference/operator/projection/positional/#proj._S_ * * @param {Object} query - defines document to pick values from * @param {string} path - address of array where query is supposed to find an element in '.' notation (e.g. 'path.to.array') * @param {Object} arr_query - query in array (e.g. {'_id':12345}) to find element * * @returns {Promise} first(!) array element that is found with query. (returns only the element itself) */ pick(query, path, arr_query) { return new Promise((resolve, reject) => { let part_query = {} let root_query = Object.assign({}, query) // for some reason I don't quite get, //the param query stays the same Object over several function calls for (let key in arr_query) { if (root_query.hasOwnProperty(key)) { part_query[path + "." + key] = arr_query[key] } } let full_query = Object.assign(root_query, part_query) // log.debug(0, "pick Full Query: ") // log.debug(0, full_query) let proj = {} proj[path + ".$"] = 1 // log.debug(0, "pick Value: ") // log.debug(0, proj) this.c .find(full_query) .project(proj) .toArray() .then((result) => { if (result.length) { path = path.split(".") let value = result[0] while (path.length) { value = value[path.shift()] } return resolve(this.decode(value[0])) } else { return resolve({}) } }) .catch((error) => { log.error(this.name, error) }) }) } /** * replace or create document from a properly formatted json file * @todo empty */ import(filename) { let json = filename.lastIndexOf(".json") if (json > 0) { var name = filename.substring(0, json) // obsolete file.loadJSON(filename).then( function (result) { this.insert(result) }.bind(this) ) } else { log.error( this.name, "Import failed. " + filename + " is not a .json file" ) } } /** * write feedback of update or insert to debug log * * @param {Object} result - promise return of manipulating db query */ feedback(result) { if (result.hasOwnProperty("insertedCount")) { log.debug(this.name, "inserted " + result.insertedCount + " Documents") if (result.insertedCount > 0) { return "created" } } else { /* log.debug(this.name, "modified " + result.modifiedCount + ", matched " + result.matchedCount + ", upserted " + result.upsertedCount) */ if (result.upsertedCount > 0) { return "created" } if (result.modifiedCount > 0) { return "updated" } if (result.matchedCount > 0) { return "unchanged" } } return "failed" } } /** * Narrows any query down to the same document. * * @param {ObjectId} query - query to address one single record in mongodb * @param {mongo.Collection} collection - collection that inherits this Document */ class Document { constructor(query, collection) { this.query = query this.col = collection this.colname = collection.name } async setup() { let result = await this.col.find(this.query) if (!result.length) { throw new adaptor.NotFoundError( `Could not find and assign document with query ${JSON.stringify(this.query)} in ${this.colname} collection.` ) } this._id = result[0]._id } /** * get full document or the values of a specific key * * @returns {Promise} full document */ get(key) { return new Promise((resolve, reject) => { if (key) { this.col.distinct(key, this.query).then((result) => { return resolve(result) }) } else { this.col .find(this.query) .then((result) => { return resolve(result[0]) }) .catch((error) => { log.error(this.name, error) return reject(error) }) } }) } distinct(key) { return this.col.distinct(key, this.query) } set(data) { return this.col.set(this.query, data) } update(data) { return this.col.update(this.query, data) } pick(path, arr_query) { return this.col.pick(this.query, path, arr_query) } pull(pull_query) { return this.col.pull(this.query, pull_query) } push(data) { return this.col.push(this.query, data) } remove(data) { return this.col.remove(this.query, data) } } module.exports = { Database: Database, Collection: Collection, Document: Document, start: start, stop: stop, client: client }