UNPKG

massive

Version:

A small query tool for Postgres that embraces json and makes life simpler

762 lines (651 loc) 27.3 kB
'use strict'; const _ = require('lodash'); const pgp = require('pg-promise'); const util = require('util'); const murmurhash = require('murmurhash').v3; const ops = require('./statement/operations'); const documentPredicate = require('./statement/document-predicate'); const parseKey = require('./util/parse-key'); const quote = require('./util/quote'); const stringify = require('./util/stringify'); const Entity = require('./entity'); const Select = require('./statement/select'); /** * A readable database entity (table or view). * * @class * @extends Entity * @param {Object} spec - An {@linkcode Entity} specification representing a * readable object: * @param {Object} spec.db - A {@linkcode Database}. * @param {String} spec.name - The table or view's name. * @param {String} spec.schema - The name of the schema owning the table or * view. * @param {Object|Array} spec.columns - An array of column names, or an object * mapping constituent Readable names to their column name arrays. * @param {Object} spec.joins - A join object. * @param {Boolean} [spec.is_matview] - Whether the object is a materialized view * (default false). */ const Readable = function (spec) { Entity.apply(this, arguments); this.columnNames = spec.columns; this.columns = spec.columns.map(c => ({ schema: this.schema, parent: this.name, name: c, fullName: `${this.delimitedFullName}."${c}"` })); this.types = spec.types; this.isMatview = spec.is_matview || false; }; util.inherits(Readable, Entity); Readable.prototype.forJoin = Symbol('join'); Readable.prototype.forWhere = Symbol('where'); Readable.prototype.forDoc = Symbol('doc'); /** * Generate a consistent alias for a field belonging to this Readable. * * @param {String} field - The field to alias. * @return {String} An alias separating schema (if necessary), relation, and * field names with double underscores. */ Readable.prototype.aliasField = function (field) { if (this.schema === this.db.currentSchema) { return `${this.name}__${field}`; } return `${this.schema}__${this.name}__${field}`; }; /** * Count rows matching criteria. There are two ways to use this method: * * 1. find() style: db.mytable.count({field: value}); * 2. where() style: db.mytable.count("field=$1", [value]); * * @param {Object|String} conditions - A criteria object or SQL predicate. * @param {Array} params - Prepared statement parameters for use with raw SQL * predicates. * @return {Promise} Row count. */ Readable.prototype.count = function (conditions = {}, params = []) { if (_.isString(conditions)) { conditions = { conditions, params }; } const query = new Select(this, conditions, {exprs: {count: 'COUNT(1)'}, order: null, single: true}); return this.db.query(query).then(res => res.count); }; /** * Count documents matching criteria. Unlike count, this function only supports * criteria objects. * * @param {Object} criteria - A criteria object. * @return {Promise} Number of matching documents. */ Readable.prototype.countDoc = function (criteria = {}) { const query = new Select(this, criteria, { exprs: {count: 'COUNT(1)'}, order: null, single: true, document: true }); return this.db.query(query).then(res => res.count); }; /** * Find rows matching criteria. * * @param {Object|UUID|Number} criteria - A criteria object or primary key value. * @param {Object} [options] - {@link https://massivejs.org/docs/options-objects|Select options}. * @return {Promise} An array containing any query results. */ Readable.prototype.find = function (criteria = {}, options = {}) { return this.db.query(new Select(this, criteria, options)); }; /** * Find a document by searching in the body. * * @param {Object|UUID|Number} [criteria] - A criteria object or primary key value. * @param {Object} [options] - {@link https://massivejs.org/docs/options-objects|Select options}. * @return {Promise} An array containing any query results. */ Readable.prototype.findDoc = function (criteria = {}, options = {}) { options.document = true; return this.find(criteria, options); }; /** * Return a single record. * * @param {Object|UUID|Number} criteria - A criteria object or primary key value. * @param {Object} [options] - {@link https://massivejs.org/docs/options-objects|Select options}. * @return {Promise} An object representing the (first) record found, or * null if no records match. */ Readable.prototype.findOne = function (criteria, options = {}) { return this.find(criteria, _.assign(options, {single: true})); }; /** * Refresh a materialized view. * * @param {Boolean} [concurrently] - Do it without locking reads. * @return {Promise} A query with no results. */ Readable.prototype.refresh = function (concurrently) { if (!this.isMatview) { return this.db.$p.reject(new Error(`${this.delimitedName} is not a materialized view`)); } const concurrentlyStr = concurrently ? 'CONCURRENTLY' : ''; return this.db.query(`REFRESH MATERIALIZED VIEW ${concurrentlyStr} ${this.delimitedFullName}`); }; /** * Perform a full-text search on queryable fields. If options.document is true, * looks in the document body fields instead of the table columns. * * @param {Object} plan - Search definition. * @param {Array} plan.fields - List of the fields to search. * @param {String} plan.term - Search term. * @param {String} [plan.parser] - Parse search term as `plain` (more forgiving * than the default), `phrase`, or `websearch`. * @param {String} [plan.tsv] - Unsafely interpolate a prebuilt text search * vector instead of using `fields`. * @param {Object} [plan.where] - Criteria object to filter results. * @param {Object} [options] - {@link https://massivejs.org/docs/options-objects|Select options}. * @param {Boolean} doDocumentProcessing - True to process results as documents. * @return {Promise} An array containing any query results. */ Readable.prototype.search = function (plan, options = {}, doDocumentProcessing = false) { if (!plan.fields && !plan.tsv) { return this.db.$p.reject(new Error('Plan must contain a fields array or tsv string')); } else if (!plan.term) { return this.db.$p.reject(new Error('Plan must contain a term string')); } if (!plan.tsv) { if (plan.fields.length === 1) { plan.to_tsv = plan.fields[0]; if (plan.to_tsv.indexOf('>>') === -1) { plan.to_tsv = quote(plan.to_tsv); // just a column, quote it to preserve casing } } else { plan.to_tsv = `concat(${plan.fields.join(", ' ', ")})`; // eslint-disable-line quotes } } if (plan.to_tsv) { plan.tsv = `to_tsvector(${plan.to_tsv})`; } switch (plan.parser) { case 'plain': plan.parser = 'plainto_tsquery'; break; case 'phrase': plan.parser = 'phraseto_tsquery'; break; case 'websearch': plan.parser = 'websearch_to_tsquery'; break; default: plan.parser = 'to_tsquery'; break; } const criteria = { conditions: `${plan.tsv} @@ ${plan.parser}($1)`, params: [plan.term], where: plan.where, isDocument: options.document }; if (doDocumentProcessing) { options.document = true; } const query = new Select(this, criteria, options); return this.db.query(query); }; /** * Shortcut to perform a full text search on a document table. * * @param {Object} plan - Search definition. * @param {Array} [plan.fields] - List of the document keys to search. * @param {String} plan.term - Search term. * @param {Object} [plan.where] - Criteria object to filter results. * @param {Object} [options] - {@link https://massivejs.org/docs/options-objects|Select options}. * @return {Promise} An array containing any query results. */ Readable.prototype.searchDoc = function (plan, options = {}) { if (!plan.fields) { plan.tsv = 'search'; } else { plan.fields = plan.fields.map(key => { return `(body ->> '${key}')`; }); } return this.search(plan, options, true); }; /** * Run a query with a raw SQL predicate, eg: * * db.mytable.where('id=$1', [123]).then(...); * * @param {String} conditions - A raw SQL predicate. * @param {Array} [params] - Prepared statement parameters. * @param {Object} [options] - {@link https://massivejs.org/docs/options-objects|Select options}. * @return {Promise} An array containing any query results. */ Readable.prototype.where = function (conditions, params = [], options = {}) { if (!_.isArray(params) && !_.isPlainObject(params)) { params = [params]; } const query = new Select(this, {conditions, params}, options); return this.db.query(query); }; /** * Build a disjunction (logical OR). * * @param {Object} criteria - A criteria object. * @param {Number} offset - Offset prepared statement parameter ordinals. * @param {Symbol} kind - forJoin, forWhere, or forDoc. * @return {String} The JOIN condition text, to be stored and interpolated into * queries after the ON. */ Readable.prototype.disjoin = function (criteria, offset, kind, ...args) { return _.reduce(criteria, (disjunction, subconditions) => { // each member of an 'or' array is itself a conjunction, so build it and // integrate it into the disjunction predicate structure /* eslint-disable-next-line no-use-before-define */ const conjunction = this.conjoin(subconditions, disjunction.offset + disjunction.params.length, kind, ...args); disjunction.params = disjunction.params.concat(conjunction.params); disjunction.predicates.push(`(${conjunction.predicates.join(' AND ')})`); return disjunction; }, { predicates: [], params: [], offset }); }; /** * Build a conjunction (logical AND). * * @param {Object} criteria - A criteria object. * @param {Number} offset - Offset prepared statement parameter ordinals. * @param {Symbol} kind - forJoin, forWhere, or forDoc. * @return {String} The JOIN condition text, to be stored and interpolated into * queries after the ON. */ Readable.prototype.conjoin = function (criteria, offset, kind, ...args) { return _.reduce(criteria, (conjunction, val, key) => { // TODO un-$ed names are officially deprecated but may or may not ever be // practical to remove if (['$or', 'or'].indexOf(key) > -1) { const disjunction = this.disjoin(val, conjunction.offset + conjunction.params.length, kind, ...args); conjunction.params = conjunction.params.concat(disjunction.params); conjunction.predicates.push(`(${disjunction.predicates.join(' OR ')})`); return conjunction; } else if (['$and', 'and'].indexOf(key) > -1) { const predicates = []; // track subconjunction predicates separately since they're grouped together conjunction = _.reduce(val, (c, subconditions) => { // each member of an 'and' array is itself a conjunction, so build it and // integrate it into the conjunction predicate structure /* eslint-disable-next-line no-use-before-define */ const innerConjunction = this.conjoin(subconditions, c.offset + c.params.length, kind, ...args); c.params = c.params.concat(innerConjunction.params); predicates.push(`(${innerConjunction.predicates.join(' AND ')})`); return c; }, conjunction); conjunction.predicates = conjunction.predicates.concat([`(${predicates.join(' AND ')})`]); return conjunction; } let name = key; if (kind === this.forDoc) { name = `body.${name}`; } else if (kind === this.forJoin) { name = `${args[0]}.${name}`; // alias for join clause } let condition = parseKey.withAppendix( name, this, ops, val, offset + conjunction.params.length + 1 ); if (kind === this.forDoc) { condition = documentPredicate(condition, key); } else { if (kind === this.forJoin && _.isString(val)) { // for join criteria, val can be another Readable column name to match const sourceKey = parseKey(val, this); if (this.columns.some(c => c.fullName === sourceKey.path && !sourceKey.remainder)) { conjunction.predicates.push(`${condition.lhs} ${condition.appended.operator} ${sourceKey.lhs}`); return conjunction; } } // join or table where // mutators can do things to condition.value, so it has to be in final form // before those get applied; and JSON predicates expect strings. if (condition.isJSON && condition.value) { condition.value = stringify(condition.value); } if (condition.appended.mutator) { condition = condition.appended.mutator(condition); } else if (condition.value) { condition.params.push(condition.value); condition.value = `$${condition.offset}`; } } conjunction.predicates.push(`${condition.lhs} ${condition.appended.operator} ${condition.value}`); conjunction.params = conjunction.params.concat(condition.params); return conjunction; }, { predicates: [], params: [], offset }); }; /** * Create a {predicate, params} object from join or where criteria. * * Spread argument is used to disambiguate columns while building compound * Readables by prepending the joining relation's alias, e.g. "alias"."field". * * @param {Object} criteria - Query criteria mapping column names (optionally * including operation eg 'my_field <>') to the parameter values. Predicates * generated from a criteria object are joined together with `$and`; an `$or` * key denotes an array of nested criteria objects, the collected predicates * from each of which are parenthesized and joined with `$or`. * @param {Number} offset - Added to the token index value in the prepared * statement (with offset 0, parameters will start $1, $2, $3). * @param {Symbol} kind - forJoin, forWhere, or forDoc. * @return {Object} A predicate string and an array of parameters. */ Readable.prototype.predicate = function (criteria, offset, kind, ...args) { if (_.isPlainObject(criteria) && _.isEmpty(criteria)) { return { predicate: 'TRUE', params: [] }; } if (Object.prototype.hasOwnProperty.call(criteria, 'conditions') && Object.prototype.hasOwnProperty.call(criteria, 'params')) { if (_.isPlainObject(criteria.where) && !_.isEmpty(criteria.where)) { // searchDoc can pass an alternate inner isDocument in the criteria let subWhere; if (Object.prototype.hasOwnProperty.call(criteria, 'isDocument')) { const innerKind = criteria.isDocument ? this.forDoc : this.forWhere; subWhere = this.predicate(criteria.where, offset, innerKind); } else { subWhere = this.predicate(criteria.where, offset, kind); } return { predicate: `(${pgp.as.format(criteria.conditions, criteria.params)}) AND (${subWhere.predicate})`, params: subWhere.params }; // if (Object.prototype.hasOwnProperty.call(criteria, 'isDocument')) { // const innerKind = criteria.isDocument ? this.forDoc : this.forWhere; // subWhere = this.predicate(criteria.where, offset, innerKind); // } else { // subWhere = this.predicate(criteria.where, offset, kind); // } // return { // predicate: `(${subWhere.predicate}) AND (${criteria.conditions})`, // params: subWhere.params.concat(criteria.params) // }; } return { predicate: criteria.conditions, params: criteria.params }; } const assemblage = this.conjoin(criteria, offset, kind, ...args); assemblage.predicate = `${assemblage.predicates.join(' AND ')}`; return assemblage; }; /** * Find the foreign key relationships which exist between this readable and its * parent in a join context. * * @param {Readable} parentReadable - The readable being joined to. * @param {String} parentAlias - The alias corresponding to parentReadable in * the current join tree. * @return {Array} A list of foreign key relationships represented as objects * mapping fields of this Readable to their corresponding fields in * `parentReadable`. */ Readable.prototype.findCandidateJoinKeys = function (parentReadable, parentAlias) { return _.chain(this.fks) .concat(parentReadable.fks) .compact() .reduce((allJoinPredicates, fk) => { // standardize the join criteria with the table being joined on the // left side regardless of the foreign key's directionality let leftColumns, rightColumns; if (fk.origin_schema === this.schema && fk.origin_name === this.name) { leftColumns = fk.origin_columns; rightColumns = fk.dependent_columns; } else if (fk.origin_schema === parentReadable.schema && fk.origin_name === parentReadable.name) { leftColumns = fk.dependent_columns; rightColumns = fk.origin_columns; } if (leftColumns && rightColumns) { // columns on the right side always belong to the parent relation // and must be prefixed with its alias for parseKey allJoinPredicates[fk.fk] = _.zipObject(leftColumns, rightColumns.map(c => `${parentAlias}.${c}`)); } return allJoinPredicates; }, {}) .toPairs() .value(); }; /** * Create a compound Readable by declaring other relations to attach. Queries * against the compound Readable will `JOIN` the attached relations and * decompose results into object trees automatically. * * Compound Readables are cached. If the same join plan is encountered * elsewhere, Massive will pull the compound Readable from the cache instead of * processing the definition again. * * @param {Object} definition - An object mapping relation paths (optional * schema and dot, required name) or aliases to objects defining the join `type` * (inner, left outer, etc); `on` mapping the foreign column(s) in the relation * being joined to the source column(s) in the relation being joined to; and an * optional `relation` path if an alias is used for the key. These objects may * be nested. * @return {Readable} The compound Readable. */ Readable.prototype.join = function (definition) { const name = murmurhash(`${this.path}.${JSON.stringify(definition)}`); if (Object.hasOwnProperty.call(this.db.entityCache, name)) { return this.db.entityCache[name]; } // single table-to-table inner joins on a foreign key relationship can be // expressed just by giving the name of the table to join, but the reducer // still expects to see an object with the appropriate key and _some_ value if (_.isString(definition)) { definition = _.set({}, definition, true); } const seenAliases = [this.name]; const primaryPk = (definition.pk ? _.castArray(definition.pk) : this.pk).map(this.aliasField, this); const decompositionSchema = { pk: primaryPk, columns: this.columns.reduce((map, c) => { map[this.aliasField(c.name)] = c.name; return map; }, {}) }; let schemaNodeParent = decompositionSchema; // if passed an explicit pk for a view at the root of the join tree, ensure // it doesn't get processed as part of that tree delete definition.pk; // This reducer is building several things at once, going node by node in the // definition tree: // // * the formal definition tree with standardized paths // * the decomposition schema tree, which has exactly the same shape as the // definition tree // * the list of member relations // * the list of involved columns from the member relations // * the list of seen aliases, to forestall reuse of same const reducer = (acc, val, key) => { // The original definition's keys each refer to a relation to be attached // in the compound Readable. However, they can refer to it in different // ways: // // * as a schema.relation path // * as a relation name alone, where the schema is db.currentSchema // * as an alias, in which case the definition must include either of the // above as a property `relation` // // The final definition reorganizes the original's properties to eliminate // the first category: all keys must be relation names or aliases. const lexed = parseKey.lex(val.relation || key); const relation = lexed.tokens.pop(); const schema = lexed.tokens.length ? lexed.tokens.shift() : undefined; const alias = val.relation ? key : relation; const readable = _.get(this.db, _.compact([schema, relation])); if (!readable) { throw new Error(`Bad join definition: unknown database entity ${val.relation || key}.`); } else if (seenAliases.some(a => a === alias)) { throw new Error(`Bad join definition: ${alias} is repeated.`); } else if (!val.pk && !readable.pk) { throw new Error(`Missing explicit pk in join definition for ${alias}.`); } seenAliases.push(alias); // setting up for recursion: store the parent schema node pointer and // enter the "current" scope which defines the parent for any nested nodes const outerParent = schemaNodeParent; if (!val.omit) { // create this node in the parallel decomposition schema tree schemaNodeParent[alias] = {}; const schemaNodeCurrent = schemaNodeParent[alias]; schemaNodeCurrent.pk = (val.pk ? _.castArray(val.pk) : readable.pk).map(c => `${alias}__${c}`); schemaNodeCurrent.decomposeTo = val.decomposeTo; schemaNodeCurrent.columns = readable.columns.reduce((map, c) => { const columnAlias = `${alias}__${c.name}`; map[columnAlias] = c.name; return map; }, {}); schemaNodeParent = schemaNodeCurrent; } // if not given an explicit join predicate, attempt to fall back on an // unambiguous foreign key relationship between the relations let on = val.on; if (!on) { const candidateFks = readable.findCandidateJoinKeys(val.parentReadable || this, val.parentAlias || this.delimitedFullName); switch (candidateFks.length) { case 1: on = candidateFks[0][1]; break; case 0: throw new Error(`An explicit 'on' mapping is required for ${val.relation || key}.`); default: throw new Error(`Ambiguous foreign keys for ${val.relation || key}. Define join keys explicitly.`); } } // standardize this node in the definition tree, recursing if necessary acc[alias] = _.reduce(val, (node, v, k) => { switch (k) { case 'type': case 'on': case 'schema': case 'relation': case 'pk': case 'omit': case 'parentReadable': case 'parentAlias': case 'decomposeTo': return node; default: // Any other property is a descendant definition node. Attach its // parent info and recurse, standardizing it and adding to the // current node. v.parentReadable = readable; v.parentAlias = alias; return reducer(node, v, k); } }, { schema, relation, alias, readable, type: val.type || 'INNER', on, joinRelations: [], joinColumns: [], joinTypes: [] }); // restore the parent schema node pointer schemaNodeParent = outerParent; // Add the join relation, its columns and types to the node accumulator. // It's stored there because we want the column order and JOIN order to match // the tree layout (a "pre-order" traversal: parents before children, in // sequence) -- Postgres won't toposort JOIN clauses for us. However, since // we're only ready to accumulate relations and columns _after_ the // recursive reduction, we're forced into a "post-order" traversal // (children first, parents as all their children are accumulated). To // restore the desired order, we accumulate descendant relations and columns // on the node and collect them after recursion. acc.joinRelations = _.concat(acc.joinRelations, [acc[alias]], acc[alias].joinRelations); acc.joinColumns = _.concat(acc.joinColumns, readable.columns.map(c => { return { schema: c.schema, parent: c.parent, name: c.name, fullName: `"${alias}"."${c.name}"`, alias: `${alias}__${c.name}` }; }), acc[alias].joinColumns); acc.joinTypes = _.extend(acc.joinTypes, _.mapKeys(readable.types, (_v, k) => `${alias}.${k}`)); // strictly speaking, only leftover joinRelations or joinColumns on the // origin node will cause problems, but it's better to keep things clean. delete acc[alias].joinRelations; delete acc[alias].joinColumns; delete acc[alias].joinTypes; return acc; }; definition = _.reduce(definition, reducer, { joinRelations: [], joinColumns: this.columns.map(c => { c.alias = this.aliasField(c.name); return c; }), joinTypes: _.mapKeys(this.types, (_v, k) => `${this.name}.${k}`) }); // this form of joinRelations is temporary; it's only declared here because // there has to be something to reference in the `clone` construction just // below. Before the compound Readable is fully finished, `joinRelations` will // be augmented with additional information about each attached relation. let joinRelations = definition.joinRelations; const joinColumns = definition.joinColumns; const joinTypes = definition.joinTypes; delete definition.joinRelations; delete definition.joinColumns; delete definition.joinTypes; const clone = new Proxy(this, { get: (target, prop) => { switch (prop) { case 'loader': return 'join'; case 'joins': return joinRelations; case 'columns': return joinColumns; case 'types': return joinTypes; } if ( typeof target[prop] === 'function' && ['find', 'findOne', 'search', 'where'].indexOf(prop) > -1 ) { // apply a decomposition schema to the options argument for all // non-document non-count Readable functions return new Proxy(target[prop], { apply: (fn, thisArg, args) => { const optionsIdx = prop === 'where' ? 2 : 1; const optionsObj = args[optionsIdx] || {}; const override = optionsObj.decompose; args[optionsIdx] = _.assign(optionsObj, { decompose: override || decompositionSchema }); return fn.apply(thisArg, args); } }); } return target[prop]; } }); let offset = 0; joinRelations = joinRelations.map(joinRelation => { // Build the ON clause ahead of time, since JOIN criteria for a given // compound Readable never change. Since parseKey only looks to compound // Readables for alternate schema/relation paths, this has to happen after // instantiation of the clone. const delimitedAlias = quote(joinRelation.alias); joinRelation.target = delimitedAlias === joinRelation.readable.delimitedFullName ? delimitedAlias : `${joinRelation.readable.delimitedFullName} AS ${delimitedAlias}`; joinRelation.on = clone.predicate(joinRelation.on, offset, this.forJoin, joinRelation.alias); offset += joinRelation.on.params.length; return joinRelation; }); this.db.entityCache[name] = clone; return clone; }; module.exports = Readable;