apostrophe/modules/@apostrophecms/schema/index.js

The Apostrophe Content Management System.

// This module provides schemas, a flexible and fast way to create new data
// types by specifying the fields that should make them up. Schemas power
// [@apostrophecms/piece-type](../@apostrophecms/piece-type/index.html),
// [@apostrophecms/widget-type](../@apostrophecms/widget-type/index.html),
// custom field types in page settings for
// [@apostrophecms/page-type](../@apostrophecms/page-type/index.html) and more.
//
// A schema is simply an array of "plain old objects."
// Each object describes one field in the schema via `type`, `name`, `label`
// and other properties.
//
// See the [schema guide](../../tutorials/getting-started/schema-guide.html)
// for a complete overview and list of schema field types. The methods
// documented here on this page are most often used when you choose to work
// independently with schemas, such as in a custom project that requires forms.

const _ = require('lodash');
const { klona } = require('klona');
const { stripIndents } = require('common-tags');
const addFieldTypes = require('./lib/addFieldTypes');
const newInstance = require('./lib/newInstance.js');

module.exports = {
  options: {
    alias: 'schema'
  },
  async init(self) {
    self.fieldTypes = {};
    self.fieldsById = {};
    self.arrayManagers = {};
    self.objectManagers = {};
    self.fieldMetadataComponents = [];
    self.uiManagerIndicators = [];

    self.enableBrowserData();

    addFieldTypes(self);

    self.validatedSchemas = {};

    // Universal ESM library, an async import is required.
    const { default: checkIfCondition, isExternalCondition } = await import('../../../lib/universal/check-if-conditions.mjs');
    self.checkIfCondition = checkIfCondition;
    self.isExternalCondition = isExternalCondition;
  },

  methods(self) {
    const defaultGroup = self.options.defaultGroup || {
      name: 'ungrouped',
      label: 'apostrophe:ungrouped'
    };

    return {
      // Compose a schema based on addFields, removeFields, orderFields
      // and, occasionally, alterFields options. This method is great for
      // merging the schema requirements of subclasses with the schema
      // requirements of a superclass. See the @apostrophecms/schema
      // documentation for a thorough explanation of the use of each option. The
      // alterFields option should be avoided if your needs can be met via
      // another option.

      compose(options, module) {
        let schema = [];

        // Useful for finding good unit test cases
        /* self.apos.util.log( */
        /*   JSON.stringify( */
        /**
         * _.pick(options, 'addFields', 'removeFields', 'arrangeFields'),
         */
        /*     null, */
        /*     '  ' */
        /*   ) */
        /* ); */

        if (options.addFields) {
          // loop over our addFields
          _.each(options.addFields, function (field) {
            let i;
            // remove it from the schema if we've already added it, last one
            // wins
            for (i = 0; i < schema.length; i++) {
              if (schema[i].name === field.name) {
                schema.splice(i, 1);
                break;
              }
            }
            // add the new field to the schema
            schema.push(field);
          });
        }

        if (options.removeFields) {
          schema = _.filter(schema, function (field) {
            return !_.includes(options.removeFields, field.name);
          });
        }

        if (options.requireFields) {
          _.each(options.requireFields, function (name) {
            const field = _.find(schema, function (field) {
              return field.name === name;
            });
            if (field) {
              field.required = true;
            }
          });
        }

        // If nothing else will do, just modify the schema with a function
        if (options.alterFields) {
          options.alterFields(schema);
        }

        const groups = self.composeGroups(schema, options.arrangeFields);

        // all fields in the schema will end up in this variable
        let newSchema = [];

        // loop over any groups and orders we want to respect
        _.each(groups, function (group) {

          _.each(group.fields, function (field) {
            // find the field we are ordering
            let f = _.find(schema, { name: field });

            if (!f) {
              // May have already been migrated due to subclasses re-grouping
              // fields
              f = _.find(newSchema, { name: field });
            }

            // make sure it exists
            if (f) {
              // set the group for this field
              const g = _.clone(group, true);
              delete g.fields;
              f.group = g;

              // push the field to the new schema, if it is a
              // duplicate due to subclasses pushing more
              // groupings, remove the earlier instance
              let fIndex = _.findIndex(newSchema, { name: field });
              if (fIndex !== -1) {
                newSchema.splice(fIndex, 1);
              }

              newSchema.push(f);

              // remove the field from the old schema, if that is where we got
              // it from
              fIndex = _.findIndex(schema, { name: field });
              if (fIndex !== -1) {
                schema.splice(fIndex, 1);
              }
            }
          });
        });

        // put remaining fields in the default group
        _.each(schema, function (field) {
          const g = _.clone(groups[0], true);
          delete g.fields;
          field.group = g;
        });

        // add any fields not in defined groups to the end of the schema
        schema = newSchema.concat(schema);

        // If a field is not consecutive with other fields in its group,
        // move it after the last already encountered in its group,
        // to simplify rendering logic

        newSchema = [];
        const groupIndexes = {};
        _.each(schema, function (field) {
          if (field.group && field.group.name) {
            if (_.has(groupIndexes, field.group.name)) {
              newSchema.splice(groupIndexes[field.group.name], 0, field);
              groupIndexes[field.group.name]++;
            } else {
              newSchema.push(field);
              groupIndexes[field.group.name] = newSchema.length;
            }
          }
        });
        schema = newSchema;

        // Move the leftover group to the end, it's just too
        // obnoxious otherwise with one-off fields popping up
        // before title etc.

        schema = _.filter(schema, function (field) {
          return !(field.group && field.group.name === defaultGroup.name);
        }).concat(_.filter(schema, function (field) {
          return field.group && field.group.name === defaultGroup.name;
        }));

        // Shallowly clone the fields. This allows modules
        // like workflow to patch schema fields of various modules
        // without inadvertently impacting other apos instances
        // when running with @apostrophecms/multisite
        schema = _.map(schema, function (field) {
          return _.clone(field);
        });

        _.each(schema, function(field) {
          // For use in resolving options like "choices" when they
          // contain a method name. For bc don't mess with possible
          // existing usages in custom schema field types predating
          // this feature
          self.setModuleName(field, module);
        });
        return schema;
      },

      composeGroups (schema, arrangeFields) {
        // always make sure there is a default group
        let groups = [ {
          name: defaultGroup.name,
          label: defaultGroup.label,
          fields: _.map(schema, 'name')
        } ];

        // if we are getting arrangeFields and it's not empty
        if (arrangeFields && arrangeFields.length > 0) {
          // if it's full of strings, use them for the default group
          if (_.isString(arrangeFields[0])) {
            // if it's full of objects, those are groups, so use them
            groups[0].fields = arrangeFields;
          } else if (_.isPlainObject(arrangeFields[0])) {
            // reset the default group's fields, but keep it around,
            // in case they have fields they forgot to put in a group
            groups[0].fields = [];
            groups = groups.concat(arrangeFields);
          }
        }

        // If there is a later group with the same name, the later
        // one wins and the earlier is forgotten. Otherwise you can't
        // ever toss a field out of a group without putting it into
        // another one, which makes it impossible to un-group a
        // field and have it appear outside of tabs in the interface.
        //
        // A reconfigured group is ordered to the bottom of the list
        // of groups again, which has the intended effect if you
        // arrange all of the groups in your module config. However
        // it comes before any groups with the `last: true` flag that
        // were not reconfigured. Reconfiguring a group without that
        // flag clears it.

        const newGroups = [];
        _.each(groups, function (group) {
          const index = _.findIndex(newGroups, { name: group.name });
          if (index !== -1) {
            newGroups.splice(index, 1);
          }
          let i = _.findIndex(newGroups, { last: true });
          if (i === -1) {
            i = groups.length;
          }
          newGroups.splice(i, 0, group);
        });

        return newGroups;
      },

      // Recursively set moduleName property of the field and any subfields,
      // as might be found in array or object fields. `module` is an actual
      // module
      setModuleName(field, module) {
        field.moduleName = field.moduleName || (module && module.__meta.name);
        if ((field.type === 'array') || (field.type === 'object')) {
          _.each(field.schema || [], function(subfield) {
            self.setModuleName(subfield, module);
          });
        }
      },

      // refine is like compose, but it starts with an existing schema array
      // and amends it via the same options as compose.

      refine(schema, _options) {
        // Don't modify the original schema which may be in use elsewhere
        schema = _.cloneDeep(schema);
        // Deep clone is not required here, we just want
        // to modify the addFields property
        const options = _.clone(_options);
        options.addFields = schema.concat(options.addFields || []);
        // The arrangeFields option is trickier because we've already
        // done a compose() and so the groups are now denormalized as
        // properties of each field. Reconstruct the old
        // arrangeFields option so we can concatenate the new one
        const oldArrangeFields = [];
        _.each(schema, function (field) {
          if (field.group) {
            let group = _.find(oldArrangeFields, { name: field.group.name });
            if (!group) {
              group = _.clone(field.group);
              group.fields = [];
              oldArrangeFields.push(group);
            }
            group.fields.push(field.name);
          }
        });
        options.arrangeFields = oldArrangeFields.concat(options.arrangeFields || []);
        return self.compose(options);
      },

      // Converts a flat schema (array of field objects) into a two
      // dimensional schema, broken up by groups
      toGroups(fields) {
        // bail on empty schemas
        if (fields.length === 0) {
          return [];
        }

        // bail if we're already in groups
        if (fields[0].type === 'group') {
          return fields;
        }

        const groups = [];
        let currentGroup = -1;
        _.each(fields, function (field) {
          if (field.contextual) {
            return;
          }

          if (!field.group) {
            field.group = {
              name: defaultGroup.name,
              label: defaultGroup.label
            };
          }
          // first group, or not the current group
          if (groups.length === 0 || groups[currentGroup].name !== field.group.name) {
            groups.push({
              type: 'group',
              name: field.group.name,
              label: field.group.label,
              fields: []
            });
            currentGroup++;
          }
          // add field to current group
          groups[currentGroup].fields.push(field);
        });
        return groups;
      },

      // Return a new schema containing only the fields named in the
      // `fields` array, while maintaining existing group relationships.
      // Any empty groups are dropped. Do NOT include group names
      // in `fields`.

      subset(schema, fields) {

        let groups;

        // check if we're already grouped
        if (schema[0].type === 'group') {
          // Don't modify the original schema which may be in use elsewhere
          groups = _.cloneDeep(schema);

          // loop over each group and remove fields from them that aren't in
          // this subset
          _.each(groups, function (group) {
            group.fields = _.filter(group.fields, function (field) {
              return _.includes(fields, field.name);
            });
          });

          // remove empty groups
          groups = _.filter(groups, function (group) {
            return group.fields.length > 0;
          });

          return groups;
        } else {
          // otherwise this is a simple filter
          return _.filter(schema, function (field) {
            return _.includes(fields, field.name);
          });
        }
      },

      // Return a new object with all default settings
      // defined in the schema
      newInstance(schema) {
        return newInstance(schema, self);
      },

      subsetInstance(schema, instance) {
        const subset = {};
        _.each(schema, function (field) {
          if (field.type === 'group') {
            return;
          }
          const subsetCopy = self.fieldTypes[field.type].subsetCopy;
          if (!subsetCopy) {
            // These rules suffice for our standard fields
            subset[field.name] = instance[field.name];
            if (field.idsStorage) {
              subset[field.idsStorage] = instance[field.idsStorage];
            }
            if (field.fieldsStorage) {
              subset[field.fieldsStorage] = instance[field.fieldsStorage];
            }
          } else {
            subsetCopy(field.name, instance, subset, field);
          }
        });
        return subset;
      },

      // Determine whether an object is empty according to the schema.
      // Note this is not the same thing as matching the defaults. A
      // nonempty string or array is never considered empty. A numeric
      // value of 0 is considered empty

      empty(schema, object) {
        return !_.find(schema, function (field) {
          // Return true if not empty
          const value = object[field.name];
          if (value !== null && value !== undefined && value !== false) {
            const emptyTest = self.fieldTypes[field.type].empty;
            if (!emptyTest) {
              // Type has no method to check emptiness, so assume not empty
              return true;
            }
            return !emptyTest(field, value);
          }
        });
      },

      // Wrapper around isEqual method to get modified fields between two
      // documents instead of just getting a boolean, it will return an array of
      // the modified fields

      getChanges(req, schema, one, two) {
        return self.isEqual(req, schema, one, two, { getChanges: true });
      },

      // Compare two objects and return true only if their schema fields are
      // equal.
      //
      // Note that for relationship fields this comparison is based on the
      // idsStorage and fieldsStorage, which are updated at the time a document
      // is saved to the database, so it will not work on a document not yet
      // inserted or updated unless `prepareForStorage` is used.
      //
      // This method is invoked by the doc module to compare draft and published
      // documents and set the modified property of the draft, just before
      // updating the published version.
      //
      // When passing the option `getChange: true` it'll return an array of
      // changed fields in this case the method won't short circuit by directly
      // returning false when finding a changed field

      isEqual(req, schema, one, two, options = {}) {
        const changedFields = [];
        for (const field of schema) {
          const fieldType = self.fieldTypes[field.type];

          if (fieldType.isEqual) {
            if (!fieldType.isEqual(req, field, one, two)) {
              if (options.getChanges) {
                changedFields.push(field.name);
              } else {
                return false;
              }
            }
            continue;
          }

          if (
            !_.isEqual(one[field.name], two[field.name]) &&
            !((one[field.name] == null) && (two[field.name] == null))
          ) {
            if (options.getChanges) {
              changedFields.push(field.name);
            } else {
              return false;
            }
          }
        }

        return options.getChanges ? changedFields : true;
      },

      // Index the object's fields for participation in Apostrophe search unless
      // `searchable: false` is set for the field in question

      indexFields(schema, object, texts) {
        _.each(schema, function (field) {
          if (field.searchable === false) {
            return;
          }
          const fieldType = self.fieldTypes[field.type];
          if (!fieldType.index) {
            return;
          }
          fieldType.index(object[field.name], field, texts);
        });
      },

      async evaluateCondition(
        req, field, clause, destination, conditionalFields,
        followingValues = {}
      ) {
        const allValues = {
          ...followingValues,
          ...destination
        };

        // 1. Evaluate all conditions without the externals.
        const savedExternalConditions = [];
        const result = self.checkIfCondition(
          allValues,
          clause,
          (propName, conditionValue, docValue) => {
            if (conditionalFields?.[propName] === false) {
              return false;
            }
            if (self.isExternalCondition(propName)) {
              savedExternalConditions.push({
                name: propName,
                value: conditionValue
              });
            }
            // Non-boolean return values are ignored
          }
        );

        // 2. If the conditions evaluate to false or no external conditionss, stop.
        if (!result || savedExternalConditions.length === 0) {
          return result;
        }

        // 3. Otherwise, evaluate the external conditions.
        // Handle external conditions:
        //  - `if: { 'methodName()': true }`
        //  - `if: { 'moduleName:methodName()': 'expected value' }`
        const promises = savedExternalConditions.map(({ name }) => {
          return self.evaluateMethod(
            req,
            name,
            field.name,
            field.moduleName,
            destination._id
          );
        });

        try {
          const externalResults = await Promise.all(promises);
          return externalResults.every((externalResult, index) => {
            return externalResult === savedExternalConditions[index].value;
          });
        } catch (error) {
          throw self.apos.error('invalid', error.message);
        }
      },

      async isFieldRequired(req, field, destination) {
        return field.requiredIf
          ? await self.evaluateCondition(req, field, field.requiredIf, destination)
          : field.required;
      },

      // Convert submitted `data` object according to `schema`, sanitizing it
      // and populating the appropriate properties of `destination` with it.
      //
      // Most field types may be converted as plaintext or in the
      // format used for Apostrophe schema forms, which in most cases
      // is identical to that in which they will be stored in the database.
      // In Apostrophe 3.x, field converters automatically determine whether
      // they are being given plaintext or form data.
      //
      // If the submission cannot be converted due to an error that can't be
      // sanitized, this method throws an exception consisting of an array
      // of objects with `path` and `error` properties. `path` is the field
      // name, or a dot-separated path to the field name if the error was in a
      // nested `array` or `object` schema field, and `error` is the error code
      // which may  be a short string such as `required` or `min` that can be
      // used to set error class names, etc. If the error is not a string, it is
      // a database error etc. and should not be displayed in the browser
      // directly.
      //
      // ancestors consists of an array of objects where each represents
      // the context object at each level of nested sanitization, excluding
      // `destination` (the current level). This allows resolution of relative
      // `following` paths during sanitization.

      async convert(
        req,
        schema,
        data,
        destination,
        {
          fetchRelationships = true,
          ancestors = [],
          rootConvert = true
        } = {}
      ) {
        const options = {
          fetchRelationships,
          ancestors
        };
        if (Array.isArray(req)) {
          throw new Error('convert invoked without a req, do you have one in your context?');
        }

        const convertErrors = [];
        for (const field of schema) {
          if (field.readOnly) {
            continue;
          }

          // Fields that are contextual are left alone, not blanked out, if
          // they do not appear at all in the data object.
          if (field.contextual && !_.has(data, field.name)) {
            continue;
          }

          const { convert } = self.fieldTypes[field.type];
          if (!convert) {
            continue;
          }

          try {
            const isRequired = await self.isFieldRequired(req, field, destination);
            await convert(
              req,
              {
                ...field,
                required: isRequired
              },
              data,
              destination,
              {
                ...options,
                rootConvert: false
              }
            );
          } catch (err) {
            const error = Array.isArray(err)
              ? self.apos.error('invalid', { errors: err })
              : err;

            error.path = field.name;
            error.schemaPath = field.aposPath;
            convertErrors.push(error);
          }
        }

        if (!rootConvert) {
          if (convertErrors.length) {
            throw convertErrors;
          }

          return;
        }

        const nonVisibleFields = await self.getNonVisibleFields({
          req,
          schema,
          destination
        });

        const validErrors = await self.handleConvertErrors({
          req,
          schema,
          convertErrors,
          destination,
          nonVisibleFields
        });

        if (validErrors.length) {
          throw validErrors;
        }
      },

      async getNonVisibleFields({
        req, schema, destination,
        nonVisibleFields = new Set(), fieldPath = '',
        parentFollowingValues = {}
      }) {
        for (const field of schema) {
          const curPath = fieldPath ? `${fieldPath}.${field.name}` : field.name;
          const isVisible = await self.isVisible(
            req,
            schema,
            destination,
            field.name,
            parentFollowingValues
          );
          if (!isVisible) {
            nonVisibleFields.add(curPath);
            continue;
          }
          if (!field.schema) {
            continue;
          }

          // Get following values for the current parent before
          // going deeper into the schema.
          parentFollowingValues = self.getNextFollowingValues(
            schema,
            destination,
            parentFollowingValues
          );

          // Relationship does not support conditional fields right now
          if ([ 'array' /*, 'relationship' */].includes(field.type) && field.schema) {
            for (const arrayItem of destination[field.name] || []) {
              await self.getNonVisibleFields({
                req,
                schema: field.schema,
                destination: arrayItem,
                nonVisibleFields,
                fieldPath: `${curPath}.${arrayItem._id}`,
                parentFollowingValues
              });
            }
          } else if (field.type === 'object') {
            await self.getNonVisibleFields({
              req,
              schema: field.schema,
              destination: destination[field.name],
              nonVisibleFields,
              fieldPath: curPath,
              parentFollowingValues
            });
          }
        }

        return nonVisibleFields;
      },

      async handleConvertErrors({
        req,
        schema,
        convertErrors,
        nonVisibleFields,
        destination,
        destinationPath = '',
        hiddenAncestors = false
      }) {
        const validErrors = [];
        for (const error of convertErrors) {
          const [ destId, destPath ] = error.path.includes('.')
            ? error.path.split('.')
            : [ null, error.path ];
          const curDestination = destId
            ? (destination?.items || destination || []).find(({ _id }) => _id === destId)
            : destination;

          const errorPath = destinationPath
            ? `${destinationPath}.${error.path}`
            : error.path;

          // Case were this error field hasn't been treated
          // Should check if path starts with, because parent can be invisible
          const nonVisibleField = hiddenAncestors || nonVisibleFields.has(errorPath);

          // We set default values only on final error fields
          if (nonVisibleField && !error.data?.errors) {
            const curSchema = self.getFieldLevelSchema(schema, error.schemaPath);
            self.setDefaultToInvisibleField(curDestination, curSchema, error.path);
            continue;
          }

          if (error.data?.errors) {
            const subErrors = await self.handleConvertErrors({
              req,
              schema,
              convertErrors: error.data.errors,
              nonVisibleFields,
              destination: curDestination[destPath],
              destinationPath: errorPath,
              hiddenAncestors: nonVisibleField
            });

            // If invalid error has no sub error, this one can be removed
            if (!subErrors.length) {
              continue;
            }

            error.data.errors = subErrors;
          }
          validErrors.push(error);
        }

        return validErrors;
      },

      setDefaultToInvisibleField(destination, schema, fieldPath) {
        // Field path might contain the ID of the object in which it is
        // contained We just want the field name here
        const [ _id, fieldName ] = fieldPath.includes('.')
          ? fieldPath.split('.')
          : [ null, fieldPath ];
        // It is not reasonable to enforce required,
        // min, max or anything else for fields
        // hidden via "if" as the user cannot correct it
        // and it will not be used. If the user changes
        // the conditional field later then they won't
        // be able to save until the erroneous field
        // is corrected
        const field = schema.find(field => field.name === fieldName);
        if (field) {
          // To protect against security issues, an invalid value
          // for a field that is not visible should be quietly discarded.
          // We only worry about this if the value is not valid, as otherwise
          // it's a kindness to save the work so the user can toggle back to it
          destination[field.name] = klona((field.def !== undefined)
            ? field.def
            : self.fieldTypes[field.type]?.def);
        }
      },

      getFieldLevelSchema(schema, fieldPath) {
        if (!fieldPath || fieldPath === '/') {
          return schema;
        }
        let curSchema = schema;
        const parts = fieldPath.split('/');
        parts.pop();
        for (const part of parts) {
          const curField = curSchema.find(({ name }) => name === part);
          curSchema = curField.schema;
        }

        return curSchema;
      },

      // Retrieve all `following` fields from a schema (ignore sub-schema),
      // merge it with the provided `parentFollowingValues` and apply the
      // `<` prefix to the keys of the resulting object to increase the nesting
      // level so that they can be passed to a sub-schema.
      getNextFollowingValues(schema, values, parentFollowingValues = {}) {
        const newFollowingValues = {};
        for (const field of schema) {
          if (!field.following) {
            continue;
          }
          const following = Array.isArray(field.following)
            ? field.following
            : [ field.following ];
          for (const followingField of following) {
            // Add the parent prefix to the following field
            newFollowingValues[`<${followingField}`] = values[followingField];
          }
        }

        for (const [ name, value ] of Object.entries(parentFollowingValues)) {
          newFollowingValues[`<${name}`] = value;
        }

        return newFollowingValues;
      },
      // Determine whether the given field is visible
      // based on `if` conditions of all fields
      async isVisible(
        req, schema, destination, name, followingValues = {}
      ) {
        const conditionalFields = {};
        const errors = {};

        while (true) {
          let change = false;
          for (const field of schema) {
            if (field.if) {
              try {
                const result = await self.evaluateCondition(
                  req,
                  field,
                  field.if,
                  destination,
                  conditionalFields,
                  followingValues
                );
                const previous = conditionalFields[field.name];
                if (previous !== result) {
                  change = true;
                }
                conditionalFields[field.name] = result;
              } catch (error) {
                errors[field.name] = error;
              }
            }
          }

          // send the error related to the given field via the `name` param
          if (errors[name]) {
            throw errors[name];
          }

          if (!change) {
            break;
          }
        }
        if (_.has(conditionalFields, name)) {
          return conditionalFields[name];
        } else {
          return true;
        }
      },

      async evaluateMethod(
        req,
        methodKey,
        fieldName,
        fieldModuleName,
        docId = null,
        optionalParenthesis = false,
        following = {},
        featureType = 'field'
      ) {
        const [ methodDefinition, rest ] = methodKey.split('(');
        const hasParenthesis = rest !== undefined;

        if (!hasParenthesis && !optionalParenthesis) {
          throw new Error(`The method "${methodDefinition}" defined in the "${fieldName}" ${featureType} should be written with parenthesis: "${methodDefinition}()".`);
        }
        if (hasParenthesis && !methodKey.endsWith('()')) {
          self.apos.util.warn(`The method "${methodDefinition}" defined in the "${fieldName}" ${featureType} should be written without argument: "${methodDefinition}()".`);
          methodKey = methodDefinition + '()';
        }

        const [ methodName, moduleName = fieldModuleName ] = methodDefinition
          .split(':')
          .reverse();

        const module = self.apos.modules[moduleName];

        if (!module) {
          throw new Error(`The "${moduleName}" module defined in the "${fieldName}" ${featureType} does not exist.`);
        } else if (!module[methodName]) {
          throw new Error(`The "${methodName}" method from "${moduleName}" module defined in the "${fieldName}" ${featureType} does not exist.`);
        }

        return module[methodName](req, { docId }, following);
      },

      // Driver invoked by the "relationship" methods of the standard
      // relationship field types.
      //
      // All arguments must be present, however fieldsStorage
      // may be undefined to indicate none is needed.
      async relationshipDriver(
        req,
        method,
        reverse,
        items,
        idsStorage,
        fieldsStorage,
        objectField,
        options
      ) {
        if (!options) {
          options = {};
        }
        const find = options.find;
        const builders = options.builders || {};
        const getCriteria = options.getCriteria || {};
        await method(items, idsStorage, fieldsStorage, objectField, ids => {
          const idsCriteria = {};
          if (reverse) {
            idsCriteria[idsStorage] = { $in: ids };
          } else {
            idsCriteria.aposDocId = { $in: ids };
          }
          const criteria = {
            $and: [
              getCriteria,
              idsCriteria
            ]
          };
          const query = find(req, criteria);
          // Builders hardcoded as part of this relationship's options don't
          // require any sanitization
          query.applyBuilders(builders);
          return query.toArray();
        }, self.apos.doc.toAposDocId);
      },

      // Fetch all the relationships in the schema on the specified object or
      // array of objects. The withRelationships option may be omitted.
      //
      // If withRelationships is omitted, null or undefined, all the
      // relationships in the schema are performed, and also any relationships
      // specified by the 'withRelationships' option of each relationship field
      // in the schema, if any. And that's where it stops. Infinite recursion is
      // not possible.
      //
      // If withRelationships is specified and set to "false",
      // no relationships at all are performed.
      //
      // If withRelationships is set to an array of relationship names found
      // in the schema, then only those relationships are performed, ignoring
      // any 'withRelationships' options found in the schema.
      //
      // If a relationship name in the withRelationships array uses dot
      // notation, like this:
      //
      // _events._locations
      //
      // Then the related events are fetched, and the locations related to
      // those events are fetched, assuming that _events is defined as a
      // relationship in the original schema and _locations is defined as a
      // relationship in the schema for the events module. Multiple "dot
      // notation" relationships may share a prefix.
      //
      // Relationships are also supported in the schemas of array fields.

      async relate(req, schema, objectOrArray, withRelationships) {

        if (withRelationships === false) {
          // Relationships explicitly deactivated for this call
          return;
        }

        const objects = _.isArray(objectOrArray) ? objectOrArray : [ objectOrArray ];

        if (!objects.length) {
          // Don't waste effort
          return;
        }

        // build an array of relationships of interest, found at any level
        // in the schema, even those nested in array schemas. Add
        // an _arrays property to each one which contains the names
        // of the array fields leading to this relationship, if any, so
        // we know where to store the results. Also set a
        // _dotPath property which can be used to identify relevant
        // relationships when the withRelationships option is present

        let relationships = [];

        function findRelationships(schema, arrays) {
          // Shallow clone of each relationship to allow
          // for independent _dotPath and _arrays properties
          // for different requests
          const _relationships = _.filter(schema, function (field) {
            return !!self.fieldTypes[field.type].relate;
          }).map(relationship => ({ ...relationship }));
          _.each(_relationships, function (relationship) {
            if (!arrays.length) {
              relationship._dotPath = relationship.name;
            } else {
              relationship._dotPath = arrays.join('.') + '.' + relationship.name;
            }
            // If we have more than one object we're not interested in
            // relationships with the ifOnlyOne restriction right now.
            if (objects.length > 1 && relationship.ifOnlyOne) {
              return;
            }
            relationship._arrays = _.clone(arrays);
          });
          relationships = relationships.concat(_relationships);
          _.each(schema, function (field) {
            if (field.type === 'array' || field.type === 'object') {
              findRelationships(field.schema, arrays.concat(field.name));
            }
          });
        }

        findRelationships(schema, []);

        // The withRelationships option allows restriction of relationships.
        // Set to false it blocks all relationships. Set to an array, it allows
        // the relationships named within. Dot notation can be used to specify
        // relationships in array properties, or relationships reached via other
        // relationships.
        //
        // By default, all configured relationships will take place, but
        // withRelationships: false
        // will be passed when fetching the objects on the other end of the
        // relationship, so that infinite recursion never takes place.

        const withRelationshipsNext = {};
        // Explicit withRelationships option passed to us
        if (Array.isArray(withRelationships)) {
          relationships = _.filter(relationships, function (relationship) {
            const dotPath = relationship._dotPath;
            let winner;
            _.each(withRelationships, function (withRelationshipName) {
              if (withRelationshipName === dotPath) {
                winner = true;
                return;
              }
              if (withRelationshipName.substr(0, dotPath.length + 1) === dotPath + '.') {
                if (!withRelationshipsNext[dotPath]) {
                  withRelationshipsNext[dotPath] = [];
                }
                withRelationshipsNext[dotPath].push(
                  withRelationshipName.substr(dotPath.length + 1)
                );
                winner = true;
              }
            });
            return winner;
          });
        } else {
          // No explicit withRelationships option for us, so we do all the
          // relationships we're configured to do, and pass on the
          // withRelationships options we have configured for those
          _.each(relationships, function (relationship) {
            if (relationship.withRelationships) {
              withRelationshipsNext[relationship._dotPath] = relationship
                .withRelationships;
            }
          });
        }

        for (const relationship of relationships) {
          const arrays = relationship._arrays;

          const _objects = findObjectsInArrays(objects, arrays);

          if (!relationship.name.match(/^_/)) {
            throw Error('Relationships should always be given names beginning with an underscore (_). Otherwise we would waste space in your database storing the results statically. There would also be a conflict with the array field withRelationships syntax. Relationship name is: ' + relationship._dotPath);
          }
          if (Array.isArray(relationship.withType)) {
            // Polymorphic join
            for (const type of relationship.withType) {
              const manager = self.apos.doc.getManager(type);
              if (!manager) {
                throw Error('I cannot find the instance type ' + type);
              }
              const find = manager.find;

              const relationships = withRelationshipsNext[relationship._dotPath] || false;
              const options = {
                find,
                builders: { relationships }
              };
              const subname = relationship.name + ':' + type;
              const _relationship = _.assign({}, relationship, {
                name: subname,
                withType: type
              });

              // Allow options to the get() method to be
              // specified in the relationship configuration
              if (_relationship.builders) {
                _.extend(options.builders, _relationship.builders);
              }
              if (_relationship.buildersByType && _relationship.buildersByType[type]) {
                _.extend(options.builders, _relationship.buildersByType[type]);
              }
              await self.apos.util.recursionGuard(req, `${_relationship.type}:${_relationship.withType}`, () => {
                // Allow options to the getter to be specified in the schema,
                return self.fieldTypes[_relationship.type]
                  .relate(req, _relationship, _objects, options);
              });
              _.each(_objects, function (object) {
                if (object[subname]) {
                  if (Array.isArray(object[subname])) {
                    object[relationship.name] = (object[relationship.name] || [])
                      .concat(object[subname]);
                  } else {
                    object[relationship.name] = object[subname];
                  }
                }
              });
            }
            if (relationship.idsStorage) {
              _.each(_objects, function (object) {
                if (object[relationship.name]) {
                  const locale = `${req.locale}:${req.mode}`;
                  object[relationship.name] = self.apos.util.orderById(
                    object[relationship.idsStorage].map(id => `${id}:${locale}`),
                    object[relationship.name]
                  );
                }
              });
            }
          }

          const manager = self.apos.doc.getManager(relationship.withType);
          if (!manager) {
            throw Error('I cannot find the instance type ' + relationship.withType);
          }

          // If it has a getter, use it, otherwise supply one
          const find = manager.find;

          const relationships = withRelationshipsNext[relationship._dotPath] || false;
          const options = {
            find,
            builders: { relationships }
          };

          // Allow options to the get() method to be
          // specified in the relationship configuration
          if (relationship.builders) {
            _.extend(options.builders, relationship.builders);
          }

          // If there is a projection for a reverse relationship, make sure it
          // includes the idsStorage and fieldsStorage for the relationship,
          // otherwise no related documents will be returned. Make sure the
          // projection is positive, not negative, before attempting to add more
          // positive assertions to it
          if ((relationship.type === 'relationshipReverse') && options.builders.project && Object.values(options.builders.project).some(v => !!v)) {
            if (relationship.idsStorage) {
              options.builders.project[relationship.idsStorage] = 1;
            }
            if (relationship.fieldsStorage) {
              options.builders.project[relationship.fieldsStorage] = 1;
            }
          }

          // Allow options to the getter to be specified in the schema
          await self.apos.util.recursionGuard(req, `${relationship.type}:${relationship.withType}`, () => {
            return self.fieldTypes[relationship.type]
              .relate(req, relationship, _objects, options);
          });
        }

        function findObjectsInArrays(objects, arrays) {
          if (!arrays) {
            return [];
          }
          if (!arrays.length) {
            return objects;
          }
          const array = arrays[0];
          let _objects = [];
          _.each(objects, function (object) {
            _objects = _objects.concat(object[array] || []);
          });
          return findObjectsInArrays(_objects, arrays.slice(1));
        }
      },

      // In the given document or widget, update any underlying
      // storage needs required for relationships, arrays, etc.,
      // such as populating the idsStorage and fieldsStorage
      // properties of relationship fields, or setting the
      // arrayName property of array items. This method is
      // always invoked for you by @apostrophecms/doc-type in a
      // beforeSave handler. This method also recursively invokes
      // itself as needed for relationships nested in widgets,
      // array fields and object fields.
      //
      // If a relationship field is present by name (such as `_products`)
      // in the document, that is taken as authoritative, and any
      // existing values in the `idsStorage` and `fieldsStorage`
      // are overwritten. If the relationship field is not present, the
      // existing values are left alone. This allows the developer
      // to safely update a document that was fetched with
      // `.relationships(false)`, provided the projection included
      // the ids.
      //
      // Currently `req` does not impact this, but that may change.

      prepareForStorage(req, doc, options = {}) {
        const can = (field) => {
          const canEdit = () => self.apos.permission.can(
            req,
            field.editPermission.action,
            field.editPermission.type
          );
          const canView = () => self.apos.permission.can(
            req,
            field.viewPermission.action,
            field.viewPermission.type
          );
          return options.permissions === false ||
            (!field.withType && !field.editPermission && !field.viewPermission) ||
            (field.withType && self.apos.permission.can(req, 'view', field.withType)) ||
            (field.editPermission && canEdit()) ||
            (field.viewPermission && canView()) ||
            false;
        };

        const handlers = {
          arrayItem: (field, object) => {
            if (!object || !can(field)) {
              return;
            }

            object._id = object._id || self.apos.util.generateId();
            object.metaType = 'arrayItem';
            object.scopedArrayName = field.scopedArrayName;
          },
          object: (field, object) => {
            if (!object || !can(field)) {
              return;
            }

            object.metaType = 'object';
            object.scopedObjectName = field.scopedObjectName;
          },
          relationship: (field, doc) => {
            if (!Array.isArray(doc[field.name]) || !can(field)) {
              return;
            }

            doc[field.idsStorage] = doc[field.name]
              .map(relatedDoc => self.apos.doc.toAposDocId(relatedDoc));
            if (field.fieldsStorage) {
              const fieldsById = doc[field.fieldsStorage] || {};
              for (const relatedDoc of doc[field.name]) {
                if (relatedDoc._fields) {
                  fieldsById[self.apos.doc.toAposDocId(relatedDoc)] = relatedDoc._fields;
                }
              }
              doc[field.fieldsStorage] = fieldsById;
            }
          }
        };

        self.apos.doc.walkByMetaType(doc, handlers);
      },

      simulateRelationshipsFromStorage(req, doc) {
        const handlers = {
          relationship: (field, object) => {
            const manager = self.apos.doc.getManager(field.withType);
            const setId = (id) => manager.options.localized !== false
              ? `${id}:${doc.aposLocale}`
              : id;

            const itemIds = object[field.idsStorage] || [];
            object[field.name] = itemIds.map(id => ({
              _id: setId(id),
              _fields: object[field.fieldsStorage]?.[id] || {}
            }));
          }
        };

        self.apos.doc.walkByMetaType(doc, handlers);
      },

      // Add a new field type. The `type` object may contain the following
      // properties:
      //
      // ### `name`
      //
      // Required. The name of the field type, such as `select`. Use a unique
      // prefix to avoid collisions with future official Apostrophe field types.
      //
      // ### `convert`
      //
      // Required. An `async` function which takes `(req, field, data,
      // destination)`. The value of the field is drawn from the untrusted input
      // object `data` and sanitized if possible, then copied to the appropriate
      // property (or properties) of `destination`.
      //
      // `field` contains the schema field definition, useful to access
      // `def`, `min`, `max`, etc.
      //
      // If the field cannot be sanitized an error can be thrown. To signal an
      // error that can be examined by browser code and used for UI, throw a
      // string like `required`. If the field is a composite field (`array` or
      // `object`), throw an array of objects with `path` and `