apostrophe/modules/@apostrophecms/attachment/index.js

The Apostrophe Content Management System.

const _ = require('lodash');
const path = require('path');
const fs = require('fs');
const Promise = require('bluebird');
const createDOMPurify = require('dompurify');
const { JSDOM } = require('jsdom');

module.exports = {
  options: {
    alias: 'attachment',
    label: 'apostrophe:attachment'
  },
  cascades: [ 'imageSizes' ],
  imageSizes: {
    add: {
      max: {
        width: 1600,
        height: 1600
      },
      full: {
        width: 1140,
        height: 1140
      },
      'two-thirds': {
        width: 760,
        height: 760
      },
      'one-half': {
        width: 570,
        height: 700
      },
      'one-third': {
        width: 380,
        height: 700
      },
      'one-sixth': {
        width: 190,
        height: 350
      }
    }
  },

  async init(self) {
    // For convenience and bc
    self.uploadfs = self.apos.uploadfs;
    // uploadfs expects an array
    self.imageSizes = Object.keys(self.imageSizes).map(name => ({
      name,
      ...self.imageSizes[name]
    }));
    self.name = 'attachment';
    self.fileGroups = self.options.fileGroups || [
      {
        name: 'images',
        label: 'apostrophe:images',
        extensions: [
          'gif',
          'jpg',
          'png',
          'svg',
          'webp'
        ],
        extensionMaps: { jpeg: 'jpg' },
        // uploadfs should treat this as an image and create scaled versions
        image: true
      },
      {
        name: 'office',
        label: 'apostrophe:office',
        extensions: [
          'txt',
          'rtf',
          'pdf',
          'xls',
          'ppt',
          'doc',
          'pptx',
          'sldx',
          'ppsx',
          'potx',
          'xlsx',
          'xltx',
          'csv',
          'docx',
          'dotx'
        ],
        extensionMaps: {},
        // uploadfs should just accept this file as-is
        image: false
      }
    ];

    if (self.options.addFileGroups) {
      self.options.addFileGroups.forEach(newGroup => {
        self.addFileGroup(newGroup);
      });
    };

    // Do NOT add keys here unless they have the value `true`
    self.croppable = {
      gif: true,
      jpg: true,
      png: true,
      webp: true
    };

    // Do NOT add keys here unless they have the value `true`
    self.sized = {
      gif: true,
      jpg: true,
      png: true,
      webp: true
    };

    self.sizeAvailableInArchive = self.options.sizeAvailableInArchive || 'one-sixth';

    self.rescaleTask = require('./lib/tasks/rescale.js')(self);
    self.downloadAllTask = require('./lib/tasks/download-all.js')(self);
    self.addFieldType();
    self.enableBrowserData();

    self.db = await self.apos.db.collection('aposAttachments');
    await self.db.createIndex({ docIds: 1 });
    await self.db.createIndex({ archivedDocIds: 1 });
    self.addLegacyMigrations();
    self.addSvgSanitizationMigration();

    // Lazy cache for types whose schema contains attachment fields.
    self.typesWithAttachmentFields = new Map();
  },

  tasks(self) {
    return {
      rescale: {
        usage: 'Usage: node app @apostrophecms/attachment:rescale\n\nRegenerate all sizes of all image attachments. Useful after a new size\nis added to the configuration. Takes a long time!',
        task: self.rescaleTask
      },
      'download-all': {
        usage: 'Usage: node app @apostrophecms/attachment:download-all --to=public/uploads/attachments [--resume] [--parallel=3]\n\nDownload all attachments to a local folder, usually to sync\nfrom a non-local uploadfs backend. Takes a long time!',
        task: self.downloadAllTask
      },
      'migrate-to-disabled-file-key': {
        usage: 'Usage: node app @apostrophecms/attachment:migrate-to-disabled-file-key\n\nThis task should be run after adding the disabledFileKey option to uploadfs\nfor the first time. It should only be relevant for storage backends where\nthat option is not mandatory, i.e. only local storage as of this writing.',
        task: self.migrateToDisabledFileKeyTask
      },
      'migrate-from-disabled-file-key': {
        usage: 'Usage: node app @apostrophecms/attachment:migrate-from-disabled-file-key\n\nThis task should be run after removing the disabledFileKey option from uploadfs.\nIt should only be relevant for storage backends where\n' + 'that option is not mandatory, i.e. only local storage as of this writing.',
        task: self.migrateFromDisabledFileKeyTask
      },
      'recompute-all-doc-references': {
        usage: 'Recompute mapping between attachments and docs,\nshould only be needed for rare repair situations',
        task: self.recomputeAllDocReferences
      }
    };
  },

  // TODO RESTify where possible
  apiRoutes(self) {
    // TODO this must be updated to employ the new useMiddleware format and that
    // section has to be implemented
    return {
      post: {
        upload: [
          self.canUpload,
          require('multer')({ dest: require('os').tmpdir() }).single('file'),
          async function (req) {
            try {
              // The file comes from multer middleware
              const file = req.file;

              if (!file) {
                throw self.apos.error('invalid');
              }

              const attachment = await self.insert(req, file);
              self.all({ attachment }, { annotate: true });

              return attachment;
            } finally {
              if (req.file) {
                try {
                  fs.unlinkSync(req.file.path);
                } catch (e) {
                  self.apos.util.warn(`Uploaded temporary file ${req.file.path} was already removed, this should have been the responsibility of the upload route`);
                }
              }
            }
          }
        ],
        // Crop a previously uploaded image, based on the `id` POST parameter
        // and the `crop` POST parameter. `id` should refer to an existing
        // file in /attachments. `crop` should contain top, left, width and
        // height properties.
        //
        // This route uploads a new, cropped version of
        // the existing image to uploadfs, named:
        //
        // /attachments/ID-NAME.top.left.width.height.extension
        //
        // The `crop` object is appended to the `crops` array property
        // of the file object.
        crop: [
          self.canUpload,
          async function (req) {
            const _id = self.apos.launder.id(req.body._id);
            const { crop } = req.body;

            if (!_id || !crop || typeof crop !== 'object' || Array.isArray(crop)) {
              throw self.apos.error('invalid');
            }

            const sanitizedCrop = self.sanitizeCrop(crop);

            if (!sanitizedCrop) {
              throw self.apos.error('invalid');
            }

            await self.crop(req, _id, sanitizedCrop);

            return true;
          }
        ]
      }
    };
  },
  handlers(self) {
    return {
      '@apostrophecms/doc-type:afterSave': {
        async updateDocReferencesAfterSave(req, doc, options) {
          return self.updateDocReferences(doc);
        }
      },
      '@apostrophecms/doc-type:afterArchive': {
        async updateDocReferencesAfterArchive(req, doc) {
          return self.updateDocReferences(doc);
        }
      },
      '@apostrophecms/doc-type:afterRescue': {
        async updateDocReferencesAfterRescue(req, doc) {
          return self.updateDocReferences(doc);
        }
      },
      '@apostrophecms/doc-type:afterDelete': {
        async updateDocReferencesAfterDelete(req, doc) {
          return self.updateDocReferences(doc, {
            deleted: true
          });
        }
      }
    };
  },
  methods(self) {
    return {
      addFieldType() {
        self.apos.schema.addFieldType({
          name: self.name,
          convert: self.convert,
          index: self.index,
          register: self.register
        });
      },
      async convert(req, field, data, object) {
        let info = data[field.name];
        if (typeof info !== 'object') {
          info = {};
        }
        info = _.pick(info, '_id', 'crop');
        info._id = self.apos.launder.id(info._id);
        if (!info._id) {
          object[field.name] = null;
        }
        info.crop = info.crop ? self.sanitizeCrop(info.crop) : undefined;
        const dbInfo = await self.db.findOne({ _id: info._id });
        if (!dbInfo) {
          object[field.name] = null;
          return;
        }
        _.assign(info, _.omit(dbInfo, 'crop'));

        // Check if the file type is acceptable
        const correctedExtensions = self.checkExtension(field, info);

        if (correctedExtensions) {
          const message = req.t('apostrophe:fileTypeNotAccepted', {
            // i18next has no built-in support for interpolating an array
            // argument
            extensions: correctedExtensions.join(req.t('apostrophe:listJoiner')),
            extension: dbInfo.extension
          });
          throw self.apos.error('invalid', message);
        }

        if (info.crop) {
          if (!_.find(info.crops, info.crop)) {
            info.crop = null;
          }
        }
        info.used = true;
        await self.db.replaceOne({ _id: info._id }, info);
        object[field.name] = info;
      },
      index(value, field, texts) {
        const silent = field.silent === undefined ? true : field.silent;
        texts.push({
          weight: field.weight || 15,
          text: (value && value.title) || '',
          silent
        });
      },
      // When the field is registered in the schema,
      // canonicalize .group and .extensions and .extension
      // into .accept for convenience, as a comma-separated
      // list of dotted file extensions suitable to pass to
      // the "accept" HTML5 attribute, including mapped extensions
      // like jpeg. If none of these options are set, .accept is
      // set to an array of all accepted file extensions across
      // all groups
      register(metaType, type, field) {
        let fileGroups = self.fileGroups;
        if (field.fileGroups) {
          fileGroups = fileGroups.filter(group => field.fileGroups.includes(group.name));
        }
        if (field.fileGroup) {
          fileGroups = fileGroups.filter(group => group.name === field.fileGroup);
        }
        let extensions = [];
        fileGroups.forEach(group => {
          extensions = [ ...extensions, ...group.extensions ];
        });
        if (field.extensions) {
          extensions = extensions
            .filter(extension => field.extensions.includes(extension));
        }
        if (field.extension) {
          extensions = extensions.filter(extension => extension === field.extension);
        }
        fileGroups.forEach(group => {
          for (const [ from, to ] of Object.entries(group.extensionMaps || {})) {
            if (extensions.includes(to) && (!extensions.includes(from))) {
              extensions.push(from);
            }
          }
        });
        field.accept = extensions.map(extension => `.${extension}`).join(',');
      },
      // Checked a given attachment's file extension against the extensions
      // allowed by a particular schema field. If the attachment's file
      // extension is allowed, `null` is returned. If the file extension is not
      // allowed, `checkExtension` returns an array of the file extensions that
      // _are_ allowed (or an empty array if the allowed extensions are
      // unknown).
      checkExtension(field, attachment) {
        const groups = field.fileGroups ||
          (field.fileGroup && [ field.fileGroup ]);
        let extensions;

        if (groups) {
          if (!_.includes(groups, attachment.group)) {
            extensions = [];
            _.each(groups, function (group) {
              const groupInfo = _.find(self.fileGroups, { name: group });
              if (!groupInfo) {
                return [];
              }
              extensions = extensions.concat(groupInfo.extensions);
            });
            return extensions;
          }
        }
        extensions = field.extensions ||
          (field.extension && [ field.extension ]);

        if (extensions) {
          if (!_.includes(extensions, attachment.extension)) {
            return extensions;
          }
        }
        return null;
      },
      // Insert a file as an Apostrophe attachment. The `file` object
      // should be an object with `name` and `path` properties.
      // `name` must be the name the user claims for the file, while `path`
      // must be the actual full path to the file on disk and need not have
      // any file extension necessarily.
      //
      // Note that when using Express to handle file uploads,
      // req.files['yourfieldname'] will be such an object as long as you
      // configure jquery fileupload to submit one per request.
      //
      // The `options` argument may be omitted completely.
      // If `options.permissions` is explicitly set to `false`,
      // permissions are not checked.
      //
      // This method returns `attachment` where `attachment` is an attachment
      // object, suitable for passing to the `url` API and for use as the value
      // of a `type: 'attachment'` schema field.
      async insert(req, file, options = {}) {
        // Handle both multer (originalname) and connect-multiparty (name) formats
        const fileName = file.originalname || file.name;
        let extension = path.extname(fileName);
        if (extension && extension.length) {
          extension = extension.substr(1);
        }
        extension = extension.toLowerCase();
        // Do we *ever* accept this file extension?
        const group = self.getFileGroup(extension);

        if (!group) {
          const accepted = _.union(_.map(self.fileGroups, 'extensions')).flat();
          throw self.apos.error('invalid', req.t('apostrophe:fileTypeNotAccepted', {
            extensions: accepted.join(req.t('apostrophe:listJoiner')),
            extension
          }));
        }

        if (
          options.attachmentId &&
          await self.apos.attachment.db.findOne({ _id: options.attachmentId })
        ) {
          throw self.apos.error('invalid', 'duplicate');
        }

        const info = {
          _id: options.attachmentId ?? self.apos.util.generateId(),
          group: group.name,
          createdAt: new Date(),
          name: self.apos.util.slugify(path.basename(fileName, path.extname(fileName))),
          title: self.apos.util.sortify(
            path.basename(fileName, path.extname(fileName))
          ),
          extension,
          type: 'attachment',
          docIds: options.docIds ?? [],
          archivedDocIds: options.archivedDocIds ?? []
        };
        if (!(options.permissions === false)) {
          if (!self.apos.permission.can(req, 'upload-attachment')) {
            throw self.apos.error('forbidden');
          }
        }
        info.length = await self.apos.util.fileLength(file.path);
        info.md5 = await self.apos.util.md5File(file.path);
        if (info.extension === 'svg') {
          try {
            await self.sanitizeSvg(file.path);
          } catch (e) {
            // Currently DOMPurify passes invalid SVG content without comment
            // as long as it's not an SVG XSS attack vector, but make provision
            // to report a relevant error if that changes
            throw self.apos.error('invalid', req.t('apostrophe:fileInvalid'));
          }
        }
        if (self.isSized(extension)) {
          // For images we correct automatically for common file extension
          // mistakes
          const result = await Promise.promisify(self.uploadfs.copyImageIn)(
            file.path,
            '/attachments/' + info._id + '-' + info.name,
            { sizes: self.imageSizes }
          );
          info.extension = result.extension;
          info.width = result.width;
          info.height = result.height;
          if (info.width > info.height) {
            info.landscape = true;
          } else {
            info.portrait = true;
          }
        } else {
          // For non-image files we have to trust the file extension
          // (but we only serve it as that content type, so this should
          // be reasonably safe)
          await Promise.promisify(self.uploadfs.copyIn)(file.path, '/attachments/' + info._id + '-' + info.name + '.' + info.extension);
        }
        info.createdAt = new Date();

        await self.emit('beforeInsert', req, info);

        await self.db.insertOne(info);
        return info;
      },

      async update(req, file, attachment) {
        const existing = await self.db.findOne({ _id: attachment._id });
        if (!existing) {
          throw self.apos.error('notfound');
        }

        const projection = {
          _id: 1,
          archived: 1
        };

        const existingRelatedDocs = await self.apos.doc.db
          .find({
            _id: {
              $in: [
                ...existing.docIds,
                ...existing.archivedDocIds,
                ...attachment.docIds,
                ...attachment.archivedDocIds
              ]
            }
          }, { projection })
          .toArray();

        const { docIds, archivedDocIds } = existingRelatedDocs
          .reduce(({ docIds, archivedDocIds }, doc) => {
            return {
              docIds: [ ...docIds, ...!doc.archived ? [ doc._id ] : [] ],
              archivedDocIds: [ ...archivedDocIds, ...doc.archived ? [ doc._id ] : [] ]
            };
          }, {
            docIds: [],
            archivedDocIds: []
          });

        await self.alterAttachment(existing, 'remove');
        await self.db.deleteOne({ _id: existing._id });
        await self.insert(req, file, {
          attachmentId: attachment._id,
          docIds: _.uniq([ ...docIds, ...existing.docIds || [] ]),
          archivedDocIds: _.uniq([ ...archivedDocIds, ...existing.archivedDocIds || [] ])
        });
      },

      // Given a path to a local svg file, sanitize any XSS attack vectors that
      // may be present in the file. The caller is responsible for catching any
      // exception thrown and treating that as an invalid file but there is no
      // guarantee that invalid SVG files will be detected or cleaned up, only
      // XSS attacks.
      async sanitizeSvg(path) {
        const readFile = require('util').promisify(fs.readFile);
        const writeFile = require('util').promisify(fs.writeFile);
        const window = new JSDOM('').window;
        const DOMPurify = createDOMPurify(window);
        DOMPurify.addHook('afterSanitizeAttributes', node => {
          if (node.hasAttribute('xlink:href') && !node.getAttribute('xlink:href').match(/^#/)) {
            node.remove();
          }
        });
        const dirty = await readFile(path);
        const clean = DOMPurify.sanitize(dirty, {
          ADD_TAGS: [ 'use' ]
        });
        return writeFile(path, clean);
      },
      getFileGroup(extension) {
        return _.find(self.fileGroups, function (group) {
          const candidate = group.extensionMaps[extension] || extension;
          if (_.includes(group.extensions, candidate)) {
            return true;
          }
        });
      },
      async crop(req, _id, crop) {
        const info = await self.db.findOne({ _id });

        if (!info) {
          throw self.apos.error('notfound');
        }

        if (!self.croppable[info.extension]) {
          throw self.apos.error('invalid', req.t('apostrophe:fileTypeCannotBeCropped', {
            extension: info.extension
          }));
        }
        const crops = info.crops || [];
        const existing = _.find(crops, crop);

        if (existing) {
          // We're done, this crop is already available
          return;
        }
        // Pull the original out of cloud storage to a temporary folder where
        // it can be cropped and popped back into uploadfs
        const originalFile = `/attachments/${info._id}-${info.name}.${info.extension}`;
        const tempFile = `${self.uploadfs.getTempPath()}/${self.apos.util.generateId()}.${info.extension}`;
        const croppedFile = `/attachments/${info._id}-${info.name}.${crop.left}.${crop.top}.${crop.width}.${crop.height}.${info.extension}`;

        await Promise.promisify(self.uploadfs.copyOut)(originalFile, tempFile);
        await Promise.promisify(self.uploadfs.copyImageIn)(tempFile, croppedFile, {
          crop,
          sizes: self.imageSizes
        });

        await self.db.updateOne({
          _id: info._id
        }, {
          $set: {
            crops: [
              ...crops,
              crop
            ]
          }
        });
        await Promise.promisify(fs.unlink)(tempFile);
      },
      sanitizeCrop(crop) {
        const neededProps = [ 'top', 'left', 'width', 'height' ];
        const { integer: sanitizeInteger } = self.apos.launder;

        if (neededProps.some((prop) => !Object.keys(crop).includes(prop))) {
          return null;
        }

        return {
          top: sanitizeInteger(crop.top, 0, 0, 10000),
          left: sanitizeInteger(crop.left, 0, 0, 10000),
          width: sanitizeInteger(crop.width, 0, 0, 10000),
          height: sanitizeInteger(crop.height, 0, 0, 10000)
        };
      },
      // This method return a default icon url if an attachment is missing
      // to avoid template errors
      getMissingAttachmentUrl() {
        const defaultIconUrl = '/modules/@apostrophecms/attachment/img/missing-icon.svg';
        self.apos.util.warn('Template warning: Impossible to retrieve the attachment url since it is missing, a default icon has been set. Please fix this ASAP!');
        // Convert static asset path to full URL, which matters when static
        // assets are in uploadfs
        return self.apos.asset.url(defaultIconUrl);
      },
      // This method is available as a template helper: apos.attachment.url
      //
      // Given an attachment object,
      // return the URL. If options.size is set, return the URL for
      // that size (one-third, one-half, two-thirds, full). full is
      // "full width" (1140px), not the original. For the original,
      // pass `original`. If size is not specified, you will receive
      // the `full` size if an image, otherwise the original.
      //
      // If the "uploadfsPath" option is true, an
      // uploadfs path is returned instead of a URL.
      url(attachment, options) {
        options = options || {};
        if (!attachment) {
          return self.getMissingAttachmentUrl();
        }
        // file module supports these, optionally
        // (performance tradeoff). It's not enough to
        // pass the prettyUrl: true option to this method,
        // and that's not necessary. Setting it to false is
        // an internal option for determining the real URL
        // behind the pretty URL.
        if (attachment._prettyUrl && (options.prettyUrl !== false)) {
          return attachment._prettyUrl;
        }
        let path = '/attachments/' + attachment._id + '-' + attachment.name;
        if (!options.uploadfsPath) {
          path = self.uploadfs.getUrl() + path;
        }
        // Attachments can have "one true crop," or a crop can be passed with
        // the options. For convenience, be tolerant if options.crop is passed
        // but doesn't actually have valid cropping properties
        let c;
        if (options.crop !== false) {
          c = options.crop || attachment._crop || attachment.crop;
          if (c && c.width) {
            path += '.' + c.left + '.' + c.top + '.' + c.width + '.' + c.height;
          }
        }
        let effectiveSize;
        if (!self.isSized(attachment) || options.size === 'original') {
          effectiveSize = false;
        } else {
          effectiveSize = options.size || 'full';
        }
        if (effectiveSize) {
          path += '.' + effectiveSize;
        }
        return path + '.' + attachment.extension;
      },
      // This method is available as a template helper: apos.attachment.first
      //
      // Find the first attachment referenced within any object with
      // attachments as possible properties or sub-properties.
      //
      // For best performance be reasonably specific; don't pass an entire page
      // or piece object if you can pass page.thumbnail to avoid an exhaustive
      // search, especially if the page has many relationships.
      //
      // Returns the first attachment matching the criteria.
      //
      // For ease of use, a null or undefined `within` argument is accepted.
      //
      // Examples:
      //
      // 1. In the body please
      //
      // apos.attachment.first(page.body)
      //
      // 2. Must be a PDF
      //
      // apos.attachment.first(page.body, { extension: 'pdf' })
      //
      // 3. May be any office-oriented file type
      //
      // apos.attachment.first(page.body, { group: 'office' })
      //
      // apos.image.first is a convenience wrapper for fetching only images.
      //
      // OPTIONS:
      //
      // You may specify `extension`, `extensions` (an array of extensions)
      // or `group` to filter the results.
      first(within, options) {
        options = options ? _.clone(options) : {};
        options.limit = 1;
        return self.all(within, options)[0];
      },
      // This method is available as a template helper: apos.attachment.all
      //
      // Find all attachments referenced within an object, whether they are
      // properties or sub-properties (via relationships, etc).
      //
      // For best performance be reasonably specific; don't pass an entire page
      // or piece object if you can pass piece.thumbnail to avoid an exhaustive
      // search, especially if the piece has many relationships.
      //
      // Returns an array of attachments, or an empty array if none are found.
      //
      // When available, the `description`, `credit`, `alt` and `creditUrl`
      // properties of the containing piece are returned as `_description`,
      // `_credit`, `_alt` and `_creditUrl`.
      //
      // For ease of use, a null or undefined `within` argument is accepted,
      // resulting in an empty array.
      //
      // Examples:
      //
      // 1. In the body please
      //
      // apos.attachment.all(page.body)
      //
      // 2. Must be a PDF
      //
      // apos.attachment.all(page.body, { extension: 'pdf' })
      //
      // 3. May be any office-oriented file type
      //
      // apos.attachment.all(page.body, { group: 'office' })
      //
      // apos.image.all is a convenience wrapper for fetching only images.
      //
      // OPTIONS:
      //
      // You may specify `extension`, `extensions` (an array of extensions)
      // or `group` to filter the results.
      //
      // If `options.annotate` is true, a `._urls` property is added to all
      // image attachments wherever they are found in `within`, with
      // subproperties for each image size name, including `original`.
      // For non-images, a `._url` property is set.
      all(within, options) {
        options = options || {};
        function test(attachment) {
          if (!attachment || typeof attachment !== 'object') {
            return false;
          }
          if (attachment.type !== 'attachment') {
            return false;
          }
          if (options.extension) {
            if (self.resolveExtension(attachment.extension) !== options.extension) {
              return false;
            }
          }
          if (options.group) {
            if (attachment.group !== options.group) {
              return false;
            }
          }
          if (options.extensions) {
            if (
              !_.contains(options.extensions, self.resolveExtension(attachment.extension))
            ) {
              return false;
            }
          }
          return true;
        }
        const winners = [];
        if (!within) {
          return [];
        }
        self.apos.doc.walk(within, function (o, key, value, dotPath, ancestors) {
          if (test(value)) {
            if (o.credit) {
              value._credit = o.credit;
            }
            if (o.creditUrl) {
              value._creditUrl = o.creditUrl;
            }
            if (o.alt) {
              value._alt = o.alt;
            }

            value._isCroppable = self.isCroppable(value);

            o[key] = value;

            // If one of our ancestors has a relationship to the piece that
            // immediately contains us, provide that as the crop. This ensures
            // that cropping coordinates stored in an @apostrophecms/image
            // widget are passed through when we make a simple call to
            // apos.attachment.url with the returned object
            for (let i = ancestors.length - 1; i >= 0; i--) {
              const ancestor = ancestors[i];
              const ancestorFields = ancestor.attachment &&
                ancestor.attachment._id === value._id && ancestor._fields;

              if (ancestorFields) {
                value = _.clone(value);
                o.attachment = value;
                value._crop = ancestorFields.width ? _.pick(ancestorFields, 'width', 'height', 'top', 'left') : undefined;
                value._focalPoint = (typeof ancestorFields.x === 'number') ? _.pick(ancestorFields, 'x', 'y') : undefined;
                break;
              }
            }

            if (options.annotate) {
              // Add URLs
              value._urls = {};
              if (value._crop) {
                value._urls.uncropped = {};
              }
              if (value.group === 'images') {
                _.each(self.imageSizes, function (size) {
                  value._urls[size.name] = self.url(value, { size: size.name });
                  if (value._crop) {
                    value._urls.uncropped[size.name] = self.url(value, {
                      size: size.name,
                      crop: false
                    });
                  }
                });
                value._urls.original = self.url(value, { size: 'original' });
                if (value._crop) {
                  value._urls.uncropped.original = self.url(value, {
                    size: 'original',
                    crop: false
                  });
                }
              } else {
                value._url = self.url(value);
              }
            }

            winners.push(value);
          }
        });
        return winners;
      },
      // Iterates over all of the attachments that exist, processing
      // up to `limit` attachments at any given time.
      //
      // If only 2 arguments are given the limit defaults to 1.
      //
      // For use only in command line tasks, migrations and other batch
      // operations in which permissions are a complete nonissue. NEVER use on
      // the front end.
      //
      // This method will `await` when calling your `each` function,
      // which must return a promise (i.e. just use an `async` function).
      //
      // This method is designed to succeed even if `each` is a fairly slow
      // operation and there are many attachments. It does not rely on keeping
      // a single MongoDB cursor open for a long time.
      async each(criteria, limit, each) {
        if (!each) {
          each = limit;
          limit = 1;
        }
        // "Why do we fetch a bucket of attachments at a time?" File operations
        // can be very slow. This can lead to MongoDB cursor timeouts in
        // tasks like @apostrophecms/attachment:rescale. We need a robust
        // solution that does not require keeping a MongoDB cursor open too
        // long. -Tom
        const batchSize = 100;
        let lastId = '';
        while (true) {
          const docs = await self.db.find({
            ...(criteria || {}),
            _id: { $gt: lastId }
          }).limit(batchSize).sort({ _id: 1 }).toArray();
          if (!docs.length) {
            return;
          }
          await Promise.map(docs, each, { concurrency: limit });
          lastId = docs[docs.length - 1]._id;
          if (docs.length < batchSize) {
            // Don't waste an extra query
            break;
          }
        }
      },
      // Check whether a schema array (recursively through
      // array and object sub-schemas) contains at least one
      // field with `type: 'attachment'`.
      schemaHasAttachmentField(schema) {
        if (!Array.isArray(schema)) {
          return false;
        }
        for (const field of schema) {
          if (field.type === 'attachment') {
            return true;
          }
          if (
            (field.type === 'array' || field.type === 'object') &&
            field.schema
          ) {
            if (self.schemaHasAttachmentField(field.schema)) {
              return true;
            }
          }
        }
        return false;
      },

      // Return a Set of doc type names whose module schema
      // contains at least one `type: 'attachment'` field.
      // Result is lazy cached.
      hasAttachmentFields(type) {
        if (self.typesWithAttachmentFields.has(type)) {
          return self.typesWithAttachmentFields.get(type);
        }
        const module = self.apos.modules[type];
        if (!module?.schema) {
          self.typesWithAttachmentFields.set(type, false);
        } else {
          self.typesWithAttachmentFields.set(
            type,
            self.schemaHasAttachmentField(module.schema)
          );
        }
        return self.typesWithAttachmentFields.get(type);
      },

      // Check whether a relationship field's `withType`
      // (which may be a virtual type like
      // `@apostrophecms/any-page-type`) resolves to a type
      // that has attachment fields.
      relationshipHasAttachmentFields(withType) {
        if (self.hasAttachmentFields(withType)) {
          return true;
        }
        if (
          withType === '@apostrophecms/any-page-type' ||
          withType === '@apostrophecms/page'
        ) {
          const cacheKey = '@apostrophecms/any-page-type';
          if (!self.typesWithAttachmentFields.has(cacheKey)) {
            self.typesWithAttachmentFields.set(
              cacheKey,
              self.apos
                .instancesOf('@apostrophecms/page-type')
                .some(module => self.hasAttachmentFields(module.__meta.name))
            );
            self.typesWithAttachmentFields.set(
              '@apostrophecms/page',
              self.typesWithAttachmentFields.get(cacheKey)
            );
          }
          return self.typesWithAttachmentFields.get(cacheKey);
        }
        return false;
      },

      // Collect the full `_id` values (e.g. `abc:en:published`)
      // of docs that are referenced via relationship fields
      // from the given `doc` and belong to types whose schema
      // contains at least one `type: 'attachment'` field.
      //
      // These IDs can be matched against the `docIds` array
      // stored on each attachment record, allowing a "used"
      // scope that only includes attachments actively referenced
      // by published content.
      //
      // Returns an array of full `_id` strings. The method is
      // synchronous because it only inspects the in-memory
      // document data — no database queries are needed.
      collectUsedDocIds(req, doc) {
        const docIds = new Set();
        if (self.hasAttachmentFields(doc.type)) {
          docIds.add(doc.aposDocId);
        }
        const locale = req.locale || self.apos.i18n?.defaultLocale || 'en';
        const mode = req.mode || 'published';

        self.collectDocAttachmentRefIds(doc, docIds);

        // Convert aposDocId values to full _id format
        return [ ...docIds ].map(
          aposDocId => `${aposDocId}:${locale}:${mode}`
        );
      },

      // Given a single document (raw MongoDB data), walk its
      // schema recursively and collect `idsStorage` values from
      // relationship fields that point to types with attachment
      // fields.  Also handles the special case of rich-text
      // widget inline images (`imageIds`) and widgets whose own
      // schema contains a direct `type: 'attachment'` field.
      //
      // Found IDs are added to the `target` Set
      // (aposDocId values, locale-agnostic).
      collectDocAttachmentRefIds(doc, target) {
        const handlers = {
          relationship: (field, contextDoc) => {
            if (self.relationshipHasAttachmentFields(field.withType)) {
              for (const id of (contextDoc[field.idsStorage] || [])) {
                target.add(id);
              }
            }
          },
          // Rich-text widgets store inline image references in
          // `imageIds` outside of any schema relationship field.
          // Widgets whose own schema has a direct `type: 'attachment'`
          // field (e.g. a custom widget storing a file directly) also
          // need to include the parent doc's ID.
          widget: (field, widget) => {
            if (self.hasAttachmentFields(widget.type)) {
              target.add(doc.aposDocId);
            }
            if (
              widget.type === '@apostrophecms/rich-text' &&
              Array.isArray(widget.imageIds)
            ) {
              for (const id of widget.imageIds) {
                target.add(id);
              }
            }
          }
        };
        self.apos.doc.walkByMetaType(doc, handlers);
      },

      // Return metadata for attachments suitable for a static build.
      // Uses batch-of-100 pattern for memory efficiency (see self.each()).
      //
      // Options (all optional):
      //
      // `docIds`: array of full locale-qualified document `_id`s.
      //   When provided, only attachments whose `docIds` array
      //   intersects with the given list are returned ("used"
      //   scope).  When not provided (undefined), all non-archived
      //   attachments are returned ("all" scope).
      //
      // Archived attachments are always excluded.
      //
      // `sizes`: array of size names to include (e.g. `['full',
      //   'one-half']`).  When provided, only these sizes are
      //   emitted.  Ignored for non-sized attachments (SVG, office
      //   files) which always get a single entry with `path` only.
      //
      // `skipSizes`: array of size names to exclude.  Applied
      //   after `sizes` (or after the full size list when `sizes`
      //   is not given).  Common use: `['original']` to skip the
      //   potentially very large original upload.
      //
      // Cropped variants, when present on the attachment record,
      // are also subject to `sizes` and `skipSizes` filtering.
      //
      // Returns an array of objects:
      //
      // ```js
      // {
      //   _id: 'abc123',
      //   urls: [
      //     // Sized attachments include a `size` property:
      //     { size: 'full', path: '/attachments/abc-photo.full.jpg' },
      //     { size: 'one-half', path: '/attachments/abc-photo.one-half.jpg' },
      //     // crop variants (same sizes):
      //     { size: 'full', path: '/attachments/abc-photo.10.20.300.400.full.jpg' },
      //     ...
      //     // Non-sized attachments (SVG, office docs) have `path` only:
      //     { path: '/attachments/def-document.pdf' },
      //   ]
      // }
      // ```
      //
      // `path` is an uploadfs-relative path (no host prefix).
      async getStaticMetadata({
        docIds, sizes, skipSizes
      } = {}) {
        const criteria = {
          archived: { $ne: true }
        };
        if (Array.isArray(docIds)) {
          criteria.docIds = { $in: docIds };
        }

        const allSizeNames = self.imageSizes.map((s) => s.name)
          .concat([ 'original' ]);
        let effectiveSizes;
        if (Array.isArray(sizes) && sizes.length) {
          effectiveSizes = sizes.filter((s) => allSizeNames.includes(s));
        } else {
          effectiveSizes = [ ...allSizeNames ];
        }
        if (Array.isArray(skipSizes) && skipSizes.length) {
          effectiveSizes = effectiveSizes.filter(
            (s) => !skipSizes.includes(s)
          );
        }

        const results = [];

        await self.each(criteria, async (attachment) => {
          const urls = [];
          const isSized = self.isSized(attachment);

          if (isSized) {
            // Sized image: emit one entry per requested size
            for (const size of effectiveSizes) {
              urls.push({
                size,
                path: self.url(attachment, {
                  uploadfsPath: true,
                  crop: false,
                  size
                })
              });
            }
            // Crop variants — respect the same size constraints
            for (const crop of (attachment.crops || [])) {
              for (const size of effectiveSizes) {
                urls.push({
                  size,
                  path: self.url(attachment, {
                    uploadfsPath: true,
                    crop,
                    size
                  })
                });
              }
            }
          } else {
            // Non-sized (SVG, office docs): path only, no size property
            urls.push({
              path: self.url(attachment, { uploadfsPath: true })
            });
          }

          if (urls.length) {
            results.push({
              _id: attachment._id,
              urls
            });
          }
        });

        return results;
      },
      // Returns true if, based on the provided attachment object,
      // a valid focal point has been specified. Useful to avoid
      // the default of `background-position: center center` if
      // not desired.
      hasFocalPoint(attachment) {
        // No attachment object; tolerate for nunjucks friendliness
        if (!attachment) {
          return false;
        }
        // Specified directly on the attachment (it's not a relationship
        // situation)
        if (typeof attachment.x === 'number') {
          return true;
        }
        // Specified on a `_focalPoint` property hoisted via a join
        return attachment._focalPoint && typeof attachment._focalPoint.x === 'number';
      },
      // If a focal point is present on the attachment, convert it to
      // CSS syntax for `object-position`. No trailing `;` is returned.
      // The coordinates are in percentage terms.
      focalPointToObjectPosition(attachment) {
        if (!self.hasFocalPoint(attachment)) {
          return 'center center';
        }
        const point = self.getFocalPoint(attachment);
        return `${point.x}% ${point.y}%`;
      },
      // Returns the effective attachment width.
      getWidth(attachment) {
        return attachment._crop ? attachment._crop.width : attachment.width;
      },
      // Returns the effective attachment height.
      getHeight(attachment) {
        return attachment._crop ? attachment._crop.height : attachment.height;
      },
      // Returns an object with `x` and `y` properties containing the
      // focal point chosen by the user, as percentages. If there is no
      // focal point, null is returned.
      getFocalPoint(attachment) {
        if (!self.hasFocalPoint(attachment)) {
          return null;
        }
        const x = attachment._focalPoint ? attachment._focalPoint.x : attachment.x;
        const y = attachment._focalPoint ? attachment._focalPoint.y : attachment.y;
        return {
          x,
          y
        };
      },
      // Returns true if this type of attachment is croppable.
      // Available as a template helper.
      isCroppable(attachment) {
        return (attachment &&
          self.croppable[self.resolveExtension(attachment.extension)]) ||
          false;
      },
      // Returns true if this type of attachment is sized,
      // i.e. uploadfs produces versions of it for each configured
      // size, as it does with GIF, JPEG and PNG files.
      //
      // Accepts either an entire attachment object or an extension.
      isSized(attachment) {
        if ((typeof attachment) === 'object') {
          return self.sized[self.resolveExtension(attachment.extension)];
        } else {
          return self.sized[self.resolveExtension(attachment)];
        }
      },
      // Resolve a file extension such as jpeg to its canonical form (jpg).
      // If no extension map is configured for this extension, return it as-is.
      resolveExtension(extension) {
        const group = self.getFileGroup(extension);
        if (group) {
          return group.extensionMaps[extension] || extension;
        }
        return extension;
      },
      // When the last doc that contains an attachment goes to the
      // archive, its permissions should change to reflect that so
      // it is no longer web-accessible to those who know the URL.
      //
      // If an attachment has no more archived *or* live docs associated
      // with it, truly delete the attachment.
      //
      // This method is invoked after any doc is inserted, updated, archived
      // or restored.
      //
      // If a document is truly deleted, call with the `{ deleted: true}`
      // option.
      async updateDocReferences(doc, options = {
        deleted: false
      }) {
        const attachments = self.all(self.apos.util.clonePermanent(doc));
        const ids = _.uniq(_.map(attachments, '_id'));
        // Build an array of mongo commands to run. Each
        // entry in the array is a 2-element array. Element 0
        // is the criteria, element 1 is the command
        const commands = [];
        if (options.deleted) {
          commands.push([
            { _id: { $in: ids } },
            {
              $pull: {
                docIds: doc._id,
                archivedDocIds: doc._id
              }
            }
          ]);
        } else if (!doc.archived) {
          commands.push([
            { _id: { $in: ids } },
            { $addToSet: { docIds: doc._id } }
          ], [
            { _id: { $in: ids } },
            { $pull: { archivedDocIds: doc._id } }
          ]);
        } else {
          commands.push([
            { _id: { $in: ids } },
            { $addToSet: { archivedDocIds: doc._id } }
          ], [
            { _id: { $in: ids } },
            { $pull: { docIds: doc._id } }
          ]);
        }
        commands.push([
          {
            $or: [
              { archivedDocIds: { $in: [ doc._id ] } },
              { docIds: { $in: [ doc._id ] } }
            ],
            _id: { $nin: ids }
          },
          {
            $pull: {
              archivedDocIds: doc._id,
              docIds: doc._id
            }
          }
        ], [
          { _id: { $in: ids } },
          { $set: { utilized: true } }
        ]);
        for (const command of commands) {
          await self.db.updateMany(command[0], command[1]);
        }
        await self.alterAttachments();
      },
      // Enable/disable access in uploadfs to all attachments
      // based on whether the documents containing them
      // are in the archive or not. Specifically, if an attachment
      // has been utilized at least once but no longer has
      // any entries in `docIds` and `archived` is not yet true,
      // it becomes web-inaccessible, `utilized` is set to false
      // and `archived` is set to true. Similarly, if an attachment
      // has entries in `docIds` but `archived` is true,
      // it becomes web-accessible and archived becomes false.
      //
      // This method is invoked at the end of `updateDocReferences`
      // and also at the end of the migration that adds `docIds`
      // to legacy sites. You should not need to invoke it yourself.
      //
      // This method also handles actually deleting attachments
      // if they have been utilized but are no longer associated
      // with any document, not even in the archive, as will occur
      // if the document is truly deleted.
      async alterAttachments() {
        await hide();
        await show();
        await _delete();
        async function hide() {
          const attachments = await self.db.find({
            utilized: true,
            'docIds.0': { $exists: 0 },
            archived: { $ne: true }
          }).toArray();
          for (const attachment of attachments) {
            await alterOne(attachment, 'disable');
          }
        }
        async function show() {
          const attachments = await self.db.find({
            utilized: true,
            'docIds.0': { $exists: 1 },
            archived: { $ne: false }
          }).toArray();
          for (const attachment of attachments) {
            await alterOne(attachment, 'enable');
          }
        }
        async function _delete() {
          const attachments = await self.db.find({
            utilized: true,
            'docIds.0': { $exists: 0 },
            'archivedDocIds.0': { $exists: 0 }
          }).toArray();
          for (const attachment of attachments) {
            await alterOne(attachment, 'remove');
          }
        }
        async function alterOne(attachment, action) {
          await self.alterAttachment(attachment, action);
          if (action === 'remove') {
            await self.db.removeOne({ _id: attachment._id });
          } else {
            await self.db.updateOne({
              _id: attachment._id
            }, {
              $set: {
                archived: (action === 'disable')
              }
            });
          }
        }
      },
      // Enable access, disable access, or truly remove the given attachment
      // via