@endo/compartment-mapper/src/archive-lite.js

The compartment mapper assembles Node applications in a sandbox

/**
 * Provides functions to create an archive (zip file with a
 * compartment-map.json) from a partially completed compartment map (it must
 * mention all packages/compartments as well as inter-compartment references
 * but does not contain an entry for every module reachable from its entry
 * module) and the means to read them from the containing file system.
 *
 * These functions do not have a bias for any particular mapping, so you will
 * need to use `mapNodeModules` from `@endo/compartment-map/node-modules.js` or
 * a similar device to construct one.
 *
 * The default `parserForLanguage` mapping is empty.
 * You will need to provide the `defaultParserForLanguage` from
 * `@endo/compartment-mapper/import-parsers.js` or
 * `@endo/compartment-mapper/archive-parsers.js`.
 *
 * If you use `@endo/compartment-mapper/archive-parsers.js`, the archive
 * will contain pre-compiled ESM and CJS modules wrapped in a JSON
 * envelope, suitable for use with the SES shim in any environment
 * including a web page, without a client-side dependency on Babel.
 *
 * If you use `@endo/compartment-mapper/import-parsers.js`, the archive
 * will contain original sources, so to import the archive with
 * `src/import-archive-lite.js`, you will need to provide the archive parsers
 * and entrain a runtime dependency on Babel.
 *
 * In fruition of https://github.com/endojs/endo/issues/400, we will be able to
 * use original source archives on XS and Node.js, but not on the web until the
 * web platform makes further progress on virtual module loaers.
 *
 * @module
 */

/* eslint no-shadow: 0 */

/**
 * @import {
 *   ArchiveLiteOptions,
 *   ArchiveResult,
 *   ArchiveWriter,
 *   CaptureSourceLocationHook,
 *   CompartmentsRenameFn,
 *   FileUrlString,
 *   HashPowers,
 *   PackageCompartmentMapDescriptor,
 *   ReadFn,
 *   ReadPowers,
 *   Sources,
 *   WriteFn,
 * } from './types.js'
 */

import { writeZip } from '@endo/zip';
import { resolve } from './node-module-specifier.js';
import { link } from './link.js';
import {
  exitModuleImportHookMaker,
  makeImportHookMaker,
} from './import-hook.js';
import { unpackReadPowers } from './powers.js';
import { detectAttenuators } from './policy.js';
import { digestCompartmentMap } from './digest.js';
import { stringCompare } from './compartment-map.js';
import { ATTENUATORS_COMPARTMENT } from './policy-format.js';

const textEncoder = new TextEncoder();

const { assign, create, freeze, keys } = Object;

/**
 * @param {ArchiveWriter} archive
 * @param {Sources} sources
 */
const addSourcesToArchive = async (archive, sources) => {
  await null;
  for (const compartment of keys(sources).sort()) {
    const modules = sources[compartment];
    for (const specifier of keys(modules).sort()) {
      if ('location' in modules[specifier]) {
        const { bytes, location } = modules[specifier];
        const path = `${compartment}/${location}`;
        if (bytes !== undefined) {
          // eslint-disable-next-line no-await-in-loop
          await archive.write(path, bytes);
        }
      }
    }
  }
};

/**
 * @param {Sources} sources
 * @param {CaptureSourceLocationHook} captureSourceLocation
 */
const captureSourceLocations = async (sources, captureSourceLocation) => {
  for (const compartmentName of keys(sources).sort()) {
    const modules = sources[compartmentName];
    for (const moduleSpecifier of keys(modules).sort()) {
      if ('sourceLocation' in modules[moduleSpecifier]) {
        const { sourceLocation } = modules[moduleSpecifier];
        captureSourceLocation(compartmentName, moduleSpecifier, sourceLocation);
      }
    }
  }
};

/**
 * @param {PackageCompartmentMapDescriptor} compartmentMap
 * @param {Sources} sources
 * @returns {ArchiveResult}
 */
export const makeArchiveCompartmentMap = (compartmentMap, sources) => {
  /** @type {CompartmentsRenameFn<FileUrlString, string>} */
  const renameCompartments = compartments => {
    /** @type {Record<FileUrlString, string>} */
    const compartmentRenames = create(null);

    /**
     * Get the new name of format `packageName-v${version}` compartments (except
     * for the attenuators compartment)
     * @param {string} name
     * @param {string} version
     * @returns {string}
     */
    const getCompartmentName = (name, version) => {
      const compartment = compartments[name];
      return ATTENUATORS_COMPARTMENT === compartment.name
        ? compartment.name
        : `${compartment.name}-v${version}`;
    };

    // The sort below combines two comparators to avoid depending on sort
    // stability, which became standard as recently as 2019.
    // If that date seems quaint, please accept my regards from the distant past.
    // We are very proud of you.
    const compartmentsByLabel =
      /** @type {Array<{name: FileUrlString, packageName: string, compartmentName: string}>} */ (
        Object.entries(compartments)
          .map(([name, compartment]) => ({
            name,
            packageName: compartments[name].name,
            compartmentName: getCompartmentName(name, compartment.version),
          }))
          .sort((a, b) => stringCompare(a.compartmentName, b.compartmentName))
      );

    /** @type {string|undefined} */
    let prev;
    let index = 1;
    for (const { name, packageName, compartmentName } of compartmentsByLabel) {
      if (packageName === prev) {
        compartmentRenames[name] = `${compartmentName}-n${index}`; // Added numeric suffix for duplicates
        index += 1;
      } else {
        compartmentRenames[name] = compartmentName;
        prev = packageName;
        index = 1;
      }
    }
    return compartmentRenames;
  };

  const {
    compartmentMap: archiveCompartmentMap,
    sources: archiveSources,
    oldToNewCompartmentNames,
    newToOldCompartmentNames,
    compartmentRenames,
  } = digestCompartmentMap(compartmentMap, sources, { renameCompartments });

  return {
    archiveCompartmentMap,
    archiveSources,
    oldToNewCompartmentNames,
    newToOldCompartmentNames,
    compartmentRenames,
  };
};

const noop = () => {};

/**
 * @param {ReadFn | ReadPowers} powers
 * @param {PackageCompartmentMapDescriptor} compartmentMap
 * @param {ArchiveLiteOptions} [options]
 * @returns {Promise<{sources: Sources, compartmentMapBytes: Uint8Array, sha512?: string}>}
 */
const digestFromMap = async (powers, compartmentMap, options = {}) => {
  const {
    moduleTransforms,
    modules: exitModules = {},
    captureSourceLocation = undefined,
    searchSuffixes = undefined,
    importHook: exitModuleImportHook = undefined,
    policy = undefined,
    sourceMapHook = undefined,
    parserForLanguage: parserForLanguageOption = {},
    log: _log = noop,
  } = options;

  const parserForLanguage = freeze(
    assign(create(null), parserForLanguageOption),
  );

  const { read, computeSha512 } = unpackReadPowers(powers);

  const {
    compartments,
    entry: { module: entryModuleSpecifier, compartment: entryCompartmentName },
  } = compartmentMap;

  /** @type {Sources} */
  const sources = Object.create(null);

  const consolidatedExitModuleImportHook = exitModuleImportHookMaker({
    modules: exitModules,
    exitModuleImportHook,
    entryCompartmentName,
  });

  const makeImportHook = makeImportHookMaker(read, entryCompartmentName, {
    sources,
    compartmentDescriptors: compartments,
    archiveOnly: true,
    computeSha512,
    searchSuffixes,
    entryCompartmentName,
    entryModuleSpecifier,
    importHook: consolidatedExitModuleImportHook,
    sourceMapHook,
  });

  // Induce importHook to record all the necessary modules to import the given module specifier.
  const { compartment, attenuatorsCompartment } = link(compartmentMap, {
    resolve,
    makeImportHook,
    moduleTransforms,
    parserForLanguage,
    archiveOnly: true,
  });
  await compartment.load(entryModuleSpecifier);
  if (policy) {
    // retain all attenuators.
    await Promise.all(
      detectAttenuators(policy).map(attenuatorSpecifier =>
        attenuatorsCompartment.load(attenuatorSpecifier),
      ),
    );
  }

  const { archiveCompartmentMap, archiveSources } = makeArchiveCompartmentMap(
    compartmentMap,
    sources,
  );

  const archiveCompartmentMapText = JSON.stringify(
    archiveCompartmentMap,
    (key, value) => (key.startsWith('_') ? undefined : value),
    2,
  );
  const archiveCompartmentMapBytes = textEncoder.encode(
    archiveCompartmentMapText,
  );

  if (captureSourceLocation !== undefined) {
    captureSourceLocations(archiveSources, captureSourceLocation);
  }

  let archiveSha512;
  if (computeSha512 !== undefined) {
    archiveSha512 = computeSha512(archiveCompartmentMapBytes);
  }

  return {
    compartmentMapBytes: archiveCompartmentMapBytes,
    sources: archiveSources,
    ...(archiveSha512 !== undefined && { sha512: archiveSha512 }),
  };
};

/**
 * @param {ReadFn | ReadPowers} powers
 * @param {PackageCompartmentMapDescriptor} compartmentMap
 * @param {ArchiveLiteOptions} [options]
 * @returns {Promise<{bytes: Uint8Array, sha512?: string}>}
 */
export const makeAndHashArchiveFromMap = async (
  powers,
  compartmentMap,
  options,
) => {
  const { compartmentMapBytes, sources, sha512 } = await digestFromMap(
    powers,
    compartmentMap,
    options,
  );

  const archive = writeZip();
  await archive.write('compartment-map.json', compartmentMapBytes);
  await addSourcesToArchive(archive, sources);
  const bytes = await archive.snapshot();

  return { bytes, ...(sha512 !== undefined && { sha512 }) };
};

/**
 * @param {ReadFn | ReadPowers} powers
 * @param {PackageCompartmentMapDescriptor} compartmentMap
 * @param {ArchiveLiteOptions} [options]
 * @returns {Promise<Uint8Array>}
 */
export const makeArchiveFromMap = async (powers, compartmentMap, options) => {
  const { bytes } = await makeAndHashArchiveFromMap(
    powers,
    compartmentMap,
    options,
  );
  return bytes;
};

/**
 * @param {ReadFn | ReadPowers} powers
 * @param {PackageCompartmentMapDescriptor} compartmentMap
 * @param {ArchiveLiteOptions} [options]
 * @returns {Promise<Uint8Array>}
 */
export const mapFromMap = async (powers, compartmentMap, options) => {
  const { compartmentMapBytes } = await digestFromMap(
    powers,
    compartmentMap,
    options,
  );
  return compartmentMapBytes;
};

/**
 * @param {HashPowers} powers
 * @param {PackageCompartmentMapDescriptor} compartmentMap
 * @param {ArchiveLiteOptions} [options]
 * @returns {Promise<string>}
 */
export const hashFromMap = async (powers, compartmentMap, options) => {
  const { compartmentMapBytes } = await digestFromMap(
    powers,
    compartmentMap,
    options,
  );
  const { computeSha512 } = powers;
  return computeSha512(compartmentMapBytes);
};

/**
 * @param {WriteFn} write
 * @param {ReadFn | ReadPowers} readPowers
 * @param {string} archiveLocation
 * @param {PackageCompartmentMapDescriptor} compartmentMap
 * @param {ArchiveLiteOptions} [options]
 */
export const writeArchiveFromMap = async (
  write,
  readPowers,
  archiveLocation,
  compartmentMap,
  options,
) => {
  const archiveBytes = await makeArchiveFromMap(
    readPowers,
    compartmentMap,
    options,
  );
  await write(archiveLocation, archiveBytes);
};