@mui/internal-docs-infra/pipeline/loadIsomorphicCodeVariant/loadIsomorphicCodeVariant.mjs

MUI Infra - internal documentation creation tools.

import * as path from 'path-module';
import { compressHastAsync } from "../hastUtils/index.mjs";
import { buildRootFallback, buildCriticalFallback, fallbackToText } from "../../CodeHighlighter/fallbackFormat.mjs";
import { getInitialVisibleFrames } from "../parseSource/frameVisibility.mjs";
import { transformSource } from "./transformSource.mjs";
import { diffHast } from "./diffHast.mjs";
import { isFrameSpan } from "../parseSource/isFrameSpan.mjs";
import { getFileNameFromUrl, getLanguageFromExtension, normalizeLanguage } from "../loaderUtils/index.mjs";
import { mergeExternals } from "../loaderUtils/mergeExternals.mjs";
import { applyUrlPrefixToVariant } from "../loaderUtils/applyUrlPrefix.mjs";
import { performanceMeasure } from "../loadPrecomputedCodeHighlighter/performanceLogger.mjs";
import { starryNightGutter } from "../parseSource/addLineGutters.mjs";
import { applyEnhancers } from "./runSourceEnhancers.mjs";
import { embedTransformsInRoot, splitTransformsForEmbed } from "./embedTransforms.mjs";

/**
 * Check if a path is absolute (either filesystem absolute or URL)
 */
function isAbsolutePath(filePath) {
  return path.isAbsolute(filePath) || filePath.includes('://');
}

/**
 * Removes the per-frame `data.fallback` text from each `span.frame` before the
 * hast is serialized. The variant-level root fallback already carries this text
 * (and `redistributeRootFallback` puts it back on decode), so keeping it on the
 * stored tree would duplicate it in every payload.
 */
function stripFrameFallbacks(root) {
  for (const child of root.children) {
    if (child.type !== 'element' || child.tagName !== 'span' || !child.data) {
      continue;
    }
    if (isFrameSpan(child) && 'fallback' in child.data) {
      delete child.data.fallback;
    }
  }
}

/**
 * Generate a conflict-free filename for globalsCode files.
 * Strategy:
 * 1. Try original filename
 * 2. If conflict, try "global_" prefix
 * 3. If still conflict, add numbers: "global_filename_1.ext", "global_filename_2.ext", etc.
 */
function generateConflictFreeFilename(originalFilename, existingFiles) {
  // First try the original filename
  if (!existingFiles.has(originalFilename)) {
    return originalFilename;
  }

  // Try with global_ prefix
  const globalFilename = `global_${originalFilename}`;
  if (!existingFiles.has(globalFilename)) {
    return globalFilename;
  }

  // Use path.parse to cleanly split filename into name and extension
  const parsed = path.parse(originalFilename);
  const nameWithoutExt = parsed.name;
  const extension = parsed.ext;

  // Add numbers until we find a free name, preserving extension
  let counter = 1;
  let candidateName;
  do {
    candidateName = `global_${nameWithoutExt}_${counter}${extension}`;
    counter += 1;
  } while (existingFiles.has(candidateName));
  return candidateName;
}

// Helper function to check if we're in production
function isProduction() {
  return typeof process !== 'undefined' && process.env.NODE_ENV === 'production';
}

/**
 * Computes a URL-style relative path from `sourceFileUrl` (the URL of a file)
 * to `targetFileUrl` such that `new URL(result, sourceFileUrl).href === targetFileUrl`.
 *
 * Returns `undefined` if the URLs differ in scheme or origin, in which case
 * a relative reference cannot be produced.
 */
function computeRelativeUrl(sourceFileUrl, targetFileUrl) {
  let source;
  let target;
  try {
    source = new URL(sourceFileUrl);
    target = new URL(targetFileUrl);
  } catch {
    return undefined;
  }
  if (source.protocol !== target.protocol || source.host !== target.host) {
    return undefined;
  }
  const sourceSegments = source.pathname.split('/');
  const targetSegments = target.pathname.split('/');
  // Drop the source file segment so we walk from its containing directory.
  sourceSegments.pop();
  let commonLength = 0;
  while (commonLength < sourceSegments.length && commonLength < targetSegments.length - 1 && sourceSegments[commonLength] === targetSegments[commonLength]) {
    commonLength += 1;
  }
  const upSegments = sourceSegments.length - commonLength;
  const downSegments = targetSegments.slice(commonLength);
  const prefix = upSegments === 0 ? './' : '../'.repeat(upSegments);
  return `${prefix}${downSegments.join('/')}`;
}

// Helper function to convert a nested key based on the directory of the source file key.
//
// Note: this operates on POSIX-style paths via `path-module`, not URLs. It does
// not decode percent-encoded segments, so callers should pass simple relative
// paths (e.g. `'../styles.css'`). Percent-encoded segments would be treated as
// opaque path components.
function convertKeyBasedOnDirectory(nestedKey, sourceFileKey) {
  // If it's an absolute path (starts with / or contains ://), keep as-is
  if (isAbsolutePath(nestedKey)) {
    return nestedKey;
  }

  // Treat bare filenames as relative to current directory (same as ./filename)
  let processedNestedKey = nestedKey;
  if (!nestedKey.startsWith('.')) {
    processedNestedKey = `./${nestedKey}`;
  }

  // Get the directory of the source file
  const sourceDir = path.dirname(sourceFileKey);

  // If sourceDir is '.' (current directory), just return the processed nested key
  // This avoids path.resolve which can produce absolute paths on Windows
  if (sourceDir === '.') {
    // Remove leading './' if present for consistency
    return processedNestedKey.startsWith('./') ? processedNestedKey.slice(2) : processedNestedKey;
  }

  // Use path.join instead of path.resolve to avoid producing absolute paths
  // path.join keeps paths relative, while path.resolve can make them absolute
  const joinedPath = path.join(sourceDir, processedNestedKey);

  // Normalize the path to clean up any ../ or ./ segments
  const normalizedPath = path.normalize(joinedPath);

  // Ensure we return a clean relative path (remove leading './' if present after normalization)
  if (normalizedPath.startsWith('./')) {
    return normalizedPath.slice(2);
  }
  return normalizedPath === '.' ? '' : normalizedPath;
}

/**
 * Normalize a relative path key by removing unnecessary ./ prefix and cleaning up the path
 */
function normalizePathKey(key) {
  // Handle edge cases
  if (key === '.' || key === '') {
    return '';
  }

  // Use path.normalize to clean up the path, then remove leading './' if present
  const normalized = path.normalize(key);

  // Convert './filename' to 'filename' using path.relative
  if (normalized.startsWith('./')) {
    return path.relative('.', normalized);
  }
  return normalized === '.' ? '' : normalized;
}

/**
 * Loads and processes extra files recursively with support for relative paths
 * and circular dependency detection. Uses Promise.all for parallel loading.
 */

async function loadSingleFile(variantName, fileName, source, url, loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, transforms, options = {}, allFilesListed = false, knownExtraFiles = new Set(), language, variantComments) {
  const {
    disableTransforms = false,
    disableParsing = false,
    framePlainFallback = false
  } = options;
  let finalSource = source;
  let finalFallback;
  let finalFallbackCritical;
  let finalTotalLines;
  let finalFocusedLines;
  let finalCollapsible;
  let extraFilesFromSource;
  let extraDependenciesFromSource;
  let externalsFromSource;
  let commentsFromSource = variantComments;
  const functionName = 'Load Variant File';
  let currentMark = performanceMeasure(undefined, {
    mark: 'Start',
    measure: 'Start'
  }, [functionName, url || fileName], true);

  // Load source if not provided
  if (!finalSource) {
    if (!loadSource) {
      throw new Error('"loadSource" function is required when source is not provided');
    }
    if (!url) {
      throw new Error('URL is required when loading source');
    }
    try {
      // Check cache first to avoid duplicate loadSource calls
      let loadPromise = loadSourceCache.get(url);
      if (!loadPromise) {
        loadPromise = loadSource(url);
        loadSourceCache.set(url, loadPromise);
      }
      const loadResult = await loadPromise;
      finalSource = loadResult.source;
      extraFilesFromSource = loadResult.extraFiles;
      extraDependenciesFromSource = loadResult.extraDependencies;
      externalsFromSource = loadResult.externals;
      commentsFromSource = loadResult.comments;
      currentMark = performanceMeasure(currentMark, {
        mark: 'Loaded File',
        measure: 'File Loading'
      }, [functionName, url]);

      // Validate that extraFiles from loadSource contain only absolute URLs as values
      if (extraFilesFromSource) {
        for (const [extraFileName, fileData] of Object.entries(extraFilesFromSource)) {
          // Validate that keys are relative paths (not absolute)
          if (isAbsolutePath(extraFileName)) {
            throw new Error(`Invalid extraFiles from loadSource: key "${extraFileName}" appears to be an absolute path. ` + `extraFiles keys should be relative paths from the current file.`);
          }

          // Validate that values are absolute URLs (not relative paths)
          if (typeof fileData === 'string' && fileData.startsWith('.')) {
            throw new Error(`Invalid extraFiles from loadSource: "${extraFileName}" has relative path "${fileData}". ` + `All extraFiles values must be absolute URLs.`);
          }
        }
      }

      // Validate that extraDependencies from loadSource contain only absolute URLs
      if (extraDependenciesFromSource) {
        for (const dependency of extraDependenciesFromSource) {
          if (dependency.startsWith('.')) {
            throw new Error(`Invalid extraDependencies from loadSource: "${dependency}" is a relative path. ` + `All extraDependencies must be absolute URLs.`);
          }
          if (dependency === url) {
            throw new Error(`Invalid extraDependencies from loadSource: "${dependency}" is the same as the input URL. ` + `extraDependencies should not include the file being loaded.`);
          }
        }
      }

      // Check for new files when allFilesListed is enabled
      if (allFilesListed && (extraFilesFromSource || extraDependenciesFromSource)) {
        const newFiles = [];
        if (extraFilesFromSource) {
          // Check if any extraFiles keys are not in the known set
          for (const extraFileKey of Object.keys(extraFilesFromSource)) {
            if (!knownExtraFiles.has(extraFileKey)) {
              newFiles.push(extraFileKey);
            }
          }
        }
        if (newFiles.length > 0) {
          const message = `Unexpected files discovered via loadSource when allFilesListed=true (variant: ${variantName}, file: ${fileName}). ` + `New files: ${newFiles.join(', ')}. ` + `Please update the loadVariantMeta function to provide the complete list of files upfront.`;
          if (isProduction()) {
            console.warn(message);
          } else {
            throw new Error(message);
          }
        }
      }
    } catch (error) {
      // Re-throw validation errors without wrapping them
      if (error instanceof Error && (error.message.startsWith('Invalid extraFiles from loadSource:') || error.message.startsWith('Invalid extraDependencies from loadSource:') || error.message.startsWith('Unexpected files discovered via loadSource when allFilesListed=true'))) {
        throw error;
      }
      throw new Error(`Failed to load source code (variant: ${variantName}, file: ${fileName}, url: ${url}): ${error instanceof Error ? error.message : String(error)}`, {
        cause: error
      });
    }
  }

  // Apply source transformers if no transforms exist and transforms are not disabled
  let finalTransforms = transforms;
  if (sourceTransformers && !finalTransforms && !disableTransforms && finalSource) {
    finalTransforms = await transformSource(finalSource, normalizePathKey(fileName), sourceTransformers, commentsFromSource);
    currentMark = performanceMeasure(currentMark, {
      mark: 'Transformed File',
      measure: 'File Transforming'
    }, [functionName, url || fileName]);
  }

  // Parse source if it's a string and parsing is not disabled
  if (typeof finalSource === 'string' && !disableParsing) {
    if (!sourceParser) {
      throw new Error('"sourceParser" function is required when source is a string and parsing is not disabled');
    }
    try {
      const sourceString = finalSource;
      const parseSource = await sourceParser;
      let parsedSource = parseSource(finalSource, fileName, language);
      currentMark = performanceMeasure(currentMark, {
        mark: 'Parsed File',
        measure: 'File Parsing'
      }, [functionName, url || fileName]);

      // `commentsFromSource` is already 1-indexed (both `Code` comments and
      // `parseImportsAndComments`/`loadSource` output use the 1-indexed convention).
      // Aliased so the diff path (below) can reuse it when wrapping `parseSource` for
      // transformed sources — the comments live in the code itself and don't shift for
      // transforms that only blank lines.
      const oneIndexedComments = commentsFromSource;

      // Apply source enhancers if provided (run sequentially as a pipeline).
      // Enhancers with a stable `enhancerName` are recorded on the HAST root
      // and skipped if they have already been applied (e.g. by a previous
      // server-side pass).
      if (sourceEnhancers && sourceEnhancers.length > 0) {
        parsedSource = await applyEnhancers(parsedSource, oneIndexedComments, fileName, sourceEnhancers);
        currentMark = performanceMeasure(currentMark, {
          mark: 'Enhanced File',
          measure: 'File Enhancing'
        }, [functionName, url || fileName]);
      }
      finalSource = parsedSource;
      if (finalTransforms && !disableTransforms) {
        // Wrap parseSource so transformed sources receive the same source
        // enhancers as the original. The frame structure produced by
        // enhanceCodeEmphasis depends on `@focus`/`@padding-*` comments;
        // running enhancers on both sides keeps the per-frame children
        // layout aligned for a positional diff. Without this the diff
        // balloons at the frame level (source has N frames, transform
        // has 1) and jsondiffpatch deletes the extras.
        const parseSourceForDiff = sourceEnhancers && sourceEnhancers.length > 0 ? async (transformedSourceString, transformedFileName, _language, transformedComments) => {
          const transformedTree = await parseSource(transformedSourceString, transformedFileName);
          // Prefer the transform-provided comment map (already
          // 1-indexed against the transformed source) so enhancers
          // emit the same frame structure on both sides. Falling
          // back to the source's `oneIndexedComments` is safe for
          // transforms that only blank lines in place, where the
          // comment positions don't shift.
          return applyEnhancers(transformedTree, transformedComments ?? oneIndexedComments, transformedFileName, sourceEnhancers);
        } : parseSource;
        finalTransforms = await diffHast(sourceString, finalSource, normalizePathKey(fileName), finalTransforms, parseSourceForDiff);
        currentMark = performanceMeasure(currentMark, {
          mark: 'Transform Parsed File',
          measure: 'Parsed File Transforming'
        }, [functionName, url || fileName]);
      }

      // When the source is about to be serialized (compressed or stringified to
      // JSON), embed the transform deltas inside the hast root's `data` field
      // so they ride along inside the compressed payload — DEFLATE then
      // shares the dictionary across the tree and the deltas, and the deltas
      // never appear as plain JSON in the rendered HTML / module graph.
      // The variant-level `finalTransforms` becomes a manifest (no `delta`).
      if (finalTransforms && (options.output === 'hastCompressed' || options.output === 'hastJson') && finalSource && typeof finalSource === 'object' && !('hastJson' in finalSource) && !('hastCompressed' in finalSource)) {
        const root = finalSource;
        const split = splitTransformsForEmbed(finalTransforms);
        if (split) {
          embedTransformsInRoot(root, split.embedded);
          finalTransforms = split.manifest;
        } else {
          // Every entry was empty; drop transforms entirely so we don't emit
          // an empty manifest.
          finalTransforms = undefined;
        }
      }

      // Derive a variant-level root fallback from the per-frame `data.fallback`
      // text before any serialization. This fallback is rendered by a
      // `ContentLoading` component before the hast is decoded, and its text
      // doubles as the DEFLATE dictionary so the compressed payload can be
      // decompressed on the client once the fallback travels over via context.
      if (finalSource && typeof finalSource === 'object' && !('hastJson' in finalSource) && !('hastCompressed' in finalSource)) {
        finalFallback = buildRootFallback(finalSource);
        // Sparse highlighted-visible companion (see `VariantCode.fallbackCritical`),
        // computed here while the source is still a live `HastRoot` (no
        // decompression) and BEFORE `stripFrameFallbacks` removes the per-frame
        // text it reuses. `false` builds the `collapseToEmpty: false` form (the only
        // one carrying highlighting); the boundary skips promoting it under
        // `collapseToEmpty`. Empty (no visible frames) → omit it entirely.
        const critical = buildCriticalFallback(finalSource, getInitialVisibleFrames(finalSource, false));
        finalFallbackCritical = Object.keys(critical).length > 0 ? critical : undefined;

        // Hoist the window counts off `root.data` while the source is still a live
        // `HastRoot`. They then ride on the variant (see the return below) so every
        // downstream reader (`prepareInitialSource`, `getVariantFileLineCounts`,
        // layout-shift classification) gets `totalLines`/`focusedLines`/`collapsible`
        // WITHOUT decompressing the payload — the compact fallback and the compressed
        // source both drop `root.data`, so without this the only way to recover the
        // counts is to decode the hast (the first-render decompression we want to avoid).
        const rootData = finalSource.data;
        if (rootData?.totalLines !== undefined) {
          const total = Number(rootData.totalLines);
          if (Number.isFinite(total) && total >= 0) {
            finalTotalLines = total;
            const focused = Number(rootData.focusedLines);
            finalFocusedLines = Number.isFinite(focused) && focused >= 0 ? focused : total;
            finalCollapsible = rootData.collapsible === true;
          }
        }
      }
      if (options.output === 'hastCompressed' && process.env.NODE_ENV === 'production') {
        if (finalFallback) {
          stripFrameFallbacks(finalSource);
        }
        const json = JSON.stringify(finalSource);
        // Use the fallback text as a DEFLATE dictionary for better compression.
        // The same dictionary is rebuilt on decode from the variant `fallback`.
        const dictionary = finalFallback ? fallbackToText(finalFallback) : undefined;
        finalSource = {
          hastCompressed: await compressHastAsync(json, dictionary)
        };
        currentMark = performanceMeasure(currentMark, {
          mark: 'Compressed File',
          measure: 'File Compression'
        }, [functionName, url || fileName]);
      } else if (options.output === 'hastJson' || options.output === 'hastCompressed') {
        // in development, we skip compression but still convert to JSON
        if (finalFallback) {
          stripFrameFallbacks(finalSource);
        }
        finalSource = {
          hastJson: JSON.stringify(finalSource)
        };
        performanceMeasure(currentMark, {
          mark: 'JSON Stringified File',
          measure: 'File Stringification'
        }, [functionName, url || fileName]);
      }
    } catch (error) {
      throw new Error(`Failed to parse source code (variant: ${variantName}, file: ${fileName}, url: ${url}): ${error instanceof Error ? error.message : ''}`);
    }
  }

  // Parsing disabled (deferred highlight) but a framed fallback was requested: keep
  // `finalSource` a plain string so the client still knows it needs syntax
  // highlighting — a HAST source reads as "already loaded" (see `isSourceLoaded`) and
  // would never re-highlight. But the loading fallback MUST be framed — rendered code
  // needs frames — so build a line-guttered plain-text HAST (no syntax engine: the
  // light `starryNightGutter`), run the same enhancers (focus window / truncation),
  // and derive the root fallback from it. The string source is returned unchanged;
  // only `fallback` is produced. Gated by `framePlainFallback` so lazy variant loads
  // (which never paint a fallback) keep skipping the work.
  if (framePlainFallback && typeof finalSource === 'string' && !finalFallback) {
    const plainRoot = {
      type: 'root',
      children: [{
        type: 'text',
        value: finalSource
      }]
    };
    starryNightGutter(plainRoot, finalSource.split(/\r?\n|\r/));
    let framedRoot = plainRoot;
    if (sourceEnhancers && sourceEnhancers.length > 0) {
      framedRoot = await applyEnhancers(framedRoot, commentsFromSource, fileName, sourceEnhancers);
    }
    finalFallback = buildRootFallback(framedRoot);
    // Surface the enhancer's window counts (the compact fallback drops `root.data`,
    // and a plain-string source can't recompute `focusedLines`/`collapsible` downstream).
    finalTotalLines = framedRoot.data?.totalLines ?? 0;
    finalFocusedLines = framedRoot.data?.focusedLines ?? finalTotalLines;
    finalCollapsible = framedRoot.data?.collapsible === true;
  }
  return {
    source: finalSource,
    fallback: finalFallback,
    fallbackCritical: finalFallbackCritical,
    totalLines: finalTotalLines,
    focusedLines: finalFocusedLines,
    collapsible: finalCollapsible,
    transforms: finalTransforms,
    extraFiles: extraFilesFromSource,
    extraDependencies: extraDependenciesFromSource,
    externals: externalsFromSource,
    // `commentsFromSource` is already 1-indexed (the stored `Code` convention).
    comments: commentsFromSource
  };
}

/**
 * Loads and processes extra files recursively with support for relative paths
 * and circular dependency detection. Uses Promise.all for parallel loading.
 */
async function loadExtraFiles(variantName, extraFiles, baseUrl, entryUrl,
// Track the original entry file URL
loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, options = {}, allFilesListed = false, knownExtraFiles = new Set(), globalsFileKeys = new Set() // Track which files came from globals
) {
  const {
    maxDepth = 10,
    loadedFiles = new Set()
  } = options;
  if (maxDepth <= 0) {
    throw new Error('Maximum recursion depth reached while loading extra files');
  }
  const processedExtraFiles = {};
  const allFilesUsed = [];
  const allExternals = {};

  // Start loading all extra files in parallel
  const extraFilePromises = Object.entries(extraFiles).map(async ([fileName, fileData]) => {
    try {
      let fileUrl;
      let sourceData;
      let inlineComments;
      let transforms;
      let nextLoadedFiles;
      // True when the entry references an external file to load (string form
      // or object form carrying only a `relativeUrl`). Used both to pick the
      // load branch and to record the resolved URL in `filesUsedFromFile`.
      let treatAsExternalUrl = false;
      if (typeof fileData === 'string') {
        // fileData is a URL/path - use it directly, don't modify it
        fileUrl = fileData;
        treatAsExternalUrl = true;

        // Check for circular dependencies
        if (loadedFiles.has(fileUrl)) {
          throw new Error(`Circular dependency detected: ${fileUrl}`);
        }

        // Create a new set with the current file added for the recursive call
        // Don't mutate the parent's loadedFiles set
        nextLoadedFiles = new Set(loadedFiles);
        nextLoadedFiles.add(fileUrl);
      } else if (fileData.source === undefined && fileData.relativeUrl) {
        // Object form carrying only a `relativeUrl` (e.g., from loadServerCodeSource
        // when the extraFiles key was rewritten). Derive the file URL by
        // resolving the relative URL against the parent file's URL.
        fileUrl = new URL(fileData.relativeUrl, baseUrl).href;
        transforms = fileData.transforms;
        treatAsExternalUrl = true;
        if (loadedFiles.has(fileUrl)) {
          throw new Error(`Circular dependency detected: ${fileUrl}`);
        }
        nextLoadedFiles = new Set(loadedFiles);
        nextLoadedFiles.add(fileUrl);
      } else {
        // fileData is an object with source and/or transforms
        sourceData = fileData.source;
        // Inline extra files carry their own 1-indexed comments (their marker lines were
        // stripped from the source upstream); forward them so the enhancers apply the
        // `@focus`/`@highlight` frames instead of silently dropping them.
        inlineComments = fileData.comments;
        transforms = fileData.transforms;
        fileUrl = baseUrl; // Use base URL as fallback
        // For inline source, just pass a copy of loadedFiles without adding current file
        nextLoadedFiles = new Set(loadedFiles);
      }

      // Derive language from fileName for extra files
      const extraFileExtension = fileName.slice(fileName.lastIndexOf('.'));
      const extraFileLanguage = getLanguageFromExtension(extraFileExtension);

      // Load the file (this will handle recursive extra files)
      const fileResult = await loadSingleFile(variantName, fileName, sourceData, fileUrl, loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, transforms, {
        ...options,
        maxDepth: maxDepth - 1,
        loadedFiles: nextLoadedFiles
      }, allFilesListed, knownExtraFiles, extraFileLanguage, inlineComments);

      // Collect files used from this file load
      const filesUsedFromFile = [];
      if (treatAsExternalUrl) {
        filesUsedFromFile.push(fileUrl);
      }
      if (fileResult.extraDependencies) {
        filesUsedFromFile.push(...fileResult.extraDependencies);
      }

      // Collect externals from this file load
      const externalsFromFile = {};
      if (fileResult.externals) {
        Object.assign(externalsFromFile, fileResult.externals);
      }
      return {
        fileName,
        fileUrl,
        result: fileResult,
        filesUsed: filesUsedFromFile,
        externals: externalsFromFile
      };
    } catch (error) {
      throw new Error(`Failed to load extra file (variant: ${variantName}, file: ${fileName}, url: ${baseUrl}): ${error instanceof Error ? error.message : ''}`);
    }
  });

  // Wait for all extra files to load
  const extraFileResults = await Promise.all(extraFilePromises);

  // Process results and handle nested extra files
  const nestedExtraFilesPromises = [];
  for (const {
    fileName,
    fileUrl,
    result,
    filesUsed,
    externals
  } of extraFileResults) {
    const normalizedFileName = normalizePathKey(fileName);
    const originalFileData = extraFiles[fileName];

    // Preserve metadata flag if it exists in the original data, or if this file came from globals
    let metadata;
    if (typeof originalFileData !== 'string') {
      metadata = originalFileData.metadata;
    } else if (globalsFileKeys.has(fileName)) {
      metadata = true;
    }

    // Anchor `relativeUrl` to the entry variant URL so the consumer can always
    // recover the actual file URL via `new URL(relativeUrl, variant.url)`. We
    // emit it whenever the entry-anchored key (`./normalizedFileName`) doesn't
    // already resolve to the file URL — this also handles cases where
    // `loadServerCodeSource` stored the file as a plain string URL because its
    // local key happened to match, since once the file bubbles up through
    // multiple parents the local key no longer reflects its true location.
    let entryRelativeUrl;
    if (entryUrl && fileUrl) {
      try {
        const expectedFromEntry = new URL(`./${normalizedFileName}`, entryUrl).href;
        if (expectedFromEntry !== fileUrl) {
          entryRelativeUrl = computeRelativeUrl(entryUrl, fileUrl);
        }
      } catch {
        // entryUrl wasn't a valid URL base; leave relativeUrl unset.
      }
    }

    // Derive language from fileName extension for extra files
    const extraFileExtension = normalizedFileName.slice(normalizedFileName.lastIndexOf('.'));
    const extraFileLanguage = getLanguageFromExtension(extraFileExtension);
    processedExtraFiles[normalizedFileName] = {
      source: result.source,
      ...(result.fallback && {
        fallback: result.fallback
      }),
      ...(result.totalLines !== undefined && {
        totalLines: result.totalLines
      }),
      ...(result.focusedLines !== undefined && {
        focusedLines: result.focusedLines
      }),
      ...(result.collapsible !== undefined && {
        collapsible: result.collapsible
      }),
      ...(extraFileLanguage && {
        language: extraFileLanguage
      }),
      ...(result.transforms && {
        transforms: result.transforms
      }),
      ...(metadata !== undefined && {
        metadata
      }),
      ...(entryRelativeUrl !== undefined && {
        relativeUrl: entryRelativeUrl
      }),
      ...(result.comments && {
        comments: result.comments
      })
    };

    // Add files used from this file load
    allFilesUsed.push(...filesUsed);

    // Add externals from this file load using proper merging
    const mergedExternals = mergeExternals([allExternals, externals]);
    Object.assign(allExternals, mergedExternals);

    // Collect promises for nested extra files with their source key
    if (result.extraFiles) {
      nestedExtraFilesPromises.push(loadExtraFiles(variantName, result.extraFiles, fileUrl,
      // Use the resolved file URL as base for its extra files
      entryUrl,
      // Keep the entry URL for final conversion
      loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, {
        ...options,
        maxDepth: maxDepth - 1,
        loadedFiles: new Set(loadedFiles)
      }, allFilesListed, knownExtraFiles, globalsFileKeys // Pass through globals file tracking
      ).then(nestedResult => ({
        files: nestedResult.extraFiles,
        allFilesUsed: nestedResult.allFilesUsed,
        allExternals: nestedResult.allExternals,
        sourceFileKey: normalizedFileName // Pass the normalized key
      })));
    }
  }

  // Wait for all nested extra files and merge them, converting paths based on key structure
  if (nestedExtraFilesPromises.length > 0) {
    const nestedExtraFilesResults = await Promise.all(nestedExtraFilesPromises);
    for (const {
      files: nestedExtraFiles,
      allFilesUsed: nestedFilesUsed,
      allExternals: nestedExternals,
      sourceFileKey
    } of nestedExtraFilesResults) {
      // Add nested files used
      allFilesUsed.push(...nestedFilesUsed);

      // Add nested externals using proper merging
      const mergedNestedExternals = mergeExternals([allExternals, nestedExternals]);
      Object.assign(allExternals, mergedNestedExternals);
      for (const [nestedKey, nestedValue] of Object.entries(nestedExtraFiles)) {
        // Convert the storage key based on the directory structure of the source key.
        // The nested file's `relativeUrl` is already entry-anchored at this point,
        // so it carries the authoritative location and no further rewriting is needed.
        const convertedKey = convertKeyBasedOnDirectory(nestedKey, sourceFileKey);
        const normalizedConvertedKey = normalizePathKey(convertedKey);
        processedExtraFiles[normalizedConvertedKey] = nestedValue;
      }
    }
  }
  return {
    extraFiles: processedExtraFiles,
    allFilesUsed,
    allExternals
  };
}

/**
 * Loads a variant with support for recursive extra file loading.
 * The loadSource function can now return extraFiles that will be loaded recursively.
 * Supports both relative and absolute paths for extra files.
 * Uses Promise.all for efficient parallel loading of extra files.
 *
 * @param url - File URL for the variant
 * @param variantName - Name of the variant (used for error messages)
 * @param variant - Variant data object or URL string
 * @param options - Loading and processing options (source parser, transformers, enhancers, etc.)
 */
export async function loadIsomorphicCodeVariant(url, variantName, variant, options = {}) {
  if (!variant) {
    throw new Error(`Variant is missing from code: ${variantName}`);
  }
  const {
    sourceParser,
    loadSource,
    loadVariantMeta,
    sourceTransformers,
    sourceEnhancers,
    globalsCode,
    disableParsing
  } = options;

  // Create a cache for loadSource calls scoped to this loadIsomorphicCodeVariant call
  const loadSourceCache = new Map();
  const functionName = 'Load Variant';
  let currentMark = performanceMeasure(undefined, {
    mark: 'Start',
    measure: 'Start'
  }, [functionName, url || variantName], true);
  if (typeof variant === 'string') {
    if (!loadVariantMeta) {
      // Create a basic loadVariantMeta function as fallback
      const {
        fileName
      } = getFileNameFromUrl(variant);
      if (!fileName) {
        throw new Error(`Cannot determine fileName from URL "${variant}" for variant "${variantName}". ` + `Please provide a loadVariantMeta function or ensure the URL has a valid file extension.`);
      }
      variant = {
        url: variant,
        fileName
      };
    } else {
      try {
        variant = await loadVariantMeta(variantName, variant);
      } catch (error) {
        throw new Error(`Failed to load variant code (variant: ${variantName}, url: ${variant}): ${error instanceof Error ? error.message : String(error)}`, {
          cause: error
        });
      }
      currentMark = performanceMeasure(currentMark, {
        mark: 'Loaded Variant Meta',
        measure: 'Variant Meta Loading'
      }, [functionName, url || variantName]);
    }
  }
  const loadedFiles = new Set();
  if (url) {
    loadedFiles.add(url);
  }
  const allFilesUsed = url ? [url] : []; // Start with the main file URL if available
  let allExternals = {}; // Collect externals from all sources

  // Build set of known extra files from variant definition
  const knownExtraFiles = new Set();
  if (variant.extraFiles) {
    for (const extraFileName of Object.keys(variant.extraFiles)) {
      knownExtraFiles.add(extraFileName);
    }
  }

  // Load main file
  const fileName = variant.fileName || (url ? getFileNameFromUrl(url).fileName : undefined);

  // Derive language from variant.language or from fileName extension
  // Normalize the language to its canonical form (e.g., 'js' -> 'javascript')
  let language = variant.language ? normalizeLanguage(variant.language) : undefined;
  if (!language && fileName) {
    const extension = fileName.slice(fileName.lastIndexOf('.'));
    language = getLanguageFromExtension(extension);
  }

  // If we don't have a fileName and no URL, we can still parse if we have language
  if (!fileName && !url) {
    let finalSource = variant.source;
    let finalFallback;
    let finalFallbackCritical;

    // Parse the source if we have language and sourceParser
    if (typeof finalSource === 'string' && language && sourceParser && !disableParsing) {
      const parseSource = await sourceParser;
      finalSource = parseSource(finalSource, '', language);
    } else if (typeof finalSource === 'string') {
      // No language or parser - build plain-text HAST root with line gutters
      const root = {
        type: 'root',
        children: [{
          type: 'text',
          value: finalSource || ''
        }]
      };
      const sourceLines = (finalSource || '').split(/\r?\n|\r/);
      starryNightGutter(root, sourceLines);
      finalSource = root;
    }

    // Apply source enhancers if provided and parsing is not disabled
    if (!disableParsing && sourceEnhancers && sourceEnhancers.length > 0) {
      // `variant.comments` is already 1-indexed (the stored `Code` convention).
      const oneIndexedComments = variant.comments;
      finalSource = await applyEnhancers(finalSource, oneIndexedComments, '', sourceEnhancers);
    }

    // Apply output format compression in production. Other format conversions
    // happen lazily via the loader so tests can inspect the parsed HAST directly.
    if (finalSource && typeof finalSource === 'object' && 'type' in finalSource) {
      // Always derive a variant-level root fallback from the per-frame text so a
      // `ContentLoading` component can render before the hast is decoded.
      finalFallback = buildRootFallback(finalSource);
      // Sparse highlighted-visible companion (see `VariantCode.fallbackCritical`),
      // built from the live `HastRoot` before `stripFrameFallbacks` runs.
      const critical = buildCriticalFallback(finalSource, getInitialVisibleFrames(finalSource, false));
      finalFallbackCritical = Object.keys(critical).length > 0 ? critical : undefined;
      if (options.output === 'hastCompressed' && process.env.NODE_ENV === 'production') {
        if (finalFallback) {
          stripFrameFallbacks(finalSource);
        }
        const json = JSON.stringify(finalSource);
        // Use the fallback text as a DEFLATE dictionary; rebuilt on decode.
        const dictionary = finalFallback ? fallbackToText(finalFallback) : undefined;
        finalSource = {
          hastCompressed: await compressHastAsync(json, dictionary)
        };
      }
    }
    const finalVariant = {
      ...variant,
      language,
      source: finalSource,
      ...(finalFallback ? {
        fallback: finalFallback
      } : {}),
      ...(finalFallbackCritical ? {
        fallbackCritical: finalFallbackCritical
      } : {})
    };
    return {
      code: finalVariant,
      dependencies: [],
      // No dependencies without URL
      externals: {} // No externals without URL
    };
  }
  if (!fileName) {
    throw new Error(`No fileName available for variant "${variantName}". ` + `Please provide a fileName in the variant definition or ensure the URL has a valid file extension.`);
  }
  const mainFileResult = await loadSingleFile(variantName, fileName, variant.source, url, loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, variant.transforms, {
    ...options,
    loadedFiles
  }, variant.allFilesListed || false, knownExtraFiles, language, variant.comments);

  // Add files used from main file loading
  if (mainFileResult.extraDependencies) {
    allFilesUsed.push(...mainFileResult.extraDependencies);
  }
  currentMark = performanceMeasure(currentMark, {
    mark: 'Loaded Main File',
    measure: 'Main File Loading'
  }, [functionName, url || fileName], true);

  // Validate extraFiles keys from variant definition
  if (variant.extraFiles) {
    for (const extraFileName of Object.keys(variant.extraFiles)) {
      // Check if key is an absolute URL (should be relative)
      if (isAbsolutePath(extraFileName)) {
        throw new Error(`Invalid extraFiles key in variant: "${extraFileName}" appears to be an absolute path. ` + `extraFiles keys in variant definition should be relative paths from the main file.`);
      }
    }
  }

  // Collect extra files from variant definition and from loaded source
  const extraFilesToLoad = {
    ...(variant.extraFiles || {}),
    ...(mainFileResult.extraFiles || {})
  };

  // Add externals from main file loading
  if (mainFileResult.externals) {
    allExternals = mergeExternals([allExternals, mainFileResult.externals]);
  }
  const externalsMergedMark = performanceMeasure(currentMark, {
    mark: 'Externals Merged',
    measure: 'Merging Externals'
  }, [functionName, url || fileName]);
  currentMark = externalsMergedMark;

  // Track which files come from globals for metadata marking
  const globalsFileKeys = new Set(); // Track globals file keys for loadExtraFiles

  // Process globalsCode array and add to extraFiles if provided
  if (globalsCode && globalsCode.length > 0) {
    // Collect existing filenames to avoid conflicts
    const existingFiles = new Set();

    // Add main variant filename if it exists
    if (variant.fileName) {
      existingFiles.add(variant.fileName);
    }

    // Add already loaded extra files
    for (const key of Object.keys(extraFilesToLoad)) {
      existingFiles.add(key);
    }

    // Process all globals items in parallel
    const globalsPromises = globalsCode.map(async globalsItem => {
      let globalsVariant;
      if (typeof globalsItem === 'string') {
        // Handle string case - load the variant metadata
        if (!loadVariantMeta) {
          // Create a basic variant as fallback
          const {
            fileName: globalsFileName
          } = getFileNameFromUrl(globalsItem);
          if (!globalsFileName) {
            throw new Error(`Cannot determine fileName from globalsCode URL "${globalsItem}". ` + `Please provide a loadVariantMeta function or ensure the URL has a valid file extension.`);
          }
          globalsVariant = {
            url: globalsItem,
            fileName: globalsFileName
          };
        } else {
          try {
            globalsVariant = await loadVariantMeta(variantName, globalsItem);
            currentMark = performanceMeasure(currentMark, {
              mark: 'Globals Variant Meta Loaded',
              measure: 'Globals Variant Meta Loading'
            }, [functionName, globalsItem, url || fileName]);
          } catch (error) {
            throw new Error(`Failed to load globalsCode variant metadata (variant: ${variantName}, url: ${globalsItem}): ${JSON.stringify(error)}`);
          }
        }
      } else {
        globalsVariant = globalsItem;
      }

      // Load the globals code separately without affecting allFilesListed
      try {
        const globalsResult = await loadIsomorphicCodeVariant(globalsVariant.url, variantName, globalsVariant, {
          ...options,
          globalsCode: undefined
        } // Prevent infinite recursion
        );
        currentMark = performanceMeasure(currentMark, {
          mark: 'Globals Variant Loaded',
          measure: 'Globals Variant Loading'
        }, [functionName, globalsVariant.url || variantName, url || fileName]);
        return globalsResult;
      } catch (error) {
        throw new Error(`Failed to load globalsCode (variant: ${variantName}): ${error instanceof Error ? error.message : JSON.stringify(error)}`);
      }
    });

    // Wait for all globals to load
    const globalsResults = await Promise.all(globalsPromises);

    // Merge results from all globals
    for (const globalsResult of globalsResults) {
      // Add globals extraFiles (but NOT the main file)
      if (globalsResult.code.extraFiles) {
        // Add globals extra files with conflict-free naming and metadata flag
        for (const [key, value] of Object.entries(globalsResult.code.extraFiles)) {
          const conflictFreeKey = generateConflictFreeFilename(key, existingFiles);

          // Always add metadata: true flag for globals files
          if (typeof value === 'string') {
            // For string URLs, we can't easily wrap them but need to track for later metadata addition
            extraFilesToLoad[conflictFreeKey] = value;
            globalsFileKeys.add(conflictFreeKey); // Track for loadExtraFiles
          } else {
            // For object values, add metadata directly
            extraFilesToLoad[conflictFreeKey] = {
              ...value,
              metadata: true
            };
          }
          existingFiles.add(conflictFreeKey); // Track the added file for subsequent iterations
        }
      }

      // Add globals dependencies
      allFilesUsed.push(...globalsResult.dependencies);

      // Add globals externals
      allExternals = mergeExternals([allExternals, globalsResult.externals]);
    }
  }
  currentMark = performanceMeasure(externalsMergedMark, {
    mark: 'Globals Loaded',
    measure: 'Globals Loading'
  }, [functionName, url || fileName], true);
  let allExtraFiles = {};

  // Load all extra files if any exist and we have a URL
  if (Object.keys(extraFilesToLoad).length > 0) {
    if (!url) {
      // If there's no URL, we can only load extra files that have inline source or absolute URLs
      const loadableFiles = {};
      for (const [key, value] of Object.entries(extraFilesToLoad)) {
        if (typeof value !== 'string' && value.source !== undefined) {
          // Inline source - can always load
          loadableFiles[key] = value;
        } else if (typeof value === 'string' && isAbsolutePath(value)) {
          // Absolute URL - can load without base URL
          loadableFiles[key] = value;
        } else {
          console.warn(`Skipping extra file "${key}" - no URL provided and file requires loading from external source`);
        }
      }
      if (Object.keys(loadableFiles).length > 0) {
        // Process loadable files: inline sources without URL-based loading, absolute URLs with loading
        for (const [key, value] of Object.entries(loadableFiles)) {
          if (typeof value !== 'string') {
            // Inline source - preserve metadata if it was marked as globals
            const metadata = value.metadata || globalsFileKeys.has(key) ? true : undefined;
            // Derive language from filename extension
            const extension = key.slice(key.lastIndexOf('.'));
            const extraFileLanguage = getLanguageFromExtension(extension);
            allExtraFiles[normalizePathKey(key)] = {
              source: value.source,
              ...(extraFileLanguage && {
                language: extraFileLanguage
              }),
              ...(value.transforms && {
                transforms: value.transforms
              }),
              ...(metadata !== undefined && {
                metadata
              })
            };
          }
        }

        // For absolute URLs, we need to load them
        const urlFilesToLoad = {};
        for (const [key, value] of Object.entries(loadableFiles)) {
          if (typeof value === 'string') {
            urlFilesToLoad[key] = value;
          }
        }
        if (Object.keys(urlFilesToLoad).length > 0) {
          // Load absolute URL files even without base URL
          const extraFilesResult = await loadExtraFiles(variantName, urlFilesToLoad, '',
          // No base URL needed for absolute URLs
          '',
          // No entry URL
          loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, {
            ...options,
            loadedFiles
          }, variant.allFilesListed || false, knownExtraFiles, globalsFileKeys // Pass globals file tracking
          );
          allExtraFiles = {
            ...allExtraFiles,
            ...extraFilesResult.extraFiles
          };
          allFilesUsed.push(...extraFilesResult.allFilesUsed);
          allExternals = mergeExternals([allExternals, extraFilesResult.allExternals]);
        }
      }
    } else {
      const extraFilesResult = await loadExtraFiles(variantName, extraFilesToLoad, url, url,
      // Entry URL is the same as the main file URL
      loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, {
        ...options,
        loadedFiles
      }, variant.allFilesListed || false, knownExtraFiles, globalsFileKeys // Pass globals file tracking
      );
      allExtraFiles = extraFilesResult.extraFiles;
      allFilesUsed.push(...extraFilesResult.allFilesUsed);
      allExternals = mergeExternals([allExternals, extraFilesResult.allExternals]);
    }
    currentMark = performanceMeasure(currentMark, {
      mark: 'Extra Files Loaded',
      measure: 'Extra Files Loading'
    }, [functionName, url || fileName], true);
  }

  // Note: metadata marking is now handled during loadExtraFiles processing

  const finalVariant = {
    ...variant,
    language,
    source: mainFileResult.source,
    ...(mainFileResult.fallback && {
      fallback: mainFileResult.fallback
    }),
    ...(mainFileResult.fallbackCritical && {
      fallbackCritical: mainFileResult.fallbackCritical
    }),
    ...(mainFileResult.totalLines !== undefined && {
      totalLines: mainFileResult.totalLines
    }),
    ...(mainFileResult.focusedLines !== undefined && {
      focusedLines: mainFileResult.focusedLines
    }),
    ...(mainFileResult.collapsible !== undefined && {
      collapsible: mainFileResult.collapsible
    }),
    transforms: mainFileResult.transforms,
    extraFiles: Object.keys(allExtraFiles).length > 0 ? allExtraFiles : undefined,
    externals: Object.keys(allExternals).length > 0 ? Object.keys(allExternals) : undefined,
    // Include comments so they can be used by enhancers on server or client
    ...(mainFileResult.comments && {
      comments: mainFileResult.comments
    })
  };

  // Apply `urlPrefix` (if any) at the boundary so the file:// URLs the loader
  // received from disk don't leak to the client. We do this here — rather than
  // inside `loadSource` — so the rewrite is shared across every loader
  // implementation and applied consistently to extraFiles too.
  const variantWithPrefix = options.urlPrefix ? applyUrlPrefixToVariant(finalVariant, options.urlPrefix) : finalVariant;
  return {
    code: variantWithPrefix,
    dependencies: Array.from(new Set(allFilesUsed)),
    // Remove duplicates
    externals: allExternals
  };
}