UNPKG

astro

Version:

Astro is a modern site builder with web best practices, performance, and DX front-of-mind.

626 lines (625 loc) • 28.1 kB
import nodeFs from "node:fs"; import os from "node:os"; import { fileURLToPath } from "node:url"; import { recordTiming, time } from "@astrojs/internal-helpers/timings"; import PLimit from "p-limit"; import PQueue from "p-queue"; import colors from "piccolore"; import { generateImagesForPath, prepareAssetsGenerationEnv, StaticImageRegistry } from "../../assets/build/generate.js"; import { removeTrailingForwardSlash, trimSlashes } from "../../core/path.js"; import { runHookBuildGenerated, toIntegrationResolvedRoute } from "../../integrations/hooks.js"; import { hashCryptoKey } from "../encryption.js"; import { ensureAsyncRenderScope } from "../render-scope/node-scope.js"; import { AstroError, AstroErrorData } from "../errors/index.js"; import { getRedirectLocationOrThrow } from "../redirects/index.js"; import { createRequest } from "../request.js"; import { hasMetaRefreshTo, redirectTemplate } from "../routing/3xx.js"; import { routeIsRedirect } from "../routing/helpers.js"; import { getOutputFilename } from "../output-filename.js"; import { getOutFile, getOutFolder, getUrlForPath } from "./common.js"; import { createDefaultPrerenderer } from "./default-prerenderer.js"; import { loadBuildImageService } from "./image-service.js"; import { createParallelPrerenderer } from "./parallel-prerenderer.js"; import { IncrementalBuildCache } from "./incremental.js"; import { computeConfigHash } from "./config-hash/index.js"; import { computeLockfileHash } from "./lockfile/index.js"; import { hasPrerenderedPages } from "./internal.js"; import { buildPhase } from "./timings/index.js"; import { getTimeStat, shouldAppendForwardSlash } from "./util.js"; export async function generatePages(options, internals, prerenderOutputDir, builder) { // Collects images resolved outside of any render, e.g. at module top level. const buildCollectors = { staticImages: [], referencedImages: new Set() }; return ensureAsyncRenderScope().run(buildCollectors, () => generatePagesInBuildScope(options, internals, prerenderOutputDir, builder, buildCollectors)); } async function generatePagesInBuildScope(options, internals, prerenderOutputDir, builder, buildCollectors) { const generatePagesTimer = performance.now(); const hasPagesToGenerate = hasPrerenderedPages(internals); // Exit early when no prerendered pages were discovered to avoid setting up // and tearing down the prerenderer for an empty set of routes. if (!hasPagesToGenerate) { return; } const ssr = options.settings.buildOutput === "server"; const logger = options.logger; const images = new StaticImageRegistry(); images.addReferencedImages(internals.referencedImages); // Get or create the prerenderer let prerenderer; let parallelPrerenderer; const settingsPrerenderer = options.settings.prerenderer; if (!settingsPrerenderer && options.settings.config.experimental.parallelPrerender) { prerenderer = parallelPrerenderer = createParallelPrerenderer({ internals, options, prerenderOutputDir }); } else if (!settingsPrerenderer) { // No custom prerenderer - create default prerenderer = createDefaultPrerenderer({ internals, options, prerenderOutputDir }); } else if (typeof settingsPrerenderer === "function") { // Factory function - create default and pass it const defaultPrerenderer = createDefaultPrerenderer({ internals, options, prerenderOutputDir }); prerenderer = settingsPrerenderer(defaultPrerenderer, { outputDirectories: options.outputDirectories }); } else { // Direct prerenderer object - use as-is prerenderer = settingsPrerenderer; } if (settingsPrerenderer && options.settings.config.experimental.parallelPrerender) { logger.warn("build", `\`experimental.parallelPrerender\` is not supported by the "${prerenderer.name}" prerenderer and will be ignored.`); } // Set up the prerenderer await buildPhase("prerenderer setup", async () => prerenderer.setup?.()); const verb = ssr ? "prerendering" : "generating"; logger.info("SKIP_FORMAT", `\n${colors.bgGreen(colors.black(` ${verb} static routes `))}`); const routeToHeaders = new Map(); // Incremental build support let cache = null; if (options.settings.config.experimental.incrementalBuild) { const [configHash, lockfileHash, keyDigest] = await Promise.all([ computeConfigHash(options.settings.config), computeLockfileHash(fileURLToPath(options.settings.config.root)), options.key.then(hashCryptoKey) ]); cache = IncrementalBuildCache.load(options.settings, configHash, lockfileHash, keyDigest, internals.contentEntryRenderHashes ?? new Map(), options.force); } try { // Get all static paths with their routes from the prerenderer const staticPaths = await buildPhase("static paths", () => prerenderer.getStaticPaths()); let pathsWithRoutes; if (Array.isArray(staticPaths)) { pathsWithRoutes = staticPaths; } else { pathsWithRoutes = staticPaths.paths; images.addMetadata(staticPaths.metadata); } // Check if i18n domains are configured (incompatible with prerendering) const hasI18nDomains = ssr && options.settings.config.i18n?.domains && Object.keys(options.settings.config.i18n.domains).length > 0; // Filter paths for conflicts (same path from multiple routes) const { config } = options.settings; // Maps each normalized pathname to the route that first claimed it, // so conflict warnings identify the actual winning route. const builtPaths = new Map(); const filteredPaths = []; const fallbackPaths = []; for (const pathWithRoute of pathsWithRoutes) { const { pathname, route } = pathWithRoute; // i18n domains won't work with prerendered routes if (hasI18nDomains && route.prerender) { throw new AstroError({ ...AstroErrorData.NoPrerenderedRoutesWithDomains, message: AstroErrorData.NoPrerenderedRoutesWithDomains.message(route.component) }); } const normalized = removeTrailingForwardSlash(pathname); // Path hasn't been built yet, include it if (!builtPaths.has(normalized)) { builtPaths.set(normalized, route); } else { // Path was already built by another route (or a duplicate from the same route). const winningRoute = builtPaths.get(normalized); // Warn or error based on config. if (config.prerenderConflictBehavior === "error") { throw new AstroError({ ...AstroErrorData.PrerenderRouteConflict, message: AstroErrorData.PrerenderRouteConflict.message(winningRoute.route, route.route, normalized), hint: AstroErrorData.PrerenderRouteConflict.hint(winningRoute.route, route.route) }); } else if (config.prerenderConflictBehavior === "warn") { const msg = AstroErrorData.PrerenderRouteConflict.message(winningRoute.route, route.route, normalized); logger.warn("build", msg); } continue; } const paths = route.type === "fallback" ? fallbackPaths : filteredPaths; paths.push(pathWithRoute); } const generationPhases = [filteredPaths, fallbackPaths]; // Generate each path const generationConcurrency = parallelPrerenderer ? parallelPrerenderer.workerCount * config.build.concurrency : config.build.concurrency; if (generationConcurrency > 1) { const limit = PLimit(generationConcurrency); // Process in batches to avoid V8's Promise.all element limit, which is around ~123k items // // NOTE: ideally we could consider an iterator to avoid the batching limitation const BATCH_SIZE = 1e5; for (const paths of generationPhases) { for (let i = 0; i < paths.length; i += BATCH_SIZE) { const promises = paths.slice(i, i + BATCH_SIZE).map(({ pathname, route, cacheKey }) => limit(() => generatePathWithPrerenderer(prerenderer, pathname, route, options, internals, routeToHeaders, logger, cache, cacheKey, images))); await Promise.all(promises); } } } else { for (const paths of generationPhases) { for (const { pathname, route, cacheKey } of paths) { await generatePathWithPrerenderer(prerenderer, pathname, route, options, internals, routeToHeaders, logger, cache, cacheKey, images); } } } if (parallelPrerenderer) { images.addMetadata(await parallelPrerenderer.finish()); } // After generation, propagate distURL from the deserialized routes (used during generation) // back to the original routes in allPages. The prerenderer operates on deserialized route // objects (reconstructed from the serialized manifest), so distURL mutations during generation // don't affect the original route objects that are later passed to the astro:build:done hook. for (const paths of generationPhases) { for (const { route: generatedRoute } of paths) { if (generatedRoute.distURL && generatedRoute.distURL.length > 0) { for (const pageData of Object.values(options.allPages)) { if (pageData.route.route === generatedRoute.route && pageData.route.component === generatedRoute.component) { pageData.route.distURL = generatedRoute.distURL; break; } } } } } // Incremental build: prune stale cache copies and write the new manifest. // dist/ is regenerated in full each build, so a file in the output directory // was always produced by this build and must never be deleted here. Only the // persistent cache copies of paths that are no longer keyed are pruned. if (cache) { const orphans = cache.findOrphanedFiles(); // `deleteOutputFile` uses `rm({ force: true })`, so a missing file never // throws. A throw here is a real failure (permissions, locked file), so // surface it and count only the copies actually removed. let pruned = 0; for (const orphanFile of orphans) { try { await cache.deleteOutputFile(options.settings, orphanFile); pruned++; } catch (err) { logger.warn("build", `Could not prune stale incremental cache file ${orphanFile}: ${err}`); } } if (pruned > 0) { logger.info("build", `Pruned ${pruned} stale file(s) from the incremental cache.`); } cache.writeManifest(options.settings); } images.addMetadata({ staticImages: buildCollectors.staticImages, referencedImages: [...buildCollectors.referencedImages] }); // Must happen before teardown since collectStaticImages fetches from the prerender server if (prerenderer.collectStaticImages) { images.addStaticImageList(await prerenderer.collectStaticImages()); } logger.info(null, colors.green(`✓ Completed in ${getTimeStat(generatePagesTimer, performance.now())}.\n`)); const staticImageList = images.images; if (staticImageList.size) { logger.info("SKIP_FORMAT", `${colors.bgGreen(colors.black(` generating optimized images `))}`); const totalCount = Array.from(staticImageList.values()).map((x) => x.transforms.size).reduce((a, b) => a + b, 0); const cpuCount = os.availableParallelism(); const assetsCreationPipeline = await prepareAssetsGenerationEnv(options, totalCount, { loadImageService: () => loadBuildImageService(builder, options.settings), referencedImages: images.referencedImages }); const queue = new PQueue({ concurrency: Math.max(cpuCount, 1) }); const errors = []; const assetsTimer = performance.now(); for (const [originalPath, transforms] of staticImageList) { // Process each source image in parallel based on the queue’s concurrency // (`cpuCount`). Process each transform for a source image sequentially. // // # Design Decision: // We have 3 source images (A.png, B.png, C.png) and 3 transforms for // each: // ``` // A1.png A2.png A3.png // B1.png B2.png B3.png // C1.png C2.png C3.png // ``` // // ## Option 1 // Enqueue all transforms indiscriminantly // ``` // |_A1.png |_B2.png |_C1.png // |_B3.png |_A2.png |_C3.png // |_C2.png |_A3.png |_B1.png // ``` // * Advantage: Maximum parallelism, saturate CPU // * Disadvantage: Spike in context switching // // ## Option 2 // Enqueue all transforms, but constrain processing order by source image // ``` // |_A3.png |_B1.png |_C2.png // |_A1.png |_B3.png |_C1.png // |_A2.png |_B2.png |_C3.png // ``` // * Advantage: Maximum parallelism, saturate CPU (same as Option 1) in // hope to avoid context switching // * Disadvantage: Context switching still occurs and performance still // suffers // // ## Option 3 // Enqueue each source image, but perform the transforms for that source // image sequentially // ``` // \_A1.png \_B1.png \_C1.png // \_A2.png \_B2.png \_C2.png // \_A3.png \_B3.png \_C3.png // ``` // * Advantage: Less context switching // * Disadvantage: If you have a low number of source images with high // number of transforms then this is suboptimal. // // ## BEST OPTION: // **Option 3**. Most projects will have a higher number of source images // with a few transforms on each. Even though Option 2 should be faster // and _should_ prevent context switching, this was not observed in // nascent tests. Context switching was high and the overall performance // was half of Option 3. // // If looking to optimize further, please consider the following: // * Avoid `queue.add()` in an async for loop. Notice the `await // queue.onIdle();` after this loop. We do not want to create a scenario // where tasks are added to the queue after the queue.onIdle() resolves. // This can break tests and create annoying race conditions. // * Exposing a concurrency property in `astro.config.mjs` to allow users // to override Node’s os.availableParallelism() default. // * Create a proper performance benchmark for asset transformations of // projects in varying sizes of source images and transforms. queue.add(() => generateImagesForPath(originalPath, transforms, assetsCreationPipeline)).catch((e) => { logger.warn("build", `Unable to generate optimized image for ${originalPath}: ${e}`); errors.push(new Error(`Error generating image for ${originalPath}: ${e}`, { cause: e })); }); } await queue.onIdle(); if (errors.length === 1) { throw errors[0]; } else if (errors.length > 1) { throw new AggregateError(errors, `${errors.length} errors occurred during asset generation`); } const assetsTimeEnd = performance.now(); logger.info(null, colors.green(`✓ Completed in ${getTimeStat(assetsTimer, assetsTimeEnd)}.\n`)); } } finally { // Always teardown to avoid leaking adapter resources when generation fails. await prerenderer.teardown?.(); } await runHookBuildGenerated({ settings: options.settings, logger, routeToHeaders, dir: options.outputDirectories.client }); } const THRESHOLD_SLOW_RENDER_TIME_MS = 500; /** * Resolves a prerenderer's `render()` return value into its widened form. A * prerenderer may return a bare `Response` or a {@link PrerenderResult}; this * yields a `PrerenderResult` either way so callers handle a single shape. */ function normalizePrerenderResult(result) { return result instanceof Response ? { response: result } : result; } /** * Renders a single prerendered path to an in-memory result. * * This function is intentionally free of filesystem writes — it only calls * `prerenderer.render()` and computes output paths. The caller is responsible * for persisting the returned `body` to disk (or any other destination). * * Returning `null` signals that no output file should be created for this path: * - the response body was empty * - the redirect was suppressed by `config.build.redirects` * - a file with the same output path already exists in `publicDir` (public files * take priority over generated pages, so the generated page is skipped) * * @param params * @param params.prerenderer - The prerenderer used to obtain a `Response` for the path. * @param params.pathname - The URL pathname being rendered (e.g. `/about`). * @param params.route - Route data for the page being rendered. * @param params.options - Build options; `options.fsMod` is used to check whether a * file already exists in `publicDir` at the output path. * @param [params.routeToHeaders=new Map()] - Mutable map populated with response headers when * the adapter requests static-header tracking. Callers that do * not need to inspect the headers after the call can omit this. * @param params.logger - Logger instance. */ export async function renderPath({ prerenderer, pathname, route, options, routeToHeaders = new Map(), logger }) { const { config } = options.settings; // Do not render the fallback route if there is already a translated page // with the same path if (route.type === "fallback" && route.pathname !== "/") { if (options.routesList.routes.some((routeData) => { if (routeData.pattern.test(pathname)) { // Check if we've matched a dynamic route if (routeData.params && routeData.params.length !== 0) { // Make sure the pathname matches an entry in distURL if (routeData.distURL && !routeData.distURL.find((url) => url.href.replace(options.outputDirectories.client.toString(), "").replace(/(?:\/index\.html|\.html)$/, "") === trimSlashes(pathname))) { return false; } } // Route matches return true; } else { return false; } })) { return null; } } // Build the request URL const url = getUrlForPath(pathname, config.base, options.origin, config.build.format, config.trailingSlash, route.type, route.isIndex); const request = createRequest({ url, headers: new Headers(), logger, isPrerendered: true, routePattern: route.component }); // Render using the prerenderer let response; let metadata; try { const rendered = normalizePrerenderResult(await time("page-render", pathname, () => prerenderer.render(request, { routeData: route, collectMetadata: true }))); response = rendered.response; metadata = rendered.metadata; } catch (err) { logger.error("build", `Caught error rendering ${pathname}: ${err}`); if (err && !AstroError.is(err) && !err.id && typeof err === "object") { err.id = route.component; } throw err; } // Handle the response let body; const responseHeaders = response.headers; if (response.status >= 300 && response.status < 400) { // Handle redirects if (routeIsRedirect(route) && !config.build.redirects) { return null; } const locationSite = getRedirectLocationOrThrow(responseHeaders); const siteURL = config.site; const location = siteURL ? new URL(locationSite, siteURL) : locationSite; const fromPath = new URL(request.url).pathname; // A body on a redirect means `src/pages/3xx.astro` rendered one // (`experimental.redirectPage`); it already honors `compressHTML` through // the normal page renderer. if (response.body) { body = await response.text(); if (!hasMetaRefreshTo(body, locationSite)) { logger.warn("redirects", `\`src/pages/3xx.astro\` rendered the redirect from \`${fromPath}\` to \`${locationSite}\` without a matching \`<meta http-equiv="refresh" content="...;url=${locationSite}">\` tag. A static build sends no \`Location\` header, so visitors will not be redirected.`); } } else { body = redirectTemplate({ status: response.status, absoluteLocation: location, relativeLocation: locationSite, from: fromPath }); if (config.compressHTML) { body = body.replaceAll("\n", ""); } } if (route.type !== "redirect") { route.redirect = location.toString(); } } else { if (!response.body) { return null; } body = Buffer.from(await response.arrayBuffer()); } // Compute output paths const encodedPath = encodeURI(pathname); const outFolder = getOutFolder({ outRoot: options.outputDirectories.client, buildFormat: config.build.format, pathname: encodedPath, routeData: route }); const outFile = getOutFile(config.build.format, outFolder, encodedPath, route); if (route.distURL) { route.distURL.push(outFile); } else { route.distURL = [outFile]; } // Track headers for static headers feature const integrationRoute = toIntegrationResolvedRoute(route, config.trailingSlash); if (options.settings.adapter?.adapterFeatures?.staticHeaders) { routeToHeaders.set(pathname, { headers: responseHeaders, route: integrationRoute }); } // Public files take priority over generated routes if (checkPublicConflict({ outFile, route, outRoot: options.outputDirectories.client, publicDir: config.publicDir, logger })) { return null; } return { body, outFile, outFolder, metadata }; } /** * Generate a single path using the prerenderer interface. * Orchestrates rendering via `renderPath()` and writes the result to the filesystem. */ async function generatePathWithPrerenderer(prerenderer, pathname, route, options, internals, routeToHeaders, logger, cache, cacheKey, images) { const timeStart = performance.now(); const { config } = options.settings; const filePath = getOutputFilename(config.build.format, pathname, route); // Compute the output file path (needed for both skip check and recording) const encodedPath = encodeURI(pathname); const outFolder = getOutFolder({ outRoot: options.outputDirectories.client, buildFormat: config.build.format, pathname: encodedPath, routeData: route }); const outFile = getOutFile(config.build.format, outFolder, encodedPath, route); // Relative path from the resolved client output directory for cache storage const relativeOutFile = outFile.href.slice(options.outputDirectories.client.href.length); // Look up the dependency hash for this route const dependencyHash = internals.pageDependencyHashes?.get(route.component) ?? ""; const hasServerIsland = internals.serverIslandPageComponents?.has(route.component) ?? false; // Incremental build: check if we can skip this path if (cacheKey !== undefined && cache?.canSkip(route.component, pathname, dependencyHash, cacheKey, hasServerIsland)) { const existsInDist = nodeFs.existsSync(outFile); const restored = !existsInDist && await cache.restoreOutputFile(options.settings, relativeOutFile, outFile); if (existsInDist || restored) { // The page is not rendered, so its optimized-image transforms are never // re-registered. Replay them into the static image list so the asset pipeline // still emits the images its restored HTML references. const restoredImages = cache.previousStaticImages(route.component, pathname); const restoredReferencedImages = cache.previousReferencedImages(route.component, pathname); images.addMetadata({ staticImages: restoredImages, referencedImages: restoredReferencedImages }); // Likewise, the route contributes no response headers when it is not // rendered. Replay them so a `staticHeaders` adapter still writes this // route into its headers file. const restoredHeaders = cache.previousHeaders(route.component, pathname); if (restoredHeaders && options.settings.adapter?.adapterFeatures?.staticHeaders) { routeToHeaders.set(pathname, { headers: new Headers(restoredHeaders), route: toIntegrationResolvedRoute(route, config.trailingSlash) }); } // Record in the new cache so orphan detection knows this path is still alive, // carrying forward the content entries, image metadata, and headers the path // resolved last build. cache.record(route.component, dependencyHash, pathname, cacheKey, relativeOutFile, cache.previousContentEntryKeys(route.component, pathname), restoredImages, restoredReferencedImages, restoredHeaders); // Track page name for stats even when skipped if (route.type === "page") { addPageName(pathname, options); } // Track distURL for the route even when skipped if (route.distURL) { route.distURL.push(outFile); } else { route.distURL = [outFile]; } logger.info(null, ` ${colors.green("├─")} ${colors.dim(filePath)}`, false); logger.info("SKIP_FORMAT", restored ? ` ${colors.green("(restored)")}` : ` ${colors.green("(cached)")}`); recordPageTiming(pathname, route, timeStart, true); return; } } logger.info(null, ` ${colors.blue("├─")} ${colors.dim(filePath)}`, false); // Track page name for stats if (route.type === "page") { addPageName(pathname, options); } // The prerenderer collects the content // entries and image transforms resolved while rendering this path in its own // runtime and reports them on the render's metadata, so they can be folded // into the path's cache entry and replayed when the path is skipped on a // later build. const result = await renderPath({ prerenderer, pathname, route, options, routeToHeaders, logger }); images.addMetadata(result?.metadata); const contentEntryKeys = result?.metadata?.contentEntryKeys; const staticImages = result?.metadata?.staticImages; const referencedImages = result?.metadata?.referencedImages; // Headers are collected only for `staticHeaders` adapters (see `renderPath`). // Persist them so a skipped path can replay its route into the headers file. const headers = cache ? [...routeToHeaders.get(pathname)?.headers ?? []] : undefined; if (!result) { // A path that produced no output this build is deliberately not recorded. // A stale cache copy from a build where it did produce output would // otherwise be kept (the path is still keyed) and restored on a later skip, // resurrecting output the path no longer emits. Leaving it unrecorded makes // `findOrphanedFiles` prune that copy and forces a re-render next build. logRenderTime(logger, timeStart, true); recordPageTiming(pathname, route, timeStart, false); return; } await nodeFs.promises.mkdir(result.outFolder, { recursive: true }); await nodeFs.promises.writeFile(result.outFile, result.body); // Without a cache key or render metadata, the path cannot be skipped safely. if (cache && cacheKey !== undefined && result.metadata !== undefined) { await cache.writeOutputFile(options.settings, relativeOutFile, result.body); cache.record(route.component, dependencyHash, pathname, cacheKey, relativeOutFile, contentEntryKeys, staticImages, referencedImages, headers); } logRenderTime(logger, timeStart, false); recordPageTiming(pathname, route, timeStart, false); } function recordPageTiming(pathname, route, timeStart, cached) { recordTiming("page", pathname, performance.now() - timeStart, { route: route.route, cached }); } function logRenderTime(logger, timeStart, notCreated) { const timeEnd = performance.now(); const isSlow = timeEnd - timeStart > THRESHOLD_SLOW_RENDER_TIME_MS; const timeIncrease = (isSlow ? colors.red : colors.dim)(`(+${getTimeStat(timeStart, timeEnd)})`); const notCreatedMsg = notCreated ? colors.yellow("(file not created, response body was empty)") : ""; logger.info("SKIP_FORMAT", ` ${timeIncrease} ${notCreatedMsg}`); } function addPageName(pathname, opts) { const trailingSlash = opts.settings.config.trailingSlash; const buildFormat = opts.settings.config.build.format; const pageName = shouldAppendForwardSlash(trailingSlash, buildFormat) ? pathname.replace(/\/?$/, "/").replace(/^\//, "") : pathname.replace(/^\//, ""); opts.pageNames.push(pageName); } /** * Check if a file exists in the public directory that would conflict with the output file. * Public files take priority over generated routes. Returns true if there's a conflict. */ function checkPublicConflict({ outFile, route, outRoot, publicDir, logger }) { // Compute the relative path by comparing URL hrefs directly to avoid // fileURLToPath issues with encoded characters like %2F. const relativePath = outFile.href.slice(outRoot.href.length); const publicFileUrl = new URL(relativePath, publicDir); if (nodeFs.existsSync(publicFileUrl)) { logger.warn("build", `Skipping ${route.component} because a file with the same name exists in the public folder: ${relativePath}`); return true; } return false; }