astro/dist/assets/build/generate.js

Astro is a modern site builder with web best practices, performance, and DX front-of-mind.

import fs, { readFileSync } from "node:fs";
import { basename } from "node:path/posix";
import { recordTiming } from "@astrojs/internal-helpers/timings";
import colors from "piccolore";
import { getTimeStat } from "../../core/build/util.js";
import { AstroError } from "../../core/errors/errors.js";
import { copyCachedFile } from "../../core/fs/index.js";
import { AstroErrorData } from "../../core/errors/index.js";
import { astroToRuntimeLogger } from "../../core/logger/core.js";
import { isRemotePath, removeLeadingForwardSlash } from "../../core/path.js";
import { isESMImportedImage } from "../utils/imageKind.js";
import { getImageServiceConfig } from "../utils/service-config.js";
import { loadRemoteImage, revalidateRemoteImage } from "./remote.js";
export async function prepareAssetsGenerationEnv(options, totalCount, { loadImageService, referencedImages }) {
	const { settings, logger } = options;
	let useCache = true;
	const assetsCacheDir = new URL("assets/", settings.config.cacheDir);
	const count = {
		total: totalCount,
		current: 1
	};
	// Ensure that the cache directory exists
	try {
		await fs.promises.mkdir(assetsCacheDir, { recursive: true });
	} catch (err) {
		logger.warn(null, `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${err}`);
		useCache = false;
	}
	const isServerOutput = settings.buildOutput === "server";
	let serverRoot, clientRoot;
	if (isServerOutput) {
		// Read source images from prerender output and write optimized images to client output.
		serverRoot = options.outputDirectories.prerender;
		clientRoot = options.outputDirectories.client;
	} else {
		// Static builds move source images to client output before image generation.
		serverRoot = options.outputDirectories.client;
		clientRoot = options.outputDirectories.client;
	}
	let imageService;
	return {
		logger,
		getImageService: () => imageService ??= loadImageService(),
		referencedImages,
		isSSR: isServerOutput,
		count,
		useCache,
		assetsCacheDir,
		serverRoot,
		clientRoot,
		imageConfig: {
			...settings.config.image,
			service: getImageServiceConfig(settings.config.image.service, "build")
		},
		assetsFolder: settings.config.build.assets
	};
}
function getFullImagePath(originalFilePath, env) {
	return new URL(removeLeadingForwardSlash(originalFilePath), env.serverRoot);
}
export async function generateImagesForPath(originalFilePath, transformsAndPath, env) {
	let originalImage;
	for (const [_, transform] of transformsAndPath.transforms) {
		await generateImage(transform.finalPath, transform.transform);
	}
	// Delete original images that are only used for optimization
	// The referencedImages set tracks images that were used via raw `src` access (e.g., <img src={img.src}>).
	if (transformsAndPath.originalSrcPath && !env.referencedImages.has(transformsAndPath.originalSrcPath)) {
		try {
			if (transformsAndPath.originalSrcPath) {
				env.logger.debug("assets", `Deleting ${originalFilePath} as it's not referenced outside of image processing.`);
				await fs.promises.unlink(getFullImagePath(originalFilePath, env));
			}
		} catch {}
	}
	async function generateImage(filepath, options) {
		const timeStart = performance.now();
		const generationData = await generateImageInternal(filepath, options);
		const timeEnd = performance.now();
		const timeChange = getTimeStat(timeStart, timeEnd);
		const timeIncrease = `(+${timeChange})`;
		const statsText = generationData.cached !== "miss" ? generationData.cached === "hit" ? `(reused cache entry)` : `(revalidated cache entry)` : `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
		const count = `(${env.count.current}/${env.count.total})`;
		env.logger.info(null, `  ${colors.green("▶")} ${filepath} ${colors.dim(statsText)} ${colors.dim(timeIncrease)} ${colors.dim(count)}`);
		recordTiming("image", filepath, timeEnd - timeStart, { cached: generationData.cached !== "miss" });
		env.count.current++;
	}
	async function generateImageInternal(filepath, options) {
		const isLocalImage = isESMImportedImage(options.src);
		const finalFileURL = new URL("." + filepath, env.clientRoot);
		const finalFolderURL = new URL("./", finalFileURL);
		await fs.promises.mkdir(finalFolderURL, { recursive: true });
		const cacheFile = basename(filepath);
		const cachedFileURL = new URL(cacheFile, env.assetsCacheDir);
		// For remote images, we also save a JSON file with the expiration date, etag and last-modified date from the server
		const cacheMetaFile = cacheFile + ".json";
		const cachedMetaFileURL = new URL(cacheMetaFile, env.assetsCacheDir);
		// Check if we have a cached entry first
		try {
			if (isLocalImage) {
				await copyCachedFile(cachedFileURL, finalFileURL);
				return { cached: "hit" };
			} else {
				const JSONData = JSON.parse(readFileSync(cachedMetaFileURL, "utf-8"));
				if (typeof JSONData.expires !== "number") {
					await Promise.allSettled([fs.promises.unlink(cachedFileURL), fs.promises.unlink(cachedMetaFileURL)]);
					throw new Error(`Malformed cache entry for ${filepath}, cache will be regenerated for this file.`);
				}
				// Upgrade old base64 encoded asset cache to the new format
				if (JSONData.data) {
					const { data, ...meta } = JSONData;
					await Promise.all([fs.promises.writeFile(cachedFileURL, Buffer.from(data, "base64")), writeCacheMetaFile(cachedMetaFileURL, meta, env)]);
				}
				// If the cache entry is not expired, use it
				if (JSONData.expires > Date.now()) {
					await copyCachedFile(cachedFileURL, finalFileURL);
					return { cached: "hit" };
				}
				// Try to revalidate the cache
				if (JSONData.etag || JSONData.lastModified) {
					try {
						const revalidatedData = await revalidateRemoteImage(options.src, {
							etag: JSONData.etag,
							lastModified: JSONData.lastModified
						});
						if (revalidatedData.data !== null) {
							// Image cache was stale, update original image to avoid redownload
							originalImage = revalidatedData;
						} else {
							// Freshen cache on disk and output cached image
							await Promise.all([writeCacheMetaFile(cachedMetaFileURL, revalidatedData, env), copyCachedFile(cachedFileURL, finalFileURL)]);
							return { cached: "revalidated" };
						}
					} catch (e) {
						// Reuse stale cache if revalidation fails
						env.logger.warn(null, `An error was encountered while revalidating a cached remote asset. Proceeding with stale cache. ${e}`);
						await copyCachedFile(cachedFileURL, finalFileURL);
						return { cached: "hit" };
					}
				}
				await Promise.allSettled([fs.promises.unlink(cachedFileURL), fs.promises.unlink(cachedMetaFileURL)]);
			}
		} catch (e) {
			if (e.code !== "ENOENT") {
				throw new Error(`An error was encountered while reading the cache file. Error: ${e}`);
			}
		}
		// The original filepath or URL from the image transform
		const originalImagePath = isLocalImage ? options.src.src : options.src;
		if (!originalImage) {
			originalImage = await loadImage(originalFilePath, env);
		}
		let resultData = {
			data: undefined,
			expires: originalImage.expires,
			etag: originalImage.etag,
			lastModified: originalImage.lastModified
		};
		const imageService = await env.getImageService();
		try {
			resultData.data = (await imageService.transform(originalImage.data, {
				...options,
				src: originalImagePath
			}, env.imageConfig, astroToRuntimeLogger(env.logger))).data;
		} catch (e) {
			if (AstroError.is(e)) {
				throw e;
			}
			const error = new AstroError({
				...AstroErrorData.CouldNotTransformImage,
				message: AstroErrorData.CouldNotTransformImage.message(originalFilePath)
			}, { cause: e });
			throw error;
		}
		try {
			// Write the cache entry
			if (env.useCache) {
				if (isLocalImage) {
					await fs.promises.writeFile(cachedFileURL, resultData.data);
				} else {
					await Promise.all([fs.promises.writeFile(cachedFileURL, resultData.data), writeCacheMetaFile(cachedMetaFileURL, resultData, env)]);
				}
			}
		} catch (e) {
			env.logger.warn(null, `An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}`);
		} finally {
			// Write the final file
			await fs.promises.writeFile(finalFileURL, resultData.data);
		}
		return {
			cached: "miss",
			weight: {
				// Divide by 1024 to get size in kilobytes
				before: Math.trunc(originalImage.data.byteLength / 1024),
				after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024)
			}
		};
	}
}
async function writeCacheMetaFile(cachedMetaFileURL, resultData, env) {
	try {
		return await fs.promises.writeFile(cachedMetaFileURL, JSON.stringify({
			expires: resultData.expires,
			etag: resultData.etag,
			lastModified: resultData.lastModified
		}), "utf-8");
	} catch (e) {
		env.logger.warn(null, `An error was encountered while writing the cache file for a remote asset. Proceeding without caching this asset. Error: ${e}`);
	}
}
export class StaticImageRegistry {
	images = new Map();
	referencedImages = new Set();
	addStaticImages(images) {
		if (!images) return;
		for (const image of images) {
			let transformsForPath = this.images.get(image.originalPath);
			if (!transformsForPath) {
				transformsForPath = {
					originalSrcPath: image.originalSrcPath,
					transforms: new Map()
				};
				this.images.set(image.originalPath, transformsForPath);
			}
			if (!transformsForPath.transforms.has(image.hash)) {
				transformsForPath.transforms.set(image.hash, {
					finalPath: image.finalPath,
					transform: image.transform
				});
			}
		}
	}
	addReferencedImages(fsPaths) {
		if (!fsPaths) return;
		for (const fsPath of fsPaths) {
			this.referencedImages.add(fsPath);
		}
	}
	addMetadata(metadata) {
		this.addStaticImages(metadata?.staticImages);
		this.addReferencedImages(metadata?.referencedImages);
	}
	addStaticImageList(list) {
		for (const [originalPath, { originalSrcPath, transforms }] of list) {
			this.addStaticImages(Array.from(transforms, ([hash, { finalPath, transform }]) => ({
				originalPath,
				originalSrcPath,
				hash,
				finalPath,
				transform
			})));
		}
	}
}
async function loadImage(path, env) {
	if (isRemotePath(path)) {
		return await loadRemoteImage(path, undefined, env.imageConfig);
	}
	return {
		data: await fs.promises.readFile(getFullImagePath(path, env)),
		expires: 0
	};
}