astro/dist/content/types-generator.js

Astro is a modern site builder with web best practices, performance, and DX front-of-mind.

import * as path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import colors from "piccolore";
import { isRunnableDevEnvironment, normalizePath } from "vite";
import * as z from "zod/v4";
import { AstroError } from "../core/errors/errors.js";
import { AstroErrorData } from "../core/errors/index.js";
import { isRelativePath } from "../core/path.js";
import { COLLECTIONS_DIR, CONTENT_LAYER_TYPE, CONTENT_TYPES_FILE, VIRTUAL_MODULE_ID } from "./consts.js";
import { getContentEntryIdAndSlug, getContentPaths, getDataEntryExts, getDataEntryId, getEntryCollectionName, getEntryConfigByExtMap, getEntrySlug, getEntryType, reloadContentConfigObserver } from "./utils.js";
import { getContentConfigLoadPromise } from "./config-prewarm.js";
import { ASTRO_VITE_ENVIRONMENT_NAMES } from "../core/constants.js";
export async function createContentTypesGenerator({ contentConfigObserver, fs, logger, settings, viteServer }) {
	const collectionEntryMap = {};
	const contentPaths = getContentPaths(settings.config, fs, settings.config.legacy?.collectionsBackwardsCompat);
	const contentEntryConfigByExt = getEntryConfigByExtMap(settings.contentEntryTypes);
	const contentEntryExts = [...contentEntryConfigByExt.keys()];
	const dataEntryExts = getDataEntryExts(settings);
	let events = [];
	let debounceTimeout;
	const typeTemplateContent = await fs.promises.readFile(contentPaths.typesTemplate, "utf-8");
	async function init() {
		events.push({
			name: "add",
			entry: contentPaths.config.url
		});
		await runEvents();
	}
	async function handleEvent(event) {
		if (event.name === "addDir" || event.name === "unlinkDir") {
			const collection = normalizePath(path.relative(fileURLToPath(contentPaths.contentDir), fileURLToPath(event.entry)));
			const collectionKey = JSON.stringify(collection);
			// If directory is multiple levels deep, it is not a collection. Ignore event.
			const isCollectionEvent = collection.split("/").length === 1;
			if (!isCollectionEvent) return { shouldGenerateTypes: false };
			switch (event.name) {
				case "addDir":
					collectionEntryMap[collectionKey] = {
						type: "unknown",
						entries: {}
					};
					logger.debug("content", `${colors.cyan(collection)} collection added`);
					break;
				case "unlinkDir":
					delete collectionEntryMap[collectionKey];
					break;
			}
			return { shouldGenerateTypes: true };
		}
		const fileType = getEntryType(fileURLToPath(event.entry), contentPaths, contentEntryExts, dataEntryExts);
		if (fileType === "ignored") {
			return { shouldGenerateTypes: false };
		}
		if (fileType === "config") {
			// Dev server creation may already be loading this config. Wait for that
			// exact load when present; otherwise this path owns the reload. Do not
			// infer ownership from the global observer status because `astro sync`
			// can retain a settled status from an earlier invocation in the process.
			const prewarm = getContentConfigLoadPromise();
			if (prewarm) {
				await prewarm;
			} else {
				await reloadContentConfigObserver({
					fs,
					settings,
					environment: viteServer.environments[ASTRO_VITE_ENVIRONMENT_NAMES.astro],
					logger
				});
			}
			return { shouldGenerateTypes: true };
		}
		const { entry } = event;
		const { contentDir } = contentPaths;
		const collection = getEntryCollectionName({
			entry,
			contentDir
		});
		if (collection === undefined) {
			logger.warn("content", `${colors.bold(normalizePath(path.relative(fileURLToPath(contentPaths.contentDir), fileURLToPath(event.entry))))} must live in a ${colors.bold("content/...")} collection subdirectory.`);
			return { shouldGenerateTypes: false };
		}
		if (fileType === "data") {
			const id = getDataEntryId({
				entry,
				contentDir,
				collection
			});
			const collectionKey = JSON.stringify(collection);
			const entryKey = JSON.stringify(id);
			switch (event.name) {
				case "add":
					if (!(collectionKey in collectionEntryMap)) {
						collectionEntryMap[collectionKey] = {
							type: "data",
							entries: {}
						};
					}
					const collectionInfo = collectionEntryMap[collectionKey];
					if (collectionInfo.type === "content") {
						viteServer.environments.client.hot.send({
							type: "error",
							err: new AstroError({
								...AstroErrorData.MixedContentDataCollectionError,
								message: AstroErrorData.MixedContentDataCollectionError.message(collectionKey),
								location: { file: entry.pathname }
							})
						});
						return { shouldGenerateTypes: false };
					}
					if (!(entryKey in collectionEntryMap[collectionKey])) {
						collectionEntryMap[collectionKey] = {
							type: "data",
							entries: {
								...collectionInfo.entries,
								[entryKey]: {}
							}
						};
					}
					return { shouldGenerateTypes: true };
				case "unlink":
					if (collectionKey in collectionEntryMap && entryKey in collectionEntryMap[collectionKey].entries) {
						delete collectionEntryMap[collectionKey].entries[entryKey];
					}
					return { shouldGenerateTypes: true };
				case "change": return { shouldGenerateTypes: false };
			}
		}
		const contentEntryType = contentEntryConfigByExt.get(path.extname(event.entry.pathname));
		if (!contentEntryType) return { shouldGenerateTypes: false };
		const { id, slug: generatedSlug } = getContentEntryIdAndSlug({
			entry,
			contentDir,
			collection
		});
		const collectionKey = JSON.stringify(collection);
		if (!(collectionKey in collectionEntryMap)) {
			collectionEntryMap[collectionKey] = {
				type: "content",
				entries: {}
			};
		}
		const collectionInfo = collectionEntryMap[collectionKey];
		if (collectionInfo.type === "data") {
			viteServer.environments.client.hot.send({
				type: "error",
				err: new AstroError({
					...AstroErrorData.MixedContentDataCollectionError,
					message: AstroErrorData.MixedContentDataCollectionError.message(collectionKey),
					location: { file: entry.pathname }
				})
			});
			return { shouldGenerateTypes: false };
		}
		const entryKey = JSON.stringify(id);
		switch (event.name) {
			case "add":
				const addedSlug = await getEntrySlug({
					generatedSlug,
					id,
					collection,
					fileUrl: event.entry,
					contentEntryType,
					fs
				});
				if (!(entryKey in collectionEntryMap[collectionKey].entries)) {
					collectionEntryMap[collectionKey] = {
						type: "content",
						entries: {
							...collectionInfo.entries,
							[entryKey]: { slug: addedSlug }
						}
					};
				}
				return { shouldGenerateTypes: true };
			case "unlink":
				if (collectionKey in collectionEntryMap && entryKey in collectionEntryMap[collectionKey].entries) {
					delete collectionEntryMap[collectionKey].entries[entryKey];
				}
				return { shouldGenerateTypes: true };
			case "change":
				// User may modify `slug` in their frontmatter.
				// Only regen types if this change is detected.
				const changedSlug = await getEntrySlug({
					generatedSlug,
					id,
					collection,
					fileUrl: event.entry,
					contentEntryType,
					fs
				});
				const entryMetadata = collectionInfo.entries[entryKey];
				if (entryMetadata?.slug !== changedSlug) {
					collectionInfo.entries[entryKey].slug = changedSlug;
					return { shouldGenerateTypes: true };
				}
				return { shouldGenerateTypes: false };
		}
	}
	function queueEvent(rawEvent) {
		const event = {
			entry: pathToFileURL(rawEvent.entry),
			name: rawEvent.name
		};
		if (contentPaths.config.url.pathname !== event.entry.pathname) {
			return;
		}
		events.push(event);
		debounceTimeout && clearTimeout(debounceTimeout);
		const runEventsSafe = async () => {
			try {
				await runEvents();
			} catch {}
		};
		debounceTimeout = setTimeout(
			runEventsSafe,
			50
			/* debounce to batch chokidar events */
		);
	}
	async function runEvents() {
		const eventResponses = [];
		for (const event of events) {
			const response = await handleEvent(event);
			eventResponses.push(response);
		}
		events = [];
		const observable = contentConfigObserver.get();
		if (eventResponses.some((r) => r.shouldGenerateTypes)) {
			await writeContentFiles({
				fs,
				collectionEntryMap,
				contentPaths,
				typeTemplateContent,
				contentConfig: observable.status === "loaded" ? observable.config : undefined,
				contentEntryTypes: settings.contentEntryTypes,
				viteServer,
				logger,
				settings
			});
			if (!isRunnableDevEnvironment(viteServer.environments[ASTRO_VITE_ENVIRONMENT_NAMES.ssr])) {
				return;
			}
			invalidateVirtualMod(viteServer.environments[ASTRO_VITE_ENVIRONMENT_NAMES.ssr]);
		}
	}
	return {
		init,
		queueEvent
	};
}
// The virtual module contains a lookup map from slugs to content imports.
// Invalidate whenever content types change.
function invalidateVirtualMod(environment) {
	const virtualMod = environment.moduleGraph.getModuleById("\0" + VIRTUAL_MODULE_ID);
	if (!virtualMod) return;
	environment.moduleGraph.invalidateModule(virtualMod);
}
/**
* Takes the source (`from`) and destination (`to`) of a config path and
* returns a normalized relative version:
*  -   If is not relative, it adds `./` to the beginning.
*  -   If it ends with `.ts`, it replaces it with `.js`.
*  -   It adds `""` around the string.
* @param from Config path source.
* @param to Config path destination.
* @returns Normalized config path.
*/
function normalizeConfigPath(from, to) {
	const configPath = path.relative(from, to).replace(/\.ts$/, ".js");
	// on windows `path.relative` will use backslashes, these must be replaced with forward slashes
	const normalizedPath = configPath.replaceAll("\\", "/");
	return `"${isRelativePath(configPath) ? "" : "./"}${normalizedPath}"`;
}
const createSchemaResultCache = new Map();
async function getCreateSchemaResult(collection, collectionKey) {
	const cached = createSchemaResultCache.get(collectionKey);
	if (cached) {
		return cached;
	}
	if (collection?.type === CONTENT_LAYER_TYPE && typeof collection.loader === "object" && !collection.loader.schema && collection.loader.createSchema) {
		const result = await collection.loader.createSchema();
		createSchemaResultCache.set(collectionKey, result);
		return result;
	}
}
async function getContentLayerSchema(collection, collectionKey) {
	if (collection?.type !== CONTENT_LAYER_TYPE || typeof collection.loader === "function") {
		return;
	}
	if (collection.loader.schema) {
		return collection.loader.schema;
	}
	const result = await getCreateSchemaResult(collection, collectionKey);
	return result?.schema;
}
async function typeForCollection(collection, collectionKey) {
	if (collection?.schema) {
		return { type: `InferEntrySchema<${collectionKey}>` };
	}
	if (!collection?.type || typeof collection.loader === "function" || !collection.loader) {
		return { type: "any" };
	}
	if (typeof collection.loader === "object" && collection.loader.schema) {
		return { type: `InferLoaderSchema<${collectionKey}>` };
	}
	const result = await getCreateSchemaResult(collection, collectionKey);
	if (!result) {
		return { type: "any" };
	}
	const base = `loaders/${collectionKey.slice(1, -1)}`;
	return {
		type: `import("./${base}.js").Entry`,
		injectedType: {
			filename: `${base}.ts`,
			content: result.types
		}
	};
}
async function writeContentFiles({ fs, contentPaths, collectionEntryMap, typeTemplateContent, contentEntryTypes, contentConfig, viteServer, logger, settings }) {
	let dataTypesStr = "";
	const collectionSchemasDir = new URL(COLLECTIONS_DIR, settings.dotAstroDir);
	fs.mkdirSync(collectionSchemasDir, { recursive: true });
	for (const [collection, config] of Object.entries(contentConfig?.collections ?? {})) {
		collectionEntryMap[JSON.stringify(collection)] ??= {
			type: config.type ?? "unknown",
			entries: {}
		};
	}
	let contentCollectionsMap = {};
	for (const collectionKey of Object.keys(collectionEntryMap).sort()) {
		const collectionConfig = contentConfig?.collections[JSON.parse(collectionKey)];
		const collection = collectionEntryMap[collectionKey];
		if (collectionConfig?.type && collection.type !== "unknown" && collectionConfig.type !== CONTENT_LAYER_TYPE && collection.type !== collectionConfig.type) {
			viteServer.environments.client.hot.send({
				type: "error",
				err: new AstroError({
					...AstroErrorData.ContentCollectionTypeMismatchError,
					message: AstroErrorData.ContentCollectionTypeMismatchError.message(collectionKey, collection.type, collectionConfig.type),
					hint: collection.type === "data" ? "Try adding `type: 'data'` to your collection config." : undefined,
					location: { file: "" }
				})
			});
			return;
		}
		const { type: dataType, injectedType } = await typeForCollection(collectionConfig, collectionKey);
		if (injectedType) {
			if (settings.injectedTypes.some((t) => t.filename === CONTENT_TYPES_FILE)) {
				// If it's the first time, we inject types the usual way. sync() will handle creating files and references. If it's not the first time, we just override the dts content
				const url = new URL(injectedType.filename, settings.dotAstroDir);
				await fs.promises.mkdir(path.dirname(fileURLToPath(url)), { recursive: true });
				await fs.promises.writeFile(url, injectedType.content, "utf-8");
			} else {
				settings.injectedTypes.push(injectedType);
			}
		}
		dataTypesStr += `${collectionKey}: Record<string, {\n  id: string;\n  body?: string;\n  collection: ${collectionKey};\n  data: ${dataType};\n  rendered?: RenderedContent;\n  filePath?: string;\n  digest?: string | number;\n}>;\n`;
		if (collectionConfig && (collectionConfig.schema || await getContentLayerSchema(collectionConfig, collectionKey))) {
			await generateJSONSchema(fs, collectionConfig, collectionKey, collectionSchemasDir, logger);
			contentCollectionsMap[collectionKey] = collection;
		}
	}
	if (settings.config.experimental.contentIntellisense) {
		let contentCollectionManifest = {
			collections: [],
			entries: {}
		};
		Object.entries(contentCollectionsMap).forEach(([collectionKey, collection]) => {
			const collectionConfig = contentConfig?.collections[JSON.parse(collectionKey)];
			const key = JSON.parse(collectionKey);
			contentCollectionManifest.collections.push({
				hasSchema: Boolean(
					// Is there a user provided schema or
					collectionConfig?.schema || typeof collectionConfig?.loader === "object" && (collectionConfig.loader.schema || createSchemaResultCache.has(collectionKey))
				),
				name: key
			});
			Object.keys(collection.entries).forEach((entryKey) => {
				const entryPath = new URL(JSON.parse(entryKey), contentPaths.contentDir + `${key}/`).toString();
				// Save entry path in lowercase to avoid case sensitivity issues between Windows and Unix
				contentCollectionManifest.entries[entryPath.toLowerCase()] = key;
			});
		});
		await fs.promises.writeFile(new URL("./collections.json", collectionSchemasDir), JSON.stringify(contentCollectionManifest, null, 2));
	}
	const configPathRelativeToCacheDir = normalizeConfigPath(settings.dotAstroDir.pathname, contentPaths.config.url.pathname);
	const liveConfigPathRelativeToCacheDir = contentPaths.liveConfig?.exists ? normalizeConfigPath(settings.dotAstroDir.pathname, contentPaths.liveConfig.url.pathname) : undefined;
	for (const contentEntryType of contentEntryTypes) {
		if (contentEntryType.contentModuleTypes) {
			typeTemplateContent = contentEntryType.contentModuleTypes + "\n" + typeTemplateContent;
		}
	}
	typeTemplateContent = typeTemplateContent.replace("// @@DATA_ENTRY_MAP@@", dataTypesStr).replace("'@@CONTENT_CONFIG_TYPE@@'", contentConfig ? `typeof import(${configPathRelativeToCacheDir})` : "never").replace("'@@LIVE_CONTENT_CONFIG_TYPE@@'", liveConfigPathRelativeToCacheDir ? `typeof import(${liveConfigPathRelativeToCacheDir})` : "never");
	// If it's the first time, we inject types the usual way. sync() will handle creating files and references. If it's not the first time, we just override the dts content
	if (settings.injectedTypes.some((t) => t.filename === CONTENT_TYPES_FILE)) {
		await fs.promises.writeFile(new URL(CONTENT_TYPES_FILE, settings.dotAstroDir), typeTemplateContent, "utf-8");
	} else {
		settings.injectedTypes.push({
			filename: CONTENT_TYPES_FILE,
			content: typeTemplateContent
		});
	}
}
async function generateJSONSchema(fsMod, collectionConfig, collectionKey, collectionSchemasDir, logger) {
	let zodSchemaForJson = typeof collectionConfig.schema === "function" ? collectionConfig.schema({ image: () => z.string() }) : collectionConfig.schema;
	if (!zodSchemaForJson && collectionConfig.type === CONTENT_LAYER_TYPE) {
		zodSchemaForJson = await getContentLayerSchema(collectionConfig, collectionKey);
	}
	// The `file()` loader uses a schema which applies to every item in the file rather than a schema
	// for the whole file. We special case this to provide the correct JSON schema to users.
	// TODO: it would be nice if loaders could indicate this behavior so it wasn't unique to the built-in loader.
	if (collectionConfig.type === CONTENT_LAYER_TYPE && collectionConfig.loader.name === "file-loader") {
		// `file()` supports both top-level arrays and record objects. Generate an anyOf schema
		// so VS Code validates correctly regardless of which shape the source file uses.
		// `$schema` is injected into the object branch only — top-level array JSON files cannot
		// reference a schema property per the JSON Schema spec.
		const itemSchema = zodSchemaForJson;
		zodSchemaForJson = z.union([z.array(itemSchema), z.object({ $schema: z.string().optional() }).catchall(itemSchema)]);
	}
	if (zodSchemaForJson instanceof z.ZodObject) {
		const existingMeta = z.globalRegistry.get(zodSchemaForJson);
		zodSchemaForJson = zodSchemaForJson.extend({ $schema: z.string().optional() });
		if (existingMeta) {
			z.globalRegistry.add(zodSchemaForJson, existingMeta);
		}
	}
	try {
		const schema = z.toJSONSchema(zodSchemaForJson, {
			unrepresentable: "any",
			override: (ctx) => {
				const def = ctx.zodSchema._zod.def;
				if (def.type === "date") {
					ctx.jsonSchema.type = "string";
					ctx.jsonSchema.format = "date-time";
				}
			},
			// Collection schemas are used for parsing collection input, so we need to tell Zod to use the
			// input shape when generating a JSON schema.
			io: "input"
		});
		const schemaStr = JSON.stringify(schema, null, 2);
		const schemaJsonPath = new URL(`./${collectionKey.replace(/"/g, "")}.schema.json`, collectionSchemasDir);
		await fsMod.promises.writeFile(schemaJsonPath, schemaStr);
	} catch (err) {
		// This should error gracefully and not crash the dev server
		logger.warn("content", `An error was encountered while creating the JSON schema for the ${collectionKey} collection. Proceeding without it. Error: ${err}`);
	}
}