astro/dist/content/loaders/glob.js

Astro is a modern site builder with web best practices, performance, and DX front-of-mind.

import { existsSync, promises as fs } from "node:fs";
import { relative } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import pLimit from "p-limit";
import colors from "piccolore";
import picomatch from "picomatch";
import { glob as tinyglobby } from "tinyglobby";
import * as AstroErrorData from "../../core/errors/errors-data.js";
import { AstroError } from "../../core/errors/index.js";
import { getContentEntryIdAndSlug, posixRelative } from "../utils.js";
/**
* Resolves a glob entry path, relative to `base`, to a file URL.
*/
function resolveEntryUrl(entry, base) {
	// The `./` prefix keeps a leading `name:` from being parsed as a URL scheme (#17762).
	// `encodeURI` leaves `#` and `?` as-is, which would start a hash or query (#18199).
	return new URL("./" + encodeURI(entry).replace(/[#?]/g, encodeURIComponent), base);
}
function generateIdDefault({ entry, base, data }, isLegacy) {
	if (data.slug) {
		return String(data.slug);
	}
	const entryURL = resolveEntryUrl(entry, base);
	if (isLegacy) {
		// Legacy behavior: use ID based on path, not slug
		const { id } = getContentEntryIdAndSlug({
			entry: entryURL,
			contentDir: base,
			collection: ""
		});
		return id;
	}
	const { slug } = getContentEntryIdAndSlug({
		entry: entryURL,
		contentDir: base,
		collection: ""
	});
	return slug;
}
function checkPrefix(pattern, prefix) {
	if (Array.isArray(pattern)) {
		return pattern.some((p) => p.startsWith(prefix));
	}
	return pattern.startsWith(prefix);
}
export const secretLegacyFlag = Symbol("astro.legacy-glob");
/**
* Loads multiple entries, using a glob pattern to match files.
* @param pattern A glob pattern to match files, relative to the content directory.
*/
export function glob(globOptions) {
	if (checkPrefix(globOptions.pattern, "../")) {
		throw new Error("Glob patterns cannot start with `../`. Set the `base` option to a parent directory instead.");
	}
	if (checkPrefix(globOptions.pattern, "/")) {
		throw new Error("Glob patterns cannot start with `/`. Set the `base` option to a parent directory or use a relative path instead.");
	}
	const isLegacy = !!globOptions[secretLegacyFlag];
	const userGenerateId = globOptions?.generateId ?? ((opts) => generateIdDefault(opts, isLegacy));
	// Coerce to string so numeric ids from YAML don't cause Set strict-equality mismatches
	// against string store keys in the untouched-entries cleanup. See #17624.
	const generateId = (opts) => String(userGenerateId(opts));
	const fileToIdMap = new Map();
	return {
		name: "glob-loader",
		load: async ({ config, collection, logger, watcher, parseData, store, generateDigest, entryTypes }) => {
			const renderFunctionByContentType = new WeakMap();
			const untouchedEntries = new Set(store.keys());
			async function syncData(entry, base, entryType, oldId) {
				if (!entryType) {
					logger.warn(`No entry type found for ${entry}`);
					return;
				}
				const fileUrl = resolveEntryUrl(entry, base);
				const contents = await fs.readFile(fileUrl, "utf-8").catch((err) => {
					logger.error(`Error reading ${entry}: ${err.message}`);
					return;
				});
				if (!contents && contents !== "") {
					logger.warn(`No contents found for ${entry}`);
					return;
				}
				const { body, data } = await entryType.getEntryInfo({
					contents,
					fileUrl
				});
				const id = generateId({
					entry,
					base,
					data
				});
				if (oldId && oldId !== id) {
					store.delete(oldId);
				}
				untouchedEntries.delete(id);
				const existingEntry = store.get(id);
				const digest = generateDigest(contents);
				const filePath = fileURLToPath(fileUrl);
				if (existingEntry && existingEntry.digest === digest && existingEntry.filePath) {
					if (existingEntry.deferredRender) {
						store.addModuleImport(existingEntry.filePath);
					}
					if (existingEntry.assetImports?.length) {
						// Add asset imports for existing entries
						store.addAssetImports(existingEntry.assetImports, existingEntry.filePath);
					}
					fileToIdMap.set(filePath, id);
					return;
				}
				const relativePath = posixRelative(fileURLToPath(config.root), filePath);
				const parsedData = await parseData({
					id,
					data,
					filePath
				});
				if (existingEntry && existingEntry.filePath && existingEntry.filePath !== relativePath) {
					// Check the old file still exists - if not, this is likely a rename and
					// the unlink event just hasn't been processed yet
					const oldFilePath = new URL(existingEntry.filePath, config.root);
					if (existsSync(oldFilePath)) {
						const message = AstroErrorData.DuplicateContentEntrySlugError.message(collection, id, existingEntry.filePath, relativePath);
						if (config.prerenderConflictBehavior === "error") {
							throw new AstroError({
								...AstroErrorData.DuplicateContentEntrySlugError,
								message
							});
						} else if (config.prerenderConflictBehavior !== "ignore") {
							logger.warn(message);
						}
					}
				}
				if (entryType.getRenderFunction && !globOptions.deferRender) {
					let render = renderFunctionByContentType.get(entryType);
					if (!render) {
						render = await entryType.getRenderFunction(config);
						// Cache the render function for this content type, so it can re-use parsers and other expensive setup
						renderFunctionByContentType.set(entryType, render);
					}
					let rendered = undefined;
					try {
						rendered = await render?.({
							id,
							data,
							body,
							filePath,
							digest
						});
					} catch (error) {
						logger.error(`Error rendering ${entry}: ${error.message}`);
						throw error;
					}
					store.set({
						id,
						data: parsedData,
						body: globOptions.retainBody === false ? undefined : body,
						filePath: relativePath,
						digest,
						rendered,
						assetImports: rendered?.metadata?.imagePaths
					});
				} else if (entryType.getRenderFunction && globOptions.deferRender || "contentModuleTypes" in entryType) {
					store.set({
						id,
						data: parsedData,
						body: globOptions.retainBody === false ? undefined : body,
						filePath: relativePath,
						digest,
						deferredRender: true
					});
				} else {
					store.set({
						id,
						data: parsedData,
						body: globOptions.retainBody === false ? undefined : body,
						filePath: relativePath,
						digest
					});
				}
				fileToIdMap.set(filePath, id);
			}
			// For legacy collections, use the collection directory as base if not explicitly set
			let baseDir;
			if (isLegacy && !globOptions.base) {
				baseDir = new URL(`./src/content/${collection}`, config.root);
			} else {
				baseDir = globOptions.base ? new URL(globOptions.base, config.root) : config.root;
			}
			if (!baseDir.pathname.endsWith("/")) {
				baseDir.pathname = `${baseDir.pathname}/`;
			}
			const filePath = fileURLToPath(baseDir);
			const relativePath = relative(fileURLToPath(config.root), filePath);
			const exists = existsSync(baseDir);
			if (!exists) {
				// We warn and don't return because we will still set up the watcher in case the directory is created later
				logger.warn(`The base directory "${fileURLToPath(baseDir)}" does not exist.`);
			}
			const files = await tinyglobby(globOptions.pattern, {
				cwd: fileURLToPath(baseDir),
				expandDirectories: false
			});
			if (exists && files.length === 0) {
				logger.warn(`No files found matching "${globOptions.pattern}" in directory "${relativePath}"`);
			}
			function configForFile(file) {
				const ext = file.split(".").at(-1);
				if (!ext) {
					logger.warn(`No extension found for ${file}`);
					return;
				}
				return entryTypes.get(`.${ext}`);
			}
			const limit = pLimit(10);
			const skippedFiles = [];
			const contentDir = new URL("content/", config.srcDir);
			const configFiles = new Set([
				"config.js",
				"config.ts",
				"config.mjs"
			].map((file) => new URL(file, contentDir).href));
			function isConfigFile(file) {
				const fileUrl = resolveEntryUrl(file, baseDir);
				return configFiles.has(fileUrl.href);
			}
			await Promise.all(files.map((entry) => {
				if (isConfigFile(entry)) {
					return;
				}
				return limit(async () => {
					const entryType = configForFile(entry);
					await syncData(entry, baseDir, entryType);
				});
			}));
			const skipCount = skippedFiles.length;
			if (skipCount > 0) {
				const patternList = Array.isArray(globOptions.pattern) ? globOptions.pattern.join(", ") : globOptions.pattern;
				logger.warn(`The glob() loader cannot be used for files in ${colors.bold("src/content")} when legacy mode is enabled.`);
				if (skipCount > 10) {
					logger.warn(`Skipped ${colors.green(skippedFiles.length)} files that matched ${colors.green(patternList)}.`);
				} else {
					logger.warn(`Skipped the following files that matched ${colors.green(patternList)}:`);
					skippedFiles.forEach((file) => logger.warn(`• ${colors.green(file)}`));
				}
			}
			// Remove entries that were not found this time
			untouchedEntries.forEach((id) => store.delete(id));
			if (!watcher) {
				return;
			}
			watcher.add(filePath);
			// Split negation patterns out and pass them as picomatch's `ignore` option
			// so watcher filtering matches tinyglobby's semantics (set subtraction),
			// not picomatch's default (any-match union). See #17484.
			// A leading `!(` is an extglob ("anything except"), not a negation (#18289).
			const patterns = Array.isArray(globOptions.pattern) ? globOptions.pattern : [globOptions.pattern];
			const isNegation = (p) => p.startsWith("!") && !p.startsWith("!(");
			const positivePatterns = patterns.filter((p) => !isNegation(p));
			const negationPatterns = patterns.filter(isNegation).map((p) => p.slice(1));
			const matchesGlob = (entry) => !entry.startsWith("../") && picomatch.isMatch(entry, positivePatterns, { ignore: negationPatterns.length > 0 ? negationPatterns : undefined });
			const basePath = fileURLToPath(baseDir);
			async function onChange(changedPath) {
				const entry = posixRelative(basePath, changedPath);
				if (!matchesGlob(entry)) {
					return;
				}
				const entryType = configForFile(changedPath);
				const baseUrl = pathToFileURL(basePath);
				const oldId = fileToIdMap.get(changedPath);
				try {
					await syncData(entry, baseUrl, entryType, oldId);
					logger.info(`Reloaded data from ${colors.green(entry)}`);
				} catch (e) {
					logger.error(`Failed to reload ${entry}: ${e.message}`);
				}
			}
			watcher.on("change", onChange);
			watcher.on("add", onChange);
			watcher.on("unlink", async (deletedPath) => {
				const entry = posixRelative(basePath, deletedPath);
				if (!matchesGlob(entry)) {
					return;
				}
				const id = fileToIdMap.get(deletedPath);
				if (id) {
					store.delete(id);
					fileToIdMap.delete(deletedPath);
				}
			});
		}
	};
}