astro
Version:
Astro is a modern site builder with web best practices, performance, and DX front-of-mind.
315 lines (314 loc) • 10.8 kB
JavaScript
import { promises as fs, existsSync } from "node:fs";
import PQueue from "p-queue";
import xxhash from "xxhash-wasm";
import { AstroError, AstroErrorData } from "../core/errors/index.js";
import {
ASSET_IMPORTS_FILE,
COLLECTIONS_MANIFEST_FILE,
CONTENT_LAYER_TYPE,
DATA_STORE_FILE,
MODULES_IMPORTS_FILE
} from "./consts.js";
import {
getEntryConfigByExtMap,
getEntryDataAndImages,
globalContentConfigObserver,
safeStringify
} from "./utils.js";
class ContentLayer {
#logger;
#store;
#settings;
#watcher;
#lastConfigDigest;
#unsubscribe;
#generateDigest;
#queue;
constructor({ settings, logger, store, watcher }) {
watcher?.setMaxListeners(50);
this.#logger = logger;
this.#store = store;
this.#settings = settings;
this.#watcher = watcher;
this.#queue = new PQueue({ concurrency: 1 });
}
/**
* Whether the content layer is currently loading content
*/
get loading() {
return this.#queue.size > 0 || this.#queue.pending > 0;
}
/**
* Watch for changes to the content config and trigger a sync when it changes.
*/
watchContentConfig() {
this.#unsubscribe?.();
this.#unsubscribe = globalContentConfigObserver.subscribe(async (ctx) => {
if (ctx.status === "loaded" && ctx.config.digest !== this.#lastConfigDigest) {
this.sync();
}
});
}
unwatchContentConfig() {
this.#unsubscribe?.();
}
dispose() {
this.#queue.clear();
this.#unsubscribe?.();
}
async #getGenerateDigest() {
if (this.#generateDigest) {
return this.#generateDigest;
}
const { h64ToString } = await xxhash();
this.#generateDigest = (data) => {
const dataString = typeof data === "string" ? data : JSON.stringify(data);
return h64ToString(dataString);
};
return this.#generateDigest;
}
async #getLoaderContext({
collectionName,
loaderName = "content",
parseData,
refreshContextData
}) {
return {
collection: collectionName,
store: this.#store.scopedStore(collectionName),
meta: this.#store.metaStore(collectionName),
logger: this.#logger.forkIntegrationLogger(loaderName),
config: this.#settings.config,
parseData,
generateDigest: await this.#getGenerateDigest(),
watcher: this.#watcher,
refreshContextData,
entryTypes: getEntryConfigByExtMap([
...this.#settings.contentEntryTypes,
...this.#settings.dataEntryTypes
])
};
}
/**
* Enqueues a sync job that runs the `load()` method of each collection's loader, which will load the data and save it in the data store.
* The loader itself is responsible for deciding whether this will clear and reload the full collection, or
* perform an incremental update. After the data is loaded, the data store is written to disk. Jobs are queued,
* so that only one sync can run at a time. The function returns a promise that resolves when this sync job is complete.
*/
sync(options = {}) {
return this.#queue.add(() => this.#doSync(options));
}
async #doSync(options) {
const contentConfig = globalContentConfigObserver.get();
const logger = this.#logger.forkIntegrationLogger("content");
if (contentConfig?.status === "error") {
logger.error(`Error loading content config. Skipping sync.
${contentConfig.error.message}`);
return;
}
if (contentConfig?.status !== "loaded") {
logger.error("Content config not loaded, skipping sync");
return;
}
logger.info("Syncing content");
const {
vite: _vite,
integrations: _integrations,
adapter: _adapter,
...hashableConfig
} = this.#settings.config;
const astroConfigDigest = safeStringify(hashableConfig);
const { digest: currentConfigDigest } = contentConfig.config;
this.#lastConfigDigest = currentConfigDigest;
let shouldClear = false;
const previousConfigDigest = await this.#store.metaStore().get("content-config-digest");
const previousAstroConfigDigest = await this.#store.metaStore().get("astro-config-digest");
const previousAstroVersion = await this.#store.metaStore().get("astro-version");
if (previousAstroConfigDigest && previousAstroConfigDigest !== astroConfigDigest) {
logger.info("Astro config changed");
shouldClear = true;
}
if (currentConfigDigest && previousConfigDigest !== currentConfigDigest) {
logger.info("Content config changed");
shouldClear = true;
}
if (previousAstroVersion !== "5.1.2") {
logger.info("Astro version changed");
shouldClear = true;
}
if (shouldClear) {
logger.info("Clearing content store");
this.#store.clearAll();
}
if ("5.1.2") {
await this.#store.metaStore().set("astro-version", "5.1.2");
}
if (currentConfigDigest) {
await this.#store.metaStore().set("content-config-digest", currentConfigDigest);
}
if (astroConfigDigest) {
await this.#store.metaStore().set("astro-config-digest", astroConfigDigest);
}
await Promise.all(
Object.entries(contentConfig.config.collections).map(async ([name, collection]) => {
if (collection.type !== CONTENT_LAYER_TYPE) {
return;
}
let { schema } = collection;
if (!schema && typeof collection.loader === "object") {
schema = collection.loader.schema;
if (typeof schema === "function") {
schema = await schema();
}
}
if (options?.loaders && (typeof collection.loader !== "object" || !options.loaders.includes(collection.loader.name))) {
return;
}
const collectionWithResolvedSchema = { ...collection, schema };
const parseData = async ({ id, data, filePath = "" }) => {
const { data: parsedData } = await getEntryDataAndImages(
{
id,
collection: name,
unvalidatedData: data,
_internal: {
rawData: void 0,
filePath
}
},
collectionWithResolvedSchema,
false,
!!this.#settings.config.experimental.svg
);
return parsedData;
};
const context = await this.#getLoaderContext({
collectionName: name,
parseData,
loaderName: collection.loader.name,
refreshContextData: options?.context
});
if (typeof collection.loader === "function") {
return simpleLoader(collection.loader, context);
}
if (!collection.loader.load) {
throw new Error(`Collection loader for ${name} does not have a load method`);
}
return collection.loader.load(context);
})
);
await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
const cacheFile = getDataStoreFile(this.#settings);
await this.#store.writeToDisk(cacheFile);
const assetImportsFile = new URL(ASSET_IMPORTS_FILE, this.#settings.dotAstroDir);
await this.#store.writeAssetImports(assetImportsFile);
const modulesImportsFile = new URL(MODULES_IMPORTS_FILE, this.#settings.dotAstroDir);
await this.#store.writeModuleImports(modulesImportsFile);
logger.info("Synced content");
if (this.#settings.config.experimental.contentIntellisense) {
await this.regenerateCollectionFileManifest();
}
}
async regenerateCollectionFileManifest() {
const collectionsManifest = new URL(COLLECTIONS_MANIFEST_FILE, this.#settings.dotAstroDir);
this.#logger.debug("content", "Regenerating collection file manifest");
if (existsSync(collectionsManifest)) {
try {
const collections = await fs.readFile(collectionsManifest, "utf-8");
const collectionsJson = JSON.parse(collections);
collectionsJson.entries ??= {};
for (const { hasSchema, name } of collectionsJson.collections) {
if (!hasSchema) {
continue;
}
const entries = this.#store.values(name);
if (!entries?.[0]?.filePath) {
continue;
}
for (const { filePath } of entries) {
if (!filePath) {
continue;
}
const key = new URL(filePath, this.#settings.config.root).href.toLowerCase();
collectionsJson.entries[key] = name;
}
}
await fs.writeFile(collectionsManifest, JSON.stringify(collectionsJson, null, 2));
} catch {
this.#logger.error("content", "Failed to regenerate collection file manifest");
}
}
this.#logger.debug("content", "Regenerated collection file manifest");
}
}
async function simpleLoader(handler, context) {
const data = await handler();
context.store.clear();
if (Array.isArray(data)) {
for (const raw of data) {
if (!raw.id) {
throw new AstroError({
...AstroErrorData.ContentLoaderInvalidDataError,
message: AstroErrorData.ContentLoaderInvalidDataError.message(
context.collection,
`Entry missing ID:
${JSON.stringify({ ...raw, id: void 0 }, null, 2)}`
)
});
}
const item = await context.parseData({ id: raw.id, data: raw });
context.store.set({ id: raw.id, data: item });
}
return;
}
if (typeof data === "object") {
for (const [id, raw] of Object.entries(data)) {
if (raw.id && raw.id !== id) {
throw new AstroError({
...AstroErrorData.ContentLoaderInvalidDataError,
message: AstroErrorData.ContentLoaderInvalidDataError.message(
context.collection,
`Object key ${JSON.stringify(id)} does not match ID ${JSON.stringify(raw.id)}`
)
});
}
const item = await context.parseData({ id, data: raw });
context.store.set({ id, data: item });
}
return;
}
throw new AstroError({
...AstroErrorData.ExpectedImageOptions,
message: AstroErrorData.ContentLoaderInvalidDataError.message(
context.collection,
`Invalid data type: ${typeof data}`
)
});
}
function getDataStoreFile(settings, isDev) {
isDev ??= process?.env.NODE_ENV === "development";
return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir);
}
function contentLayerSingleton() {
let instance = null;
return {
init: (options) => {
instance?.dispose();
instance = new ContentLayer(options);
return instance;
},
get: () => instance,
dispose: () => {
instance?.dispose();
instance = null;
}
};
}
const globalContentLayer = contentLayerSingleton();
export {
ContentLayer,
getDataStoreFile,
globalContentLayer,
simpleLoader
};