UNPKG

gatsby

Version:
976 lines (843 loc) • 28.7 kB
import { createFromReadableStream } from "react-server-dom-webpack" import prefetchHelper from "./prefetch" import emitter from "./emitter" import { setMatchPaths, findPath, findMatchPath } from "./find-path" /** * Available resource loading statuses */ export const PageResourceStatus = { /** * At least one of critical resources failed to load */ Error: `error`, /** * Resources loaded successfully */ Success: `success`, } const preferDefault = m => (m && m.default) || m const stripSurroundingSlashes = s => { s = s[0] === `/` ? s.slice(1) : s s = s.endsWith(`/`) ? s.slice(0, -1) : s return s } const createPageDataUrl = rawPath => { const [path, maybeSearch] = rawPath.split(`?`) const fixedPath = path === `/` ? `index` : stripSurroundingSlashes(path) return `${__PATH_PREFIX__}/page-data/${fixedPath}/page-data.json${ maybeSearch ? `?${maybeSearch}` : `` }` } /** * Utility to check the path that goes into doFetch for e.g. potential malicious intentions. * It checks for "//" because with this you could do a fetch request to a different domain. */ const shouldAbortFetch = rawPath => rawPath.startsWith(`//`) function doFetch(url, method = `GET`) { return new Promise(resolve => { const req = new XMLHttpRequest() req.open(method, url, true) req.onreadystatechange = () => { if (req.readyState == 4) { resolve(req) } } req.send(null) }) } const doesConnectionSupportPrefetch = () => { if ( `connection` in navigator && typeof navigator.connection !== `undefined` ) { if ((navigator.connection.effectiveType || ``).includes(`2g`)) { return false } if (navigator.connection.saveData) { return false } } return true } // Regex that matches common search crawlers const BOT_REGEX = /bot|crawler|spider|crawling/i const toPageResources = (pageData, component = null, head) => { const page = { componentChunkName: pageData.componentChunkName, path: pageData.path, webpackCompilationHash: pageData.webpackCompilationHash, matchPath: pageData.matchPath, staticQueryHashes: pageData.staticQueryHashes, getServerDataError: pageData.getServerDataError, slicesMap: pageData.slicesMap ?? {}, } return { component, head, json: pageData.result, page, } } function waitForResponse(response) { return new Promise(resolve => { try { const result = response.readRoot() resolve(result) } catch (err) { if ( Object.hasOwnProperty.call(err, `_response`) && Object.hasOwnProperty.call(err, `_status`) ) { setTimeout(() => { waitForResponse(response).then(resolve) }, 200) } else { throw err } } }) } export class BaseLoader { constructor(loadComponent, matchPaths) { // Map of pagePath -> Page. Where Page is an object with: { // status: PageResourceStatus.Success || PageResourceStatus.Error, // payload: PageResources, // undefined if PageResourceStatus.Error // } // PageResources is { // component, // json: pageData.result, // page: { // componentChunkName, // path, // webpackCompilationHash, // staticQueryHashes // }, // staticQueryResults // } this.pageDb = new Map() this.inFlightDb = new Map() this.staticQueryDb = {} this.pageDataDb = new Map() this.partialHydrationDb = new Map() this.slicesDataDb = new Map() this.sliceInflightDb = new Map() this.slicesDb = new Map() this.isPrefetchQueueRunning = false this.prefetchQueued = [] this.prefetchTriggered = new Set() this.prefetchCompleted = new Set() this.loadComponent = loadComponent setMatchPaths(matchPaths) } inFlightNetworkRequests = new Map() memoizedGet(url) { let inFlightPromise = this.inFlightNetworkRequests.get(url) if (!inFlightPromise) { inFlightPromise = doFetch(url, `GET`) this.inFlightNetworkRequests.set(url, inFlightPromise) } // Prefer duplication with then + catch over .finally to prevent problems in ie11 + firefox return inFlightPromise .then(response => { this.inFlightNetworkRequests.delete(url) return response }) .catch(err => { this.inFlightNetworkRequests.delete(url) throw err }) } setApiRunner(apiRunner) { this.apiRunner = apiRunner this.prefetchDisabled = apiRunner(`disableCorePrefetching`).some(a => a) } fetchPageDataJson(loadObj) { const { pagePath, retries = 0 } = loadObj const url = createPageDataUrl(pagePath) return this.memoizedGet(url).then(req => { const { status, responseText } = req // Handle 200 if (status === 200) { try { const jsonPayload = JSON.parse(responseText) if (jsonPayload.path === undefined) { throw new Error(`not a valid pageData response`) } const maybeSearch = pagePath.split(`?`)[1] if (maybeSearch && !jsonPayload.path.includes(maybeSearch)) { jsonPayload.path += `?${maybeSearch}` } return Object.assign(loadObj, { status: PageResourceStatus.Success, payload: jsonPayload, }) } catch (err) { // continue regardless of error } } // Handle 404 if (status === 404 || status === 200) { // If the request was for a 404/500 page and it doesn't exist, we're done if (pagePath === `/404.html` || pagePath === `/500.html`) { return Object.assign(loadObj, { status: PageResourceStatus.Error, }) } // Need some code here to cache the 404 request. In case // multiple loadPageDataJsons result in 404s return this.fetchPageDataJson( Object.assign(loadObj, { pagePath: `/404.html`, notFound: true }) ) } // handle 500 response (Unrecoverable) if (status === 500) { return this.fetchPageDataJson( Object.assign(loadObj, { pagePath: `/500.html`, internalServerError: true, }) ) } // Handle everything else, including status === 0, and 503s. Should retry if (retries < 3) { return this.fetchPageDataJson( Object.assign(loadObj, { retries: retries + 1 }) ) } // Retried 3 times already, result is an error. return Object.assign(loadObj, { status: PageResourceStatus.Error, }) }) } fetchPartialHydrationJson(loadObj) { const { pagePath, retries = 0 } = loadObj const url = createPageDataUrl(pagePath).replace(`.json`, `-rsc.json`) return this.memoizedGet(url).then(req => { const { status, responseText } = req // Handle 200 if (status === 200) { try { return Object.assign(loadObj, { status: PageResourceStatus.Success, payload: responseText, }) } catch (err) { // continue regardless of error } } // Handle 404 if (status === 404 || status === 200) { // If the request was for a 404/500 page and it doesn't exist, we're done if (pagePath === `/404.html` || pagePath === `/500.html`) { return Object.assign(loadObj, { status: PageResourceStatus.Error, }) } // Need some code here to cache the 404 request. In case // multiple loadPageDataJsons result in 404s return this.fetchPartialHydrationJson( Object.assign(loadObj, { pagePath: `/404.html`, notFound: true }) ) } // handle 500 response (Unrecoverable) if (status === 500) { return this.fetchPartialHydrationJson( Object.assign(loadObj, { pagePath: `/500.html`, internalServerError: true, }) ) } // Handle everything else, including status === 0, and 503s. Should retry if (retries < 3) { return this.fetchPartialHydrationJson( Object.assign(loadObj, { retries: retries + 1 }) ) } // Retried 3 times already, result is an error. return Object.assign(loadObj, { status: PageResourceStatus.Error, }) }) } loadPageDataJson(rawPath) { const pagePath = findPath(rawPath) if (this.pageDataDb.has(pagePath)) { const pageData = this.pageDataDb.get(pagePath) if (process.env.BUILD_STAGE !== `develop` || !pageData.stale) { return Promise.resolve(pageData) } } return this.fetchPageDataJson({ pagePath }).then(pageData => { this.pageDataDb.set(pagePath, pageData) return pageData }) } loadPartialHydrationJson(rawPath) { const pagePath = findPath(rawPath) if (this.partialHydrationDb.has(pagePath)) { const pageData = this.partialHydrationDb.get(pagePath) if (process.env.BUILD_STAGE !== `develop` || !pageData.stale) { return Promise.resolve(pageData) } } return this.fetchPartialHydrationJson({ pagePath }).then(pageData => { this.partialHydrationDb.set(pagePath, pageData) return pageData }) } loadSliceDataJson(sliceName) { if (this.slicesDataDb.has(sliceName)) { const jsonPayload = this.slicesDataDb.get(sliceName) return Promise.resolve({ sliceName, jsonPayload }) } const url = `${__PATH_PREFIX__}/slice-data/${sliceName}.json` return doFetch(url, `GET`).then(res => { const jsonPayload = JSON.parse(res.responseText) this.slicesDataDb.set(sliceName, jsonPayload) return { sliceName, jsonPayload } }) } findMatchPath(rawPath) { return findMatchPath(rawPath) } // TODO check all uses of this and whether they use undefined for page resources not exist loadPage(rawPath) { const pagePath = findPath(rawPath) if (this.pageDb.has(pagePath)) { const page = this.pageDb.get(pagePath) if (process.env.BUILD_STAGE !== `develop` || !page.payload.stale) { if (page.error) { return Promise.resolve({ error: page.error, status: page.status, }) } return Promise.resolve(page.payload) } } if (this.inFlightDb.has(pagePath)) { return this.inFlightDb.get(pagePath) } const loadDataPromises = [ this.loadAppData(), this.loadPageDataJson(pagePath), ] if (global.hasPartialHydration) { loadDataPromises.push(this.loadPartialHydrationJson(pagePath)) } const inFlightPromise = Promise.all(loadDataPromises).then(allData => { const [appDataResponse, pageDataResponse, rscDataResponse] = allData if ( pageDataResponse.status === PageResourceStatus.Error || rscDataResponse?.status === PageResourceStatus.Error ) { return { status: PageResourceStatus.Error, } } let pageData = pageDataResponse.payload const { componentChunkName, staticQueryHashes: pageStaticQueryHashes = [], slicesMap = {}, } = pageData const finalResult = {} const dedupedSliceNames = Array.from(new Set(Object.values(slicesMap))) const loadSlice = slice => { if (this.slicesDb.has(slice.name)) { return this.slicesDb.get(slice.name) } else if (this.sliceInflightDb.has(slice.name)) { return this.sliceInflightDb.get(slice.name) } const inFlight = this.loadComponent(slice.componentChunkName).then( component => { return { component: preferDefault(component), sliceContext: slice.result.sliceContext, data: slice.result.data, } } ) this.sliceInflightDb.set(slice.name, inFlight) inFlight.then(results => { this.slicesDb.set(slice.name, results) this.sliceInflightDb.delete(slice.name) }) return inFlight } return Promise.all( dedupedSliceNames.map(sliceName => this.loadSliceDataJson(sliceName)) ).then(slicesData => { const slices = [] const dedupedStaticQueryHashes = [...pageStaticQueryHashes] for (const { jsonPayload, sliceName } of Object.values(slicesData)) { slices.push({ name: sliceName, ...jsonPayload }) for (const staticQueryHash of jsonPayload.staticQueryHashes) { if (!dedupedStaticQueryHashes.includes(staticQueryHash)) { dedupedStaticQueryHashes.push(staticQueryHash) } } } const loadChunkPromises = [ Promise.all(slices.map(loadSlice)), this.loadComponent(componentChunkName, `head`), ] if (!global.hasPartialHydration) { loadChunkPromises.push(this.loadComponent(componentChunkName)) } // In develop we have separate chunks for template and Head components // to enable HMR (fast refresh requires single exports). // In production we have shared chunk with both exports. Double loadComponent here // will be deduped by webpack runtime resulting in single request and single module // being loaded for both `component` and `head`. // get list of components to get const componentChunkPromises = Promise.all(loadChunkPromises).then( components => { const [sliceComponents, headComponent, pageComponent] = components finalResult.createdAt = new Date() for (const sliceComponent of sliceComponents) { if (!sliceComponent || sliceComponent instanceof Error) { finalResult.status = PageResourceStatus.Error finalResult.error = sliceComponent } } if ( !global.hasPartialHydration && (!pageComponent || pageComponent instanceof Error) ) { finalResult.status = PageResourceStatus.Error finalResult.error = pageComponent } let pageResources if (finalResult.status !== PageResourceStatus.Error) { finalResult.status = PageResourceStatus.Success if ( pageDataResponse.notFound === true || rscDataResponse?.notFound === true ) { finalResult.notFound = true } pageData = Object.assign(pageData, { webpackCompilationHash: appDataResponse ? appDataResponse.webpackCompilationHash : ``, }) if (typeof rscDataResponse?.payload === `string`) { pageResources = toPageResources(pageData, null, headComponent) pageResources.partialHydration = rscDataResponse.payload const readableStream = new ReadableStream({ start(controller) { const te = new TextEncoder() controller.enqueue(te.encode(rscDataResponse.payload)) }, pull(controller) { // close on next read when queue is empty controller.close() }, cancel() {}, }) return waitForResponse( createFromReadableStream(readableStream) ).then(result => { pageResources.partialHydration = result return pageResources }) } else { pageResources = toPageResources( pageData, pageComponent, headComponent ) } } // undefined if final result is an error return pageResources } ) // get list of static queries to get const staticQueryBatchPromise = Promise.all( dedupedStaticQueryHashes.map(staticQueryHash => { // Check for cache in case this static query result has already been loaded if (this.staticQueryDb[staticQueryHash]) { const jsonPayload = this.staticQueryDb[staticQueryHash] return { staticQueryHash, jsonPayload } } return this.memoizedGet( `${__PATH_PREFIX__}/page-data/sq/d/${staticQueryHash}.json` ) .then(req => { const jsonPayload = JSON.parse(req.responseText) return { staticQueryHash, jsonPayload } }) .catch(() => { throw new Error( `We couldn't load "${__PATH_PREFIX__}/page-data/sq/d/${staticQueryHash}.json"` ) }) }) ).then(staticQueryResults => { const staticQueryResultsMap = {} staticQueryResults.forEach(({ staticQueryHash, jsonPayload }) => { staticQueryResultsMap[staticQueryHash] = jsonPayload this.staticQueryDb[staticQueryHash] = jsonPayload }) return staticQueryResultsMap }) return ( Promise.all([componentChunkPromises, staticQueryBatchPromise]) .then(([pageResources, staticQueryResults]) => { let payload if (pageResources) { payload = { ...pageResources, staticQueryResults } finalResult.payload = payload emitter.emit(`onPostLoadPageResources`, { page: payload, pageResources: payload, }) } this.pageDb.set(pagePath, finalResult) if (finalResult.error) { return { error: finalResult.error, status: finalResult.status, } } return payload }) // when static-query fail to load we throw a better error .catch(err => { return { error: err, status: PageResourceStatus.Error, } }) ) }) }) inFlightPromise .then(() => { this.inFlightDb.delete(pagePath) }) .catch(error => { this.inFlightDb.delete(pagePath) throw error }) this.inFlightDb.set(pagePath, inFlightPromise) return inFlightPromise } // returns undefined if the page does not exists in cache loadPageSync(rawPath, options = {}) { const pagePath = findPath(rawPath) if (this.pageDb.has(pagePath)) { const pageData = this.pageDb.get(pagePath) if (pageData.payload) { return pageData.payload } if (options?.withErrorDetails) { return { error: pageData.error, status: pageData.status, } } } return undefined } shouldPrefetch(pagePath) { // Skip prefetching if we know user is on slow or constrained connection if (!doesConnectionSupportPrefetch()) { return false } // Don't prefetch if this is a crawler bot if (navigator.userAgent && BOT_REGEX.test(navigator.userAgent)) { return false } // Check if the page exists. if (this.pageDb.has(pagePath)) { return false } return true } prefetch(pagePath) { if (!this.shouldPrefetch(pagePath)) { return { then: resolve => resolve(false), abort: () => {}, } } if (this.prefetchTriggered.has(pagePath)) { return { then: resolve => resolve(true), abort: () => {}, } } const defer = { resolve: null, reject: null, promise: null, } defer.promise = new Promise((resolve, reject) => { defer.resolve = resolve defer.reject = reject }) this.prefetchQueued.push([pagePath, defer]) const abortC = new AbortController() abortC.signal.addEventListener(`abort`, () => { const index = this.prefetchQueued.findIndex(([p]) => p === pagePath) // remove from the queue if (index !== -1) { this.prefetchQueued.splice(index, 1) } }) if (!this.isPrefetchQueueRunning) { this.isPrefetchQueueRunning = true setTimeout(() => { this._processNextPrefetchBatch() }, 3000) } return { then: (resolve, reject) => defer.promise.then(resolve, reject), abort: abortC.abort.bind(abortC), } } _processNextPrefetchBatch() { const idleCallback = window.requestIdleCallback || (cb => setTimeout(cb, 0)) idleCallback(() => { const toPrefetch = this.prefetchQueued.splice(0, 4) const prefetches = Promise.all( toPrefetch.map(([pagePath, dPromise]) => { // Tell plugins with custom prefetching logic that they should start // prefetching this path. if (!this.prefetchTriggered.has(pagePath)) { this.apiRunner(`onPrefetchPathname`, { pathname: pagePath }) this.prefetchTriggered.add(pagePath) } // If a plugin has disabled core prefetching, stop now. if (this.prefetchDisabled) { return dPromise.resolve(false) } return this.doPrefetch(findPath(pagePath)).then(() => { if (!this.prefetchCompleted.has(pagePath)) { this.apiRunner(`onPostPrefetchPathname`, { pathname: pagePath }) this.prefetchCompleted.add(pagePath) } dPromise.resolve(true) }) }) ) if (this.prefetchQueued.length) { prefetches.then(() => { setTimeout(() => { this._processNextPrefetchBatch() }, 3000) }) } else { this.isPrefetchQueueRunning = false } }) } doPrefetch(pagePath) { const pageDataUrl = createPageDataUrl(pagePath) if (global.hasPartialHydration) { return Promise.all([ prefetchHelper(pageDataUrl, { crossOrigin: `anonymous`, as: `fetch`, }).then(() => // This was just prefetched, so will return a response from // the cache instead of making another request to the server this.loadPageDataJson(pagePath) ), prefetchHelper(pageDataUrl.replace(`.json`, `-rsc.json`), { crossOrigin: `anonymous`, as: `fetch`, }).then(() => // This was just prefetched, so will return a response from // the cache instead of making another request to the server this.loadPartialHydrationJson(pagePath) ), ]) } else { return prefetchHelper(pageDataUrl, { crossOrigin: `anonymous`, as: `fetch`, }).then(() => // This was just prefetched, so will return a response from // the cache instead of making another request to the server this.loadPageDataJson(pagePath) ) } } hovering(rawPath) { this.loadPage(rawPath) } getResourceURLsForPathname(rawPath) { const pagePath = findPath(rawPath) const page = this.pageDataDb.get(pagePath) if (page) { const pageResources = toPageResources(page.payload) return [ ...createComponentUrls(pageResources.page.componentChunkName), createPageDataUrl(pagePath), ] } else { return null } } isPageNotFound(rawPath) { const pagePath = findPath(rawPath) const page = this.pageDb.get(pagePath) return !page || page.notFound } loadAppData(retries = 0) { return this.memoizedGet(`${__PATH_PREFIX__}/page-data/app-data.json`).then( req => { const { status, responseText } = req let appData if (status !== 200 && retries < 3) { // Retry 3 times incase of non-200 responses return this.loadAppData(retries + 1) } // Handle 200 if (status === 200) { try { const jsonPayload = JSON.parse(responseText) if (jsonPayload.webpackCompilationHash === undefined) { throw new Error(`not a valid app-data response`) } appData = jsonPayload } catch (err) { // continue regardless of error } } return appData } ) } } const createComponentUrls = componentChunkName => (window.___chunkMapping[componentChunkName] || []).map( chunk => __PATH_PREFIX__ + chunk ) export class ProdLoader extends BaseLoader { constructor(asyncRequires, matchPaths, pageData) { const loadComponent = (chunkName, exportType = `components`) => { if (!global.hasPartialHydration) { exportType = `components` } if (!asyncRequires[exportType][chunkName]) { throw new Error( `We couldn't find the correct component chunk with the name "${chunkName}"` ) } return ( asyncRequires[exportType][chunkName]() // loader will handle the case when component is error .catch(err => err) ) } super(loadComponent, matchPaths) if (pageData) { this.pageDataDb.set(findPath(pageData.path), { pagePath: pageData.path, payload: pageData, status: `success`, }) } } doPrefetch(pagePath) { return super.doPrefetch(pagePath).then(result => { if (result.status !== PageResourceStatus.Success) { return Promise.resolve() } const pageData = result.payload const chunkName = pageData.componentChunkName const componentUrls = createComponentUrls(chunkName) return Promise.all(componentUrls.map(prefetchHelper)).then(() => pageData) }) } loadPageDataJson(rawPath) { return super.loadPageDataJson(rawPath).then(data => { if (data.notFound) { if (shouldAbortFetch(rawPath)) { return data } // check if html file exist using HEAD request: // if it does we should navigate to it instead of showing 404 return doFetch(rawPath, `HEAD`).then(req => { if (req.status === 200) { // page (.html file) actually exist (or we asked for 404 ) // returning page resources status as errored to trigger // regular browser navigation to given page return { status: PageResourceStatus.Error, } } // if HEAD request wasn't 200, return notFound result // and show 404 page return data }) } return data }) } loadPartialHydrationJson(rawPath) { return super.loadPartialHydrationJson(rawPath).then(data => { if (data.notFound) { if (shouldAbortFetch(rawPath)) { return data } // check if html file exist using HEAD request: // if it does we should navigate to it instead of showing 404 return doFetch(rawPath, `HEAD`).then(req => { if (req.status === 200) { // page (.html file) actually exist (or we asked for 404 ) // returning page resources status as errored to trigger // regular browser navigation to given page return { status: PageResourceStatus.Error, } } // if HEAD request wasn't 200, return notFound result // and show 404 page return data }) } return data }) } } let instance export const setLoader = _loader => { instance = _loader } export const publicLoader = { enqueue: rawPath => instance.prefetch(rawPath), // Real methods getResourceURLsForPathname: rawPath => instance.getResourceURLsForPathname(rawPath), loadPage: rawPath => instance.loadPage(rawPath), // TODO add deprecation to v4 so people use withErrorDetails and then we can remove in v5 and change default behaviour loadPageSync: (rawPath, options = {}) => instance.loadPageSync(rawPath, options), prefetch: rawPath => instance.prefetch(rawPath), isPageNotFound: rawPath => instance.isPageNotFound(rawPath), hovering: rawPath => instance.hovering(rawPath), loadAppData: () => instance.loadAppData(), } export default publicLoader export function getStaticQueryResults() { if (instance) { return instance.staticQueryDb } else { return {} } } export function getSliceResults() { if (instance) { return instance.slicesDb } else { return {} } }