@helia/verified-fetch
Version:
A fetch-like API for obtaining verified & trustless IPFS content on the web
340 lines • 16.2 kB
JavaScript
import { ipns as heliaIpns } from '@helia/ipns';
import { AbortError } from '@libp2p/interface';
import { prefixLogger } from '@libp2p/logger';
import { CustomProgressEvent } from 'progress-events';
import QuickLRU from 'quick-lru';
import { ByteRangeContextPlugin } from './plugins/plugin-handle-byte-range-context.js';
import { CarPlugin } from './plugins/plugin-handle-car.js';
import { DagCborPlugin } from './plugins/plugin-handle-dag-cbor.js';
import { DagPbPlugin } from './plugins/plugin-handle-dag-pb.js';
import { DagWalkPlugin } from './plugins/plugin-handle-dag-walk.js';
import { IpnsRecordPlugin } from './plugins/plugin-handle-ipns-record.js';
import { JsonPlugin } from './plugins/plugin-handle-json.js';
import { RawPlugin } from './plugins/plugin-handle-raw.js';
import { TarPlugin } from './plugins/plugin-handle-tar.js';
import { contentTypeParser } from './utils/content-type-parser.js';
import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js';
import { getETag } from './utils/get-e-tag.js';
import { getResolvedAcceptHeader } from './utils/get-resolved-accept-header.js';
import { getRedirectResponse } from './utils/handle-redirects.js';
import { parseResource } from './utils/parse-resource.js';
import { resourceToSessionCacheKey } from './utils/resource-to-cache-key.js';
import { setCacheControlHeader } from './utils/response-headers.js';
import { badRequestResponse, notAcceptableResponse, notSupportedResponse, badGatewayResponse } from './utils/responses.js';
import { selectOutputType } from './utils/select-output-type.js';
import { serverTiming } from './utils/server-timing.js';
const SESSION_CACHE_MAX_SIZE = 100;
const SESSION_CACHE_TTL_MS = 60 * 1000;
function convertOptions(options) {
if (options == null) {
return undefined;
}
let signal;
if (options?.signal === null) {
signal = undefined;
}
else {
signal = options?.signal;
}
return {
...options,
signal
};
}
export class VerifiedFetch {
helia;
ipns;
log;
contentTypeParser;
blockstoreSessions;
serverTimingHeaders = [];
withServerTiming;
plugins = [];
constructor({ helia, ipns }, init) {
this.helia = helia;
this.log = helia.logger.forComponent('helia:verified-fetch');
this.ipns = ipns ?? heliaIpns(helia);
this.contentTypeParser = init?.contentTypeParser ?? contentTypeParser;
this.blockstoreSessions = new QuickLRU({
maxSize: init?.sessionCacheSize ?? SESSION_CACHE_MAX_SIZE,
maxAge: init?.sessionTTLms ?? SESSION_CACHE_TTL_MS,
onEviction: (key, store) => {
store.close();
}
});
this.withServerTiming = init?.withServerTiming ?? false;
const pluginOptions = {
...init,
logger: prefixLogger('helia:verified-fetch'),
getBlockstore: (cid, resource, useSession, options) => this.getBlockstore(cid, resource, useSession, options),
handleServerTiming: async (name, description, fn) => this.handleServerTiming(name, description, fn, this.withServerTiming),
helia,
contentTypeParser: this.contentTypeParser
};
const defaultPlugins = [
new DagWalkPlugin(pluginOptions),
new ByteRangeContextPlugin(pluginOptions),
new IpnsRecordPlugin(pluginOptions),
new CarPlugin(pluginOptions),
new RawPlugin(pluginOptions),
new TarPlugin(pluginOptions),
new JsonPlugin(pluginOptions),
new DagCborPlugin(pluginOptions),
new DagPbPlugin(pluginOptions)
];
const customPlugins = init?.plugins?.map((pluginFactory) => pluginFactory(pluginOptions)) ?? [];
if (customPlugins.length > 0) {
// allow custom plugins to replace default plugins
const defaultPluginMap = new Map(defaultPlugins.map(plugin => [plugin.id, plugin]));
const customPluginMap = new Map(customPlugins.map(plugin => [plugin.id, plugin]));
this.plugins = defaultPlugins.map(plugin => customPluginMap.get(plugin.id) ?? plugin);
// Add any remaining custom plugins that don't replace a default plugin
this.plugins.push(...customPlugins.filter(plugin => !defaultPluginMap.has(plugin.id)));
}
else {
this.plugins = defaultPlugins;
}
this.log.trace('created VerifiedFetch instance');
}
getBlockstore(root, resource, useSession = true, options = {}) {
const key = resourceToSessionCacheKey(resource);
if (!useSession) {
return this.helia.blockstore;
}
let session = this.blockstoreSessions.get(key);
if (session == null) {
session = this.helia.blockstore.createSession(root, options);
this.blockstoreSessions.set(key, session);
}
return session;
}
async handleServerTiming(name, description, fn, withServerTiming) {
if (!withServerTiming) {
return fn();
}
const { error, result, header } = await serverTiming(name, description, fn);
this.serverTimingHeaders.push(header);
if (error != null) {
throw error;
}
return result;
}
/**
* The last place a Response touches in verified-fetch before being returned to the user. This is where we add the
* Server-Timing header to the response if it has been collected. It should be used for any final processing of the
* response before it is returned to the user.
*/
handleFinalResponse(response, { query, cid, reqFormat, ttl, protocol, ipfsPath, pathDetails, byteRangeContext, options } = {}) {
if (this.serverTimingHeaders.length > 0) {
const headerString = this.serverTimingHeaders.join(', ');
response.headers.set('Server-Timing', headerString);
this.serverTimingHeaders = [];
}
// if there are multiple ranges, we should omit the content-length header. see https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Transfer-Encoding
if (response.headers.get('Transfer-Encoding') !== 'chunked') {
if (byteRangeContext != null) {
const contentLength = byteRangeContext.getLength();
if (contentLength != null) {
this.log.trace('Setting Content-Length from byteRangeContext: %d', contentLength);
response.headers.set('Content-Length', contentLength.toString());
}
}
}
// set Content-Disposition header
let contentDisposition;
this.log.trace('checking for content disposition');
// force download if requested
if (query?.download === true) {
contentDisposition = 'attachment';
}
else {
this.log.trace('download not requested');
}
// override filename if requested
if (query?.filename != null) {
if (contentDisposition == null) {
contentDisposition = 'inline';
}
contentDisposition = `${contentDisposition}; ${getContentDispositionFilename(query.filename)}`;
}
else {
this.log.trace('no filename specified in query');
}
if (contentDisposition != null) {
response.headers.set('Content-Disposition', contentDisposition);
}
else {
this.log.trace('no content disposition specified');
}
if (cid != null && response.headers.get('etag') == null) {
response.headers.set('etag', getETag({ cid: pathDetails?.terminalElement.cid ?? cid, reqFormat, weak: false }));
}
if (protocol != null) {
setCacheControlHeader({ response, ttl, protocol });
}
if (ipfsPath != null) {
response.headers.set('X-Ipfs-Path', ipfsPath);
}
// set CORS headers. If hosting your own gateway with verified-fetch behind the scenes, you can alter these before you send the response to the client.
response.headers.set('Access-Control-Allow-Origin', '*');
response.headers.set('Access-Control-Allow-Methods', 'GET, HEAD, OPTIONS');
response.headers.set('Access-Control-Allow-Headers', 'Range, X-Requested-With');
response.headers.set('Access-Control-Expose-Headers', 'Content-Range, Content-Length, X-Ipfs-Path, X-Stream-Output');
if (reqFormat !== 'car') {
// if we are not doing streaming responses, set the Accept-Ranges header to bytes to enable range requests
response.headers.set('Accept-Ranges', 'bytes');
}
else {
// set accept-ranges to none to disable range requests for streaming responses
response.headers.set('Accept-Ranges', 'none');
}
if (options?.method === 'HEAD') {
// don't send the body for HEAD requests
const headers = response?.headers;
return new Response(null, { status: 200, headers });
}
return response;
}
/**
* Runs plugins in a loop. After each plugin that returns `null` (partial/no final),
* we re-check `canHandle()` for all plugins in the next iteration if the context changed.
*/
async runPluginPipeline(context, maxPasses = 3) {
let finalResponse;
let passCount = 0;
const pluginsUsed = new Set();
let prevModificationId = context.modified;
while (passCount < maxPasses) {
this.log(`Starting pipeline pass #${passCount + 1}`);
passCount++;
// gather plugins that say they can handle the *current* context, but haven't been used yet
const readyPlugins = this.plugins.filter(p => !pluginsUsed.has(p.id)).filter(p => p.canHandle(context));
if (readyPlugins.length === 0) {
this.log.trace('No plugins can handle the current context.. checking by CID code');
const plugins = this.plugins.filter(p => p.codes.includes(context.cid.code));
if (plugins.length > 0) {
readyPlugins.push(...plugins);
}
else {
this.log.trace('No plugins found that can handle request by CID code; exiting pipeline.');
break;
}
}
this.log.trace('Plugins ready to handle request: ', readyPlugins.map(p => p.id).join(', '));
// track if any plugin changed the context or returned a response
let contextChanged = false;
let pluginHandled = false;
for (const plugin of readyPlugins) {
try {
this.log.trace('Invoking plugin:', plugin.id);
pluginsUsed.add(plugin.id);
const maybeResponse = await plugin.handle(context);
if (maybeResponse != null) {
// if a plugin returns a final Response, short-circuit
finalResponse = maybeResponse;
pluginHandled = true;
break;
}
}
catch (err) {
if (context.options?.signal?.aborted) {
throw new AbortError(context.options?.signal?.reason);
}
this.log.error('Error in plugin:', plugin.constructor.name, err);
// if fatal, short-circuit the pipeline
if (err.name === 'PluginFatalError') {
// if plugin provides a custom error response, return it
return err.response ?? badGatewayResponse(context.resource, 'Failed to fetch');
}
}
finally {
// on each plugin call, check for changes in the context
const newModificationId = context.modified;
contextChanged = newModificationId !== prevModificationId;
if (contextChanged) {
prevModificationId = newModificationId;
}
}
if (finalResponse != null) {
this.log.trace('Plugin produced final response:', plugin.id);
break;
}
}
if (pluginHandled && finalResponse != null) {
break;
}
if (!contextChanged) {
this.log.trace('No context changes and no final response; exiting pipeline.');
break;
}
}
return finalResponse;
}
/**
* We're starting to get to the point where we need a queue or pipeline of
* operations to perform and a single place to handle errors.
*
* TODO: move operations called by fetch to a queue of operations where we can
* always exit early (and cleanly) if a given signal is aborted
*/
async fetch(resource, opts) {
this.log('fetch %s', resource);
if (opts?.method === 'OPTIONS') {
return this.handleFinalResponse(new Response(null, { status: 200 }));
}
const options = convertOptions(opts);
const withServerTiming = options?.withServerTiming ?? this.withServerTiming;
options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { resource }));
let parsedResult;
try {
parsedResult = await this.handleServerTiming('parse-resource', '', async () => parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, { withServerTiming, ...options }), withServerTiming);
this.serverTimingHeaders.push(...parsedResult.serverTimings.map(({ header }) => header));
}
catch (err) {
if (options?.signal?.aborted) {
throw new AbortError(options?.signal?.reason);
}
this.log.error('error parsing resource %s', resource, err);
return this.handleFinalResponse(badRequestResponse(resource.toString(), err));
}
options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:resolve', { cid: parsedResult.cid, path: parsedResult.path }));
const acceptHeader = getResolvedAcceptHeader({ query: parsedResult.query, headers: options?.headers, logger: this.helia.logger });
const accept = selectOutputType(parsedResult.cid, acceptHeader);
this.log('output type %s', accept);
if (acceptHeader != null && accept == null) {
return this.handleFinalResponse(notAcceptableResponse(resource.toString()));
}
const responseContentType = accept?.split(';')[0] ?? 'application/octet-stream';
const redirectResponse = await getRedirectResponse({ resource, options, logger: this.helia.logger, cid: parsedResult.cid });
if (redirectResponse != null) {
return this.handleFinalResponse(redirectResponse);
}
const context = {
...parsedResult,
resource: resource.toString(),
accept,
options,
withServerTiming,
onProgress: options?.onProgress,
modified: 0,
plugins: this.plugins.map(p => p.id)
};
this.log.trace('finding handler for cid code "%s" and response content type "%s"', parsedResult.cid.code, responseContentType);
const response = await this.runPluginPipeline(context);
options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: parsedResult.cid, path: parsedResult.path }));
return this.handleFinalResponse(response ?? notSupportedResponse(resource.toString()), context);
}
/**
* Start the Helia instance
*/
async start() {
await this.helia.start();
}
/**
* Shut down the Helia instance
*/
async stop() {
await this.helia.stop();
}
}
//# sourceMappingURL=verified-fetch.js.map