cacheable-request
Version:
Wrap native HTTP requests with RFC compliant cache support
351 lines • 16.5 kB
JavaScript
// biome-ignore-all lint/suspicious/noImplicitAnyLet: legacy format
// biome-ignore-all lint/suspicious/noExplicitAny: legacy format
import crypto from "node:crypto";
import EventEmitter from "node:events";
import stream, { PassThrough as PassThroughStream } from "node:stream";
import urlLib, { URL } from "node:url";
import { getStreamAsBuffer } from "get-stream";
import CachePolicy from "http-cache-semantics";
import { Keyv } from "keyv";
import mimicResponse from "mimic-response";
import normalizeUrl from "normalize-url";
import Response from "responselike";
import { CacheError, RequestError, } from "./types.js";
class CacheableRequest {
constructor(cacheRequest, cacheAdapter) {
this.cache = new Keyv({ namespace: "cacheable-request" });
this.hooks = new Map();
this.request = () => (options, callback) => {
let url;
if (typeof options === "string") {
url = normalizeUrlObject(parseWithWhatwg(options));
options = {};
}
else if (options instanceof urlLib.URL) {
url = normalizeUrlObject(parseWithWhatwg(options.toString()));
options = {};
}
else {
const [pathname, ...searchParts] = (options.path ?? "").split("?");
const search = searchParts.length > 0 ? `?${searchParts.join("?")}` : "";
url = normalizeUrlObject({ ...options, pathname, search });
}
options = {
headers: {},
method: "GET",
cache: true,
strictTtl: false,
automaticFailover: false,
...options,
...urlObjectToRequestOptions(url),
};
options.headers = Object.fromEntries(entries(options.headers).map(([key, value]) => [
key.toLowerCase(),
value,
]));
const ee = new EventEmitter();
const normalizedUrlString = normalizeUrl(urlLib.format(url), {
stripWWW: false,
removeTrailingSlash: false,
stripAuthentication: false,
});
let key = `${options.method}:${normalizedUrlString}`;
// POST, PATCH, and PUT requests may be cached, depending on the response
// cache-control headers. As a result, the body of the request should be
// added to the cache key in order to avoid collisions.
if (options.body &&
options.method !== undefined &&
["POST", "PATCH", "PUT"].includes(options.method)) {
if (options.body instanceof stream.Readable) {
// Streamed bodies should completely skip the cache because they may
// or may not be hashable and in either case the stream would need to
// close before the cache key could be generated.
options.cache = false;
}
else {
key += `:${crypto.createHash("md5").update(options.body).digest("hex")}`;
}
}
let revalidate = false;
let madeRequest = false;
const makeRequest = (options_) => {
madeRequest = true;
let requestErrored = false;
/* c8 ignore next 4 */
let requestErrorCallback = () => {
/* do nothing */
};
const requestErrorPromise = new Promise((resolve) => {
requestErrorCallback = () => {
if (!requestErrored) {
requestErrored = true;
resolve();
}
};
});
const handler = async (response) => {
if (revalidate) {
response.status = response.statusCode;
const originalPolicy = CachePolicy.fromObject(revalidate.cachePolicy);
const revalidatedPolicy = originalPolicy.revalidatedPolicy(options_, response);
if (!revalidatedPolicy.modified) {
response.resume();
await new Promise((resolve) => {
// Skipping 'error' handler cause 'error' event should't be emitted for 304 response
response.once("end", resolve);
});
// Get headers from revalidated policy
const headers = convertHeaders(revalidatedPolicy.policy.responseHeaders());
// Preserve headers from the original cached response that may have been
// lost during revalidation (e.g., content-encoding, content-type, etc.)
// This works around a limitation in http-cache-semantics where some headers
// are not preserved when a 304 response has minimal headers
const originalHeaders = convertHeaders(originalPolicy.responseHeaders());
// Headers that should be preserved from the cached response
// according to RFC 7232 section 4.1
const preserveHeaders = [
"content-encoding",
"content-type",
"content-length",
"content-language",
"content-location",
"etag",
];
for (const headerName of preserveHeaders) {
if (originalHeaders[headerName] !== undefined &&
headers[headerName] === undefined) {
headers[headerName] = originalHeaders[headerName];
}
}
response = new Response({
statusCode: revalidate.statusCode,
headers,
body: revalidate.body,
url: revalidate.url,
});
response.cachePolicy = revalidatedPolicy.policy;
response.fromCache = true;
}
}
if (!response.fromCache) {
response.cachePolicy = new CachePolicy(options_, response, options_);
response.fromCache = false;
}
let clonedResponse;
if (options_.cache && response.cachePolicy.storable()) {
clonedResponse = cloneResponse(response);
(async () => {
try {
const bodyPromise = getStreamAsBuffer(response);
await Promise.race([
requestErrorPromise,
new Promise((resolve) => response.once("end", resolve)),
new Promise((resolve) => response.once("close", resolve)),
]);
const body = await bodyPromise;
let value = {
url: response.url,
statusCode: response.fromCache
? revalidate.statusCode
: response.statusCode,
body,
cachePolicy: response.cachePolicy.toObject(),
};
let ttl = options_.strictTtl
? response.cachePolicy.timeToLive()
: undefined;
if (options_.maxTtl) {
ttl = ttl ? Math.min(ttl, options_.maxTtl) : options_.maxTtl;
}
if (this.hooks.size > 0) {
for (const key_ of this.hooks.keys()) {
value = await this.runHook(key_, value, response);
}
}
await this.cache.set(key, value, ttl);
/* c8 ignore next -- @preserve */
}
catch (error) {
/* c8 ignore next -- @preserve */
ee.emit("error", new CacheError(error));
/* c8 ignore next -- @preserve */
}
})();
}
else if (options_.cache && revalidate) {
(async () => {
try {
await this.cache.delete(key);
/* c8 ignore next -- @preserve */
}
catch (error) {
/* c8 ignore next -- @preserve */
ee.emit("error", new CacheError(error));
/* c8 ignore next -- @preserve */
}
})();
}
ee.emit("response", clonedResponse ?? response);
if (typeof callback === "function") {
callback(clonedResponse ?? response);
}
};
try {
const request_ = this.cacheRequest(options_, handler);
request_.once("error", requestErrorCallback);
request_.once("abort", requestErrorCallback);
request_.once("destroy", requestErrorCallback);
ee.emit("request", request_);
}
catch (error) {
ee.emit("error", new RequestError(error));
}
};
(async () => {
const get = async (options_) => {
await Promise.resolve();
const cacheEntry = options_.cache
? await this.cache.get(key)
: undefined;
if (cacheEntry === undefined && !options_.forceRefresh) {
makeRequest(options_);
return;
}
const policy = CachePolicy.fromObject(cacheEntry.cachePolicy);
if (policy.satisfiesWithoutRevalidation(options_) &&
!options_.forceRefresh) {
const headers = convertHeaders(policy.responseHeaders());
const bodyBuffer = cacheEntry.body;
const body = Buffer.from(bodyBuffer);
const response = new Response({
statusCode: cacheEntry.statusCode,
headers,
body,
url: cacheEntry.url,
});
response.cachePolicy = policy;
response.fromCache = true;
ee.emit("response", response);
if (typeof callback === "function") {
callback(response);
}
}
else if (policy.satisfiesWithoutRevalidation(options_) &&
Date.now() >= policy.timeToLive() &&
options_.forceRefresh) {
await this.cache.delete(key);
options_.headers = policy.revalidationHeaders(options_);
makeRequest(options_);
}
else {
revalidate = cacheEntry;
options_.headers = policy.revalidationHeaders(options_);
makeRequest(options_);
}
};
const errorHandler = (error) => ee.emit("error", new CacheError(error));
if (this.cache instanceof Keyv) {
const cachek = this.cache;
cachek.once("error", errorHandler);
ee.on("error", () => {
cachek.removeListener("error", errorHandler);
});
ee.on("response", () => {
cachek.removeListener("error", errorHandler);
});
}
try {
await get(options);
}
catch (error) {
/* v8 ignore next -- @preserve */
if (options.automaticFailover && !madeRequest) {
makeRequest(options);
}
ee.emit("error", new CacheError(error));
}
})();
return ee;
};
this.addHook = (name, function_) => {
if (!this.hooks.has(name)) {
this.hooks.set(name, function_);
}
};
this.removeHook = (name) => this.hooks.delete(name);
this.getHook = (name) => this.hooks.get(name);
this.runHook = async (name, ...arguments_) => this.hooks.get(name)?.(...arguments_);
if (cacheAdapter) {
if (cacheAdapter instanceof Keyv) {
this.cache = cacheAdapter;
}
else {
this.cache = new Keyv({
store: cacheAdapter,
namespace: "cacheable-request",
});
}
}
this.request = this.request.bind(this);
this.cacheRequest = cacheRequest;
}
}
const entries = Object.entries;
const cloneResponse = (response) => {
const clone = new PassThroughStream({ autoDestroy: false });
mimicResponse(response, clone);
return response.pipe(clone);
};
const urlObjectToRequestOptions = (url) => {
const options = { ...url };
options.path = `${url.pathname || "/"}${url.search || ""}`;
delete options.pathname;
delete options.search;
return options;
};
const normalizeUrlObject = (url) =>
// If url was parsed by url.parse or new URL:
// - hostname will be set
// - host will be hostname[:port]
// - port will be set if it was explicit in the parsed string
// Otherwise, url was from request options:
// - hostname or host may be set
// - host shall not have port encoded
({
protocol: url.protocol,
auth: url.auth,
hostname: url.hostname || url.host || "localhost",
port: url.port,
pathname: url.pathname,
search: url.search,
});
const convertHeaders = (headers) => {
const result = [];
for (const name of Object.keys(headers)) {
result[name.toLowerCase()] = headers[name];
}
return result;
};
export const parseWithWhatwg = (raw) => {
const u = new URL(raw);
// If normalizeUrlObject expects the same fields as url.parse()
return {
protocol: u.protocol, // E.g. 'https:'
slashes: true, // Always true for WHATWG URLs
/* c8 ignore next 3 */
auth: u.username || u.password ? `${u.username}:${u.password}` : undefined,
host: u.host, // E.g. 'example.com:8080'
port: u.port, // E.g. '8080'
hostname: u.hostname, // E.g. 'example.com'
hash: u.hash, // E.g. '#quux'
search: u.search, // E.g. '?bar=baz'
query: Object.fromEntries(u.searchParams), // { bar: 'baz' }
pathname: u.pathname, // E.g. '/foo'
path: u.pathname + u.search, // '/foo?bar=baz'
href: u.href, // Full serialized URL
};
};
export default CacheableRequest;
export * from "./types.js";
export const onResponse = "onResponse";
//# sourceMappingURL=index.js.map