@zhengxs/http
Version:
A lightweight cross-platform http request library
914 lines (789 loc) • 25.6 kB
text/typescript
import {
type Agent,
fetch,
getDefaultAgent,
type HeadersInit,
type RequestInfo,
type RequestInit,
kind as shimsKind,
} from './_shims/index';
import {
APIConnectionError,
APIConnectionTimeoutError,
APIError,
APIUserAbortError,
castToError,
HttpException,
} from './error';
import { Stream } from './streaming';
import type {
APIResponseProps,
DefaultQuery,
Fetch,
FinalRequestOptions,
HTTPMethod,
PromiseOrValue,
ReqHeaders,
RequestClient,
RequestOptions,
} from './types';
import { isMultipartBody } from './uploads';
import {
createResponseHeaders,
debug,
isAbsoluteURL,
isEmptyObj,
safeJSON,
sleep,
uuid4,
validatePositiveInteger,
} from './util';
export {
maybeMultipartFormRequestOptions,
multipartFormRequestOptions,
createForm,
type Uploadable,
} from './uploads';
export async function defaultParseResponse<T>(
props: APIResponseProps,
): Promise<T> {
const { response } = props;
if (props.options.stream) {
debug(
'response',
response.status,
response.url,
response.headers,
response.body,
);
// Note: there is an invariant here that isn't represented in the type system
// that if you set `stream: true` the response type must also be `Stream<T>`
return Stream.fromSSEResponse(response, props.controller) as any;
}
// fetch refuses to read the body when the status code is 204.
if (response.status === 204) {
return null as T;
}
if (props.options.__binaryResponse) {
return response as unknown as T;
}
const contentType = response.headers.get('content-type');
if (contentType?.includes('application/json')) {
const json = await response.json();
debug('response', response.status, response.url, response.headers, json);
return json as T;
}
const text = await response.text();
debug('response', response.status, response.url, response.headers, text);
// TODO handle blob, arraybuffer, other content types, etc.
return text as unknown as T;
}
/**
* A subclass of `Promise` providing additional helper methods
* for interacting with the SDK.
*/
export class APIPromise<T> extends Promise<T> {
private parsedPromise: Promise<T> | undefined;
constructor(
private responsePromise: Promise<APIResponseProps>,
private parseResponse: (
props: APIResponseProps,
) => PromiseOrValue<T> = defaultParseResponse,
) {
super(resolve => {
// this is maybe a bit weird but this has to be a no-op to not implicitly
// parse the response body; instead .then, .catch, .finally are overridden
// to parse the response
resolve(null as any);
});
}
_thenUnwrap<U>(transform: (data: T) => U): APIPromise<U> {
return new APIPromise(this.responsePromise, async props =>
transform(await this.parseResponse(props)),
);
}
/**
* Gets the raw `Response` instance instead of parsing the response
* data.
*
* If you want to parse the response body but still get the `Response`
* instance, you can use {@link withResponse()}.
*/
asResponse(): Promise<Response> {
return this.responsePromise.then(p => p.response);
}
/**
* Gets the parsed response data and the raw `Response` instance.
*
* If you just want to get the raw `Response` instance without parsing it,
* you can use {@link asResponse()}.
*/
async withResponse(): Promise<{ data: T; response: Response }> {
const [data, response] = await Promise.all([
this.parse(),
this.asResponse(),
]);
return { data, response };
}
private parse(): Promise<T> {
if (!this.parsedPromise) {
this.parsedPromise = this.responsePromise.then(this.parseResponse);
}
return this.parsedPromise;
}
override then<TResult1 = T, TResult2 = never>(
onfulfilled?:
| ((value: T) => TResult1 | PromiseLike<TResult1>)
| undefined
| null,
onrejected?:
| ((reason: any) => TResult2 | PromiseLike<TResult2>)
| undefined
| null,
): Promise<TResult1 | TResult2> {
return this.parse().then(onfulfilled, onrejected);
}
override catch<TResult = never>(
onrejected?:
| ((reason: any) => TResult | PromiseLike<TResult>)
| undefined
| null,
): Promise<T | TResult> {
return this.parse().catch(onrejected);
}
override finally(onfinally?: (() => void) | undefined | null): Promise<T> {
return this.parse().finally(onfinally);
}
}
export interface APIClientOptions {
baseURL: string;
maxRetries?: number | undefined;
timeout?: number | undefined;
httpAgent?: Agent | undefined;
fetch?: Fetch | undefined;
}
export class APIClient {
baseURL: string;
maxRetries: number;
timeout: number;
httpAgent: Agent | undefined;
protected fetch: Fetch;
protected idempotencyHeader?: string;
constructor({
baseURL,
maxRetries = 2,
timeout = 600000, // 10 minutes
httpAgent,
fetch: overrideFetch,
}: APIClientOptions) {
this.baseURL = baseURL;
this.maxRetries = validatePositiveInteger('maxRetries', maxRetries);
this.timeout = validatePositiveInteger('timeout', timeout);
this.httpAgent = httpAgent;
this.fetch = overrideFetch ?? fetch;
}
/**
* Override this to add your own auth headers.
*
* ```ts
* {
* Authorization: 'Bearer 123',
* }
* ```
*/
protected authHeaders(
_opts: FinalRequestOptions,
): PromiseOrValue<ReqHeaders> {
return {};
}
/**
* Override this to add your own default headers.
*/
protected async defaultHeaders(
opts: FinalRequestOptions,
): Promise<ReqHeaders> {
const authHeaders = await this.authHeaders(opts);
return {
Accept: 'application/json',
'Content-Type': 'application/json',
'User-Agent': this.getUserAgent(),
...authHeaders,
};
}
protected defaultQuery(): DefaultQuery | undefined {
return undefined;
}
/**
* Override this to add your own headers validation:
*/
protected validateHeaders(_headers: ReqHeaders, _customHeaders: ReqHeaders) {}
protected defaultIdempotencyKey(): string {
return `stainless-node-retry-${uuid4()}`;
}
get<Req extends NonNullable<unknown>, Rsp>(
path: string,
opts?: PromiseOrValue<RequestOptions<Req>>,
): APIPromise<Rsp> {
return this.methodRequest('get', path, opts);
}
post<Req extends NonNullable<unknown>, Rsp>(
path: string,
opts?: PromiseOrValue<RequestOptions<Req>>,
): APIPromise<Rsp> {
return this.methodRequest('post', path, opts);
}
patch<Req extends NonNullable<unknown>, Rsp>(
path: string,
opts?: PromiseOrValue<RequestOptions<Req>>,
): APIPromise<Rsp> {
return this.methodRequest('patch', path, opts);
}
put<Req extends NonNullable<unknown>, Rsp>(
path: string,
opts?: PromiseOrValue<RequestOptions<Req>>,
): APIPromise<Rsp> {
return this.methodRequest('put', path, opts);
}
delete<Req extends NonNullable<unknown>, Rsp>(
path: string,
opts?: PromiseOrValue<RequestOptions<Req>>,
): APIPromise<Rsp> {
return this.methodRequest('delete', path, opts);
}
private methodRequest<Req extends NonNullable<unknown>, Rsp>(
method: HTTPMethod,
path: string,
opts?: PromiseOrValue<RequestOptions<Req>>,
): APIPromise<Rsp> {
return this.request(
Promise.resolve(opts).then(opts => ({ method, path, ...opts })),
);
}
getAPIList<Item, PageClass extends AbstractPage<Item> = AbstractPage<Item>>(
path: string,
Page: new (...args: any[]) => PageClass,
opts?: RequestOptions<any>,
): PagePromise<PageClass, Item> {
return this.requestAPIList(Page, { method: 'get', path, ...opts });
}
private calculateContentLength(body: unknown): string | null {
if (typeof body === 'string') {
if (typeof Buffer !== 'undefined') {
return Buffer.byteLength(body, 'utf8').toString();
}
if (typeof TextEncoder !== 'undefined') {
const encoder = new TextEncoder();
const encoded = encoder.encode(body);
return encoded.length.toString();
}
}
return null;
}
protected async buildRequest<Req extends NonNullable<unknown>>(
options: FinalRequestOptions<Req>,
): Promise<{ req: RequestInit; url: string; timeout: number }> {
const { method, path, query, headers: headers = {} } = options;
const body = isMultipartBody(options.body)
? options.body.body
: options.body
? JSON.stringify(options.body, null, 2)
: null;
const contentLength = this.calculateContentLength(body);
const url = this.buildURL(path!, query);
if ('timeout' in options)
validatePositiveInteger('timeout', options.timeout);
const timeout = options.timeout ?? this.timeout;
const httpAgent =
options.httpAgent ?? this.httpAgent ?? getDefaultAgent!(url);
const minAgentTimeout = timeout + 1000;
if (
typeof (httpAgent as any)?.options?.timeout === 'number' &&
minAgentTimeout > ((httpAgent as any).options.timeout ?? 0)
) {
// Allow any given request to bump our agent active socket timeout.
// This may seem strange, but leaking active sockets should be rare and not particularly problematic,
// and without mutating agent we would need to create more of them.
// This tradeoff optimizes for performance.
(httpAgent as any).options.timeout = minAgentTimeout;
}
if (this.idempotencyHeader && method !== 'get') {
if (!options.idempotencyKey)
options.idempotencyKey = this.defaultIdempotencyKey();
headers[this.idempotencyHeader] = options.idempotencyKey;
}
const defaultHeaders = await this.defaultHeaders(options);
const reqHeaders: Record<string, string> = {
...(contentLength && { 'Content-Length': contentLength }),
...defaultHeaders,
...headers,
};
// let builtin fetch set the Content-Type for multipart bodies
if (isMultipartBody(options.body) && shimsKind !== 'node') {
delete reqHeaders['Content-Type'];
}
// Strip any headers being explicitly omitted with null
Object.keys(reqHeaders).forEach(
key => reqHeaders[key] === null && delete reqHeaders[key],
);
const req: RequestInit = {
method,
...(body && { body: body as any }),
headers: reqHeaders,
duplex: options.duplex,
...(httpAgent && { agent: httpAgent }),
// @ts-ignore node-fetch uses a custom AbortSignal type that is
// not compatible with standard web types
signal: options.signal ?? null,
};
this.validateHeaders(reqHeaders, headers);
return { req, url, timeout };
}
/**
* Used as a callback for mutating the given `RequestInit` object.
*
* This is useful for cases where you want to add certain headers based off of
* the request properties, e.g. `method` or `url`.
*/
protected async prepareRequest(
_request: RequestInit,
_config: { url: string; options: FinalRequestOptions },
): Promise<void> {}
protected parseHeaders(
headers: HeadersInit | null | undefined,
): Record<string, string> {
return !headers
? {}
: Symbol.iterator in headers
? Object.fromEntries(
Array.from(headers as Iterable<string[]>).map(header => [
...header,
]),
)
: { ...headers };
}
protected makeStatusError(
status: number | undefined,
error: NonNullable<unknown> | undefined,
message: string | undefined,
headers: ReqHeaders | undefined,
) {
return APIError.generate(status, error, message, headers);
}
request<Req extends NonNullable<unknown>, Rsp>(
options: PromiseOrValue<FinalRequestOptions<Req>>,
remainingRetries: number | null = null,
): APIPromise<Rsp> {
return new APIPromise(this.makeRequest(options, remainingRetries));
}
protected async makeRequest(
optionsInput: PromiseOrValue<FinalRequestOptions>,
retriesRemaining: number | null,
): Promise<APIResponseProps> {
const options = await optionsInput;
if (retriesRemaining == null) {
retriesRemaining = options.maxRetries ?? this.maxRetries;
}
const { req, url, timeout } = await this.buildRequest(options);
await this.prepareRequest(req, { url, options });
debug('request', url, options, req.headers);
if (options.signal?.aborted) {
throw new APIUserAbortError();
}
const controller = new AbortController();
const response = await this.fetchWithTimeout(
url,
req,
timeout,
controller,
).catch(castToError);
if (response instanceof Error) {
if (options.signal?.aborted) {
throw new APIUserAbortError();
}
if (retriesRemaining) {
return this.retryRequest(options, retriesRemaining);
}
if (response.name === 'AbortError') {
throw new APIConnectionTimeoutError();
}
throw new APIConnectionError({ cause: response });
}
const responseHeaders = createResponseHeaders(response.headers);
if (!response.ok) {
if (retriesRemaining && this.shouldRetry(response)) {
return this.retryRequest(options, retriesRemaining, responseHeaders);
}
const errText = await response.text().catch(e => castToError(e).message);
const errJSON = safeJSON(errText);
const errMessage = errJSON ? undefined : errText;
debug('response', response.status, url, responseHeaders, errMessage);
const err = this.makeStatusError(
response.status,
errJSON,
errMessage,
responseHeaders,
);
throw err;
}
return { response, options, controller };
}
simple<Req extends NonNullable<unknown>, Rsp>(
path: string,
opts?: PromiseOrValue<RequestOptions<Req>>,
): APIPromise<Rsp> {
const optionsInput = Promise.resolve(opts).then<FinalRequestOptions<Req>>(
opts => ({ method: 'get', path, ...opts }),
);
return new APIPromise(this.makeSimpleRequest(optionsInput));
}
protected async makeSimpleRequest(
optionsInput: PromiseOrValue<FinalRequestOptions>,
retriesRemaining?: number | null,
): Promise<APIResponseProps> {
const options = await optionsInput;
if (retriesRemaining == null) {
retriesRemaining = options.maxRetries ?? this.maxRetries;
}
const body = isMultipartBody(options.body)
? options.body.body
: options.body
? JSON.stringify(options.body, null, 2)
: null;
// @ts-expect-error
const url = this.buildURL(options.path!, options.query);
if ('timeout' in options) {
validatePositiveInteger('timeout', options.timeout);
}
const timeout = options.timeout ?? this.timeout;
const httpAgent =
options.httpAgent ?? this.httpAgent ?? getDefaultAgent!(url);
const minAgentTimeout = timeout + 1000;
if (
typeof (httpAgent as any)?.options?.timeout === 'number' &&
minAgentTimeout > ((httpAgent as any).options.timeout ?? 0)
) {
// Allow any given request to bump our agent active socket timeout.
// This may seem strange, but leaking active sockets should be rare and not particularly problematic,
// and without mutating agent we would need to create more of them.
// This tradeoff optimizes for performance.
(httpAgent as any).options.timeout = minAgentTimeout;
}
const req: RequestInit = {
method: options.method || 'get',
...(body && { body: body as any }),
headers: options.headers,
...(httpAgent && { agent: httpAgent }),
// @ts-ignore node-fetch uses a custom AbortSignal type that is
// not compatible with standard web types
signal: options.signal ?? null,
};
debug('request', url, options, req.headers);
const controller = new AbortController();
const response = await this.fetchWithTimeout(
url,
req,
timeout,
controller,
).catch(castToError);
if (response instanceof Error) {
if (req.signal?.aborted) {
throw new APIUserAbortError();
}
if (response.name === 'AbortError') {
throw new APIConnectionTimeoutError();
}
throw new APIConnectionError({ cause: response });
}
const responseHeaders = createResponseHeaders(response.headers);
if (!response.ok) {
const errText = await response.text().catch(e => castToError(e).message);
const errJSON = safeJSON(errText);
const errMessage = errJSON ? undefined : errText;
debug('response', response.status, url, responseHeaders, errMessage);
const err = this.makeStatusError(
response.status,
errJSON,
errMessage,
responseHeaders,
);
throw err;
}
return { response, options, controller };
}
requestAPIList<
Item = unknown,
PageClass extends AbstractPage<Item> = AbstractPage<Item>,
>(
Page: new (
...args: ConstructorParameters<typeof AbstractPage>
) => PageClass,
options: FinalRequestOptions,
): PagePromise<PageClass, Item> {
const request = this.makeRequest(options, null);
return new PagePromise<PageClass, Item>(this, request, Page);
}
buildURL<Req extends Record<string, unknown>>(
path: string,
query: Req | null | undefined,
): string {
const url = isAbsoluteURL(path)
? new URL(path)
: new URL(
this.baseURL +
(this.baseURL.endsWith('/') && path.startsWith('/')
? path.slice(1)
: path),
);
const defaultQuery = this.defaultQuery();
if (!isEmptyObj(defaultQuery)) {
query = { ...defaultQuery, ...query } as Req;
}
if (query) {
url.search = this.stringifyQuery(query);
}
return url.toString();
}
protected stringifyQuery(query: Record<string, unknown>): string {
return Object.entries(query)
.filter(([_, value]) => typeof value !== 'undefined')
.map(([key, value]) => {
if (
typeof value === 'string' ||
typeof value === 'number' ||
typeof value === 'boolean'
) {
return `${encodeURIComponent(key)}=${encodeURIComponent(value)}`;
}
if (value === null) {
return `${encodeURIComponent(key)}=`;
}
throw new HttpException(
`Cannot stringify type ${typeof value}; Expected string, number, boolean, or null. If you need to pass nested query parameters, you can manually encode them, e.g. { query: { 'foo[key1]': value1, 'foo[key2]': value2 } }, and please open a GitHub issue requesting better support for your use case.`,
);
})
.join('&');
}
async fetchWithTimeout(
url: RequestInfo,
init: RequestInit | undefined,
ms: number,
controller: AbortController,
): Promise<Response> {
const { signal, ...options } = init || {};
if (signal) signal.addEventListener('abort', () => controller.abort());
const timeout = setTimeout(() => controller.abort(), ms);
return (
this.getRequestClient()
// use undefined this binding; fetch errors if bound to something else in browser/cloudflare
.fetch.call(undefined, url, {
signal: controller.signal as any,
...options,
})
.finally(() => {
clearTimeout(timeout);
})
);
}
protected getRequestClient(): RequestClient {
return { fetch: this.fetch };
}
private shouldRetry(response: Response): boolean {
// Note this is not a standard header.
const shouldRetryHeader = response.headers.get('x-should-retry');
// If the server explicitly says whether or not to retry, obey.
if (shouldRetryHeader === 'true') return true;
if (shouldRetryHeader === 'false') return false;
// Retry on request timeouts.
if (response.status === 408) return true;
// Retry on lock timeouts.
if (response.status === 409) return true;
// Retry on rate limits.
if (response.status === 429) return true;
// Retry internal errors.
if (response.status >= 500) return true;
return false;
}
private async retryRequest(
options: FinalRequestOptions,
retriesRemaining: number,
responseHeaders?: ReqHeaders | undefined,
): Promise<APIResponseProps> {
// About the Retry-After header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
let timeoutMillis: number | undefined;
const retryAfterHeader = responseHeaders?.['retry-after'];
if (retryAfterHeader) {
const timeoutSeconds = parseInt(retryAfterHeader);
if (!Number.isNaN(timeoutSeconds)) {
timeoutMillis = timeoutSeconds * 1000;
} else {
timeoutMillis = Date.parse(retryAfterHeader) - Date.now();
}
}
// If the API asks us to wait a certain amount of time (and it's a reasonable amount),
// just do what it says, but otherwise calculate a default
if (
!timeoutMillis ||
!Number.isInteger(timeoutMillis) ||
timeoutMillis <= 0 ||
timeoutMillis > 60 * 1000
) {
const maxRetries = options.maxRetries ?? this.maxRetries;
timeoutMillis = this.calculateDefaultRetryTimeoutMillis(
retriesRemaining,
maxRetries,
);
}
await sleep(timeoutMillis);
return this.makeRequest(options, retriesRemaining - 1);
}
private calculateDefaultRetryTimeoutMillis(
retriesRemaining: number,
maxRetries: number,
): number {
const initialRetryDelay = 0.5;
const maxRetryDelay = 8.0;
const numRetries = maxRetries - retriesRemaining;
// Apply exponential backoff, but not more than the max.
const sleepSeconds = Math.min(
initialRetryDelay * Math.pow(2, numRetries),
maxRetryDelay,
);
// Apply some jitter, take up to at most 25 percent of the retry time.
const jitter = 1 - Math.random() * 0.25;
return sleepSeconds * jitter * 1000;
}
protected getUserAgent(): string {
return `${this.constructor.name}/JS`;
}
static create(
baseURL: string,
options?: Omit<APIClientOptions, 'baseURL'>,
): APIClient {
return new APIClient({ baseURL, ...options });
}
}
export type PageInfo =
| { url: URL }
| { params: Record<string, unknown> | null };
export abstract class AbstractPage<Item> implements AsyncIterable<Item> {
#client: APIClient;
protected options: FinalRequestOptions;
protected response: Response;
protected body: unknown;
constructor(
client: APIClient,
response: Response,
body: unknown,
options: FinalRequestOptions,
) {
this.#client = client;
this.options = options;
this.response = response;
this.body = body;
}
/**
* @deprecated Use nextPageInfo instead
*/
abstract nextPageParams(): Partial<Record<string, unknown>> | null;
abstract nextPageInfo(): PageInfo | null;
abstract getPaginatedItems(): Item[];
hasNextPage(): boolean {
const items = this.getPaginatedItems();
if (!items.length) return false;
return this.nextPageInfo() != null;
}
async getNextPage(): Promise<this> {
const nextInfo = this.nextPageInfo();
if (!nextInfo) {
throw new HttpException(
'No next page expected; please check `.hasNextPage()` before calling `.getNextPage()`.',
);
}
const nextOptions = { ...this.options };
if ('params' in nextInfo) {
nextOptions.query = { ...nextOptions.query, ...nextInfo.params };
} else if ('url' in nextInfo) {
const params = [
...Object.entries(nextOptions.query || {}),
...nextInfo.url.searchParams.entries(),
];
for (const [key, value] of params) {
nextInfo.url.searchParams.set(key, value as any);
}
nextOptions.query = undefined;
nextOptions.path = nextInfo.url.toString();
}
return await this.#client.requestAPIList(
this.constructor as any,
nextOptions,
);
}
async *iterPages() {
// eslint-disable-next-line @typescript-eslint/no-this-alias
let page: AbstractPage<Item> = this;
yield page;
while (page.hasNextPage()) {
page = await page.getNextPage();
yield page;
}
}
async *[Symbol.asyncIterator]() {
for await (const page of this.iterPages()) {
for (const item of page.getPaginatedItems()) {
yield item;
}
}
}
}
/**
* This subclass of Promise will resolve to an instantiated Page once the request completes.
*
* It also implements AsyncIterable to allow auto-paginating iteration on an unawaited list call, eg:
*
* ```ts
* for await (const item of client.items.list()) {
* console.log(item)
* }
* ```
*/
export class PagePromise<
PageClass extends AbstractPage<Item>,
Item = ReturnType<PageClass['getPaginatedItems']>[number],
>
extends APIPromise<PageClass>
implements AsyncIterable<Item>
{
constructor(
client: APIClient,
request: Promise<APIResponseProps>,
Page: new (
...args: ConstructorParameters<typeof AbstractPage>
) => PageClass,
) {
super(
request,
async props =>
new Page(
client,
props.response,
await defaultParseResponse(props),
props.options,
),
);
}
/**
* Allow auto-paginating iteration on an un awaited list call, eg:
*
* ```ts
* for await (const item of client.items.list()) {
* console.log(item)
* }
* ```
*/
async *[Symbol.asyncIterator]() {
const page = await this;
for await (const item of page) {
yield item;
}
}
}