@clickup/rest-client/src/RestRequest.ts

A syntax sugar tool around Node fetch() API, tailored to work with TypeScript and response validators

import dns from "dns";
import { promisify } from "util";
import { Memoize } from "fast-typescript-memoize";
import { parse } from "ipaddr.js";
import random from "lodash/random";
import type { RequestInit, RequestRedirect } from "node-fetch";
import { Headers } from "node-fetch";
import RestContentSizeOverLimitError from "./errors/RestContentSizeOverLimitError";
import RestError from "./errors/RestError";
import RestTimeoutError from "./errors/RestTimeoutError";
import calcRetryDelay from "./internal/calcRetryDelay";
import inspectPossibleJSON from "./internal/inspectPossibleJSON";
import RestFetchReader from "./internal/RestFetchReader";
import throwIfErrorResponse from "./internal/throwIfErrorResponse";
import toFloatMs from "./internal/toFloatMs";
import type RestOptions from "./RestOptions";
import type { RestLogEvent } from "./RestOptions";
import RestResponse from "./RestResponse";
import RestStream from "./RestStream";

const MAX_DEBUG_LEN = 1024 * 100;

/**
 * Type TAssertShape allows to limit json()'s assert callbacks to only those
 * which return an object compatible with TAssertShape.
 */
export default class RestRequest<TAssertShape = any> {
  readonly options: RestOptions;

  constructor(
    options: RestOptions,
    public readonly method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE",
    public url: string,
    public readonly headers: Headers,
    public readonly body: string | Buffer | NodeJS.ReadableStream,
    public readonly shape?: string,
  ) {
    this.options = { ...options };
  }

  /**
   * Modifies the request by adding a custom HTTP header.
   */
  setHeader(name: string, value: string) {
    this.headers.append(name, value);
    return this;
  }

  /**
   * Modifies the request by adding a custom request option.
   */
  setOptions(options: Partial<RestOptions>) {
    for (const [k, v] of Object.entries(options)) {
      (this.options as any)[k] = v;
    }

    return this;
  }

  /**
   * Forces RestClient to debug-output the request and response to console.
   * Never use in production.
   */
  setDebug(flag = true) {
    this.options.isDebug = flag;
    return this;
  }

  /**
   * Sends the request and reads the response a JSON. In absolute most of the
   * cases, this method is used to reach API responses. The assert callback
   * (typically generated by typescript-is) is intentionally made mandatory to
   * not let people to do anti-patterns.
   */
  async json<TJson extends TAssertShape>(
    assert:
      | ((obj: any) => TJson)
      | { mask(obj: any): TJson }
      | { $assert(obj: any): TJson },
    ...checkers: Array<(json: TJson, res: RestResponse) => false | Error>
  ): Promise<TJson> {
    const res = await this.response();

    // Support Superstruct in a duck-typing way.
    if (typeof assert !== "function") {
      if ("mask" in assert) {
        assert = assert.mask.bind(assert);
      } else if ("$assert" in assert) {
        assert = assert.$assert.bind(assert);
      }
    }

    const json = assert(res.json);
    for (const checker of checkers) {
      const error = checker(json, res);
      if (error !== false) {
        throw error;
      }
    }

    return json as any;
  }

  /**
   * Sends the request and returns plaintext response.
   */
  async text() {
    const res = await this.response();
    return res.text;
  }

  /**
   * Returns the entire RestResponse object with response status and headers
   * information in it. Try to minimize usage of this method, because it doesn't
   * make any assumptions on the response structure.
   */
  @Memoize()
  async response(): Promise<RestResponse> {
    const stream = await this.stream(
      // By passing Number.MAX_SAFE_INTEGER to stream(), we ensure that the
      // entire data will be preloaded, or the loading will fail due to
      // throwIfResIsBigger limitation, whichever will happen faster.
      Number.MAX_SAFE_INTEGER,
    );
    await stream.close();
    return stream.res;
  }

  /**
   * Sends the requests and returns RestStream object. You MUST iterate over
   * this object entirely (or call its return() method), otherwise the
   * connection will remain dangling.
   */
  async stream(preloadChars = 128 * 1024): Promise<RestStream> {
    const finalAttempt = Math.max(this.options.retries + 1, 1);
    let retryDelayMs = Math.max(this.options.retryDelayFirstMs, 10);

    for (let attempt = 1; attempt <= finalAttempt; attempt++) {
      let timeStart = process.hrtime();
      let res = null as RestResponse | null;

      try {
        // Middlewares work with RestResponse (can alter it) and not with
        // RestStream intentionally. So the goal here is to create a
        // RestResponse object, and then later derive a RestStream from it.
        let reader = null as RestFetchReader | null;
        res = await runMiddlewares(
          this,
          this.options.middlewares,
          async (req) => {
            reader = null;
            try {
              timeStart = process.hrtime();
              res = null;

              try {
                const fetchReq = await req._createFetchRequest();
                reader = req._createFetchReader(fetchReq);
                await reader.preload(preloadChars);
              } finally {
                res = reader
                  ? req._createRestResponse(reader)
                  : new RestResponse(req, null, 0, new Headers(), "", false);
              }

              throwIfErrorResponse(req.options, res);

              req._logResponse({
                attempt,
                req,
                res,
                exception: null,
                timestamp: Date.now(),
                elapsed: toFloatMs(process.hrtime(timeStart)),
                isFinalAttempt: attempt === finalAttempt,
                privateDataInResponse: req.options.privateDataInResponse,
                comment: "",
              });

              return res;
            } catch (e: unknown) {
              await reader?.close();
              throw e;
            }
          },
        );

        // The only place where we return the response. Otherwise we retry or
        // throw an exception.
        return new RestStream(res, reader!);
      } catch (error: unknown) {
        this._logResponse({
          attempt,
          req: this,
          res,
          exception: error,
          timestamp: Date.now(),
          elapsed: toFloatMs(process.hrtime(timeStart)),
          isFinalAttempt: attempt === finalAttempt,
          privateDataInResponse: this.options.privateDataInResponse,
          comment: "",
        });

        if (res === null) {
          // An error in internal function or middleware; this must not happen.
          throw error;
        }

        if (attempt === finalAttempt) {
          // Last retry attempt; always throw.
          throw error;
        }

        const newRetryDelay = calcRetryDelay(
          error,
          this.options,
          res,
          retryDelayMs,
        );
        if (newRetryDelay === "no_retry") {
          throw error;
        }

        retryDelayMs = newRetryDelay;
      }

      const delayStart = process.hrtime();
      retryDelayMs *= random(
        1 - this.options.retryDelayJitter,
        1 + this.options.retryDelayJitter,
        true,
      );
      await this.options.heartbeater.delay(retryDelayMs);

      this._logResponse({
        attempt,
        req: this,
        res: "backoff_delay",
        exception: null,
        timestamp: Date.now(),
        elapsed: toFloatMs(process.hrtime(delayStart)),
        isFinalAttempt: false,
        privateDataInResponse: this.options.privateDataInResponse,
        comment: "",
      });

      retryDelayMs *= this.options.retryDelayFactor;
      retryDelayMs = Math.min(this.options.retryDelayMaxMs, retryDelayMs);
    }

    throw Error("BUG: should never happen");
  }

  /**
   * We can actually only create RequestInit. We can't create an instance of
   * node-fetch.Request object since it doesn't allow injection of
   * AbortController later, and we don't want to deal with AbortController here.
   */
  private async _createFetchRequest(): Promise<RequestInit & { url: string }> {
    const url = new URL(this.url);
    if (!["http:", "https:"].includes(url.protocol)) {
      throw new RestError(`Unsupported protocol: ${url.protocol}`);
    }

    // TODO: rework DNS resolution to use a custom Agent. We'd be able to
    // support safe redirects then.

    // Resolve IP address, check it for being a public IP address and substitute
    // it in the URL; the hostname will be passed separately via Host header.
    const hostname = url.hostname;
    const headers = new Headers(this.headers);
    const addr = await promisify(dns.lookup)(hostname, {
      family: this.options.family,
    });
    let redirectMode: RequestRedirect = "follow";
    if (!this.options.allowInternalIPs) {
      const range = parse(addr.address).range();
      // External requests are returned as "unicast" by ipaddr.js.
      const isInternal = range !== "unicast";
      if (isInternal) {
        throw new RestError(
          `Domain ${hostname} resolves to a non-public (${range}) IP address ${addr.address}`,
        );
      }

      url.hostname = addr.address;
      if (!headers.get("host")) {
        headers.set("host", hostname);
      }

      // ATTENTION: don't turn on redirects, it's a security breach when using
      // with allowInternalIPs=false which is default!
      redirectMode = "error";
    }

    const hasKeepAlive = this.options.keepAlive.timeoutMs > 0;
    if (hasKeepAlive) {
      headers.append("connection", "Keep-Alive");
    }

    // Use lazily created/cached per-RestClient Agent instance to utilize HTTP
    // persistent connections and save on HTTPS connection re-establishment.
    const agent = (
      url.protocol === "https:"
        ? this.options.agents.https.bind(this.options.agents)
        : this.options.agents.http.bind(this.options.agents)
    )({
      keepAlive: hasKeepAlive,
      timeout: hasKeepAlive ? this.options.keepAlive.timeoutMs : undefined,
      maxSockets: this.options.keepAlive.maxSockets,
      rejectUnauthorized: this.options.allowInternalIPs ? false : undefined,
      family: this.options.family,
    });

    return {
      url: url.toString(),
      method: this.method,
      headers,
      body: this.body || undefined,
      timeout: this.options.timeoutMs,
      redirect: redirectMode,
      agent,
    };
  }

  /**
   * Creates an instance of RestFetchReader.
   */
  private _createFetchReader(req: RequestInit & { url: string }) {
    return new RestFetchReader(req.url, req, {
      timeoutMs: this.options.timeoutMs,
      heartbeat: async () => this.options.heartbeater.heartbeat(),
      onTimeout: (reader) => {
        throw new RestTimeoutError(
          `Timed out while reading response body (${this.options.timeoutMs} ms)`,
          this._createRestResponse(reader)!,
        );
      },
      onAfterRead: (reader) => {
        if (
          this.options.throwIfResIsBigger &&
          reader.charsRead > this.options.throwIfResIsBigger
        ) {
          throw new RestContentSizeOverLimitError(
            `Content size is over limit of ${this.options.throwIfResIsBigger} characters`,
            this._createRestResponse(reader)!,
          );
        }
      },
      responseEncoding: this.options.responseEncoding,
    });
  }

  /**
   * Creates a RestResponse from a RestFetchReader. Assumes that
   * RestFetchReader.preload() has already been called.
   */
  private _createRestResponse(reader: RestFetchReader) {
    return new RestResponse(
      this,
      reader.agent,
      reader.status,
      reader.headers,
      reader.textFetched,
      reader.textIsPartial,
    );
  }

  /**
   * Logs a response event, an error event or a backoff event. If RestResponse
   * is not yet known (e.g. an exception happened in a DNS resolution), res must
   * be passed as null.
   */
  private _logResponse(event: RestLogEvent) {
    this.options.logger(event);

    // Debug-logging to console?
    if (
      this.options.isDebug ||
      (process.env["NODE_ENV"] === "development" &&
        event.res === "backoff_delay")
    ) {
      let reqMessage =
        `${this.method} ${this.url}\n` +
        Object.entries(this.headers.raw())
          .map(([k, vs]) => vs.map((v) => `${k}: ${v}`))
          .join("\n");
      if (this.body) {
        reqMessage +=
          "\n" + inspectPossibleJSON(this.headers, this.body, MAX_DEBUG_LEN);
      }

      let resMessage = "";
      if (event.res === "backoff_delay") {
        resMessage =
          "Previous request failed, backoff delay elapsed, retrying attempt " +
          (event.attempt + 1) +
          "...";
      } else if (event.res) {
        resMessage =
          `HTTP ${event.res.status} ` +
          `(took ${Math.round(event.elapsed)} ms)`;
        if (event.res.text) {
          resMessage +=
            "\n" +
            inspectPossibleJSON(
              event.res.headers,
              event.res.text,
              MAX_DEBUG_LEN,
            );
        }
      } else if (event.exception) {
        resMessage = "" + event.exception;
      }

      // eslint-disable-next-line no-console
      console.log(reqMessage.replace(/^/gm, "+++ "));
      // eslint-disable-next-line no-console
      console.log(resMessage.replace(/^/gm, "=== ") + "\n");
    }
  }
}

/**
 * Runs the middlewares chain of responsibility. Each middleware receives a
 * mutable RestRequest object and next() callback which, when called, triggers
 * execution of the remaining middlewares in the chain. This allows a middleware
 * to not only modify the request object, but also alter the response received
 * from the subsequent middlewares.
 */
async function runMiddlewares(
  req: RestRequest,
  middlewares: RestOptions["middlewares"],
  last: (req: RestRequest) => Promise<RestResponse>,
): Promise<RestResponse> {
  if (middlewares.length > 0) {
    const [head, ...tail] = middlewares;
    return head(req, async (req) => runMiddlewares(req, tail, last));
  }

  return last(req);
}