@unito/integration-sdk/src/resources/provider.ts

Integration SDK

import https from 'https';
import FormData from 'form-data';

import { buildHttpError } from '../errors.js';
import * as HttpErrors from '../httpErrors.js';
import { Credentials } from '../middlewares/credentials.js';
import Logger from '../resources/logger.js';

/**
 * RateLimiter is a wrapper function that you can provide to limit the rate of calls to the provider based on the
 * caller's credentials.
 *
 * When necessary, the Provider's Response headers can be inspected to update the rate limit before being returned.
 *
 * NOTE: make sure to return one of the supported HttpErrors from the SDK, otherwise the error will be translated to a
 * generic server (500) error.
 *
 * @param options - The credentials and the logger from the RequestOptions passed with the provider call.
 * @param targetFunction - The function to call the provider.
 * @returns The response from the provider.
 * @throws RateLimitExceededError when the rate limit is exceeded.
 * @throws WouldExceedRateLimitError when the next call would exceed the rate limit.
 * @throws HttpError when the provider returns an error.
 */
export type RateLimiter = <T>(
  options: { credentials: Credentials; logger: Logger },
  targetFunction: () => Promise<Response<T>>,
) => Promise<Response<T>>;

/**
 * RequestOptions are the options passed to the Provider's call.
 *
 * @property credentials - The credentials to use for the call.
 * @property logger - The logger to use during the call.
 * @property queryParams - The query parameters to add when calling the provider.
 * @property additionnalheaders - The headers to add when calling the provider.
 * @property rawBody - Whether to return the raw response body.
 */
export type RequestOptions = {
  credentials: Credentials;
  logger: Logger;
  signal?: AbortSignal;
  queryParams?: { [key: string]: string };
  additionnalheaders?: { [key: string]: string };
  rawBody?: boolean;
};

/**
 * Response object returned by the Provider's method.
 *
 * Contains;
 * - the body typed as specified when calling the method
 * - the status code of the response
 * - the headers of the response.
 */
export type Response<T> = {
  body: T;
  status: number;
  headers: Headers;
};

export type PreparedRequest = {
  url: string;
  headers: Record<string, string>;
};

export type RequestBody = Record<string, unknown> | RequestBody[];

/**
 * The Provider class is a wrapper around the fetch function to call a provider's HTTP API.
 *
 * Defines methods for the following HTTP methods: GET, POST, PUT, PATCH, DELETE.
 *
 * Needs to be initialized with a prepareRequest function to define the Provider's base URL and any specific headers to
 * add to the requests, can also be configured to use a provided rate limiting function, and custom error handler.
 *
 * Multiple `Provider` instances can be created, with different configurations to call different providers APIs with
 * different rateLimiting functions, as needed.
 * @see {@link RateLimiter}
 * @see {@link prepareRequest}
 * @see {@link customErrorHandler}
 */
export class Provider {
  /**
   * The Rate Limiter function to use to limit the rate of calls made to the provider based on the caller's credentials.
   */
  protected rateLimiter: RateLimiter | undefined = undefined;
  /**
   * Function called before each request to define the Provider's base URL and any specific headers to add to the requests.
   *
   * This is applied at large to all requests made to the provider. If you need to add specific headers to a single request,
   * pass it through the RequestOptions object when calling the Provider's methods.
   */
  protected prepareRequest: (options: {
    credentials: Credentials;
    logger: Logger;
  }) => PreparedRequest | Promise<PreparedRequest>;
  /**
   * (Optional) Custom error handler to handle specific errors returned by the provider.
   *
   * If provided, this method should only care about custom errors returned by the provider and return the corresponding
   * HttpError from the SDK. If the error encountered is a standard error, it should return undefined and let the SDK handle it.
   *
   * @see buildHttpError for the list of standard errors the SDK can handle.
   */
  protected customErrorHandler:
    | ((
        responseStatus: number,
        message: string,
        options: {
          credentials: Credentials;
          logger: Logger;
        },
      ) => HttpErrors.HttpError | undefined)
    | undefined;

  /**
   * Initializes a Provider with the given options.
   *
   * @property {@link prepareRequest} - function to define the Provider's base URL and specific headers to add to the request.
   * @property {@link RateLimiter} - function to limit the rate of calls to the provider based on the caller's credentials.
   * @property {@link customErrorHandler} - function to handle specific errors returned by the provider.
   */
  constructor(options: {
    prepareRequest: typeof Provider.prototype.prepareRequest;
    rateLimiter?: RateLimiter | undefined;
    customErrorHandler?: typeof Provider.prototype.customErrorHandler;
  }) {
    this.prepareRequest = options.prepareRequest;
    this.rateLimiter = options.rateLimiter;
    this.customErrorHandler = options.customErrorHandler;
  }

  /**
   * Performs a GET request to the provider.
   *
   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
   * adds the following headers:
   * - Accept: application/json
   *
   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
   * @param options RequestOptions used to adjust the call made to the provider (use to override default headers).
   * @returns The {@link Response} extracted from the provider.
   */
  public async get<T>(endpoint: string, options: RequestOptions): Promise<Response<T>> {
    return this.fetchWrapper<T>(endpoint, null, {
      ...options,
      method: 'GET',
      defaultHeaders: {
        Accept: 'application/json',
      },
    });
  }

  /**
   * Performs a GET request to the provider and return the response as a ReadableStream.
   *
   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
   * adds the following headers:
   * - Accept: application/octet-stream
   *
   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
   * @param options RequestOptions used to adjust the call made to the provider (e.g. used to override default headers).
   * @returns The streaming {@link Response} extracted from the provider.
   */
  public async streamingGet(endpoint: string, options: RequestOptions): Promise<Response<ReadableStream<Uint8Array>>> {
    return this.fetchWrapper<ReadableStream<Uint8Array>>(endpoint, null, {
      ...options,
      method: 'GET',
      defaultHeaders: {
        Accept: 'application/octet-stream',
      },
      rawBody: true,
    });
  }

  /**
   * Performs a POST request to the provider.
   *
   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
   * adds the following headers:
   * - Content-Type: application/json',
   * - Accept: application/json
   *
   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
   * @param options RequestOptions used to adjust the call made to the provider (use to override default headers).
   * @returns The {@link Response} extracted from the provider.
   */
  public async post<T>(endpoint: string, body: RequestBody, options: RequestOptions): Promise<Response<T>> {
    return this.fetchWrapper<T>(endpoint, body, {
      ...options,
      method: 'POST',
      defaultHeaders: {
        'Content-Type': 'application/json',
        Accept: 'application/json',
      },
    });
  }

  public async postForm<T>(endpoint: string, form: FormData, options: RequestOptions): Promise<Response<T>> {
    const { url: providerUrl, headers: providerHeaders } = await this.prepareRequest(options);
    const absoluteUrl = this.generateAbsoluteUrl(providerUrl, endpoint, options.queryParams);
    const headers = { ...form.getHeaders(), ...providerHeaders, ...options.additionnalheaders };

    const reqOptions = {
      method: 'POST',
      headers,
    };
    /**
     * For some obscure reason we can't use the fetch API to send a form data, so we have to use the native https module
     * It seems that there is a miscalculation of the Content-Length headers that generates an error :
     *  --> headers length is different from the actual body length
     * The goto solution recommended across the internet for this, is to simply drop the header.
     * However, some integrations like Servicenow, will not accept the request if it doesn't contain that header
     */
    const callToProvider = async (): Promise<Response<T>> => {
      return new Promise((resolve, reject) => {
        try {
          const request = https.request(absoluteUrl, reqOptions, response => {
            response.setEncoding('utf8');
            let responseBody = '';

            response.on('data', chunk => {
              responseBody += chunk;
            });

            response.on('end', () => {
              try {
                const body = JSON.parse(responseBody);
                if (body.error) {
                  reject(this.handleError(400, body.error.message, options));
                }
                resolve({ status: 201, headers: response.headers as unknown as Headers, body: body as T });
              } catch (error) {
                reject(this.handleError(500, `Failed to parse response body: "${error}"`, options));
              }
            });
          });

          request.on('error', error => {
            reject(this.handleError(400, `Error while calling the provider: "${error}"`, options));
          });

          form.pipe(request);
        } catch (error) {
          reject(this.handleError(500, `Unexpected error while calling the provider: "${error}"`, options));
        }
      });
    };

    return this.rateLimiter ? this.rateLimiter(options, callToProvider) : callToProvider();
  }

  /**
   * Performs a PUT request to the provider.
   *
   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
   * adds the following headers:
   * - Content-Type: application/json',
   * - Accept: application/json
   *
   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
   * @param options RequestOptions used to adjust the call made to the provider (use to override default headers).
   * @returns The {@link Response} extracted from the provider.
   */
  public async put<T>(endpoint: string, body: RequestBody, options: RequestOptions): Promise<Response<T>> {
    return this.fetchWrapper<T>(endpoint, body, {
      ...options,
      method: 'PUT',
      defaultHeaders: {
        'Content-Type': 'application/json',
        Accept: 'application/json',
      },
    });
  }

  /**
   * Performs a PUT request to the provider with a Buffer body, typically used for sending binary data.
   *
   * IMPORTANT: This method should ONLY be used as a last resort when FormData cannot be used.
   * It bypasses normal form handling and is used to **manually send chunked** binary data, which may not be appropriate
   * for all providers. Always be mindful not to load entire binary files in memory!
   *
   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
   * adds the following headers:
   * - Content-Type: application/octet-stream
   * - Accept: application/json
   *
   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
   * @param body The Buffer containing the binary data to be sent.
   * @param options RequestOptions used to adjust the call made to the provider (use to override default headers).
   * @returns The {@link Response} extracted from the provider.
   */
  public async putBuffer<T>(endpoint: string, body: Buffer, options: RequestOptions): Promise<Response<T>> {
    return this.fetchWrapper<T>(endpoint, body, {
      ...options,
      method: 'PUT',
      defaultHeaders: {
        'Content-Type': 'application/octet-stream',
        Accept: 'application/json',
      },
    });
  }

  /**
   * Performs a PATCH request to the provider.
   *
   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
   * adds the following headers:
   * - Content-Type: application/json',
   * - Accept: application/json
   *
   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
   * @param options RequestOptions used to adjust the call made to the provider (use to override default headers).
   * @returns The {@link Response} extracted from the provider.
   */
  public async patch<T>(endpoint: string, body: RequestBody, options: RequestOptions): Promise<Response<T>> {
    return this.fetchWrapper<T>(endpoint, body, {
      ...options,
      method: 'PATCH',
      defaultHeaders: {
        'Content-Type': 'application/json',
        Accept: 'application/json',
      },
    });
  }

  /**
   * Performs a DELETE request to the provider.
   *
   * Uses the prepareRequest function to get the base URL and any specific headers to add to the request and by default
   * adds the following headers:
   * - Accept: application/json
   *
   * @param endpoint Path to the provider's resource. Will be added to the URL returned by the prepareRequest function.
   * @param options RequestOptions used to adjust the call made to the provider (use to override default headers).
   * @returns The {@link Response} extracted from the provider.
   */
  public async delete<T = undefined>(
    endpoint: string,
    options: RequestOptions,
    body: RequestBody | null = null,
  ): Promise<Response<T>> {
    const defaultHeaders: { Accept: string; 'Content-Type'?: string } = {
      Accept: 'application/json',
    };

    // Only add Content-Type header when body is provided
    if (body !== null) {
      defaultHeaders['Content-Type'] = 'application/json';
    }

    return this.fetchWrapper<T>(endpoint, body, {
      ...options,
      method: 'DELETE',
      defaultHeaders,
    });
  }

  private generateAbsoluteUrl(providerUrl: string, endpoint: string, queryParams?: { [key: string]: string }): string {
    let absoluteUrl;

    if (/^https?:\/\//.test(endpoint)) {
      absoluteUrl = endpoint;
    } else {
      absoluteUrl = [providerUrl, endpoint.charAt(0) === '/' ? endpoint.substring(1) : endpoint].join('/');
    }

    if (queryParams) {
      absoluteUrl = `${absoluteUrl}?${new URLSearchParams(queryParams)}`;
    }
    return absoluteUrl;
  }

  private async fetchWrapper<T>(
    endpoint: string,
    body: RequestBody | Buffer | null,
    options: RequestOptions & { defaultHeaders: { 'Content-Type'?: string; Accept?: string }; method: string },
  ): Promise<Response<T>> {
    const { url: providerUrl, headers: providerHeaders } = await this.prepareRequest(options);
    const absoluteUrl = this.generateAbsoluteUrl(providerUrl, endpoint, options.queryParams);
    const headers = { ...options.defaultHeaders, ...providerHeaders, ...options.additionnalheaders };

    let fetchBody: string | Buffer | null = null;

    if (body) {
      if (headers['Content-Type'] === 'application/x-www-form-urlencoded') {
        fetchBody = new URLSearchParams(body as Record<string, string>).toString();
      } else if (
        headers['Content-Type'] === 'application/json' ||
        headers['Content-Type'] === 'application/json-patch+json'
      ) {
        fetchBody = JSON.stringify(body);
      } else if (headers['Content-Type'] === 'application/octet-stream' && body instanceof Buffer) {
        fetchBody = body;
      } else {
        throw this.handleError(400, `Content type not supported: ${headers['Content-Type']}`, options);
      }
    }

    const callToProvider = async (): Promise<Response<T>> => {
      const beforeRequestTimestamp = process.hrtime.bigint();

      let response: globalThis.Response;

      try {
        response = await fetch(absoluteUrl, {
          method: options.method,
          headers,
          body: fetchBody,
          ...(options.signal ? { signal: options.signal } : {}),
        });
      } catch (error) {
        if (error instanceof Error) {
          switch (error.name) {
            case 'AbortError':
              throw this.handleError(408, 'Request aborted', options);
            case 'TimeoutError':
              throw this.handleError(408, 'Request timeout', options);
          }

          throw this.handleError(
            500,
            `Unexpected error while calling the provider. ErrorName: "${error.name}" \n message: "${error.message}" \n stack: ${error.stack} \n cause: ${error.cause} \n causeStack: ${(error.cause as Error)?.stack}`,
            options,
          );
        }

        throw this.handleError(
          500,
          'Unexpected error while calling the provider - this is not normal, investigate',
          options,
        );
      }

      const afterRequestTimestamp = process.hrtime.bigint();
      const requestDurationInNS = Number(afterRequestTimestamp - beforeRequestTimestamp);
      const requestDurationInMs = (requestDurationInNS / 1_000_000) | 0;

      options.logger.info(
        `Connector API Request ${options.method} ${absoluteUrl} ${response.status} - ${requestDurationInMs} ms`,
        {
          duration: requestDurationInNS,
          http: {
            method: options.method,
            status_code: response.status,
            content_type: headers['Content-Type'],
            url_details: {
              path: absoluteUrl,
            },
          },
        },
      );

      if (response.status >= 400) {
        const textResult = await response.text();
        throw this.handleError(response.status, textResult, options);
      } else if (response.status === 204 || response.body === null) {
        // No content: return without inspecting the body
        return { status: response.status, headers: response.headers, body: undefined as unknown as T };
      }

      const responseContentType = response.headers.get('content-type');
      let body: T;

      if (options.rawBody || headers.Accept === 'application/octet-stream') {
        // When we expect octet-stream, we accept any Content-Type the provider sends us, we just want to stream it
        body = response.body as T;
      } else if (headers.Accept?.match(/application\/.*json/)) {
        // Validate that the response content type is at least similar to what we expect
        // (Provider's response Content-Type might be more specific, e.g. application/json;charset=utf-8)
        // Default to application/json if no Content-Type header is provided
        if (responseContentType && !responseContentType.match(/application\/.*json/)) {
          const textResult = await response.text();
          throw this.handleError(
            500,
            `Unsupported content-type, expected 'application/json' but got '${responseContentType}'.
             Original response (${response.status}): "${textResult}"`,
            options,
          );
        }

        try {
          body = response.body ? await response.json() : undefined;
        } catch (err) {
          throw this.handleError(500, `Invalid JSON response`, options);
        }
      } else if (headers.Accept?.includes('text/html')) {
        // Accept text based content types
        body = (await response.text()) as T;
      } else {
        throw this.handleError(500, 'Unsupported Content-Type', options);
      }

      return { status: response.status, headers: response.headers, body };
    };

    return this.rateLimiter ? this.rateLimiter(options, callToProvider) : callToProvider();
  }

  private handleError(responseStatus: number, message: string, options: RequestOptions): HttpErrors.HttpError {
    const customError = this.customErrorHandler?.(responseStatus, message, options);

    return customError ?? buildHttpError(responseStatus, message);
  }
}