wikibase-edit/src/lib/request/request.ts

Edit Wikibase from NodeJS

import { stringifyQuery, wait, type Query } from '../utils.js'
import checkKnownIssues from './check_known_issues.js'
import { customFetch, type HttpHeaders, type HttpMethod, type HttpRequestAgent } from './fetch.js'
import { getSignatureHeaders } from './oauth.js'
import { parseResponseBody } from './parse_response_body.js'
import type { PostData } from './post.js'
import type { AbsoluteUrl } from '../types/common.js'
import type { OAuthCredentials } from '../types/config.js'

const timeout = 30000

export interface RequestParams {
  url: AbsoluteUrl
  body?: Query | PostData
  oauth?: OAuthCredentials['oauth']
  headers?: HttpHeaders
  autoRetry?: boolean
  httpRequestAgent?: HttpRequestAgent
}

export async function request (method: HttpMethod, params: RequestParams) {
  method ??= 'get'
  const { url, body, oauth: oauthTokens, headers, autoRetry = true, httpRequestAgent } = params
  let maxlag
  if (typeof body === 'object' && 'maxlag' in body) {
    maxlag = body.maxlag
  }
  let attempts = 1

  let bodyStr
  if (method === 'post' && body != null) {
    bodyStr = stringifyQuery(body)
    headers['Content-Type'] = 'application/x-www-form-urlencoded'
  }

  async function tryRequest () {
    if (oauthTokens) {
      const signatureHeaders = getSignatureHeaders({
        url,
        method,
        data: body,
        oauthTokens,
      })
      Object.assign(headers, signatureHeaders)
    }

    try {
      const res = await customFetch(url, { method, body: bodyStr, headers, timeout, agent: httpRequestAgent })
      return await parseResponseBody(res)
    } catch (err) {
      checkKnownIssues(url, err)
      if (autoRetry === false) throw err
      if (errorIsWorthARetry(err)) {
        const delaySeconds = getRetryDelay(err.headers) * attempts
        retryWarn(method, url, err, delaySeconds, attempts++, maxlag)
        await wait(delaySeconds * 1000)
        return tryRequest()
      } else {
        err.context ??= {}
        err.context.request = { url, body }
        throw err
      }
    }
  }

  return tryRequest()
}

export interface APIResponseError {
  code: string
  info: string
}

function errorIsWorthARetry (err) {
  if (errorsWorthARetry.has(err.name) || errorsWorthARetry.has(err.type) || errorsCodeWorthARetry.has(err.code)) return true
  // failed-save might be a recoverable error from the server
  // See https://github.com/maxlath/wikibase-cli/issues/150
  if (err.name === 'failed-save') {
    const { messages } = err.body.error
    return !messages.some(isNonRecoverableFailedSave)
  }
  if (err.cause) return errorIsWorthARetry(err.cause)
  return false
}

const isNonRecoverableFailedSave = message => message.name.startsWith('wikibase-validator') || nonRecoverableFailedSaveMessageNames.has(message.name)

const errorsWorthARetry = new Set([
  'maxlag',
  'TimeoutError',
  'request-timeout',
  'wrong response format',
])

const errorsCodeWorthARetry = new Set([
  'ECONNREFUSED',
  'ENETUNREACH',
  'ENOTFOUND',
  'ETIMEDOUT',
  'UND_ERR_CONNECT_TIMEOUT',
])

const nonRecoverableFailedSaveMessageNames = new Set([
  'protectedpagetext',
  'permissionserrors',
])

const defaultRetryDelay = 5
function getRetryDelay (headers) {
  const retryAfterSeconds = headers?.['retry-after']
  if (/^\d+$/.test(retryAfterSeconds)) return parseInt(retryAfterSeconds)
  else return defaultRetryDelay
}

function retryWarn (method, url, err, delaySeconds, attempts, maxlag) {
  method = method.toUpperCase()
  const maxlagStr = typeof maxlag === 'number' ? `${maxlag}s` : maxlag
  console.warn(`[wikibase-edit][WARNING] ${method} ${url}
    ${err.message}${err.cause ? ` (cause: ${err.cause.message})` : ''}
    retrying in ${delaySeconds}s (attempt: ${attempts}, maxlag: ${maxlagStr})`)
}