@bitrix24/b24jssdk/dist/cjs/core/actions/v2/fetch-list.cjs.map

Bitrix24 REST API JavaScript SDK

{"version":3,"file":"fetch-list.cjs","sources":["../../../../../src/core/actions/v2/fetch-list.ts"],"sourcesContent":["import type { WalkBoundsOptions } from '../_walk-bounds'\nimport { assertNotAborted, maxPagesExceededError, resolveMaxPages } from '../_walk-bounds'\nimport type { TypeCallParams, TypeCallParamsV2, TypeFilterV2 } from '../../../types/http'\nimport type { AjaxResult } from '../../http/ajax-result'\nimport { AbstractAction } from '../abstract-action'\nimport { SdkError } from '../../sdk-error'\nimport { cursorStalledError, cursorWentBackwardsError, CURSOR_STALLED_HINT_LIST } from '../_cursor-stalled'\nimport { cursorProgressed } from '../_cursor-progress'\nimport { warnOnShadowedUppercaseParams } from './_uppercase-list-params'\n\nexport type ActionFetchListV2 = WalkBoundsOptions & {\n  /** REST list method that returns rows, e.g. `crm.item.list`, `tasks.task.list`. */\n  method: string\n  /**\n   * Request parameters. `start` and `order` are excluded: the walk writes both\n   * itself on every page. Use `filter` and `select` to narrow the selection.\n   *\n   * Conditions must go in the **lowercase** `filter`, with any uppercase\n   * `FILTER` removed — see {@link warnOnShadowedUppercaseParams} for what the\n   * portal does with two keys differing only by case, and why it is silent.\n   */\n  params?: Omit<TypeCallParamsV2, 'start' | 'order'>\n  /**\n   * Name of the id field **as it appears in each response item**; its value\n   * drives the cursor. Default `'ID'` — `crm.item.list` and other camelCase\n   * methods return `id`, so they need `idKey: 'id'`.\n   */\n  idKey?: string\n  /**\n   * Field name used in the **request**, for `order` and the `>` page filter.\n   * Defaults to `idKey`. Set it only when a method spells the id differently in\n   * the two: `tasks.task.list` sorts and filters by `ID` but returns `id`, so it\n   * needs `idKey: 'id', cursorIdKey: 'ID'`.\n   */\n  cursorIdKey?: string\n  /** Key the rows are nested under in the response, e.g. `items` for CRM items. */\n  customKeyForResult?: string\n  /**\n   * Sent as the `bx24_request_id` query parameter, for tracing. It does **not**\n   * deduplicate anything — for that see `idempotencyKey` on `restApi:v3`.\n   */\n  requestId?: string\n}\n\n/**\n * Calls a REST API list method and returns an async generator for efficient large data retrieval. `restApi:v2`\n *\n * Iterates through all pages of a v2 list method using cursor-based pagination and yields each\n * page as an array, allowing callers to process records incrementally without holding the entire\n * dataset in memory. Unlike `CallListV2`, which accumulates all pages before returning, this\n * class exposes an `AsyncGenerator` so processing can begin as soon as the first page arrives.\n */\nexport class FetchListV2 extends AbstractAction {\n  /**\n   * Calls a REST API list method and returns an async generator, for walking a\n   * large dataset without holding all of it in memory.\n   *\n   * **Every option is documented on the [fetchList v2 page](https://bitrix24.github.io/b24jssdk/docs/working-with-the-rest-api/fetch-list-rest-api-ver2/),\n   * and each one on {@link ActionFetchListV2}.** Not repeated here: nothing\n   * watches a sentence in a comment, while the page is link-checked and its\n   * code compiled on every CI run.\n   *\n   * What matters while editing this file:\n   *\n   * - The cursor only advances if rows arrive sorted by `cursorIdKey` ascending,\n   *   so the walk writes its own `order` and strips a caller's with a `warning`.\n   * - `idKey` reads the RESPONSE, `cursorIdKey` writes the REQUEST, and the two\n   *   fail differently. A wrong `cursorIdKey` means the page condition never\n   *   matches, the same page keeps arriving, and the walk stops with\n   *   {@link cursorStalledError}. A wrong `idKey` is quieter: if the value\n   *   cannot be read as a number the walk warns and stops short, and if it\n   *   names a *different numeric* field it advances a cursor the request never\n   *   sorts by — which skips rows rather than reporting anything.\n   * - Page size on `restApi:v2` is a fixed 50 — there is no `limit` to ask with,\n   *   so a page shorter than that ends the walk, but only once the cursor read\n   *   from it has been vouched for.\n   * - The portal keeps only the later of two top-level keys differing by case,\n   *   so a caller's uppercase `FILTER` can drop either their conditions or this\n   *   walk's cursor. Both are warned about, neither can be fixed from in here —\n   *   see {@link warnOnShadowedUppercaseParams}.\n   * - `maxPages` never ends a walk silently. Every page up to the ceiling has\n   *   already been yielded and is the consumer's; the throw is what stops a\n   *   truncated walk from reading as a finished one.\n   *\n   * @template T - The type of items in the returned arrays (default is `unknown`).\n   * @param {ActionFetchListV2} options - every field is documented on the type.\n   * @returns {AsyncGenerator<T[]>} An async generator yielding one page of rows\n   *     at a time until the dataset is exhausted.\n   *\n   * @example\n   * import { EnumCrmEntityTypeId, Text } from '@bitrix24/b24jssdk'\n   *\n   * interface CrmItem { id: number, title: string }\n   * const sixMonthAgo = new Date()\n   * sixMonthAgo.setMonth((new Date()).getMonth() - 6)\n   * sixMonthAgo.setHours(0, 0, 0)\n   * const generator = b24.actions.v2.fetchList.make<CrmItem>({\n   *   method: 'crm.item.list',\n   *   params: {\n   *     entityTypeId: EnumCrmEntityTypeId.company,\n   *     filter: {\n   *       '=%title': 'A%',\n   *       '>=createdTime': Text.toB24Format(sixMonthAgo) // created at least 6 months ago\n   *     },\n   *     select: ['id', 'title']\n   *   },\n   *   idKey: 'id',\n   *   customKeyForResult: 'items',\n   *   requestId: 'list-123'\n   * })\n   *\n   * for await (const chunk of generator) {\n   *   // Process chunk (e.g., save to database, analyze, etc.)\n   *   console.log(`Processing ${chunk.length} items`)\n   * }\n   *\n   * @see {@link https://apidocs.bitrix24.com/settings/performance/huge-data.html Bitrix24: Fast algorithm for large data}\n   */\n  public override async* make<T = unknown>(options: ActionFetchListV2): AsyncGenerator<T[]> {\n    const batchSize = 50\n\n    const idKey = options?.idKey ?? 'ID'\n    const cursorIdKey = options?.cursorIdKey ?? idKey\n    const customKeyForResult = options?.customKeyForResult ?? null\n    const params = options?.params ?? {}\n\n    // Warn and strip user-provided `order` — cursor pagination requires ordering by cursorIdKey only\n    if ('order' in params && params['order']) {\n      this._logger.warning('fetchList.make: user-provided `order` parameter is ignored because cursor-based pagination requires ordering by cursorIdKey. Use `filter` to narrow results instead.').catch(() => {})\n    }\n\n    const moreIdKey = `>${cursorIdKey}`\n    const { order: _ignoredOrder, ...restParams } = params as TypeCallParams\n    const requestParams: TypeCallParamsV2 & { filter: TypeFilterV2 } = {\n      ...restParams,\n      order: { [cursorIdKey]: 'ASC' },\n      filter: { ...(params['filter'] || {}), [moreIdKey]: 0 },\n      start: -1\n    }\n\n    warnOnShadowedUppercaseParams('fetchList.make', requestParams as Record<string, unknown>, this._logger)\n\n    let pages = 0\n    const maxPages = resolveMaxPages('fetchList.make', options?.maxPages)\n\n    while (true) {\n      assertNotAborted(options?.signal, 'fetchList.make', options.method)\n      const response: AjaxResult<T> = await this._b24.actions.v2.call.make<T>({\n        method: options.method,\n        params: requestParams,\n        requestId: options.requestId\n      })\n\n      if (!response.isSuccess) {\n        this._logger.error('fetchList.make', {\n          method: options.method,\n          requestId: options.requestId,\n          messages: response.getErrorMessages()\n        }).catch(() => {})\n        throw new SdkError({\n          code: 'JSSDK_CORE_B24_FETCH_LIST_METHOD_API_V2',\n          description: `API Error: ${response.getErrorMessages().join('; ')}`,\n          status: 500\n        })\n      }\n      const responseData = response.getData()\n      if (!responseData) {\n        break\n      }\n\n      const resultData: T[] = null === customKeyForResult\n        ? responseData.result as T[]\n        : (responseData.result as any)[customKeyForResult] as T[]\n\n      if (resultData.length === 0) {\n        break\n      }\n\n      pages += 1\n      yield resultData\n      // Again after the page has been handed over. The gap between `yield` and\n      // the top of the loop is not empty - the stall guard and the ceiling both\n      // sit in it - so a consumer that aborts while holding a page would\n      // otherwise be told the read is too large, or that the cursor stalled,\n      // when what actually happened is that they cancelled. No request is saved\n      // by this check; the correct diagnosis is.\n      assertNotAborted(options?.signal, 'fetchList.make', options.method)\n\n      // Read rather than acted on: the cursor is checked first, and only\n      // then does a short page end the walk. A page shorter than the one\n      // asked for used to end it as \"end of data\" whatever the cursor did, so\n      // a stalled page that happened to be capped returned a truncated,\n      // overlapping result and reported success (#496). The two are\n      // separable: `>idKey` asks for rows strictly past the cursor, so a row\n      // at or before it cannot be in an answer that honoured the condition,\n      // however short the page.\n      const isShortPage = resultData.length < batchSize\n\n      // Update the filter for the next iteration\n      const lastItem = resultData[resultData.length - 1] as Record<string, any>\n      const cursorValue = lastItem ? Number.parseInt(lastItem[idKey], 10) : Number.NaN\n      if (Number.isFinite(cursorValue)) {\n        // See the note in `v2/call-list.ts`: a full page whose last id equals\n        // the one already filtered on means `>idKey` was dropped, and the same\n        // page repeats for ever. Here the pages have already been yielded, so\n        // the consumer holds the duplicates — the error says so.\n        if (cursorValue === requestParams.filter[moreIdKey]) {\n          throw cursorStalledError('fetchList.make', CURSOR_STALLED_HINT_LIST)\n        }\n\n        // …and a cursor that moved the wrong way. A server alternating between\n        // two pages never repeats the immediately preceding value, so the\n        // check above never fires and the walk runs for ever (#495). Every\n        // cycle steps backwards somewhere, and this is that step.\n        if (!cursorProgressed(cursorValue, requestParams.filter[moreIdKey] as number, 'ASC')) {\n          throw cursorWentBackwardsError('fetchList.make', CURSOR_STALLED_HINT_LIST)\n        }\n\n        // End of data, now that the cursor has been vouched for.\n        if (isShortPage) {\n          break\n        }\n\n        requestParams.filter[moreIdKey] = cursorValue\n\n        // Last, so every cheaper stop wins: a stalled cursor is still reported\n        // as a stall, the more specific diagnosis. A walk that ends exactly on\n        // its ceiling finishes only when its final page is short — that is what\n        // proves end-of-data. On an exact multiple of the page size the last\n        // page is full, nothing has proved the data ended, and this fires; the\n        // rows read are returned with the error attached, not discarded.\n        if (pages >= maxPages) {\n          throw maxPagesExceededError('fetchList.make', options.method, maxPages)\n        }\n      } else {\n        // No usable numeric cursor id could be read from the page's items via\n        // `idKey` — almost always an `idKey` that doesn't match the\n        // response field (e.g. a request that sorts by `ID` while the response\n        // carries a lowercase `id`). Without a cursor we can't advance, so stop and\n        // tell the caller how to fix it instead of silently truncating.\n        // A short page is simply the end of the data, so say nothing about a\n        // cursor the walk never needed.\n        if (!isShortPage) {\n          this._logger.warning(`fetchList.make: pagination stops here — no numeric id could be read from the returned items via idKey \"${idKey}\". Make sure idKey matches the id field in the response; if the sortable field name differs from it, also set cursorIdKey (e.g. idKey: 'id', cursorIdKey: 'ID').`).catch(() => {})\n        }\n        break\n      }\n    }\n  }\n}\n"],"names":["AbstractAction","warnOnShadowedUppercaseParams","resolveMaxPages","assertNotAborted","SdkError","cursorStalledError","CURSOR_STALLED_HINT_LIST","cursorProgressed","cursorWentBackwardsError","maxPagesExceededError"],"mappings":";;;;;;;;;;;;;;;;;;;AAoDO,MAAM,oBAAoBA,6BAAA,CAAe;AAAA,EApDhD;AAoDgD,IAAA,MAAA,CAAA,IAAA,EAAA,aAAA,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAkE9C,OAAuB,KAAkB,OAAA,EAAiD;AACxF,IAAA,MAAM,SAAA,GAAY,EAAA;AAElB,IAAA,MAAM,KAAA,GAAQ,SAAS,KAAA,IAAS,IAAA;AAChC,IAAA,MAAM,WAAA,GAAc,SAAS,WAAA,IAAe,KAAA;AAC5C,IAAA,MAAM,kBAAA,GAAqB,SAAS,kBAAA,IAAsB,IAAA;AAC1D,IAAA,MAAM,MAAA,GAAS,OAAA,EAAS,MAAA,IAAU,EAAC;AAGnC,IAAA,IAAI,OAAA,IAAW,MAAA,IAAU,MAAA,CAAO,OAAO,CAAA,EAAG;AACxC,MAAA,IAAA,CAAK,OAAA,CAAQ,OAAA,CAAQ,sKAAsK,CAAA,CAAE,MAAM,MAAM;AAAA,MAAC,CAAC,CAAA;AAAA,IAC7M;AAEA,IAAA,MAAM,SAAA,GAAY,IAAI,WAAW,CAAA,CAAA;AACjC,IAAA,MAAM,EAAE,KAAA,EAAO,aAAA,EAAe,GAAG,YAAW,GAAI,MAAA;AAChD,IAAA,MAAM,aAAA,GAA6D;AAAA,MACjE,GAAG,UAAA;AAAA,MACH,KAAA,EAAO,EAAE,CAAC,WAAW,GAAG,KAAA,EAAM;AAAA,MAC9B,MAAA,EAAQ,EAAE,GAAI,MAAA,CAAO,QAAQ,CAAA,IAAK,EAAC,EAAI,CAAC,SAAS,GAAG,CAAA,EAAE;AAAA,MACtD,KAAA,EAAO;AAAA,KACT;AAEA,IAAAC,kDAAA,CAA8B,gBAAA,EAAkB,aAAA,EAA0C,IAAA,CAAK,OAAO,CAAA;AAEtG,IAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,IAAA,MAAM,QAAA,GAAWC,2BAAA,CAAgB,gBAAA,EAAkB,OAAA,EAAS,QAAQ,CAAA;AAEpE,IAAA,OAAO,IAAA,EAAM;AACX,MAAAC,4BAAA,CAAiB,OAAA,EAAS,MAAA,EAAQ,gBAAA,EAAkB,OAAA,CAAQ,MAAM,CAAA;AAClE,MAAA,MAAM,WAA0B,MAAM,IAAA,CAAK,KAAK,OAAA,CAAQ,EAAA,CAAG,KAAK,IAAA,CAAQ;AAAA,QACtE,QAAQ,OAAA,CAAQ,MAAA;AAAA,QAChB,MAAA,EAAQ,aAAA;AAAA,QACR,WAAW,OAAA,CAAQ;AAAA,OACpB,CAAA;AAED,MAAA,IAAI,CAAC,SAAS,SAAA,EAAW;AACvB,QAAA,IAAA,CAAK,OAAA,CAAQ,MAAM,gBAAA,EAAkB;AAAA,UACnC,QAAQ,OAAA,CAAQ,MAAA;AAAA,UAChB,WAAW,OAAA,CAAQ,SAAA;AAAA,UACnB,QAAA,EAAU,SAAS,gBAAA;AAAiB,SACrC,CAAA,CAAE,KAAA,CAAM,MAAM;AAAA,QAAC,CAAC,CAAA;AACjB,QAAA,MAAM,IAAIC,iBAAA,CAAS;AAAA,UACjB,IAAA,EAAM,yCAAA;AAAA,UACN,aAAa,CAAA,WAAA,EAAc,QAAA,CAAS,kBAAiB,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA;AAAA,UACjE,MAAA,EAAQ;AAAA,SACT,CAAA;AAAA,MACH;AACA,MAAA,MAAM,YAAA,GAAe,SAAS,OAAA,EAAQ;AACtC,MAAA,IAAI,CAAC,YAAA,EAAc;AACjB,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,aAAkB,IAAA,KAAS,kBAAA,GAC7B,aAAa,MAAA,GACZ,YAAA,CAAa,OAAe,kBAAkB,CAAA;AAEnD,MAAA,IAAI,UAAA,CAAW,WAAW,CAAA,EAAG;AAC3B,QAAA;AAAA,MACF;AAEA,MAAA,KAAA,IAAS,CAAA;AACT,MAAA,MAAM,UAAA;AAON,MAAAD,4BAAA,CAAiB,OAAA,EAAS,MAAA,EAAQ,gBAAA,EAAkB,OAAA,CAAQ,MAAM,CAAA;AAUlE,MAAA,MAAM,WAAA,GAAc,WAAW,MAAA,GAAS,SAAA;AAGxC,MAAA,MAAM,QAAA,GAAW,UAAA,CAAW,UAAA,CAAW,MAAA,GAAS,CAAC,CAAA;AACjD,MAAA,MAAM,WAAA,GAAc,WAAW,MAAA,CAAO,QAAA,CAAS,SAAS,KAAK,CAAA,EAAG,EAAE,CAAA,GAAI,MAAA,CAAO,GAAA;AAC7E,MAAA,IAAI,MAAA,CAAO,QAAA,CAAS,WAAW,CAAA,EAAG;AAKhC,QAAA,IAAI,WAAA,KAAgB,aAAA,CAAc,MAAA,CAAO,SAAS,CAAA,EAAG;AACnD,UAAA,MAAME,iCAAA,CAAmB,kBAAkBC,uCAAwB,CAAA;AAAA,QACrE;AAMA,QAAA,IAAI,CAACC,iCAAiB,WAAA,EAAa,aAAA,CAAc,OAAO,SAAS,CAAA,EAAa,KAAK,CAAA,EAAG;AACpF,UAAA,MAAMC,uCAAA,CAAyB,kBAAkBF,uCAAwB,CAAA;AAAA,QAC3E;AAGA,QAAA,IAAI,WAAA,EAAa;AACf,UAAA;AAAA,QACF;AAEA,QAAA,aAAA,CAAc,MAAA,CAAO,SAAS,CAAA,GAAI,WAAA;AAQlC,QAAA,IAAI,SAAS,QAAA,EAAU;AACrB,UAAA,MAAMG,iCAAA,CAAsB,gBAAA,EAAkB,OAAA,CAAQ,MAAA,EAAQ,QAAQ,CAAA;AAAA,QACxE;AAAA,MACF,CAAA,MAAO;AAQL,QAAA,IAAI,CAAC,WAAA,EAAa;AAChB,UAAA,IAAA,CAAK,QAAQ,OAAA,CAAQ,CAAA,4GAAA,EAA0G,KAAK,CAAA,gKAAA,CAAkK,CAAA,CAAE,MAAM,MAAM;AAAA,UAAC,CAAC,CAAA;AAAA,QACxT;AACA,QAAA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;;"}