@genkit-ai/ai/lib/formats/jsonl.mjs.map

Genkit AI framework generative AI APIs.

{"version":3,"sources":["../../src/formats/jsonl.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { GenkitError } from '@genkit-ai/core';\nimport JSON5 from 'json5';\nimport { extractJson } from '../extract';\nimport type { Formatter } from './types';\n\nfunction objectLines(text: string): string[] {\n  return text\n    .split('\\n')\n    .map((line) => line.trim())\n    .filter((line) => line.startsWith('{'));\n}\n\nexport const jsonlFormatter: Formatter<unknown[], unknown[]> = {\n  name: 'jsonl',\n  config: {\n    contentType: 'application/jsonl',\n  },\n  handler: (schema) => {\n    if (\n      schema &&\n      (schema.type !== 'array' || schema.items?.type !== 'object')\n    ) {\n      throw new GenkitError({\n        status: 'INVALID_ARGUMENT',\n        message: `Must supply an 'array' schema type containing 'object' items when using the 'jsonl' parser format.`,\n      });\n    }\n\n    let instructions: string | undefined;\n    if (schema?.items) {\n      instructions = `Output should be JSONL format, a sequence of JSON objects (one per line) separated by a newline \\`\\\\n\\` character. Each line should be a JSON object conforming to the following schema:\n\n\\`\\`\\`\n${JSON.stringify(schema.items)}\n\\`\\`\\`\n    `;\n    }\n\n    return {\n      parseChunk: (chunk) => {\n        const results: unknown[] = [];\n\n        const text = chunk.accumulatedText;\n\n        let startIndex = 0;\n        if (chunk.previousChunks?.length) {\n          const lastNewline = chunk.previousText.lastIndexOf('\\n');\n          if (lastNewline !== -1) {\n            startIndex = lastNewline + 1;\n          }\n        }\n\n        const lines = text.slice(startIndex).split('\\n');\n\n        for (const line of lines) {\n          const trimmed = line.trim();\n          if (trimmed.startsWith('{')) {\n            try {\n              const result = JSON5.parse(trimmed);\n              if (result) {\n                results.push(result);\n              }\n            } catch (e) {\n              break;\n            }\n          }\n        }\n\n        return results;\n      },\n\n      parseMessage: (message) => {\n        const items = objectLines(message.text)\n          .map((l) => extractJson(l))\n          .filter((l) => !!l);\n\n        return items;\n      },\n\n      instructions,\n    };\n  },\n};\n"],"mappings":"AAgBA,SAAS,mBAAmB;AAC5B,OAAO,WAAW;AAClB,SAAS,mBAAmB;AAG5B,SAAS,YAAY,MAAwB;AAC3C,SAAO,KACJ,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,WAAW,GAAG,CAAC;AAC1C;AAEO,MAAM,iBAAkD;AAAA,EAC7D,MAAM;AAAA,EACN,QAAQ;AAAA,IACN,aAAa;AAAA,EACf;AAAA,EACA,SAAS,CAAC,WAAW;AACnB,QACE,WACC,OAAO,SAAS,WAAW,OAAO,OAAO,SAAS,WACnD;AACA,YAAM,IAAI,YAAY;AAAA,QACpB,QAAQ;AAAA,QACR,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,QAAI;AACJ,QAAI,QAAQ,OAAO;AACjB,qBAAe;AAAA;AAAA;AAAA,EAGnB,KAAK,UAAU,OAAO,KAAK,CAAC;AAAA;AAAA;AAAA,IAG1B;AAEA,WAAO;AAAA,MACL,YAAY,CAAC,UAAU;AACrB,cAAM,UAAqB,CAAC;AAE5B,cAAM,OAAO,MAAM;AAEnB,YAAI,aAAa;AACjB,YAAI,MAAM,gBAAgB,QAAQ;AAChC,gBAAM,cAAc,MAAM,aAAa,YAAY,IAAI;AACvD,cAAI,gBAAgB,IAAI;AACtB,yBAAa,cAAc;AAAA,UAC7B;AAAA,QACF;AAEA,cAAM,QAAQ,KAAK,MAAM,UAAU,EAAE,MAAM,IAAI;AAE/C,mBAAW,QAAQ,OAAO;AACxB,gBAAM,UAAU,KAAK,KAAK;AAC1B,cAAI,QAAQ,WAAW,GAAG,GAAG;AAC3B,gBAAI;AACF,oBAAM,SAAS,MAAM,MAAM,OAAO;AAClC,kBAAI,QAAQ;AACV,wBAAQ,KAAK,MAAM;AAAA,cACrB;AAAA,YACF,SAAS,GAAG;AACV;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAEA,eAAO;AAAA,MACT;AAAA,MAEA,cAAc,CAAC,YAAY;AACzB,cAAM,QAAQ,YAAY,QAAQ,IAAI,EACnC,IAAI,CAAC,MAAM,YAAY,CAAC,CAAC,EACzB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AAEpB,eAAO;AAAA,MACT;AAAA,MAEA;AAAA,IACF;AAAA,EACF;AACF;","names":[]}