@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 4.26 kB
Source Map (JSON)
{"version":3,"file":"chatgpt.cjs","names":["TextLoader","Document"],"sources":["../../../src/document_loaders/fs/chatgpt.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { TextLoader } from \"@langchain/classic/document_loaders/fs/text\";\n\ninterface ChatGPTMessage {\n author: {\n role: string;\n };\n content: {\n parts: string[];\n };\n create_time: number;\n}\n\ninterface ChatGPTLog {\n title: string;\n mapping: Record<string, { message: ChatGPTMessage }>;\n}\n\nfunction concatenateRows(message: ChatGPTMessage, title: string): string {\n /**\n * Combine message information in a readable format ready to be used.\n * @param {ChatGPTMessage} message - Message to be concatenated\n * @param {string} title - Title of the conversation\n *\n * @returns {string} Concatenated message\n */\n if (!message) {\n return \"\";\n }\n\n const sender = message.author ? message.author.role : \"unknown\";\n const text = message.content.parts[0];\n const date = new Date(message.create_time * 1000)\n .toISOString()\n .slice(0, 19)\n .replace(\"T\", \" \");\n return `${title} - ${sender} on ${date}: ${text}\\n\\n`;\n}\n\nexport class ChatGPTLoader extends TextLoader {\n public numLogs: number;\n\n constructor(filePathOrBlob: string | Blob, numLogs = 0) {\n super(filePathOrBlob);\n this.numLogs = numLogs;\n }\n\n protected async parse(raw: string): Promise<string[]> {\n let data;\n try {\n data = JSON.parse(raw);\n } catch (e) {\n console.error(e);\n throw new Error(\"Failed to parse JSON\");\n }\n\n const truncatedData = this.numLogs > 0 ? data.slice(0, this.numLogs) : data;\n\n return truncatedData.map((d: ChatGPTLog) =>\n Object.values(d.mapping)\n .filter(\n (msg, idx) => !(idx === 0 && msg.message.author.role === \"system\")\n )\n .map((msg) => concatenateRows(msg.message, d.title))\n .join(\"\")\n );\n }\n\n public async load(): Promise<Document[]> {\n let text: string;\n let metadata: Record<string, string>;\n if (typeof this.filePathOrBlob === \"string\") {\n const { readFile } = await TextLoader.imports();\n try {\n text = await readFile(this.filePathOrBlob, \"utf8\");\n } catch (e) {\n console.error(e);\n throw new Error(\"Failed to read file\");\n }\n metadata = { source: this.filePathOrBlob };\n } else {\n try {\n text = await this.filePathOrBlob.text();\n } catch (e) {\n console.error(e);\n throw new Error(\"Failed to read blob\");\n }\n metadata = { source: \"blob\", blobType: this.filePathOrBlob.type };\n }\n\n const parsed = await this.parse(text);\n return parsed.map(\n (pageContent, i) =>\n new Document({\n pageContent,\n metadata: {\n ...metadata,\n logIndex: i + 1,\n },\n })\n );\n }\n}\n"],"mappings":";;;;;;AAkBA,SAAS,gBAAgB,SAAyB,OAAuB;;;;;;;;AAQvE,KAAI,CAAC,QACH,QAAO;CAGT,MAAM,SAAS,QAAQ,SAAS,QAAQ,OAAO,OAAO;CACtD,MAAM,OAAO,QAAQ,QAAQ,MAAM;AAKnC,QAAO,GAAG,MAAM,KAAK,OAAO,uBAJf,IAAI,KAAK,QAAQ,cAAc,IAAK,EAC9C,aAAa,CACb,MAAM,GAAG,GAAG,CACZ,QAAQ,KAAK,IAAI,CACmB,IAAI,KAAK;;AAGlD,IAAa,gBAAb,cAAmCA,4CAAAA,WAAW;CAC5C;CAEA,YAAY,gBAA+B,UAAU,GAAG;AACtD,QAAM,eAAe;AACrB,OAAK,UAAU;;CAGjB,MAAgB,MAAM,KAAgC;EACpD,IAAI;AACJ,MAAI;AACF,UAAO,KAAK,MAAM,IAAI;WACf,GAAG;AACV,WAAQ,MAAM,EAAE;AAChB,SAAM,IAAI,MAAM,uBAAuB;;AAKzC,UAFsB,KAAK,UAAU,IAAI,KAAK,MAAM,GAAG,KAAK,QAAQ,GAAG,MAElD,KAAK,MACxB,OAAO,OAAO,EAAE,QAAQ,CACrB,QACE,KAAK,QAAQ,EAAE,QAAQ,KAAK,IAAI,QAAQ,OAAO,SAAS,UAC1D,CACA,KAAK,QAAQ,gBAAgB,IAAI,SAAS,EAAE,MAAM,CAAC,CACnD,KAAK,GAAG,CACZ;;CAGH,MAAa,OAA4B;EACvC,IAAI;EACJ,IAAI;AACJ,MAAI,OAAO,KAAK,mBAAmB,UAAU;GAC3C,MAAM,EAAE,aAAa,MAAMA,4CAAAA,WAAW,SAAS;AAC/C,OAAI;AACF,WAAO,MAAM,SAAS,KAAK,gBAAgB,OAAO;YAC3C,GAAG;AACV,YAAQ,MAAM,EAAE;AAChB,UAAM,IAAI,MAAM,sBAAsB;;AAExC,cAAW,EAAE,QAAQ,KAAK,gBAAgB;SACrC;AACL,OAAI;AACF,WAAO,MAAM,KAAK,eAAe,MAAM;YAChC,GAAG;AACV,YAAQ,MAAM,EAAE;AAChB,UAAM,IAAI,MAAM,sBAAsB;;AAExC,cAAW;IAAE,QAAQ;IAAQ,UAAU,KAAK,eAAe;IAAM;;AAInE,UADe,MAAM,KAAK,MAAM,KAAK,EACvB,KACX,aAAa,MACZ,IAAIC,0BAAAA,SAAS;GACX;GACA,UAAU;IACR,GAAG;IACH,UAAU,IAAI;IACf;GACF,CAAC,CACL"}