@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 5.32 kB
Source Map (JSON)
{"version":3,"file":"pdf.cjs","names":["BaseDocumentLoader","blob: Blob","documents: Document[]","Document"],"sources":["../../../src/document_loaders/web/pdf.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\n/**\n * A document loader for loading data from PDFs.\n * @example\n * ```typescript\n * const loader = new WebPDFLoader(new Blob());\n * const docs = await loader.load();\n * console.log({ docs });\n * ```\n */\nexport class WebPDFLoader extends BaseDocumentLoader {\n protected blob: Blob;\n\n protected splitPages = true;\n\n private pdfjs: typeof PDFLoaderImports;\n\n protected parsedItemSeparator: string;\n\n constructor(\n blob: Blob,\n {\n splitPages = true,\n pdfjs = PDFLoaderImports,\n parsedItemSeparator = \"\",\n } = {}\n ) {\n super();\n this.blob = blob;\n this.splitPages = splitPages ?? this.splitPages;\n this.pdfjs = pdfjs;\n this.parsedItemSeparator = parsedItemSeparator;\n }\n\n /**\n * Loads the contents of the PDF as documents.\n * @returns An array of Documents representing the retrieved data.\n */\n async load(): Promise<Document[]> {\n const { getDocument, version } = await this.pdfjs();\n const parsedPdf = await getDocument({\n data: new Uint8Array(await this.blob.arrayBuffer()),\n useWorkerFetch: false,\n isEvalSupported: false,\n useSystemFonts: true,\n }).promise;\n const meta = await parsedPdf.getMetadata().catch(() => null);\n\n const documents: Document[] = [];\n\n for (let i = 1; i <= parsedPdf.numPages; i += 1) {\n const page = await parsedPdf.getPage(i);\n const content = await page.getTextContent();\n\n if (content.items.length === 0) {\n continue;\n }\n\n // Eliminate excessive newlines\n // Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16\n let lastY;\n const textItems = [];\n for (const item of content.items) {\n if (\"str\" in item) {\n if (lastY === item.transform[5] || !lastY) {\n textItems.push(item.str);\n } else {\n textItems.push(`\\n${item.str}`);\n }\n lastY = item.transform[5];\n }\n }\n const text = textItems.join(this.parsedItemSeparator);\n\n documents.push(\n new Document({\n pageContent: text,\n metadata: {\n pdf: {\n version,\n info: meta?.info,\n metadata: meta?.metadata,\n totalPages: parsedPdf.numPages,\n },\n loc: {\n pageNumber: i,\n },\n },\n })\n );\n }\n\n if (this.splitPages) {\n return documents;\n }\n\n if (documents.length === 0) {\n return [];\n }\n\n return [\n new Document({\n pageContent: documents.map((doc) => doc.pageContent).join(\"\\n\\n\"),\n metadata: {\n pdf: {\n version,\n info: meta?.info,\n metadata: meta?.metadata,\n totalPages: parsedPdf.numPages,\n },\n },\n }),\n ];\n\n return documents;\n }\n}\n\nasync function PDFLoaderImports() {\n try {\n const { default: mod } = await import(\n \"pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js\"\n );\n const { getDocument, version } = mod;\n return { getDocument, version };\n } catch (e) {\n console.error(e);\n throw new Error(\n \"Failed to load pdf-parse. This loader currently supports pdf-parse v1 only. Please install v1, e.g. `npm install pdf-parse@^1` (v2 is not yet supported).\"\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAYA,IAAa,eAAb,cAAkCA,0DAAmB;CACnD,AAAU;CAEV,AAAU,aAAa;CAEvB,AAAQ;CAER,AAAU;CAEV,YACEC,MACA,EACE,aAAa,MACb,QAAQ,kBACR,sBAAsB,IACvB,GAAG,CAAE,GACN;EACA,OAAO;EACP,KAAK,OAAO;EACZ,KAAK,aAAa,cAAc,KAAK;EACrC,KAAK,QAAQ;EACb,KAAK,sBAAsB;CAC5B;;;;;CAMD,MAAM,OAA4B;EAChC,MAAM,EAAE,aAAa,SAAS,GAAG,MAAM,KAAK,OAAO;EACnD,MAAM,YAAY,MAAM,YAAY;GAClC,MAAM,IAAI,WAAW,MAAM,KAAK,KAAK,aAAa;GAClD,gBAAgB;GAChB,iBAAiB;GACjB,gBAAgB;EACjB,EAAC,CAAC;EACH,MAAM,OAAO,MAAM,UAAU,aAAa,CAAC,MAAM,MAAM,KAAK;EAE5D,MAAMC,YAAwB,CAAE;AAEhC,OAAK,IAAI,IAAI,GAAG,KAAK,UAAU,UAAU,KAAK,GAAG;GAC/C,MAAM,OAAO,MAAM,UAAU,QAAQ,EAAE;GACvC,MAAM,UAAU,MAAM,KAAK,gBAAgB;AAE3C,OAAI,QAAQ,MAAM,WAAW,EAC3B;GAKF,IAAI;GACJ,MAAM,YAAY,CAAE;AACpB,QAAK,MAAM,QAAQ,QAAQ,MACzB,KAAI,SAAS,MAAM;AACjB,QAAI,UAAU,KAAK,UAAU,MAAM,CAAC,OAClC,UAAU,KAAK,KAAK,IAAI;SAExB,UAAU,KAAK,CAAC,EAAE,EAAE,KAAK,KAAK,CAAC;IAEjC,QAAQ,KAAK,UAAU;GACxB;GAEH,MAAM,OAAO,UAAU,KAAK,KAAK,oBAAoB;GAErD,UAAU,KACR,IAAIC,oCAAS;IACX,aAAa;IACb,UAAU;KACR,KAAK;MACH;MACA,MAAM,MAAM;MACZ,UAAU,MAAM;MAChB,YAAY,UAAU;KACvB;KACD,KAAK,EACH,YAAY,EACb;IACF;GACF,GACF;EACF;AAED,MAAI,KAAK,WACP,QAAO;AAGT,MAAI,UAAU,WAAW,EACvB,QAAO,CAAE;AAGX,SAAO,CACL,IAAIA,oCAAS;GACX,aAAa,UAAU,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,KAAK,OAAO;GACjE,UAAU,EACR,KAAK;IACH;IACA,MAAM,MAAM;IACZ,UAAU,MAAM;IAChB,YAAY,UAAU;GACvB,EACF;EACF,EACF;CAGF;AACF;AAED,eAAe,mBAAmB;AAChC,KAAI;EACF,MAAM,EAAE,SAAS,KAAK,GAAG,MAAM,OAC7B;EAEF,MAAM,EAAE,aAAa,SAAS,GAAG;AACjC,SAAO;GAAE;GAAa;EAAS;CAChC,SAAQ,GAAG;EACV,QAAQ,MAAM,EAAE;AAChB,QAAM,IAAI,MACR;CAEH;AACF"}