UNPKG

@langchain/community

Version:

Third-party integrations for LangChain.js

github.com/langchain-ai/langchainjs/tree/main/libs/langchain-community/

langchain-ai/langchainjs

1 lines • 10.2 kB
1
{"version":3,"file":"arxiv.cjs","names":["query: string","arxivIds: string","XMLParser","pdfUrl: string","PDFLoader","docs: Document[]","results: ArxivEntry[]","Document","entry: any","authors: string[]","author: any","links: any[]","link: any"],"sources":["../../src/utils/arxiv.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport { Document } from \"@langchain/core/documents\";\nimport { XMLParser } from \"fast-xml-parser\";\n\nimport { PDFLoader } from \"../document_loaders/fs/pdf.js\";\n\n// Interface for processed arXiv entry\ninterface ArxivEntry {\n id: string;\n title: string;\n summary: string;\n published: string;\n updated: string;\n authors: string[];\n pdfUrl: string;\n links: any[];\n}\n\n// Used to check if the query is an arXiv ID, or a natural language query\nexport function isArXivIdentifier(query: string): boolean {\n const arxivIdRegex = /^\\d{4}\\.\\d{4,5}(v\\d+)?$|^\\d{7}(\\.\\d+)?(v\\d+)?$/;\n return arxivIdRegex.test(query.trim());\n}\n\n// Used to fetch direct arXiv articles by IDs (supports multiple IDs)\nexport async function fetchDirectArxivArticle(\n arxivIds: string\n): Promise<ArxivEntry[]> {\n try {\n const idList = arxivIds\n .split(/[\\s,]+/)\n .map((id) => id.trim())\n .filter(Boolean)\n .join(\",\");\n const url = `http://export.arxiv.org/api/query?id_list=${idList}`;\n const response = await fetch(url);\n\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n\n const xml = await response.text();\n\n const parser = new XMLParser({\n ignoreAttributes: false,\n attributeNamePrefix: \"@_\",\n });\n const result = parser.parse(xml);\n let entries = result.feed.entry;\n\n if (!entries) {\n return [];\n }\n\n // Ensure entries is an array\n if (!Array.isArray(entries)) {\n entries = [entries];\n }\n\n const processedEntries = entries.map(processEntry);\n\n return processedEntries;\n } catch {\n throw new Error(`Failed to fetch articles with IDs ${arxivIds}`);\n }\n}\n\n// Used to fetch arXiv results by natural language query with maxResults parameter\nexport async function fetchArxivResultsByQuery(\n query: string,\n start = 0,\n maxResults = 10\n): Promise<ArxivEntry[]> {\n try {\n const encodedQuery = encodeURIComponent(query);\n const url = `http://export.arxiv.org/api/query?search_query=all:${encodedQuery}&start=${start}&max_results=${maxResults}`;\n const response = await fetch(url);\n\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n\n const xml = await response.text();\n\n const parser = new XMLParser({\n ignoreAttributes: false,\n attributeNamePrefix: \"@_\",\n });\n const result = parser.parse(xml);\n let entries = result.feed.entry;\n\n if (!entries) {\n return [];\n }\n\n // Ensure entries is an array\n if (!Array.isArray(entries)) {\n entries = [entries];\n }\n\n const processedEntries = entries.map(processEntry);\n\n return processedEntries;\n } catch {\n throw new Error(`Failed to fetch articles with query \"${query}\"`);\n }\n}\n\n// Used to search for arXiv articles with a maxResults parameter\nexport async function searchArxiv(\n query: string,\n maxResults = 3\n): Promise<ArxivEntry[]> {\n if (isArXivIdentifier(query)) {\n return await fetchDirectArxivArticle(query);\n } else {\n return await fetchArxivResultsByQuery(query, 0, maxResults);\n }\n}\n\n// Used to fetch and parse PDF to text\nexport async function fetchAndParsePDF(pdfUrl: string): Promise<string> {\n try {\n // Fetch the PDF\n const response = await fetch(pdfUrl);\n\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n\n const buffer = await response.arrayBuffer();\n\n // Convert the ArrayBuffer to a Blob\n const blob = new Blob([buffer], { type: \"application/pdf\" });\n\n // Use PDFLoader to process the PDF\n const loader = new PDFLoader(blob, { splitPages: false }); // Pass the Blob\n const docs: Document[] = await loader.load();\n\n // Combine all document content into a single string\n const content = docs.map((doc) => doc.pageContent).join(\"\\n\\n\");\n return content;\n } catch {\n throw new Error(`Failed to fetch or parse PDF from ${pdfUrl}`);\n }\n}\n\n// Used to load raw text from each search result, and convert to Document instances\nexport async function loadDocsFromResults(\n results: ArxivEntry[]\n): Promise<Document[]> {\n const docs: Document[] = [];\n for (const result of results) {\n const pdfUrl = result.pdfUrl;\n try {\n const pdfContent = await fetchAndParsePDF(pdfUrl);\n const metadata = {\n id: result.id,\n title: result.title,\n authors: result.authors,\n published: result.published,\n updated: result.updated,\n source: \"arxiv\",\n url: result.id,\n summary: result.summary,\n };\n const doc = new Document({\n pageContent: pdfContent,\n metadata,\n });\n docs.push(doc);\n } catch {\n throw new Error(`Error loading document from ${pdfUrl}`);\n }\n }\n return docs;\n}\n\n// Used to convert metadata and summaries to Document instances\nexport function getDocsFromSummaries(results: ArxivEntry[]): Document[] {\n const docs: Document[] = [];\n for (const result of results) {\n const metadata = {\n id: result.id,\n title: result.title,\n authors: result.authors,\n published: result.published,\n updated: result.updated,\n source: \"arxiv\",\n url: result.id,\n };\n const doc = new Document({\n pageContent: result.summary,\n metadata,\n });\n docs.push(doc);\n }\n return docs;\n}\n\n// Helper function to process each arXiv entry\nfunction processEntry(entry: any): ArxivEntry {\n const id = entry.id;\n const title = entry.title.replace(/\\s+/g, \" \").trim();\n const summary = entry.summary.replace(/\\s+/g, \" \").trim();\n const published = entry.published;\n const updated = entry.updated;\n\n // Extract authors\n let authors: string[] = [];\n if (Array.isArray(entry.author)) {\n authors = entry.author.map((author: any) => author.name);\n } else if (entry.author) {\n authors = [entry.author.name];\n }\n\n // Extract links\n let links: any[] = [];\n if (Array.isArray(entry.link)) {\n links = entry.link;\n } else if (entry.link) {\n links = [entry.link];\n }\n\n // Extract PDF link\n let pdfUrl = `${id.replace(\"/abs/\", \"/pdf/\")}.pdf`;\n const pdfLinkObj = links.find((link: any) => link[\"@_title\"] === \"pdf\");\n if (pdfLinkObj && pdfLinkObj[\"@_href\"]) {\n pdfUrl = pdfLinkObj[\"@_href\"];\n }\n\n return {\n id,\n title,\n summary,\n published,\n updated,\n authors,\n pdfUrl,\n links,\n };\n}\n"],"mappings":";;;;;;AAmBA,SAAgB,kBAAkBA,OAAwB;CACxD,MAAM,eAAe;AACrB,QAAO,aAAa,KAAK,MAAM,MAAM,CAAC;AACvC;AAGD,eAAsB,wBACpBC,UACuB;AACvB,KAAI;EACF,MAAM,SAAS,SACZ,MAAM,SAAS,CACf,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,CACtB,OAAO,QAAQ,CACf,KAAK,IAAI;EACZ,MAAM,MAAM,CAAC,0CAA0C,EAAE,QAAQ;EACjE,MAAM,WAAW,MAAM,MAAM,IAAI;AAEjC,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,CAAC,oBAAoB,EAAE,SAAS,QAAQ;EAG1D,MAAM,MAAM,MAAM,SAAS,MAAM;EAEjC,MAAM,SAAS,IAAIC,0BAAU;GAC3B,kBAAkB;GAClB,qBAAqB;EACtB;EACD,MAAM,SAAS,OAAO,MAAM,IAAI;EAChC,IAAI,UAAU,OAAO,KAAK;AAE1B,MAAI,CAAC,QACH,QAAO,CAAE;AAIX,MAAI,CAAC,MAAM,QAAQ,QAAQ,EACzB,UAAU,CAAC,OAAQ;EAGrB,MAAM,mBAAmB,QAAQ,IAAI,aAAa;AAElD,SAAO;CACR,QAAO;AACN,QAAM,IAAI,MAAM,CAAC,kCAAkC,EAAE,UAAU;CAChE;AACF;AAGD,eAAsB,yBACpBF,OACA,QAAQ,GACR,aAAa,IACU;AACvB,KAAI;EACF,MAAM,eAAe,mBAAmB,MAAM;EAC9C,MAAM,MAAM,CAAC,mDAAmD,EAAE,aAAa,OAAO,EAAE,MAAM,aAAa,EAAE,YAAY;EACzH,MAAM,WAAW,MAAM,MAAM,IAAI;AAEjC,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,CAAC,oBAAoB,EAAE,SAAS,QAAQ;EAG1D,MAAM,MAAM,MAAM,SAAS,MAAM;EAEjC,MAAM,SAAS,IAAIE,0BAAU;GAC3B,kBAAkB;GAClB,qBAAqB;EACtB;EACD,MAAM,SAAS,OAAO,MAAM,IAAI;EAChC,IAAI,UAAU,OAAO,KAAK;AAE1B,MAAI,CAAC,QACH,QAAO,CAAE;AAIX,MAAI,CAAC,MAAM,QAAQ,QAAQ,EACzB,UAAU,CAAC,OAAQ;EAGrB,MAAM,mBAAmB,QAAQ,IAAI,aAAa;AAElD,SAAO;CACR,QAAO;AACN,QAAM,IAAI,MAAM,CAAC,qCAAqC,EAAE,MAAM,CAAC,CAAC;CACjE;AACF;AAGD,eAAsB,YACpBF,OACA,aAAa,GACU;AACvB,KAAI,kBAAkB,MAAM,CAC1B,QAAO,MAAM,wBAAwB,MAAM;KAE3C,QAAO,MAAM,yBAAyB,OAAO,GAAG,WAAW;AAE9D;AAGD,eAAsB,iBAAiBG,QAAiC;AACtE,KAAI;EAEF,MAAM,WAAW,MAAM,MAAM,OAAO;AAEpC,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,CAAC,oBAAoB,EAAE,SAAS,QAAQ;EAG1D,MAAM,SAAS,MAAM,SAAS,aAAa;EAG3C,MAAM,OAAO,IAAI,KAAK,CAAC,MAAO,GAAE,EAAE,MAAM,kBAAmB;EAG3D,MAAM,SAAS,IAAIC,0CAAU,MAAM,EAAE,YAAY,MAAO;EACxD,MAAMC,OAAmB,MAAM,OAAO,MAAM;EAG5C,MAAM,UAAU,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,KAAK,OAAO;AAC/D,SAAO;CACR,QAAO;AACN,QAAM,IAAI,MAAM,CAAC,kCAAkC,EAAE,QAAQ;CAC9D;AACF;AAGD,eAAsB,oBACpBC,SACqB;CACrB,MAAMD,OAAmB,CAAE;AAC3B,MAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,SAAS,OAAO;AACtB,MAAI;GACF,MAAM,aAAa,MAAM,iBAAiB,OAAO;GACjD,MAAM,WAAW;IACf,IAAI,OAAO;IACX,OAAO,OAAO;IACd,SAAS,OAAO;IAChB,WAAW,OAAO;IAClB,SAAS,OAAO;IAChB,QAAQ;IACR,KAAK,OAAO;IACZ,SAAS,OAAO;GACjB;GACD,MAAM,MAAM,IAAIE,oCAAS;IACvB,aAAa;IACb;GACD;GACD,KAAK,KAAK,IAAI;EACf,QAAO;AACN,SAAM,IAAI,MAAM,CAAC,4BAA4B,EAAE,QAAQ;EACxD;CACF;AACD,QAAO;AACR;AAGD,SAAgB,qBAAqBD,SAAmC;CACtE,MAAMD,OAAmB,CAAE;AAC3B,MAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,WAAW;GACf,IAAI,OAAO;GACX,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO;GAClB,SAAS,OAAO;GAChB,QAAQ;GACR,KAAK,OAAO;EACb;EACD,MAAM,MAAM,IAAIE,oCAAS;GACvB,aAAa,OAAO;GACpB;EACD;EACD,KAAK,KAAK,IAAI;CACf;AACD,QAAO;AACR;AAGD,SAAS,aAAaC,OAAwB;CAC5C,MAAM,KAAK,MAAM;CACjB,MAAM,QAAQ,MAAM,MAAM,QAAQ,QAAQ,IAAI,CAAC,MAAM;CACrD,MAAM,UAAU,MAAM,QAAQ,QAAQ,QAAQ,IAAI,CAAC,MAAM;CACzD,MAAM,YAAY,MAAM;CACxB,MAAM,UAAU,MAAM;CAGtB,IAAIC,UAAoB,CAAE;AAC1B,KAAI,MAAM,QAAQ,MAAM,OAAO,EAC7B,UAAU,MAAM,OAAO,IAAI,CAACC,WAAgB,OAAO,KAAK;UAC/C,MAAM,QACf,UAAU,CAAC,MAAM,OAAO,IAAK;CAI/B,IAAIC,QAAe,CAAE;AACrB,KAAI,MAAM,QAAQ,MAAM,KAAK,EAC3B,QAAQ,MAAM;UACL,MAAM,MACf,QAAQ,CAAC,MAAM,IAAK;CAItB,IAAI,SAAS,GAAG,GAAG,QAAQ,SAAS,QAAQ,CAAC,IAAI,CAAC;CAClD,MAAM,aAAa,MAAM,KAAK,CAACC,SAAc,KAAK,eAAe,MAAM;AACvE,KAAI,cAAc,WAAW,WAC3B,SAAS,WAAW;AAGtB,QAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACD;AACF"}