@langchain/community/dist/document_transformers/mozilla_readability.cjs.map

Third-party integrations for LangChain.js

{"version":3,"file":"mozilla_readability.cjs","names":["MappingDocumentTransformer","options: Options","document: Document","JSDOM","Readability","Document"],"sources":["../../src/document_transformers/mozilla_readability.ts"],"sourcesContent":["import { Readability } from \"@mozilla/readability\";\nimport { JSDOM } from \"jsdom\";\nimport type { Options } from \"mozilla-readability\";\nimport {\n  MappingDocumentTransformer,\n  Document,\n} from \"@langchain/core/documents\";\n\n/**\n * A transformer that uses the Mozilla Readability library to extract the\n * main content from a web page.\n * @example\n * ```typescript\n * const loader = new HTMLWebBaseLoader(\"https://example.com/article\");\n * const docs = await loader.load();\n *\n * const splitter = new RecursiveCharacterTextSplitter({\n *  maxCharacterCount: 5000,\n * });\n * const transformer = new MozillaReadabilityTransformer();\n *\n * // The sequence processes the loaded documents through the splitter and then the transformer.\n * const sequence = transformer.pipe(splitter);\n *\n * // Invoke the sequence to transform the documents into a more readable format.\n * const newDocuments = await sequence.invoke(docs);\n *\n * console.log(newDocuments);\n * ```\n */\nexport class MozillaReadabilityTransformer extends MappingDocumentTransformer {\n  static lc_name() {\n    return \"MozillaReadabilityTransformer\";\n  }\n\n  constructor(protected options: Options = {}) {\n    super(options);\n  }\n\n  async _transformDocument(document: Document): Promise<Document> {\n    const doc = new JSDOM(document.pageContent);\n\n    const readability = new Readability(doc.window.document, this.options);\n\n    const result = readability.parse();\n\n    return new Document({\n      pageContent: result?.textContent ?? \"\",\n      metadata: {\n        ...document.metadata,\n      },\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,IAAa,gCAAb,cAAmDA,sDAA2B;CAC5E,OAAO,UAAU;AACf,SAAO;CACR;CAED,YAAsBC,UAAmB,CAAE,GAAE;EAC3C,MAAM,QAAQ;EADM;CAErB;CAED,MAAM,mBAAmBC,UAAuC;EAC9D,MAAM,MAAM,IAAIC,YAAM,SAAS;EAE/B,MAAM,cAAc,IAAIC,kCAAY,IAAI,OAAO,UAAU,KAAK;EAE9D,MAAM,SAAS,YAAY,OAAO;AAElC,SAAO,IAAIC,oCAAS;GAClB,aAAa,QAAQ,eAAe;GACpC,UAAU,EACR,GAAG,SAAS,SACb;EACF;CACF;AACF"}