@langchain/community

Version:

Third-party integrations for LangChain.js

github.com/langchain-ai/langchainjs-community/tree/main/libs/community/

1 lines • 4.72 kB

Source Map (JSON)

{"version":3,"file":"csv.cjs","names":["TextLoader"],"sources":["../../../src/document_loaders/fs/csv.ts"],"sourcesContent":["import { TextLoader } from \"@langchain/classic/document_loaders/fs/text\";\n\n/**\n * Loads a CSV file into a list of documents.\n * Each document represents one row of the CSV file.\n *\n * When `column` is not specified, each row is converted into a key/value pair\n * with each key/value pair outputted to a new line in the document's pageContent.\n *\n * @example\n * // CSV file:\n * // id,html\n * // 1,Corruption discovered at the core of the Banking Clan!\n * // 2,Corruption discovered at the core of the Banking Clan!\n *\n * const loader = new CSVLoader(\"path/to/file.csv\");\n * const docs = await loader.load();\n *\n * // docs[0].pageContent:\n * // id: 1\n * // html: Corruption discovered at the core of the Banking Clan!\n *\n * When `column` is specified, one document is created for each row, and the\n * value of the specified column is used as the document's pageContent.\n *\n * @example\n * // CSV file:\n * // id,html\n * // 1,Corruption discovered at the core of the Banking Clan!\n * // 2,Corruption discovered at the core of the Banking Clan!\n *\n * const loader = new CSVLoader(\"path/to/file.csv\", \"html\");\n * const docs = await loader.load();\n *\n * // docs[0].pageContent:\n * // Corruption discovered at the core of the Banking Clan!\n */\n\ntype CSVLoaderOptions = {\n column?: string;\n separator?: string;\n};\n\n/**\n * A class that extends the TextLoader class. It represents a document\n * loader that loads documents from a CSV file. It has a constructor that\n * takes a `filePathOrBlob` parameter representing the path to the CSV\n * file or a Blob object, and an optional `options` parameter of type\n * `CSVLoaderOptions` or a string representing the column to use as the\n * document's pageContent.\n */\nexport class CSVLoader extends TextLoader {\n protected options: CSVLoaderOptions = {};\n\n constructor(\n filePathOrBlob: string | Blob,\n options?: CSVLoaderOptions | string\n ) {\n super(filePathOrBlob);\n if (typeof options === \"string\") {\n this.options = { column: options };\n } else {\n this.options = options ?? this.options;\n }\n }\n\n /**\n * A protected method that parses the raw CSV data and returns an array of\n * strings representing the pageContent of each document. It uses the\n * `dsvFormat` function from the `d3-dsv` module to parse the CSV data. If\n * the `column` option is specified, it checks if the column exists in the\n * CSV file and returns the values of that column as the pageContent. If\n * the `column` option is not specified, it converts each row of the CSV\n * data into key/value pairs and joins them with newline characters.\n * @param raw The raw CSV data to be parsed.\n * @returns An array of strings representing the pageContent of each document.\n */\n protected async parse(raw: string): Promise<string[]> {\n const { column, separator = \",\" } = this.options;\n\n const { dsvFormat } = await CSVLoaderImports();\n const psv = dsvFormat(separator);\n const parsed = psv.parse(raw.trim());\n\n if (column !== undefined) {\n if (!parsed.columns.includes(column)) {\n throw new Error(`Column ${column} not found in CSV file.`);\n }\n // Note TextLoader will raise an exception if the value is null.\n return parsed.map((row) => row[column]!);\n }\n\n return parsed.map((row) =>\n Object.keys(row)\n .map((key) => `${key.trim()}: ${row[key]?.trim()}`)\n .join(\"\\n\")\n );\n }\n}\n\nasync function CSVLoaderImports() {\n try {\n const { dsvFormat } = await import(\"d3-dsv\");\n return { dsvFormat };\n } catch (e) {\n console.error(e);\n throw new Error(\n \"Please install d3-dsv as a dependency with, e.g. `pnpm install d3-dsv@2`\"\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAmDA,IAAa,YAAb,cAA+BA,4CAAAA,WAAW;CACxC,UAAsC,EAAE;CAExC,YACE,gBACA,SACA;AACA,QAAM,eAAe;AACrB,MAAI,OAAO,YAAY,SACrB,MAAK,UAAU,EAAE,QAAQ,SAAS;MAElC,MAAK,UAAU,WAAW,KAAK;;;;;;;;;;;;;CAenC,MAAgB,MAAM,KAAgC;EACpD,MAAM,EAAE,QAAQ,YAAY,QAAQ,KAAK;EAEzC,MAAM,EAAE,cAAc,MAAM,kBAAkB;EAE9C,MAAM,SADM,UAAU,UAAU,CACb,MAAM,IAAI,MAAM,CAAC;AAEpC,MAAI,WAAW,KAAA,GAAW;AACxB,OAAI,CAAC,OAAO,QAAQ,SAAS,OAAO,CAClC,OAAM,IAAI,MAAM,UAAU,OAAO,yBAAyB;AAG5D,UAAO,OAAO,KAAK,QAAQ,IAAI,QAAS;;AAG1C,SAAO,OAAO,KAAK,QACjB,OAAO,KAAK,IAAI,CACb,KAAK,QAAQ,GAAG,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,MAAM,GAAG,CAClD,KAAK,KAAK,CACd;;;AAIL,eAAe,mBAAmB;AAChC,KAAI;EACF,MAAM,EAAE,cAAc,MAAM,OAAO;AACnC,SAAO,EAAE,WAAW;UACb,GAAG;AACV,UAAQ,MAAM,EAAE;AAChB,QAAM,IAAI,MACR,2EACD"}