@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
572 lines (571 loc) • 21.4 kB
JavaScript
/**
* Document Loaders
*
* Provides loaders for various document formats including:
* - Text files
* - Markdown files
* - HTML files and web pages
* - JSON files
* - CSV files
* - PDF files
*
* @example
* ```typescript
* import { loadDocument, WebLoader, PDFLoader } from 'neurolink/rag';
*
* // Load from file path
* const doc = await loadDocument('/path/to/document.md');
*
* // Load from URL
* const webDoc = await WebLoader.load('https://example.com/article');
*
* // Load PDF
* const pdfDoc = await PDFLoader.load('/path/to/document.pdf');
* ```
*/
import { existsSync } from "fs";
import { readFile } from "fs/promises";
import { basename, extname } from "path";
import { logger } from "../../utils/logger.js";
import { CSVProcessor, stripBom, stripMetadataLine, splitCsvLines, dedupeColumnNames, sanitizeColumnName, } from "../../utils/csvProcessor.js";
import { splitCSVFields } from "../../utils/csvUtils.js";
import { MDocument } from "./MDocument.js";
/**
* Text file loader
*/
export class TextLoader {
async load(source, options) {
const content = await this.loadContent(source, options?.encoding);
return MDocument.fromText(content, {
source: this.getSourceName(source),
...options?.metadata,
});
}
canHandle(source) {
const ext = extname(source).toLowerCase();
return ext === ".txt" || ext === "";
}
async loadContent(source, encoding = "utf-8") {
if (existsSync(source)) {
return await readFile(source, encoding);
}
// Assume source is content if not a file
return source;
}
getSourceName(source) {
return existsSync(source) ? basename(source) : "inline-content";
}
}
/**
* Markdown file loader
*/
export class MarkdownLoader extends TextLoader {
async load(source, options) {
const content = await this.loadContent(source, options?.encoding);
return MDocument.fromMarkdown(content, {
source: this.getSourceName(source),
...options?.metadata,
});
}
canHandle(source) {
const ext = extname(source).toLowerCase();
return ext === ".md" || ext === ".markdown" || ext === ".mdx";
}
}
/**
* HTML file loader
*/
export class HTMLLoader extends TextLoader {
async load(source, options) {
const content = await this.loadContent(source, options?.encoding);
return MDocument.fromHTML(content, {
source: this.getSourceName(source),
...options?.metadata,
});
}
canHandle(source) {
const ext = extname(source).toLowerCase();
return ext === ".html" || ext === ".htm" || ext === ".xhtml";
}
}
/**
* JSON file loader
*/
export class JSONLoader extends TextLoader {
async load(source, options) {
const content = await this.loadContent(source, options?.encoding);
// Validate JSON
try {
JSON.parse(content);
}
catch (error) {
throw new Error(`Invalid JSON: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
}
return MDocument.fromJSONContent(content, {
source: this.getSourceName(source),
...options?.metadata,
});
}
canHandle(source) {
const ext = extname(source).toLowerCase();
return ext === ".json" || ext === ".jsonl";
}
}
/**
* CSV file loader
*/
export class CSVLoader extends TextLoader {
async load(source, options) {
const content = await this.loadContent(source, options?.encoding);
const { delimiter, hasHeader = true, columns, outputFormat = "text", sanitizeColumnNames = false, columnNameCase = "snake_case", } = options || {};
// #361: an explicit delimiter always wins; otherwise detect it from the
// content + extension so a .tsv fed through RAG isn't collapsed into one
// comma-column (CSVLoader.canHandle already accepts .tsv).
const effectiveDelimiter = delimiter ?? CSVProcessor.detectDelimiter(content, extname(source));
// Route header/row parsing through the same shared, quote-aware path
// detectDelimiter() above already uses internally: a quote-aware logical
// row split (splitCsvLines — a newline inside a quoted field stays part
// of that field instead of breaking the row) with a leading Excel `sep=`
// preamble line stripped (stripMetadataLine), so it can't be parsed as
// the header. Field-level splitting also goes through the shared
// RFC-4180-aware splitCSVFields() instead of the loader's own naive
// backslash-escape parser, which didn't handle doubled-quote (`""`)
// escaping or quoted delimiters in the headerless-columns branch.
const { dataLines: strippedLines } = stripMetadataLine(splitCsvLines(stripBom(content)));
const lines = strippedLines.filter((line) => line.trim());
const rawHeaders = hasHeader
? splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((h) => h.trim())
: columns ||
splitCSVFields(lines[0] ?? "", effectiveDelimiter).map((_, i) => `col${i + 1}`);
// #378: apply the same opt-in header sanitization CSVProcessor offers, so
// the RAG ingestion path doesn't corrupt "Price ($)"-style headers as keys.
const headers = sanitizeColumnNames
? dedupeColumnNames(rawHeaders.map((h, i) => sanitizeColumnName(h, columnNameCase, i)))
: rawHeaders;
const dataLines = hasHeader ? lines.slice(1) : lines;
const rows = dataLines.map((line) => splitCSVFields(line, effectiveDelimiter).map((field) => field.trim()));
let formattedContent;
switch (outputFormat) {
case "json":
formattedContent = JSON.stringify(rows.map((row) => {
// Defense-in-depth against a caller-controlled header literally
// named "__proto__"/"constructor"/"prototype" (#1199);
// Object.fromEntries can't actually pollute Object.prototype here
// (spec semantics), but a null-prototype target avoids relying on
// that for future readers.
const record = Object.create(null);
headers.forEach((h, i) => {
record[h] = row[i];
});
return record;
}), null, 2);
break;
case "markdown":
formattedContent = this.toMarkdownTable(headers, rows);
break;
default:
formattedContent = this.toTextTable(headers, rows);
}
return MDocument.fromCSV(formattedContent, {
source: this.getSourceName(source),
rowCount: rows.length,
columnCount: headers.length,
columns: headers,
...options?.metadata,
});
}
canHandle(source) {
const ext = extname(source).toLowerCase();
return ext === ".csv" || ext === ".tsv";
}
toMarkdownTable(headers, rows) {
const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
const headerRow = `| ${safeHeaders.join(" | ")} |`;
const separator = `| ${safeHeaders.map(() => "---").join(" | ")} |`;
const dataRows = rows.map((row) => `| ${row.map((cell) => this.collapseEmbeddedNewlines(cell)).join(" | ")} |`);
return [headerRow, separator, ...dataRows].join("\n");
}
toTextTable(headers, rows) {
const safeHeaders = headers.map((h) => this.collapseEmbeddedNewlines(h));
const safeRows = rows.map((row) => row.map((cell) => this.collapseEmbeddedNewlines(cell)));
const allRows = [safeHeaders, ...safeRows];
const colWidths = safeHeaders.map((_, i) => Math.max(...allRows.map((row) => (row[i] || "").length)));
const formatRow = (row) => row.map((cell, i) => (cell || "").padEnd(colWidths[i])).join(" | ");
return [
formatRow(safeHeaders),
colWidths.map((w) => "-".repeat(w)).join("-+-"),
...safeRows.map(formatRow),
].join("\n");
}
/**
* Collapse embedded newlines to a single space so one logical CSV row
* renders as exactly one line in the text/markdown table output.
*
* The shared quote-aware parser (`splitCsvLines` + `splitCSVFields`)
* preserves newlines embedded inside a quoted field verbatim — correct
* for the `json` output format (where `JSON.stringify` escapes them), but
* left as-is here it would split a single record across multiple
* physical table rows. Mirrors the same normalization `CSVProcessor`
* already applies in its own markdown formatter.
*/
collapseEmbeddedNewlines(cell) {
return cell.replace(/\r\n|\n|\r/g, " ");
}
}
/**
* PDF file loader
*
* Note: Requires external PDF processing library for full functionality.
* Falls back to placeholder implementation if pdf-parse is not available.
*/
export class PDFLoader {
async load(source, options) {
if (!existsSync(source)) {
throw new Error(`PDF file not found: ${source}`);
}
logger.debug("[PDFLoader] Loading PDF", {
source,
pageRange: options?.pageRange,
});
let PDFParse;
try {
PDFParse = await this.loadPdfParser();
}
catch (error) {
// pdf-parse genuinely isn't installed / doesn't expose a usable
// export — degrade gracefully rather than throwing.
logger.warn("[PDFLoader] pdf-parse not available, using fallback", {
error: error instanceof Error ? error.message : String(error),
});
return new MDocument(`[PDF Document: ${basename(source)}]\n\nNote: PDF parsing requires the 'pdf-parse' package. Install it with:\n npm install pdf-parse`, {
type: "pdf",
metadata: {
source: basename(source),
parseError: "pdf-parse not available",
...options?.metadata,
},
});
}
const buffer = await readFile(source);
const parser = new PDFParse({ data: buffer });
try {
const data = await parser.getText();
const text = data.text;
// Handle page range if specified
if (options?.pageRange) {
const _pages = this.parsePageRange(options.pageRange, data.total);
// Note: pdf-parse doesn't support page selection directly
// This is a placeholder for more sophisticated page handling
logger.debug("[PDFLoader] Page range requested but not fully supported", {
pageRange: options.pageRange,
totalPages: data.total,
});
}
return new MDocument(text, {
type: "pdf",
metadata: {
source: basename(source),
pageCount: data.total,
...options?.metadata,
},
});
}
catch (error) {
// pdf-parse IS installed and loaded fine, but failed on this
// specific file (corrupt PDF, password-protected, unsupported
// structure, etc.). Keep this distinct from the "not installed"
// fallback above — both surface as a placeholder document, but the
// message, parseError value, and log level differ so a genuine
// parse failure is never mistaken for a missing dependency.
const message = error instanceof Error ? error.message : String(error);
logger.error("[PDFLoader] pdf-parse failed to parse PDF", {
source,
error: message,
});
return new MDocument(`[PDF Document: ${basename(source)}]\n\nNote: PDF parsing failed for this file: ${message}`, {
type: "pdf",
metadata: {
source: basename(source),
parseError: `pdf-parse failed: ${message}`,
...options?.metadata,
},
});
}
finally {
await parser.destroy();
}
}
canHandle(source) {
const ext = extname(source).toLowerCase();
return ext === ".pdf";
}
async loadPdfParser() {
// pdf-parse v2 exports a `PDFParse` class (`new PDFParse({ data }).getText()`)
// instead of v1's bare callable — resolve it dynamically and validate at
// runtime that the export shape is actually present instead of blindly
// asserting.
let pdfParseModule;
try {
pdfParseModule = await import("pdf-parse");
}
catch {
throw new Error("pdf-parse module not available");
}
const candidate = typeof pdfParseModule === "object" &&
pdfParseModule !== null &&
"PDFParse" in pdfParseModule
? pdfParseModule.PDFParse
: undefined;
if (typeof candidate !== "function") {
// Outside the try/catch so this message isn't swallowed by the
// module-not-available rethrow.
throw new Error("pdf-parse module does not export a PDFParse class");
}
return candidate;
}
parsePageRange(range, totalPages) {
const pages = [];
const parts = range.split(",");
for (const part of parts) {
if (part.includes("-")) {
const [start, end] = part.split("-").map(Number);
for (let i = start; i <= Math.min(end, totalPages); i++) {
pages.push(i);
}
}
else {
const page = Number(part);
if (page <= totalPages) {
pages.push(page);
}
}
}
return [...new Set(pages)].sort((a, b) => a - b);
}
}
/**
* Web page loader
*
* Fetches and extracts content from web pages.
* Supports basic HTML parsing without external dependencies.
*/
export class WebLoader {
defaultUserAgent = "Mozilla/5.0 (compatible; NeuroLink/1.0; +https://github.com/juspay/neurolink)";
async load(source, options) {
if (!this.canHandle(source)) {
throw new Error(`Invalid URL: ${source}`);
}
logger.debug("[WebLoader] Fetching URL", {
url: source,
timeout: options?.timeout,
});
const response = await fetch(source, {
signal: options?.timeout
? AbortSignal.timeout(options.timeout)
: undefined,
headers: {
"User-Agent": options?.userAgent || this.defaultUserAgent,
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
...options?.headers,
},
});
if (!response.ok) {
throw new Error(`HTTP error ${response.status}: ${response.statusText}`);
}
const html = await response.text();
let content = html;
// Extract main content if requested
if (options?.extractMainContent) {
content = this.extractMainContent(html, options.contentSelector);
}
// Convert HTML to plain text for better processing
const text = this.htmlToText(content);
return new MDocument(text, {
type: "html",
metadata: {
source,
url: source,
fetchedAt: new Date().toISOString(),
contentType: response.headers.get("content-type") || "text/html",
...options?.metadata,
},
});
}
canHandle(source) {
try {
const url = new URL(source);
return url.protocol === "http:" || url.protocol === "https:";
}
catch {
return false;
}
}
/**
* Extract main content from HTML
*/
extractMainContent(html, selector) {
// Simple extraction based on common content patterns
// For production use, consider using a library like cheerio
// Try to extract content from common containers
const patterns = selector
? [`<${selector}[^>]*>([\\s\\S]*?)</${selector}>`]
: [
/<main[^>]*>([\s\S]*?)<\/main>/i,
/<article[^>]*>([\s\S]*?)<\/article>/i,
/<div[^>]*class="[^"]*content[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
/<div[^>]*id="content"[^>]*>([\s\S]*?)<\/div>/i,
/<body[^>]*>([\s\S]*?)<\/body>/i,
];
for (const pattern of patterns) {
const match = html.match(new RegExp(pattern, "i"));
if (match) {
return match[1] || match[0];
}
}
return html;
}
/**
* Convert HTML to plain text
*/
htmlToText(html) {
return (html
// Remove script and style elements
.replace(/<script[\s\S]*?<\/script>/gi, "")
.replace(/<style[\s\S]*?<\/style>/gi, "")
// Remove HTML comments
.replace(/<!--[\s\S]*?-->/g, "")
// Replace common block elements with newlines
.replace(/<\/(p|div|h[1-6]|br|li|tr|blockquote)>/gi, "\n")
.replace(/<(br|hr)\s*\/?>/gi, "\n")
// Remove remaining tags
.replace(/<[^>]+>/g, "")
// Decode common HTML entities
.replace(/ /gi, " ")
.replace(/&/gi, "&")
.replace(/</gi, "<")
.replace(/>/gi, ">")
.replace(/"/gi, '"')
.replace(/'/gi, "'")
.replace(/'/gi, "'")
// Decode numeric entities
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
// Normalize whitespace
.replace(/\n\s*\n/g, "\n\n")
.replace(/[ \t]+/g, " ")
.trim());
}
}
/**
* Registry of document loaders
*/
const loaderRegistry = [
new MarkdownLoader(),
new HTMLLoader(),
new JSONLoader(),
new CSVLoader(),
new PDFLoader(),
new WebLoader(),
new TextLoader(), // Default fallback
];
/**
* Detect document type from source
*/
function _detectDocumentType(source) {
const ext = extname(source).toLowerCase();
const typeMap = {
".md": "markdown",
".markdown": "markdown",
".mdx": "markdown",
".html": "html",
".htm": "html",
".xhtml": "html",
".json": "json",
".jsonl": "json",
".csv": "csv",
".tsv": "csv",
".tex": "latex",
".latex": "latex",
".pdf": "pdf",
};
// Check if it's a URL
try {
const url = new URL(source);
if (url.protocol === "http:" || url.protocol === "https:") {
return "html";
}
}
catch {
// Not a URL
}
return typeMap[ext] || "text";
}
/**
* Load document from file path, URL, or content
*
* Automatically detects the document type and uses the appropriate loader.
*
* @param source - File path, URL, or raw content
* @param options - Loader options
* @returns Promise resolving to MDocument
*
* @example
* ```typescript
* // Load from file
* const doc = await loadDocument('/path/to/document.md');
*
* // Load from URL
* const webDoc = await loadDocument('https://example.com/article');
*
* // Load with options
* const pdfDoc = await loadDocument('/path/to/doc.pdf', {
* pageRange: '1-5',
* metadata: { project: 'research' }
* });
* ```
*/
export async function loadDocument(source, options) {
// Find appropriate loader
const loader = loaderRegistry.find((l) => l.canHandle(source));
if (!loader) {
// Fall back to text loader
return new TextLoader().load(source, options);
}
logger.debug("[loadDocument] Loading document", {
source: source.slice(0, 100),
loaderType: loader.constructor.name,
});
return loader.load(source, options);
}
/**
* Load multiple documents
*
* @param sources - Array of file paths, URLs, or content
* @param options - Loader options (applied to all)
* @returns Promise resolving to array of MDocuments
*/
export async function loadDocuments(sources, options) {
const results = await Promise.allSettled(sources.map((source) => loadDocument(source, options)));
const documents = [];
const errors = [];
results.forEach((result, index) => {
if (result.status === "fulfilled") {
documents.push(result.value);
}
else {
errors.push({
source: sources[index],
error: result.reason instanceof Error
? result.reason.message
: String(result.reason),
});
}
});
if (errors.length > 0) {
logger.warn("[loadDocuments] Some documents failed to load", {
loaded: documents.length,
failed: errors.length,
errors,
});
}
return documents;
}