@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
222 lines (221 loc) • 10.2 kB
TypeScript
/**
* CSV Processing Utility
* Converts CSV files to LLM-friendly text formats
* Uses streaming for memory efficiency with large files
*/
import type { FileProcessingResult, CSVFileOpenOptions, CSVProcessorOptions, CSVRow } from "../types/index.js";
/**
* Rewrite a raw header into a valid identifier. Trims, replaces non-identifier
* runs with `_`, prefixes a `col_` when it starts with a digit or is empty,
* then applies the requested case style. Never returns an empty string.
*/
export declare function sanitizeColumnName(name: string, style?: "camelCase" | "snake_case", index?: number): string;
/**
* Deduplicate a list of (already sanitized) names by suffixing `_2`, `_3`, … on
* collisions, preserving order. The suffixed candidate is bumped until it does
* not collide with ANY already-emitted name — so an existing `name_2` can't be
* silently overwritten by a generated `name_2` (which the naive counter did for
* `["name","name","name_2"]`).
*/
export declare function dedupeColumnNames(names: string[]): string[];
/**
* Predicate form of the row-shape invariant: a non-null, non-array object whose
* every value is a string or undefined. csv-parser's default output always
* satisfies this; the guard is defense-in-depth at the parse boundary.
*/
export declare function isValidCsvRow(row: unknown): row is CSVRow;
/**
* Assertion form used inside the streaming `data` handlers. Throws a
* `[CSVProcessor]`-prefixed error (so existing `includes("CSV")` catches still
* match) when a row violates the invariant.
*/
export declare function assertValidCsvRow(row: unknown, rowNumber: number): asserts row is CSVRow;
/**
* True when a parsed CSV row is blank: no keys, or every value is empty /
* whitespace-only (#373). Shared by the structured post-filter and by
* `streamParse` so blank rows do not consume `maxRows`.
*/
export declare function isBlankCsvDataRow(row: CSVRow): boolean;
/**
* Raw-line counterpart of {@link isBlankCsvDataRow} (#373): a fully blank /
* whitespace line, or a delimiter-only line such as `,,` / ` , ` whose
* fields are all empty after a CSV-aware split.
*/
export declare function isBlankCsvRawLine(line: string, delimiter: string): boolean;
/**
* Detect if first line is CSV metadata (not actual data/headers)
* Common patterns:
* - Excel separator line: "SEP=,"
* - Lines with significantly different delimiter count than line 2
* - Lines that don't match CSV structure of subsequent lines
*/
/**
* Strip a leading UTF-8 BOM (U+FEFF) if present. Exported so every entry
* point that sniffs raw text (including the RAG CSVLoader) normalizes BOMs
* the same way before metadata/delimiter detection.
*/
export declare function stripBom(text: string): string;
/**
* Strip a leading metadata line (e.g. Excel's `sep=;` preamble) from
* already-split lines, and read the delimiter it explicitly declares.
* Excel semantics: an explicit `sep=<char>` always wins over frequency-based
* detection; a metadata line without a recognized delimiter character is
* simply dropped from the sample without forcing a delimiter. This is the
* single place process(), detectDelimiter(), parseCSVFile(), and
* parseCSVString() all go through, so none of them detect a delimiter over
* the raw, un-stripped lines (which lets the preamble skew detection).
* Exported so non-CSVProcessor callers (the RAG CSVLoader) share it too.
*/
export declare function stripMetadataLine(lines: string[]): {
dataLines: string[];
hasMetadataLine: boolean;
explicitDelimiter?: string;
};
/**
* Split CSV text into logical rows for metadata detection and raw row limiting.
*
* Supports Unix (LF), Windows (CRLF), and classic Mac (CR) line endings, and is
* quote-aware: a line ending that appears *inside* a quoted field (RFC 4180) is
* kept as part of that field rather than treated as a row boundary. A doubled
* `""` inside quotes is the RFC-4180 escaped quote and does not toggle the quote
* state. For unquoted input this yields exactly the same result as a plain
* `split(/\r\n|\n|\r/)`.
*
* Exported so non-CSVProcessor callers (the RAG CSVLoader) get the same
* quote-aware row boundaries instead of a physical `content.split("\n")`,
* which would break rows on a newline embedded in a quoted field.
*/
export declare function splitCsvLines(csvString: string): string[];
/**
* CSV processor for converting CSV data to LLM-optimized formats
*
* Supports three output formats:
* - raw: Original CSV format with proper escaping (RECOMMENDED for best LLM performance)
* - json: JSON array format (best for structured data processing)
* - markdown: Markdown table format (best for small datasets <100 rows)
*
* All formats use csv-parser for reliable parsing, then convert to the target format.
*
* @example
* ```typescript
* const csvBuffer = Buffer.from('name,age\nAlice,30\nBob,25');
* const result = await CSVProcessor.process(csvBuffer, {
* maxRows: 1000,
* formatStyle: 'raw'
* });
* console.log(result.content); // CSV string with proper escaping
* ```
*/
export declare class CSVProcessor {
/**
* Process CSV Buffer to LLM-friendly format
* Content already loaded by FileDetector
*
* @param content - CSV file as Buffer
* @param options - Processing options
* @returns Formatted CSV data ready for LLM (JSON or Markdown)
*/
static process(content: Buffer, options?: CSVProcessorOptions): Promise<FileProcessingResult>;
/**
* Detect the delimiter of DSV content (comma/tab/semicolon/pipe), respecting
* RFC-4180 quoting. An optional extension hint (`tsv` → tab, `psv` → pipe)
* breaks ambiguity. Used by callers (e.g. the RAG CSV loader) that parse
* DSV content themselves and need the same detection (#361). A leading
* Excel `sep=` metadata line is stripped before sampling so it can't skew
* detection, and an explicit `sep=<char>` value wins outright.
*/
static detectDelimiter(csvString: string, extensionHint?: string): string;
/**
* Parse CSV file from disk using streaming (memory efficient)
*
* @param filePath - Path to CSV file
* @param maxRows - Maximum rows to parse (default: 1000)
* @returns Array of row objects
*/
static parseCSVFile(filePath: string, maxRows?: number, timeoutMs?: number, openOptions?: CSVFileOpenOptions): Promise<CSVRow[]>;
/**
* File-parse variant that also reports whether the parse timed out, used by
* `process()` to surface `metadata.parseTimedOut`.
*/
static parseCSVFileWithMeta(filePath: string, maxRows?: number, timeoutMs?: number, encoding?: string, openOptions?: CSVFileOpenOptions): Promise<{
rows: CSVRow[];
timedOut: boolean;
}>;
/**
* Peek the head of a raw file stream to resolve encoding, metadata-line
* skipping, and the delimiter (#361), then return a decoded text stream
* that still delivers the whole file from byte 0 (via `unshift`) — a
* single disk read (#368/#362).
*/
private static prepareFileSource;
/**
* Parse CSV string to array of row objects
* Exposed for use by tools that need direct CSV parsing
*
* @param csvString - CSV data as string
* @param maxRows - Maximum rows to parse (default: 1000)
* @param timeoutMs - Wall-clock parse cap; returns partial rows on timeout
* @returns Array of row objects
*/
static parseCSVString(csvString: string, maxRows?: number, timeoutMs?: number): Promise<CSVRow[]>;
/**
* String-parse variant that also reports whether the parse timed out, used by
* `process()` to surface `metadata.parseTimedOut`.
*/
static parseCSVStringWithMeta(csvString: string, maxRows?: number, timeoutMs?: number): Promise<{
rows: CSVRow[];
timedOut: boolean;
}>;
/**
* Shared streaming consumer for both parse methods. Guarantees:
* - source, rawSource, and parser are all destroyed on ANY settle path
* (#371) — `rawSource` matters for the file-parse path, where `source`
* is a decode Transform piped FROM `rawSource`; `.pipe()` does not
* propagate `.destroy()` upstream, so without this the underlying
* `fs.createReadStream` fd would leak on abort/timeout/error,
* - a wall-clock timeout returns partial rows instead of hanging (#379),
* - every row is validated at the boundary (#384),
* - reject messages carry columns/last-row-shape/format context (#375),
* - the delimiter (#361, comma/tab/semicolon/pipe) detected by the caller
* is honored, instead of csv-parser's hard-coded comma default.
*
* @param rawSource - The original file stream `source` was piped from, when
* applicable (file-parse path only; `undefined` for string parsing).
*/
private static streamParse;
/**
* Format parsed CSV data for LLM consumption
* Only used for JSON and Markdown formats (raw format handled separately)
*
* @param columnNames - Optional schema headers (#373). When preserved blank
* rows lead `rows`, Markdown must still use the real column names.
*/
private static formatForLLM;
/**
* Format as markdown table
* Best for small datasets (<100 rows)
*
* @param columnNames - Optional explicit headers (#373). Falls back to
* `Object.keys(rows[0])` when omitted.
*/
private static toMarkdownTable;
/**
* Format sample data according to the specified format
*
* @param sampleRows - Array of sample row objects
* @param format - Output format for sample data
* @param includeHeaders - Whether to include headers in CSV/markdown formats
* @param columnNames - Optional schema headers for csv/markdown (#373)
* @returns Formatted sample data as string or array
*/
private static formatSampleData;
/**
* Convert row objects to CSV string format
*
* @param rows - Array of row objects
* @param includeHeaders - Whether to include header row
* @param columnNames - Optional explicit headers (#373)
* @returns CSV formatted string
*/
private static toCSVString;
}