@goatlab/typesense
Version:
Modern TypeScript wrapper for Typesense search engine API
226 lines • 7.87 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ExportFormatter = void 0;
// ExportFormatter - Handles CSV/gzip helpers with streaming support
const stream_1 = require("stream");
const zlib_1 = require("zlib");
class ExportFormatter {
static formatDocuments(documents, format) {
switch (format) {
case 'json':
return documents;
case 'jsonl':
return documents.map(doc => JSON.stringify(doc)).join('\n');
case 'csv':
return this.formatCSV(documents);
default:
throw new Error(`Unsupported export format: ${format}`);
}
}
static formatCSV(documents) {
if (documents.length === 0) {
return '';
}
// Get all unique keys from all documents
const allKeys = new Set();
documents.forEach(doc => {
Object.keys(doc).forEach(key => allKeys.add(key));
});
const headers = Array.from(allKeys).sort();
const csvLines = [headers.join(',')];
for (const doc of documents) {
const row = headers.map(header => {
const value = doc[header];
return this.escapeCsvValue(value);
});
csvLines.push(row.join(','));
}
return csvLines.join('\n');
}
static createStreamingCSVTransform() {
let isFirstRow = true;
let headers = [];
return new stream_1.Transform({
objectMode: true,
transform(chunk, encoding, callback) {
try {
if (isFirstRow) {
// Extract headers from first document
headers = Object.keys(chunk).sort();
this.push(headers.join(',') + '\n');
isFirstRow = false;
}
// Convert document to CSV row
const row = headers.map(header => {
const value = chunk[header];
return ExportFormatter.escapeCsvValue(value);
});
this.push(row.join(',') + '\n');
callback();
}
catch (error) {
callback(error);
}
}
});
}
static createStreamingJSONLTransform() {
return new stream_1.Transform({
objectMode: true,
transform(chunk, encoding, callback) {
try {
this.push(JSON.stringify(chunk) + '\n');
callback();
}
catch (error) {
callback(error);
}
}
});
}
static createGzipStream() {
return (0, zlib_1.createGzip)();
}
static createDocumentParser(format) {
switch (format) {
case 'jsonl':
return this.createJSONLParser();
case 'json':
return this.createJSONParser();
default:
throw new Error(`Parsing not supported for format: ${format}`);
}
}
static createJSONLParser() {
let buffer = '';
return new stream_1.Transform({
objectMode: true,
transform(chunk, encoding, callback) {
buffer += chunk.toString();
const lines = buffer.split('\n');
// Keep the last incomplete line in buffer
buffer = lines.pop() || '';
for (const line of lines) {
if (line.trim()) {
try {
const document = JSON.parse(line);
this.push(document);
}
catch (error) {
return callback(new Error(`Invalid JSON in line: ${line}`));
}
}
}
callback();
},
flush(callback) {
if (buffer.trim()) {
try {
const document = JSON.parse(buffer);
this.push(document);
}
catch (error) {
return callback(new Error(`Invalid JSON in final line: ${buffer}`));
}
}
callback();
}
});
}
static createJSONParser() {
let buffer = '';
return new stream_1.Transform({
objectMode: true,
transform(chunk, encoding, callback) {
buffer += chunk.toString();
callback();
},
flush(callback) {
try {
const documents = JSON.parse(buffer);
if (Array.isArray(documents)) {
documents.forEach(doc => this.push(doc));
}
else {
this.push(documents);
}
}
catch (error) {
return callback(new Error(`Invalid JSON: ${error.message}`));
}
callback();
}
});
}
static escapeCsvValue(value) {
if (value === null || value === undefined) {
return '';
}
let stringValue = String(value);
// Handle arrays by joining with semicolons
if (Array.isArray(value)) {
stringValue = value.map(item => String(item)).join(';');
}
// Handle objects by stringifying
if (typeof value === 'object' && !Array.isArray(value)) {
stringValue = JSON.stringify(value);
}
// Escape quotes and wrap in quotes if needed
const needsQuoting = stringValue.includes(',') ||
stringValue.includes('"') ||
stringValue.includes('\n') ||
stringValue.includes('\r');
if (needsQuoting) {
// Escape existing quotes by doubling them
stringValue = stringValue.replace(/"/g, '""');
return `"${stringValue}"`;
}
return stringValue;
}
static async streamToString(stream) {
const chunks = [];
return new Promise((resolve, reject) => {
stream.on('data', chunk => chunks.push(chunk));
stream.on('error', reject);
stream.on('end', () => {
resolve(Buffer.concat(chunks).toString());
});
});
}
static async *streamToAsyncIterator(stream) {
const reader = stream[Symbol.asyncIterator]?.() || stream;
if (typeof reader[Symbol.asyncIterator] === 'function') {
for await (const chunk of reader) {
yield chunk;
}
}
else {
// Fallback for streams without async iterator support
const chunks = [];
await new Promise((resolve, reject) => {
stream.on('data', (chunk) => chunks.push(chunk));
stream.on('error', reject);
stream.on('end', resolve);
});
for (const chunk of chunks) {
yield chunk;
}
}
}
static createDocumentStream(documents) {
let index = 0;
return new stream_1.Readable({
objectMode: true,
read() {
if (index < documents.length) {
this.push(documents[index++]);
}
else {
this.push(null); // End stream
}
}
});
}
}
exports.ExportFormatter = ExportFormatter;
//# sourceMappingURL=export-formatter.js.map