@beenotung/tslib/csv-file.js

utils library in Typescript

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.write_csv_file = write_csv_file;
exports.write_csv_file_sync = write_csv_file_sync;
exports.read_csv_file = read_csv_file;
exports.read_csv_file_sync = read_csv_file_sync;
const tslib_1 = require("tslib");
/**
 * for processing large data set, then will run out of memory if use csv.ts directly
 * */
const fs_1 = tslib_1.__importDefault(require("fs"));
const csv_1 = require("./csv");
const file_stream_1 = require("./file-stream");
function wrapLine(s) {
    return s.replace(/\n/g, '\\n');
}
function unwrapLine(s) {
    return s.replace(/\\n/g, '\n');
}
function wrapObjectLine(o, titles) {
    const res = {};
    for (const key of titles) {
        let value = o[key];
        if (typeof value === 'string') {
            value = wrapLine(value);
        }
        res[key] = value;
    }
    return res;
}
function unwrapObjectLine(o, titles) {
    for (const key of titles) {
        let value = o[key];
        if (typeof value === 'string') {
            value = unwrapLine(value);
            o[key] = value;
        }
    }
}
function write_csv_file(filename, records, options) {
    return new Promise((resolve, reject) => {
        let stream;
        try {
            stream = fs_1.default.createWriteStream(filename);
            const is_wrap_line = options?.wrap_line;
            const separator = options?.separator;
            const delimiter = options?.delimiter;
            let titles = options?.titles || (0, csv_1.json_to_csv_titles)(records);
            if (is_wrap_line) {
                titles = titles.map(wrapLine);
            }
            let first = true;
            for (let record of records) {
                if (is_wrap_line) {
                    record = wrapObjectLine(record, titles);
                }
                let rows = (0, csv_1.json_to_csv)([record], titles);
                if (first) {
                    first = false; // include titles
                    if (options?.skip_titles) {
                        rows = [rows[1]]; // skip titles
                    }
                }
                else {
                    rows = [rows[1]]; // skip titles
                }
                const text = (0, csv_1.to_csv)(rows, separator, delimiter);
                stream.write(text);
            }
            stream.close();
            stream.on('close', resolve);
            stream.on('error', reject);
        }
        catch (e) {
            stream?.close();
            reject(e);
        }
    });
}
function write_csv_file_sync(filename, records, options) {
    const is_wrap_line = options?.wrap_line;
    const separator = options?.separator;
    const delimiter = options?.delimiter;
    let titles = options?.titles || (0, csv_1.json_to_csv_titles)(records);
    if (is_wrap_line) {
        titles = titles.map(wrapLine);
    }
    let first = true;
    fs_1.default.writeFileSync(filename, '');
    for (let record of records) {
        if (is_wrap_line) {
            record = wrapObjectLine(record, titles);
        }
        let rows = (0, csv_1.json_to_csv)([record], titles);
        if (first) {
            first = false; // include titles
            if (options?.skip_titles) {
                rows = [rows[1]]; // skip titles
            }
        }
        else {
            rows = [rows[1]]; // skip titles
        }
        const text = (0, csv_1.to_csv)(rows, separator, delimiter);
        fs_1.default.appendFileSync(filename, text);
    }
}
async function* read_csv_file(filename, options) {
    const separator = options?.separator;
    const delimiter = options?.delimiter;
    const is_wrap_line = options?.wrap_line;
    const stream = fs_1.default.createReadStream(filename);
    let titles;
    let first = true;
    for await (const line of (0, file_stream_1.stream_lines)(stream)) {
        if (first) {
            titles = (0, csv_1.from_csv)(line, separator, delimiter)[0];
            if (is_wrap_line) {
                titles = titles.map(unwrapLine);
            }
            first = false;
            continue;
        }
        const text = line + '\n';
        const records = (0, csv_1.csv_to_json)((0, csv_1.from_csv)(text, separator, delimiter), titles);
        const record = records[0];
        if (record) {
            if (is_wrap_line) {
                unwrapObjectLine(record, titles);
            }
            yield record;
        }
    }
    stream.close();
}
function* read_csv_file_sync(filename, options) {
    const separator = options?.separator;
    const delimiter = options?.delimiter;
    const is_wrap_line = options?.wrap_line;
    const text = fs_1.default.readFileSync(filename).toString();
    const lines = text.split('\n');
    let titles = (0, csv_1.from_csv)(lines[0])[0];
    if (is_wrap_line) {
        titles = titles.map(unwrapLine);
    }
    for (let i = 1; i < lines.length; i++) {
        const text = lines[i] + '\n';
        const records = (0, csv_1.csv_to_json)((0, csv_1.from_csv)(text, separator, delimiter), titles);
        const record = records[0];
        if (record) {
            if (is_wrap_line) {
                unwrapObjectLine(record, titles);
            }
            yield record;
        }
    }
}