iterparse/lib/line.js

Delightful data parsing

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.lineWrite = exports.lineRead = exports._lineIterParser = void 0;
const tslib_1 = require("tslib");
const helpers_1 = require("./helpers");
const fs_extra_1 = require("fs-extra");
const buffer_1 = require("./buffer");
const ts_prime_1 = require("ts-prime");
const types_1 = require("./types");
function _lineIterParser(options) {
    return tslib_1.__asyncGenerator(this, arguments, function* _lineIterParser_1() {
        var e_1, _a;
        const size = fs_extra_1.statSync(options.filePath);
        const { lineBreak = '\n' } = options || {};
        const progress = new helpers_1.Progress(options.filePath, size.size, Date.now());
        let prev = "";
        try {
            for (var _b = tslib_1.__asyncValues(buffer_1.bufferRead(Object.assign(Object.assign({}, options), { progress: (q) => {
                    var _a;
                    q.set({
                        items: progress.items
                    });
                    (_a = options === null || options === void 0 ? void 0 : options.progress) === null || _a === void 0 ? void 0 : _a.call(options, q);
                } }))), _c; _c = yield tslib_1.__await(_b.next()), !_c.done;) {
                const buffer = _c.value;
                const lineList = buffer.toString().split(lineBreak);
                prev = lineList.pop() || '';
                for (const l of lineList) {
                    yield yield tslib_1.__await(l);
                    progress.addItem(1);
                }
            }
        }
        catch (e_1_1) { e_1 = { error: e_1_1 }; }
        finally {
            try {
                if (_c && !_c.done && (_a = _b.return)) yield tslib_1.__await(_a.call(_b));
            }
            finally { if (e_1) throw e_1.error; }
        }
    });
}
exports._lineIterParser = _lineIterParser;
/**
 * Function will read file line by line
 * @param options - optional options list.
 * @include ./LineReadOptions.md
 * @example
 *  import { lineRead } from 'iterparse'
 *  lineRead({ filePath: "path/to/file" })
 *      .map((q)=> console.log(q))
 *      .count()
 * @example
 *  import { lineRead } from 'iterparse'
 *  for await (const line of lineRead({ filePath: "path/to/file" })) {
 *
 *  }
 * @category Line
 */
function lineRead(options) {
    return types_1.IX.from(_lineIterParser(options));
}
exports.lineRead = lineRead;
function _lineIterWriter(data, options) {
    function iter() {
        return tslib_1.__asyncGenerator(this, arguments, function* iter_1() {
            var e_2, _a;
            let destination = 0;
            const { mode = 'overwrite' } = options;
            if (mode === 'overwrite') {
                if (fs_extra_1.existsSync(options.filePath)) {
                    fs_extra_1.unlinkSync(options.filePath);
                }
            }
            const progress = new helpers_1.WriteProgress(options.filePath, Date.now());
            const log = () => {
                var _a;
                (_a = options.progress) === null || _a === void 0 ? void 0 : _a.call(options, progress);
            };
            const inter = setInterval(log, options.progressFrequency || 3000);
            try {
                for (var data_1 = tslib_1.__asyncValues(data), data_1_1; data_1_1 = yield tslib_1.__await(data_1.next()), !data_1_1.done;) {
                    const line = data_1_1.value;
                    if (destination === 0) {
                        yield tslib_1.__await(fs_extra_1.ensureFile(options.filePath));
                        destination = yield tslib_1.__await(fs_extra_1.open(options.filePath, "a"));
                    }
                    if (!ts_prime_1.isString(line)) {
                        throw new Error(`Incorrect data type. Line type is: "${ts_prime_1.type(line)}"`);
                    }
                    const buffer = Buffer.from(`${line}\n`);
                    progress.add(buffer.byteLength);
                    yield tslib_1.__await(fs_extra_1.appendFile(destination, buffer));
                    progress.addItem(1);
                }
            }
            catch (e_2_1) { e_2 = { error: e_2_1 }; }
            finally {
                try {
                    if (data_1_1 && !data_1_1.done && (_a = data_1.return)) yield tslib_1.__await(_a.call(data_1));
                }
                finally { if (e_2) throw e_2.error; }
            }
            clearInterval(inter);
            log();
        });
    }
    return types_1.IX.from(iter());
}
function lineWrite() {
    return ts_prime_1.purry(_lineIterWriter, arguments);
}
exports.lineWrite = lineWrite;