iterparse/lib/buffer.js

Delightful data parsing

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.bufferWrite = exports.bufferRead = void 0;
const tslib_1 = require("tslib");
const fs_extra_1 = require("fs-extra");
const helpers_1 = require("./helpers");
const P = tslib_1.__importStar(require("ts-prime"));
const types_1 = require("./types");
const ts_prime_1 = require("ts-prime");
function _bufferIterParser(options) {
    return tslib_1.__asyncGenerator(this, arguments, function* _bufferIterParser_1() {
        var e_1, _a;
        const { progressFrequency = 3000 } = options || {};
        const size = fs_extra_1.statSync(options.filePath);
        const progress = new helpers_1.Progress(options.filePath, size.size, Date.now());
        const log = () => {
            var _a;
            (_a = options === null || options === void 0 ? void 0 : options.progress) === null || _a === void 0 ? void 0 : _a.call(options, progress);
        };
        const logTh = P.throttle(log, progressFrequency);
        try {
            for (var _b = tslib_1.__asyncValues(fs_extra_1.createReadStream(options.filePath)), _c; _c = yield tslib_1.__await(_b.next()), !_c.done;) {
                const buffer = _c.value;
                const b = buffer;
                yield yield tslib_1.__await(b);
                progress.addItem(1);
                progress.add(b.byteLength);
                logTh();
            }
        }
        catch (e_1_1) { e_1 = { error: e_1_1 }; }
        finally {
            try {
                if (_c && !_c.done && (_a = _b.return)) yield tslib_1.__await(_a.call(_b));
            }
            finally { if (e_1) throw e_1.error; }
        }
        log();
    });
}
function _bufferWrite(data, options) {
    function iter() {
        return tslib_1.__asyncGenerator(this, arguments, function* iter_1() {
            var e_2, _a;
            let dest = 0;
            const mode = options.mode || 'overwrite';
            if (mode === 'overwrite') {
                if (fs_extra_1.existsSync(options.filePath)) {
                    fs_extra_1.unlinkSync(options.filePath);
                }
            }
            const progress = new helpers_1.WriteProgress(options.filePath, Date.now());
            const log = () => {
                var _a;
                (_a = options.progress) === null || _a === void 0 ? void 0 : _a.call(options, progress);
            };
            const inter = setInterval(log, options.progressFrequency || 3000);
            try {
                for (var data_1 = tslib_1.__asyncValues(data), data_1_1; data_1_1 = yield tslib_1.__await(data_1.next()), !data_1_1.done;) {
                    const item = data_1_1.value;
                    if (dest === 0) {
                        yield tslib_1.__await(fs_extra_1.ensureFile(options.filePath));
                        dest = yield tslib_1.__await(fs_extra_1.open(options.filePath, "a"));
                    }
                    yield tslib_1.__await(fs_extra_1.appendFile(dest, item));
                    yield yield tslib_1.__await(Buffer.from(item));
                }
            }
            catch (e_2_1) { e_2 = { error: e_2_1 }; }
            finally {
                try {
                    if (data_1_1 && !data_1_1.done && (_a = data_1.return)) yield tslib_1.__await(_a.call(data_1));
                }
                finally { if (e_2) throw e_2.error; }
            }
            clearInterval(inter);
            log();
        });
    }
    return types_1.IX.from(iter());
}
/**
 * Function will read big files in memory efficient way.
 * @include ./BufferReadOptions.md
 * @example
 *  import { bufferRead } from 'iterparse'
 *
 *  bufferRead({ filePath: "path/to/file" })
 *      .map((buffer)=> console.log(buffer.byteLength))
 *      .count()
 * @example
 *  import { bufferRead } from 'iterparse'
 *
 *  for await (const buffer of bufferRead({ filePath: "path/to/file" })) {
 *      console.log(q.byteLength)
 *  }
 * @category Buffer
 */
function bufferRead(options) {
    return types_1.IX.from(_bufferIterParser(options));
}
exports.bufferRead = bufferRead;
function bufferWrite() {
    return ts_prime_1.purry(_bufferWrite, arguments);
}
exports.bufferWrite = bufferWrite;