linereader2/lib/LineReader.js

Asynchronous, buffered, chunk-by-chunk file reader with customizable buffer size on Node.js.

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __generator = (this && this.__generator) || function (thisArg, body) {
    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
    function verb(n) { return function (v) { return step([n, v]); }; }
    function step(op) {
        if (f) throw new TypeError("Generator is already executing.");
        while (g && (g = 0, op[0] && (_ = 0)), _) try {
            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
            if (y = 0, t) op = [op[0] & 2, t.value];
            switch (op[0]) {
                case 0: case 1: t = op; break;
                case 4: _.label++; return { value: op[1], done: false };
                case 5: _.label++; y = op[1]; op = [0]; continue;
                case 7: op = _.ops.pop(); _.trys.pop(); continue;
                default:
                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
                    if (t[2]) _.ops.pop();
                    _.trys.pop(); continue;
            }
            op = body.call(thisArg, _);
        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
    }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
/* eslint-disable @typescript-eslint/no-non-null-assertion */
var async_mutex_1 = require("async-mutex");
var chunkreader2_1 = require("chunkreader2");
var denque_1 = __importDefault(require("denque"));
var typedfastbitset_1 = require("typedfastbitset");
var LineReader = /** @class */ (function () {
    function LineReader(options) {
        this.options = options;
        this.separator = this.options.lineSeparator;
        this.reader = new chunkreader2_1.ChunkReader(options);
        this.buffer = new denque_1.default();
        this.mutex = new async_mutex_1.Mutex();
        this.count = 0;
        this.skips = new typedfastbitset_1.TypedFastBitSet();
        if (this.options.skipNumbers) {
            for (var _i = 0, _a = this.options.skipNumbers; _i < _a.length; _i++) {
                var skipNumber = _a[_i];
                if (Array.isArray(skipNumber)) {
                    this.skips.addRange(skipNumber[0], skipNumber[1] + 1);
                }
                else {
                    this.skips.add(skipNumber);
                }
            }
        }
    }
    Object.defineProperty(LineReader.prototype, "filePath", {
        get: function () {
            return this.reader.filePath;
        },
        enumerable: false,
        configurable: true
    });
    Object.defineProperty(LineReader.prototype, "bytesLength", {
        get: function () {
            return this.reader.bytesLength;
        },
        enumerable: false,
        configurable: true
    });
    Object.defineProperty(LineReader.prototype, "lineSeparator", {
        get: function () {
            return this.separator;
        },
        enumerable: false,
        configurable: true
    });
    Object.defineProperty(LineReader.prototype, "linesRead", {
        get: function () {
            return this.count;
        },
        enumerable: false,
        configurable: true
    });
    Object.defineProperty(LineReader.prototype, "bytesRead", {
        get: function () {
            return this.reader.bytesRead;
        },
        enumerable: false,
        configurable: true
    });
    Object.defineProperty(LineReader.prototype, "isClosed", {
        get: function () {
            return this.reader.isClosed && this.cutted === undefined && this.buffer.length === 0;
        },
        enumerable: false,
        configurable: true
    });
    Object.defineProperty(LineReader.prototype, "isOpened", {
        get: function () {
            return this.reader.isOpened;
        },
        enumerable: false,
        configurable: true
    });
    LineReader.prototype._determineSeparator = function () {
        return __awaiter(this, void 0, void 0, function () {
            var separator, chunk, middleChunk;
            return __generator(this, function (_a) {
                switch (_a.label) {
                    case 0:
                        if (!!this.reader.isClosed) return [3 /*break*/, 2];
                        return [4 /*yield*/, this.reader.read()];
                    case 1:
                        chunk = _a.sent();
                        if (chunk.lastIndexOf('\r\n') !== -1) {
                            separator = '\r\n';
                            return [3 /*break*/, 2];
                        }
                        middleChunk = chunk.slice(1, chunk.length - 1);
                        if (middleChunk.lastIndexOf('\n') !== -1) {
                            separator = '\n';
                            return [3 /*break*/, 2];
                        }
                        if (middleChunk.lastIndexOf('\r') !== -1) {
                            separator = '\r';
                            return [3 /*break*/, 2];
                        }
                        return [3 /*break*/, 0];
                    case 2:
                        this.reader.reset();
                        this.separator = separator || '\r\n';
                        return [2 /*return*/];
                }
            });
        });
    };
    LineReader.prototype._nextChunk = function () {
        return __awaiter(this, void 0, void 0, function () {
            var chunks, cutAt, cutted, chunk, foundAt, data;
            return __generator(this, function (_a) {
                switch (_a.label) {
                    case 0:
                        if (!!this.separator) return [3 /*break*/, 2];
                        return [4 /*yield*/, this._determineSeparator()];
                    case 1:
                        _a.sent();
                        _a.label = 2;
                    case 2:
                        chunks = '';
                        cutAt = 0;
                        cutted = false;
                        _a.label = 3;
                    case 3:
                        if (!!this.reader.isClosed) return [3 /*break*/, 5];
                        return [4 /*yield*/, this.reader.read()];
                    case 4:
                        chunk = _a.sent();
                        foundAt = chunk.lastIndexOf(this.separator);
                        chunks += chunk;
                        if (foundAt !== -1 && foundAt < chunk.length - this.separator.length) {
                            cutAt = chunks.length - chunk.length + foundAt;
                            cutted = true;
                            return [3 /*break*/, 5];
                        }
                        return [3 /*break*/, 3];
                    case 5:
                        data = (this.cutted || '') + (cutted ? chunks.slice(0, cutAt) : chunks);
                        if (this.reader.isClosed) {
                            if (cutted)
                                data += chunks.slice(cutAt);
                            this.cutted = undefined;
                        }
                        else {
                            this.cutted = cutted ? chunks.slice(cutAt + this.separator.length) : undefined;
                        }
                        return [2 /*return*/, data];
                }
            });
        });
    };
    LineReader.prototype.readLinesWithNumbers = function (limit) {
        return __awaiter(this, void 0, void 0, function () {
            var _this = this;
            return __generator(this, function (_a) {
                switch (_a.label) {
                    case 0: return [4 /*yield*/, this.mutex.runExclusive(function () { return __awaiter(_this, void 0, void 0, function () {
                            var output, texts, _i, texts_1, text, number, lines;
                            return __generator(this, function (_a) {
                                switch (_a.label) {
                                    case 0:
                                        if (!this.reader.isOpened)
                                            this.reader.open();
                                        if (!this.reader.bytesLength)
                                            return [2 /*return*/, []];
                                        if (!(!limit || limit > this.buffer.length)) return [3 /*break*/, 2];
                                        return [4 /*yield*/, this._nextChunk()];
                                    case 1:
                                        output = (_a.sent()) || '';
                                        if (output || !this.isClosed) {
                                            texts = output.split(this.separator);
                                            for (_i = 0, texts_1 = texts; _i < texts_1.length; _i++) {
                                                text = texts_1[_i];
                                                number = ++this.count;
                                                if (this.options.skipNumbers && this.skips.has(number))
                                                    continue;
                                                if (this.options.skipBlank && !text.trim().length)
                                                    continue;
                                                this.buffer.push([text, number]);
                                            }
                                        }
                                        _a.label = 2;
                                    case 2:
                                        lines = this.buffer.remove(0, limit || this.buffer.length) || [];
                                        return [2 /*return*/, lines];
                                }
                            });
                        }); })];
                    case 1: return [2 /*return*/, _a.sent()];
                }
            });
        });
    };
    LineReader.prototype.readLines = function (limit) {
        return __awaiter(this, void 0, void 0, function () {
            var lines;
            return __generator(this, function (_a) {
                switch (_a.label) {
                    case 0: return [4 /*yield*/, this.readLinesWithNumbers(limit)];
                    case 1:
                        lines = _a.sent();
                        return [2 /*return*/, lines.map(function (_a) {
                                var text = _a[0];
                                return text;
                            })];
                }
            });
        });
    };
    LineReader.prototype.readLineWithNumber = function () {
        return __awaiter(this, void 0, void 0, function () {
            var line;
            return __generator(this, function (_a) {
                switch (_a.label) {
                    case 0:
                        line = undefined;
                        _a.label = 1;
                    case 1:
                        if (!(!line && !this.isClosed)) return [3 /*break*/, 3];
                        return [4 /*yield*/, this.readLinesWithNumbers(1)];
                    case 2:
                        line = (_a.sent())[0];
                        return [3 /*break*/, 1];
                    case 3: return [2 /*return*/, line];
                }
            });
        });
    };
    LineReader.prototype.readLine = function () {
        return __awaiter(this, void 0, void 0, function () {
            var line;
            return __generator(this, function (_a) {
                switch (_a.label) {
                    case 0: return [4 /*yield*/, this.readLineWithNumber()];
                    case 1:
                        line = _a.sent();
                        return [2 /*return*/, line !== undefined ? line[0] : undefined];
                }
            });
        });
    };
    LineReader.prototype.resetReader = function () {
        this.reader.reset();
        this.buffer.clear();
        delete this.cutted;
        delete this.separator;
        this.count = 0;
    };
    LineReader.prototype.openReader = function () {
        this.reader.open();
    };
    LineReader.prototype.closeReader = function () {
        this.reader.close();
    };
    return LineReader;
}());
exports.default = LineReader;