disk-fastq/dist/index.js

fastq wrapper persisted queue on disk to save some memory

"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }// src/index.ts
var _fastq = require('fastq'); var _fastq2 = _interopRequireDefault(_fastq);
var _diskqueue = require('disk-queue');
var _stream = require('stream');
var DiskFastq = class extends _stream.EventEmitter {
  constructor(...args) {
    super();
    this.isSaturated = false;
    this.closed = false;
    this.numInMemory = 0;
    let context = null;
    let worker;
    let concurrency;
    let diskQueueOptions;
    let callback = () => {
    };
    if (args.length === 3) {
      [worker, concurrency, diskQueueOptions] = args;
    } else if (args.length === 4) {
      if (typeof args[args.length - 1] !== "function") {
        [context, worker, concurrency, diskQueueOptions] = args;
      } else {
        [worker, concurrency, diskQueueOptions, callback] = args;
      }
    } else {
      [context, worker, concurrency, diskQueueOptions, callback] = args;
    }
    const fastq = _fastq2.default.call(void 0, context, worker, concurrency);
    const queue = new (0, _diskqueue.DiskQueue)(diskQueueOptions);
    this.queue = queue;
    this.fastq = fastq;
    this.concurrency = concurrency;
    this.diskQueueOptions = diskQueueOptions;
    this.jobCallback = (err, result, task) => {
      this.numInMemory--;
      callback(err, result, task);
    };
    this.jobCallback.bind(this);
    this.fastq.drain = this.onFastqDrain.bind(this);
    this.fastq.saturated = () => {
      this.isSaturated = true;
    };
  }
  addDiskTaskToMemory() {
    while (this.numInMemory < this.concurrency && this.queue.remainCount > 0) {
      const data = this.queue.shift();
      this.fastq.push(data, (err, result) => {
        this.jobCallback(err, result, data);
        this.addDiskTaskToMemory();
      });
      this.numInMemory++;
    }
  }
  onFastqDrain() {
    if (this.closed) {
      if (this.queue.remainCount === 0) {
        this.queue.close();
        this.emit("drain");
      }
    }
    this.isSaturated = false;
    this.addDiskTaskToMemory();
  }
  push(arg) {
    if (this.closed) {
      return;
    }
    if (this.numInMemory < this.concurrency && this.queue.remainCount <= 0) {
      this.fastq.push(arg, (err, result) => {
        this.jobCallback(err, result, arg);
        this.addDiskTaskToMemory();
      });
      this.numInMemory++;
    } else {
      this.addDiskTaskToMemory();
      this.queue.push(arg);
    }
  }
  close() {
    this.closed = true;
  }
  reset() {
    this.fastq.kill();
    this.fastq.drain = this.onFastqDrain.bind(this);
    this.queue.reset();
  }
  get length() {
    return this.fastq.length() + this.queue.remainCount;
  }
};


exports.DiskFastq = DiskFastq;