@shutterstock/p-map-iterable
Version:
Set of classes used for async prefetching with backpressure (IterableMapper) and async flushing with backpressure (IterableQueueMapper, IterableQueueMapperSimple)
193 lines • 7.83 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.BlockingQueue = void 0;
const queue_1 = require("./queue");
/**
* `enqueue` blocks when the queue is full, until an item is read with `dequeue`, or done
* `dequeue` blocks when the queue is empty, until an item is added with `enqueue`, or done
* `done` must be called when no more items will be added
* `dequeue` returns `undefined` there are no more items and processing is complete
*
* @category Low-Level
*/
class BlockingQueue {
/**
* Create a new `BlockingQueue`
*
* @param options BlockingQueue options
*/
constructor(options = {}) {
// NOTE: unreadQueue has to be a proper FIFO queue with O(1) removal from the front
// as we may have O(m) unread items, where m may approach n, and Array.shift()
// would be O(m) (approaching O(n)) in that case as the array gets large.
this._unreadQueue = new queue_1.Queue();
// NOTE: Yes, it is possible to call next() multiple times in parallel (and we
// have a test that demonstrates that), so we may have multiple readers waiting
// and thus need a queue not just a single item.
// The items in the queue are 'resolve' functions that we call when an item
// is ready.
this._readersWaiting = new queue_1.Queue();
// NOTE: Yes, it is possible to call add() multiple times in parallel (and we
// have a test that demonstrates that), so we may have multiple writers waiting
// and thus need a queue not just a single item.
// The items in the queue are 'resolve' functions that we call when an item
// is ready.
this._writersWaiting = new queue_1.Queue();
this._doneAdding = false;
const { maxUnread = 8 } = options;
this._options = { maxUnread };
// Avoid undefined errors on options
if (this._options.maxUnread === undefined) {
throw new TypeError('`maxUnread` must be set');
}
// Validate maxUnread option
if (!((Number.isSafeInteger(this._options.maxUnread) ||
this._options.maxUnread === Number.POSITIVE_INFINITY) &&
this._options.maxUnread >= 0)) {
throw new TypeError(`Expected \`maxUnread\` to be an integer from 0 and up or \`Infinity\`, got \`${maxUnread}\` (${typeof maxUnread})`);
}
}
get length() {
return this._unreadQueue.length;
}
/**
* Indicate that no more items will be enqueued.
*
* This releases all readers blocked on `enqueue`
*/
done() {
// Just mark that we're done adding
this._doneAdding = true;
// If there are writers waiting then then they will release the waiting readers
// The reader that sees the writer count hit zero will clean up
if (this._writersWaiting.length !== 0) {
return;
}
// If there are no writers waiting then we release all the readers
this.releaseAllReaders();
}
/**
* Add an item to the queue, wait if the queue is full.
*
* @param item Element to add
*/
async enqueue(item) {
if (this._doneAdding) {
throw new Error('`enqueue` called after `done` called');
}
// We always add to the internal queue, we just wait to return if the queue is full
this._unreadQueue.enqueue(item);
if (this._readersWaiting.length > 0) {
// Release a reader if one is waiting for an item
// Remove the first waiting reader Promise from the queue
const reader = this._readersWaiting.dequeue();
if (reader === undefined) {
throw new TypeError('reader queue returned undefined');
}
// Resolve the Promise for a waiting reader
reader(undefined);
}
if (this._unreadQueue.length > this._options.maxUnread) {
// We wait if the queue is at max length
await this.waitForRead();
}
}
/**
* Gets an element when one is ready, waits if none are ready.
*
* @returns Element or undefined if queue is empty and `done` has been called
*/
async dequeue() {
// Bail out and release all waiters if there are no more items coming
const done = this.areWeDone();
if (done) {
return undefined;
}
// Check if queue has an item
let item;
// Get the item if one is ready
if (this._unreadQueue.length !== 0 && this._readersWaiting.length === 0) {
// If there are items and no waiters, pop one and return it
item = this._unreadQueue.dequeue();
}
else {
// If there are no items and/or there are already waiters, we have to wait for one to be ready
await this.waitForWrite();
item = this._unreadQueue.dequeue();
}
if (this._writersWaiting.length > 0) {
// Release a writer if one is waiting
// Remove the first waiting reader Promise from the queue
const writer = this._writersWaiting.dequeue();
if (writer === undefined) {
throw new TypeError('writer queue returned undefined');
}
// Resolve the Promise for a waiting writer
writer(undefined);
}
return item;
}
releaseAllReaders() {
// Release all the waiting readers since we're done
let waiter;
while ((waiter = this._readersWaiting.dequeue()) !== undefined) {
waiter(undefined);
}
}
areWeDone() {
if (this._doneAdding) {
// No more calls to `enqueue` will be made
if (this._writersWaiting.length === 0) {
// No `enqueue` calls are waiting to add an item
if (this._unreadQueue.length === 0) {
// There are no unread items left
this.releaseAllReaders();
return true;
}
}
}
return false;
}
// /**
// * Waits for an item to be ready in the queue
// * @returns Item from the ready queue
// */
// private async readWhenReady(): Promise<Element | undefined> {
// // If there are waiters, create a waiter promise and add to end of list
// // Note: the background reader removes the promise from the readers waiting queue
// const itemReady = new Promise((resolve) => {
// this._readersWaiting.enqueue(resolve);
// });
// // Wait for our item to be ready
// await itemReady;
// const item = this._unreadQueue.dequeue();
// // Pull the item from the front
// return item;
// }
/**
* Waits for an item to be ready in the queue
*/
async waitForWrite() {
// If there are waiters, create a waiter promise and add to end of list
// Note: the background reader removes the promise from the readers waiting queue
const itemReady = new Promise((resolve) => {
this._readersWaiting.enqueue(resolve);
});
// Wait for our item to be ready
await itemReady;
}
/**
* Waits to be able to add an item to the queue when full
*/
async waitForRead() {
// If there are waiters, create a waiter promise and add to end of list
// Note: the background reader removes the promise from the readers waiting queue
const writeReady = new Promise((resolve) => {
this._writersWaiting.enqueue(resolve);
});
// Wait for our turn to add to the queue
await writeReady;
}
}
exports.BlockingQueue = BlockingQueue;
//# sourceMappingURL=blocking-queue.js.map