@push.rocks/smartstream/ts/smartstream.classes.smartduplex.ts

A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.

import * as plugins from './smartstream.plugins.js';
import { Duplex, type DuplexOptions } from 'stream';

export interface IStreamTools {
  truncate: () => void;
  push: (pipeObject: any) => Promise<boolean>;
}

export interface IStreamWriteFunction<T, rT> {
  (chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
}

export interface IStreamFinalFunction<rT> {
  (toolsArg: IStreamTools): Promise<rT>;
}

export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
  /**
   * wether to print debug logs
   */
  debug?: boolean;
  /**
   * the name of the stream
   */
  name?: string;
  /**
   * a function that is being called to read more stuff from whereever to be processed by the stream
   * @returns
   */
  readFunction?: () => Promise<void>;

  /**
   * the write function is called for every chunk that is being written to the stream
   * it can push or return chunks (but does not have to) to be written to the readable side of the stream
   */
  writeFunction?: IStreamWriteFunction<TInput, TOutput>;

  /**
   * a final function that is run at the end of the stream
   */
  finalFunction?: IStreamFinalFunction<TOutput>;
}

export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
  // STATIC
  static fromBuffer(buffer: Buffer, options?: ISmartDuplexOptions<any, any>): SmartDuplex {
    const smartDuplex = new SmartDuplex(options);
    process.nextTick(() => {
      smartDuplex.push(buffer);
      smartDuplex.push(null); // Signal the end of the data
    });
    return smartDuplex;
  }

  public static fromWebReadableStream<T = any>(
    readableStream: ReadableStream<T>
  ): SmartDuplex<T, T> {
    const smartDuplex = new SmartDuplex<T, T>({
      /**
       * this function is called whenever the stream is being read from and at the same time if nothing is enqueued
       * therefor it is important to always unlock the reader after reading
       */
      readFunction: async () => {
        const reader = readableStream.getReader();
        const { value, done } = await reader.read();
        if (value !== undefined) {
          smartDuplex.push(value);
        }
        reader.releaseLock();
        if (done) {
          smartDuplex.push(null);
        }
      },
    });
    return smartDuplex;
  }

  // INSTANCE
  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>; // an array that only takes a defined amount of items
  public options: ISmartDuplexOptions<TInput, TOutput>;
  private observableSubscription?: plugins.smartrx.rxjs.Subscription;
  private debugLog(messageArg: string) {
    // optional debug log
    if (this.options.debug) {
      console.log(messageArg);
    }
  }

  constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
    super(
      Object.assign(
        {
          highWaterMark: 1,
        },
        optionsArg
      )
    );
    this.options = optionsArg;
    this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(
      this.options.highWaterMark || 1
    );
  }

  public async _read(size: number): Promise<void> {
    this.debugLog(`${this.options.name}: read was called`);
    if (this.options.readFunction) {
      await this.options.readFunction();
    }
    await this.backpressuredArray.waitForItems();
    this.debugLog(`${this.options.name}: successfully waited for items.`);
    let canPushMore = true;
    while (this.backpressuredArray.data.length > 0 && canPushMore) {
      const nextChunk = this.backpressuredArray.shift();
      canPushMore = this.push(nextChunk);
    }
  }

  public async backpressuredPush(pushArg: TOutput) {
    const canPushMore = this.backpressuredArray.push(pushArg);
    if (!canPushMore) {
      this.debugLog(`${this.options.name}: cannot push more`);
      await this.backpressuredArray.waitForSpace();
      this.debugLog(`${this.options.name}: can push more again`);
    }
    return canPushMore;
  }

  private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
  // Ensure the _write method types the chunk as TInput and encodes TOutput
  public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
    if (!this.options.writeFunction) {
      return callback(new Error('No stream function provided'));
    }

    let isTruncated = false;
    const tools: IStreamTools = {
      truncate: () => {
        this.push(null);
        isTruncated = true;
        callback();
      },
      push: async (pushArg: TOutput) => {
        return await this.backpressuredPush(pushArg);
      },
    };

    try {
      const writeDeferred = plugins.smartpromise.defer();
      this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
      const modifiedChunk = await this.options.writeFunction(chunk, tools);
      if (isTruncated) {
        return;
      }
      if (modifiedChunk) {
        await tools.push(modifiedChunk);
      }
      callback();
      writeDeferred.resolve();
      writeDeferred.promise.then(() => {
        this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
      });
    } catch (err) {
      callback(err);
    }
  }

  public async _final(callback: (error?: Error | null) => void) {
    await Promise.all(this.asyncWritePromiseObjectmap.getArray());
    if (this.options.finalFunction) {
      const tools: IStreamTools = {
        truncate: () => callback(),
        push: async (pipeObject) => {
          return this.backpressuredArray.push(pipeObject);
        },
      };

      try {
        const finalChunk = await this.options.finalFunction(tools);
        if (finalChunk) {
          this.backpressuredArray.push(finalChunk);
        }
      } catch (err) {
        this.backpressuredArray.push(null);
        callback(err);
        return;
      }
    }
    this.backpressuredArray.push(null);
    callback();
  }

  public async getWebStreams(): Promise<{ readable: ReadableStream; writable: WritableStream }> {
    const duplex = this;
    const readable = new ReadableStream({
      start(controller) {
        duplex.on('readable', () => {
          let chunk;
          while (null !== (chunk = duplex.read())) {
            controller.enqueue(chunk);
          }
        });

        duplex.on('end', () => {
          controller.close();
        });
      },
      cancel(reason) {
        duplex.destroy(new Error(reason));
      },
    });

    const writable = new WritableStream({
      write(chunk) {
        return new Promise<void>((resolve, reject) => {
          const isBackpressured = !duplex.write(chunk, (error) => {
            if (error) {
              reject(error);
            } else {
              resolve();
            }
          });

          if (isBackpressured) {
            duplex.once('drain', resolve);
          }
        });
      },
      close() {
        return new Promise<void>((resolve, reject) => {
          duplex.end(resolve);
        });
      },
      abort(reason) {
        duplex.destroy(new Error(reason));
      },
    });

    return { readable, writable };
  }
}