sflow/src/lines.ts

sflow is a powerful and highly-extensible library designed for processing and manipulating streams of data effortlessly. Inspired by the functional programming paradigm, it provides a rich set of utilities for transforming streams, including chunking, fil

import { chunkIfs } from "./chunkIfs";
import { flatMaps } from "./flatMaps";
import { maps } from "./maps";
import { throughs } from "./throughs";

type LinesOptions = { EOL?: "KEEP" | "LF" | "CRLF" | "NONE" };
/** split string stream into lines stream, handy to concat LLM's tokens stream into line by line stream or split a long string by lines */
export const lines: (opts?: LinesOptions) => TransformStream<string, string> =
  ({ EOL = "KEEP" }: LinesOptions = {}) => {
    const CRLFMap = {
      KEEP: "$1",
      LF: "\n",
      CRLF: "\r\n",
      NONE: "",
    };
    return throughs<string, string>((r) =>
      r
        .pipeThrough(flatMaps((s: string) => s.split(/(?<=\n)/g)))
        .pipeThrough(
          chunkIfs((ch: string) => ch.indexOf("\n") === -1, {
            inclusive: true,
          }),
        )
        .pipeThrough(
          maps((chunks) => chunks.join("").replace(/(\r?\n?)$/, CRLFMap[EOL])),
        ),
    );
  };