ts-ebml-esm/src/EBMLReader.ts

ebml decoder and encoder

import { EventEmitter } from "events";
import * as EBML from "./EBML.js";
import * as tools from "./tools.js";

/**
 * This is an informal code for reference.
 * EBMLReader is a class for getting information to enable seeking Webm recorded by MediaRecorder.
 * So please do not use for regular WebM files.
 */
export default class EBMLReader extends EventEmitter {
  private metadataloaded: boolean;

  private stack: (EBML.MasterElement & EBML.ElementDetail)[];
  private chunks: EBML.EBMLElementDetail[];

  private segmentOffset: number;
  private last2SimpleBlockVideoTrackTimestamp: [number, number];
  private last2SimpleBlockAudioTrackTimestamp: [number, number];
  private lastClusterTimestamp: number;
  private lastClusterPosition: number;
  private firstVideoBlockRead: boolean;
  private firstAudioBlockRead: boolean;
  timestampScale: number;
  metadataSize: number;
  metadatas: EBML.EBMLElementDetail[];

  private currentTrack: {
    TrackNumber: number;
    TrackType: number;
    DefaultDuration: number | null;
    CodecDelay: number | null;
  };
  // this type is equal to
  // trackTypes: { [trackID: number]: number };
  private trackTypes: number[];
  private trackDefaultDuration: (number | null)[];
  private trackCodecDelay: (number | null)[];

  private ended: boolean;
  trackInfo:
    | { type: "video" | "audio" | "both"; trackNumber: number }
    | { type: "nothing" };
  /**
   * usefull for thumbnail creation.
   */
  use_webp: boolean;
  /** Enabling this flag will slow down the operation */
  use_duration_every_simpleblock: boolean;
  logging: boolean;
  logGroup: string = "";
  private hasLoggingStarted: boolean = false;
  /**
   * usefull for recording chunks.
   */
  use_segment_info: boolean;
  /** see: https://bugs.chromium.org/p/chromium/issues/detail?id=606000#c22 */
  drop_default_duration: boolean;

  cues: { CueTrack: number; CueClusterPosition: number; CueTime: number }[];

  constructor() {
    super();
    this.metadataloaded = false;
    this.chunks = [];

    this.stack = [];
    this.segmentOffset = 0;
    this.last2SimpleBlockVideoTrackTimestamp = [0, 0];
    this.last2SimpleBlockAudioTrackTimestamp = [0, 0];
    this.lastClusterTimestamp = 0;
    this.lastClusterPosition = 0;
    // webm default TimestampScale is 1ms
    this.timestampScale = 1000000;
    this.metadataSize = 0;
    this.metadatas = [];
    this.cues = [];
    this.firstVideoBlockRead = false;
    this.firstAudioBlockRead = false;

    this.currentTrack = {
      TrackNumber: -1,
      TrackType: -1,
      DefaultDuration: null,
      CodecDelay: null
    };
    this.trackTypes = [];
    this.trackDefaultDuration = [];
    this.trackCodecDelay = [];
    this.trackInfo = { type: "nothing" };
    this.ended = false;

    this.logging = false;

    this.use_duration_every_simpleblock = false;
    this.use_webp = false;
    this.use_segment_info = true;
    this.drop_default_duration = true;
  }

  /**
   * emit final state.
   */
  stop() {
    this.ended = true;
    this.emit_segment_info();

    // clean up any unclosed Master Elements at the end of the stream.
    while (this.stack.length) {
      this.stack.pop();
      if (this.logging) {
        console.groupEnd();
      }
    }

    // close main group if set, logging is enabled, and has actually logged anything.
    if (this.logging && this.hasLoggingStarted && this.logGroup) {
      console.groupEnd();
    }
  }

  /**
   * emit chunk info
   */
  private emit_segment_info() {
    const data = this.chunks;
    this.chunks = [];
    if (!this.metadataloaded) {
      this.metadataloaded = true;
      this.metadatas = data;
      // find first video track
      const videoTrackNum = this.trackTypes.indexOf(1);
      // find first audio track
      const audioTrackNum = this.trackTypes.indexOf(2);
      this.trackInfo =
        videoTrackNum >= 0 && audioTrackNum >= 0
          ? { type: "both", trackNumber: videoTrackNum }
          : videoTrackNum >= 0
            ? { type: "video", trackNumber: videoTrackNum }
            : audioTrackNum >= 0
              ? { type: "audio", trackNumber: audioTrackNum }
              : { type: "nothing" };
      if (!this.use_segment_info) {
        return;
      }
      this.emit("metadata", { data, metadataSize: this.metadataSize });
    } else {
      if (!this.use_segment_info) {
        return;
      }
      const timestamp = this.lastClusterTimestamp;
      const duration = this.duration;
      const timestampScale = this.timestampScale;
      this.emit("cluster", { timestamp, data });
      this.emit("duration", { timestampScale, duration });
    }
  }
  read(elm: EBML.EBMLElementDetail) {
    let drop = false;
    if (this.ended) {
      // reader is finished
      return;
    }
    if (elm.type === "m") {
      // 閉じタグの自動挿入
      if (elm.isEnd) {
        this.stack.pop();
      } else {
        const parent = this.stack[this.stack.length - 1];
        if (parent != null && parent.level >= elm.level) {
          // 閉じタグなしでレベルが下がったら閉じタグを挿入
          this.stack.pop();

          // From http://w3c.github.io/media-source/webm-byte-stream-format.html#webm-media-segments
          // This fixes logging for webm streams with Cluster of unknown length and no Cluster closing elements.
          if (this.logging) {
            console.groupEnd();
          }

          parent.dataEnd = elm.dataEnd;
          parent.dataSize = elm.dataEnd - parent.dataStart;
          parent.unknownSize = false;
          const o = Object.assign({}, parent, {
            name: parent.name,
            type: parent.type,
            isEnd: true
          });
          this.chunks.push(o);
        }
        this.stack.push(elm);
      }
    }
    if (elm.type === "m" && elm.name === "Segment") {
      if (this.segmentOffset !== 0) {
        console.warn("Multiple segments detected!");
      }
      this.segmentOffset = elm.dataStart;
      this.emit("segment_offset", this.segmentOffset);
    } else if (elm.type === "b" && elm.name === "SimpleBlock") {
      const {
        timecode: timestamp,
        trackNumber,
        frames
      } = tools.ebmlBlock(elm.data);
      if (this.trackTypes[trackNumber] === 1) {
        // trackType === 1 => video track
        if (!this.firstVideoBlockRead) {
          this.firstVideoBlockRead = true;
          if (
            this.trackInfo.type === "both" ||
            this.trackInfo.type === "video"
          ) {
            const CueTime = this.lastClusterTimestamp + timestamp;
            this.cues.push({
              CueTrack: trackNumber,
              CueClusterPosition: this.lastClusterPosition,
              CueTime
            });
            this.emit("cue_info", {
              CueTrack: trackNumber,
              CueClusterPosition: this.lastClusterPosition,
              CueTime: this.lastClusterTimestamp
            });
            this.emit("cue", {
              CueTrack: trackNumber,
              CueClusterPosition: this.lastClusterPosition,
              CueTime
            });
          }
        }
        this.last2SimpleBlockVideoTrackTimestamp = [
          this.last2SimpleBlockVideoTrackTimestamp[1],
          timestamp
        ];
      } else if (this.trackTypes[trackNumber] === 2) {
        // trackType === 2 => audio track
        if (!this.firstAudioBlockRead) {
          this.firstAudioBlockRead = true;
          if (this.trackInfo.type === "audio") {
            const CueTime = this.lastClusterTimestamp + timestamp;
            this.cues.push({
              CueTrack: trackNumber,
              CueClusterPosition: this.lastClusterPosition,
              CueTime
            });
            this.emit("cue_info", {
              CueTrack: trackNumber,
              CueClusterPosition: this.lastClusterPosition,
              CueTime: this.lastClusterTimestamp
            });
            this.emit("cue", {
              CueTrack: trackNumber,
              CueClusterPosition: this.lastClusterPosition,
              CueTime
            });
          }
        }
        this.last2SimpleBlockAudioTrackTimestamp = [
          this.last2SimpleBlockAudioTrackTimestamp[1],
          timestamp
        ];
      }
      if (this.use_duration_every_simpleblock) {
        this.emit("duration", {
          timestampScale: this.timestampScale,
          duration: this.duration
        });
      }
      if (this.use_webp) {
        for (const frame of frames) {
          const startcode = frame.subarray(3, 6).toString("hex");
          // this is not a good way to detect VP8
          // see rfc6386 -- VP8 Data Format and Decoding Guide
          if (startcode !== "9d012a") {
            break;
          }
          const webpBuf = tools.VP8BitStreamToRiffWebPBuffer(frame);
          const webp = new Blob([webpBuf], { type: "image/webp" });
          const currentTime = this.duration;
          this.emit("webp", { currentTime, webp });
        }
      }
    } else if (elm.type === "m" && elm.name === "Cluster" && !elm.isEnd) {
      this.firstVideoBlockRead = false;
      this.firstAudioBlockRead = false;
      this.emit_segment_info();
      this.emit("cluster_ptr", elm.tagStart);
      this.lastClusterPosition = elm.tagStart;
    } else if (elm.type === "u" && elm.name === "Timestamp") {
      this.lastClusterTimestamp = elm.value;
    } else if (elm.type === "u" && elm.name === "TimestampScale") {
      this.timestampScale = elm.value;
    } else if (elm.type === "m" && elm.name === "TrackEntry") {
      if (elm.isEnd) {
        this.trackTypes[this.currentTrack.TrackNumber] =
          this.currentTrack.TrackType;
        this.trackDefaultDuration[this.currentTrack.TrackNumber] =
          this.currentTrack.DefaultDuration;
        this.trackCodecDelay[this.currentTrack.TrackNumber] =
          this.currentTrack.CodecDelay;
      } else {
        this.currentTrack = {
          TrackNumber: -1,
          TrackType: -1,
          DefaultDuration: null,
          CodecDelay: null
        };
      }
    } else if (elm.type === "u" && elm.name === "TrackType") {
      this.currentTrack.TrackType = elm.value;
    } else if (elm.type === "u" && elm.name === "TrackNumber") {
      this.currentTrack.TrackNumber = elm.value;
    } else if (elm.type === "u" && elm.name === "CodecDelay") {
      this.currentTrack.CodecDelay = elm.value;
    } else if (elm.type === "u" && elm.name === "DefaultDuration") {
      // media source api は DefaultDuration を計算するとバグる。
      // https://bugs.chromium.org/p/chromium/issues/detail?id=606000#c22
      // chrome 58 ではこれを回避するために DefaultDuration 要素を抜き取った。
      // chrome 58 以前でもこのタグを抜き取ることで回避できる
      if (this.drop_default_duration) {
        console.warn("DefaultDuration detected!, remove it");
        drop = true;
      } else {
        this.currentTrack.DefaultDuration = elm.value;
      }
    } else if (elm.name === "unknown") {
      console.warn(elm);
    }
    if (!this.metadataloaded && elm.dataEnd > 0) {
      this.metadataSize = elm.dataEnd;
    }
    if (!drop) {
      this.chunks.push(elm);
    }
    if (this.logging) {
      this.put(elm);
    }
  }
  /**
   * DefaultDuration が定義されている場合は最後のフレームのdurationも考慮する
   * 単位 timestampScale
   *
   * !!! if you need duration with seconds !!!
   * ```js
   * const nanosec = reader.duration * reader.timestampScale;
   * const sec = nanosec / 1000 / 1000 / 1000;
   * ```
   */
  get duration() {
    if (this.trackInfo.type === "nothing") {
      console.warn("no video, no audio track");
      return 0;
    }
    // defaultDuration は 生の nano sec
    let defaultDuration = 0;
    // nanoseconds
    let codecDelay = 0;
    let lastTimestamp = 0;

    const _defaultDuration =
      this.trackDefaultDuration[this.trackInfo.trackNumber];
    if (typeof _defaultDuration === "number") {
      defaultDuration = _defaultDuration;
    } else {
      // https://bugs.chromium.org/p/chromium/issues/detail?id=606000#c22
      // default duration がないときに使う delta
      if (this.trackInfo.type === "both") {
        if (
          this.last2SimpleBlockAudioTrackTimestamp[1] >
          this.last2SimpleBlockVideoTrackTimestamp[1]
        ) {
          // audio diff
          defaultDuration =
            (this.last2SimpleBlockAudioTrackTimestamp[1] -
              this.last2SimpleBlockAudioTrackTimestamp[0]) *
            this.timestampScale;
          // audio delay
          // 2 => audio
          const delay = this.trackCodecDelay[this.trackTypes.indexOf(2)];
          if (typeof delay === "number") {
            codecDelay = delay;
          }
          // audio timestamp
          lastTimestamp = this.last2SimpleBlockAudioTrackTimestamp[1];
        } else {
          // video diff
          defaultDuration =
            (this.last2SimpleBlockVideoTrackTimestamp[1] -
              this.last2SimpleBlockVideoTrackTimestamp[0]) *
            this.timestampScale;
          // video delay
          // 1 => video
          const delay = this.trackCodecDelay[this.trackTypes.indexOf(1)];
          if (typeof delay === "number") {
            codecDelay = delay;
          }
          // video timestamp
          lastTimestamp = this.last2SimpleBlockVideoTrackTimestamp[1];
        }
      } else if (this.trackInfo.type === "video") {
        defaultDuration =
          (this.last2SimpleBlockVideoTrackTimestamp[1] -
            this.last2SimpleBlockVideoTrackTimestamp[0]) *
          this.timestampScale;
        const delay = this.trackCodecDelay[this.trackInfo.trackNumber];
        if (typeof delay === "number") {
          codecDelay = delay;
        }
        lastTimestamp = this.last2SimpleBlockVideoTrackTimestamp[1];
      } else if (this.trackInfo.type === "audio") {
        defaultDuration =
          (this.last2SimpleBlockAudioTrackTimestamp[1] -
            this.last2SimpleBlockAudioTrackTimestamp[0]) *
          this.timestampScale;
        const delay = this.trackCodecDelay[this.trackInfo.trackNumber];
        if (typeof delay === "number") {
          codecDelay = delay;
        }
        lastTimestamp = this.last2SimpleBlockAudioTrackTimestamp[1];
      }
      // else { never }
    }
    // convert to timestampscale
    const duration_nanosec =
      (this.lastClusterTimestamp + lastTimestamp) * this.timestampScale +
      defaultDuration -
      codecDelay;
    const duration = duration_nanosec / this.timestampScale;
    return Math.floor(duration);
  }
  /**
   * @deprecated
   * emit on every segment
   * https://www.matroska.org/technical/specs/notes.html#Position_References
   */
  override addListener(
    event: "segment_offset",
    listener: (ev: number) => void
  ): this;
  /**
   * @deprecated
   * emit on every cluster element start.
   * Offset byte from __file start__. It is not an offset from the Segment element.
   */
  override addListener(
    event: "cluster_ptr",
    listener: (ev: number) => void
  ): this;
  /** @deprecated
   * emit on every cue point for cluster to create seekable webm file from MediaRecorder
   * */
  override addListener(
    event: "cue_info",
    listener: (ev: CueInfo) => void
  ): this;
  /** emit on every cue point for cluster to create seekable webm file from MediaRecorder */
  override addListener(event: "cue", listener: (ev: CueInfo) => void): this;
  /** latest EBML > Info > TimestampScale and EBML > Info > Duration to create seekable webm file from MediaRecorder */
  override addListener(
    event: "duration",
    listener: (ev: DurationInfo) => void
  ): this;
  /** EBML header without Cluster Element for recording metadata chunk */
  override addListener(
    event: "metadata",
    listener: (ev: SegmentInfo & { metadataSize: number }) => void
  ): this;
  /** emit every Cluster Element and its children for recording chunk */
  override addListener(
    event: "cluster",
    listener: (ev: SegmentInfo & { timestamp: number }) => void
  ): this;
  /** for thumbnail */
  override addListener(
    event: "webp",
    listener: (ev: ThumbnailInfo) => void
  ): this;
  override addListener(event: string, listener: (ev: any) => void): this {
    return super.addListener(event, listener);
  }

  put(elm: EBML.EBMLElementDetail) {
    if (!this.hasLoggingStarted) {
      this.hasLoggingStarted = true;
      if (this.logging && this.logGroup) {
        console.groupCollapsed(this.logGroup);
      }
    }
    if (elm.type === "m") {
      if (elm.isEnd) {
        console.groupEnd();
      } else {
        console.group(elm.name + ":" + elm.tagStart);
      }
    } else if (elm.type === "b") {
      // for debug
      //if(elm.name === "SimpleBlock"){
      //const o = EBML.tools.ebmlBlock(elm.value);
      //console.log(elm.name, elm.type, o.trackNumber, o.timestamp);
      //}else{
      console.log(elm.name, elm.type);
      //}
    } else {
      console.log(elm.name, elm.tagStart, elm.type, elm.value);
    }
  }
}
/** CueClusterPosition: Offset byte from __file start__. It is not an offset from the Segment element. */
export interface CueInfo {
  CueTrack: number;
  CueClusterPosition: number;
  CueTime: number;
}
export interface SegmentInfo {
  data: EBML.EBMLElementDetail[];
}
export interface DurationInfo {
  duration: number;
  timestampScale: number;
}
export interface ThumbnailInfo {
  webp: Blob;
  currentTime: number;
}