webcodecs-flv/src/flv-demuxer.js

webcodecs decode flv

import AMF from './amf-parser.js';
import * as EventModule from './event.js';
import { demuxerChange } from './event.js';
import SPSParser from './sps-parser.js';
function Swap16(src) {
  return ((src >>> 8) & 0xff) | ((src & 0xff) << 8);
}

function Swap32(src) {
  return (
    ((src & 0xff000000) >>> 24) |
    ((src & 0x00ff0000) >>> 8) |
    ((src & 0x0000ff00) << 8) |
    ((src & 0x000000ff) << 24)
  );
}

function ReadBig32(array, index) {
  return (
    (array[index] << 24) |
    (array[index + 1] << 16) |
    (array[index + 2] << 8) |
    array[index + 3]
  );
}
class FLVDemuxer {
  constructor() {
    Object.assign(this, new EventModule.EventDispatcher());
    this.FLVHeader = {
      //header：占9个字节
      streamInfo: {
        //1个字节
        hasVideo: null,
        hasAudio: null,
      },
      version: 1, //1个字节
      Type: 'flv', // 3个字节
      headerSize: null, //4个字节整个header的长度，一般为9； 大于9表示下面还有扩展信息 (9)
      offset: 0,
    };
    this.videoInfo = {
      codec: null, //编解码
      DataRate: null,
      width: null,
      height: null,
      fps: null,
      profile: null,
      level: null,
      refFrames: null,
      chromaFormat: null,
      sarNum: null,
      sarDen: null,
      data: null,
      ts: null,
    };
    this.audioInfo = {
      codec: null, //编解码
      DataRate: null,
      audioSampleRate: null,
      audioChannelCount: null,
      data: null,
      ts: null,
    };
    this._firstParse = false;
    this._littleEndian = (function () {
      //小端
      let buf = new ArrayBuffer(2);
      new DataView(buf).setInt16(0, 256, true); // little-endian write
      return new Int16Array(buf)[0] === 256; // platform-spec read, if equal then LE
    })();
    this._flvSoundRateTable = [5500, 11025, 22050, 44100, 48000];
    this._mpegSamplingRates = [
      96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000,
      11025, 8000, 7350,
    ];
    this.noParseData = null;
    this._naluLengthSize = 4;
    this.chunkList = [];
    this.isParse = false;
    this.firstData = [];
    this.isFirstData = true;
  }
  _parseHeader(buffer) {
    let data = new Uint8Array(buffer);
    if (
      data[0] !== 0x46 ||
      data[1] !== 0x4c ||
      data[2] !== 0x56 ||
      data[3] !== 0x01
    ) {
      return false;
    }
    this.FLVHeader.streamInfo.hasAudio = (data[4] & 4) >>> 2 !== 0;
    this.FLVHeader.streamInfo.hasVideo = (data[4] & 1) !== 0;
    this.FLVHeader.offset = ReadBig32(data, 5); //获取header长度，一般为9
    this.dispatchEvent(
      new demuxerChange('demuxer', { event: 'flvHeader', data: this.FLVHeader })
    );
  }
  parseChunks(chunk) {
    if (!this.isParse) {
      if (this.chunkList.length == 0) {
        this._parseChunks(chunk);
      } else {
        let parseData = this.chunkList.splice(0, 1);
        this._parseChunks(parseData);
      }
    } else {
      this.chunkList.push(chunk);
    }
  }
  _parseChunks(chunk, byteStart) {
    let offset = 0;
    let originChunk = chunk;
    if (this.noParseData) {
      chunk = this._concatUint8(this.noParseData, chunk);
    } else {
      chunk = originChunk;
    }
    if (!this._firstParse) {
      this._firstParse = true;
      this._parseHeader(chunk); //解析header
      // 处理PreviousTagSize0    占4个字节，一般为0
      if (
        chunk[9] !== 0 ||
        chunk[10] !== 0 ||
        chunk[11] !== 0 ||
        chunk[12] !== 0
      ) {
        return 'First time parsing but chunk byteStart invalid!';
      }
      offset = this.FLVHeader.offset + 4; //第一个PreviousTag
    }
    this.isParse = true;
    while (offset < chunk.byteLength) {
      let v = new DataView(chunk.buffer, offset);
      if (offset + 11 + 4 <= chunk.byteLength) {
        this.noParseData = null;
      }
      if (offset + 11 + 4 > chunk.byteLength) {
        // data not enough for parsing an flv tag
        this.noParseData = new Uint8Array(chunk.buffer, offset);
        break;
      }
      let tagType = v.getUint8(0);
      let dataSize = v.getUint32(0, !this._littleEndian) & 0x00ffffff;
      if (offset + 11 + dataSize + 4 <= chunk.byteLength) {
        this.noParseData = null;
      }
      if (offset + 11 + dataSize + 4 > chunk.byteLength) {
        this.noParseData = new Uint8Array(chunk.buffer, offset);
        break;
      }
      if (tagType !== 8 && tagType !== 9 && tagType !== 18) {
        offset += 11 + dataSize + 4;
        continue;
      }
      let ts2 = v.getUint8(4);
      let ts1 = v.getUint8(5);
      let ts0 = v.getUint8(6);
      let ts3 = v.getUint8(7); //高位8位

      let timestamp = ts0 | (ts1 << 8) | (ts2 << 16) | (ts3 << 24);

      let streamId = v.getUint32(7, !this._littleEndian) & 0x00ffffff;
      //   console.log(streamId, 'streamId');
      let dataOffset = offset + 11; //body 中的tag header占11个字节
      switch (tagType) {
        case 8: // Audio
          this._parseAudioData(chunk.buffer, dataOffset, dataSize, timestamp);
          break;
        case 9: // Video
          this._parseVideoData(
            chunk.buffer,
            dataOffset,
            dataSize,
            timestamp,
            byteStart + offset
          );

          break;
        case 18: // ScriptDataObject
          this._parseScriptData(chunk.buffer, dataOffset, dataSize);
          break;
      }
      offset += 11 + dataSize + 4;
    }
    this.isParse = false;
  }
  _parseAudioData(arrayBuffer, dataOffset, dataSize, tagTimestamp) {
    if (dataSize <= 1) {
      return 'Flv: Invalid audio packet, missing SoundData payload!';
    }
    let le = this._littleEndian;
    let v = new DataView(arrayBuffer, dataOffset, dataSize);
    let soundSpec = v.getUint8(0);

    let soundFormat = soundSpec >>> 4; //音频格式占4个字节
    // console.log(soundFormat, 'soundFormat 10 is AAC');
    let soundRateIndex = (soundSpec & 12) >>> 2; //占2个字节
    // console.log(soundRateIndex, 'soundRateIndex 采样率'); // AAC 总是3
    let soundRate = 0;
    if (soundRateIndex >= 0 && soundRateIndex <= 4) {
      soundRate = this._flvSoundRateTable[soundRateIndex];
    } else {
      return;
    }
    // console.log(soundRate, 'soundRate');
    let soundSize = (soundSpec & 2) >>> 1; // 采样长度 0 = snd8Bit1 = snd16Bit 压缩过的音频都是16bit
    // console.log(soundSize, 'soundSize 采样长度');
    let soundType = soundSpec & 1; //音频类型   0 = sndMono 1 = sndStereo
    // console.log(soundType, 'soundType 音频类型');
    this.audioInfo.audioChannelCount = soundType === 0 ? 1 : 2; //声道数
    this.audioInfo.audioSampleRate = soundRate; //采样率
    if (soundFormat === 10) {
      let aacData = this._parseAACAudioData(
        arrayBuffer,
        dataOffset + 1,
        dataSize - 1,
        tagTimestamp
      );
      // console.log(aacData, 'aacData');
    }
  }
  _parseAACAudioData(arrayBuffer, dataOffset, dataSize, tagTimestamp) {
    if (dataSize <= 1) {
      return;
    }

    // let result = {};
    let array = new Uint8Array(arrayBuffer, dataOffset, dataSize);
    // console.log(array, 'audio array ');
    // result.packetType = array[0];

    if (array[0] === 0) {
      this._parseAACAudioSpecificConfig(
        arrayBuffer,
        dataOffset + 1,
        dataSize - 1
      );
    } else {
      this.dispatchEvent(
        new demuxerChange('demuxer', {
          event: 'audioData',
          data: { data: array.subarray(1), timestamp: tagTimestamp },
        })
      );
    }
  }
  _parseAACAudioSpecificConfig(arrayBuffer, dataOffset, dataSize) {
    let array = new Uint8Array(arrayBuffer, dataOffset, dataSize);
    let config = null;

    /* Audio Object Type:
       0: Null
       1: AAC Main
       2: AAC LC
       3: AAC SSR (Scalable Sample Rate)
       4: AAC LTP (Long Term Prediction)
       5: HE-AAC / SBR (Spectral Band Replication)
       6: AAC Scalable
    */

    let audioObjectType = 0;
    let originalAudioObjectType = 0;
    let audioExtensionObjectType = null;
    let samplingIndex = 0;
    let extensionSamplingIndex = null;

    // 5 bits
    audioObjectType = originalAudioObjectType = array[0] >>> 3;
    // 4 bits
    samplingIndex = ((array[0] & 0x07) << 1) | (array[1] >>> 7);
    if (samplingIndex < 0 || samplingIndex >= this._mpegSamplingRates.length) {
      this._onError(
        DemuxErrors.FORMAT_ERROR,
        'Flv: AAC invalid sampling frequency index!'
      );
      return;
    }

    let samplingFrequence = this._mpegSamplingRates[samplingIndex];

    // 4 bits
    let channelConfig = (array[1] & 0x78) >>> 3;
    if (channelConfig < 0 || channelConfig >= 8) {
      this._onError(
        DemuxErrors.FORMAT_ERROR,
        'Flv: AAC invalid channel configuration'
      );
      return;
    }

    if (audioObjectType === 5) {
      // HE-AAC?
      // 4 bits
      extensionSamplingIndex = ((array[1] & 0x07) << 1) | (array[2] >>> 7);
      // 5 bits
      audioExtensionObjectType = (array[2] & 0x7c) >>> 2;
    }

    // workarounds for various browsers
    let userAgent = self.navigator.userAgent.toLowerCase();

    if (userAgent.indexOf('firefox') !== -1) {
      // firefox: use SBR (HE-AAC) if freq less than 24kHz
      if (samplingIndex >= 6) {
        audioObjectType = 5;
        config = new Array(4);
        extensionSamplingIndex = samplingIndex - 3;
      } else {
        // use LC-AAC
        audioObjectType = 2;
        config = new Array(2);
        extensionSamplingIndex = samplingIndex;
      }
    } else if (userAgent.indexOf('android') !== -1) {
      // android: always use LC-AAC
      audioObjectType = 2;
      config = new Array(2);
      extensionSamplingIndex = samplingIndex;
    } else {
      // for other browsers, e.g. chrome...
      // Always use HE-AAC to make it easier to switch aac codec profile
      audioObjectType = 5;
      extensionSamplingIndex = samplingIndex;
      config = new Array(4);

      if (samplingIndex >= 6) {
        extensionSamplingIndex = samplingIndex - 3;
      } else if (channelConfig === 1) {
        // Mono channel
        audioObjectType = 2;
        config = new Array(2);
        extensionSamplingIndex = samplingIndex;
      }
    }

    config[0] = audioObjectType << 3;
    config[0] |= (samplingIndex & 0x0f) >>> 1;
    config[1] = (samplingIndex & 0x0f) << 7;
    config[1] |= (channelConfig & 0x0f) << 3;
    if (audioObjectType === 5) {
      config[1] |= (extensionSamplingIndex & 0x0f) >>> 1;
      config[2] = (extensionSamplingIndex & 0x01) << 7;
      // extended audio object type: force to 2 (LC-AAC)
      config[2] |= 2 << 2;
      config[3] = 0;
    }
    this.dispatchEvent(
      new demuxerChange('demuxer', {
        event: 'audioConfig',
        data: {
          config: config,
          samplingRate: samplingFrequence,
          channelCount: channelConfig,
          codec: 'mp4a.40.' + audioObjectType,
          originalCodec: 'mp4a.40.' + originalAudioObjectType,
        },
      })
    );
  }
  _parseVideoData(
    arrayBuffer,
    dataOffset,
    dataSize,
    tagTimestamp,
    tagPosition
  ) {
    if (dataSize <= 1) {
      console.log('Flv: Invalid video packet, missing VideoData payload!');
      return;
    }
    let spec = new Uint8Array(arrayBuffer, dataOffset, dataSize)[0];
    // console.log(spec, 'video spec');
    // 1: keyframe (for AVC, a seekable frame)
    // 2: inter frame (for AVC, a non-seekable frame)
    // 3: disposable inter frame (H.263 only)
    // 4: generated keyframe (reserved for server use only)
    // 5: video info/command frame
    let frameType = (spec & 240) >>> 4;
    // 1: JPEG (currently unused)
    // 2: Sorenson H.263
    // 3: Screen video
    // 4: On2 VP6
    // 5: On2 VP6 with alpha channel
    // 6: Screen video version 2
    // 7: AVC
    let codecId = spec & 15;
    // console.log(frameType, 'video frameType 帧类型');

    // console.log(codecId, 'video codecId 编码ID');
    let array = new Uint8Array(arrayBuffer, dataOffset + 1, dataSize - 1);
    // console.log(array, 'video data');
    // this.dispatchEvent(
    //   new demuxerChange('demuxer', {
    //     event: 'videoData',
    //     data: { array, tagTimestamp, frameType, codecId },
    //   })
    // );
    this._parseAVCVideoPacket(
      arrayBuffer,
      dataOffset + 1,
      dataSize - 1,
      tagTimestamp,
      tagPosition,
      frameType
    );
  }
  _parseAVCVideoPacket(
    arrayBuffer,
    dataOffset,
    dataSize,
    tagTimestamp,
    tagPosition,
    frameType
  ) {
    let le = this._littleEndian;
    let v = new DataView(arrayBuffer, dataOffset, dataSize);
    let packetType = v.getUint8(0);
    let cts_unsigned = v.getUint32(0, !le) & 0x00ffffff;
    let cts = (cts_unsigned << 8) >> 8; // convert to 24-bit signed int
    // console.log(packetType, 'video packetType');
    // console.log(cts, 'video  tag body cts');
    if (packetType === 0) {
      this._parseAVCDecoderConfigurationRecord(
        arrayBuffer,
        dataOffset + 4,
        dataSize - 4
      );
    } else if (packetType === 1) {
      this._parseAVCVideoData(
        arrayBuffer,
        dataOffset + 4,
        dataSize - 4,
        tagTimestamp,
        tagPosition,
        frameType,
        cts
      );
    }
  }
  _parseAVCDecoderConfigurationRecord(arrayBuffer, dataOffset, dataSize) {
    if (dataSize < 7) {
      console.log('Flv: Invalid AVCDecoderConfigurationRecord, lack of data!');
      return;
    }
    let le = this._littleEndian;
    let v = new DataView(arrayBuffer, dataOffset, dataSize);
    let version = v.getUint8(0); // configurationVersion
    let avcProfile = v.getUint8(1); // avcProfileIndication
    let profileCompatibility = v.getUint8(2); // profile_compatibility
    let avcLevel = v.getUint8(3);
    this._naluLengthSize = (v.getUint8(4) & 3) + 1; // lengthSizeMinusOne
    if (this._naluLengthSize !== 3 && this._naluLengthSize !== 4) {
      // holy shit!!
      return;
    }
    let spsCount = v.getUint8(5) & 31; // numOfSequenceParameterSets
    let offset = 6;
    var config = {};
    let codecString = 'avc1.';
    for (let i = 0; i < spsCount; i++) {
      let len = v.getUint16(offset, !le); // sequenceParameterSetLength
      offset += 2;
      if (len === 0) {
        continue;
      }
      var sps = new Uint8Array(arrayBuffer, dataOffset + offset, len);
      offset += len;
      config = SPSParser.parseSPS(sps);
      if (i !== 0) {
        // ignore other sps's config
        continue;
      }
      // console.log(config, 'config');
      let codecArray = sps.subarray(1, 4);
      for (let j = 0; j < 3; j++) {
        let h = codecArray[j].toString(16);
        if (h.length < 2) {
          h = '0' + h;
        }
        codecString += h;
      }
      // console.log(codecString, 'codecString');
    }
    let ppsCount = v.getUint8(offset); // numOfPictureParameterSets
    offset++;

    for (let i = 0; i < ppsCount; i++) {
      let len = v.getUint16(offset, !le); // pictureParameterSetLength
      offset += 2;

      if (len === 0) {
        continue;
      }
      var pps = new Uint8Array(arrayBuffer, dataOffset + offset, len);
      // pps is useless for extracting video information
      offset += len;
    }
    this.firstData = this._concatUint8(
      new Uint8Array([0, 0, 0, 1]),
      sps,
      new Uint8Array([0, 0, 0, 1]),
      pps
    );
    this.dispatchEvent(
      new demuxerChange('demuxer', {
        event: 'videoConfig',
        data: Object.assign(config, { codecString }),
      })
    );
  }
  _parseAVCVideoData(
    arrayBuffer,
    dataOffset,
    dataSize,
    tagTimestamp,
    tagPosition,
    frameType,
    cts
  ) {
    let le = this._littleEndian;
    let v = new DataView(arrayBuffer, dataOffset, dataSize);
    let v2 = new DataView(arrayBuffer);
    let units = [],
      length = 0;
    let offset = 0;
    const lengthSize = this._naluLengthSize;
    let dts = 0 + tagTimestamp;
    let keyframe = frameType === 1; // from FLV Frame Type constants
    while (offset < dataSize) {
      let naluSize = v.getUint32(offset, !le); // Big-Endian read
      if (lengthSize === 3) {
        naluSize >>>= 8;
      }
      let unitType = v.getUint8(offset + lengthSize) & 0x1f;

      if (unitType === 5) {
        // IDR
        keyframe = true;
      }

      let data = new Uint8Array(
        arrayBuffer,
        dataOffset + offset,
        lengthSize + naluSize
      );
      data[0] = 0;
      data[1] = 0;
      data[2] = 0;
      data[3] = 1;
      //   console.log(data, 'data111111');
      //   console.log(dataOffset + offset);

      v2.setInt8(dataOffset + offset, 0);
      v2.setInt8(dataOffset + offset + 1, 0);
      v2.setInt8(dataOffset + offset + 2, 0);
      v2.setInt8(dataOffset + offset + 3, 1);
      //   console.log(v2);
      let unit = { type: unitType, data: data };
      units.push(unit);
      length += data.byteLength;
      offset += lengthSize + naluSize;
    }

    if (units.length) {
      let avcSample = {
        units: units,
        length: length,
        isKeyframe: keyframe,
        dts: dts,
        cts: cts,
        pts: dts + cts,
        data: this.isFirstData
          ? this._concatUint8(
              this.firstData,
              new Uint8Array(arrayBuffer, dataOffset, dataSize)
            )
          : new Uint8Array(arrayBuffer, dataOffset, dataSize),
      };
      //   console.log(avcSample, 'avcSample');
      this.isFirstData = false;
      this.dispatchEvent(
        new demuxerChange('demuxer', {
          event: 'videoData',
          data: avcSample,
        })
      );
    }
  }
  _parseScriptData(arrayBuffer, dataOffset, dataSize) {
    let scriptData = AMF.parseScriptData(arrayBuffer, dataOffset, dataSize);
    // console.log(scriptData, 'scriptData');
    this.dispatchEvent(
      new demuxerChange('demuxer', { event: 'scriptData', data: scriptData })
    );
  }
  _concatUint8(...args) {
    const length = args.reduce((len, cur) => (len += cur.byteLength), 0);
    const result = new Uint8Array(length);

    let offset = 0;
    args.forEach((uint8) => {
      result.set(uint8, offset);
      offset += uint8.byteLength;
    });

    return result;
  }
}
export default FLVDemuxer;