UNPKG

mediabunny

Version:

Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.

1,635 lines (1,367 loc) 72.2 kB
/*! * Copyright (c) 2026-present, Vanilagy and contributors * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import { TrackType } from '../output'; import { SAMPLES_PER_AAC_FRAME } from '../adts/adts-demuxer'; import { MAX_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader'; import { aacChannelMap, aacFrequencyTable } from '../../shared/aac-misc'; import { AacCodecInfo, AudioCodec, extractAudioCodecString, extractVideoCodecString, MediaCodec, VideoCodec, } from '../codec'; import { AC3_ACMOD_CHANNEL_COUNTS, AC3_SAMPLES_PER_FRAME, AvcDecoderConfigurationRecord, AvcNalUnitType, determineVideoPacketType, extractAvcDecoderConfigurationRecord, extractHevcDecoderConfigurationRecord, extractNalUnitTypeForAvc, extractNalUnitTypeForHevc, EAC3_NUMBLKS_TABLE, getEac3ChannelCount, getEac3SampleRate, HevcDecoderConfigurationRecord, HevcNalUnitType, parseAc3SyncFrame, parseAvcSps, parseEac3SyncFrame, parseHevcSps, AC3_FRAME_SIZES, } from '../codec-data'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrackBacking, InputTrackBacking, InputVideoTrackBacking, } from '../input-track'; import { PacketRetrievalOptions } from '../media-sink'; import { DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata'; import { assert, binarySearchExact, binarySearchLessOrEqual, COLOR_PRIMARIES_MAP_INVERSE, findLastIndex, floorToMultiple, last, MATRIX_COEFFICIENTS_MAP_INVERSE, Rotation, roundIfAlmostInteger, toDataView, TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, } from '../misc'; import { MP3_FRAME_HEADER_SIZE, getMp3ChannelCount, readMp3FrameHeader, } from '../../shared/mp3-misc'; import { EncodedPacket, PacketType, PLACEHOLDER_DATA } from '../packet'; import { FileSlice, readBytes, Reader, readU16Be, readU32Be, readU8 } from '../reader'; import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc'; import { AC3_SAMPLE_RATES } from '../../shared/ac3-misc'; import { Bitstream } from '../../shared/bitstream'; // Resources: // ISO/IEC 13818-1 const MISSING_PTS_ERROR_MESSAGE = 'PES packet is missing PTS where it was expected. PES packets without PTS are not' + ' currently supported. If you think this file should be supported, please report it.'; type ElementaryStream = { demuxer: MpegTsDemuxer; pid: number; streamType: number; initialized: boolean; firstSection: Section | null; /** * Some muxers suck ass and don't correctly label key frames, meaning we'll need to use our skill to * compensate for another programmer's skill issue. */ canBeTrustedWithKeyPackets: boolean; info: { type: 'video'; codec: VideoCodec; decoderConfig: VideoDecoderConfig | null; avcCodecInfo: AvcDecoderConfigurationRecord | null; hevcCodecInfo: HevcDecoderConfigurationRecord | null; colorSpace: VideoColorSpaceInit; width: number; height: number; squarePixelWidth: number; squarePixelHeight: number; reorderSize: number; } | { type: 'audio'; codec: AudioCodec; decoderConfig: AudioDecoderConfig | null; aacCodecInfo: AacCodecInfo | null; numberOfChannels: number; sampleRate: number; }; /** * Reference PES packets, spread throughout the file, to be used to speed up repeated random access. Sorted by both * byte offset and PTS. */ referencePesPackets: TimestampedPesPacketHeader[]; }; type ElementaryVideoStream = ElementaryStream & { info: { type: 'video' } }; type ElementaryAudioStream = ElementaryStream & { info: { type: 'audio' } }; type TsPacketHeader = { payloadUnitStartIndicator: number; pid: number; adaptationFieldControl: number; }; type TsPacket = TsPacketHeader & { body: Uint8Array<ArrayBufferLike>; }; type Section = { startPos: number; endPos: number | null; // null if the section was not read fully pid: number; payload: Uint8Array<ArrayBufferLike>; randomAccessIndicator: number; }; // Remember them so the warning doesn't get spammed const ignoredStreamTypes = new Set<number>(); export class MpegTsDemuxer extends Demuxer { reader: Reader; metadataPromise: Promise<void> | null = null; elementaryStreams: ElementaryStream[] = []; trackBackingEntries: InputTrackBacking[] = []; packetOffset = 0; packetStride = -1; sectionEndPositions: number[] = []; seekChunkSize = 5 * 1024 * 1024; // 5 MiB, picked because most HLS segments are below this size minReferencePointByteDistance = -1; constructor(input: Input) { super(input); this.reader = input._reader; } async readMetadata() { return this.metadataPromise ??= (async () => { const lengthToCheck = TS_PACKET_SIZE + 16 + 1; let startingSlice = this.reader.requestSlice(0, lengthToCheck); if (startingSlice instanceof Promise) startingSlice = await startingSlice; assert(startingSlice); const startingBytes = readBytes(startingSlice, lengthToCheck); if (startingBytes[0] === 0x47 && startingBytes[TS_PACKET_SIZE] === 0x47) { // Regular MPEG-TS this.packetOffset = 0; this.packetStride = TS_PACKET_SIZE; } else if (startingBytes[0] === 0x47 && startingBytes[TS_PACKET_SIZE + 16] === 0x47) { // MPEG-TS with Forward Error Correction this.packetOffset = 0; this.packetStride = TS_PACKET_SIZE + 16; } else if (startingBytes[4] === 0x47 && startingBytes[4 + TS_PACKET_SIZE + 4] === 0x47) { // MPEG-2-TS (DVHS) this.packetOffset = 4; this.packetStride = TS_PACKET_SIZE + 4; } else { throw new Error('Unreachable.'); } const MIN_REFERENCE_POINT_PACKET_DISTANCE = 256; this.minReferencePointByteDistance = MIN_REFERENCE_POINT_PACKET_DISTANCE * this.packetStride; let currentPos = this.packetOffset; let programMapPid: number | null = null; // Some files contain these multiple times, but we only care about their first appearance let hasProgramAssociationTable = false; let hasProgramMap = false; while (true) { const packetHeader = await this.readPacketHeader(currentPos); if (!packetHeader) { break; } if (packetHeader.payloadUnitStartIndicator === 0) { // Not the start of a section currentPos += this.packetStride; continue; } const section = await this.readSection( currentPos, true, !hasProgramMap, // Expect contiguous sections as long as we don't have the PMT ); if (!section) { break; } const BYTES_BEFORE_SECTION_LENGTH = 3; const BITS_IN_CRC_32 = 32; // Duh // Some streams don't contain a PAT for some reason, so we must do some guesswork to figure out where // the PMT is. let isProbablyProgramMap = false; if (!hasProgramMap && section.pid !== 0) { const isPesPacket = section.payload[0] === 0x00 && section.payload[1] === 0x00 && section.payload[2] === 0x01; if (!isPesPacket) { // Assume it's a PSI const bitstream = new Bitstream(section.payload); const pointerField = bitstream.readAlignedByte(); bitstream.skipBits(8 * pointerField); const tableId = bitstream.readBits(8); isProbablyProgramMap = tableId === 0x02; // 0x02 == TS_program_map_section } } if (section.pid === 0 && !hasProgramAssociationTable) { const bitstream = new Bitstream(section.payload); const pointerField = bitstream.readAlignedByte(); bitstream.skipBits(8 * pointerField); bitstream.skipBits(14); const sectionLength = bitstream.readBits(10); bitstream.skipBits(40); while (8 * (sectionLength + BYTES_BEFORE_SECTION_LENGTH) - bitstream.pos > BITS_IN_CRC_32) { const programNumber = bitstream.readBits(16); bitstream.skipBits(3); // Reserved const id = bitstream.readBits(13); if (programNumber !== 0) { if (programMapPid !== null) { throw new Error('Only files with a single program are supported.'); } else { programMapPid = id; } } } if (programMapPid === null) { throw new Error('Program Association Table must link to a Program Map Table.'); } hasProgramAssociationTable = true; } else if ((section.pid === programMapPid || isProbablyProgramMap) && !hasProgramMap) { const bitstream = new Bitstream(section.payload); const pointerField = bitstream.readAlignedByte(); bitstream.skipBits(8 * pointerField); bitstream.skipBits(12); const sectionLength = bitstream.readBits(12); bitstream.skipBits(43); // eslint-disable-next-line @typescript-eslint/no-unused-vars const pcrPid = bitstream.readBits(13); bitstream.skipBits(6); // "The remaining 10 bits specify the number of bytes of the descriptors immediately following the // program_info_length field" const programInfoLength = bitstream.readBits(10); bitstream.skipBits(8 * programInfoLength); while (8 * (sectionLength + BYTES_BEFORE_SECTION_LENGTH) - bitstream.pos > BITS_IN_CRC_32) { const streamType = bitstream.readBits(8); bitstream.skipBits(3); const elementaryPid = bitstream.readBits(13); bitstream.skipBits(6); const esInfoLength = bitstream.readBits(10); // Check ES descriptors to detect AC-3/E-AC-3 in System B const esInfoEndPos = bitstream.pos + 8 * esInfoLength; let hasAc3Descriptor = false; let hasEac3Descriptor = false; while (bitstream.pos < esInfoEndPos) { const descriptorTag = bitstream.readBits(8); const descriptorLength = bitstream.readBits(8); if (descriptorTag === 0x6a) { hasAc3Descriptor = true; } else if (descriptorTag === 0x7a || descriptorTag === 0xcc) { hasEac3Descriptor = true; } bitstream.skipBits(8 * descriptorLength); } let info: ElementaryStream['info'] | null = null; switch (streamType) { case MpegTsStreamType.AVC: case MpegTsStreamType.HEVC: { const codec = streamType === MpegTsStreamType.AVC ? 'avc' : 'hevc'; info = { type: 'video', codec, decoderConfig: null, avcCodecInfo: null, hevcCodecInfo: null, colorSpace: { primaries: null, transfer: null, matrix: null, fullRange: null, }, width: -1, height: -1, squarePixelWidth: -1, squarePixelHeight: -1, reorderSize: -1, }; }; break; case MpegTsStreamType.MP3_MPEG1: case MpegTsStreamType.MP3_MPEG2: case MpegTsStreamType.AAC: case MpegTsStreamType.AC3_SYSTEM_A: case MpegTsStreamType.EAC3_SYSTEM_A: { let codec: AudioCodec; if ( streamType === MpegTsStreamType.MP3_MPEG1 || streamType === MpegTsStreamType.MP3_MPEG2 ) { codec = 'mp3'; } else if (streamType === MpegTsStreamType.AAC) { codec = 'aac'; } else if (streamType === MpegTsStreamType.AC3_SYSTEM_A) { codec = 'ac3'; } else if (streamType === MpegTsStreamType.EAC3_SYSTEM_A) { codec = 'eac3'; } else { throw new Error('Unreachable.'); } info = { type: 'audio', codec, decoderConfig: null, aacCodecInfo: null, numberOfChannels: -1, sampleRate: -1, }; }; break; case MpegTsStreamType.PRIVATE_DATA: { if (hasEac3Descriptor) { info = { type: 'audio', codec: 'eac3', decoderConfig: null, aacCodecInfo: null, numberOfChannels: -1, sampleRate: -1, }; } else if (hasAc3Descriptor) { info = { type: 'audio', codec: 'ac3', decoderConfig: null, aacCodecInfo: null, numberOfChannels: -1, sampleRate: -1, }; } }; break; default: { // If we don't recognize the codec, we don't surface the track at all. This is because // we can't determine its metadata and also have no idea how to packetize its data. if (!ignoredStreamTypes.has(streamType)) { console.warn( `Note: MPEG-TS streams with stream_type 0x${streamType.toString(16)} are not` + ` currently supported.`, ); ignoredStreamTypes.add(streamType); } } } if (info) { this.elementaryStreams.push({ demuxer: this, pid: elementaryPid, streamType, initialized: false, firstSection: null, canBeTrustedWithKeyPackets: false, info, referencePesPackets: [], }); } } hasProgramMap = true; } else { const elementaryStream = this.elementaryStreams.find(x => x.pid === section.pid); outer: if (elementaryStream && !elementaryStream.initialized) { const pesPacket = readPesPacket(section, true); if (!pesPacket) { throw new Error( `Couldn't read first PES packet for Elementary Stream with PID ${elementaryStream.pid}`, ); } elementaryStream.firstSection = section; elementaryStream.canBeTrustedWithKeyPackets = section.randomAccessIndicator === 1; if (this.input._initInput) { const initDemuxer = (await this.input._initInput._getDemuxer()) as MpegTsDemuxer; const matchingStream = initDemuxer.elementaryStreams.find(x => ( x.pid === section.pid && x.info.codec === elementaryStream.info.codec )); if (matchingStream) { elementaryStream.info = matchingStream.info; elementaryStream.initialized = true; break outer; // We have the stream info, we're done } } const context = new PacketReadingContext(elementaryStream, pesPacket); if (elementaryStream.info.type === 'video') { // We loop because in some files, the video parameters are not in the first packet while (true) { const contextAlias = context; // TyyyyypeScript 😩 contextAlias.suppliedPacket = null; await context.markNextPacket(); if (elementaryStream.info.codec === 'avc') { if (!context.suppliedPacket) { throw new Error( 'Invalid AVC video stream; could not extract AVCDecoderConfigurationRecord' + ' from any packet.', ); } elementaryStream.info.avcCodecInfo = extractAvcDecoderConfigurationRecord(context.suppliedPacket.data); if (!elementaryStream.info.avcCodecInfo) { continue; // Search the next packet for it } const spsUnit = elementaryStream.info.avcCodecInfo.sequenceParameterSets[0]; assert(spsUnit); const spsInfo = parseAvcSps(spsUnit)!; elementaryStream.info.width = spsInfo.displayWidth; elementaryStream.info.height = spsInfo.displayHeight; const num = spsInfo.pixelAspectRatio.num; const den = spsInfo.pixelAspectRatio.den; if (num > 0 && den > 0) { if (num > den) { elementaryStream.info.squarePixelWidth = Math.round( elementaryStream.info.width * num / den, ); elementaryStream.info.squarePixelHeight = elementaryStream.info.height; } else { elementaryStream.info.squarePixelWidth = elementaryStream.info.width; elementaryStream.info.squarePixelHeight = Math.round( elementaryStream.info.height * den / num, ); } } elementaryStream.info.colorSpace = { primaries: COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as VideoColorPrimaries | undefined, transfer: TRANSFER_CHARACTERISTICS_MAP_INVERSE[spsInfo.transferCharacteristics] as VideoTransferCharacteristics | undefined, matrix: MATRIX_COEFFICIENTS_MAP_INVERSE[spsInfo.matrixCoefficients] as VideoMatrixCoefficients | undefined, fullRange: !!spsInfo.fullRangeFlag, }; elementaryStream.info.reorderSize = spsInfo.maxDecFrameBuffering; break; } else if (elementaryStream.info.codec === 'hevc') { if (!context.suppliedPacket) { throw new Error( 'Invalid HEVC video stream; could not extract HVCDecoderConfigurationRecord' + ' from first packet.', ); } elementaryStream.info.hevcCodecInfo = extractHevcDecoderConfigurationRecord(context.suppliedPacket.data); if (!elementaryStream.info.hevcCodecInfo) { continue; // Search the next packet for it } const spsArray = elementaryStream.info.hevcCodecInfo.arrays.find( a => a.nalUnitType === HevcNalUnitType.SPS_NUT, )!; const spsUnit = spsArray.nalUnits[0]; assert(spsUnit); const spsInfo = parseHevcSps(spsUnit)!; elementaryStream.info.width = spsInfo.displayWidth; elementaryStream.info.height = spsInfo.displayHeight; if (spsInfo.pixelAspectRatio.num > spsInfo.pixelAspectRatio.den) { elementaryStream.info.squarePixelWidth = Math.round( elementaryStream.info.width * spsInfo.pixelAspectRatio.num / spsInfo.pixelAspectRatio.den, ); elementaryStream.info.squarePixelHeight = elementaryStream.info.height; } else { elementaryStream.info.squarePixelWidth = elementaryStream.info.width; elementaryStream.info.squarePixelHeight = Math.round( elementaryStream.info.height * spsInfo.pixelAspectRatio.den / spsInfo.pixelAspectRatio.num, ); } elementaryStream.info.colorSpace = { primaries: COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as VideoColorPrimaries | undefined, transfer: TRANSFER_CHARACTERISTICS_MAP_INVERSE[spsInfo.transferCharacteristics] as VideoTransferCharacteristics | undefined, matrix: MATRIX_COEFFICIENTS_MAP_INVERSE[spsInfo.matrixCoefficients] as VideoMatrixCoefficients | undefined, fullRange: !!spsInfo.fullRangeFlag, }; elementaryStream.info.reorderSize = spsInfo.maxDecFrameBuffering; break; } else { throw new Error('Unhandled.'); } } elementaryStream.info.decoderConfig = { codec: extractVideoCodecString({ width: elementaryStream.info.width, height: elementaryStream.info.height, codec: elementaryStream.info.codec, codecDescription: null, colorSpace: elementaryStream.info.colorSpace, avcType: 1, avcCodecInfo: elementaryStream.info.avcCodecInfo, hevcCodecInfo: elementaryStream.info.hevcCodecInfo, vp9CodecInfo: null, av1CodecInfo: null, }), codedWidth: elementaryStream.info.width, codedHeight: elementaryStream.info.height, colorSpace: elementaryStream.info.colorSpace, }; if ( elementaryStream.info.width !== elementaryStream.info.squarePixelWidth || elementaryStream.info.height !== elementaryStream.info.squarePixelHeight ) { elementaryStream.info.decoderConfig.displayAspectWidth = elementaryStream.info.squarePixelWidth; elementaryStream.info.decoderConfig.displayAspectHeight = elementaryStream.info.squarePixelHeight; } elementaryStream.initialized = true; } else { await context.markNextPacket(); if (!context.suppliedPacket) { throw new Error( `Couldn't parse first media packet for Elementary Stream with` + ` PID ${elementaryStream.pid}`, ); } if (elementaryStream.info.codec === 'aac') { const slice = FileSlice.tempFromBytes(context.suppliedPacket.data); const header = readAdtsFrameHeader(slice); if (!header) { throw new Error( 'Invalid AAC audio stream; could not read ADTS frame header from first packet.', ); } elementaryStream.info.aacCodecInfo = { isMpeg2: false, objectType: header.objectType, }; elementaryStream.info.numberOfChannels = aacChannelMap[header.channelConfiguration]!; elementaryStream.info.sampleRate = aacFrequencyTable[header.samplingFrequencyIndex]!; } else if (elementaryStream.info.codec === 'mp3') { const word = readU32Be(FileSlice.tempFromBytes(context.suppliedPacket.data)); const result = readMp3FrameHeader(word, context.suppliedPacket.data.byteLength); if (!result.header) { throw new Error( 'Invalid MP3 audio stream; could not read frame header from first packet.', ); } elementaryStream.info.numberOfChannels = getMp3ChannelCount(result.header.channel); elementaryStream.info.sampleRate = result.header.sampleRate; } else if (elementaryStream.info.codec === 'ac3') { const frameInfo = parseAc3SyncFrame(context.suppliedPacket.data); if (!frameInfo) { throw new Error( 'Invalid AC-3 audio stream; could not read sync frame from first packet.', ); } if (frameInfo.fscod === 3) { throw new Error( 'Invalid AC-3 audio stream; reserved sample rate code found in first packet.', ); } elementaryStream.info.numberOfChannels = AC3_ACMOD_CHANNEL_COUNTS[frameInfo.acmod]! + frameInfo.lfeon; elementaryStream.info.sampleRate = AC3_SAMPLE_RATES[frameInfo.fscod]!; } else if (elementaryStream.info.codec === 'eac3') { const frameInfo = parseEac3SyncFrame(context.suppliedPacket.data); if (!frameInfo) { throw new Error( 'Invalid E-AC-3 audio stream; could not read sync frame from first packet.', ); } const sampleRate = getEac3SampleRate(frameInfo); if (sampleRate === null) { throw new Error( 'Invalid E-AC-3 audio stream; reserved sample rate code found in first packet.', ); } elementaryStream.info.numberOfChannels = getEac3ChannelCount(frameInfo); elementaryStream.info.sampleRate = sampleRate; } else { throw new Error('Unhandled.'); } elementaryStream.info.decoderConfig = { codec: extractAudioCodecString({ codec: elementaryStream.info.codec, codecDescription: null, aacCodecInfo: elementaryStream.info.aacCodecInfo, }), numberOfChannels: elementaryStream.info.numberOfChannels, sampleRate: elementaryStream.info.sampleRate, }; elementaryStream.initialized = true; } } } const isDone = hasProgramMap && this.elementaryStreams.every(x => x.initialized); if (isDone) { break; } currentPos += this.packetStride; } if (!hasProgramMap) { if (!hasProgramAssociationTable) { throw new Error('No Program Association Table found in the file.'); } throw new Error('No Program Map Table found in the file.'); } for (const stream of this.elementaryStreams) { if (stream.info.type === 'video') { this.trackBackingEntries.push( new MpegTsVideoTrackBacking(stream as ElementaryVideoStream), ); } else { this.trackBackingEntries.push( new MpegTsAudioTrackBacking(stream as ElementaryAudioStream), ); } } })(); } async getTrackBackings() { await this.readMetadata(); return this.trackBackingEntries; } async getMetadataTags(): Promise<MetadataTags> { return {}; // Nothing for now } async getMimeType(): Promise<string> { await this.readMetadata(); const codecStrings = await Promise.all(this.trackBackingEntries.map( x => x.getDecoderConfig().then(c => c?.codec ?? null), )); return buildMpegTsMimeType(codecStrings); } async readSection(startPos: number, full: boolean, contiguous = false): Promise<Section | null> { let endPos = startPos; let currentPos = startPos; const chunks: Uint8Array[] = []; let chunksByteLength = 0; let firstPacket: TsPacket | null = null; let mustAddSectionEnd = true; let randomAccessIndicator = 0; while (true) { const packet = await this.readPacket(currentPos); currentPos += this.packetStride; if (!packet) { break; } if (!firstPacket) { if (packet.payloadUnitStartIndicator === 0) { break; } firstPacket = packet; } else { if (packet.pid !== firstPacket.pid) { if (contiguous) { break; // End of section } else { continue; // Ignore this packet } } if (packet.payloadUnitStartIndicator === 1) { break; } } const hasAdaptationField = !!(packet.adaptationFieldControl & 0b10); const hasPayload = !!(packet.adaptationFieldControl & 0b01); let adaptationFieldLength = 0; if (hasAdaptationField) { adaptationFieldLength = 1 + packet.body[0]!; // Extract random_access_indicator from first packet's adaptation field if (packet === firstPacket && adaptationFieldLength > 1) { randomAccessIndicator = (packet.body[1]! >> 6) & 1; } } if (hasPayload) { if (adaptationFieldLength === 0) { chunks.push(packet.body); chunksByteLength += packet.body.byteLength; } else { chunks.push(packet.body.subarray(adaptationFieldLength)); chunksByteLength += packet.body.byteLength - adaptationFieldLength; } } endPos = currentPos; // 64 is just "a bit of data", enough for the PES packet header if (!full && chunksByteLength >= 64) { mustAddSectionEnd = false; // Not the actual section end break; } // Check if we already know this is a section end const isKnownSectionEnd = binarySearchExact(this.sectionEndPositions, endPos, x => x) !== -1; if (isKnownSectionEnd) { mustAddSectionEnd = false; break; } } if (mustAddSectionEnd) { const index = binarySearchLessOrEqual(this.sectionEndPositions, endPos, x => x); this.sectionEndPositions.splice(index + 1, 0, endPos); } if (!firstPacket) { return null; } let merged: Uint8Array; if (chunks.length === 1) { merged = chunks[0]!; } else { const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); merged = new Uint8Array(totalLength); let offset = 0; for (const chunk of chunks) { merged.set(chunk, offset); offset += chunk.length; } } return { startPos, endPos: full ? endPos : null, pid: firstPacket.pid, payload: merged, randomAccessIndicator, }; } async readPacketHeader(pos: number): Promise<TsPacketHeader | null> { let slice = this.reader.requestSlice(pos, 4); if (slice instanceof Promise) slice = await slice; if (!slice) { return null; } const syncByte = readU8(slice); if (syncByte !== 0x47) { throw new Error('Invalid TS packet sync byte. Likely an internal bug, please report this file.'); } const nextTwoBytes = readU16Be(slice); // eslint-disable-next-line @typescript-eslint/no-unused-vars const transportErrorIndicator = nextTwoBytes >> 15; const payloadUnitStartIndicator = (nextTwoBytes >> 14) & 0x1; // eslint-disable-next-line @typescript-eslint/no-unused-vars const transportPriority = (nextTwoBytes >> 13) & 0x1; const pid = nextTwoBytes & 0x1FFF; const nextByte = readU8(slice); // eslint-disable-next-line @typescript-eslint/no-unused-vars const transportScramblingControl = nextByte >> 6; const adaptationFieldControl = (nextByte >> 4) & 0x3; // eslint-disable-next-line @typescript-eslint/no-unused-vars const continuityCounter = nextByte & 0xF; return { payloadUnitStartIndicator, pid, adaptationFieldControl, }; } async readPacket(pos: number): Promise<TsPacket | null> { // Code in here is duplicated from readPacketHeader for performance reasons let slice = this.reader.requestSlice(pos, TS_PACKET_SIZE); if (slice instanceof Promise) slice = await slice; if (!slice) { return null; } const bytes = readBytes(slice, TS_PACKET_SIZE); const syncByte = bytes[0]!; if (syncByte !== 0x47) { throw new Error('Invalid TS packet sync byte. Likely an internal bug, please report this file.'); } const nextTwoBytes = (bytes[1]! << 8) + bytes[2]!; // eslint-disable-next-line @typescript-eslint/no-unused-vars const transportErrorIndicator = nextTwoBytes >> 15; const payloadUnitStartIndicator = (nextTwoBytes >> 14) & 0x1; // eslint-disable-next-line @typescript-eslint/no-unused-vars const transportPriority = (nextTwoBytes >> 13) & 0x1; const pid = nextTwoBytes & 0x1FFF; const nextByte = bytes[3]!; // eslint-disable-next-line @typescript-eslint/no-unused-vars const transportScramblingControl = nextByte >> 6; const adaptationFieldControl = (nextByte >> 4) & 0x3; // eslint-disable-next-line @typescript-eslint/no-unused-vars const continuityCounter = nextByte & 0xF; return { payloadUnitStartIndicator, pid, adaptationFieldControl, body: bytes.subarray(4), }; } } type PesPacketHeader = { sectionStartPos: number; sectionEndPos: number | null; // null if the section wasn't read fully pts: number | null; randomAccessIndicator: number; }; type TimestampedPesPacketHeader = PesPacketHeader & { pts: number; }; type PesPacket = PesPacketHeader & { data: Uint8Array<ArrayBufferLike>; }; type TimestampedPesPacket = PesPacket & { pts: number; }; const readPesPacketHeader = <T extends boolean>( section: Section, expectPts: T, ): (T extends true ? TimestampedPesPacketHeader : PesPacketHeader) | null => { if (section.payload.byteLength < 3) { return null; } const bitstream = new Bitstream(section.payload); const startCodePrefix = bitstream.readBits(24); if (startCodePrefix !== 0x000001) { return null; } const streamId = bitstream.readBits(8); bitstream.skipBits(16); if ( streamId === 0b10111100 // program_stream_map || streamId === 0b10111110 // padding_stream || streamId === 0b10111111 // private_stream_2 || streamId === 0b11110000 // ECM || streamId === 0b11110001 // EMM || streamId === 0b11111111 // program_stream_directory || streamId === 0b11110010 // DSMCC_stream || streamId === 0b11111000 // ITU-T Rec. H.222.1 type E stream ) { return null; } bitstream.skipBits(8); const ptsDtsFlags = bitstream.readBits(2); bitstream.skipBits(14); let pts: number | null = null; if (ptsDtsFlags === 0b10 || ptsDtsFlags === 0b11) { pts = 0; bitstream.skipBits(4); pts += bitstream.readBits(3) * (1 << 30); bitstream.skipBits(1); pts += bitstream.readBits(15) * (1 << 15); bitstream.skipBits(1); pts += bitstream.readBits(15); } else { if (expectPts) { throw new Error(MISSING_PTS_ERROR_MESSAGE); } } return { sectionStartPos: section.startPos, sectionEndPos: section.endPos, pts, randomAccessIndicator: section.randomAccessIndicator, } as T extends true ? TimestampedPesPacketHeader : PesPacketHeader; }; const readPesPacket = <T extends boolean>( section: Section, expectPts: T, ): (T extends true ? TimestampedPesPacket : PesPacket) | null => { assert(section.endPos !== null); // Can only read full PES packets from fully read sections const header = readPesPacketHeader(section, expectPts); if (!header) { return null; } const bitstream = new Bitstream(section.payload); bitstream.skipBits(32); const pesPacketLength = bitstream.readBits(16); const BYTES_UNTIL_END_OF_PES_PACKET_LENGTH = 6; bitstream.skipBits(16); const pesHeaderDataLength = bitstream.readBits(8); const pesHeaderEndPos = bitstream.pos + 8 * pesHeaderDataLength; bitstream.pos = pesHeaderEndPos; const bytePos = pesHeaderEndPos / 8; assert(Number.isInteger(bytePos)); const data = section.payload.subarray( bytePos, // "A value of 0 indicates that the PES packet length is neither specified nor bounded and is allowed only in // PES packets whose payload consists of bytes from a video elementary stream contained in // transport stream packets." pesPacketLength > 0 ? BYTES_UNTIL_END_OF_PES_PACKET_LENGTH + pesPacketLength : section.payload.byteLength, ); return { ...header, data, } as T extends true ? TimestampedPesPacket : PesPacket; }; abstract class MpegTsTrackBacking implements InputTrackBacking { packetBuffers = new WeakMap<EncodedPacket, PacketBuffer>(); /** Used for recreating PacketBuffers if necessary. */ packetSectionStarts = new WeakMap<EncodedPacket, number>(); constructor(public elementaryStream: ElementaryStream) {} abstract getType(): TrackType; abstract getDecoderConfig(): Promise<VideoDecoderConfig | AudioDecoderConfig | null>; getId() { return this.elementaryStream.pid; } getNumber() { const demuxer = this.elementaryStream.demuxer; const trackType = this.elementaryStream.info.type; let number = 0; for (const backing of demuxer.trackBackingEntries) { if (backing.getType() === trackType) { number++; } assert(backing instanceof MpegTsTrackBacking); if (backing.elementaryStream === this.elementaryStream) { break; } } return number; } getCodec(): MediaCodec | null { throw new Error('Not implemented on base class.'); } getInternalCodecId() { return this.elementaryStream.streamType; } getName() { return null; } getLanguageCode() { return UNDETERMINED_LANGUAGE; } getDisposition(): TrackDisposition { return { ...DEFAULT_TRACK_DISPOSITION, primary: false, }; } getTimeResolution() { return TIMESCALE; } isRelativeToUnixEpoch() { return false; } getPairingMask() { return 1n; } getBitrate() { return null; } getAverageBitrate() { return null; } async getDurationFromMetadata() { return null; } async getLiveRefreshInterval() { return null; } abstract allPacketsAreKeyPackets(): boolean; abstract getReorderSize(): number; createEncodedPacket( suppliedPacket: SuppliedPacket, duration: number, options: PacketRetrievalOptions, ) { let packetType: PacketType; if (this.allPacketsAreKeyPackets()) { packetType = 'key'; } else { packetType = suppliedPacket.randomAccessIndicator === 1 ? 'key' : 'delta'; } return new EncodedPacket( options.metadataOnly ? PLACEHOLDER_DATA : suppliedPacket.data, packetType, suppliedPacket.pts / TIMESCALE, Math.max(duration / TIMESCALE, 0), suppliedPacket.sequenceNumber, suppliedPacket.data.byteLength, ); } async getFirstPacket(options: PacketRetrievalOptions): Promise<EncodedPacket | null> { const section = this.elementaryStream.firstSection; assert(section); const pesPacket = readPesPacket(section, true); assert(pesPacket); const context = new PacketReadingContext(this.elementaryStream, pesPacket); const buffer = new PacketBuffer(this, context); const result = await buffer.readNext(); if (!result) { return null; } const packet = this.createEncodedPacket(result.packet, result.duration, options); this.packetBuffers.set(packet, buffer); this.packetSectionStarts.set(packet, result.packet.sectionStartPos); return packet; } async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise<EncodedPacket | null> { let buffer = this.packetBuffers.get(packet); if (buffer) { // Fast path const result = await buffer.readNext(); if (!result) { return null; } // Remove PacketBuffer access from the old packet, it belongs to the next packet now this.packetBuffers.delete(packet); const newPacket = this.createEncodedPacket(result.packet, result.duration, options); this.packetBuffers.set(newPacket, buffer); this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos); return newPacket; } // No buffer, we gotta do some rereading const sectionStartPos = this.packetSectionStarts.get(packet); if (sectionStartPos === undefined) { throw new Error('Packet was not created from this track.'); } const demuxer = this.elementaryStream.demuxer; const section = await demuxer.readSection(sectionStartPos, true); assert(section); const pesPacket = readPesPacket(section, true); assert(pesPacket); const context = new PacketReadingContext(this.elementaryStream, pesPacket); buffer = new PacketBuffer(this, context); // Advance until we pass the current packet's sequence number const targetSequenceNumber = packet.sequenceNumber; while (true) { const result = await buffer.readNext(); if (!result) { return null; } if (result.packet.sequenceNumber > targetSequenceNumber) { // We found the next packet! const newPacket = this.createEncodedPacket(result.packet, result.duration, options); this.packetBuffers.set(newPacket, buffer); this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos); return newPacket; } } } async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise<EncodedPacket | null> { let currentPacket: EncodedPacket | null = packet; // Just loop until we hit one while (true) { currentPacket = await this.getNextPacket(currentPacket, options); if (!currentPacket) { return null; } if (currentPacket.type === 'key') { return currentPacket; } } } getPacket(timestamp: number, options: PacketRetrievalOptions): Promise<EncodedPacket | null> { return this.doPacketLookup(timestamp, false, options); } getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise<EncodedPacket | null> { return this.doPacketLookup(timestamp, true, options); } /** * Searches for the packet with the largest timestamp not larger than `timestamp` in the file, using a combination * of chunk-based binary search and linear refinement. The reason the coarse search is done in large chunks is to * make it more performant for small files and over high-latency readers such as the network. */ async doPacketLookup( timestamp: number, keyframesOnly: boolean, options: PacketRetrievalOptions, ): Promise<EncodedPacket | null> { const searchPts = roundIfAlmostInteger(timestamp * TIMESCALE); const demuxer = this.elementaryStream.demuxer; const { reader, seekChunkSize } = demuxer; const pid = this.elementaryStream.pid; const findFirstPesPacketHeaderInChunk = async ( startPos: number, endPos: number, readSectionInFull: boolean, ) => { let currentPos = startPos; while (currentPos < endPos) { const packetHeader = await demuxer.readPacketHeader(currentPos); if (!packetHeader) { return null; } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { const section = await demuxer.readSection(currentPos, readSectionInFull); if (!section) { return null; } const pesPacketHeader = readPesPacketHeader(section, false); if (pesPacketHeader && pesPacketHeader.pts !== null) { return { pesPacketHeader: pesPacketHeader as TimestampedPesPacketHeader, section, }; } } currentPos += demuxer.packetStride; } return null; }; // Get the first PES packet of the track const firstSection = this.elementaryStream.firstSection; assert(firstSection); const firstPesPacketHeader = readPesPacketHeader(firstSection, true); assert(firstPesPacketHeader); if (searchPts < firstPesPacketHeader.pts) { // We're before the first packet, definitely nothing here return null; } let scanStartPos: number; const referencePesPackets = this.elementaryStream.referencePesPackets; const referencePointIndex = binarySearchLessOrEqual(referencePesPackets, searchPts, x => x.pts); const referencePoint = referencePointIndex !== -1 ? referencePesPackets[referencePointIndex]! : null; if (referencePoint && searchPts - referencePoint.pts < TIMESCALE / 2) { // Reference point ain't too far away, prefer it over the chunk search scanStartPos = referencePoint.sectionStartPos; } else { let startChunkIndex = 0; if (reader.fileSize !== null) { const numChunks = Math.ceil(reader.fileSize / seekChunkSize); if (numChunks > 1) { // Binary search to find the chunk with highest index whose first PES has pts <= searchPts let low = 0; let high = numChunks - 1; startChunkIndex = low; while (low <= high) { const mid = Math.floor((low + high) / 2); const chunkStartPos = floorToMultiple(mid * seekChunkSize, demuxer.packetStride) + firstPesPacketHeader.sectionStartPos; const chunkEndPos = chunkStartPos + seekChunkSize; const result = await findFirstPesPacketHeaderInChunk(chunkStartPos, chunkEndPos, false); if (!result) { // No PES packet found in this chunk, search left high = mid - 1; continue; } if (result.pesPacketHeader.pts <= searchPts) { // This chunk's first PES is <= searchPts, it's a candidate startChunkIndex = mid; low = mid + 1; // Search right } else { // Search left high = mid - 1; } } } } scanStartPos = floorToMultiple( startChunkIndex * seekChunkSize, demuxer.packetStride, ) + firstPesPacketHeader.sectionStartPos; } // Find the first PES packet at or after scanStartPos const result = await findFirstPesPacketHeaderInChunk( scanStartPos, reader.fileSize ?? Infinity, false, ); let currentPesHeader = result?.pesPacketHeader ?? null; if (!currentPesHeader) { // Fall back to first packet currentPesHeader = firstPesPacketHeader; } const reorderSize = this.getReorderSize(); const retrieveEncodedPacket = async ( sectionStartPos: number, predicate: (packet: SuppliedPacket) => boolean, ) => { // Load the relevant section in full const section = await demuxer.readSection(sectionStartPos, true); assert(section); const pesPacket = readPesPacket(section, true); assert(pesPacket); const context = new PacketReadingContext(this.elementaryStream, pesPacket); const buffer = new PacketBuffer(this, context); // Advance until the top-most presentation timestamp crosses or equals searchPts while (true) { const topPts = last(buffer.presentationOrderPackets)?.pts ?? -Infinity; if (topPts >= searchPts) { break; } const didRead = await buffer.readNextPacket(); if (!didRead) { break; } } const targetIndex = findLastIndex(buffer.presentationOrderPackets, predicate); if (targetIndex === -1) { return null; } const targetPacket = buffer.presentationOrderPackets[targetIndex]!; const lastDuration = targetIndex === 0 ? 0 : targetPacket.pts - buffer.presentationOrderPackets[targetIndex - 1]!.pts; // Pop packets in decode order until we hit the target packet while (buffer.decodeOrderPackets[0] !== targetPacket) { buffer.decodeOrderPackets.shift(); } buffer.lastDuration = lastDuration; // Kinda ugly but necessary fix const result = await buffer.readNext(); assert(result); const packet = this.createEncodedPacket(result.packet, result.duration, options); this.packetBuffers.set(packet, buffer); this.packetSectionStarts.set(packet, result.packet.sectionStartPos); return packet; }; if (!keyframesOnly || this.allPacketsAreKeyPackets()) { // Normat packet lookup case. Slightly easier since we just need to search (mostly) forward to find the // packet. // Linear scan to find the PES packet with largest pts <= searchPts. This will be used as the "midpoint" // of the next refinement step (which is needed because of B-frames). outer: while (true) { let currentPos = currentPesHeader.sectionStartPos + demuxer.packetStride; while (true) { const packetHeader = await demuxer.readPacketHeader(currentPos); if (!packetHeader) { break outer; // End of file } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { const section = await demuxer.readSection(currentPos, false); if (section) { const nextPesHeader = readPesPacketHeader(section, false); if (nextPesHeader && nextPesHeader.pts !== null) { if (nextPesHeader.pts > searchPts) { break outer; } currentPesHeader = nextPesHeader as TimestampedPesPacketHeader; maybeInsertReferencePacket(this.elementaryStream, currentPesHeader); break; } } } currentPos += demuxer.packetStride; } } // Rewind by reorderSize + 1 PES packets (even for audio! To ensure proper durations) outer: for (let i = 0; i < reorderSize + 1; i++) { let pos = currentPesHeader.sectionStartPos - demuxer.packetStride; while (pos >= demuxer.packetOffset) { const packetHeader = await demuxer.readPacketHeader(pos); if (!packetHeader) { break outer; } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { const section = await demuxer.readSection(pos, false); if (section) { const header = readPesPacketHeader(section, false); if (header && header.pts !== null) { currentPesHeader = header as TimestampedPesPacketHeader; break; } } } pos -= demuxer.packetStride; } } return retrieveEncodedPacket(currentPesHeader.sectionStartPos, p => p.pts <= searchPts); } else { // Key packet lookup case. Slightly harder since the starting chunk may not have a key packet at all, which // means we might need to search the previous chunks until we find something. let currentChunkStartPos = scanStartPos; let nextChunkStartPos: number | null = null; // "next" as in later in the file, even tho we scan backwards const readSectionsInFull = !this.elementaryStream.canBeTrustedWithKeyPackets; while (true) { let bestKeyPesHeader: TimestampedPesPacketHeader | null = null; const isFirstChunk = currentChunkStartPos <= firstPesPacketHeader.sectionStartPos; let pesHeader: TimestampedPesPacketHeader | null; let pesHeaderSection: Section | null = null; if (isFirstChunk) { pesHeader = firstPesPacketHeader; pesHeaderSection = firstSection; } else { const result = await findFirstPesPacketHeaderInChunk( currentChunkStartPos, reader.fileSize ?? Infinity, readSectionsInFull, ); pesHeader = result?.pesPacketHeader ?? null; pesHeaderSection = result?.section ?? null; } let passedSearchPts = false; let lookaheadCount = 0; outer: while (pesHeader) { if (nextChunkStartPos !== null && pesHeader.sectionStartPos >= nextChunkStartPos) { // Stop at the next chunk boundary break; } if (pesHeader.pts <= searchPts) { let isKeyPacket: boolean; if (this.elementaryStream.canBeTrustedWithKeyPackets) { isKeyPacket = pesHeader.randomAccessIndicator === 1; } else { assert(pesHeaderSection); const pesPacket = readPesPacket(pesHeaderSection, true); assert(pesPacket); const context = new PacketReadingContext(this.elementaryStream, pesPacket); await context.markNextPacket(); isKeyPacket = context.suppliedPacket?.randomAccessIndicator === 1; } if (isKeyPacket) { bestKeyPesHeader = pesHeader; } } if (pesHeader.pts > searchPts) { passedSearchPts = true; } // If we've passed searchPts, do lookahead for reorderSize more packets just to be sure if (passedSearchPts) { lookaheadCount++; if (lookaheadCount > reorderSize) { break; } } // Find next PES packet let currentPos = pesHeader.sectionStartPos + demuxer.packetStride; while (true) { const packetHeader = await demuxer.readPacketHeader(currentPos); if (!packetHeader) { break outer; // End of file } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { const section = await demuxer.readSection(currentPos, readSectionsInFull); if (section) { const nextPesHeader = readPesPacketHeader(section, false); if (nextPesHeader && nextPesHeader.pts !== null) { pesHeader = nextPesHeader as TimestampedPesPacketHeader; pesHeaderSection = section; maybeInsertReferencePacket(this.elementaryStream, pesHeader); break; } } } currentPos += demuxer.packetStride; } } if (bestKeyPesHeader) { let startPesHeader = bestKeyPesHeader; if (lookaheadCount === 0) { // Packet is at the end of stream, let's rewind a little to obtain the correct packet duration outer: for (let i = 0; i < reorderSize; i++) { let pos = startPesHeader.sectionStartPos - demuxer.packetStride; while (pos >= demuxer.packetOffset) { const packetHeader = await demuxer.readPacketHeader(pos); if (!packetHeader) { break outer; } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { const section = await demuxer.readSection(pos, readSectionsInFull); if (section) { const header = readPesPacketHeader(section, false); if (header && header.pts !== null) { startPesHeader = header as TimestampedPesPacketHeader; break; } } } pos -= demuxer.packetStride; } } } const encodedPacket = await retrieveEncodedPacket( startPesHeader.sectionStartPos, p => p.p