mediabunny
Version:
Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.
1,635 lines (1,367 loc) • 72.2 kB
text/typescript
/*!
* Copyright (c) 2026-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
import { TrackType } from '../output';
import { SAMPLES_PER_AAC_FRAME } from '../adts/adts-demuxer';
import { MAX_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader';
import { aacChannelMap, aacFrequencyTable } from '../../shared/aac-misc';
import {
AacCodecInfo,
AudioCodec,
extractAudioCodecString,
extractVideoCodecString,
MediaCodec,
VideoCodec,
} from '../codec';
import {
AC3_ACMOD_CHANNEL_COUNTS,
AC3_SAMPLES_PER_FRAME,
AvcDecoderConfigurationRecord,
AvcNalUnitType,
determineVideoPacketType,
extractAvcDecoderConfigurationRecord,
extractHevcDecoderConfigurationRecord,
extractNalUnitTypeForAvc,
extractNalUnitTypeForHevc,
EAC3_NUMBLKS_TABLE,
getEac3ChannelCount,
getEac3SampleRate,
HevcDecoderConfigurationRecord,
HevcNalUnitType,
parseAc3SyncFrame,
parseAvcSps,
parseEac3SyncFrame,
parseHevcSps,
AC3_FRAME_SIZES,
} from '../codec-data';
import { Demuxer } from '../demuxer';
import { Input } from '../input';
import {
InputAudioTrackBacking,
InputTrackBacking,
InputVideoTrackBacking,
} from '../input-track';
import { PacketRetrievalOptions } from '../media-sink';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata';
import {
assert,
binarySearchExact,
binarySearchLessOrEqual,
COLOR_PRIMARIES_MAP_INVERSE,
findLastIndex,
floorToMultiple,
last,
MATRIX_COEFFICIENTS_MAP_INVERSE,
Rotation,
roundIfAlmostInteger,
toDataView,
TRANSFER_CHARACTERISTICS_MAP_INVERSE,
UNDETERMINED_LANGUAGE,
} from '../misc';
import {
MP3_FRAME_HEADER_SIZE,
getMp3ChannelCount,
readMp3FrameHeader,
} from '../../shared/mp3-misc';
import { EncodedPacket, PacketType, PLACEHOLDER_DATA } from '../packet';
import { FileSlice, readBytes, Reader, readU16Be, readU32Be, readU8 } from '../reader';
import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc';
import { AC3_SAMPLE_RATES } from '../../shared/ac3-misc';
import { Bitstream } from '../../shared/bitstream';
// Resources:
// ISO/IEC 13818-1
const MISSING_PTS_ERROR_MESSAGE = 'PES packet is missing PTS where it was expected. PES packets without PTS are not'
+ ' currently supported. If you think this file should be supported, please report it.';
type ElementaryStream = {
demuxer: MpegTsDemuxer;
pid: number;
streamType: number;
initialized: boolean;
firstSection: Section | null;
/**
* Some muxers suck ass and don't correctly label key frames, meaning we'll need to use our skill to
* compensate for another programmer's skill issue.
*/
canBeTrustedWithKeyPackets: boolean;
info: {
type: 'video';
codec: VideoCodec;
decoderConfig: VideoDecoderConfig | null;
avcCodecInfo: AvcDecoderConfigurationRecord | null;
hevcCodecInfo: HevcDecoderConfigurationRecord | null;
colorSpace: VideoColorSpaceInit;
width: number;
height: number;
squarePixelWidth: number;
squarePixelHeight: number;
reorderSize: number;
} | {
type: 'audio';
codec: AudioCodec;
decoderConfig: AudioDecoderConfig | null;
aacCodecInfo: AacCodecInfo | null;
numberOfChannels: number;
sampleRate: number;
};
/**
* Reference PES packets, spread throughout the file, to be used to speed up repeated random access. Sorted by both
* byte offset and PTS.
*/
referencePesPackets: TimestampedPesPacketHeader[];
};
type ElementaryVideoStream = ElementaryStream & { info: { type: 'video' } };
type ElementaryAudioStream = ElementaryStream & { info: { type: 'audio' } };
type TsPacketHeader = {
payloadUnitStartIndicator: number;
pid: number;
adaptationFieldControl: number;
};
type TsPacket = TsPacketHeader & {
body: Uint8Array<ArrayBufferLike>;
};
type Section = {
startPos: number;
endPos: number | null; // null if the section was not read fully
pid: number;
payload: Uint8Array<ArrayBufferLike>;
randomAccessIndicator: number;
};
// Remember them so the warning doesn't get spammed
const ignoredStreamTypes = new Set<number>();
export class MpegTsDemuxer extends Demuxer {
reader: Reader;
metadataPromise: Promise<void> | null = null;
elementaryStreams: ElementaryStream[] = [];
trackBackingEntries: InputTrackBacking[] = [];
packetOffset = 0;
packetStride = -1;
sectionEndPositions: number[] = [];
seekChunkSize = 5 * 1024 * 1024; // 5 MiB, picked because most HLS segments are below this size
minReferencePointByteDistance = -1;
constructor(input: Input) {
super(input);
this.reader = input._reader;
}
async readMetadata() {
return this.metadataPromise ??= (async () => {
const lengthToCheck = TS_PACKET_SIZE + 16 + 1;
let startingSlice = this.reader.requestSlice(0, lengthToCheck);
if (startingSlice instanceof Promise) startingSlice = await startingSlice;
assert(startingSlice);
const startingBytes = readBytes(startingSlice, lengthToCheck);
if (startingBytes[0] === 0x47 && startingBytes[TS_PACKET_SIZE] === 0x47) {
// Regular MPEG-TS
this.packetOffset = 0;
this.packetStride = TS_PACKET_SIZE;
} else if (startingBytes[0] === 0x47 && startingBytes[TS_PACKET_SIZE + 16] === 0x47) {
// MPEG-TS with Forward Error Correction
this.packetOffset = 0;
this.packetStride = TS_PACKET_SIZE + 16;
} else if (startingBytes[4] === 0x47 && startingBytes[4 + TS_PACKET_SIZE + 4] === 0x47) {
// MPEG-2-TS (DVHS)
this.packetOffset = 4;
this.packetStride = TS_PACKET_SIZE + 4;
} else {
throw new Error('Unreachable.');
}
const MIN_REFERENCE_POINT_PACKET_DISTANCE = 256;
this.minReferencePointByteDistance = MIN_REFERENCE_POINT_PACKET_DISTANCE * this.packetStride;
let currentPos = this.packetOffset;
let programMapPid: number | null = null;
// Some files contain these multiple times, but we only care about their first appearance
let hasProgramAssociationTable = false;
let hasProgramMap = false;
while (true) {
const packetHeader = await this.readPacketHeader(currentPos);
if (!packetHeader) {
break;
}
if (packetHeader.payloadUnitStartIndicator === 0) {
// Not the start of a section
currentPos += this.packetStride;
continue;
}
const section = await this.readSection(
currentPos,
true,
!hasProgramMap, // Expect contiguous sections as long as we don't have the PMT
);
if (!section) {
break;
}
const BYTES_BEFORE_SECTION_LENGTH = 3;
const BITS_IN_CRC_32 = 32; // Duh
// Some streams don't contain a PAT for some reason, so we must do some guesswork to figure out where
// the PMT is.
let isProbablyProgramMap = false;
if (!hasProgramMap && section.pid !== 0) {
const isPesPacket
= section.payload[0] === 0x00 && section.payload[1] === 0x00 && section.payload[2] === 0x01;
if (!isPesPacket) {
// Assume it's a PSI
const bitstream = new Bitstream(section.payload);
const pointerField = bitstream.readAlignedByte();
bitstream.skipBits(8 * pointerField);
const tableId = bitstream.readBits(8);
isProbablyProgramMap = tableId === 0x02; // 0x02 == TS_program_map_section
}
}
if (section.pid === 0 && !hasProgramAssociationTable) {
const bitstream = new Bitstream(section.payload);
const pointerField = bitstream.readAlignedByte();
bitstream.skipBits(8 * pointerField);
bitstream.skipBits(14);
const sectionLength = bitstream.readBits(10);
bitstream.skipBits(40);
while (8 * (sectionLength + BYTES_BEFORE_SECTION_LENGTH) - bitstream.pos > BITS_IN_CRC_32) {
const programNumber = bitstream.readBits(16);
bitstream.skipBits(3); // Reserved
const id = bitstream.readBits(13);
if (programNumber !== 0) {
if (programMapPid !== null) {
throw new Error('Only files with a single program are supported.');
} else {
programMapPid = id;
}
}
}
if (programMapPid === null) {
throw new Error('Program Association Table must link to a Program Map Table.');
}
hasProgramAssociationTable = true;
} else if ((section.pid === programMapPid || isProbablyProgramMap) && !hasProgramMap) {
const bitstream = new Bitstream(section.payload);
const pointerField = bitstream.readAlignedByte();
bitstream.skipBits(8 * pointerField);
bitstream.skipBits(12);
const sectionLength = bitstream.readBits(12);
bitstream.skipBits(43);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const pcrPid = bitstream.readBits(13);
bitstream.skipBits(6);
// "The remaining 10 bits specify the number of bytes of the descriptors immediately following the
// program_info_length field"
const programInfoLength = bitstream.readBits(10);
bitstream.skipBits(8 * programInfoLength);
while (8 * (sectionLength + BYTES_BEFORE_SECTION_LENGTH) - bitstream.pos > BITS_IN_CRC_32) {
const streamType = bitstream.readBits(8);
bitstream.skipBits(3);
const elementaryPid = bitstream.readBits(13);
bitstream.skipBits(6);
const esInfoLength = bitstream.readBits(10);
// Check ES descriptors to detect AC-3/E-AC-3 in System B
const esInfoEndPos = bitstream.pos + 8 * esInfoLength;
let hasAc3Descriptor = false;
let hasEac3Descriptor = false;
while (bitstream.pos < esInfoEndPos) {
const descriptorTag = bitstream.readBits(8);
const descriptorLength = bitstream.readBits(8);
if (descriptorTag === 0x6a) {
hasAc3Descriptor = true;
} else if (descriptorTag === 0x7a || descriptorTag === 0xcc) {
hasEac3Descriptor = true;
}
bitstream.skipBits(8 * descriptorLength);
}
let info: ElementaryStream['info'] | null = null;
switch (streamType) {
case MpegTsStreamType.AVC:
case MpegTsStreamType.HEVC: {
const codec = streamType === MpegTsStreamType.AVC ? 'avc' : 'hevc';
info = {
type: 'video',
codec,
decoderConfig: null,
avcCodecInfo: null,
hevcCodecInfo: null,
colorSpace: {
primaries: null,
transfer: null,
matrix: null,
fullRange: null,
},
width: -1,
height: -1,
squarePixelWidth: -1,
squarePixelHeight: -1,
reorderSize: -1,
};
}; break;
case MpegTsStreamType.MP3_MPEG1:
case MpegTsStreamType.MP3_MPEG2:
case MpegTsStreamType.AAC:
case MpegTsStreamType.AC3_SYSTEM_A:
case MpegTsStreamType.EAC3_SYSTEM_A: {
let codec: AudioCodec;
if (
streamType === MpegTsStreamType.MP3_MPEG1
|| streamType === MpegTsStreamType.MP3_MPEG2
) {
codec = 'mp3';
} else if (streamType === MpegTsStreamType.AAC) {
codec = 'aac';
} else if (streamType === MpegTsStreamType.AC3_SYSTEM_A) {
codec = 'ac3';
} else if (streamType === MpegTsStreamType.EAC3_SYSTEM_A) {
codec = 'eac3';
} else {
throw new Error('Unreachable.');
}
info = {
type: 'audio',
codec,
decoderConfig: null,
aacCodecInfo: null,
numberOfChannels: -1,
sampleRate: -1,
};
}; break;
case MpegTsStreamType.PRIVATE_DATA: {
if (hasEac3Descriptor) {
info = {
type: 'audio',
codec: 'eac3',
decoderConfig: null,
aacCodecInfo: null,
numberOfChannels: -1,
sampleRate: -1,
};
} else if (hasAc3Descriptor) {
info = {
type: 'audio',
codec: 'ac3',
decoderConfig: null,
aacCodecInfo: null,
numberOfChannels: -1,
sampleRate: -1,
};
}
}; break;
default: {
// If we don't recognize the codec, we don't surface the track at all. This is because
// we can't determine its metadata and also have no idea how to packetize its data.
if (!ignoredStreamTypes.has(streamType)) {
console.warn(
`Note: MPEG-TS streams with stream_type 0x${streamType.toString(16)} are not`
+ ` currently supported.`,
);
ignoredStreamTypes.add(streamType);
}
}
}
if (info) {
this.elementaryStreams.push({
demuxer: this,
pid: elementaryPid,
streamType,
initialized: false,
firstSection: null,
canBeTrustedWithKeyPackets: false,
info,
referencePesPackets: [],
});
}
}
hasProgramMap = true;
} else {
const elementaryStream = this.elementaryStreams.find(x => x.pid === section.pid);
outer:
if (elementaryStream && !elementaryStream.initialized) {
const pesPacket = readPesPacket(section, true);
if (!pesPacket) {
throw new Error(
`Couldn't read first PES packet for Elementary Stream with PID ${elementaryStream.pid}`,
);
}
elementaryStream.firstSection = section;
elementaryStream.canBeTrustedWithKeyPackets = section.randomAccessIndicator === 1;
if (this.input._initInput) {
const initDemuxer = (await this.input._initInput._getDemuxer()) as MpegTsDemuxer;
const matchingStream = initDemuxer.elementaryStreams.find(x => (
x.pid === section.pid && x.info.codec === elementaryStream.info.codec
));
if (matchingStream) {
elementaryStream.info = matchingStream.info;
elementaryStream.initialized = true;
break outer; // We have the stream info, we're done
}
}
const context = new PacketReadingContext(elementaryStream, pesPacket);
if (elementaryStream.info.type === 'video') {
// We loop because in some files, the video parameters are not in the first packet
while (true) {
const contextAlias = context; // TyyyyypeScript 😩
contextAlias.suppliedPacket = null;
await context.markNextPacket();
if (elementaryStream.info.codec === 'avc') {
if (!context.suppliedPacket) {
throw new Error(
'Invalid AVC video stream; could not extract AVCDecoderConfigurationRecord'
+ ' from any packet.',
);
}
elementaryStream.info.avcCodecInfo
= extractAvcDecoderConfigurationRecord(context.suppliedPacket.data);
if (!elementaryStream.info.avcCodecInfo) {
continue; // Search the next packet for it
}
const spsUnit = elementaryStream.info.avcCodecInfo.sequenceParameterSets[0];
assert(spsUnit);
const spsInfo = parseAvcSps(spsUnit)!;
elementaryStream.info.width = spsInfo.displayWidth;
elementaryStream.info.height = spsInfo.displayHeight;
const num = spsInfo.pixelAspectRatio.num;
const den = spsInfo.pixelAspectRatio.den;
if (num > 0 && den > 0) {
if (num > den) {
elementaryStream.info.squarePixelWidth = Math.round(
elementaryStream.info.width * num / den,
);
elementaryStream.info.squarePixelHeight = elementaryStream.info.height;
} else {
elementaryStream.info.squarePixelWidth = elementaryStream.info.width;
elementaryStream.info.squarePixelHeight = Math.round(
elementaryStream.info.height * den / num,
);
}
}
elementaryStream.info.colorSpace = {
primaries:
COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as
VideoColorPrimaries | undefined,
transfer:
TRANSFER_CHARACTERISTICS_MAP_INVERSE[spsInfo.transferCharacteristics] as
VideoTransferCharacteristics | undefined,
matrix:
MATRIX_COEFFICIENTS_MAP_INVERSE[spsInfo.matrixCoefficients] as
VideoMatrixCoefficients | undefined,
fullRange: !!spsInfo.fullRangeFlag,
};
elementaryStream.info.reorderSize = spsInfo.maxDecFrameBuffering;
break;
} else if (elementaryStream.info.codec === 'hevc') {
if (!context.suppliedPacket) {
throw new Error(
'Invalid HEVC video stream; could not extract HVCDecoderConfigurationRecord'
+ ' from first packet.',
);
}
elementaryStream.info.hevcCodecInfo
= extractHevcDecoderConfigurationRecord(context.suppliedPacket.data);
if (!elementaryStream.info.hevcCodecInfo) {
continue; // Search the next packet for it
}
const spsArray = elementaryStream.info.hevcCodecInfo.arrays.find(
a => a.nalUnitType === HevcNalUnitType.SPS_NUT,
)!;
const spsUnit = spsArray.nalUnits[0];
assert(spsUnit);
const spsInfo = parseHevcSps(spsUnit)!;
elementaryStream.info.width = spsInfo.displayWidth;
elementaryStream.info.height = spsInfo.displayHeight;
if (spsInfo.pixelAspectRatio.num > spsInfo.pixelAspectRatio.den) {
elementaryStream.info.squarePixelWidth = Math.round(
elementaryStream.info.width
* spsInfo.pixelAspectRatio.num / spsInfo.pixelAspectRatio.den,
);
elementaryStream.info.squarePixelHeight = elementaryStream.info.height;
} else {
elementaryStream.info.squarePixelWidth = elementaryStream.info.width;
elementaryStream.info.squarePixelHeight = Math.round(
elementaryStream.info.height
* spsInfo.pixelAspectRatio.den / spsInfo.pixelAspectRatio.num,
);
}
elementaryStream.info.colorSpace = {
primaries:
COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as
VideoColorPrimaries | undefined,
transfer:
TRANSFER_CHARACTERISTICS_MAP_INVERSE[spsInfo.transferCharacteristics] as
VideoTransferCharacteristics | undefined,
matrix:
MATRIX_COEFFICIENTS_MAP_INVERSE[spsInfo.matrixCoefficients] as
VideoMatrixCoefficients | undefined,
fullRange: !!spsInfo.fullRangeFlag,
};
elementaryStream.info.reorderSize = spsInfo.maxDecFrameBuffering;
break;
} else {
throw new Error('Unhandled.');
}
}
elementaryStream.info.decoderConfig = {
codec: extractVideoCodecString({
width: elementaryStream.info.width,
height: elementaryStream.info.height,
codec: elementaryStream.info.codec,
codecDescription: null,
colorSpace: elementaryStream.info.colorSpace,
avcType: 1,
avcCodecInfo: elementaryStream.info.avcCodecInfo,
hevcCodecInfo: elementaryStream.info.hevcCodecInfo,
vp9CodecInfo: null,
av1CodecInfo: null,
}),
codedWidth: elementaryStream.info.width,
codedHeight: elementaryStream.info.height,
colorSpace: elementaryStream.info.colorSpace,
};
if (
elementaryStream.info.width !== elementaryStream.info.squarePixelWidth
|| elementaryStream.info.height !== elementaryStream.info.squarePixelHeight
) {
elementaryStream.info.decoderConfig.displayAspectWidth
= elementaryStream.info.squarePixelWidth;
elementaryStream.info.decoderConfig.displayAspectHeight
= elementaryStream.info.squarePixelHeight;
}
elementaryStream.initialized = true;
} else {
await context.markNextPacket();
if (!context.suppliedPacket) {
throw new Error(
`Couldn't parse first media packet for Elementary Stream with`
+ ` PID ${elementaryStream.pid}`,
);
}
if (elementaryStream.info.codec === 'aac') {
const slice = FileSlice.tempFromBytes(context.suppliedPacket.data);
const header = readAdtsFrameHeader(slice);
if (!header) {
throw new Error(
'Invalid AAC audio stream; could not read ADTS frame header from first packet.',
);
}
elementaryStream.info.aacCodecInfo = {
isMpeg2: false,
objectType: header.objectType,
};
elementaryStream.info.numberOfChannels
= aacChannelMap[header.channelConfiguration]!;
elementaryStream.info.sampleRate
= aacFrequencyTable[header.samplingFrequencyIndex]!;
} else if (elementaryStream.info.codec === 'mp3') {
const word = readU32Be(FileSlice.tempFromBytes(context.suppliedPacket.data));
const result = readMp3FrameHeader(word, context.suppliedPacket.data.byteLength);
if (!result.header) {
throw new Error(
'Invalid MP3 audio stream; could not read frame header from first packet.',
);
}
elementaryStream.info.numberOfChannels = getMp3ChannelCount(result.header.channel);
elementaryStream.info.sampleRate = result.header.sampleRate;
} else if (elementaryStream.info.codec === 'ac3') {
const frameInfo = parseAc3SyncFrame(context.suppliedPacket.data);
if (!frameInfo) {
throw new Error(
'Invalid AC-3 audio stream; could not read sync frame from first packet.',
);
}
if (frameInfo.fscod === 3) {
throw new Error(
'Invalid AC-3 audio stream; reserved sample rate code found in first packet.',
);
}
elementaryStream.info.numberOfChannels
= AC3_ACMOD_CHANNEL_COUNTS[frameInfo.acmod]! + frameInfo.lfeon;
elementaryStream.info.sampleRate = AC3_SAMPLE_RATES[frameInfo.fscod]!;
} else if (elementaryStream.info.codec === 'eac3') {
const frameInfo = parseEac3SyncFrame(context.suppliedPacket.data);
if (!frameInfo) {
throw new Error(
'Invalid E-AC-3 audio stream; could not read sync frame from first packet.',
);
}
const sampleRate = getEac3SampleRate(frameInfo);
if (sampleRate === null) {
throw new Error(
'Invalid E-AC-3 audio stream; reserved sample rate code found in first packet.',
);
}
elementaryStream.info.numberOfChannels = getEac3ChannelCount(frameInfo);
elementaryStream.info.sampleRate = sampleRate;
} else {
throw new Error('Unhandled.');
}
elementaryStream.info.decoderConfig = {
codec: extractAudioCodecString({
codec: elementaryStream.info.codec,
codecDescription: null,
aacCodecInfo: elementaryStream.info.aacCodecInfo,
}),
numberOfChannels: elementaryStream.info.numberOfChannels,
sampleRate: elementaryStream.info.sampleRate,
};
elementaryStream.initialized = true;
}
}
}
const isDone = hasProgramMap && this.elementaryStreams.every(x => x.initialized);
if (isDone) {
break;
}
currentPos += this.packetStride;
}
if (!hasProgramMap) {
if (!hasProgramAssociationTable) {
throw new Error('No Program Association Table found in the file.');
}
throw new Error('No Program Map Table found in the file.');
}
for (const stream of this.elementaryStreams) {
if (stream.info.type === 'video') {
this.trackBackingEntries.push(
new MpegTsVideoTrackBacking(stream as ElementaryVideoStream),
);
} else {
this.trackBackingEntries.push(
new MpegTsAudioTrackBacking(stream as ElementaryAudioStream),
);
}
}
})();
}
async getTrackBackings() {
await this.readMetadata();
return this.trackBackingEntries;
}
async getMetadataTags(): Promise<MetadataTags> {
return {}; // Nothing for now
}
async getMimeType(): Promise<string> {
await this.readMetadata();
const codecStrings = await Promise.all(this.trackBackingEntries.map(
x => x.getDecoderConfig().then(c => c?.codec ?? null),
));
return buildMpegTsMimeType(codecStrings);
}
async readSection(startPos: number, full: boolean, contiguous = false): Promise<Section | null> {
let endPos = startPos;
let currentPos = startPos;
const chunks: Uint8Array[] = [];
let chunksByteLength = 0;
let firstPacket: TsPacket | null = null;
let mustAddSectionEnd = true;
let randomAccessIndicator = 0;
while (true) {
const packet = await this.readPacket(currentPos);
currentPos += this.packetStride;
if (!packet) {
break;
}
if (!firstPacket) {
if (packet.payloadUnitStartIndicator === 0) {
break;
}
firstPacket = packet;
} else {
if (packet.pid !== firstPacket.pid) {
if (contiguous) {
break; // End of section
} else {
continue; // Ignore this packet
}
}
if (packet.payloadUnitStartIndicator === 1) {
break;
}
}
const hasAdaptationField = !!(packet.adaptationFieldControl & 0b10);
const hasPayload = !!(packet.adaptationFieldControl & 0b01);
let adaptationFieldLength = 0;
if (hasAdaptationField) {
adaptationFieldLength = 1 + packet.body[0]!;
// Extract random_access_indicator from first packet's adaptation field
if (packet === firstPacket && adaptationFieldLength > 1) {
randomAccessIndicator = (packet.body[1]! >> 6) & 1;
}
}
if (hasPayload) {
if (adaptationFieldLength === 0) {
chunks.push(packet.body);
chunksByteLength += packet.body.byteLength;
} else {
chunks.push(packet.body.subarray(adaptationFieldLength));
chunksByteLength += packet.body.byteLength - adaptationFieldLength;
}
}
endPos = currentPos;
// 64 is just "a bit of data", enough for the PES packet header
if (!full && chunksByteLength >= 64) {
mustAddSectionEnd = false; // Not the actual section end
break;
}
// Check if we already know this is a section end
const isKnownSectionEnd = binarySearchExact(this.sectionEndPositions, endPos, x => x) !== -1;
if (isKnownSectionEnd) {
mustAddSectionEnd = false;
break;
}
}
if (mustAddSectionEnd) {
const index = binarySearchLessOrEqual(this.sectionEndPositions, endPos, x => x);
this.sectionEndPositions.splice(index + 1, 0, endPos);
}
if (!firstPacket) {
return null;
}
let merged: Uint8Array;
if (chunks.length === 1) {
merged = chunks[0]!;
} else {
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
merged = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.length;
}
}
return {
startPos,
endPos: full ? endPos : null,
pid: firstPacket.pid,
payload: merged,
randomAccessIndicator,
};
}
async readPacketHeader(pos: number): Promise<TsPacketHeader | null> {
let slice = this.reader.requestSlice(pos, 4);
if (slice instanceof Promise) slice = await slice;
if (!slice) {
return null;
}
const syncByte = readU8(slice);
if (syncByte !== 0x47) {
throw new Error('Invalid TS packet sync byte. Likely an internal bug, please report this file.');
}
const nextTwoBytes = readU16Be(slice);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const transportErrorIndicator = nextTwoBytes >> 15;
const payloadUnitStartIndicator = (nextTwoBytes >> 14) & 0x1;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const transportPriority = (nextTwoBytes >> 13) & 0x1;
const pid = nextTwoBytes & 0x1FFF;
const nextByte = readU8(slice);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const transportScramblingControl = nextByte >> 6;
const adaptationFieldControl = (nextByte >> 4) & 0x3;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const continuityCounter = nextByte & 0xF;
return {
payloadUnitStartIndicator,
pid,
adaptationFieldControl,
};
}
async readPacket(pos: number): Promise<TsPacket | null> {
// Code in here is duplicated from readPacketHeader for performance reasons
let slice = this.reader.requestSlice(pos, TS_PACKET_SIZE);
if (slice instanceof Promise) slice = await slice;
if (!slice) {
return null;
}
const bytes = readBytes(slice, TS_PACKET_SIZE);
const syncByte = bytes[0]!;
if (syncByte !== 0x47) {
throw new Error('Invalid TS packet sync byte. Likely an internal bug, please report this file.');
}
const nextTwoBytes = (bytes[1]! << 8) + bytes[2]!;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const transportErrorIndicator = nextTwoBytes >> 15;
const payloadUnitStartIndicator = (nextTwoBytes >> 14) & 0x1;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const transportPriority = (nextTwoBytes >> 13) & 0x1;
const pid = nextTwoBytes & 0x1FFF;
const nextByte = bytes[3]!;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const transportScramblingControl = nextByte >> 6;
const adaptationFieldControl = (nextByte >> 4) & 0x3;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const continuityCounter = nextByte & 0xF;
return {
payloadUnitStartIndicator,
pid,
adaptationFieldControl,
body: bytes.subarray(4),
};
}
}
type PesPacketHeader = {
sectionStartPos: number;
sectionEndPos: number | null; // null if the section wasn't read fully
pts: number | null;
randomAccessIndicator: number;
};
type TimestampedPesPacketHeader = PesPacketHeader & {
pts: number;
};
type PesPacket = PesPacketHeader & {
data: Uint8Array<ArrayBufferLike>;
};
type TimestampedPesPacket = PesPacket & {
pts: number;
};
const readPesPacketHeader = <T extends boolean>(
section: Section,
expectPts: T,
): (T extends true ? TimestampedPesPacketHeader : PesPacketHeader) | null => {
if (section.payload.byteLength < 3) {
return null;
}
const bitstream = new Bitstream(section.payload);
const startCodePrefix = bitstream.readBits(24);
if (startCodePrefix !== 0x000001) {
return null;
}
const streamId = bitstream.readBits(8);
bitstream.skipBits(16);
if (
streamId === 0b10111100 // program_stream_map
|| streamId === 0b10111110 // padding_stream
|| streamId === 0b10111111 // private_stream_2
|| streamId === 0b11110000 // ECM
|| streamId === 0b11110001 // EMM
|| streamId === 0b11111111 // program_stream_directory
|| streamId === 0b11110010 // DSMCC_stream
|| streamId === 0b11111000 // ITU-T Rec. H.222.1 type E stream
) {
return null;
}
bitstream.skipBits(8);
const ptsDtsFlags = bitstream.readBits(2);
bitstream.skipBits(14);
let pts: number | null = null;
if (ptsDtsFlags === 0b10 || ptsDtsFlags === 0b11) {
pts = 0;
bitstream.skipBits(4);
pts += bitstream.readBits(3) * (1 << 30);
bitstream.skipBits(1);
pts += bitstream.readBits(15) * (1 << 15);
bitstream.skipBits(1);
pts += bitstream.readBits(15);
} else {
if (expectPts) {
throw new Error(MISSING_PTS_ERROR_MESSAGE);
}
}
return {
sectionStartPos: section.startPos,
sectionEndPos: section.endPos,
pts,
randomAccessIndicator: section.randomAccessIndicator,
} as T extends true ? TimestampedPesPacketHeader : PesPacketHeader;
};
const readPesPacket = <T extends boolean>(
section: Section,
expectPts: T,
): (T extends true ? TimestampedPesPacket : PesPacket) | null => {
assert(section.endPos !== null); // Can only read full PES packets from fully read sections
const header = readPesPacketHeader(section, expectPts);
if (!header) {
return null;
}
const bitstream = new Bitstream(section.payload);
bitstream.skipBits(32);
const pesPacketLength = bitstream.readBits(16);
const BYTES_UNTIL_END_OF_PES_PACKET_LENGTH = 6;
bitstream.skipBits(16);
const pesHeaderDataLength = bitstream.readBits(8);
const pesHeaderEndPos = bitstream.pos + 8 * pesHeaderDataLength;
bitstream.pos = pesHeaderEndPos;
const bytePos = pesHeaderEndPos / 8;
assert(Number.isInteger(bytePos));
const data = section.payload.subarray(
bytePos,
// "A value of 0 indicates that the PES packet length is neither specified nor bounded and is allowed only in
// PES packets whose payload consists of bytes from a video elementary stream contained in
// transport stream packets."
pesPacketLength > 0
? BYTES_UNTIL_END_OF_PES_PACKET_LENGTH + pesPacketLength
: section.payload.byteLength,
);
return {
...header,
data,
} as T extends true ? TimestampedPesPacket : PesPacket;
};
abstract class MpegTsTrackBacking implements InputTrackBacking {
packetBuffers = new WeakMap<EncodedPacket, PacketBuffer>();
/** Used for recreating PacketBuffers if necessary. */
packetSectionStarts = new WeakMap<EncodedPacket, number>();
constructor(public elementaryStream: ElementaryStream) {}
abstract getType(): TrackType;
abstract getDecoderConfig(): Promise<VideoDecoderConfig | AudioDecoderConfig | null>;
getId() {
return this.elementaryStream.pid;
}
getNumber() {
const demuxer = this.elementaryStream.demuxer;
const trackType = this.elementaryStream.info.type;
let number = 0;
for (const backing of demuxer.trackBackingEntries) {
if (backing.getType() === trackType) {
number++;
}
assert(backing instanceof MpegTsTrackBacking);
if (backing.elementaryStream === this.elementaryStream) {
break;
}
}
return number;
}
getCodec(): MediaCodec | null {
throw new Error('Not implemented on base class.');
}
getInternalCodecId() {
return this.elementaryStream.streamType;
}
getName() {
return null;
}
getLanguageCode() {
return UNDETERMINED_LANGUAGE;
}
getDisposition(): TrackDisposition {
return {
...DEFAULT_TRACK_DISPOSITION,
primary: false,
};
}
getTimeResolution() {
return TIMESCALE;
}
isRelativeToUnixEpoch() {
return false;
}
getPairingMask() {
return 1n;
}
getBitrate() {
return null;
}
getAverageBitrate() {
return null;
}
async getDurationFromMetadata() {
return null;
}
async getLiveRefreshInterval() {
return null;
}
abstract allPacketsAreKeyPackets(): boolean;
abstract getReorderSize(): number;
createEncodedPacket(
suppliedPacket: SuppliedPacket,
duration: number,
options: PacketRetrievalOptions,
) {
let packetType: PacketType;
if (this.allPacketsAreKeyPackets()) {
packetType = 'key';
} else {
packetType = suppliedPacket.randomAccessIndicator === 1
? 'key'
: 'delta';
}
return new EncodedPacket(
options.metadataOnly ? PLACEHOLDER_DATA : suppliedPacket.data,
packetType,
suppliedPacket.pts / TIMESCALE,
Math.max(duration / TIMESCALE, 0),
suppliedPacket.sequenceNumber,
suppliedPacket.data.byteLength,
);
}
async getFirstPacket(options: PacketRetrievalOptions): Promise<EncodedPacket | null> {
const section = this.elementaryStream.firstSection;
assert(section);
const pesPacket = readPesPacket(section, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
const buffer = new PacketBuffer(this, context);
const result = await buffer.readNext();
if (!result) {
return null;
}
const packet = this.createEncodedPacket(result.packet, result.duration, options);
this.packetBuffers.set(packet, buffer);
this.packetSectionStarts.set(packet, result.packet.sectionStartPos);
return packet;
}
async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise<EncodedPacket | null> {
let buffer = this.packetBuffers.get(packet);
if (buffer) {
// Fast path
const result = await buffer.readNext();
if (!result) {
return null;
}
// Remove PacketBuffer access from the old packet, it belongs to the next packet now
this.packetBuffers.delete(packet);
const newPacket = this.createEncodedPacket(result.packet, result.duration, options);
this.packetBuffers.set(newPacket, buffer);
this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos);
return newPacket;
}
// No buffer, we gotta do some rereading
const sectionStartPos = this.packetSectionStarts.get(packet);
if (sectionStartPos === undefined) {
throw new Error('Packet was not created from this track.');
}
const demuxer = this.elementaryStream.demuxer;
const section = await demuxer.readSection(sectionStartPos, true);
assert(section);
const pesPacket = readPesPacket(section, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
buffer = new PacketBuffer(this, context);
// Advance until we pass the current packet's sequence number
const targetSequenceNumber = packet.sequenceNumber;
while (true) {
const result = await buffer.readNext();
if (!result) {
return null;
}
if (result.packet.sequenceNumber > targetSequenceNumber) {
// We found the next packet!
const newPacket = this.createEncodedPacket(result.packet, result.duration, options);
this.packetBuffers.set(newPacket, buffer);
this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos);
return newPacket;
}
}
}
async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise<EncodedPacket | null> {
let currentPacket: EncodedPacket | null = packet;
// Just loop until we hit one
while (true) {
currentPacket = await this.getNextPacket(currentPacket, options);
if (!currentPacket) {
return null;
}
if (currentPacket.type === 'key') {
return currentPacket;
}
}
}
getPacket(timestamp: number, options: PacketRetrievalOptions): Promise<EncodedPacket | null> {
return this.doPacketLookup(timestamp, false, options);
}
getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise<EncodedPacket | null> {
return this.doPacketLookup(timestamp, true, options);
}
/**
* Searches for the packet with the largest timestamp not larger than `timestamp` in the file, using a combination
* of chunk-based binary search and linear refinement. The reason the coarse search is done in large chunks is to
* make it more performant for small files and over high-latency readers such as the network.
*/
async doPacketLookup(
timestamp: number,
keyframesOnly: boolean,
options: PacketRetrievalOptions,
): Promise<EncodedPacket | null> {
const searchPts = roundIfAlmostInteger(timestamp * TIMESCALE);
const demuxer = this.elementaryStream.demuxer;
const { reader, seekChunkSize } = demuxer;
const pid = this.elementaryStream.pid;
const findFirstPesPacketHeaderInChunk = async (
startPos: number,
endPos: number,
readSectionInFull: boolean,
) => {
let currentPos = startPos;
while (currentPos < endPos) {
const packetHeader = await demuxer.readPacketHeader(currentPos);
if (!packetHeader) {
return null;
}
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(currentPos, readSectionInFull);
if (!section) {
return null;
}
const pesPacketHeader = readPesPacketHeader(section, false);
if (pesPacketHeader && pesPacketHeader.pts !== null) {
return {
pesPacketHeader: pesPacketHeader as TimestampedPesPacketHeader,
section,
};
}
}
currentPos += demuxer.packetStride;
}
return null;
};
// Get the first PES packet of the track
const firstSection = this.elementaryStream.firstSection;
assert(firstSection);
const firstPesPacketHeader = readPesPacketHeader(firstSection, true);
assert(firstPesPacketHeader);
if (searchPts < firstPesPacketHeader.pts) {
// We're before the first packet, definitely nothing here
return null;
}
let scanStartPos: number;
const referencePesPackets = this.elementaryStream.referencePesPackets;
const referencePointIndex = binarySearchLessOrEqual(referencePesPackets, searchPts, x => x.pts);
const referencePoint = referencePointIndex !== -1 ? referencePesPackets[referencePointIndex]! : null;
if (referencePoint && searchPts - referencePoint.pts < TIMESCALE / 2) {
// Reference point ain't too far away, prefer it over the chunk search
scanStartPos = referencePoint.sectionStartPos;
} else {
let startChunkIndex = 0;
if (reader.fileSize !== null) {
const numChunks = Math.ceil(reader.fileSize / seekChunkSize);
if (numChunks > 1) {
// Binary search to find the chunk with highest index whose first PES has pts <= searchPts
let low = 0;
let high = numChunks - 1;
startChunkIndex = low;
while (low <= high) {
const mid = Math.floor((low + high) / 2);
const chunkStartPos = floorToMultiple(mid * seekChunkSize, demuxer.packetStride)
+ firstPesPacketHeader.sectionStartPos;
const chunkEndPos = chunkStartPos + seekChunkSize;
const result = await findFirstPesPacketHeaderInChunk(chunkStartPos, chunkEndPos, false);
if (!result) {
// No PES packet found in this chunk, search left
high = mid - 1;
continue;
}
if (result.pesPacketHeader.pts <= searchPts) {
// This chunk's first PES is <= searchPts, it's a candidate
startChunkIndex = mid;
low = mid + 1; // Search right
} else {
// Search left
high = mid - 1;
}
}
}
}
scanStartPos = floorToMultiple(
startChunkIndex * seekChunkSize,
demuxer.packetStride,
) + firstPesPacketHeader.sectionStartPos;
}
// Find the first PES packet at or after scanStartPos
const result = await findFirstPesPacketHeaderInChunk(
scanStartPos,
reader.fileSize ?? Infinity,
false,
);
let currentPesHeader = result?.pesPacketHeader ?? null;
if (!currentPesHeader) {
// Fall back to first packet
currentPesHeader = firstPesPacketHeader;
}
const reorderSize = this.getReorderSize();
const retrieveEncodedPacket = async (
sectionStartPos: number,
predicate: (packet: SuppliedPacket) => boolean,
) => {
// Load the relevant section in full
const section = await demuxer.readSection(sectionStartPos, true);
assert(section);
const pesPacket = readPesPacket(section, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
const buffer = new PacketBuffer(this, context);
// Advance until the top-most presentation timestamp crosses or equals searchPts
while (true) {
const topPts = last(buffer.presentationOrderPackets)?.pts ?? -Infinity;
if (topPts >= searchPts) {
break;
}
const didRead = await buffer.readNextPacket();
if (!didRead) {
break;
}
}
const targetIndex = findLastIndex(buffer.presentationOrderPackets, predicate);
if (targetIndex === -1) {
return null;
}
const targetPacket = buffer.presentationOrderPackets[targetIndex]!;
const lastDuration = targetIndex === 0
? 0
: targetPacket.pts - buffer.presentationOrderPackets[targetIndex - 1]!.pts;
// Pop packets in decode order until we hit the target packet
while (buffer.decodeOrderPackets[0] !== targetPacket) {
buffer.decodeOrderPackets.shift();
}
buffer.lastDuration = lastDuration; // Kinda ugly but necessary fix
const result = await buffer.readNext();
assert(result);
const packet = this.createEncodedPacket(result.packet, result.duration, options);
this.packetBuffers.set(packet, buffer);
this.packetSectionStarts.set(packet, result.packet.sectionStartPos);
return packet;
};
if (!keyframesOnly || this.allPacketsAreKeyPackets()) {
// Normat packet lookup case. Slightly easier since we just need to search (mostly) forward to find the
// packet.
// Linear scan to find the PES packet with largest pts <= searchPts. This will be used as the "midpoint"
// of the next refinement step (which is needed because of B-frames).
outer:
while (true) {
let currentPos = currentPesHeader.sectionStartPos + demuxer.packetStride;
while (true) {
const packetHeader = await demuxer.readPacketHeader(currentPos);
if (!packetHeader) {
break outer; // End of file
}
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(currentPos, false);
if (section) {
const nextPesHeader = readPesPacketHeader(section, false);
if (nextPesHeader && nextPesHeader.pts !== null) {
if (nextPesHeader.pts > searchPts) {
break outer;
}
currentPesHeader = nextPesHeader as TimestampedPesPacketHeader;
maybeInsertReferencePacket(this.elementaryStream, currentPesHeader);
break;
}
}
}
currentPos += demuxer.packetStride;
}
}
// Rewind by reorderSize + 1 PES packets (even for audio! To ensure proper durations)
outer:
for (let i = 0; i < reorderSize + 1; i++) {
let pos = currentPesHeader.sectionStartPos - demuxer.packetStride;
while (pos >= demuxer.packetOffset) {
const packetHeader = await demuxer.readPacketHeader(pos);
if (!packetHeader) {
break outer;
}
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(pos, false);
if (section) {
const header = readPesPacketHeader(section, false);
if (header && header.pts !== null) {
currentPesHeader = header as TimestampedPesPacketHeader;
break;
}
}
}
pos -= demuxer.packetStride;
}
}
return retrieveEncodedPacket(currentPesHeader.sectionStartPos, p => p.pts <= searchPts);
} else {
// Key packet lookup case. Slightly harder since the starting chunk may not have a key packet at all, which
// means we might need to search the previous chunks until we find something.
let currentChunkStartPos = scanStartPos;
let nextChunkStartPos: number | null = null; // "next" as in later in the file, even tho we scan backwards
const readSectionsInFull = !this.elementaryStream.canBeTrustedWithKeyPackets;
while (true) {
let bestKeyPesHeader: TimestampedPesPacketHeader | null = null;
const isFirstChunk = currentChunkStartPos <= firstPesPacketHeader.sectionStartPos;
let pesHeader: TimestampedPesPacketHeader | null;
let pesHeaderSection: Section | null = null;
if (isFirstChunk) {
pesHeader = firstPesPacketHeader;
pesHeaderSection = firstSection;
} else {
const result = await findFirstPesPacketHeaderInChunk(
currentChunkStartPos,
reader.fileSize ?? Infinity,
readSectionsInFull,
);
pesHeader = result?.pesPacketHeader ?? null;
pesHeaderSection = result?.section ?? null;
}
let passedSearchPts = false;
let lookaheadCount = 0;
outer:
while (pesHeader) {
if (nextChunkStartPos !== null && pesHeader.sectionStartPos >= nextChunkStartPos) {
// Stop at the next chunk boundary
break;
}
if (pesHeader.pts <= searchPts) {
let isKeyPacket: boolean;
if (this.elementaryStream.canBeTrustedWithKeyPackets) {
isKeyPacket = pesHeader.randomAccessIndicator === 1;
} else {
assert(pesHeaderSection);
const pesPacket = readPesPacket(pesHeaderSection, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
await context.markNextPacket();
isKeyPacket = context.suppliedPacket?.randomAccessIndicator === 1;
}
if (isKeyPacket) {
bestKeyPesHeader = pesHeader;
}
}
if (pesHeader.pts > searchPts) {
passedSearchPts = true;
}
// If we've passed searchPts, do lookahead for reorderSize more packets just to be sure
if (passedSearchPts) {
lookaheadCount++;
if (lookaheadCount > reorderSize) {
break;
}
}
// Find next PES packet
let currentPos = pesHeader.sectionStartPos + demuxer.packetStride;
while (true) {
const packetHeader = await demuxer.readPacketHeader(currentPos);
if (!packetHeader) {
break outer; // End of file
}
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(currentPos, readSectionsInFull);
if (section) {
const nextPesHeader = readPesPacketHeader(section, false);
if (nextPesHeader && nextPesHeader.pts !== null) {
pesHeader = nextPesHeader as TimestampedPesPacketHeader;
pesHeaderSection = section;
maybeInsertReferencePacket(this.elementaryStream, pesHeader);
break;
}
}
}
currentPos += demuxer.packetStride;
}
}
if (bestKeyPesHeader) {
let startPesHeader = bestKeyPesHeader;
if (lookaheadCount === 0) {
// Packet is at the end of stream, let's rewind a little to obtain the correct packet duration
outer:
for (let i = 0; i < reorderSize; i++) {
let pos = startPesHeader.sectionStartPos - demuxer.packetStride;
while (pos >= demuxer.packetOffset) {
const packetHeader = await demuxer.readPacketHeader(pos);
if (!packetHeader) {
break outer;
}
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(pos, readSectionsInFull);
if (section) {
const header = readPesPacketHeader(section, false);
if (header && header.pts !== null) {
startPesHeader = header as TimestampedPesPacketHeader;
break;
}
}
}
pos -= demuxer.packetStride;
}
}
}
const encodedPacket = await retrieveEncodedPacket(
startPesHeader.sectionStartPos,
p => p.p