@yume-chan/scrcpy-decoder-webcodecs/src/video/codec/h26x.ts

Raw H.264 stream decoder and renderer using WebCodecs API (requires modern browser).

import type {
    ScrcpyMediaStreamDataPacket,
    ScrcpyMediaStreamPacket,
} from "@yume-chan/scrcpy";

import type { CodecDecoder } from "./type.js";

export abstract class H26xDecoder implements CodecDecoder {
    #decoder: VideoDecoder;

    #config: (VideoDecoderConfig & { raw: Uint8Array }) | undefined;
    #configured = false;

    constructor(decoder: VideoDecoder) {
        this.#decoder = decoder;
    }

    abstract configure(data: Uint8Array): VideoDecoderConfig;

    #configureAndDecodeFirstKeyframe(
        config: VideoDecoderConfig & { raw: Uint8Array },
        packet: ScrcpyMediaStreamDataPacket,
    ) {
        this.#decoder.configure(config);
        this.#configured = true;

        // For H.264 and H.265, when the stream is in Annex B format
        // (which Scrcpy uses, as Android MediaCodec produces),
        // configuration data needs to be combined with the first frame data.
        // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
        const { raw } = config;
        const data = new Uint8Array(raw.length + packet.data.length);
        data.set(raw, 0);
        data.set(packet.data, raw.length);

        this.#decoder.decode(
            new EncodedVideoChunk({
                type: "key",
                timestamp: 0,
                data,
            }),
        );
    }

    decode(packet: ScrcpyMediaStreamPacket): void {
        if (packet.type === "configuration") {
            this.#config = {
                ...this.configure(packet.data),
                raw: packet.data,
            };
            this.#configured = false;
            return;
        }

        if (!this.#config) {
            throw new Error("Decoder not configured");
        }

        if (packet.keyframe) {
            if (this.#decoder.decodeQueueSize) {
                // If the device is too slow to decode all frames,
                // discard queued frames when next keyframe arrives.
                // (can only do this for keyframes because decoding must start from a keyframe)
                // This limits the maximum latency to 1 keyframe interval
                // (60 frames by default).
                this.#decoder.reset();

                // `reset` also resets the decoder configuration
                // so we need to re-configure it again.
                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
                return;
            }

            if (!this.#configured) {
                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
                return;
            }
        }

        if (!this.#configured) {
            if (packet.keyframe === undefined) {
                // Scrcpy <1.23 doesn't send `keyframe` flag
                // Infer the first frame after configuration as keyframe
                // (`VideoDecoder` will throw error if it's not)
                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
                return;
            }

            throw new Error("Expect a keyframe but got a delta frame");
        }

        this.#decoder.decode(
            new EncodedVideoChunk({
                // Treat `undefined` as `key`, otherwise won't decode.
                type: packet.keyframe === false ? "delta" : "key",
                timestamp: 0,
                data: packet.data,
            }),
        );
    }
}