react-native-vision-camera/lib/hooks/useFrameOutput.js

VisionCamera is the fastest and most powerful Camera for react-native.

import { useEffect, useMemo, useRef } from 'react';
import { VisionCameraWorkletsProxy } from '../third-party/VisionCameraWorkletsProxy';
import { CommonResolutions } from '../utils/CommonResolutions';
import { VisionCamera } from '../VisionCamera';
/**
 * Use a {@linkcode CameraFrameOutput}.
 *
 * The {@linkcode UseFrameOutputProps.onFrame | onFrame(...)} callback
 * will be called for every {@linkcode Frame} the Camera
 * sees. It is a synchronous JS function running on the
 * {@linkcode CameraFrameOutput}'s {@linkcode CameraFrameOutput.thread | thread} -
 * aka a "worklet".
 *
 * @note {@linkcode useFrameOutput | useFrameOutput(...)} requires
 * `react-native-vision-camera-worklets` (and
 * [react-native-worklets](https://docs.swmansion.com/react-native-worklets/docs/))
 * to be installed.
 *
 * @discussion
 * You must {@linkcode Frame.dispose | dispose} the {@linkcode Frame} after your
 * Frame Processor has finished processing, otherwise subsequent {@linkcode Frame}s
 * may be dropped (see {@linkcode UseFrameOutputProps.onFrameDropped | onFrameDropped(...)}).
 *
 * @discussion
 * Choosing an appropriate {@linkcode FrameOutputOptions.pixelFormat | pixelFormat}
 * depends on your Frame Processor's usage. While the most commonly used
 * format in visual recognition models is {@linkcode VideoPixelFormat | 'rgb'},
 * it is by far not the most efficient format for a Camera pipeline as it
 * requires an additional conversion and uses ~2.6x more bandwidth than
 * {@linkcode VideoPixelFormat | 'yuv'}.
 *
 * If you render to native Surfaces (e.g. via GPU pipelines or Media Encoders),
 * you may also be able to use {@linkcode TargetVideoPixelFormat | 'native'},
 * which chooses whatever the resolved {@linkcode CameraSessionConfig}'s
 * {@linkcode CameraSessionConfig.nativePixelFormat | nativePixelFormat} is, and
 * requires zero conversions.
 * Use {@linkcode TargetVideoPixelFormat | 'native'} with caution, as the
 * negotiated {@linkcode CameraSessionConfig.nativePixelFormat | nativePixelFormat}
 * might also be a RAW format like {@linkcode VideoPixelFormat | 'raw-bayer-packed96-12-bit'},
 * or a vendor-specific private format ({@linkcode PixelFormat | 'private'}).
 *
 * Examples:
 * - [MLKit](https://developers.google.com/ml-kit) natively supports YUV,
 * so streaming in {@linkcode TargetVideoPixelFormat | 'yuv'} is most efficient.
 * - [OpenCV](https://opencv.org) natively supports YUV, so streaming in
 * {@linkcode TargetVideoPixelFormat | 'yuv'} is most efficient.
 * - [LiteRT](https://ai.google.dev/edge/litert/overview) supports YUV, **but
 * converts to RGB internally** - so streaming in {@linkcode TargetVideoPixelFormat | 'rgb'}
 * directly is more efficient as conversion is handled in the Camera pipeline.
 * - [react-native-skia](https://shopify.github.io/react-native-skia/) supports
 * both YUV and RGB, so streaming in {@linkcode TargetVideoPixelFormat | 'yuv'}
 * is more efficient.
 *
 * @see {@linkcode TargetVideoPixelFormat}
 * @see {@linkcode VideoPixelFormat}
 *
 * @example
 * ```ts
 * const frameOutput = useFrameOutput({
 *   onFrame(frame) {
 *     'worklet'
 *     // some frame processing
 *     frame.dispose()
 *   }
 * })
 * ```
 */
export function useFrameOutput({ targetResolution = CommonResolutions.HD_16_9, pixelFormat = 'native', dropFramesWhileBusy = true, enableCameraMatrixDelivery = false, enablePhysicalBufferRotation = false, enablePreviewSizedOutputBuffers = false, allowDeferredStart = true, onFrame, onFrameDropped, }) {
    // 1. Create frame output
    const frameOutput = useMemo(() => VisionCamera.createFrameOutput({
        targetResolution: targetResolution,
        pixelFormat: pixelFormat,
        enablePhysicalBufferRotation: enablePhysicalBufferRotation,
        enableCameraMatrixDelivery: enableCameraMatrixDelivery,
        enablePreviewSizedOutputBuffers: enablePreviewSizedOutputBuffers,
        allowDeferredStart: allowDeferredStart,
        dropFramesWhileBusy: dropFramesWhileBusy,
    }), [
        targetResolution,
        pixelFormat,
        dropFramesWhileBusy,
        enableCameraMatrixDelivery,
        enablePhysicalBufferRotation,
        enablePreviewSizedOutputBuffers,
        allowDeferredStart,
    ]);
    // 2. Add Frame dropped warner
    const onFrameDroppedRef = useRef(onFrameDropped);
    onFrameDroppedRef.current = onFrameDropped;
    useEffect(() => {
        frameOutput.setOnFrameDroppedCallback((reason) => {
            const callback = onFrameDroppedRef.current;
            if (callback != null)
                callback(reason);
            else
                console.warn(`Frame Dropped! Reason: ${reason}`);
        });
    }, [frameOutput]);
    // 3. Create Worklet Runtime for NativeThread
    const runtime = useMemo(() => VisionCameraWorkletsProxy.createRuntimeForThread(frameOutput.thread), [frameOutput.thread]);
    // 4. Update onFrame() callback if it changed
    useEffect(() => {
        runtime.setOnFrameCallback(frameOutput, onFrame);
    }, [runtime, frameOutput, onFrame]);
    // 5. Return :)
    return frameOutput;
}