@needle-tools/engine/src/engine-components/webxr/WebARCameraBackground.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import {
    DoubleSide,
    Mesh, MeshBasicMaterial,
    PerspectiveCamera,
    PlaneGeometry,
    Scene,
    ShaderLib,
    ShaderMaterial,
    Texture,
    UniformsUtils,
    Vector4,
} from "three";

import { serializable } from "../../engine/engine_serialization_decorator.js";
import { getParam } from "../../engine/engine_utils.js";
import { InternalScreenshotUtils } from "../../engine/engine_utils_screenshot.js";
import { updateTextureFromXRFrame } from "../../engine/engine_utils_screenshot.xr.js";
import type { NeedleXREventArgs } from "../../engine/engine_xr.js";
import { RGBAColor } from "../../engine/js-extensions/index.js"
import { Behaviour } from "../Component.js";

const debug = getParam("debugarcamera");

/**
 * WebARCameraBackground is a component that allows to display the camera feed as a background in an AR session to more easily blend the real world with the virtual world or applying effects to the camera feed.
 */
export class WebARCameraBackground extends Behaviour {

    /** @internal */
    onBeforeXR(_mode: XRSessionMode, args: XRSessionInit): void {
        if (_mode === "immersive-ar") {
            args.optionalFeatures = args.optionalFeatures || [];
            args.optionalFeatures.push('camera-access');
            if (debug) console.warn("Requesting camera-access");
        }
    }

    /** @internal */
    onEnterXR(_args: NeedleXREventArgs): void {
        if (_args.xr.mode === "immersive-ar") {
            if (this.backgroundPlane) {
                this.context.scene.add(this.backgroundPlane);
                this.backgroundPlane.visible = false;
            }

            if (this.backgroundPlane) this.context.scene.add(this.backgroundPlane);
            this.context.pre_render_callbacks.push(this.preRender);
        }
    }

    /** @internal */
    onLeaveXR(_args: NeedleXREventArgs): void {
        if (this.backgroundPlane) this.backgroundPlane.removeFromParent();
        const i = this.context.pre_render_callbacks.indexOf(this.preRender);
        if (i >= 0)
            this.context.pre_render_callbacks.splice(i, 1);
    }

    /**
     * The tint color of the camera feed
     */
    @serializable(RGBAColor)
    public backgroundTint: RGBAColor = new RGBAColor(1, 1, 1, 1);

    public get background() {
        return this.backgroundPlane;
    }

    private backgroundPlane?: Mesh;
    private threeTexture?: Texture;
    private forceTextureInitialization = function () {
        const material = new MeshBasicMaterial();
        const geometry = new PlaneGeometry();
        const scene = new Scene();
        scene.add(new Mesh(geometry, material));
        const camera = new PerspectiveCamera();

        return function forceTextureInitialization(renderer, texture) {
            material.map = texture;
            renderer.render(scene, camera);
            if (debug) console.warn("Force texture initialization");
        };
    }();



    /** @internal */
    private preRender = () => {
        if (!this || !this.gameObject) return;

        const xr = this.context.renderer.xr;
        const frame = xr.getFrame();

        if (frame) {

            // We're generating a new texture here, and force three to initialize it
            // from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
            if (!this.threeTexture && this.context.renderer) {
                this.threeTexture = new Texture();
                this.forceTextureInitialization(this.context.renderer, this.threeTexture);
            }

            // simple mesh and fullscreen shader to display the camera texture
            // from three: WebGLBackground
            if (this.backgroundPlane === undefined) {
                const tint = this.backgroundTint;
                this.backgroundPlane = InternalScreenshotUtils.makeFullscreenPlane({
                    material: new ShaderMaterial({
                        name: 'BackgroundMaterial',
                        uniforms: {
                            ...UniformsUtils.clone(ShaderLib.background.uniforms),
                            tint: { value: new Vector4(tint.r, tint.g, tint.b, tint.a) },
                        },
                        vertexShader: ShaderLib.background.vertexShader,
                        fragmentShader: backgroundFragment,
                        side: DoubleSide,
                        depthTest: false,
                        depthWrite: false,
                        fog: false
                    })
                });
            }
            if (this.backgroundPlane.parent !== this.scene)
                this.scene.add(this.backgroundPlane);

            if (this.backgroundPlane.material instanceof ShaderMaterial)
                this.backgroundPlane.material.uniforms.tint.value.set(this.backgroundTint.r, this.backgroundTint.g, this.backgroundTint.b, this.backgroundTint.a);

            // WebXR Raw Camera Access - 
            // we composite the camera texture into the scene background by rendering it first.
            this.updateFromFrame();
        }
    }

    /** @internal */
    onBeforeRender(_frame: XRFrame | null) {
        this.updateFromFrame();
    }

    private updateFromFrame() {
        if (!this.threeTexture) return;
        if (this.context.xr?.mode === "immersive-ar") {
            updateTextureFromXRFrame(this.context.renderer, this.threeTexture);
            this.setTexture(this.threeTexture);
        }
    }

    setTexture(texture: Texture) {
        if (!this.backgroundPlane) return;
        this.threeTexture = texture;
        //@ts-ignore
        this.backgroundPlane.setTexture(this.threeTexture);
        this.backgroundPlane.visible = true;
    }
}

const backgroundFragment: string = /* glsl */`
uniform sampler2D t2D;
uniform vec4 tint;

varying vec2 vUv;

void main() {

    vec4 texColor = texture2D( t2D, vUv );
    texColor.w = 1.0;

    // inline sRGB decode
    texColor = vec4( mix( pow( texColor.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), texColor.rgb * 0.0773993808, vec3( lessThanEqual( texColor.rgb, vec3( 0.04045 ) ) ) ), texColor.w );

    gl_FragColor = texColor * tint;

    #include <tonemapping_fragment>
    #include <colorspace_fragment>
}
`;