@needle-tools/engine/src/engine-components/ContactShadows.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import { BackSide, CustomBlending, DoubleSide, FrontSide, Group, Material, Matrix4, MaxEquation, Mesh, MeshBasicMaterial, MeshDepthMaterial, MeshStandardMaterial, MinEquation, Object3D, OrthographicCamera, PlaneGeometry, RenderItem, ShaderMaterial, Vector3, WebGLRenderTarget } from "three";
import { HorizontalBlurShader } from 'three/examples/jsm/shaders/HorizontalBlurShader.js';
import { VerticalBlurShader } from 'three/examples/jsm/shaders/VerticalBlurShader.js';

import { setAutoFitEnabled } from "../engine/engine_camera.js";
import { addComponent } from "../engine/engine_components.js";
import { Context } from "../engine/engine_context.js";
import { Gizmos } from "../engine/engine_gizmos.js";
import { onStart } from "../engine/engine_lifecycle_api.js";
import { serializable } from "../engine/engine_serialization_decorator.js";
import { getBoundingBox, getVisibleInCustomShadowRendering } from "../engine/engine_three_utils.js";
import { HideFlags, IGameObject, Vec3 } from "../engine/engine_types.js";
import { getParam } from "../engine/engine_utils.js"
import { setCustomVisibility } from "../engine/js-extensions/Layers.js";
import { Behaviour, GameObject } from "./Component.js";

const debug = getParam("debugcontactshadows");

onStart(ctx => {
    const val = ctx.domElement.getAttribute("contactshadows") || ctx.domElement.getAttribute("contact-shadows");
    if (val != undefined && val != "0" && val != "false") {
        console.debug("Auto-creating ContactShadows because of `contactshadows` attribute");
        const shadows = ContactShadows.auto(ctx);
        const intensity = parseFloat(val);
        if (!isNaN(intensity)) {
            shadows.opacity = intensity;
            shadows.darkness = intensity;
        }
    }
})

// Adapted from https://github.com/mrdoob/three.js/blob/master/examples/webgl_shadow_contact.html.

// Improved with
// - ground occluder
// - backface shadowing (slightly less than front faces)
// - node can simply be scaled in Y to adjust max. ground height

/**
 * ContactShadows is a component that allows to display contact shadows in the scene.
 * @category Rendering
 * @group Components
 */
export class ContactShadows extends Behaviour {

    private static readonly _instances: Map<Context, ContactShadows> = new Map();
    /**
     * Create contact shadows for the scene. Automatically fits the shadows to the scene.  
     * The instance of contact shadows will be created only once.  
     * @param context The context to create the contact shadows in.
     * @returns The instance of the contact shadows.
     */
    static auto(context?: Context): ContactShadows {
        if (!context) context = Context.Current;
        if (!context) {
            throw new Error("No context provided and no current context set.");
        }
        let instance = this._instances.get(context);
        if (!instance || instance.destroyed) {
            const obj = new Object3D();
            instance = addComponent(obj, ContactShadows, {
                autoFit: false,
                occludeBelowGround: false
            });
            this._instances.set(context, instance);
        }
        context.scene.add(instance.gameObject);
        instance.fitShadows();
        return instance;
    }

    /**
     * When enabled the contact shadows component will be created to fit the whole scene.
     */
    @serializable()
    autoFit: boolean = false;
    /**
     * Darkness of the shadows. 
     * @default 0.5
     */
    @serializable()
    darkness: number = 0.5;
    /**
     * Opacity of the shadows. 
     * @default 0.5
     */
    @serializable()
    opacity: number = 0.5;
    /**
     * Blur of the shadows. 
     * @default 4.0
     */
    @serializable()
    blur: number = 4.0;
    /**
     * When enabled objects will not be visible below the shadow plane
     * @default false
     */
    @serializable()
    occludeBelowGround: boolean = false;
    /**
     * When enabled the backfaces of objects will cast shadows as well.
     * @default true
     */
    @serializable()
    backfaceShadows: boolean = true;

    /**
     * The minimum size of the shadows box
     */
    minSize?: Partial<Vec3>;

    /**
     * When enabled the shadows will not be updated automatically. Use `needsUpdate()` to update the shadows manually.
     * This is useful when you want to update the shadows only when the scene changes.
     */
    manualUpdate: boolean = false;
    /**
     * Call this method to update the shadows manually. The update will be done in the next frame.
     */
    set needsUpdate(val: boolean) {
        this._needsUpdate = val;
    }
    get needsUpdate(): boolean {
        return this._needsUpdate;
    }
    private _needsUpdate: boolean = false;

    /** All shadow objects are parented to this object. 
     * The gameObject itself should not be transformed because we want the ContactShadows object e.g. also have a GroundProjectedEnv component
     * in which case ContactShadows scale would affect the projection
     **/
    private readonly shadowsRoot: IGameObject = new Object3D() as IGameObject;
    private shadowCamera?: OrthographicCamera;
    private readonly shadowGroup: Group = new Group();

    private renderTarget?: WebGLRenderTarget;
    private renderTargetBlur?: WebGLRenderTarget;

    private plane?: Mesh;
    private occluderMesh?: Mesh;
    private blurPlane?: Mesh;

    private depthMaterial?: MeshDepthMaterial;
    private horizontalBlurMaterial?: ShaderMaterial;
    private verticalBlurMaterial?: ShaderMaterial;

    private textureSize = 512;

    /**
     * Call to fit the shadows to the scene.
     */
    fitShadows() {
        if (debug) console.warn("Fitting shadows to scene");
        setAutoFitEnabled(this.shadowsRoot, false);
        const box = getBoundingBox(this.context.scene.children, [this.shadowsRoot]);
        // expand box in all directions (except below ground)
        // 0.75 expands by 75% in each direction
        // The "32" is pretty much heuristically determined – adjusting the value until we don't get a visible border anymore.
        const expandFactor = Math.max(1, this.blur / 32);
        const sizeX = box.max.x - box.min.x;
        const sizeZ = box.max.z - box.min.z;
        box.expandByVector(new Vector3(expandFactor * sizeX, 0, expandFactor * sizeZ));
        if (debug) Gizmos.DrawWireBox3(box, 0xffff00, 60);
        if (this.gameObject.parent) {
            // transform box from world space into parent space
            box.applyMatrix4((this.gameObject.parent as GameObject).matrixWorld.clone().invert());
        }
        const min = box.min;
        const offset = Math.max(0.00001, (box.max.y - min.y) * .002);
        box.max.y += offset;
        // This is for cases where GroundProjection with autoFit is used
        // Since contact shadows can currently not ignore certain objects from rendering
        // we need to make sure the GroundProjection is not exactly on the same level as ContactShadows
        // We can't move GroundProjection down because of immersive-ar mesh/plane tracking where occlusion would otherwise hide GroundProjection
        this.shadowsRoot.position.set((min.x + box.max.x) / 2, min.y - offset, (min.z + box.max.z) / 2);
        this.shadowsRoot.scale.set(box.max.x - min.x, box.max.y - min.y, box.max.z - min.z);
        this.applyMinSize();
        this.shadowsRoot.matrixWorldNeedsUpdate = true;
        if (debug) console.log("Fitted shadows to scene", this.shadowsRoot.scale.clone());
    }


    /** @internal */
    awake() {
        ContactShadows._instances.set(this.context, this);
        this.shadowsRoot.hideFlags = HideFlags.DontExport;

        // ignore self for autofitting
        setAutoFitEnabled(this.shadowsRoot, false);
    }


    /** @internal */
    start(): void {
        if (debug) console.log("Create ContactShadows on " + this.gameObject.name, this)

        this.gameObject.add(this.shadowsRoot);
        this.shadowsRoot.add(this.shadowGroup);

        // the render target that will show the shadows in the plane texture
        this.renderTarget = new WebGLRenderTarget(this.textureSize, this.textureSize);
        this.renderTarget.texture.generateMipmaps = false;

        // the render target that we will use to blur the first render target
        this.renderTargetBlur = new WebGLRenderTarget(this.textureSize, this.textureSize);
        this.renderTargetBlur.texture.generateMipmaps = false;

        // make a plane and make it face up
        const planeGeometry = new PlaneGeometry(1, 1).rotateX(Math.PI / 2);

        if (this.gameObject instanceof Mesh) {
            console.warn("ContactShadows can not be added to a Mesh. Please add it to a Group or an empty Object");
            // this.enabled = false;
            setCustomVisibility(this.gameObject, false);
            // this.plane = this.gameObject as any as Mesh;
            // // Make sure we clone the material once because it might be used on another object as well
            // const mat = this.plane.material = (this.plane.material as MeshBasicMaterial).clone();
            // mat.map = this.renderTarget.texture;
            // mat.opacity = this.opacity;
            // mat.transparent = true;
            // mat.depthWrite = false;
            // mat.needsUpdate = true;
            // When someone makes a custom mesh, they can set these values right on the material.
            // mat.opacity = this.state.plane.opacity;
            // mat.transparent = true;
            // mat.depthWrite = false;
        }

        const planeMaterial = new MeshBasicMaterial({
            map: this.renderTarget.texture,
            opacity: this.opacity,
            color: 0x000000,
            transparent: true,
            depthWrite: false,
            side: FrontSide,
        });
        this.plane = new Mesh(planeGeometry, planeMaterial);
        this.plane.scale.y = - 1;
        this.plane.layers.set(2);
        this.shadowsRoot.add(this.plane);

        if (this.plane) this.plane.renderOrder = 1;

        this.occluderMesh = new Mesh(this.plane.geometry, new MeshBasicMaterial({
            depthWrite: true,
            stencilWrite: true,
            colorWrite: false,
            side: BackSide,
        }))
            // .rotateX(Math.PI)
            .translateY(-0.0001);
        this.occluderMesh.renderOrder = -100;
        this.occluderMesh.layers.set(2);
        this.shadowsRoot.add(this.occluderMesh);

        // the plane onto which to blur the texture
        this.blurPlane = new Mesh(planeGeometry);
        this.blurPlane.visible = false;
        this.shadowGroup.add(this.blurPlane);

        // max. ground distance is controlled via object scale
        const near = 0;
        const far = 1.0;
        this.shadowCamera = new OrthographicCamera(-1 / 2, 1 / 2, 1 / 2, -1 / 2, near, far);
        this.shadowCamera.layers.enableAll();
        this.shadowCamera.rotation.x = Math.PI / 2; // get the camera to look up
        this.shadowGroup.add(this.shadowCamera);

        // like MeshDepthMaterial, but goes from black to transparent
        this.depthMaterial = new MeshDepthMaterial();
        this.depthMaterial.userData.darkness = { value: this.darkness };
        // this will properly overlap calculated shadows
        this.depthMaterial.blending = CustomBlending;
        this.depthMaterial.blendEquation = MaxEquation;

        // this.depthMaterial.blendEquation = MinEquation;
        this.depthMaterial.onBeforeCompile = shader => {
            if (!this.depthMaterial) return;
            shader.uniforms.darkness = this.depthMaterial.userData.darkness;
            shader.fragmentShader = /* glsl */`
                uniform float darkness;
                ${shader.fragmentShader.replace(
                'gl_FragColor = vec4( vec3( 1.0 - fragCoordZ ), opacity );',
                // we're scaling the shadow value down a bit when it's a backface (looks better)
                'gl_FragColor = vec4( vec3( 1.0 ), ( 1.0 - fragCoordZ ) * darkness * opacity * (gl_FrontFacing ? 1.0 : 0.66) );'
            )}
            `;
        };

        this.depthMaterial.depthTest = false;
        this.depthMaterial.depthWrite = false;

        this.horizontalBlurMaterial = new ShaderMaterial(HorizontalBlurShader);
        this.horizontalBlurMaterial.depthTest = false;

        this.verticalBlurMaterial = new ShaderMaterial(VerticalBlurShader);
        this.verticalBlurMaterial.depthTest = false;

        this.shadowGroup.visible = false;

        if (this.autoFit) this.fitShadows();
        else this.applyMinSize();
    }

    onEnable(): void {
        this._needsUpdate = true;
    }

    /** @internal */
    onDestroy(): void {
        const instance = ContactShadows._instances.get(this.context);
        if (instance === this) {
            ContactShadows._instances.delete(this.context);
        }

        // dispose the render targets
        this.renderTarget?.dispose();
        this.renderTargetBlur?.dispose();

        // dispose the materials
        this.depthMaterial?.dispose();
        this.horizontalBlurMaterial?.dispose();
        this.verticalBlurMaterial?.dispose();

        // dispose the geometries
        this.blurPlane?.geometry.dispose();
        this.plane?.geometry.dispose();
        this.occluderMesh?.geometry.dispose();
    }

    /** @internal */
    onBeforeRender(_frame: XRFrame | null): void {

        if (this.manualUpdate) {
            if (!this._needsUpdate) return;
        }
        this._needsUpdate = false;

        if (!this.renderTarget || !this.renderTargetBlur ||
            !this.depthMaterial || !this.shadowCamera ||
            !this.blurPlane || !this.shadowGroup || !this.plane ||
            !this.horizontalBlurMaterial || !this.verticalBlurMaterial) {
            if (debug)
                console.error("ContactShadows: not initialized yet");
            return;
        }

        const scene = this.context.scene;
        const renderer = this.context.renderer;
        const initialRenderTarget = renderer.getRenderTarget();

        // Idea: shear the shadowCamera matrix to add some light direction to the ground shadows
        /*
        const mat = this.shadowCamera.projectionMatrix.clone();
        this.shadowCamera.projectionMatrix.multiply(new Matrix4().makeShear(0, 0, 0, 0, 0, 0));
        */

        this.shadowGroup.visible = true;
        if (this.occluderMesh) this.occluderMesh.visible = false;
        const planeWasVisible = this.plane.visible;
        this.plane.visible = false;

        if (this.gameObject instanceof Mesh) {
            // this.gameObject.visible = false;
            setCustomVisibility(this.gameObject, false);
        }

        // remove the background
        const initialBackground = scene.background;
        scene.background = null;

        // force the depthMaterial to everything
        scene.overrideMaterial = this.depthMaterial;
        if (this.backfaceShadows)
            this.depthMaterial.side = DoubleSide;
        else {
            this.depthMaterial.side = FrontSide;
        }

        // set renderer clear alpha
        const initialClearAlpha = renderer.getClearAlpha();
        renderer.setClearAlpha(0);

        const prevXRState = renderer.xr.enabled;
        renderer.xr.enabled = false;

        const prevSceneMatrixAutoUpdate = this.context.scene.matrixWorldAutoUpdate;
        this.context.scene.matrixWorldAutoUpdate = false;

        const list = renderer.renderLists.get(scene, 0);
        const prevTransparent = list.transparent;
        empty_buffer.length = 0;
        list.transparent = empty_buffer;

        // we need to hide objects that don't render color or that are wireframes
        objects_hidden.length = 0;
        for (const entry of list.opaque) {
            if (!entry.object.visible) continue;
            const mat = entry.material as MeshStandardMaterial;
            // Ignore objects that don't render color
            let hide = entry.material.colorWrite == false || mat.wireframe === true || getVisibleInCustomShadowRendering(entry.object) === false;
            // Ignore line materials (e.g. GridHelper)
            if (!hide && (entry.material["isLineMaterial"]))
                hide = true;
            // Ignore point materials
            if (!hide && (entry.material["isPointsMaterial"]))
                hide = true;
            if (hide) {
                objects_hidden.push(entry.object);
                entry.object["needle:visible"] = entry.object.visible;
                entry.object.visible = false;
            }
        }

        // render to the render target to get the depths
        renderer.setRenderTarget(this.renderTarget);
        renderer.clear();
        renderer.render(scene, this.shadowCamera);
        list.transparent = prevTransparent;

        // reset previously hidden objects
        for (const object of objects_hidden) {
            if (object["needle:visible"] != undefined) {
                object.visible = object["needle:visible"];
            }
        }

        // for the shearing idea
        // this.shadowCamera.projectionMatrix.copy(mat);

        // and reset the override material
        scene.overrideMaterial = null;

        const blurAmount = Math.max(this.blur, 0.05);

        // two-pass blur to reduce the artifacts
        this.blurShadow(blurAmount * 2);
        this.blurShadow(blurAmount * 0.5);

        this.shadowGroup.visible = false;
        if (this.occluderMesh) this.occluderMesh.visible = this.occludeBelowGround;
        this.plane.visible = planeWasVisible;

        // reset and render the normal scene
        renderer.setRenderTarget(initialRenderTarget);
        renderer.setClearAlpha(initialClearAlpha);
        scene.background = initialBackground;
        renderer.xr.enabled = prevXRState;
        this.context.scene.matrixWorldAutoUpdate = prevSceneMatrixAutoUpdate;
    }

    // renderTarget --> blurPlane (horizontalBlur) --> renderTargetBlur --> blurPlane (verticalBlur) --> renderTarget
    private blurShadow(amount: number) {
        if (!this.blurPlane || !this.shadowCamera ||
            !this.renderTarget || !this.renderTargetBlur ||
            !this.horizontalBlurMaterial || !this.verticalBlurMaterial)
            return;

        this.blurPlane.visible = true;

        // Correct for contact shadow plane aspect ratio.
        // since we have a separable blur, we can just adjust the blur amount for X and Z individually
        const ws = this.shadowsRoot.worldScale;
        const avg = (ws.x + ws.z) / 2;
        const aspectX = ws.z / avg;
        const aspectZ = ws.x / avg;

        // blur horizontally and draw in the renderTargetBlur
        this.blurPlane.material = this.horizontalBlurMaterial;
        (this.blurPlane.material as ShaderMaterial).uniforms.tDiffuse.value = this.renderTarget.texture;
        this.horizontalBlurMaterial.uniforms.h.value = amount * 1 / this.textureSize * aspectX;

        const renderer = this.context.renderer;

        const currentRt = renderer.getRenderTarget();
        renderer.setRenderTarget(this.renderTargetBlur);
        renderer.render(this.blurPlane, this.shadowCamera);

        // blur vertically and draw in the main renderTarget
        this.blurPlane.material = this.verticalBlurMaterial;
        (this.blurPlane.material as ShaderMaterial).uniforms.tDiffuse.value = this.renderTargetBlur.texture;
        this.verticalBlurMaterial.uniforms.v.value = amount * 1 / this.textureSize * aspectZ;

        renderer.setRenderTarget(this.renderTarget);
        renderer.render(this.blurPlane, this.shadowCamera);

        this.blurPlane.visible = false;

        renderer.setRenderTarget(currentRt);
    }

    private applyMinSize() {
        if (this.minSize) {
            this.shadowsRoot.scale.set(
                Math.max(this.minSize.x || 0, this.shadowsRoot.scale.x),
                Math.max(this.minSize.y || 0, this.shadowsRoot.scale.y),
                Math.max(this.minSize.z || 0, this.shadowsRoot.scale.z)
            );
        }
    }
}

const empty_buffer = [];
const objects_hidden = new Array<Object3D>();