@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
171 lines (165 loc) • 7.52 kB
JavaScript
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
import { DoubleSide, Mesh, MeshBasicMaterial, PerspectiveCamera, PlaneGeometry, Scene, ShaderLib, ShaderMaterial, Texture, UniformsUtils, Vector4, } from "three";
import { serializable } from "../../engine/engine_serialization_decorator.js";
import { getParam } from "../../engine/engine_utils.js";
import { InternalScreenshotUtils } from "../../engine/engine_utils_screenshot.js";
import { updateTextureFromXRFrame } from "../../engine/engine_utils_screenshot.xr.js";
import { RGBAColor } from "../../engine/js-extensions/index.js";
import { Behaviour } from "../Component.js";
const debug = getParam("debugarcamera");
/**
* WebARCameraBackground is a component that allows to display the camera feed as a background in an AR session to more easily blend the real world with the virtual world or applying effects to the camera feed.
*
* This component automatically requests `camera-access` permission when entering AR mode, which is required to:
* - Display the real-world camera feed as a background
* - Include the camera feed in AR screenshots taken with {@link screenshot2}
*
* **Note**: If you want to take AR screenshots with the camera feed but don't need to display it as a background,
* you can still add this component to your scene (it will request camera access) or manually request the
* `camera-access` feature in your `onBeforeXR` method.
*
* - Example: https://samples.needle.tools/ar-camera-background
*
* @summary Displays the camera feed as background in WebAR sessions
* @category XR
* @group Components
* @see {@link screenshot2} for taking screenshots in AR (requires camera access for camera feed compositing)
*/
export class WebARCameraBackground extends Behaviour {
/** @internal */
onBeforeXR(_mode, args) {
if (_mode === "immersive-ar") {
args.optionalFeatures = args.optionalFeatures || [];
args.optionalFeatures.push('camera-access');
if (debug)
console.warn("Requesting camera-access");
}
}
/** @internal */
onEnterXR(_args) {
if (_args.xr.mode === "immersive-ar") {
if (this.backgroundPlane) {
this.context.scene.add(this.backgroundPlane);
this.backgroundPlane.visible = false;
}
if (this.backgroundPlane)
this.context.scene.add(this.backgroundPlane);
this.context.pre_render_callbacks.push(this.preRender);
}
}
/** @internal */
onLeaveXR(_args) {
if (this.backgroundPlane)
this.backgroundPlane.removeFromParent();
const i = this.context.pre_render_callbacks.indexOf(this.preRender);
if (i >= 0)
this.context.pre_render_callbacks.splice(i, 1);
}
/**
* The tint color of the camera feed
*/
backgroundTint = new RGBAColor(1, 1, 1, 1);
get background() {
return this.backgroundPlane;
}
backgroundPlane;
threeTexture;
forceTextureInitialization = function () {
const material = new MeshBasicMaterial();
const geometry = new PlaneGeometry();
const scene = new Scene();
scene.add(new Mesh(geometry, material));
const camera = new PerspectiveCamera();
return function forceTextureInitialization(renderer, texture) {
material.map = texture;
renderer.render(scene, camera);
if (debug)
console.warn("Force texture initialization");
};
}();
/** @internal */
preRender = () => {
if (!this || !this.gameObject)
return;
const xr = this.context.renderer.xr;
const frame = xr.getFrame();
if (frame) {
// We're generating a new texture here, and force three to initialize it
// from https://stackoverflow.com/a/55084367 to inject a custom texture into three.js
if (!this.threeTexture && this.context.renderer) {
this.threeTexture = new Texture();
this.forceTextureInitialization(this.context.renderer, this.threeTexture);
}
// simple mesh and fullscreen shader to display the camera texture
// from three: WebGLBackground
if (this.backgroundPlane === undefined) {
const tint = this.backgroundTint;
this.backgroundPlane = InternalScreenshotUtils.makeFullscreenPlane({
material: new ShaderMaterial({
name: 'BackgroundMaterial',
uniforms: {
...UniformsUtils.clone(ShaderLib.background.uniforms),
tint: { value: new Vector4(tint.r, tint.g, tint.b, tint.a) },
},
vertexShader: ShaderLib.background.vertexShader,
fragmentShader: backgroundFragment,
side: DoubleSide,
depthTest: false,
depthWrite: false,
fog: false
})
});
}
if (this.backgroundPlane.parent !== this.scene)
this.scene.add(this.backgroundPlane);
if (this.backgroundPlane.material instanceof ShaderMaterial)
this.backgroundPlane.material.uniforms.tint.value.set(this.backgroundTint.r, this.backgroundTint.g, this.backgroundTint.b, this.backgroundTint.a);
// WebXR Raw Camera Access -
// we composite the camera texture into the scene background by rendering it first.
this.updateFromFrame();
}
};
/** @internal */
onBeforeRender(_frame) {
this.updateFromFrame();
}
updateFromFrame() {
if (!this.threeTexture)
return;
if (this.context.xr?.mode === "immersive-ar") {
updateTextureFromXRFrame(this.context.renderer, this.threeTexture);
this.setTexture(this.threeTexture);
}
}
setTexture(texture) {
if (!this.backgroundPlane)
return;
this.threeTexture = texture;
//@ts-ignore
this.backgroundPlane.setTexture(this.threeTexture);
this.backgroundPlane.visible = true;
}
}
__decorate([
serializable(RGBAColor)
], WebARCameraBackground.prototype, "backgroundTint", void 0);
const backgroundFragment = /* glsl */ `
uniform sampler2D t2D;
uniform vec4 tint;
varying vec2 vUv;
void main() {
vec4 texColor = texture2D( t2D, vUv );
texColor.w = 1.0;
// inline sRGB decode
texColor = vec4( mix( pow( texColor.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), texColor.rgb * 0.0773993808, vec3( lessThanEqual( texColor.rgb, vec3( 0.04045 ) ) ) ), texColor.w );
gl_FragColor = texColor * tint;
#include <tonemapping_fragment>
#include <colorspace_fragment>
}
`;
//# sourceMappingURL=WebARCameraBackground.js.map