@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
933 lines (833 loc) • 34.4 kB
text/typescript
import { Camera, Color, ColorRepresentation, FrontSide, Mesh, MirroredRepeatWrapping, PerspectiveCamera, PlaneGeometry, Scene, ShaderLib, ShaderMaterial, SRGBColorSpace, Texture, UniformsUtils, WebGLRenderer, WebGLRenderTarget } from "three";
import { GameObject } from "../engine-components/Component.js";
import { Renderer } from "../engine-components/Renderer.js";
import { WebARCameraBackground } from "../engine-components/webxr/WebARCameraBackground.js";
import { setAutoFitEnabled } from "./engine_camera.js";
import { getComponentsInChildren } from "./engine_components.js";
import { ContextRegistry } from "./engine_context_registry.js";
import { onAfterRender } from "./engine_lifecycle_api.js";
import { registerFrameEventCallback } from "./engine_lifecycle_functions_internal.js";
import { Context, FrameEvent } from "./engine_setup.js";
import { ICamera } from "./engine_types.js";
import { DeviceUtilities } from "./engine_utils.js";
import { updateTextureFromXRFrame } from "./engine_utils_screenshot.xr.js";
import { RGBAColor } from "./js-extensions/index.js";
import { setCustomVisibility } from "./js-extensions/Layers.js";
declare type ScreenshotImageMimeType = "image/webp" | "image/png" | "image/jpeg";
/**
* Take a screenshot from the current scene.
* **NOTE**: Use {@link screenshot2} for more options.
*
* @param context The context to take the screenshot from
* @param width The width of the screenshot
* @param height The height of the screenshot
* @param mimeType The mime type of the image
* @param camera The camera to use for the screenshot
* @returns The data url of the screenshot. Returns null if the screenshot could not be taken.
* @example
* ```ts
* const dataUrl = screenshot();
* saveImage(dataUrl, "screenshot.png");
* ```
*
*/
export function screenshot(context?: Context, width?: number, height?: number, mimeType: ScreenshotImageMimeType = "image/webp", camera?: Camera | null): string | null {
return screenshot2({ context, width, height, mimeType, camera });
}
/**
* Options for the {@link screenshot2} function.
*/
export declare type ScreenshotOptions = {
/**
* The context to take the screenshot from. If not provided, the current context will be used.
*/
context?: Pick<Context, "scene" | "renderer" | "mainCamera" | "renderNow" | "updateAspect" | "updateSize" | "currentFrameEvent" | "devicePixelRatio">,
/**
* The width of the screenshot - if not provided, the width of the current renderer will be used.
*/
width?: number,
/**
* The height of the screenshot - if not provided, the height of the current renderer will be used.
*/
height?: number,
/**
* The mime type of the image
*/
mimeType?: ScreenshotImageMimeType,
/**
* The camera to use for the screenshot. If not provided, the main camera of the context will be used.
*/
camera?: Camera | ICamera | null,
/**
* If true, the background will be transparent.
*/
transparent?: boolean,
/**
* If true, the image will be trimmed to the non-transparent area. Has no effect if `transparent` is false.
*/
trim?: boolean,
/**
* The background of the screenshot. If not provided, the currently set background of the renderer/scene will be used
*/
background?: Color | RGBAColor | ColorRepresentation,
/**
* If true onBeforeRender and onAfterRender will be invoked on all renderers in the scene.
* @default true
*/
render_events?: boolean,
};
export declare type ScreenshotOptionsDataUrl = ScreenshotOptions & {
/**
* If set the screenshot will be downloaded using the provided filename.
* NOTE: if you need more control you can manually download the returned image using {@link saveImage}
* @default undefined
*/
download_filename?: string,
}
export declare type ScreenshotOptionsTexture = ScreenshotOptions & {
type: "texture",
/**
* If set the screenshot will be saved to the provided texture.
* @default undefined
*/
target?: Texture,
}
export declare type ScreenshotOptionsBlob = ScreenshotOptions & {
type: "blob",
}
export declare type ScreenshotOptionsShare = ScreenshotOptions & {
/**
* Set `{ type: "share" }` to share the screenshot using the Web Share API. The promise will resolve with the blob of the screenshot and whether it was shared successfully or not. Note that the Web Share API is only available in secure contexts (HTTPS) and on some platforms.
*/
type: "share",
/**
* The filename to use when sharing the screenshot. If not provided, a default filename will be used.
*/
filename?: string,
/**
* The mime type of the shared file. If not provided, the mime type will be inferred from the screenshot options or default to "image/png".
*/
file_type?: ScreenshotImageMimeType,
/**
* The title to use when sharing the screenshot. This is optional and may not be supported by all platforms.
*/
title?: string,
/**
* The text to use when sharing the screenshot. This is optional and may not be supported by all platforms.
*/
text?: string,
/**
* The URL to use when sharing the screenshot. This is optional and may not be supported by all platforms.
*/
url?: string,
}
declare type ScreenshotOptionsShareReturnType = {
blob: Blob | null,
shared: boolean,
}
/**
* Take a screenshot from the current scene and return a {@link Texture}. This can be applied to a surface in 3D space.
* @param opts Provide `{ type: "texture" }` to get a texture instead of a data url.
* @returns The texture of the screenshot. Returns null if the screenshot could not be taken.
* @category Utilities
* @example
* ```ts
* // Create a texture from the current view
* const screenshotTexture = screenshot2({ type: "texture", width: 512, height: 512 });
* if (screenshotTexture) {
* myMaterial.map = screenshotTexture;
* myMaterial.needsUpdate = true;
* }
*
* // Update an existing texture
* const existingTexture = new Texture();
* screenshot2({ type: "texture", target: existingTexture, transparent: true });
* ```
*/
export function screenshot2(opts: ScreenshotOptionsTexture): Texture | null;
/**
* Take a screenshot from the current scene and return a data URL string.
*
* @param opts Screenshot options. All properties are optional.
* @returns The data URL of the screenshot (e.g., "data:image/png;base64,..."). Returns null if the screenshot could not be taken.
* @category Utilities
*
* @example Basic screenshot
* ```ts
* // Take a simple screenshot with default settings
* const dataUrl = screenshot2({});
* console.log(dataUrl); // "data:image/webp;base64,..."
* ```
*
* @example High-resolution screenshot with transparent background
* ```ts
* const dataUrl = screenshot2({
* width: 2048,
* height: 2048,
* mimeType: "image/png",
* transparent: true,
* trim: true, // Remove transparent edges
* });
* ```
*
* @example Screenshot with custom background color
* ```ts
* import { Color } from "three";
*
* const dataUrl = screenshot2({
* width: 1024,
* height: 1024,
* background: new Color(0x00ff00), // Green background
* });
* ```
*
* @example Download screenshot automatically
* ```ts
* screenshot2({
* width: 1920,
* height: 1080,
* mimeType: "image/jpeg",
* download_filename: "my-scene.jpg",
* });
* ```
*
* @example Manual download using saveImage
* ```ts
* const dataUrl = screenshot2({
* width: 1024,
* height: 1024,
* mimeType: "image/webp",
* transparent: true,
* });
* if (dataUrl) {
* saveImage(dataUrl, "screenshot.webp");
* }
* ```
*
* @example Screenshot from specific camera
* ```ts
* const myCamera = this.gameObject.getComponent(Camera);
* const dataUrl = screenshot2({
* camera: myCamera,
* width: 1024,
* height: 1024,
* });
* ```
*/
export function screenshot2(opts: ScreenshotOptionsDataUrl): string | null;
/**
* Take a screenshot asynchronously and return a Blob. This is useful when you need to process or upload the image data.
*
* @param opts Set `{ type: "blob" }` to get a blob instead of a data url. All other {@link ScreenshotOptions} are also available.
* @returns A Promise that resolves with the Blob of the screenshot. Returns null if the screenshot could not be taken.
* @category Utilities
*
* @example Upload screenshot to server
* ```ts
* const blob = await screenshot2({ type: "blob", mimeType: "image/png" });
* if (blob) {
* const formData = new FormData();
* formData.append("screenshot", blob, "screenshot.png");
* await fetch("/api/upload", { method: "POST", body: formData });
* }
* ```
*
* @example Save blob to file (browser download)
* ```ts
* const blob = await screenshot2({
* type: "blob",
* width: 1920,
* height: 1080,
* transparent: true
* });
* if (blob) {
* const url = URL.createObjectURL(blob);
* saveImage(url, "screenshot.png");
* URL.revokeObjectURL(url); // Clean up
* }
* ```
*/
export function screenshot2(opts: ScreenshotOptionsBlob): Promise<Blob | null>;
/**
* Take a screenshot and share it using the Web Share API (mobile-friendly).
*
* **Note**: The Web Share API is only available in secure contexts (HTTPS) and may not be supported on all platforms/browsers.
*
* @param opts Set `{ type: "share" }` to share the screenshot. Additional options like `filename`, `title`, `text`, and `url` can be provided.
* @returns A Promise that resolves with an object containing the blob and whether it was successfully shared.
* @category Utilities
*
* @example Share screenshot on mobile
* ```ts
* const result = await screenshot2({
* type: "share",
* filename: "my-creation.png",
* title: "Check out my 3D scene!",
* text: "I created this with Needle Engine",
* url: "https://engine.needle.tools",
* mimeType: "image/png",
* });
*
* if (result.shared) {
* console.log("Screenshot shared successfully!");
* } else {
* console.log("User cancelled or sharing not supported");
* }
* ```
*
* @example Share with fallback
* ```ts
* const result = await screenshot2({
* type: "share",
* filename: "screenshot.webp",
* file_type: "image/webp",
* });
*
* if (!result.shared && result.blob) {
* // Fallback: download the image instead
* const url = URL.createObjectURL(result.blob);
* saveImage(url, "screenshot.webp");
* URL.revokeObjectURL(url);
* }
* ```
*/
export function screenshot2(opts: ScreenshotOptionsShare): Promise<ScreenshotOptionsShareReturnType>;
/**
* Take a screenshot from the current scene with advanced options.
*
* This is the main screenshot function in Needle Engine with support for multiple output formats:
* - **Data URL** (default): Returns a base64-encoded string suitable for img src attributes
* - **Texture**: Returns a Three.js Texture that can be applied to materials
* - **Blob**: Returns a Blob for uploading or processing (async)
* - **Share**: Uses the Web Share API to share the screenshot (async, mobile-friendly)
*
* @param opts Screenshot options. Use the `type` property to specify output format. See {@link ScreenshotOptions} for all available options.
* @returns Depending on the `type` option:
* - Data URL (string) when `type` is undefined or not specified
* - Texture when `type: "texture"`
* - Promise<Blob | null> when `type: "blob"`
* - Promise<{blob, shared}> when `type: "share"`
*
* Returns null (or Promise resolving to null) if the screenshot could not be taken.
*
* @category Utilities
*
* @example WebXR / AR Screenshots
* ```ts
* // Screenshots work automatically in WebXR sessions
* // The camera feed will be composited with your 3D content
* const dataUrl = screenshot2({
* width: 1920,
* height: 1080
* });
* ```
*
* **Note for AR Screenshots**: To include the device camera feed in AR screenshots, you need to request camera access.
* This is done automatically when you use {@link WebARCameraBackground} component in your scene.
* If you're not using WebARCameraBackground, you can request camera access manually:
* ```ts
* export class MyComponent extends Behaviour {
* onBeforeXR(mode: XRSessionMode, args: XRSessionInit): void {
* if (mode === "immersive-ar") {
* args.optionalFeatures = args.optionalFeatures || [];
* args.optionalFeatures.push('camera-access');
* }
* }
* }
* ```
* Without camera access, AR screenshots will only show your 3D content without the real-world background.
*
* @example Combining multiple options
* ```ts
* // High-res transparent screenshot with custom camera
* const myCamera = this.gameObject.getComponent(Camera);
* const texture = screenshot2({
* type: "texture",
* camera: myCamera,
* width: 2048,
* height: 2048,
* transparent: true,
* trim: true,
* render_events: true, // Ensure reflection probes are updated
* });
* ```
*
* @see {@link screenshot} for a simpler alternative with fewer options
* @see {@link saveImage} for downloading data URLs
* @see {@link ScreenshotOptions} for all available options
*/
export function screenshot2(opts: ScreenshotOptionsDataUrl | ScreenshotOptionsTexture | ScreenshotOptionsBlob | ScreenshotOptionsShare)
: Texture | string | null | Promise<Blob | null> | Promise<ScreenshotOptionsShareReturnType> {
if (!opts) opts = {};
const { transparent = false } = opts;
let { mimeType, context, width, height, camera } = opts;
if (!context) {
context = ContextRegistry.Current as Context;
if (!context) {
console.error("Can not save screenshot: No needle-engine context found or provided.");
return null;
}
}
if (!camera) {
camera = context.mainCamera;
if (!camera) {
console.error("No camera found");
return null;
}
}
const renderer = context.renderer;
const isXRScreenshot = renderer.xr.enabled && renderer.xr.isPresenting;
// Perform XR screenshot in onBeforeRender (after the screenshot we want to render the original camera view)
// If we do it in onAfterRender we will see one frame of a wrong image which is not what we want
if (isXRScreenshot && context.currentFrameEvent != FrameEvent.EarlyUpdate) {
console.warn("Screenshot: defer to access XR frame")
const ret = new Promise(resolve => {
registerFrameEventCallback(_ => {
// TODO: why is the return type not correct?
const screenshotResult = screenshot2(opts);
resolve(screenshotResult);
}, FrameEvent.EarlyUpdate, { once: true });
});
/** @ts-expect-error */
return ret;
}
const domElement = renderer.domElement;
const prevWidth = domElement.width;
const prevHeight = domElement.height;
if (!width) width = prevWidth;
if (!height) height = prevHeight;
const renderWidth = width;
const renderHeight = height;
// Apply page zoom
let zoomLevel = (window.devicePixelRatio || 1);
let contextZoomLevel = 1;
// Apply context zoom level
if (context.devicePixelRatio === 'auto' || context.devicePixelRatio === 'manual')
contextZoomLevel = 1;
else
contextZoomLevel = context.devicePixelRatio / window.devicePixelRatio;
zoomLevel *= contextZoomLevel;
width /= zoomLevel;
height /= zoomLevel;
width = Math.floor(width);
height = Math.floor(height);
// save XR state and reset it for screenshot
const xrframe = renderer.xr.isPresenting && renderer.xr.getFrame();
const previousXRState = renderer.xr.enabled;
renderer.xr.enabled = false;
renderer.xr.isPresenting = false;
// reset style during screenshot
domElement.style.width = `${width}px`;
domElement.style.height = `${height}px`;
const prevRenderTarget = renderer.getRenderTarget();
const previousClearColor = renderer.getClearColor(new Color());
const previousClearAlpha = renderer.getClearAlpha();
const previousBackground = context.scene.background;
const previousAspect: number | null = "aspect" in camera ? camera.aspect : null;
try {
// Calling onBeforeRender to update objects with reflection probes. https://linear.app/needle/issue/NE-5112
const callRenderEvents = opts.render_events !== false;
const renderers = new Array<Renderer>();
if (callRenderEvents) {
getComponentsInChildren(context.scene, Renderer, renderers);
renderers.forEach(r => {
r?.onBeforeRender();
if (r.isInstancingActive && r.instances) {
for (let i = 0; i < r.instances?.length; i++) {
const handle = r.instances[i];
setCustomVisibility(handle.object, true);
}
}
});
}
if (transparent) {
context.scene.background = null;
renderer.setClearColor(0x000000, 0);
}
if (opts.background) {
context.scene.background = null;
renderer.setClearColor(opts.background);
if (opts.background instanceof RGBAColor) {
renderer.setClearAlpha(opts.background.a);
}
}
if (transparent) {
renderer.setClearAlpha(0);
}
// set the desired output size
renderer.setSize(width, height, false);
// If a camera component was provided
if ("cam" in camera) {
camera = camera.threeCamera;
}
// update the camera aspect and matrix
if (camera instanceof PerspectiveCamera) {
camera.aspect = width / height;
camera.updateProjectionMatrix();
}
const textureOutput = "type" in opts && opts.type === "texture";
let targetTexture: WebGLRenderTarget | null = null;
if (textureOutput) {
targetTexture = new WebGLRenderTarget(width, height, {
wrapS: MirroredRepeatWrapping,
wrapT: MirroredRepeatWrapping,
format: 1023,
});
renderer.setRenderTarget(targetTexture);
}
let outputCanvas = domElement;
if (isXRScreenshot) {
// Special rendering path since in XR rendering doesn't go into the domElement
// and we also want to composite the camera-access image if available.
if (targetTexture) {
console.error('Taking XR screenshots with { type: "texture" } is currently not supported.');
}
// Note: this method must be invoked during the render loop when we get the XR frame
outputCanvas = InternalScreenshotUtils.compositeWithCameraImage({
width: renderWidth,
height: renderHeight,
scene: context.scene,
camera: camera,
renderer: renderer,
});
}
else {
// Render normally, as we can just use the domElement for rendering
context.renderNow(camera || null);
}
// restore
if (camera instanceof PerspectiveCamera && previousAspect != null) {
camera.aspect = previousAspect;
camera.updateProjectionMatrix();
}
if (callRenderEvents)
renderers.forEach(r => r.onAfterRender());
if (!mimeType) {
if ("download_filename" in opts && opts.download_filename) {
const ext = opts.download_filename.split(".").pop()?.toLowerCase();
switch (ext) {
case "png":
mimeType = "image/png";
break;
case "jpg":
case "jpeg":
mimeType = "image/jpeg";
break;
case "webp":
mimeType = "image/webp";
break;
}
}
}
if (transparent && opts.trim === true) {
const trimmed = trimCanvas(outputCanvas);
if (trimmed) outputCanvas = trimmed;
}
if ("type" in opts) {
if (opts.type === "texture") {
if (!targetTexture) {
console.error("No target texture found");
return null;
}
if (opts.target) {
opts.target.image = targetTexture?.texture.image;
opts.target.needsUpdate = true;
}
targetTexture.texture.offset.set(0, -1);
targetTexture.texture.needsUpdate = true;
return targetTexture.texture;
}
else if (opts.type === "blob") {
const promise = new Promise<Blob | null>((resolve, _) => {
outputCanvas.toBlob(blob => {
resolve(blob);
}, mimeType);
});
return promise;
}
else if (opts.type === "share") {
const promise = new Promise<ScreenshotOptionsShareReturnType>((resolve, _) => {
outputCanvas.toBlob(blob => {
if (blob && "share" in navigator) {
let mimetype = "file_type" in opts ? opts.file_type || mimeType : mimeType;
if (!mimeType) {
mimetype = "image/png";
}
const ext = mimetype?.split("/")[1] || "png";
const file = new File([blob], "filename" in opts ? opts.filename || `screenshot.${ext}` : `screenshot.${ext}`, { type: mimetype });
return navigator.share({
title: "title" in opts ? opts.title : undefined,
text: "text" in opts ? opts.text : undefined,
url: "url" in opts ? opts.url : undefined,
files: [file],
})
.catch(err => {
console.warn("User cancelled share", err.message);
})
.finally(() => {
resolve({ blob, shared: true });
});
}
return {
blob: blob,
shared: false
}
}, mimeType);
});
return promise;
}
}
const dataUrl = outputCanvas.toDataURL(mimeType);
if ("download_filename" in opts && opts.download_filename) {
let download_name = opts.download_filename;
// On mobile we don't want to see the dialogue for every screenshot
if (DeviceUtilities.isMobileDevice() && typeof window !== "undefined") {
const key = download_name + "_screenshots";
const parts = download_name.split(".");
const ext = parts.pop()?.toLowerCase();
let count = 0;
if (localStorage.getItem(key)) {
count = parseInt(sessionStorage.getItem(key) || "0");
}
if (count > 0) {
// const timestamp = new Date().toLocaleString();
download_name = `${parts.join()}-${count}.${ext}`;
}
count += 1;
sessionStorage.setItem(key, count.toString());
}
saveImage(dataUrl, download_name);
}
return dataUrl;
}
finally {
renderer.setRenderTarget(prevRenderTarget);
context.scene.background = previousBackground;
renderer.setSize(prevWidth, prevHeight, false);
renderer.setClearColor(previousClearColor, previousClearAlpha);
// Make sure to reset the aspect ratio. This is crucial if the main camera is not the currently active rendering camera
// For example if we did a screenshot from a different camera that has a different aspect ratio / fov
if (previousAspect != null && camera instanceof PerspectiveCamera) {
camera.aspect = previousAspect;
camera.updateProjectionMatrix();
}
renderer.xr.enabled = previousXRState;
renderer.xr.isPresenting = isXRScreenshot;
if (!isXRScreenshot)
context.updateSize(true);
}
return null;
}
// trim to transparent pixels
function trimCanvas(originalCanvas: HTMLCanvasElement): HTMLCanvasElement | null {
if (!("document" in globalThis)) return null;
// Copy the original canvas to a new canvas
const canvas = document.createElement('canvas');
canvas.width = originalCanvas.width;
canvas.height = originalCanvas.height;
const ctx = canvas.getContext('2d');
if (!ctx) return null;
ctx.drawImage(originalCanvas, 0, 0);
const width = canvas.width;
const height = canvas.height;
const imageData = ctx.getImageData(0, 0, width, height);
const data = imageData.data;
// Calculate the bounding box of non-transparent pixels
let top = height, left = width, bottom = 0, right = 0;
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const index = (y * width + x) * 4;
const alpha = data[index + 3];
if (alpha !== 0) {
if (x < left) left = x;
if (x > right) right = x;
if (y < top) top = y;
if (y > bottom) bottom = y;
}
}
}
// Create new canvas with trimmed dimensions
const trimmedWidth = right - left + 1;
const trimmedHeight = bottom - top + 1;
const trimmedCanvas = document.createElement('canvas');
const trimmedCtx = trimmedCanvas.getContext('2d');
if (!trimmedCtx) return null;
trimmedCanvas.width = trimmedWidth;
trimmedCanvas.height = trimmedHeight;
// Draw the trimmed area onto the new canvas
trimmedCtx.drawImage(canvas, left, top, trimmedWidth, trimmedHeight, 0, 0, trimmedWidth, trimmedHeight);
return trimmedCanvas;
}
let saveImageElement: HTMLAnchorElement | null = null;
/** Download a image (must be a data url).
* @param dataUrl The data url of the image
* @param filename The filename of the image
* @example
* ```ts
* const dataUrl = screenshot();
* saveImage(dataUrl, "screenshot.png");
* ```
*/
export function saveImage(dataUrl: string | null, filename: string) {
if (!dataUrl) {
return;
}
if (!dataUrl.startsWith("data:image")) {
console.error("Can not save image: Data url is not an image", dataUrl);
return;
}
if (!saveImageElement) {
saveImageElement = document.createElement("a");
}
saveImageElement.href = dataUrl;
saveImageElement.download = filename;
saveImageElement.click();
}
export namespace InternalScreenshotUtils {
let backgroundPlane: FullscreenPlane | null = null;
let otherPlaneMesh: FullscreenPlane | null = null;
let rtTexture: WebGLRenderTarget | null = null;
let threeTexture: Texture | null = null;
let customCanvas: HTMLCanvasElement | null = null;
/**
* Screenshot rendering for AR
* @param args
* @returns The canvas with the screenshot
*/
export function compositeWithCameraImage(args: { scene: Scene, camera: Camera, renderer: WebGLRenderer, width: number, height: number }) {
const { renderer, width, height } = args;
const prevXREnabled = renderer.xr.enabled;
const prevRT = renderer.getRenderTarget();
const prevAutoClear = renderer.autoClear;
// Initialize the render target and canvas. Width and height should already take DPI into account
const expectedWidth = width;
const expectedHeight = height;
const aspect = width / height;
if (!rtTexture || rtTexture.width !== expectedWidth || rtTexture.height !== expectedHeight) {
rtTexture ??= new WebGLRenderTarget(expectedWidth, expectedHeight, { colorSpace: SRGBColorSpace });
rtTexture.width = expectedWidth;
rtTexture.height = expectedHeight;
rtTexture.samples = 4;
// necessary to match texture orientation from the exported meshes it seems
rtTexture.texture.repeat.y = -1;
rtTexture.texture.offset.y = 1;
}
if (!customCanvas || customCanvas.width !== expectedWidth || customCanvas.height !== expectedHeight) {
customCanvas = document.createElement('canvas');
customCanvas.width = expectedWidth;
customCanvas.height = expectedHeight;
customCanvas.style.position = "fixed";
customCanvas.style.top = "0px";
customCanvas.style.right = "0px";
customCanvas.style.width = "300px";
customCanvas.style.height = `${300 / aspect}px`;
customCanvas.style.zIndex = "1000";
customCanvas.style.pointerEvents = "none";
customCanvas.style.opacity = "1.0";
customCanvas.style.willChange = "contents";
}
if (!backgroundPlane) {
backgroundPlane = makeFullscreenPlane({
defines: {
DECODE_VIDEO_TEXTURE: true
},
});
}
if (!otherPlaneMesh) {
otherPlaneMesh = makeFullscreenPlane();
}
if (!threeTexture) {
threeTexture = new Texture();
}
const manager = renderer.xr;
manager.updateCamera(args.camera as PerspectiveCamera);
// adjust aspect on currentCamera
// doesn't seem to be necessary since updateCamera
// if (args.camera.type === "PerspectiveCamera") {
// const cam = args.camera as PerspectiveCamera;
// cam.aspect = aspect;
// cam.updateProjectionMatrix();
// }
renderer.xr.enabled = false;
renderer.autoClear = false;
renderer.clear();
renderer.setSize(expectedWidth, expectedHeight);
renderer.setRenderTarget(rtTexture);
// First we update the render texture which will hold the camera image
if (!updateTextureFromXRFrame(args.renderer, threeTexture)) {
console.error("Could not update texture from XR frame");
}
const camBg = GameObject.findObjectOfType(WebARCameraBackground);
if (camBg) {
// the scene uses WebARCameraBackground, so we make sure it has the latest camera-access texture
camBg.setTexture(threeTexture);
}
else {
// the scene doesn't use WebARCameraBackground, so we render the camera feed fullscreen
backgroundPlane.setTexture(threeTexture);
renderer.render(backgroundPlane, args.camera);
}
renderer.clearDepth();
renderer.setSize(expectedWidth, expectedHeight);
renderer.render(args.scene, args.camera);
// Blit the render texture first into our renderer on the GPU,
// then into a canvas so we can process it further on the CPU
renderer.setRenderTarget(null);
otherPlaneMesh.setTexture(rtTexture.texture);
renderer.render(otherPlaneMesh, args.camera);
const _context = customCanvas.getContext('2d', { alpha: false })!;
_context.drawImage(renderer.domElement, 0, 0, customCanvas.width, customCanvas.height);
renderer.setRenderTarget(prevRT);
renderer.xr.enabled = prevXREnabled;
renderer.autoClear = prevAutoClear;
return customCanvas;
}
const backgroundFragment: string = /* glsl */`
uniform sampler2D t2D;
varying vec2 vUv;
void main() {
vec4 texColor = texture2D( t2D, vUv );
#ifdef DECODE_VIDEO_TEXTURE
// inline sRGB decode (TODO: Remove this code when https://crbug.com/1256340 is solved)
texColor = vec4( mix( pow( texColor.rgb * 0.9478672986 + vec3( 0.0521327014 ), vec3( 2.4 ) ), texColor.rgb * 0.0773993808, vec3( lessThanEqual( texColor.rgb, vec3( 0.04045 ) ) ) ), texColor.w );
#endif
gl_FragColor = texColor;
#include <tonemapping_fragment>
#include <colorspace_fragment>
}
`;
declare type FullscreenPlane = Mesh & { setTexture: (texture: Texture) => void };
export function makeFullscreenPlane(options?: {
material?: ShaderMaterial,
defines?: { [key: string]: boolean | number },
}): FullscreenPlane {
const planeMaterial = options?.material || new ShaderMaterial({
name: 'BackgroundMaterial',
uniforms: UniformsUtils.clone(ShaderLib.background.uniforms),
vertexShader: ShaderLib.background.vertexShader,
fragmentShader: backgroundFragment,
defines: options?.defines,
side: FrontSide,
depthTest: false,
depthWrite: false,
fog: false
});
// add "map" material property so the renderer can evaluate it like for built-in materials
Object.defineProperty(planeMaterial, 'map', {
get: function () {
return this.threeTexture;
}
});
const planeMesh = new Mesh(new PlaneGeometry(2, 2), planeMaterial,) as unknown as FullscreenPlane;
setAutoFitEnabled(planeMesh, false);
planeMesh.geometry.deleteAttribute('normal');
// Option 1: add the planeMesh to our scene for rendering.
// This is useful for applying custom shader effects on the background (instead of using the system composite)
planeMesh.renderOrder = -1000000; // render first
// should be a class, for now lets just define a method for the weird way the texture needs to be set
planeMesh.setTexture = function (texture) {
planeMaterial.uniforms.t2D.value = texture;
}
return planeMesh;
}
}