@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
1,078 lines (928 loc) • 35.3 kB
text/typescript
import { Material, Mesh, Object3D, ShaderMaterial, SRGBColorSpace, Texture, Vector2, Vector4, VideoTexture } from "three";
import { isDevEnvironment } from "../engine/debug/index.js";
import { ObjectUtils, PrimitiveType } from "../engine/engine_create_objects.js";
import { awaitInput } from "../engine/engine_input_utils.js";
import { serializable } from "../engine/engine_serialization_decorator.js";
import { Context } from "../engine/engine_setup.js";
import { getWorldScale } from "../engine/engine_three_utils.js";
import { getParam } from "../engine/engine_utils.js";
import { PointerEventData } from "./api.js";
import { Behaviour, GameObject } from "./Component.js";
import { Renderer } from "./Renderer.js";
const debug = getParam("debugvideo");
export enum AspectMode {
None = 0,
AdjustHeight = 1,
AdjustWidth = 2,
}
export enum VideoSource {
/// <summary>
/// <para>Use the current clip as the video content source.</para>
/// </summary>
VideoClip = 0,
/// <summary>
/// <para>Use the current URL as the video content source.</para>
/// </summary>
Url = 1,
}
export enum VideoAudioOutputMode {
None = 0,
AudioSource = 1,
Direct = 2,
APIOnly = 3,
}
export enum VideoRenderMode {
CameraFarPlane = 0,
CameraNearPlane = 1,
RenderTexture = 2,
MaterialOverride = 3,
}
/**
* [VideoPlayer](https://engine.needle.tools/docs/api/VideoPlayer) plays video clips from URLs, media streams, or HLS playlists (m3u8 livestreams).
*
* **Supported formats:**
* - Standard video files (MP4, WebM, etc.)
* - Media streams (from webcam, screen capture, etc.)
* - HLS playlists (m3u8) for livestreaming
*
* [](https://engine.needle.tools/samples/video-playback/)
*
* **Rendering modes:**
* Video can be rendered to a material texture, render texture, or camera planes.
* Set `targetMaterialRenderer` to apply video to a specific mesh's material.
*
* **Browser autoplay:**
* Videos may require user interaction to play with audio.
* Set `playOnAwake = true` for automatic playback (muted if needed).
*
* @example Basic video playback
* ```ts
* const videoPlayer = addComponent(obj, VideoPlayer, {
* url: "https://example.com/video.mp4",
* playOnAwake: true,
* loop: true,
* });
* ```
*
* @example Play video on a 3D surface
* ```ts
* const video = myScreen.getComponent(VideoPlayer);
* video.targetMaterialRenderer = myScreen.getComponent(Renderer);
* video.play();
* ```
*
* @summary Plays video clips from URLs or streams
* @category Multimedia
* @group Components
* @see {@link AudioSource} for audio-only playback
* @see {@link ScreenCapture} for capturing and sharing video
* @see {@link Renderer} for video texture targets
*/
export class VideoPlayer extends Behaviour {
/**
* When true the video will start playing as soon as the component is enabled
*/
playOnAwake: boolean = true;
/**
* The aspect mode to use for the video. If
*/
aspectMode: AspectMode = AspectMode.None;
// /**
// * If true the video will toggle between play and pause when clicked.
// * Note that this requires the video element to be interactable which is not the case when using certain render modes like CameraNearPlane or CameraFarPlane. In those cases you can use the `requestPictureInPicture()` method to open the video in a separate window which allows interaction with the video element.
// */
// @serializable()
// clickToToggle: boolean = true;
// onPointerClick(_args: PointerEventData) {
// if (this.clickToToggle) {
// if (this.isPlaying) this.pause();
// else this.play();
// }
// }
private clip?: string | MediaStream | null = null;
// set a default src, this should not be undefined
private source: VideoSource = VideoSource.Url;
/**
* The video clip url to play.
*/
get url() { return this._url }
/**
* The video clip to play.
*/
set url(val: string | null) {
const prev = this._url;
const changed = prev !== val;
if (this.__didAwake) {
if (changed) {
this.setClipURL(val ?? "");
}
}
else this._url = val;
}
private _url: string | null = null;
private renderMode: VideoRenderMode = VideoRenderMode.MaterialOverride;
private targetMaterialProperty?: string;
private targetMaterialRenderer?: Renderer;
private targetTexture?: Texture;
private time: number = 0;
private _playbackSpeed: number = 1;
/**
* Get the video playback speed. Increasing this value will speed up the video, decreasing it will slow it down.
* @default 1
*/
get playbackSpeed(): number {
return this._videoElement?.playbackRate ?? this._playbackSpeed;
}
/**
* Set the video playback speed. Increasing this value will speed up the video, decreasing it will slow it down.
*/
set playbackSpeed(val: number) {
this._playbackSpeed = val;
if (this._videoElement)
this._videoElement.playbackRate = val;
}
private _isLooping: boolean = false;
get isLooping(): boolean {
return this._videoElement?.loop ?? this._isLooping;
}
set isLooping(val: boolean) {
this._isLooping = val;
if (this._videoElement)
this._videoElement.loop = val;
}
/**
* @returns the current time of the video in seconds
*/
get currentTime(): number {
return this._videoElement?.currentTime ?? this.time;
}
/**
* set the current time of the video in seconds
*/
set currentTime(val: number) {
if (this._videoElement) {
this._videoElement.currentTime = val;
}
else this.time = val;
}
/**
* @returns true if the video is currently playing
*/
get isPlaying(): boolean {
const video = this._videoElement;
if (video) {
if (video.currentTime > 0 && !video.paused && !video.ended
&& video.readyState > video.HAVE_CURRENT_DATA)
return true;
else if (video.srcObject) {
const stream = video.srcObject as MediaStream;
if (stream.active) return true;
}
}
return false;
}
get crossOrigin(): string | null {
return this._videoElement?.crossOrigin ?? this._crossOrigin;
}
set crossOrigin(val: string | null) {
this._crossOrigin = val;
if (this._videoElement) {
if (val !== null) this._videoElement.setAttribute("crossorigin", val);
else this._videoElement.removeAttribute("crossorigin");
}
}
/**
* the material that is used to render the video
*/
get videoMaterial() {
if (!this._videoMaterial) if (!this.create(false)) return null;
return this._videoMaterial;
}
/**
* the video texture that is used to render the video
*/
get videoTexture() {
if (!this._videoTexture) if (!this.create(false)) return null;
return this._videoTexture;
}
/**
* the HTMLVideoElement that is used to play the video
*/
get videoElement() {
if (!this._videoElement) if (!this.create(false)) return null;
return this._videoElement!;
}
/**
* Request the browser to enter picture in picture mode
* @link https://developer.mozilla.org/en-US/docs/Web/API/Picture-in-Picture_API
* @returns the promise returned by the browser
*/
requestPictureInPicture() {
if (this._videoElement) return this._videoElement.requestPictureInPicture();
return null;
}
/**
* @returns true if the video is muted
*/
get muted() {
return this._videoElement?.muted ?? this._muted;
}
/**
* set the video to be muted
*/
set muted(val: boolean) {
this._muted = val;
if (this._videoElement) this._videoElement.muted = val;
}
private _muted: boolean = false;
/**
* The current video clip that is being played
*/
get currentVideo() {
return this.clip;
}
private set audioOutputMode(mode: VideoAudioOutputMode) {
if (mode !== this._audioOutputMode) {
if (mode === VideoAudioOutputMode.AudioSource && isDevEnvironment()) console.warn("VideoAudioOutputMode.AudioSource is not yet implemented");
this._audioOutputMode = mode;
this.updateVideoElementSettings();
}
}
private get audioOutputMode() { return this._audioOutputMode; }
private _audioOutputMode: VideoAudioOutputMode = VideoAudioOutputMode.Direct;
/** Set this to false to pause video playback while the tab is not active
* @default true
*/
playInBackground: boolean = true;
private _crossOrigin: string | null = "anonymous";
private _videoElement: HTMLVideoElement | null = null;
private _videoTexture: VideoTexture | null = null;
private _videoMaterial: Material | null = null;
private _isPlaying: boolean = false;
private wasPlaying: boolean = false;
/** ensure's the video element has been created and will start loading the clip */
preloadVideo() {
if (debug) console.log("Video Preload: " + this.name, this.clip);
this.create(false);
}
/** @deprecated use `preloadVideo()` */
preload() { this.preloadVideo(); }
/** Set a new video stream
* starts to play automatically if the videoplayer hasnt been active before and playOnAwake is true */
setVideo(video: MediaStream) {
this.clip = video;
this.source = VideoSource.VideoClip;
if (!this._videoElement) this.create(this.playOnAwake);
else {
// TODO: how to prevent interruption error when another video is already playing
this._videoElement.srcObject = video;
if (this._isPlaying)
this.play();
this.updateAspect();
}
}
setClipURL(url: string) {
if (this._url === url) return;
this._url = url;
this.source = VideoSource.Url;
if (debug) console.log("set url", url);
if (!this._videoElement) this.create(this.playOnAwake);
else {
if (url.endsWith(".m3u8") || url.includes(".m3u")) {
this.ensureM3UCanBePlayed();
}
else {
this._videoElement.src = url;
if (this._isPlaying) {
this.stop();
this.play();
}
}
}
}
/** @internal */
onEnable(): void {
if (debug) console.log("VideoPlayer.onEnable", VideoSource[this.source], this.clip, this.url, this)
window.addEventListener('visibilitychange', this.visibilityChanged);
if (this.playOnAwake === true) {
this.create(true);
}
else {
this.preloadVideo();
}
if (this.screenspace) {
this._overlay?.start();
}
else this._overlay?.stop();
}
/** @internal */
onDisable(): void {
window.removeEventListener('visibilitychange', this.visibilityChanged);
this._overlay?.stop();
this.pause();
}
private visibilityChanged = (_: Event) => {
switch (document.visibilityState) {
case "hidden":
if (!this.playInBackground) {
this.wasPlaying = this._isPlaying;
this.pause();
}
break;
case "visible":
if (this.wasPlaying && !this._isPlaying) this.play();
break;
}
}
/** @internal */
onDestroy(): void {
if (this._videoElement) {
this.videoElement?.remove();
this._videoElement = null;
}
if (this._videoTexture) {
this._videoTexture.dispose();
this._videoTexture = null;
}
}
private _receivedInput: boolean = false;
/**
* @internal
*/
constructor() {
super();
awaitInput(() => {
this._receivedInput = true;
this.updateVideoElementSettings();
});
this._targetObjects = [];
if (getParam("videoscreenspace")) {
window.addEventListener("keydown", evt => {
if (evt.key === "f") {
this.screenspace = !this.screenspace;
}
});
}
}
private _playErrors: number = 0;
/** start playing the video source */
play() {
if (!this._videoElement) this.create(false);
if (!this._videoElement) {
if (debug) console.warn("Can not play: no video element found", this);
return
}
if (this._isPlaying && !this._videoElement?.ended && !this._videoElement?.paused) return;
this._isPlaying = true;
if (!this._receivedInput) this._videoElement.muted = true;
this.handleBeginPlaying(false);
if (this.shouldUseM3U) {
this.ensureM3UCanBePlayed();
return;
}
if (debug) console.log("Video Play()", this.clip, this._videoElement, this.time);
this._videoElement.currentTime = this.time;
this._videoElement.play().catch(err => {
if (this._playErrors++ < 10) console.error(err);
else if (this._playErrors === 10) console.error("Multiple errors playing video, further errors will be suppressed. Use 'debugvideo' param to see all errors.");
// https://developer.chrome.com/blog/play-request-was-interrupted/
if (debug)
console.error("Error playing video", err, "CODE=" + err.code, this.videoElement?.src, this);
setTimeout(() => {
if (this._isPlaying && !this.destroyed && this.activeAndEnabled)
this.play();
}, 1000);
});
if (debug) console.log("play", this._videoElement, this.time);
}
/**
* Stop the video playback. This will reset the video to the beginning
*/
stop() {
this._isPlaying = false;
this.time = 0;
if (!this._videoElement) return;
this._videoElement.currentTime = 0;
this._videoElement.pause();
if (debug) console.log("STOP", this);
}
/**
* Pause the video playback
*/
pause(): void {
this.time = this._videoElement?.currentTime ?? 0;
this._isPlaying = false;
this._videoElement?.pause();
if (debug) console.log("PAUSE", this, this.currentTime);
}
/** create the video element and assign the video source url or stream */
create(playAutomatically: boolean): boolean {
let src;
switch (this.source) {
case VideoSource.VideoClip:
src = this.clip;
break;
case VideoSource.Url:
src = this.url;
if (!src?.length && typeof this.clip === "string")
src = this.clip;
break;
}
if (!src) {
if (debug) console.warn("No video source set", this);
return false;
}
if (!this._videoElement) {
if (debug)
console.warn("Create VideoElement", this);
this._videoElement = this.createVideoElement();
this.context.domElement!.shadowRoot!.prepend(this._videoElement);
// hide it because otherwise it would overlay the website with default css
this.updateVideoElementStyles();
}
if (typeof src === "string") {
if (debug) console.log("Set Video src", src);
this._videoElement.src = src;
// Nor sure why we did this here, but with this code the video does not restart when being paused / enable toggled
// const str = this._videoElement["captureStream"]?.call(this._videoElement);
// this.clip = str;
}
else {
if (debug) console.log("Set Video srcObject", src);
this._videoElement.srcObject = src;
}
if (!this._videoTexture)
this._videoTexture = new VideoTexture(this._videoElement);
this._videoTexture.flipY = false;
this._videoTexture.colorSpace = SRGBColorSpace;
if (playAutomatically)
this.handleBeginPlaying(playAutomatically);
if (debug)
console.log("Video: handle playing done...", src, playAutomatically);
return true;
}
updateAspect() {
if (this.aspectMode === AspectMode.None) return;
this.startCoroutine(this.updateAspectImpl());
}
private _overlay: VideoOverlay | null = null;
/**
* If true the video will be rendered in screenspace mode and overlayed on top of the scene.
* Alternatively you can also request the video to be played in PictureInPicture mode by calling `requestPictureInPicture()`
*/
get screenspace(): boolean {
return this._overlay?.enabled ?? false;
}
set screenspace(val: boolean) {
if (val) {
if (!this._videoTexture) return;
if (!this._overlay) this._overlay = new VideoOverlay(this.context);
this._overlay.add(this._videoTexture);
}
else this._overlay?.remove(this._videoTexture);
if (this._overlay) this._overlay.enabled = val;
}
private _targetObjects: Array<Object3D>;
private createVideoElement(): HTMLVideoElement {
const video = document.createElement("video") as HTMLVideoElement;
if (this._crossOrigin)
video.setAttribute("crossorigin", this._crossOrigin);
if (debug) console.log("created video element", video);
return video;
}
private handleBeginPlaying(playAutomatically: boolean) {
if (!this.activeAndEnabled) return;
if (!this._videoElement) return;
this._targetObjects.length = 0;
let target: Object3D | undefined = this.gameObject;
switch (this.renderMode) {
case VideoRenderMode.MaterialOverride:
target = this.targetMaterialRenderer?.gameObject;
if (!target) target = GameObject.getComponent(this.gameObject, Renderer)?.gameObject;
break;
case VideoRenderMode.RenderTexture:
console.error("VideoPlayer renderTexture not implemented yet. Please use material override instead");
return;
}
if (!target) {
console.error("Missing target for video material renderer", this.name, VideoRenderMode[this.renderMode!], this);
return;
}
const mat = target["material"];
if (mat) {
this._targetObjects.push(target);
if (mat !== this._videoMaterial) {
this._videoMaterial = mat.clone();
target["material"] = this._videoMaterial;
}
const fieldName = "map";
const videoMaterial = this._videoMaterial as any;
if (!this.targetMaterialProperty) {
if (debug && videoMaterial[fieldName] === undefined) {
console.warn(`The target material does not have a '${fieldName}' property, video might not render correctly.`);
}
videoMaterial[fieldName] = this._videoTexture;
}
else {
switch (this.targetMaterialProperty) {
default:
if (debug && videoMaterial[this.targetMaterialProperty] === undefined) {
console.warn(`The target material does not have a '${this.targetMaterialProperty}' property, video might not render correctly.`);
}
videoMaterial[fieldName] = this._videoTexture;
break;
// doesnt render:
// case "emissiveTexture":
// console.log(this.videoMaterial);
// // (this.videoMaterial as any).map = this.videoTexture;
// (this.videoMaterial as any).emissive?.set(1,1,1);// = this.videoTexture;
// (this.videoMaterial as any).emissiveMap = this.videoTexture;
// break;
}
}
}
else {
console.warn("Can not play video, no material found, this might be a multimaterial case which is not supported yet");
return;
}
this.updateVideoElementSettings();
this.updateVideoElementStyles();
if (playAutomatically) {
if (this.shouldUseM3U) {
this.ensureM3UCanBePlayed();
}
this.play();
}
}
private updateVideoElementSettings() {
if (!this._videoElement) return;
this._videoElement.loop = this._isLooping;
this._videoElement.currentTime = this.currentTime;
this._videoElement.playbackRate = this._playbackSpeed;
// dont open in fullscreen on ios
this._videoElement.playsInline = true;
let muted = !this._receivedInput || this.audioOutputMode === VideoAudioOutputMode.None;
if (!muted && this._muted) muted = true;
this._videoElement.muted = muted;
if (this.playOnAwake)
this._videoElement.autoplay = true;
}
private updateVideoElementStyles() {
if (!this._videoElement) return;
// set style here so preview frame is rendered
// set display and selectable because otherwise is interfers with input/focus e.g. breaks orbit control
this._videoElement.style.userSelect = "none";
this._videoElement.style.visibility = "hidden";
this._videoElement.style.display = "none";
this.updateAspect();
}
private _updateAspectRoutineId: number = -1;
private *updateAspectImpl() {
const id = ++this._updateAspectRoutineId;
const lastAspect: number | undefined = undefined;
const stream = this.clip;
while (id === this._updateAspectRoutineId && this.aspectMode !== AspectMode.None && this.clip && stream === this.clip && this._isPlaying) {
if (!stream || typeof stream === "string") {
return;
}
let aspect: number | undefined = undefined;
for (const track of stream.getVideoTracks()) {
const settings = track.getSettings();
if (settings && settings.width && settings.height) {
aspect = settings.width / settings.height;
break;
}
// on firefox capture canvas stream works but looks like
// the canvas stream track doesnt contain settings?!!?
else {
aspect = this.context.renderer.domElement.clientWidth / this.context.renderer.domElement.clientHeight;
}
}
if (aspect === undefined) {
for (let i = 0; i < 10; i++)
yield;
if (!this.isPlaying) break;
continue;
}
if (lastAspect === aspect) {
yield;
continue;
}
for (const obj of this._targetObjects) {
let worldAspect = 1;
if (obj.parent) {
const parentScale = getWorldScale(obj.parent);
worldAspect = parentScale.x / parentScale.y;
}
switch (this.aspectMode) {
case AspectMode.AdjustHeight:
obj.scale.y = 1 / aspect * obj.scale.x * worldAspect;
break;
case AspectMode.AdjustWidth:
obj.scale.x = aspect * obj.scale.y * worldAspect;
break;
}
}
for (let i = 0; i < 3; i++)
yield;
}
}
private get shouldUseM3U(): boolean { return this.url != undefined && (this.url.endsWith(".m3u8") || this.url.endsWith(".m3u")) && this.source === VideoSource.Url; }
private ensureM3UCanBePlayed() {
if (!this.shouldUseM3U) return;
let hls_script = document.head.querySelector("script[data-hls_library]") as HTMLScriptElement;
if (!hls_script) {
if (debug) console.log("HLS: load script");
hls_script = document.createElement("script");
hls_script.dataset["hls_library"] = "hls.js";
hls_script.src = "https://cdn.jsdelivr.net/npm/hls.js@1";
hls_script.addEventListener("load", this.onHlsAvailable);
document.head.append(hls_script);
}
else if (globalThis["Hls"]) {
this.onHlsAvailable();
}
else {
hls_script.addEventListener("load", this.onHlsAvailable);
}
}
private _hls?: Hls;
private onHlsAvailable = () => {
if (debug) console.log("HLS: available", this.clip);
if (!this.shouldUseM3U || !this.url) return;
if (!this._hls)
this._hls = new Hls();
this.videoElement!.autoplay = true;
this._hls.loadSource(this.url);
this._hls.attachMedia(this.videoElement!);
this._videoElement?.play();
if (debug) console.log("HLS: loaded", this.clip);
}
}
declare class Hls {
constructor();
loadSource(url: string);
attachMedia(videoElement: HTMLVideoElement);
}
class VideoOverlay {
readonly context: Context;
constructor(context: Context) {
this.context = context;
this._input = new VideoOverlayInput(this);
}
get enabled() {
return this._isInScreenspaceMode;
}
set enabled(val: boolean) {
if (val) this.start();
else this.stop();
}
add(video: VideoTexture) {
if (this._videos.indexOf(video) === -1) {
this._videos.push(video);
}
}
remove(video: VideoTexture | null | undefined) {
if (!video) return;
const index = this._videos.indexOf(video);
if (index >= 0) {
this._videos.splice(index, 1);
}
}
start() {
if (this._isInScreenspaceMode) return;
if (this._videos.length < 0) return;
const texture = this._videos[this._videos.length - 1];
if (!texture) return;
this._isInScreenspaceMode = true;
if (!this._screenspaceModeQuad) {
this._screenspaceModeQuad = ObjectUtils.createPrimitive(PrimitiveType.Quad, {
material: new ScreenspaceTexture(texture)
});
if (!this._screenspaceModeQuad) return;
this._screenspaceModeQuad.geometry.scale(2, 2, 2);
}
const quad = this._screenspaceModeQuad;
this.context.scene.add(quad);
this.updateScreenspaceMaterialUniforms();
const mat = quad.material as ScreenspaceTexture;
mat?.reset();
this._input?.enable(mat);
}
stop() {
this._isInScreenspaceMode = false;
if (this._screenspaceModeQuad) {
this._input?.disable();
this._screenspaceModeQuad.removeFromParent();
}
}
updateScreenspaceMaterialUniforms() {
const mat = this._screenspaceModeQuad?.material as ScreenspaceTexture;
if (!mat) return;
// mat.videoAspect = this.videoTexture?.image?.width / this.videoTexture?.image?.height;
mat.screenAspect = this.context.domElement.clientWidth / this.context.domElement.clientHeight;
}
private _videos: VideoTexture[] = [];
private _screenspaceModeQuad: Mesh | undefined;
private _isInScreenspaceMode: boolean = false;
private _input: VideoOverlayInput;
}
class VideoOverlayInput {
private _onResizeScreenFn?: () => void;
private _onKeyUpFn?: (e: KeyboardEvent) => void;
private _onMouseWheelFn?: (e: WheelEvent) => void;
private readonly context: Context;
private readonly overlay: VideoOverlay;
constructor(overlay: VideoOverlay) {
this.overlay = overlay;
this.context = overlay.context;
}
private _material?: ScreenspaceTexture;
enable(mat: ScreenspaceTexture) {
this._material = mat;
window.addEventListener("resize", this._onResizeScreenFn = () => {
this.overlay.updateScreenspaceMaterialUniforms();
});
window.addEventListener("keyup", this._onKeyUpFn = (args) => {
if (args.key === "Escape")
this.overlay.stop();
});
window.addEventListener("wheel", this._onMouseWheelFn = (args) => {
if (this.overlay.enabled) {
mat.zoom += args.deltaY * .0005;
args.preventDefault();
}
}, { passive: false });
const delta: Vector2 = new Vector2();
window.addEventListener("mousemove", (args: MouseEvent) => {
if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
const normalizedMovement = new Vector2(args.movementX, args.movementY);
normalizedMovement.x /= this.context.domElement.clientWidth;
normalizedMovement.y /= this.context.domElement.clientHeight;
delta.set(normalizedMovement.x, normalizedMovement.y);
delta.multiplyScalar(mat.zoom / -this.context.time.deltaTime * .01);
mat.offset = mat.offset.add(delta);
}
});
window.addEventListener("pointermove", (args: PointerEvent) => {
if (this.overlay.enabled && this.context.input.getPointerPressed(0)) {
const count = this.context.input.getTouchesPressedCount();
if (count === 1) {
delta.set(args.movementX, args.movementY);
delta.multiplyScalar(mat.zoom * -this.context.time.deltaTime * .05);
mat.offset = mat.offset.add(delta);
}
}
});
let lastTouchStartTime = 0;
window.addEventListener("touchstart", e => {
if (e.touches.length < 2) {
if (this.context.time.time - lastTouchStartTime < .3) {
this.overlay.stop();
}
lastTouchStartTime = this.context.time.time;
return;
}
this._isPinching = true;
this._lastPinch = 0;
})
window.addEventListener("touchmove", e => {
if (!this._isPinching || !this._material) return;
const touch1 = e.touches[0];
const touch2 = e.touches[1];
const dx = touch1.clientX - touch2.clientX;
const dy = touch1.clientY - touch2.clientY;
const distance = Math.sqrt(dx * dx + dy * dy);
if (this._lastPinch !== 0) {
const delta = distance - this._lastPinch;
this._material.zoom -= delta * .004;
}
this._lastPinch = distance;
})
window.addEventListener("touchend", () => {
this._isPinching = false;
})
}
private _isPinching: boolean = false;
private _lastPinch = 0;
disable() {
if (this._onResizeScreenFn) {
window.removeEventListener("resize", this._onResizeScreenFn);
this._onResizeScreenFn = undefined;
}
if (this._onKeyUpFn) {
window.removeEventListener("keyup", this._onKeyUpFn);
this._onKeyUpFn = undefined;
}
if (this._onMouseWheelFn) {
window.removeEventListener("wheel", this._onMouseWheelFn);
this._onMouseWheelFn = undefined;
}
}
}
class ScreenspaceTexture extends ShaderMaterial {
set screenAspect(val: number) {
this.uniforms["screenAspect"].value = val;
this.needsUpdate = true;
}
set offset(vec: Vector2 | { x: number, y: number }) {
const val = this.uniforms["offsetScale"].value;
val.x = vec.x;
val.y = vec.y;
// console.log(val);
this.uniforms["offsetScale"].value = val;
this.needsUpdate = true;
}
private readonly _offset: Vector2 = new Vector2();
get offset(): Vector2 {
const val = this.uniforms["offsetScale"].value;
this._offset.set(val.x, val.y);
return this._offset;
}
set zoom(val: number) {
const zoom = this.uniforms["offsetScale"].value;
if (val < .001) val = .001;
zoom.z = val;
// zoom.z = this.maxZoom - val;
// zoom.z /= this.maxZoom;
this.needsUpdate = true;
}
get zoom(): number {
return this.uniforms["offsetScale"].value.z;// * this.maxZoom;
}
reset() {
this.offset = this.offset.set(0, 0);
this.zoom = 1;
this.needsUpdate = true;
}
// maxZoom : number = 10
constructor(tex: Texture) {
super();
this.uniforms = {
map: { value: tex },
screenAspect: { value: 1 },
offsetScale: { value: new Vector4(0, 0, 1, 1) }
};
this.vertexShader = `
uniform sampler2D map;
uniform float screenAspect;
uniform vec4 offsetScale;
varying vec2 vUv;
void main() {
gl_Position = vec4( position , 1.0 );
vUv = uv;
vUv.y = 1. - vUv.y;
// fit into screen
ivec2 res = textureSize(map, 0);
float videoAspect = float(res.x) / float(res.y);
float aspect = videoAspect / screenAspect;
if(aspect >= 1.0)
{
vUv.y = vUv.y * aspect;
float offset = (1. - aspect) * .5;
vUv.y = vUv.y + offset;
}
else
{
vUv.x = vUv.x / aspect;
float offset = (1. - 1. / aspect) * .5;
vUv.x = vUv.x + offset;
}
vUv.x -= .5;
vUv.y -= .5;
vUv.x *= offsetScale.z;
vUv.y *= offsetScale.z;
vUv.x += offsetScale.x;
vUv.y += offsetScale.y;
vUv.x += .5;
vUv.y += .5;
}
`
this.fragmentShader = `
uniform sampler2D map;
varying vec2 vUv;
void main() {
if(vUv.x < 0. || vUv.x > 1. || vUv.y < 0. || vUv.y > 1.)
gl_FragColor = vec4(0., 0., 0., 1.);
else
{
vec4 texcolor = texture2D(map, vUv);
gl_FragColor = texcolor;
}
}
`
}
}