@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
548 lines • 24.3 kB
JavaScript
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
import { isDevEnvironment, showBalloonWarning } from "../engine/debug/index.js";
import { RoomEvents } from "../engine/engine_networking.js";
import { disposeStream, NetworkedStreamEvents, NetworkedStreams } from "../engine/engine_networking_streams.js";
import { serializable } from "../engine/engine_serialization.js";
import { delay, getParam } from "../engine/engine_utils.js";
import { AudioSource } from "./AudioSource.js";
import { Behaviour, GameObject } from "./Component.js";
import { AspectMode, VideoPlayer } from "./VideoPlayer.js";
const debug = getParam("debugscreensharing");
/**
* ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room.
*/
export var ScreenCaptureDevice;
(function (ScreenCaptureDevice) {
/**
* Capture the screen of the user.
*/
ScreenCaptureDevice[ScreenCaptureDevice["Screen"] = 0] = "Screen";
/**
* Capture the camera of the user.
*/
ScreenCaptureDevice[ScreenCaptureDevice["Camera"] = 1] = "Camera";
/** Please note that canvas streaming might not work reliably on chrome: https://bugs.chromium.org/p/chromium/issues/detail?id=1156408 */
ScreenCaptureDevice[ScreenCaptureDevice["Canvas"] = 2] = "Canvas";
/** When using Microphone only the voice will be send */
ScreenCaptureDevice[ScreenCaptureDevice["Microphone"] = 3] = "Microphone";
})(ScreenCaptureDevice || (ScreenCaptureDevice = {}));
/**
* The current mode of the {@link ScreenCapture} component.
*/
export var ScreenCaptureMode;
(function (ScreenCaptureMode) {
ScreenCaptureMode[ScreenCaptureMode["Idle"] = 0] = "Idle";
ScreenCaptureMode[ScreenCaptureMode["Sending"] = 1] = "Sending";
ScreenCaptureMode[ScreenCaptureMode["Receiving"] = 2] = "Receiving";
})(ScreenCaptureMode || (ScreenCaptureMode = {}));
/**
* The ScreenCapture component allows you to share your screen, camera or microphone with other users in the networked room.
* When the stream is active the video will be displayed on the VideoPlayer component attached to the same GameObject.
*
* Note: For debugging append `?debugscreensharing` to the URL to see more information in the console.
*
* By default the component will start sharing the screen when the user clicks on the object this component is attached to. You can set {@link device} This behaviour can be disabled by setting `allowStartOnClick` to false.
* It is also possible to start the stream manually from your code by calling the {@link share} method.
*
* @category Networking
* @group Components
*/
export class ScreenCapture extends Behaviour {
/**
* When enabled the stream will start when the user clicks on the object this component is attached to
* It is also possible to start the stream manually from your code by calling the {@link share} method
* To modify what type of device is shared you can set the {@link device} property.
* @default true
*/
allowStartOnClick = true;
/** @internal */
onPointerEnter() {
if (this.context.connection.allowEditing == false)
return;
if (!this.allowStartOnClick)
return;
this.context.input.setCursor("pointer");
}
/** @internal */
onPointerExit() {
if (this.context.connection.allowEditing == false)
return;
if (!this.allowStartOnClick)
return;
this.context.input.unsetCursor("pointer");
}
/** @internal */
onPointerClick(evt) {
if (this.context.connection.allowEditing == false)
return;
if (!this.allowStartOnClick)
return;
if (evt && evt.pointerId !== 0)
return;
if (this.isReceiving && this.videoPlayer?.isPlaying) {
if (this.videoPlayer)
this.videoPlayer.screenspace = !this.videoPlayer.screenspace;
return;
}
if (this.isSending) {
this.close();
return;
}
this.share();
}
/** When enabled the stream will start when this component becomes active (enabled in the scene) */
autoConnect = false;
/**
* If a VideoPlayer component is assigned to this property the video will be displayed on the VideoPlayer component.
*/
set videoPlayer(val) {
if (this._videoPlayer && (this.isSending || this.isReceiving)) {
this._videoPlayer.stop();
}
this._videoPlayer = val;
if (this._videoPlayer && this._currentStream && (this.isSending || this.isReceiving)) {
this._videoPlayer.setVideo(this._currentStream);
}
}
get videoPlayer() { return this._videoPlayer; }
_videoPlayer;
_audioSource;
/**
* When enabled the video will be displayed in the screenspace of the VideoPlayer component.
*/
get screenspace() { return this.videoPlayer?.screenspace ?? false; }
set screenspace(v) { if (this.videoPlayer)
this.videoPlayer.screenspace = v; }
/**
* Which streaming device type should be used when starting to share (if {@link share} is called without a device option). Options are Screen, Camera, Microphone.
* This is e.g. used if `allowStartOnClick` is enabled and the user clicks on the object.
* @default Screen
*/
device = "Screen";
/**
* If assigned the device the device will be selected by this id or label when starting to share.
* Note: This is only supported for `Camera` devices
*/
deviceName;
/**
* Filter which device should be chosen for sharing by id or label.
* Assign a method to this property to manually filter the available devices.
*/
deviceFilter;
/**
* the current stream that is being shared or received
* @link https://developer.mozilla.org/en-US/docs/Web/API/MediaStream
*/
get currentScream() {
return this._currentStream;
}
get currentMode() {
return this._currentMode;
}
/**
* @returns true if the component is currently sending a stream
*/
get isSending() {
return this._currentStream?.active && this._currentMode === ScreenCaptureMode.Sending;
}
/**
* @returns true if the component is currently receiving a stream
*/
get isReceiving() {
if (this._currentMode === ScreenCaptureMode.Receiving) {
if (!this._currentStream || this._currentStream.active === false)
return false;
// if any track is still live consider it active
const tracks = this._currentStream.getTracks();
for (const track of tracks) {
if (track.readyState === "live")
return true;
}
}
return false;
}
get requiresVideoPlayer() {
return this.device !== "Microphone";
}
_net;
_requestOpen = false;
_currentStream = null;
_currentMode = ScreenCaptureMode.Idle;
/** @internal */
awake() {
// Resolve the device type if it is a number
if (typeof this.device === "number") {
this.device = ScreenCaptureDevice[this.device];
}
if (debug)
console.log("Screensharing", this.name, this);
AudioSource.registerWaitForAllowAudio(() => {
if (this._videoPlayer && this._currentStream && this._currentMode === ScreenCaptureMode.Receiving) {
this._videoPlayer.playInBackground = true;
this._videoPlayer.setVideo(this._currentStream);
}
});
this._net = new NetworkedStreams(this);
}
/** @internal */
onEnable() {
this._net?.enable();
//@ts-ignore
this._net?.addEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
//@ts-ignore
this._net?.addEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded);
this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
if (this.autoConnect) {
delay(1000).then(() => {
if (this.enabled && this.autoConnect && !this.isReceiving && !this.isSending && this.context.connection.isInRoom)
this.share();
return 0;
});
}
}
/** @internal */
onDisable() {
//@ts-ignore
this._net?.removeEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
//@ts-ignore
this._net?.removeEventListener(NetworkedStreamEvents.StreamEnded, this.onCallEnded);
this.context.connection.stopListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
this._net?.disable();
this.close();
}
onJoinedRoom = async () => {
await delay(1000);
if (this.autoConnect && !this.isSending && !this.isReceiving && this.context.connection.isInRoom) {
this.share();
}
};
_ensureVideoPlayer() {
const vp = new VideoPlayer();
vp.aspectMode = AspectMode.AdjustWidth;
GameObject.addComponent(this.gameObject, vp);
this._videoPlayer = vp;
}
_activeShareRequest = null;
/** Call to begin screensharing */
async share(opts) {
if (this._activeShareRequest)
return this._activeShareRequest;
this._activeShareRequest = this.internalShare(opts);
return this._activeShareRequest.then(() => {
return this._activeShareRequest = null;
});
}
async internalShare(opts) {
if (this.context.connection.isInRoom === false) {
console.warn("Can not start screensharing: requires network connection");
if (isDevEnvironment())
showBalloonWarning("Can not start screensharing: requires network connection. Add a SyncedRoom component or join a room first.");
return;
}
if (opts?.device)
this.device = opts.device;
if (!this.videoPlayer && this.requiresVideoPlayer) {
if (!this._videoPlayer) {
this._videoPlayer = GameObject.getComponent(this.gameObject, VideoPlayer) ?? undefined;
}
if (!this.videoPlayer) {
this._ensureVideoPlayer();
}
if (!this.videoPlayer) {
console.warn("Can not share video without a videoPlayer assigned");
return;
}
}
this._requestOpen = true;
try {
const settings = opts?.constraints ?? {
echoCancellation: true,
autoGainControl: false,
};
const displayMediaOptions = {
video: settings,
audio: settings,
};
const videoOptions = displayMediaOptions.video;
if (videoOptions !== undefined && typeof videoOptions !== "boolean") {
// Set default video settings
if (!videoOptions.width)
videoOptions.width = { max: 1920 };
if (!videoOptions.height)
videoOptions.height = { max: 1920 };
if (!videoOptions.aspectRatio)
videoOptions.aspectRatio = { ideal: 1.7777777778 };
if (!videoOptions.frameRate)
videoOptions.frameRate = { ideal: 24 };
if (!videoOptions.facingMode)
videoOptions.facingMode = { ideal: "user" };
}
switch (this.device) {
// Capture a connected camera
case "Camera":
this.tryShareUserCamera(displayMediaOptions, opts);
break;
// capture any screen, will show a popup
case "Screen":
{
if (!navigator.mediaDevices.getDisplayMedia) {
console.error("No getDisplayMedia support");
return;
}
const myVideo = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
if (this._requestOpen) {
this.setStream(myVideo, ScreenCaptureMode.Sending);
}
else
disposeStream(myVideo);
}
break;
// capture the canvas meaning the threejs view
case "Canvas":
// looks like this doesnt work reliably on chrome https://stackoverflow.com/a/66848674
// firefox updates fine
// https://bugs.chromium.org/p/chromium/issues/detail?id=1156408
const fps = 0;
const stream = this.context.renderer.domElement.captureStream(fps);
this.setStream(stream, ScreenCaptureMode.Sending);
break;
case "Microphone":
{
if (!navigator.mediaDevices.getUserMedia) {
console.error("No getDisplayMedia support");
return;
}
displayMediaOptions.video = false;
const myStream = await navigator.mediaDevices.getUserMedia(displayMediaOptions);
if (this._requestOpen) {
this.setStream(myStream, ScreenCaptureMode.Sending);
}
else
disposeStream(myStream);
}
break;
default:
console.error("Can not start screen sharing: Unknown device type", this.device);
}
}
catch (err) {
if (err.name === "NotAllowedError") {
// user cancelled stream selection
console.log("Selection cancelled");
this._requestOpen = false;
return;
}
console.error("Error opening video", err);
}
}
close() {
this._requestOpen = false;
if (this._currentStream) {
if (debug)
console.warn("Close current stream / disposing resources, stream was active?", this._currentStream.active);
this._net?.stopSendingStream(this._currentStream);
disposeStream(this._currentStream);
this._currentMode = ScreenCaptureMode.Idle;
this._currentStream = null;
}
}
setStream(stream, mode) {
if (stream === this._currentStream)
return;
this.close();
if (!stream)
return;
this._currentStream = stream;
this._requestOpen = true;
this._currentMode = mode;
const isVideoStream = this.device !== "Microphone";
const isSending = mode === ScreenCaptureMode.Sending;
if (isVideoStream) {
if (!this._videoPlayer)
this._ensureVideoPlayer();
if (this._videoPlayer)
this._videoPlayer.setVideo(stream);
else
console.error("No video player assigned for video stream");
}
else {
if (!this._audioSource) {
this._audioSource = new AudioSource();
this._audioSource.spatialBlend = 0;
this._audioSource.volume = 1;
this.gameObject.addComponent(this._audioSource);
}
if (!isSending) {
if (debug)
console.log("PLAY", stream.getAudioTracks());
this._audioSource.volume = 1;
this._audioSource?.play(stream);
}
}
if (isSending) {
this._net?.startSendingStream(stream);
}
// Mute audio for the video we are sending
if (isSending) {
if (this._videoPlayer)
this._videoPlayer.muted = true;
this._audioSource?.stop();
}
for (const track of stream.getTracks()) {
track.addEventListener("ended", () => {
if (debug)
console.log("Track ended", track);
this.close();
});
if (debug) {
if (track.kind === "video") {
if (isSending)
console.log("Video →", track.getSettings());
else
console.log("Video ←", track.getSettings());
}
}
}
}
onReceiveStream = (evt) => {
if (evt.stream?.active !== true)
return;
this.setStream(evt.stream, ScreenCaptureMode.Receiving);
};
onCallEnded = (_evt) => {
if (debug)
console.log("CALL ENDED", this.isReceiving, this?.screenspace);
if (this.isReceiving)
this.screenspace = false;
};
async tryShareUserCamera(constraints, options) {
// let newWindow = open('', 'example', 'width=300,height=300');
// if (window) {
// newWindow!.document.body.innerHTML = "Please allow access to your camera and microphone";
// }
// TODO: allow user to select device
const devices = (await navigator.mediaDevices.enumerateDevices()).filter(d => d.kind === "videoinput");
if (debug)
console.log("Request camera. These are your kind:videoinput devices:\n", devices);
let foundDevice = false;
for (const dev of devices) {
try {
if (!this._requestOpen) {
if (debug)
console.log("Camera selection cancelled");
break;
}
if (dev.kind !== "videoinput") {
if (debug)
console.log("Skipping non-video device", dev);
continue;
}
const id = dev.deviceId;
// If the share method is called with filter options then those should be used
const hasOptionsFilter = options?.deviceId != undefined || options?.deviceFilter != undefined;
if (hasOptionsFilter) {
if (options?.deviceId !== undefined) {
if (id !== options.deviceId) {
if (debug)
console.log("Skipping device due to options.deviceId: " + dev.label + "; " + dev.deviceId);
continue;
}
}
if (options?.deviceFilter) {
const useDevice = options.deviceFilter(dev);
if (useDevice === false) {
if (debug)
console.log("Skipping device due to options.deviceFilter: " + dev.label + "; " + dev.deviceId);
continue;
}
}
}
// If the share method was called without filter options then the component filter should be used
else if (this.deviceFilter) {
const useDevice = this.deviceFilter(dev);
if (useDevice === false) {
if (debug)
console.log("Skipping device due to ScreenShare.deviceFilter: " + dev.label + "; " + dev.deviceId);
continue;
}
else if (debug)
console.log("Selected device by filter", dev);
}
else if (this.deviceName) {
const lowercaseLabel = dev.label.toLowerCase();
const lowercaseName = this.deviceName.toLowerCase();
const labelMatches = lowercaseLabel.includes(lowercaseName);
const idMatches = dev.deviceId === this.deviceName;
if (!labelMatches && !idMatches) {
if (debug)
console.log("Skipping device due to ScreenShare.deviceName: " + dev.label + "; " + dev.deviceId);
continue;
}
else if (debug)
console.log("Selected device by name", dev);
}
if (constraints.video !== false) {
if (typeof constraints.video === "undefined" || typeof constraints.video === "boolean") {
constraints.video = {};
}
constraints.video.deviceId = id;
}
foundDevice = true;
const userMedia = await navigator.mediaDevices.getUserMedia(constraints).catch(err => {
console.error("Failed to get user media", err);
return null;
});
if (userMedia === null) {
continue;
}
else if (this._requestOpen) {
this.setStream(userMedia, ScreenCaptureMode.Sending);
if (debug)
console.log("Selected camera", dev);
}
else {
disposeStream(userMedia);
if (debug)
console.log("Camera selection cancelled");
}
break;
}
catch (err) {
// First message is firefox, second is chrome when the video source is already in use by another app
if (err.message === "Failed to allocate videosource" || err.message === "Could not start video source") {
showBalloonWarning("Failed to start video: Try another camera (Code " + err.code + ")");
console.warn(err);
continue;
}
else {
console.error("Failed to get user media", err.message, err.code, err);
}
}
}
if (!foundDevice && isDevEnvironment()) {
showBalloonWarning("No camera found for sharing. Please connect a camera (see console for more information)");
console.warn("No camera found for sharing. Please connect a camera", devices, this.deviceName, "Using deviceFilter? " + this.deviceFilter != undefined, "Using options? " + options != undefined, "Using deviceName? " + this.deviceName != undefined, "Using options.deviceId? " + options?.deviceId != undefined, "Using options.deviceFilter? " + options?.deviceFilter != undefined);
}
}
}
__decorate([
serializable()
], ScreenCapture.prototype, "allowStartOnClick", void 0);
__decorate([
serializable()
], ScreenCapture.prototype, "autoConnect", void 0);
__decorate([
serializable(VideoPlayer)
], ScreenCapture.prototype, "videoPlayer", null);
__decorate([
serializable()
], ScreenCapture.prototype, "device", void 0);
__decorate([
serializable()
], ScreenCapture.prototype, "deviceName", void 0);
//# sourceMappingURL=ScreenCapture.js.map