@babylonjs/core
Version:
Getting started? Play directly with the Babylon.js API using our [playground](https://playground.babylonjs.com/). It also contains a lot of samples to learn how to use it.
1,084 lines (1,083 loc) • 138 kB
JavaScript
/** This file must only contain pure code and pure imports */
import { SubMesh } from "../subMesh.pure.js";
import { Mesh } from "../mesh.pure.js";
import { VertexData } from "../mesh.vertexData.js";
import { Matrix, TmpVectors, Vector2, Vector3 } from "../../Maths/math.vector.pure.js";
import { Logger } from "../../Misc/logger.js";
import { Observable } from "../../Misc/observable.js";
import { GaussianSplattingMaterial } from "../../Materials/GaussianSplatting/gaussianSplattingMaterial.pure.js";
import { RawTexture } from "../../Materials/Textures/rawTexture.js";
import { ToHalfFloat } from "../../Misc/textureTools.js";
import { Scalar } from "../../Maths/math.scalar.js";
import { runCoroutineSync, runCoroutineAsync, createYieldingScheduler } from "../../Misc/coroutine.js";
import { EngineStore } from "../../Engines/engineStore.js";
import { ImportMeshAsync } from "../../Loading/sceneLoader.js";
import { GaussianSplattingSortWorker, GaussianSplattingSortWorkerCommand } from "./gaussianSplattingSortWorker.js";
const IsNative = typeof _native !== "undefined";
const Native = IsNative ? _native : null;
// @internal
const UnpackUnorm = (value, bits) => {
const t = (1 << bits) - 1;
return (value & t) / t;
};
// @internal
const Unpack111011 = (value, result) => {
result.x = UnpackUnorm(value >>> 21, 11);
result.y = UnpackUnorm(value >>> 11, 10);
result.z = UnpackUnorm(value, 11);
};
// @internal
const Unpack8888 = (value, result) => {
result[0] = UnpackUnorm(value >>> 24, 8) * 255;
result[1] = UnpackUnorm(value >>> 16, 8) * 255;
result[2] = UnpackUnorm(value >>> 8, 8) * 255;
result[3] = UnpackUnorm(value, 8) * 255;
};
// @internal
// unpack quaternion with 2,10,10,10 format (largest element, 3x10bit element)
const UnpackRot = (value, result) => {
const norm = 1.0 / (Math.sqrt(2) * 0.5);
const a = (UnpackUnorm(value >>> 20, 10) - 0.5) * norm;
const b = (UnpackUnorm(value >>> 10, 10) - 0.5) * norm;
const c = (UnpackUnorm(value, 10) - 0.5) * norm;
const m = Math.sqrt(1.0 - (a * a + b * b + c * c));
switch (value >>> 30) {
case 0:
result.set(m, a, b, c);
break;
case 1:
result.set(a, m, b, c);
break;
case 2:
result.set(a, b, m, c);
break;
case 3:
result.set(a, b, c, m);
break;
}
};
/**
* Representation of the types
*/
var PLYType;
(function (PLYType) {
PLYType[PLYType["FLOAT"] = 0] = "FLOAT";
PLYType[PLYType["INT"] = 1] = "INT";
PLYType[PLYType["UINT"] = 2] = "UINT";
PLYType[PLYType["DOUBLE"] = 3] = "DOUBLE";
PLYType[PLYType["UCHAR"] = 4] = "UCHAR";
PLYType[PLYType["UNDEFINED"] = 5] = "UNDEFINED";
})(PLYType || (PLYType = {}));
/**
* Usage types of the PLY values
*/
var PLYValue;
(function (PLYValue) {
PLYValue[PLYValue["MIN_X"] = 0] = "MIN_X";
PLYValue[PLYValue["MIN_Y"] = 1] = "MIN_Y";
PLYValue[PLYValue["MIN_Z"] = 2] = "MIN_Z";
PLYValue[PLYValue["MAX_X"] = 3] = "MAX_X";
PLYValue[PLYValue["MAX_Y"] = 4] = "MAX_Y";
PLYValue[PLYValue["MAX_Z"] = 5] = "MAX_Z";
PLYValue[PLYValue["MIN_SCALE_X"] = 6] = "MIN_SCALE_X";
PLYValue[PLYValue["MIN_SCALE_Y"] = 7] = "MIN_SCALE_Y";
PLYValue[PLYValue["MIN_SCALE_Z"] = 8] = "MIN_SCALE_Z";
PLYValue[PLYValue["MAX_SCALE_X"] = 9] = "MAX_SCALE_X";
PLYValue[PLYValue["MAX_SCALE_Y"] = 10] = "MAX_SCALE_Y";
PLYValue[PLYValue["MAX_SCALE_Z"] = 11] = "MAX_SCALE_Z";
PLYValue[PLYValue["PACKED_POSITION"] = 12] = "PACKED_POSITION";
PLYValue[PLYValue["PACKED_ROTATION"] = 13] = "PACKED_ROTATION";
PLYValue[PLYValue["PACKED_SCALE"] = 14] = "PACKED_SCALE";
PLYValue[PLYValue["PACKED_COLOR"] = 15] = "PACKED_COLOR";
PLYValue[PLYValue["X"] = 16] = "X";
PLYValue[PLYValue["Y"] = 17] = "Y";
PLYValue[PLYValue["Z"] = 18] = "Z";
PLYValue[PLYValue["SCALE_0"] = 19] = "SCALE_0";
PLYValue[PLYValue["SCALE_1"] = 20] = "SCALE_1";
PLYValue[PLYValue["SCALE_2"] = 21] = "SCALE_2";
PLYValue[PLYValue["DIFFUSE_RED"] = 22] = "DIFFUSE_RED";
PLYValue[PLYValue["DIFFUSE_GREEN"] = 23] = "DIFFUSE_GREEN";
PLYValue[PLYValue["DIFFUSE_BLUE"] = 24] = "DIFFUSE_BLUE";
PLYValue[PLYValue["OPACITY"] = 25] = "OPACITY";
PLYValue[PLYValue["F_DC_0"] = 26] = "F_DC_0";
PLYValue[PLYValue["F_DC_1"] = 27] = "F_DC_1";
PLYValue[PLYValue["F_DC_2"] = 28] = "F_DC_2";
PLYValue[PLYValue["F_DC_3"] = 29] = "F_DC_3";
PLYValue[PLYValue["ROT_0"] = 30] = "ROT_0";
PLYValue[PLYValue["ROT_1"] = 31] = "ROT_1";
PLYValue[PLYValue["ROT_2"] = 32] = "ROT_2";
PLYValue[PLYValue["ROT_3"] = 33] = "ROT_3";
PLYValue[PLYValue["MIN_COLOR_R"] = 34] = "MIN_COLOR_R";
PLYValue[PLYValue["MIN_COLOR_G"] = 35] = "MIN_COLOR_G";
PLYValue[PLYValue["MIN_COLOR_B"] = 36] = "MIN_COLOR_B";
PLYValue[PLYValue["MAX_COLOR_R"] = 37] = "MAX_COLOR_R";
PLYValue[PLYValue["MAX_COLOR_G"] = 38] = "MAX_COLOR_G";
PLYValue[PLYValue["MAX_COLOR_B"] = 39] = "MAX_COLOR_B";
PLYValue[PLYValue["SH_0"] = 40] = "SH_0";
PLYValue[PLYValue["SH_1"] = 41] = "SH_1";
PLYValue[PLYValue["SH_2"] = 42] = "SH_2";
PLYValue[PLYValue["SH_3"] = 43] = "SH_3";
PLYValue[PLYValue["SH_4"] = 44] = "SH_4";
PLYValue[PLYValue["SH_5"] = 45] = "SH_5";
PLYValue[PLYValue["SH_6"] = 46] = "SH_6";
PLYValue[PLYValue["SH_7"] = 47] = "SH_7";
PLYValue[PLYValue["SH_8"] = 48] = "SH_8";
PLYValue[PLYValue["SH_9"] = 49] = "SH_9";
PLYValue[PLYValue["SH_10"] = 50] = "SH_10";
PLYValue[PLYValue["SH_11"] = 51] = "SH_11";
PLYValue[PLYValue["SH_12"] = 52] = "SH_12";
PLYValue[PLYValue["SH_13"] = 53] = "SH_13";
PLYValue[PLYValue["SH_14"] = 54] = "SH_14";
PLYValue[PLYValue["SH_15"] = 55] = "SH_15";
PLYValue[PLYValue["SH_16"] = 56] = "SH_16";
PLYValue[PLYValue["SH_17"] = 57] = "SH_17";
PLYValue[PLYValue["SH_18"] = 58] = "SH_18";
PLYValue[PLYValue["SH_19"] = 59] = "SH_19";
PLYValue[PLYValue["SH_20"] = 60] = "SH_20";
PLYValue[PLYValue["SH_21"] = 61] = "SH_21";
PLYValue[PLYValue["SH_22"] = 62] = "SH_22";
PLYValue[PLYValue["SH_23"] = 63] = "SH_23";
PLYValue[PLYValue["SH_24"] = 64] = "SH_24";
PLYValue[PLYValue["SH_25"] = 65] = "SH_25";
PLYValue[PLYValue["SH_26"] = 66] = "SH_26";
PLYValue[PLYValue["SH_27"] = 67] = "SH_27";
PLYValue[PLYValue["SH_28"] = 68] = "SH_28";
PLYValue[PLYValue["SH_29"] = 69] = "SH_29";
PLYValue[PLYValue["SH_30"] = 70] = "SH_30";
PLYValue[PLYValue["SH_31"] = 71] = "SH_31";
PLYValue[PLYValue["SH_32"] = 72] = "SH_32";
PLYValue[PLYValue["SH_33"] = 73] = "SH_33";
PLYValue[PLYValue["SH_34"] = 74] = "SH_34";
PLYValue[PLYValue["SH_35"] = 75] = "SH_35";
PLYValue[PLYValue["SH_36"] = 76] = "SH_36";
PLYValue[PLYValue["SH_37"] = 77] = "SH_37";
PLYValue[PLYValue["SH_38"] = 78] = "SH_38";
PLYValue[PLYValue["SH_39"] = 79] = "SH_39";
PLYValue[PLYValue["SH_40"] = 80] = "SH_40";
PLYValue[PLYValue["SH_41"] = 81] = "SH_41";
PLYValue[PLYValue["SH_42"] = 82] = "SH_42";
PLYValue[PLYValue["SH_43"] = 83] = "SH_43";
PLYValue[PLYValue["SH_44"] = 84] = "SH_44";
PLYValue[PLYValue["SH_45"] = 85] = "SH_45";
PLYValue[PLYValue["SH_46"] = 86] = "SH_46";
PLYValue[PLYValue["SH_47"] = 87] = "SH_47";
PLYValue[PLYValue["SH_48"] = 88] = "SH_48";
PLYValue[PLYValue["SH_49"] = 89] = "SH_49";
PLYValue[PLYValue["SH_50"] = 90] = "SH_50";
PLYValue[PLYValue["SH_51"] = 91] = "SH_51";
PLYValue[PLYValue["SH_52"] = 92] = "SH_52";
PLYValue[PLYValue["SH_53"] = 93] = "SH_53";
PLYValue[PLYValue["SH_54"] = 94] = "SH_54";
PLYValue[PLYValue["SH_55"] = 95] = "SH_55";
PLYValue[PLYValue["SH_56"] = 96] = "SH_56";
PLYValue[PLYValue["SH_57"] = 97] = "SH_57";
PLYValue[PLYValue["SH_58"] = 98] = "SH_58";
PLYValue[PLYValue["SH_59"] = 99] = "SH_59";
PLYValue[PLYValue["SH_60"] = 100] = "SH_60";
PLYValue[PLYValue["SH_61"] = 101] = "SH_61";
PLYValue[PLYValue["SH_62"] = 102] = "SH_62";
PLYValue[PLYValue["SH_63"] = 103] = "SH_63";
PLYValue[PLYValue["SH_64"] = 104] = "SH_64";
PLYValue[PLYValue["SH_65"] = 105] = "SH_65";
PLYValue[PLYValue["SH_66"] = 106] = "SH_66";
PLYValue[PLYValue["SH_67"] = 107] = "SH_67";
PLYValue[PLYValue["SH_68"] = 108] = "SH_68";
PLYValue[PLYValue["SH_69"] = 109] = "SH_69";
PLYValue[PLYValue["SH_70"] = 110] = "SH_70";
PLYValue[PLYValue["SH_71"] = 111] = "SH_71";
PLYValue[PLYValue["UNDEFINED"] = 112] = "UNDEFINED";
})(PLYValue || (PLYValue = {}));
function _AcquireGsInterFrameYield(engine) {
// Browser-only optimization: wraps the global requestAnimationFrame, which
// doesn't exist on Babylon Native. Skip it there so the engine uses its
// default frame scheduling.
if (IsNative) {
return;
}
const existing = engine.customAnimationFrameRequester;
if (existing?._gsInterFrameYield) {
existing._refCount++;
return;
}
if (existing) {
// Slot is owned by another requester. Don't interfere.
return;
}
let _timeoutId = 0;
let _innerRafId = 0;
const wrapper = {
_gsInterFrameYield: true,
_refCount: 1,
requestAnimationFrame: (callback) => {
// Insert a setTimeout(0) to yield to the regular task queue (worker
// postMessage results, input events) before scheduling the next animation
// frame. Without this, a continuous rAF loop at high refresh rates leaves
// almost no time for regular tasks, starving worker messages for hundreds of ms.
_timeoutId = setTimeout(() => {
_innerRafId = requestAnimationFrame(callback);
}, 0);
return _timeoutId;
},
cancelAnimationFrame: (_id) => {
clearTimeout(_timeoutId);
if (_innerRafId > 0) {
cancelAnimationFrame(_innerRafId);
}
_timeoutId = 0;
_innerRafId = 0;
},
};
engine.customAnimationFrameRequester = wrapper;
}
// Counterpart to _AcquireGsInterFrameYield. Call once per mesh on dispose.
function _ReleaseGsInterFrameYield(engine) {
const existing = engine.customAnimationFrameRequester;
if (!existing?._gsInterFrameYield) {
return;
}
existing._refCount--;
if (existing._refCount === 0) {
engine.customAnimationFrameRequester = null;
}
}
/**
* Base class for Gaussian Splatting meshes. Contains all single-cloud rendering logic.
* @internal Use GaussianSplattingMesh instead; this class is an internal implementation detail.
*/
export class GaussianSplattingMeshBase extends Mesh {
/**
* Returns a byte-accurate view for retained splat data, preserving any non-zero byte offset.
* @param data The retained splat source bytes.
* @returns A Uint8Array covering the exact source byte range.
* @internal
*/
static _GetSplatDataBytes(data) {
return ArrayBuffer.isView(data) ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) : new Uint8Array(data);
}
/**
* Returns a Float32 reinterpretation for retained splat data, copying only when alignment requires it.
* @param data The retained splat source bytes.
* @returns A Float32Array over the exact source byte range.
* @internal
*/
static _GetSplatDataFloats(data) {
const bytes = GaussianSplattingMeshBase._GetSplatDataBytes(data);
const floatSize = Float32Array.BYTES_PER_ELEMENT;
if (bytes.byteLength % floatSize !== 0) {
throw new Error(`Gaussian splat data byte length (${bytes.byteLength}) is not divisible by ${floatSize} and cannot be reinterpreted as Float32 data.`);
}
if (bytes.byteOffset % floatSize !== 0) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return new Float32Array(copy.buffer, 0, bytes.byteLength / floatSize);
}
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / floatSize);
}
static _BuildSplatRangeData(ranges, vertexCount) {
if (ranges === null) {
return { ranges: null, count: vertexCount, key: "" };
}
const rangePairs = [];
let totalCount = 0;
let key = "";
for (const range of ranges) {
const start = Math.max(0, Math.floor(range.offset));
const count = Math.max(0, Math.floor(range.count));
const end = Math.min(vertexCount, start + count);
const rangeCount = Math.max(0, end - start);
if (rangeCount === 0) {
continue;
}
rangePairs.push(start, rangeCount);
totalCount += rangeCount;
key += `${start}:${rangeCount};`;
}
return { ranges: new Uint32Array(rangePairs), count: totalCount, key };
}
/**
* If true, disables depth sorting of the splats (default: false)
*/
get disableDepthSort() {
return this._disableDepthSort;
}
set disableDepthSort(value) {
if (!this._disableDepthSort && value) {
this._worker?.terminate();
this._worker = null;
this._disableDepthSort = true;
}
else if (this._disableDepthSort && !value) {
this._disableDepthSort = false;
this._sortIsDirty = true;
this._instantiateWorker();
}
}
/**
* View direction factor used to compute the SH view direction in the shader.
* @deprecated Not used anymore for SH rendering
*/
get viewDirectionFactor() {
return Vector3.OneReadOnly;
}
/**
* SH degree. 0 = no sh (default). 1 = 3 parameters. 2 = 8 parameters. 3 = 15 parameters.
* Value is clamped between 0 and the maximum degree available from loaded data.
*/
get shDegree() {
return this._shDegree;
}
set shDegree(value) {
const maxDegree = this._maxShDegree;
const clamped = Math.max(0, Math.min(Math.round(value), maxDegree));
if (this._shDegree === clamped) {
return;
}
this._shDegree = clamped;
this.material?.resetDrawCache();
}
/**
* Maximum SH degree available from the loaded data.
*/
get maxShDegree() {
return this._maxShDegree;
}
/**
* Number of splats in the mesh
*/
get splatCount() {
return this._splatIndex?.length;
}
/**
* Number of source splats currently selected for rendering.
* When no range filter is active, this is the mesh's full source splat count.
*/
get renderedSplatCount() {
return this._activeSplatRanges ? this._activeSplatRenderCount : this._vertexCount;
}
/**
* Restricts rendering to the provided source splat ranges.
* Passing `null` clears the range filter and renders the full source splat set.
* @param ranges contiguous source ranges to render, or null to render all splats
*/
setSplatIndexRanges(ranges) {
this._setSplatIndexRanges(ranges);
}
_setSplatIndexRanges(ranges) {
const rangeData = GaussianSplattingMeshBase._BuildSplatRangeData(ranges, this._vertexCount);
const isSameRangeSet = this._activeSplatRanges === null ? rangeData.ranges === null : rangeData.ranges !== null && this._activeSplatRangeKey === rangeData.key;
if (isSameRangeSet) {
return;
}
this._activeSplatRanges = rangeData.ranges;
this._activeSplatRangeKey = rangeData.key;
this._activeSplatRenderCount = rangeData.count;
this._activeRangeVersion++;
this._postIntervalsToWorker();
if (this._worker) {
// Defer swapping the rendered index buffer until the worker returns a depth sort computed for
// this exact range set (see the worker `onmessage` handler). Until then we keep rendering the
// previous fully-sorted state, so a range/LOD change never shows an unsorted flash nor a stale
// index buffer (rendering indices that belong to a different active set, which caused the
// disappear/reappear flicker). Only the worker's depth buffer is resized here so the next sort
// produces the right number of entries.
this._ensureDepthMixSize(Math.max((this._activeSplatRenderCount + 15) & ~0xf, 16));
this._sortIsDirty = true;
this._postToWorker(true);
}
else {
// Synchronous sort paths (disabled depth sort / Native): swap the index buffer immediately.
this._updateSplatIndexBuffer(this._vertexCount);
this._postToWorker(true);
}
}
/**
* Whether the depth sort is settled: a sort computed for the current active ranges and camera has been
* applied to the rendered index buffer, and no further sort is pending or in flight. For a static camera
* and a fixed active set this becomes true once the final sort completes. Used by streaming subclasses to
* detect when rendering is fully up to date (e.g. for deterministic screenshots) and by IBL shadows to
* avoid voxelizing against an index buffer the worker has not finished (re)building yet.
* @internal
*/
get _isDepthSortSettled() {
return this._readyToDisplay && !this._sortIsDirty && this._canPostToWorker;
}
// (Re)allocates the worker depth buffer to the given padded size. A fresh array is allocated when the
// size differs or the current buffer is detached (in-flight in the worker), so a queued sort can be
// re-posted with a correctly-sized buffer without disturbing the in-flight one.
_ensureDepthMixSize(paddedCount) {
if (IsNative) {
return;
}
if (!this._depthMix || this._depthMix.length !== paddedCount || this._depthMix.buffer.byteLength === 0) {
this._depthMix = new BigInt64Array(paddedCount);
}
}
/**
* Sends the active source-splat intervals to the sort worker. When no range filter is active,
* a single interval covering all source splats is sent so the worker never assumes the full set.
*/
_postIntervalsToWorker() {
if (!this._worker) {
return;
}
const intervals = this._activeSplatRanges ? new Uint32Array(this._activeSplatRanges) : new Uint32Array([0, this._vertexCount]);
this._worker.postMessage({ command: GaussianSplattingSortWorkerCommand.INTERVALS, intervals }, [intervals.buffer]);
}
/**
* Initializes this mesh to render from an externally-provided, GPU-decoded work buffer, bypassing the
* CPU `updateData` path. The four textures must hold the standard decoded GS layout addressed by linear
* splat index over a square texture: centers (x,y,z,1), covariance A (Sigma00,01,02,11) and B
* (Sigma12,22,*,*) as full covariance (so `center.w` is 1), and RGBA color. `splatPositions` are the
* stride-4 CPU centers consumed by the depth-sort worker.
* @param centers centers texture
* @param covariancesA covariance A texture
* @param covariancesB covariance B texture
* @param colors color texture
* @param splatPositions stride-4 CPU centers for depth sorting (length vertexCount*4)
* @param vertexCount number of splats addressable in the work buffer
*/
_setExternalWorkBuffer(centers, covariancesA, covariancesB, colors, splatPositions, vertexCount) {
this._covariancesATexture = covariancesA;
this._covariancesBTexture = covariancesB;
this._centersTexture = centers;
this._colorsTexture = colors;
this._splatPositions = splatPositions;
this._vertexCount = vertexCount;
this._activeSplatRanges = null;
this._activeSplatRangeKey = "";
this._activeSplatRenderCount = 0;
this._readyToDisplay = false;
// Sizes _splatIndex/_depthMix, starts the sort worker, and posts positions + intervals.
this._instantiateWorker();
// updateData (bypassed here) normally enables the mesh; do it explicitly for the work-buffer path.
this.setEnabled(true);
}
/**
* returns the splats data array buffer that contains in order : postions (3 floats), size (3 floats), color (4 bytes), orientation quaternion (4 bytes)
* Only available if the mesh was created with keepInRam: true
*/
get splatsData() {
return this._keepInRam ? this._splatsData : null;
}
/**
* returns the SH data arrays
* Only available if the mesh was created with keepInRam: true
*/
get shData() {
return this._keepInRam ? this._shData : null;
}
/**
* Returns the min/max size range of splats in this mesh, where size is pow(|det(Σ)|, 1/6)
* of the 3D covariance matrix — equivalent to the geometric mean of the principal radii.
* Computed automatically during updateData(). Returns null before any data has been loaded.
*/
get splatSizeRange() {
if (!isFinite(this._splatSizeMin) || !isFinite(this._splatSizeMax)) {
return null;
}
return { min: this._splatSizeMin, max: this._splatSizeMax };
}
/**
* Gets the covariancesA texture
*/
get covariancesATexture() {
return this._covariancesATexture;
}
/**
* Gets the covariancesB texture
*/
get covariancesBTexture() {
return this._covariancesBTexture;
}
/**
* Gets the centers texture
*/
get centersTexture() {
return this._centersTexture;
}
/**
* Gets the colors texture
*/
get colorsTexture() {
return this._colorsTexture;
}
/**
* Gets the rotation matrix A texture (rotation elements m[0],m[1],m[2],m[4])
*/
get rotationsATexture() {
return this._rotationsATexture;
}
/**
* Gets the rotation matrix B texture (rotation elements m[5],m[6],m[8],m[9])
*/
get rotationsBTexture() {
return this._rotationsBTexture;
}
/**
* Gets the rotation scale texture (rotation element m[10] followed by scale diagonal sx,sy,sz)
*/
get rotationScaleTexture() {
return this._rotationScaleTexture;
}
/**
* Enables or disables generation of rotation and scale matrix textures, required for voxel-based IBL shadows.
*/
get needsRotationScaleTextures() {
return this._needsRotationScaleTextures;
}
set needsRotationScaleTextures(value) {
if (this._needsRotationScaleTextures === value) {
return;
}
this._needsRotationScaleTextures = value;
if (value && this._covariancesATexture) {
if (this._splatsData) {
this.updateData(this._splatsData, this._shData ?? undefined, { flipY: false }, undefined, this._shDegree);
}
else {
Logger.Error("GaussianSplattingMeshBase: needsRotationScaleTextures was enabled after the mesh was already loaded, but the splat data is not kept in RAM. " +
"The rotation and scale matrix textures cannot be initialized. Please reload the mesh data via updateData() or construct with keepInRam=true.");
}
}
}
/**
* Gets the SH textures
*/
get shTextures() {
return this._shTextures;
}
/**
* True when this mesh holds raw SOG webp textures (dequantized in-shader) rather than the
* pre-decoded covariance/center/color textures produced by the standard splat loader.
*/
get useSog() {
return this._useSog;
}
/**
* SOG dequantization parameters paired with the raw textures.
* Set by the splat loader when `useSogTextures: true`. Null otherwise.
*/
get sogParams() {
return this._sogParams;
}
/**
* Install a set of raw SOG webp textures and bind the mesh to the in-shader dequantization path.
* @param pack SOG texture pack produced by ParseSogMetaAsTextures.
* @internal
*/
setSogTextureData(pack) {
this._useSog = true;
this._sogParams?.codebookTexture?.dispose();
this._sogParams = pack;
this._vertexCount = pack.splatCount;
this._shDegree = pack.shDegree ?? 0;
this._maxShDegree = this._shDegree;
// Stride-4 (xyz + 1) — required by the depth-sort worker and the centers texture path.
this._splatPositions = pack.positions;
// Reuse existing texture slots for SOG textures (the shader, under USE_SOG, samples them as RGBA8).
this._covariancesATexture?.dispose();
this._covariancesBTexture?.dispose();
this._centersTexture?.dispose();
this._colorsTexture?.dispose();
this._rotationsATexture?.dispose();
if (this._shTextures) {
for (const t of this._shTextures) {
t.dispose();
}
}
this._centersTexture = pack.meansTextureL;
this._covariancesATexture = pack.meansTextureU;
this._covariancesBTexture = pack.scalesTexture;
this._rotationsATexture = pack.quatsTexture;
this._colorsTexture = pack.sh0Texture;
const shTextures = [];
if (pack.shCentroidsTexture) {
shTextures.push(pack.shCentroidsTexture);
}
if (pack.shLabelsTexture) {
shTextures.push(pack.shLabelsTexture);
}
this._shTextures = shTextures.length ? shTextures : null;
// Force pipeline rebuild so the USE_SOG define and extra samplers are picked up.
this._material?.resetDrawCache();
const size = pack.meansTextureL.getSize();
this._textureSize.x = size.width;
this._textureSize.y = size.height;
this._updateSplatIndexBuffer(this._vertexCount);
this._instantiateWorker();
// Compute bounds from the CPU-decoded positions (stride-4) so the mesh is not frustum-culled.
const positions = pack.positions;
const minimum = new Vector3(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY);
const maximum = new Vector3(Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY);
for (let i = 0; i < this._vertexCount; i++) {
const x = positions[i * 4 + 0];
const y = positions[i * 4 + 1];
const z = positions[i * 4 + 2];
if (x < minimum.x) {
minimum.x = x;
}
if (y < minimum.y) {
minimum.y = y;
}
if (z < minimum.z) {
minimum.z = z;
}
if (x > maximum.x) {
maximum.x = x;
}
if (y > maximum.y) {
maximum.y = y;
}
if (z > maximum.z) {
maximum.z = z;
}
}
this.getBoundingInfo().reConstruct(minimum, maximum, this.getWorldMatrix());
this.setEnabled(true);
this._sortIsDirty = true;
}
/**
* Gets the kernel size
* Documentation and mathematical explanations here:
* https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093
* https://github.com/autonomousvision/mip-splatting/issues/18#issuecomment-1929388931
*/
get kernelSize() {
return this._material instanceof GaussianSplattingMaterial ? this._material.kernelSize : 0;
}
/**
* Get the compensation state
*/
get compensation() {
return this._material instanceof GaussianSplattingMaterial ? this._material.compensation : false;
}
/**
* Minimum projected splat size, in pixels, below which a splat is discarded (0 = disabled).
* Applied in real time; no rebuild required.
*/
get minPixelSize() {
return this._material instanceof GaussianSplattingMaterial ? this._material.minPixelSize : 0;
}
set minPixelSize(value) {
if (this._material instanceof GaussianSplattingMaterial) {
this._material.minPixelSize = Math.max(0, value);
}
}
/**
* set rendering material
*/
set material(value) {
this._material = value;
this._material.backFaceCulling = false;
this._material.cullBackFaces = false;
value.resetDrawCache();
}
/**
* get rendering material
*/
get material() {
return this._material;
}
static _MakeSplatGeometryForMesh(mesh) {
const vertexData = new VertexData();
const originPositions = [-2, -2, 0, 2, -2, 0, 2, 2, 0, -2, 2, 0];
const originIndices = [0, 1, 2, 0, 2, 3];
const positions = [];
const indices = [];
for (let i = 0; i < GaussianSplattingMeshBase._BatchSize; i++) {
for (let j = 0; j < 12; j++) {
if (j == 2 || j == 5 || j == 8 || j == 11) {
positions.push(i); // local splat index
}
else {
positions.push(originPositions[j]);
}
}
indices.push(originIndices.map((v) => v + i * 4));
}
vertexData.positions = positions;
vertexData.indices = indices.flat();
vertexData.applyToMesh(mesh);
}
/**
* Creates a new gaussian splatting mesh
* @param name defines the name of the mesh
* @param url defines the url to load from (optional)
* @param scene defines the hosting scene (optional)
* @param keepInRam keep datas in ram for editing purpose
*/
constructor(name, url = null, scene = null, keepInRam = false) {
super(name, scene);
/**
* Safe-orbit camera limits parsed from the source file's metadata, or `null` when the file
* carries none. The loader also auto-applies these to an active ArcRotateCamera at load time
* (unless `disableAutoCameraLimits` is set); this field exposes the raw values so they can be
* applied or tracked regardless of the active camera type. See {@link ISafeOrbitCameraLimits}.
*/
this.safeOrbitCameraLimits = null;
/** @internal */
this._vertexCount = 0;
this._worker = null;
this._modelViewProjectionMatrix = Matrix.Identity();
this._canPostToWorker = true;
this._readyToDisplay = false;
this._sortRequestId = 0;
// Incremented on every active-range change. A worker sort result is only applied when it was computed
// for the current range version (and size), so a sort for stale ranges is never rendered.
this._activeRangeVersion = 0;
this._hasRenderedOnce = false;
this._covariancesATexture = null;
this._covariancesBTexture = null;
this._centersTexture = null;
this._colorsTexture = null;
this._rotationsATexture = null;
this._rotationsBTexture = null;
this._rotationScaleTexture = null;
this._rotationDataA = null;
this._rotationDataB = null;
this._rotationScaleData = null;
this._needsRotationScaleTextures = false;
this._splatPositions = null;
this._splatIndex = null;
this._shTextures = null;
/** @internal */
this._splatsData = null;
/** @internal */
this._shData = null;
this._textureSize = new Vector2(0, 0);
this._keepInRam = false;
this._alwaysRetainSplatsData = false;
this._flipY = false;
this._delayedTextureUpdate = null;
this._useRGBACovariants = false;
this._useSog = false;
this._sogParams = null;
this._material = null;
this._tmpCovariances = [0, 0, 0, 0, 0, 0];
this._splatSizeMin = Infinity;
this._splatSizeMax = -Infinity;
this._sortIsDirty = false;
this._activeSplatRanges = null;
this._activeSplatRangeKey = "";
this._activeSplatRenderCount = 0;
// Cached bounding box for incremental addPart updates (O(1) vs O(N) scan of positions)
this._cachedBoundingMin = null;
this._cachedBoundingMax = null;
/** @internal */
this._shDegree = 0;
this._maxShDegree = 0;
this._cameraViewInfos = new Map();
/** Fired after parts are added or the mesh is rebuilt following a removal. Payload is the new part count. */
this.onPartCountChangedObservable = new Observable();
/** Fired after part-removal validation passes but before the mesh is rebuilt.
* Payload is the original (pre-removal) part index. */
this.onPartRemovedObservable = new Observable();
/**
* Cosine value of the angle threshold to update view dependent splat sorting. Default is 0.0001.
*/
this.viewUpdateThreshold = GaussianSplattingMeshBase._DefaultViewUpdateThreshold;
this._disableDepthSort = false;
this._loadingPromise = null;
this._updateTextureFromData = (texture, data, width, lineStart, lineCount) => {
const engine = this._getTextureDataUpdateEngine();
engine.updateTextureData(texture.getInternalTexture(), data, 0, lineStart, width, lineCount, 0, 0, false);
};
this._updateTextureFromDataRect = (texture, data, xOffset, yOffset, width, height) => {
const engine = this._getTextureDataUpdateEngine();
engine.updateTextureData(texture.getInternalTexture(), data, xOffset, yOffset, width, height, 0, 0, false);
};
this.subMeshes = [];
new SubMesh(0, 0, 4 * GaussianSplattingMeshBase._BatchSize, 0, 6 * GaussianSplattingMeshBase._BatchSize, this);
this.setEnabled(false);
// webGL2 and webGPU support for RG texture with float16 is fine. not webGL1
this._useRGBACovariants = !this.getEngine().isWebGPU && this.getEngine().version === 1.0;
_AcquireGsInterFrameYield(this.getEngine());
this._keepInRam = keepInRam;
if (url) {
this._loadingPromise = this.loadFileAsync(url);
}
const gaussianSplattingMaterial = new GaussianSplattingMaterial(this.name + "_material", this._scene);
// Cast is safe: GaussianSplattingMeshBase is @internal; all concrete instances are GaussianSplattingMesh.
gaussianSplattingMaterial.setSourceMesh(this);
gaussianSplattingMaterial.doNotSerialize = true;
this._material = gaussianSplattingMaterial;
// delete meshes created for cameras on camera removal
this._scene.onCameraRemovedObservable.add((camera) => {
const cameraId = camera.uniqueId;
// delete mesh for this camera
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
if (cameraViewInfos) {
// If the cached shadow-caster geometry came from the mesh we're disposing, drop it
// so isReady()/render() re-cache it from a surviving camera (see isReady()).
// Must run before dispose(), which nulls the inner mesh's geometry.
if (this._geometry === cameraViewInfos.mesh.geometry) {
this._geometry = null;
}
cameraViewInfos.mesh.dispose();
this._cameraViewInfos.delete(cameraId);
}
});
}
/**
* Get the loading promise when loading the mesh from a URL in the constructor
* @returns constructor loading promise or null if no URL was provided
*/
getLoadingPromise() {
return this._loadingPromise;
}
/**
* Returns the class name
* @returns "GaussianSplattingMeshBase"
*/
getClassName() {
return "GaussianSplattingMeshBase";
}
/**
* Returns the total number of vertices (splats) within the mesh
* @returns the total number of vertices
*/
getTotalVertices() {
return this._vertexCount;
}
/**
* Is this node ready to be used/rendered
* @param completeCheck defines if a complete check (including materials and lights) has to be done (false by default)
* @returns true when ready
*/
isReady(completeCheck = false) {
if (!super.isReady(completeCheck, true)) {
return false;
}
if (!this._readyToDisplay) {
// mesh is ready when worker has done at least 1 sorting
this._postToWorker(true);
return false;
}
// Before the first successful render, apply strict sort-state checks to ensure
// the first rendered frame uses correct splat ordering. Once the mesh has been
// rendered at least once, skip these checks — the render loop will continuously
// re-sort as the camera/world changes via _postToWorker() in render().
if (!this._hasRenderedOnce && !this._disableDepthSort) {
const cameras = this._scene.activeCameras?.length ? this._scene.activeCameras : [this._scene.activeCamera];
const worldMatrix = this.computeWorldMatrix(true);
let anyDirty = false;
for (const camera of cameras) {
if (!camera) {
continue;
}
const cameraViewInfo = this._cameraViewInfos.get(camera.uniqueId);
if (!cameraViewInfo || !cameraViewInfo.splatIndexBufferSet) {
anyDirty = true;
continue;
}
// Wait for the most recently requested sort to be applied so that the splat indices
// match the latest world/camera state.
if (cameraViewInfo.sortAppliedId !== cameraViewInfo.sortRequestId) {
anyDirty = true;
continue;
}
// If world matrix drifted (user applied transforms after load), re-sort before first render.
// Camera drift is intentionally excluded here: checking camera movement would cause isReady()
// to return false indefinitely while the camera is moving. The render loop handles camera
// re-sorting continuously via _postToWorker() in render().
if (this._isSortStateDirty(cameraViewInfo, worldMatrix, camera, true)) {
anyDirty = true;
}
}
if (anyDirty) {
// Try to post any pending sort so subsequent polling iterations make progress.
this._postToWorker(true);
return false;
}
}
// Attach the splat geometry to the GS top mesh so that the shadow generator (which renders
// shadow casters via the top mesh's subMeshes, NOT through this mesh's render() override)
// has valid geometry on the very first shadow pass. Without this, the first shadow render
// happens before render() is called and the GS produces no shadow caster output.
if (!this._geometry && this._cameraViewInfos.size) {
this._geometry = this._cameraViewInfos.values().next().value.mesh.geometry;
}
// If the material declares a shadow depth wrapper, make sure its effect is compiled for
// each subMesh against the scene's shadow generators. Otherwise the first shadow pass
// would be skipped (ShadowGenerator.isReady would return false) and we'd miss the shadow
// on a renderCount=1 capture.
// The shadow generator's depth wrapper is standalone (it wraps a ShaderMaterial), so its
// isReadyForSubMesh path stamps an effect on subMesh._drawWrappers[engine.currentRenderPassId].
// If we leave currentRenderPassId set to the main pass while doing this, we'd overwrite the
// GS material's defines on the main draw wrapper, causing the GS material to recreate its
// defines on the next call and lose any plugin-driven define state (e.g. defines toggled
// by a MaterialPluginBase.isReadyForSubMesh override). Temporarily switch to each shadow
// generator's render pass id while preparing it (matches the pattern used in Mesh.isReady).
if (this.material && this.material.shadowDepthWrapper) {
const engine = this._scene.getEngine();
const previousRenderPassId = engine.currentRenderPassId;
try {
for (const light of this._scene.lights) {
const shadowGenerator = light.getShadowGenerator();
if (!shadowGenerator) {
continue;
}
const shadowMap = shadowGenerator.getShadowMap();
const renderPassIds = shadowMap?.renderPassIds;
if (!renderPassIds || renderPassIds.length === 0) {
continue;
}
for (let p = 0; p < renderPassIds.length; ++p) {
engine.currentRenderPassId = renderPassIds[p];
for (const subMesh of this.subMeshes) {
if (!shadowGenerator.isReady(subMesh, true, false)) {
return false;
}
}
}
}
}
finally {
engine.currentRenderPassId = previousRenderPassId;
}
}
return true;
}
_getCameraDirection(camera) {
const cameraViewMatrix = camera.getViewMatrix();
const cameraProjectionMatrix = camera.getProjectionMatrix();
const cameraViewProjectionMatrix = TmpVectors.Matrix[0];
cameraViewMatrix.multiplyToRef(cameraProjectionMatrix, cameraViewProjectionMatrix);
const modelMatrix = this.computeWorldMatrix(true);
const modelViewMatrix = TmpVectors.Matrix[1];
modelMatrix.multiplyToRef(cameraViewMatrix, modelViewMatrix);
modelMatrix.multiplyToRef(cameraViewProjectionMatrix, this._modelViewProjectionMatrix);
// return vector used to compute distance to camera
const localDirection = TmpVectors.Vector3[1];
localDirection.set(modelViewMatrix.m[2], modelViewMatrix.m[6], modelViewMatrix.m[10]);
localDirection.normalize();
return localDirection;
}
_isSortStateDirty(cameraViewInfo, worldMatrix, camera, worldMatrixOnly = false) {
const world = worldMatrix.m;
const previousWorld = cameraViewInfo.sortWorldMatrix.m;
for (let i = 0; i < previousWorld.length; i++) {
if (!Scalar.WithinEpsilon(previousWorld[i], world[i], this.viewUpdateThreshold)) {
return true;
}
}
if (worldMatrixOnly) {
return false;
}
const cameraViewMatrix = camera.getViewMatrix();
if (!Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.x, cameraViewMatrix.m[2], this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.y, cameraViewMatrix.m[6], this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.z, cameraViewMatrix.m[10], this.viewUpdateThreshold)) {
return true;
}
const cameraPosition = camera.globalPosition;
return (!Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.x, cameraPosition.x, this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.y, cameraPosition.y, this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.z, cameraPosition.z, this.viewUpdateThreshold));
}
/** @internal */
_postToWorker(forced = false) {
const scene = this._scene;
const frameId = scene.getFrameId();
// force update or at least frame update for camera is outdated
let outdated = false;
this._cameraViewInfos.forEach((cameraViewInfos) => {
if (cameraViewInfos.frameIdLastUpdate !== frameId) {
outdated = true;
}
});
// array of cameras used for rendering
const cameras = this._scene.activeCameras?.length ? this._scene.activeCameras : [this._scene.activeCamera];
// list view infos for active cameras
const activeViewInfos = [];
cameras.forEach((camera) => {
if (!camera) {
return;
}
const cameraId = camera.uniqueId;
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
if (cameraViewInfos) {
activeViewInfos.push(cameraViewInfos);
}
else {
// mesh doesn't exist yet for this camera
const cameraMesh = new Mesh(this.name + "_cameraMesh_" + cameraId, this._scene);
cameraMesh.doNotSerialize = true;
// not visible with inspector or the scene graph
cameraMesh.reservedDataStore = { hidden: true };
cameraMesh.setEnabled(false);
cameraMesh.material = this.material;
if (cameraMesh.material && cameraMesh.material instanceof GaussianSplattingMaterial) {
const gsMaterial = cameraMesh.material;
// GaussianSplattingMaterial source mesh may not have been set yet.
// This happens for cloned resources from asset containers for instance,
// where material is cloned before mesh.
if (!gsMaterial.getSourceMesh()) {
// Cast is safe: see constructor comment above.
gsMaterial.setSourceMesh(this);
}
}
GaussianSplattingMeshBase._MakeSplatGeometryForMesh(cameraMesh);
const newViewInfos = {
camera: camera,
cameraDirection: new Vector3(0, 0, 0),
sortWorldMatrix: Matrix.Identity(),
sortCameraForward: new Vector3(0, 0, 0),
sortCameraPosition: new Vector3(0, 0, 0),
sortRequestId: 0,
sortAppliedId: 0,
mesh: cameraMesh,
frameIdLastUpdate: frameId,
splatIndexBufferSet: false,
};
activeViewInfos.push(newViewInfos);
this._cameraViewInfos.set(cameraId, newViewInfos);
}
});
// sort view infos: cameras without an initial splat-index buffer come first so they don't get starved
// by a `forced` re-sort of an already-initialized camera (which would consume `_canPostToWorker`).
// Among initialized cameras, the least recently updated comes first.
activeViewInfos.sort((a, b) => {
if (a.splatIndexBufferSet !== b.splatIndexBufferSet) {
return a.splatIndexBufferSet ? 1 : -1;
}
return a.frameIdLastUpdate - b.frameIdLastUpdate;
});
const hasSortFunction = this._worker || Native?.sortSplats || this._disableDepthSort;
if ((forced || outdated) && hasSortFunction && (this._scene.activeCameras?.length || this._scene.activeCamera) && this._canPostToWorker) {
const worldMatrix = this.computeWorldMatrix(true);
// view infos sorted by least recent updated frame id
activeViewInfos.forEach((cameraViewInfos) => {
const camera = cameraViewInfos.camera;
const cameraDirection = this._getCameraDirection(camera);
if ((forced || this._isSortStateDirty(cameraViewInfos, worldMatrix, camera)) && this._canPostToWorker) {
const cameraViewMatrix = camera.getViewMatrix();
cameraViewInfos.cameraDirection.copyFrom(cameraDirection);
cameraViewInfos.sortWorldMatrix.copyFrom(worldMatrix);
cameraViewInfos.sortCameraForward.set(cameraViewMatrix.m[2], cameraViewMatrix.m[6], cameraViewMatrix.m[10]);
cameraViewInfos.sortCameraPosition.copyFrom(camera.globalPosition);
cameraViewInfos.sortRequestId = ++this._sortRequestId;
cameraViewInfos.frameIdLastUpdate = frameId;
this._canPostToWorker = false;
if (this._worker) {
this._worker.postMessage({
command: GaussianSplattingSortWorkerCommand.SORT,
worldMatrix: worldMatrix.m,
cameraForward: [cameraViewMatrix.m[2], cameraViewMatrix.m[6], cameraViewMatrix.m[10]],
cameraPosition: [camera.globalPosition.x, camera.globalPosition.y, camera.globalPosition.z],
depthMix: this._depthMix,
cameraId: camera.uniqueId,
sortRequestId: cameraViewInfos.sortRequestId,
rangeVersion: this._activeRangeVersion,
useCountingSort: GaussianSplattingMeshBase.UseCountingSort,
rightHanded: this._scene.useRightHandedSystem,
logSortPerformance: GaussianSplattingMeshBase.LogSortPerformance,
}, [this._depthMix.buffer]);
}
else if (Native?.sortSplats) {
if (this._activeSplatRanges) {
// Native sort can't filter ranges: fall back to the (already range-filtered) identity index buffer.
this._updateSplatIndexBuffer(this._vertexCount);
}
else {
Native.sortSplats(this._modelViewProjectionMatrix, this._splatPositions, this._splatIndex, this._s