@babylonjs/core
Version:
Getting started? Play directly with the Babylon.js API using our [playground](https://playground.babylonjs.com/). It also contains a lot of samples to learn how to use it.
1,128 lines (1,126 loc) • 126 kB
JavaScript
/** This file must only contain pure code and pure imports */
import { SubMesh } from "../subMesh.pure.js";
import { Mesh } from "../mesh.pure.js";
import { VertexData } from "../mesh.vertexData.js";
import { Matrix, TmpVectors, Vector2, Vector3 } from "../../Maths/math.vector.pure.js";
import { Logger } from "../../Misc/logger.js";
import { Observable } from "../../Misc/observable.js";
import { GaussianSplattingMaterial } from "../../Materials/GaussianSplatting/gaussianSplattingMaterial.pure.js";
import { RawTexture } from "../../Materials/Textures/rawTexture.js";
import { ToHalfFloat } from "../../Misc/textureTools.js";
import { Scalar } from "../../Maths/math.scalar.js";
import { runCoroutineSync, runCoroutineAsync, createYieldingScheduler } from "../../Misc/coroutine.js";
import { EngineStore } from "../../Engines/engineStore.js";
import { ImportMeshAsync } from "../../Loading/sceneLoader.js";
const IsNative = typeof _native !== "undefined";
const Native = IsNative ? _native : null;
// @internal
const UnpackUnorm = (value, bits) => {
const t = (1 << bits) - 1;
return (value & t) / t;
};
// @internal
const Unpack111011 = (value, result) => {
result.x = UnpackUnorm(value >>> 21, 11);
result.y = UnpackUnorm(value >>> 11, 10);
result.z = UnpackUnorm(value, 11);
};
// @internal
const Unpack8888 = (value, result) => {
result[0] = UnpackUnorm(value >>> 24, 8) * 255;
result[1] = UnpackUnorm(value >>> 16, 8) * 255;
result[2] = UnpackUnorm(value >>> 8, 8) * 255;
result[3] = UnpackUnorm(value, 8) * 255;
};
// @internal
// unpack quaternion with 2,10,10,10 format (largest element, 3x10bit element)
const UnpackRot = (value, result) => {
const norm = 1.0 / (Math.sqrt(2) * 0.5);
const a = (UnpackUnorm(value >>> 20, 10) - 0.5) * norm;
const b = (UnpackUnorm(value >>> 10, 10) - 0.5) * norm;
const c = (UnpackUnorm(value, 10) - 0.5) * norm;
const m = Math.sqrt(1.0 - (a * a + b * b + c * c));
switch (value >>> 30) {
case 0:
result.set(m, a, b, c);
break;
case 1:
result.set(a, m, b, c);
break;
case 2:
result.set(a, b, m, c);
break;
case 3:
result.set(a, b, c, m);
break;
}
};
/**
* Representation of the types
*/
var PLYType;
(function (PLYType) {
PLYType[PLYType["FLOAT"] = 0] = "FLOAT";
PLYType[PLYType["INT"] = 1] = "INT";
PLYType[PLYType["UINT"] = 2] = "UINT";
PLYType[PLYType["DOUBLE"] = 3] = "DOUBLE";
PLYType[PLYType["UCHAR"] = 4] = "UCHAR";
PLYType[PLYType["UNDEFINED"] = 5] = "UNDEFINED";
})(PLYType || (PLYType = {}));
/**
* Usage types of the PLY values
*/
var PLYValue;
(function (PLYValue) {
PLYValue[PLYValue["MIN_X"] = 0] = "MIN_X";
PLYValue[PLYValue["MIN_Y"] = 1] = "MIN_Y";
PLYValue[PLYValue["MIN_Z"] = 2] = "MIN_Z";
PLYValue[PLYValue["MAX_X"] = 3] = "MAX_X";
PLYValue[PLYValue["MAX_Y"] = 4] = "MAX_Y";
PLYValue[PLYValue["MAX_Z"] = 5] = "MAX_Z";
PLYValue[PLYValue["MIN_SCALE_X"] = 6] = "MIN_SCALE_X";
PLYValue[PLYValue["MIN_SCALE_Y"] = 7] = "MIN_SCALE_Y";
PLYValue[PLYValue["MIN_SCALE_Z"] = 8] = "MIN_SCALE_Z";
PLYValue[PLYValue["MAX_SCALE_X"] = 9] = "MAX_SCALE_X";
PLYValue[PLYValue["MAX_SCALE_Y"] = 10] = "MAX_SCALE_Y";
PLYValue[PLYValue["MAX_SCALE_Z"] = 11] = "MAX_SCALE_Z";
PLYValue[PLYValue["PACKED_POSITION"] = 12] = "PACKED_POSITION";
PLYValue[PLYValue["PACKED_ROTATION"] = 13] = "PACKED_ROTATION";
PLYValue[PLYValue["PACKED_SCALE"] = 14] = "PACKED_SCALE";
PLYValue[PLYValue["PACKED_COLOR"] = 15] = "PACKED_COLOR";
PLYValue[PLYValue["X"] = 16] = "X";
PLYValue[PLYValue["Y"] = 17] = "Y";
PLYValue[PLYValue["Z"] = 18] = "Z";
PLYValue[PLYValue["SCALE_0"] = 19] = "SCALE_0";
PLYValue[PLYValue["SCALE_1"] = 20] = "SCALE_1";
PLYValue[PLYValue["SCALE_2"] = 21] = "SCALE_2";
PLYValue[PLYValue["DIFFUSE_RED"] = 22] = "DIFFUSE_RED";
PLYValue[PLYValue["DIFFUSE_GREEN"] = 23] = "DIFFUSE_GREEN";
PLYValue[PLYValue["DIFFUSE_BLUE"] = 24] = "DIFFUSE_BLUE";
PLYValue[PLYValue["OPACITY"] = 25] = "OPACITY";
PLYValue[PLYValue["F_DC_0"] = 26] = "F_DC_0";
PLYValue[PLYValue["F_DC_1"] = 27] = "F_DC_1";
PLYValue[PLYValue["F_DC_2"] = 28] = "F_DC_2";
PLYValue[PLYValue["F_DC_3"] = 29] = "F_DC_3";
PLYValue[PLYValue["ROT_0"] = 30] = "ROT_0";
PLYValue[PLYValue["ROT_1"] = 31] = "ROT_1";
PLYValue[PLYValue["ROT_2"] = 32] = "ROT_2";
PLYValue[PLYValue["ROT_3"] = 33] = "ROT_3";
PLYValue[PLYValue["MIN_COLOR_R"] = 34] = "MIN_COLOR_R";
PLYValue[PLYValue["MIN_COLOR_G"] = 35] = "MIN_COLOR_G";
PLYValue[PLYValue["MIN_COLOR_B"] = 36] = "MIN_COLOR_B";
PLYValue[PLYValue["MAX_COLOR_R"] = 37] = "MAX_COLOR_R";
PLYValue[PLYValue["MAX_COLOR_G"] = 38] = "MAX_COLOR_G";
PLYValue[PLYValue["MAX_COLOR_B"] = 39] = "MAX_COLOR_B";
PLYValue[PLYValue["SH_0"] = 40] = "SH_0";
PLYValue[PLYValue["SH_1"] = 41] = "SH_1";
PLYValue[PLYValue["SH_2"] = 42] = "SH_2";
PLYValue[PLYValue["SH_3"] = 43] = "SH_3";
PLYValue[PLYValue["SH_4"] = 44] = "SH_4";
PLYValue[PLYValue["SH_5"] = 45] = "SH_5";
PLYValue[PLYValue["SH_6"] = 46] = "SH_6";
PLYValue[PLYValue["SH_7"] = 47] = "SH_7";
PLYValue[PLYValue["SH_8"] = 48] = "SH_8";
PLYValue[PLYValue["SH_9"] = 49] = "SH_9";
PLYValue[PLYValue["SH_10"] = 50] = "SH_10";
PLYValue[PLYValue["SH_11"] = 51] = "SH_11";
PLYValue[PLYValue["SH_12"] = 52] = "SH_12";
PLYValue[PLYValue["SH_13"] = 53] = "SH_13";
PLYValue[PLYValue["SH_14"] = 54] = "SH_14";
PLYValue[PLYValue["SH_15"] = 55] = "SH_15";
PLYValue[PLYValue["SH_16"] = 56] = "SH_16";
PLYValue[PLYValue["SH_17"] = 57] = "SH_17";
PLYValue[PLYValue["SH_18"] = 58] = "SH_18";
PLYValue[PLYValue["SH_19"] = 59] = "SH_19";
PLYValue[PLYValue["SH_20"] = 60] = "SH_20";
PLYValue[PLYValue["SH_21"] = 61] = "SH_21";
PLYValue[PLYValue["SH_22"] = 62] = "SH_22";
PLYValue[PLYValue["SH_23"] = 63] = "SH_23";
PLYValue[PLYValue["SH_24"] = 64] = "SH_24";
PLYValue[PLYValue["SH_25"] = 65] = "SH_25";
PLYValue[PLYValue["SH_26"] = 66] = "SH_26";
PLYValue[PLYValue["SH_27"] = 67] = "SH_27";
PLYValue[PLYValue["SH_28"] = 68] = "SH_28";
PLYValue[PLYValue["SH_29"] = 69] = "SH_29";
PLYValue[PLYValue["SH_30"] = 70] = "SH_30";
PLYValue[PLYValue["SH_31"] = 71] = "SH_31";
PLYValue[PLYValue["SH_32"] = 72] = "SH_32";
PLYValue[PLYValue["SH_33"] = 73] = "SH_33";
PLYValue[PLYValue["SH_34"] = 74] = "SH_34";
PLYValue[PLYValue["SH_35"] = 75] = "SH_35";
PLYValue[PLYValue["SH_36"] = 76] = "SH_36";
PLYValue[PLYValue["SH_37"] = 77] = "SH_37";
PLYValue[PLYValue["SH_38"] = 78] = "SH_38";
PLYValue[PLYValue["SH_39"] = 79] = "SH_39";
PLYValue[PLYValue["SH_40"] = 80] = "SH_40";
PLYValue[PLYValue["SH_41"] = 81] = "SH_41";
PLYValue[PLYValue["SH_42"] = 82] = "SH_42";
PLYValue[PLYValue["SH_43"] = 83] = "SH_43";
PLYValue[PLYValue["SH_44"] = 84] = "SH_44";
PLYValue[PLYValue["SH_45"] = 85] = "SH_45";
PLYValue[PLYValue["SH_46"] = 86] = "SH_46";
PLYValue[PLYValue["SH_47"] = 87] = "SH_47";
PLYValue[PLYValue["SH_48"] = 88] = "SH_48";
PLYValue[PLYValue["SH_49"] = 89] = "SH_49";
PLYValue[PLYValue["SH_50"] = 90] = "SH_50";
PLYValue[PLYValue["SH_51"] = 91] = "SH_51";
PLYValue[PLYValue["SH_52"] = 92] = "SH_52";
PLYValue[PLYValue["SH_53"] = 93] = "SH_53";
PLYValue[PLYValue["SH_54"] = 94] = "SH_54";
PLYValue[PLYValue["SH_55"] = 95] = "SH_55";
PLYValue[PLYValue["SH_56"] = 96] = "SH_56";
PLYValue[PLYValue["SH_57"] = 97] = "SH_57";
PLYValue[PLYValue["SH_58"] = 98] = "SH_58";
PLYValue[PLYValue["SH_59"] = 99] = "SH_59";
PLYValue[PLYValue["SH_60"] = 100] = "SH_60";
PLYValue[PLYValue["SH_61"] = 101] = "SH_61";
PLYValue[PLYValue["SH_62"] = 102] = "SH_62";
PLYValue[PLYValue["SH_63"] = 103] = "SH_63";
PLYValue[PLYValue["SH_64"] = 104] = "SH_64";
PLYValue[PLYValue["SH_65"] = 105] = "SH_65";
PLYValue[PLYValue["SH_66"] = 106] = "SH_66";
PLYValue[PLYValue["SH_67"] = 107] = "SH_67";
PLYValue[PLYValue["SH_68"] = 108] = "SH_68";
PLYValue[PLYValue["SH_69"] = 109] = "SH_69";
PLYValue[PLYValue["SH_70"] = 110] = "SH_70";
PLYValue[PLYValue["SH_71"] = 111] = "SH_71";
PLYValue[PLYValue["UNDEFINED"] = 112] = "UNDEFINED";
})(PLYValue || (PLYValue = {}));
/**
* Base class for Gaussian Splatting meshes. Contains all single-cloud rendering logic.
* @internal Use GaussianSplattingMesh instead; this class is an internal implementation detail.
*/
export class GaussianSplattingMeshBase extends Mesh {
/**
* Returns a byte-accurate view for retained splat data, preserving any non-zero byte offset.
* @param data The retained splat source bytes.
* @returns A Uint8Array covering the exact source byte range.
* @internal
*/
static _GetSplatDataBytes(data) {
return ArrayBuffer.isView(data) ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) : new Uint8Array(data);
}
/**
* Returns a Float32 reinterpretation for retained splat data, copying only when alignment requires it.
* @param data The retained splat source bytes.
* @returns A Float32Array over the exact source byte range.
* @internal
*/
static _GetSplatDataFloats(data) {
const bytes = GaussianSplattingMeshBase._GetSplatDataBytes(data);
const floatSize = Float32Array.BYTES_PER_ELEMENT;
if (bytes.byteLength % floatSize !== 0) {
throw new Error(`Gaussian splat data byte length (${bytes.byteLength}) is not divisible by ${floatSize} and cannot be reinterpreted as Float32 data.`);
}
if (bytes.byteOffset % floatSize !== 0) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return new Float32Array(copy.buffer, 0, bytes.byteLength / floatSize);
}
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / floatSize);
}
/**
* If true, disables depth sorting of the splats (default: false)
*/
get disableDepthSort() {
return this._disableDepthSort;
}
set disableDepthSort(value) {
if (!this._disableDepthSort && value) {
this._worker?.terminate();
this._worker = null;
this._disableDepthSort = true;
}
else if (this._disableDepthSort && !value) {
this._disableDepthSort = false;
this._sortIsDirty = true;
this._instantiateWorker();
}
}
/**
* View direction factor used to compute the SH view direction in the shader.
* @deprecated Not used anymore for SH rendering
*/
get viewDirectionFactor() {
return Vector3.OneReadOnly;
}
/**
* SH degree. 0 = no sh (default). 1 = 3 parameters. 2 = 8 parameters. 3 = 15 parameters.
* Value is clamped between 0 and the maximum degree available from loaded data.
*/
get shDegree() {
return this._shDegree;
}
set shDegree(value) {
const maxDegree = this._maxShDegree;
const clamped = Math.max(0, Math.min(Math.round(value), maxDegree));
if (this._shDegree === clamped) {
return;
}
this._shDegree = clamped;
this.material?.resetDrawCache();
}
/**
* Maximum SH degree available from the loaded data.
*/
get maxShDegree() {
return this._maxShDegree;
}
/**
* Number of splats in the mesh
*/
get splatCount() {
return this._splatIndex?.length;
}
/**
* returns the splats data array buffer that contains in order : postions (3 floats), size (3 floats), color (4 bytes), orientation quaternion (4 bytes)
* Only available if the mesh was created with keepInRam: true
*/
get splatsData() {
return this._keepInRam ? this._splatsData : null;
}
/**
* returns the SH data arrays
* Only available if the mesh was created with keepInRam: true
*/
get shData() {
return this._keepInRam ? this._shData : null;
}
/**
* Returns the min/max size range of splats in this mesh, where size is pow(|det(Σ)|, 1/6)
* of the 3D covariance matrix — equivalent to the geometric mean of the principal radii.
* Computed automatically during updateData(). Returns null before any data has been loaded.
*/
get splatSizeRange() {
if (!isFinite(this._splatSizeMin) || !isFinite(this._splatSizeMax)) {
return null;
}
return { min: this._splatSizeMin, max: this._splatSizeMax };
}
/**
* Gets the covariancesA texture
*/
get covariancesATexture() {
return this._covariancesATexture;
}
/**
* Gets the covariancesB texture
*/
get covariancesBTexture() {
return this._covariancesBTexture;
}
/**
* Gets the centers texture
*/
get centersTexture() {
return this._centersTexture;
}
/**
* Gets the colors texture
*/
get colorsTexture() {
return this._colorsTexture;
}
/**
* Gets the rotation matrix A texture (rotation elements m[0],m[1],m[2],m[4])
*/
get rotationsATexture() {
return this._rotationsATexture;
}
/**
* Gets the rotation matrix B texture (rotation elements m[5],m[6],m[8],m[9])
*/
get rotationsBTexture() {
return this._rotationsBTexture;
}
/**
* Gets the rotation scale texture (rotation element m[10] followed by scale diagonal sx,sy,sz)
*/
get rotationScaleTexture() {
return this._rotationScaleTexture;
}
/**
* Enables or disables generation of rotation and scale matrix textures, required for voxel-based IBL shadows.
*/
get needsRotationScaleTextures() {
return this._needsRotationScaleTextures;
}
set needsRotationScaleTextures(value) {
if (this._needsRotationScaleTextures === value) {
return;
}
this._needsRotationScaleTextures = value;
if (value && this._covariancesATexture) {
if (this._splatsData) {
this.updateData(this._splatsData, this._shData ?? undefined, { flipY: false }, undefined, this._shDegree);
}
else {
Logger.Error("GaussianSplattingMeshBase: needsRotationScaleTextures was enabled after the mesh was already loaded, but the splat data is not kept in RAM. " +
"The rotation and scale matrix textures cannot be initialized. Please reload the mesh data via updateData() or construct with keepInRam=true.");
}
}
}
/**
* Gets the SH textures
*/
get shTextures() {
return this._shTextures;
}
/**
* True when this mesh holds raw SOG webp textures (dequantized in-shader) rather than the
* pre-decoded covariance/center/color textures produced by the standard splat loader.
*/
get useSog() {
return this._useSog;
}
/**
* SOG dequantization parameters paired with the raw textures.
* Set by the splat loader when `useSogTextures: true`. Null otherwise.
*/
get sogParams() {
return this._sogParams;
}
/**
* Install a set of raw SOG webp textures and bind the mesh to the in-shader dequantization path.
* @param pack SOG texture pack produced by ParseSogMetaAsTextures.
* @internal
*/
setSogTextureData(pack) {
this._useSog = true;
this._sogParams?.codebookTexture?.dispose();
this._sogParams = pack;
this._vertexCount = pack.splatCount;
this._shDegree = pack.shDegree ?? 0;
this._maxShDegree = this._shDegree;
// Stride-4 (xyz + 1) — required by the depth-sort worker and the centers texture path.
this._splatPositions = pack.positions;
// Reuse existing texture slots for SOG textures (the shader, under USE_SOG, samples them as RGBA8).
this._covariancesATexture?.dispose();
this._covariancesBTexture?.dispose();
this._centersTexture?.dispose();
this._colorsTexture?.dispose();
this._rotationsATexture?.dispose();
if (this._shTextures) {
for (const t of this._shTextures) {
t.dispose();
}
}
this._centersTexture = pack.meansTextureL;
this._covariancesATexture = pack.meansTextureU;
this._covariancesBTexture = pack.scalesTexture;
this._rotationsATexture = pack.quatsTexture;
this._colorsTexture = pack.sh0Texture;
const shTextures = [];
if (pack.shCentroidsTexture) {
shTextures.push(pack.shCentroidsTexture);
}
if (pack.shLabelsTexture) {
shTextures.push(pack.shLabelsTexture);
}
this._shTextures = shTextures.length ? shTextures : null;
// Force pipeline rebuild so the USE_SOG define and extra samplers are picked up.
this._material?.resetDrawCache();
const size = pack.meansTextureL.getSize();
this._textureSize.x = size.width;
this._textureSize.y = size.height;
this._updateSplatIndexBuffer(this._vertexCount);
this._instantiateWorker();
// Compute bounds from the CPU-decoded positions (stride-4) so the mesh is not frustum-culled.
const positions = pack.positions;
const minimum = new Vector3(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY);
const maximum = new Vector3(Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY);
for (let i = 0; i < this._vertexCount; i++) {
const x = positions[i * 4 + 0];
const y = positions[i * 4 + 1];
const z = positions[i * 4 + 2];
if (x < minimum.x) {
minimum.x = x;
}
if (y < minimum.y) {
minimum.y = y;
}
if (z < minimum.z) {
minimum.z = z;
}
if (x > maximum.x) {
maximum.x = x;
}
if (y > maximum.y) {
maximum.y = y;
}
if (z > maximum.z) {
maximum.z = z;
}
}
this.getBoundingInfo().reConstruct(minimum, maximum, this.getWorldMatrix());
this.setEnabled(true);
this._sortIsDirty = true;
}
/**
* Gets the kernel size
* Documentation and mathematical explanations here:
* https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093
* https://github.com/autonomousvision/mip-splatting/issues/18#issuecomment-1929388931
*/
get kernelSize() {
return this._material instanceof GaussianSplattingMaterial ? this._material.kernelSize : 0;
}
/**
* Get the compensation state
*/
get compensation() {
return this._material instanceof GaussianSplattingMaterial ? this._material.compensation : false;
}
/**
* set rendering material
*/
set material(value) {
this._material = value;
this._material.backFaceCulling = false;
this._material.cullBackFaces = false;
value.resetDrawCache();
}
/**
* get rendering material
*/
get material() {
return this._material;
}
static _MakeSplatGeometryForMesh(mesh) {
const vertexData = new VertexData();
const originPositions = [-2, -2, 0, 2, -2, 0, 2, 2, 0, -2, 2, 0];
const originIndices = [0, 1, 2, 0, 2, 3];
const positions = [];
const indices = [];
for (let i = 0; i < GaussianSplattingMeshBase._BatchSize; i++) {
for (let j = 0; j < 12; j++) {
if (j == 2 || j == 5 || j == 8 || j == 11) {
positions.push(i); // local splat index
}
else {
positions.push(originPositions[j]);
}
}
indices.push(originIndices.map((v) => v + i * 4));
}
vertexData.positions = positions;
vertexData.indices = indices.flat();
vertexData.applyToMesh(mesh);
}
/**
* Creates a new gaussian splatting mesh
* @param name defines the name of the mesh
* @param url defines the url to load from (optional)
* @param scene defines the hosting scene (optional)
* @param keepInRam keep datas in ram for editing purpose
*/
constructor(name, url = null, scene = null, keepInRam = false) {
super(name, scene);
/** @internal */
this._vertexCount = 0;
this._worker = null;
this._modelViewProjectionMatrix = Matrix.Identity();
this._canPostToWorker = true;
this._readyToDisplay = false;
this._sortRequestId = 0;
this._hasRenderedOnce = false;
this._covariancesATexture = null;
this._covariancesBTexture = null;
this._centersTexture = null;
this._colorsTexture = null;
this._rotationsATexture = null;
this._rotationsBTexture = null;
this._rotationScaleTexture = null;
this._rotationDataA = null;
this._rotationDataB = null;
this._rotationScaleData = null;
this._needsRotationScaleTextures = false;
this._splatPositions = null;
this._splatIndex = null;
this._shTextures = null;
/** @internal */
this._splatsData = null;
/** @internal */
this._shData = null;
this._textureSize = new Vector2(0, 0);
this._keepInRam = false;
this._alwaysRetainSplatsData = false;
this._flipY = false;
this._delayedTextureUpdate = null;
this._useRGBACovariants = false;
this._useSog = false;
this._sogParams = null;
this._material = null;
this._tmpCovariances = [0, 0, 0, 0, 0, 0];
this._splatSizeMin = Infinity;
this._splatSizeMax = -Infinity;
this._sortIsDirty = false;
// Cached bounding box for incremental addPart updates (O(1) vs O(N) scan of positions)
this._cachedBoundingMin = null;
this._cachedBoundingMax = null;
/** @internal */
this._shDegree = 0;
this._maxShDegree = 0;
this._cameraViewInfos = new Map();
/** Fired after parts are added or the mesh is rebuilt following a removal. Payload is the new part count. */
this.onPartCountChangedObservable = new Observable();
/** Fired after part-removal validation passes but before the mesh is rebuilt.
* Payload is the original (pre-removal) part index. */
this.onPartRemovedObservable = new Observable();
/**
* Cosine value of the angle threshold to update view dependent splat sorting. Default is 0.0001.
*/
this.viewUpdateThreshold = GaussianSplattingMeshBase._DefaultViewUpdateThreshold;
this._disableDepthSort = false;
this._loadingPromise = null;
this._updateTextureFromData = (texture, data, width, lineStart, lineCount) => {
const engine = this._getTextureDataUpdateEngine();
engine.updateTextureData(texture.getInternalTexture(), data, 0, lineStart, width, lineCount, 0, 0, false);
};
this._updateTextureFromDataRect = (texture, data, xOffset, yOffset, width, height) => {
const engine = this._getTextureDataUpdateEngine();
engine.updateTextureData(texture.getInternalTexture(), data, xOffset, yOffset, width, height, 0, 0, false);
};
this.subMeshes = [];
new SubMesh(0, 0, 4 * GaussianSplattingMeshBase._BatchSize, 0, 6 * GaussianSplattingMeshBase._BatchSize, this);
this.setEnabled(false);
// webGL2 and webGPU support for RG texture with float16 is fine. not webGL1
this._useRGBACovariants = !this.getEngine().isWebGPU && this.getEngine().version === 1.0;
this._keepInRam = keepInRam;
if (url) {
this._loadingPromise = this.loadFileAsync(url);
}
const gaussianSplattingMaterial = new GaussianSplattingMaterial(this.name + "_material", this._scene);
// Cast is safe: GaussianSplattingMeshBase is @internal; all concrete instances are GaussianSplattingMesh.
gaussianSplattingMaterial.setSourceMesh(this);
gaussianSplattingMaterial.doNotSerialize = true;
this._material = gaussianSplattingMaterial;
// delete meshes created for cameras on camera removal
this._scene.onCameraRemovedObservable.add((camera) => {
const cameraId = camera.uniqueId;
// delete mesh for this camera
if (this._cameraViewInfos.has(cameraId)) {
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
cameraViewInfos?.mesh.dispose();
this._cameraViewInfos.delete(cameraId);
}
});
}
/**
* Get the loading promise when loading the mesh from a URL in the constructor
* @returns constructor loading promise or null if no URL was provided
*/
getLoadingPromise() {
return this._loadingPromise;
}
/**
* Returns the class name
* @returns "GaussianSplattingMeshBase"
*/
getClassName() {
return "GaussianSplattingMeshBase";
}
/**
* Returns the total number of vertices (splats) within the mesh
* @returns the total number of vertices
*/
getTotalVertices() {
return this._vertexCount;
}
/**
* Is this node ready to be used/rendered
* @param completeCheck defines if a complete check (including materials and lights) has to be done (false by default)
* @returns true when ready
*/
isReady(completeCheck = false) {
if (!super.isReady(completeCheck, true)) {
return false;
}
if (!this._readyToDisplay) {
// mesh is ready when worker has done at least 1 sorting
this._postToWorker(true);
return false;
}
// Before the first successful render, apply strict sort-state checks to ensure
// the first rendered frame uses correct splat ordering. Once the mesh has been
// rendered at least once, skip these checks — the render loop will continuously
// re-sort as the camera/world changes via _postToWorker() in render().
if (!this._hasRenderedOnce && !this._disableDepthSort) {
const cameras = this._scene.activeCameras?.length ? this._scene.activeCameras : [this._scene.activeCamera];
const worldMatrix = this.computeWorldMatrix(true);
let anyDirty = false;
for (const camera of cameras) {
if (!camera) {
continue;
}
const cameraViewInfo = this._cameraViewInfos.get(camera.uniqueId);
if (!cameraViewInfo || !cameraViewInfo.splatIndexBufferSet) {
anyDirty = true;
continue;
}
// Wait for the most recently requested sort to be applied so that the splat indices
// match the latest world/camera state.
if (cameraViewInfo.sortAppliedId !== cameraViewInfo.sortRequestId) {
anyDirty = true;
continue;
}
// If world matrix drifted (user applied transforms after load), re-sort before first render.
// Camera drift is intentionally excluded here: checking camera movement would cause isReady()
// to return false indefinitely while the camera is moving. The render loop handles camera
// re-sorting continuously via _postToWorker() in render().
if (this._isSortStateDirty(cameraViewInfo, worldMatrix, camera, true)) {
anyDirty = true;
}
}
if (anyDirty) {
// Try to post any pending sort so subsequent polling iterations make progress.
this._postToWorker(true);
return false;
}
}
// Attach the splat geometry to the GS top mesh so that the shadow generator (which renders
// shadow casters via the top mesh's subMeshes, NOT through this mesh's render() override)
// has valid geometry on the very first shadow pass. Without this, the first shadow render
// happens before render() is called and the GS produces no shadow caster output.
if (!this._geometry && this._cameraViewInfos.size) {
this._geometry = this._cameraViewInfos.values().next().value.mesh.geometry;
}
// If the material declares a shadow depth wrapper, make sure its effect is compiled for
// each subMesh against the scene's shadow generators. Otherwise the first shadow pass
// would be skipped (ShadowGenerator.isReady would return false) and we'd miss the shadow
// on a renderCount=1 capture.
// The shadow generator's depth wrapper is standalone (it wraps a ShaderMaterial), so its
// isReadyForSubMesh path stamps an effect on subMesh._drawWrappers[engine.currentRenderPassId].
// If we leave currentRenderPassId set to the main pass while doing this, we'd overwrite the
// GS material's defines on the main draw wrapper, causing the GS material to recreate its
// defines on the next call and lose any plugin-driven define state (e.g. defines toggled
// by a MaterialPluginBase.isReadyForSubMesh override). Temporarily switch to each shadow
// generator's render pass id while preparing it (matches the pattern used in Mesh.isReady).
if (this.material && this.material.shadowDepthWrapper) {
const engine = this._scene.getEngine();
const previousRenderPassId = engine.currentRenderPassId;
try {
for (const light of this._scene.lights) {
const shadowGenerator = light.getShadowGenerator();
if (!shadowGenerator) {
continue;
}
const shadowMap = shadowGenerator.getShadowMap();
const renderPassIds = shadowMap?.renderPassIds;
if (!renderPassIds || renderPassIds.length === 0) {
continue;
}
for (let p = 0; p < renderPassIds.length; ++p) {
engine.currentRenderPassId = renderPassIds[p];
for (const subMesh of this.subMeshes) {
if (!shadowGenerator.isReady(subMesh, true, false)) {
return false;
}
}
}
}
}
finally {
engine.currentRenderPassId = previousRenderPassId;
}
}
return true;
}
_getCameraDirection(camera) {
const cameraViewMatrix = camera.getViewMatrix();
const cameraProjectionMatrix = camera.getProjectionMatrix();
const cameraViewProjectionMatrix = TmpVectors.Matrix[0];
cameraViewMatrix.multiplyToRef(cameraProjectionMatrix, cameraViewProjectionMatrix);
const modelMatrix = this.computeWorldMatrix(true);
const modelViewMatrix = TmpVectors.Matrix[1];
modelMatrix.multiplyToRef(cameraViewMatrix, modelViewMatrix);
modelMatrix.multiplyToRef(cameraViewProjectionMatrix, this._modelViewProjectionMatrix);
// return vector used to compute distance to camera
const localDirection = TmpVectors.Vector3[1];
localDirection.set(modelViewMatrix.m[2], modelViewMatrix.m[6], modelViewMatrix.m[10]);
localDirection.normalize();
return localDirection;
}
_isSortStateDirty(cameraViewInfo, worldMatrix, camera, worldMatrixOnly = false) {
const world = worldMatrix.m;
const previousWorld = cameraViewInfo.sortWorldMatrix.m;
for (let i = 0; i < previousWorld.length; i++) {
if (!Scalar.WithinEpsilon(previousWorld[i], world[i], this.viewUpdateThreshold)) {
return true;
}
}
if (worldMatrixOnly) {
return false;
}
const cameraViewMatrix = camera.getViewMatrix();
if (!Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.x, cameraViewMatrix.m[2], this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.y, cameraViewMatrix.m[6], this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraForward.z, cameraViewMatrix.m[10], this.viewUpdateThreshold)) {
return true;
}
const cameraPosition = camera.globalPosition;
return (!Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.x, cameraPosition.x, this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.y, cameraPosition.y, this.viewUpdateThreshold) ||
!Scalar.WithinEpsilon(cameraViewInfo.sortCameraPosition.z, cameraPosition.z, this.viewUpdateThreshold));
}
/** @internal */
_postToWorker(forced = false) {
const scene = this._scene;
const frameId = scene.getFrameId();
// force update or at least frame update for camera is outdated
let outdated = false;
this._cameraViewInfos.forEach((cameraViewInfos) => {
if (cameraViewInfos.frameIdLastUpdate !== frameId) {
outdated = true;
}
});
// array of cameras used for rendering
const cameras = this._scene.activeCameras?.length ? this._scene.activeCameras : [this._scene.activeCamera];
// list view infos for active cameras
const activeViewInfos = [];
cameras.forEach((camera) => {
if (!camera) {
return;
}
const cameraId = camera.uniqueId;
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
if (cameraViewInfos) {
activeViewInfos.push(cameraViewInfos);
}
else {
// mesh doesn't exist yet for this camera
const cameraMesh = new Mesh(this.name + "_cameraMesh_" + cameraId, this._scene);
cameraMesh.doNotSerialize = true;
// not visible with inspector or the scene graph
cameraMesh.reservedDataStore = { hidden: true };
cameraMesh.setEnabled(false);
cameraMesh.material = this.material;
if (cameraMesh.material && cameraMesh.material instanceof GaussianSplattingMaterial) {
const gsMaterial = cameraMesh.material;
// GaussianSplattingMaterial source mesh may not have been set yet.
// This happens for cloned resources from asset containers for instance,
// where material is cloned before mesh.
if (!gsMaterial.getSourceMesh()) {
// Cast is safe: see constructor comment above.
gsMaterial.setSourceMesh(this);
}
}
GaussianSplattingMeshBase._MakeSplatGeometryForMesh(cameraMesh);
const newViewInfos = {
camera: camera,
cameraDirection: new Vector3(0, 0, 0),
sortWorldMatrix: Matrix.Identity(),
sortCameraForward: new Vector3(0, 0, 0),
sortCameraPosition: new Vector3(0, 0, 0),
sortRequestId: 0,
sortAppliedId: 0,
mesh: cameraMesh,
frameIdLastUpdate: frameId,
splatIndexBufferSet: false,
};
activeViewInfos.push(newViewInfos);
this._cameraViewInfos.set(cameraId, newViewInfos);
}
});
// sort view infos: cameras without an initial splat-index buffer come first so they don't get starved
// by a `forced` re-sort of an already-initialized camera (which would consume `_canPostToWorker`).
// Among initialized cameras, the least recently updated comes first.
activeViewInfos.sort((a, b) => {
if (a.splatIndexBufferSet !== b.splatIndexBufferSet) {
return a.splatIndexBufferSet ? 1 : -1;
}
return a.frameIdLastUpdate - b.frameIdLastUpdate;
});
const hasSortFunction = this._worker || Native?.sortSplats || this._disableDepthSort;
if ((forced || outdated) && hasSortFunction && (this._scene.activeCameras?.length || this._scene.activeCamera) && this._canPostToWorker) {
const worldMatrix = this.computeWorldMatrix(true);
// view infos sorted by least recent updated frame id
activeViewInfos.forEach((cameraViewInfos) => {
const camera = cameraViewInfos.camera;
const cameraDirection = this._getCameraDirection(camera);
if ((forced || this._isSortStateDirty(cameraViewInfos, worldMatrix, camera)) && this._canPostToWorker) {
const cameraViewMatrix = camera.getViewMatrix();
cameraViewInfos.cameraDirection.copyFrom(cameraDirection);
cameraViewInfos.sortWorldMatrix.copyFrom(worldMatrix);
cameraViewInfos.sortCameraForward.set(cameraViewMatrix.m[2], cameraViewMatrix.m[6], cameraViewMatrix.m[10]);
cameraViewInfos.sortCameraPosition.copyFrom(camera.globalPosition);
cameraViewInfos.sortRequestId = ++this._sortRequestId;
cameraViewInfos.frameIdLastUpdate = frameId;
this._canPostToWorker = false;
if (this._worker) {
this._worker.postMessage({
worldMatrix: worldMatrix.m,
cameraForward: [cameraViewMatrix.m[2], cameraViewMatrix.m[6], cameraViewMatrix.m[10]],
cameraPosition: [camera.globalPosition.x, camera.globalPosition.y, camera.globalPosition.z],
depthMix: this._depthMix,
cameraId: camera.uniqueId,
sortRequestId: cameraViewInfos.sortRequestId,
}, [this._depthMix.buffer]);
}
else if (Native?.sortSplats) {
Native.sortSplats(this._modelViewProjectionMatrix, this._splatPositions, this._splatIndex, this._scene.useRightHandedSystem);
if (cameraViewInfos.splatIndexBufferSet) {
cameraViewInfos.mesh.thinInstanceBufferUpdated("splatIndex");
}
else {
cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
cameraViewInfos.splatIndexBufferSet = true;
}
cameraViewInfos.sortAppliedId = cameraViewInfos.sortRequestId;
this._canPostToWorker = true;
this._readyToDisplay = true;
}
}
});
}
else if (this._disableDepthSort) {
activeViewInfos.forEach((cameraViewInfos) => {
if (!cameraViewInfos.splatIndexBufferSet) {
cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
cameraViewInfos.splatIndexBufferSet = true;
}
});
this._canPostToWorker = true;
this._readyToDisplay = true;
}
}
/**
* Triggers the draw call for the mesh. Usually, you don't need to call this method by your own because the mesh rendering is handled by the scene rendering manager
* @param subMesh defines the subMesh to render
* @param enableAlphaMode defines if alpha mode can be changed
* @param effectiveMeshReplacement defines an optional mesh used to provide info for the rendering
* @returns the current mesh
*/
render(subMesh, enableAlphaMode, effectiveMeshReplacement) {
this._postToWorker();
// geometry used for shadows, bind the first found in the camera view infos
if (!this._geometry && this._cameraViewInfos.size) {
this._geometry = this._cameraViewInfos.values().next().value.mesh.geometry;
}
const cameraId = this._scene.activeCamera.uniqueId;
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
if (!cameraViewInfos || !cameraViewInfos.splatIndexBufferSet) {
return this;
}
if (this.onBeforeRenderObservable) {
this.onBeforeRenderObservable.notifyObservers(this);
}
const mesh = cameraViewInfos.mesh;
mesh.getWorldMatrix().copyFrom(this.getWorldMatrix());
// Propagate render pass material overrides (e.g., GPU picking) to the inner camera mesh.
// When this mesh is rendered into a RenderTargetTexture with a material override (via setMaterialForRendering),
// the override is set on this proxy mesh but needs to be applied to the actual camera mesh that does the rendering.
const engine = this._scene.getEngine();
const renderPassId = engine.currentRenderPassId;
const renderPassMaterial = this.getMaterialForRenderPass(renderPassId);
if (renderPassMaterial) {
mesh.setMaterialForRenderPass(renderPassId, renderPassMaterial);
}
const ret = mesh.render(subMesh, enableAlphaMode, effectiveMeshReplacement);
this._hasRenderedOnce = true;
// Clean up the temporary override to avoid affecting other render passes
if (renderPassMaterial) {
mesh.setMaterialForRenderPass(renderPassId, undefined);
}
if (this.onAfterRenderObservable) {
this.onAfterRenderObservable.notifyObservers(this);
}
return ret;
}
static _TypeNameToEnum(name) {
switch (name) {
case "float":
return 0 /* PLYType.FLOAT */;
case "int":
return 1 /* PLYType.INT */;
case "uint":
return 2 /* PLYType.UINT */;
case "double":
return 3 /* PLYType.DOUBLE */;
case "uchar":
return 4 /* PLYType.UCHAR */;
}
return 5 /* PLYType.UNDEFINED */;
}
static _ValueNameToEnum(name) {
switch (name) {
case "min_x":
return 0 /* PLYValue.MIN_X */;
case "min_y":
return 1 /* PLYValue.MIN_Y */;
case "min_z":
return 2 /* PLYValue.MIN_Z */;
case "max_x":
return 3 /* PLYValue.MAX_X */;
case "max_y":
return 4 /* PLYValue.MAX_Y */;
case "max_z":
return 5 /* PLYValue.MAX_Z */;
case "min_scale_x":
return 6 /* PLYValue.MIN_SCALE_X */;
case "min_scale_y":
return 7 /* PLYValue.MIN_SCALE_Y */;
case "min_scale_z":
return 8 /* PLYValue.MIN_SCALE_Z */;
case "max_scale_x":
return 9 /* PLYValue.MAX_SCALE_X */;
case "max_scale_y":
return 10 /* PLYValue.MAX_SCALE_Y */;
case "max_scale_z":
return 11 /* PLYValue.MAX_SCALE_Z */;
case "packed_position":
return 12 /* PLYValue.PACKED_POSITION */;
case "packed_rotation":
return 13 /* PLYValue.PACKED_ROTATION */;
case "packed_scale":
return 14 /* PLYValue.PACKED_SCALE */;
case "packed_color":
return 15 /* PLYValue.PACKED_COLOR */;
case "x":
return 16 /* PLYValue.X */;
case "y":
return 17 /* PLYValue.Y */;
case "z":
return 18 /* PLYValue.Z */;
case "scale_0":
return 19 /* PLYValue.SCALE_0 */;
case "scale_1":
return 20 /* PLYValue.SCALE_1 */;
case "scale_2":
return 21 /* PLYValue.SCALE_2 */;
case "diffuse_red":
case "red":
return 22 /* PLYValue.DIFFUSE_RED */;
case "diffuse_green":
case "green":
return 23 /* PLYValue.DIFFUSE_GREEN */;
case "diffuse_blue":
case "blue":
return 24 /* PLYValue.DIFFUSE_BLUE */;
case "f_dc_0":
return 26 /* PLYValue.F_DC_0 */;
case "f_dc_1":
return 27 /* PLYValue.F_DC_1 */;
case "f_dc_2":
return 28 /* PLYValue.F_DC_2 */;
case "f_dc_3":
return 29 /* PLYValue.F_DC_3 */;
case "opacity":
return 25 /* PLYValue.OPACITY */;
case "rot_0":
return 30 /* PLYValue.ROT_0 */;
case "rot_1":
return 31 /* PLYValue.ROT_1 */;
case "rot_2":
return 32 /* PLYValue.ROT_2 */;
case "rot_3":
return 33 /* PLYValue.ROT_3 */;
case "min_r":
return 34 /* PLYValue.MIN_COLOR_R */;
case "min_g":
return 35 /* PLYValue.MIN_COLOR_G */;
case "min_b":
return 36 /* PLYValue.MIN_COLOR_B */;
case "max_r":
return 37 /* PLYValue.MAX_COLOR_R */;
case "max_g":
return 38 /* PLYValue.MAX_COLOR_G */;
case "max_b":
return 39 /* PLYValue.MAX_COLOR_B */;
case "f_rest_0":
return 40 /* PLYValue.SH_0 */;
case "f_rest_1":
return 41 /* PLYValue.SH_1 */;
case "f_rest_2":
return 42 /* PLYValue.SH_2 */;
case "f_rest_3":
return 43 /* PLYValue.SH_3 */;
case "f_rest_4":
return 44 /* PLYValue.SH_4 */;
case "f_rest_5":
return 45 /* PLYValue.SH_5 */;
case "f_rest_6":
return 46 /* PLYValue.SH_6 */;
case "f_rest_7":
return 47 /* PLYValue.SH_7 */;
case "f_rest_8":
return 48 /* PLYValue.SH_8 */;
case "f_rest_9":
return 49 /* PLYValue.SH_9 */;
case "f_rest_10":
return 50 /* PLYValue.SH_10 */;
case "f_rest_11":
return 51 /* PLYValue.SH_11 */;
case "f_rest_12":
return 52 /* PLYValue.SH_12 */;
case "f_rest_13":
return 53 /* PLYValue.SH_13 */;
case "f_rest_14":
return 54 /* PLYValue.SH_14 */;
case "f_rest_15":
return 55 /* PLYValue.SH_15 */;
case "f_rest_16":
return 56 /* PLYValue.SH_16 */;
case "f_rest_17":
return 57 /* PLYValue.SH_17 */;
case "f_rest_18":
return 58 /* PLYValue.SH_18 */;
case "f_rest_19":
return 59 /* PLYValue.SH_19 */;
case "f_rest_20":
return 60 /* PLYValue.SH_20 */;
case "f_rest_21":
return 61 /* PLYValue.SH_21 */;
case "f_rest_22":
return 62 /* PLYValue.SH_22 */;
case "f_rest_23":
return 63 /* PLYValue.SH_23 */;
case "f_rest_24":
return 64 /* PLYValue.SH_24 */;
case "f_rest_25":
return 65 /* PLYValue.SH_25 */;
case "f_rest_26":
return 66 /* PLYValue.SH_26 */;
case "f_rest_27":
return 67 /* PLYValue.SH_27 */;
case "f_rest_28":
return 68 /* PLYValue.SH_28 */;
case "f_rest_29":
return 69 /* PLYValue.SH_29 */;
case "f_rest_30":
return 70 /* PLYValue.SH_30 */;
case "f_rest_31":
return 71 /* PLYValue.SH_31 */;
case "f_rest_32":
return 72 /* PLYValue.SH_32 */;
case "f_rest_33":
return 73 /* PLYValue.SH_33 */;
case "f_rest_34":
return 74 /* PLYValue.SH_34 */;
case "f_rest_35":
return 75 /* PLYValue.SH_35 */;
case "f_rest_36":
return 76 /* PLYValue.SH_36 */;
case "f_rest_37":
return 77 /* PLYValue.SH_37 */;
case "f_rest_38":
return 78 /* PLYValue.SH_38 */;
case "f_rest_39":
return 79 /* PLYValue.SH_39 */;
case "f_rest_40":
return 80 /* PLYValue.SH_40 */;
case "f_rest_41":
return 81 /* PLYValue.SH_41 */;
case "f_rest_42":
return 82 /* PLYValue.SH_42 */;
case "f_rest_43":
return 83 /* PLYValue.SH_43 */;
case "f_rest_44":
return 84 /* PLYValue.SH_44 */;
case "f_rest_45":
return 85 /* PLYValue.SH_45 */;
case "f_rest_46":
return 86 /* PLYValue.SH_46 */;
case "f_rest_47":
return 87 /* PLYValue.SH_47 */;
case "f_rest_48":