@babylonjs/core
Version:
Getting started? Play directly with the Babylon.js API using our [playground](https://playground.babylonjs.com/). It also contains a lot of samples to learn how to use it.
1,159 lines (1,158 loc) • 96.1 kB
JavaScript
import { SubMesh } from "../subMesh.js";
import { Mesh } from "../mesh.js";
import { VertexData } from "../mesh.vertexData.js";
import { Matrix, TmpVectors, Vector2, Vector3 } from "../../Maths/math.vector.js";
import { Logger } from "../../Misc/logger.js";
import { GaussianSplattingMaterial } from "../../Materials/GaussianSplatting/gaussianSplattingMaterial.js";
import { RawTexture } from "../../Materials/Textures/rawTexture.js";
import "../thinInstanceMesh.js";
import { ToHalfFloat } from "../../Misc/textureTools.js";
import { Scalar } from "../../Maths/math.scalar.js";
import { runCoroutineSync, runCoroutineAsync, createYieldingScheduler } from "../../Misc/coroutine.js";
import { EngineStore } from "../../Engines/engineStore.js";
import { ImportMeshAsync } from "../../Loading/sceneLoader.js";
const IsNative = typeof _native !== "undefined";
const Native = IsNative ? _native : null;
// @internal
const UnpackUnorm = (value, bits) => {
const t = (1 << bits) - 1;
return (value & t) / t;
};
// @internal
const Unpack111011 = (value, result) => {
result.x = UnpackUnorm(value >>> 21, 11);
result.y = UnpackUnorm(value >>> 11, 10);
result.z = UnpackUnorm(value, 11);
};
// @internal
const Unpack8888 = (value, result) => {
result[0] = UnpackUnorm(value >>> 24, 8) * 255;
result[1] = UnpackUnorm(value >>> 16, 8) * 255;
result[2] = UnpackUnorm(value >>> 8, 8) * 255;
result[3] = UnpackUnorm(value, 8) * 255;
};
// @internal
// unpack quaternion with 2,10,10,10 format (largest element, 3x10bit element)
const UnpackRot = (value, result) => {
const norm = 1.0 / (Math.sqrt(2) * 0.5);
const a = (UnpackUnorm(value >>> 20, 10) - 0.5) * norm;
const b = (UnpackUnorm(value >>> 10, 10) - 0.5) * norm;
const c = (UnpackUnorm(value, 10) - 0.5) * norm;
const m = Math.sqrt(1.0 - (a * a + b * b + c * c));
switch (value >>> 30) {
case 0:
result.set(m, a, b, c);
break;
case 1:
result.set(a, m, b, c);
break;
case 2:
result.set(a, b, m, c);
break;
case 3:
result.set(a, b, c, m);
break;
}
};
/**
* Representation of the types
*/
var PLYType;
(function (PLYType) {
PLYType[PLYType["FLOAT"] = 0] = "FLOAT";
PLYType[PLYType["INT"] = 1] = "INT";
PLYType[PLYType["UINT"] = 2] = "UINT";
PLYType[PLYType["DOUBLE"] = 3] = "DOUBLE";
PLYType[PLYType["UCHAR"] = 4] = "UCHAR";
PLYType[PLYType["UNDEFINED"] = 5] = "UNDEFINED";
})(PLYType || (PLYType = {}));
/**
* Usage types of the PLY values
*/
var PLYValue;
(function (PLYValue) {
PLYValue[PLYValue["MIN_X"] = 0] = "MIN_X";
PLYValue[PLYValue["MIN_Y"] = 1] = "MIN_Y";
PLYValue[PLYValue["MIN_Z"] = 2] = "MIN_Z";
PLYValue[PLYValue["MAX_X"] = 3] = "MAX_X";
PLYValue[PLYValue["MAX_Y"] = 4] = "MAX_Y";
PLYValue[PLYValue["MAX_Z"] = 5] = "MAX_Z";
PLYValue[PLYValue["MIN_SCALE_X"] = 6] = "MIN_SCALE_X";
PLYValue[PLYValue["MIN_SCALE_Y"] = 7] = "MIN_SCALE_Y";
PLYValue[PLYValue["MIN_SCALE_Z"] = 8] = "MIN_SCALE_Z";
PLYValue[PLYValue["MAX_SCALE_X"] = 9] = "MAX_SCALE_X";
PLYValue[PLYValue["MAX_SCALE_Y"] = 10] = "MAX_SCALE_Y";
PLYValue[PLYValue["MAX_SCALE_Z"] = 11] = "MAX_SCALE_Z";
PLYValue[PLYValue["PACKED_POSITION"] = 12] = "PACKED_POSITION";
PLYValue[PLYValue["PACKED_ROTATION"] = 13] = "PACKED_ROTATION";
PLYValue[PLYValue["PACKED_SCALE"] = 14] = "PACKED_SCALE";
PLYValue[PLYValue["PACKED_COLOR"] = 15] = "PACKED_COLOR";
PLYValue[PLYValue["X"] = 16] = "X";
PLYValue[PLYValue["Y"] = 17] = "Y";
PLYValue[PLYValue["Z"] = 18] = "Z";
PLYValue[PLYValue["SCALE_0"] = 19] = "SCALE_0";
PLYValue[PLYValue["SCALE_1"] = 20] = "SCALE_1";
PLYValue[PLYValue["SCALE_2"] = 21] = "SCALE_2";
PLYValue[PLYValue["DIFFUSE_RED"] = 22] = "DIFFUSE_RED";
PLYValue[PLYValue["DIFFUSE_GREEN"] = 23] = "DIFFUSE_GREEN";
PLYValue[PLYValue["DIFFUSE_BLUE"] = 24] = "DIFFUSE_BLUE";
PLYValue[PLYValue["OPACITY"] = 25] = "OPACITY";
PLYValue[PLYValue["F_DC_0"] = 26] = "F_DC_0";
PLYValue[PLYValue["F_DC_1"] = 27] = "F_DC_1";
PLYValue[PLYValue["F_DC_2"] = 28] = "F_DC_2";
PLYValue[PLYValue["F_DC_3"] = 29] = "F_DC_3";
PLYValue[PLYValue["ROT_0"] = 30] = "ROT_0";
PLYValue[PLYValue["ROT_1"] = 31] = "ROT_1";
PLYValue[PLYValue["ROT_2"] = 32] = "ROT_2";
PLYValue[PLYValue["ROT_3"] = 33] = "ROT_3";
PLYValue[PLYValue["MIN_COLOR_R"] = 34] = "MIN_COLOR_R";
PLYValue[PLYValue["MIN_COLOR_G"] = 35] = "MIN_COLOR_G";
PLYValue[PLYValue["MIN_COLOR_B"] = 36] = "MIN_COLOR_B";
PLYValue[PLYValue["MAX_COLOR_R"] = 37] = "MAX_COLOR_R";
PLYValue[PLYValue["MAX_COLOR_G"] = 38] = "MAX_COLOR_G";
PLYValue[PLYValue["MAX_COLOR_B"] = 39] = "MAX_COLOR_B";
PLYValue[PLYValue["SH_0"] = 40] = "SH_0";
PLYValue[PLYValue["SH_1"] = 41] = "SH_1";
PLYValue[PLYValue["SH_2"] = 42] = "SH_2";
PLYValue[PLYValue["SH_3"] = 43] = "SH_3";
PLYValue[PLYValue["SH_4"] = 44] = "SH_4";
PLYValue[PLYValue["SH_5"] = 45] = "SH_5";
PLYValue[PLYValue["SH_6"] = 46] = "SH_6";
PLYValue[PLYValue["SH_7"] = 47] = "SH_7";
PLYValue[PLYValue["SH_8"] = 48] = "SH_8";
PLYValue[PLYValue["SH_9"] = 49] = "SH_9";
PLYValue[PLYValue["SH_10"] = 50] = "SH_10";
PLYValue[PLYValue["SH_11"] = 51] = "SH_11";
PLYValue[PLYValue["SH_12"] = 52] = "SH_12";
PLYValue[PLYValue["SH_13"] = 53] = "SH_13";
PLYValue[PLYValue["SH_14"] = 54] = "SH_14";
PLYValue[PLYValue["SH_15"] = 55] = "SH_15";
PLYValue[PLYValue["SH_16"] = 56] = "SH_16";
PLYValue[PLYValue["SH_17"] = 57] = "SH_17";
PLYValue[PLYValue["SH_18"] = 58] = "SH_18";
PLYValue[PLYValue["SH_19"] = 59] = "SH_19";
PLYValue[PLYValue["SH_20"] = 60] = "SH_20";
PLYValue[PLYValue["SH_21"] = 61] = "SH_21";
PLYValue[PLYValue["SH_22"] = 62] = "SH_22";
PLYValue[PLYValue["SH_23"] = 63] = "SH_23";
PLYValue[PLYValue["SH_24"] = 64] = "SH_24";
PLYValue[PLYValue["SH_25"] = 65] = "SH_25";
PLYValue[PLYValue["SH_26"] = 66] = "SH_26";
PLYValue[PLYValue["SH_27"] = 67] = "SH_27";
PLYValue[PLYValue["SH_28"] = 68] = "SH_28";
PLYValue[PLYValue["SH_29"] = 69] = "SH_29";
PLYValue[PLYValue["SH_30"] = 70] = "SH_30";
PLYValue[PLYValue["SH_31"] = 71] = "SH_31";
PLYValue[PLYValue["SH_32"] = 72] = "SH_32";
PLYValue[PLYValue["SH_33"] = 73] = "SH_33";
PLYValue[PLYValue["SH_34"] = 74] = "SH_34";
PLYValue[PLYValue["SH_35"] = 75] = "SH_35";
PLYValue[PLYValue["SH_36"] = 76] = "SH_36";
PLYValue[PLYValue["SH_37"] = 77] = "SH_37";
PLYValue[PLYValue["SH_38"] = 78] = "SH_38";
PLYValue[PLYValue["SH_39"] = 79] = "SH_39";
PLYValue[PLYValue["SH_40"] = 80] = "SH_40";
PLYValue[PLYValue["SH_41"] = 81] = "SH_41";
PLYValue[PLYValue["SH_42"] = 82] = "SH_42";
PLYValue[PLYValue["SH_43"] = 83] = "SH_43";
PLYValue[PLYValue["SH_44"] = 84] = "SH_44";
PLYValue[PLYValue["UNDEFINED"] = 85] = "UNDEFINED";
})(PLYValue || (PLYValue = {}));
/**
* Base class for Gaussian Splatting meshes. Contains all single-cloud rendering logic.
* @internal Use GaussianSplattingMesh instead; this class is an internal implementation detail.
*/
export class GaussianSplattingMeshBase extends Mesh {
/**
* If true, disables depth sorting of the splats (default: false)
*/
get disableDepthSort() {
return this._disableDepthSort;
}
set disableDepthSort(value) {
if (!this._disableDepthSort && value) {
this._worker?.terminate();
this._worker = null;
this._disableDepthSort = true;
}
else if (this._disableDepthSort && !value) {
this._disableDepthSort = false;
this._sortIsDirty = true;
this._instantiateWorker();
}
}
/**
* View direction factor used to compute the SH view direction in the shader.
* @deprecated Not used anymore for SH rendering
*/
get viewDirectionFactor() {
return Vector3.OneReadOnly;
}
/**
* SH degree. 0 = no sh (default). 1 = 3 parameters. 2 = 8 parameters. 3 = 15 parameters.
* Value is clamped between 0 and the maximum degree available from loaded data.
*/
get shDegree() {
return this._shDegree;
}
set shDegree(value) {
const maxDegree = this._shTextures?.length ?? 0;
const clamped = Math.max(0, Math.min(Math.round(value), maxDegree));
if (this._shDegree === clamped) {
return;
}
this._shDegree = clamped;
this.material?.resetDrawCache();
}
/**
* Maximum SH degree available from the loaded data.
*/
get maxShDegree() {
return this._shTextures?.length ?? 0;
}
/**
* Number of splats in the mesh
*/
get splatCount() {
return this._splatIndex?.length;
}
/**
* returns the splats data array buffer that contains in order : postions (3 floats), size (3 floats), color (4 bytes), orientation quaternion (4 bytes)
* Only available if the mesh was created with keepInRam: true
*/
get splatsData() {
return this._keepInRam ? this._splatsData : null;
}
/**
* returns the SH data arrays
* Only available if the mesh was created with keepInRam: true
*/
get shData() {
return this._keepInRam ? this._shData : null;
}
/**
* Gets the covariancesA texture
*/
get covariancesATexture() {
return this._covariancesATexture;
}
/**
* Gets the covariancesB texture
*/
get covariancesBTexture() {
return this._covariancesBTexture;
}
/**
* Gets the centers texture
*/
get centersTexture() {
return this._centersTexture;
}
/**
* Gets the colors texture
*/
get colorsTexture() {
return this._colorsTexture;
}
/**
* Gets the SH textures
*/
get shTextures() {
return this._shTextures;
}
/**
* Gets the kernel size
* Documentation and mathematical explanations here:
* https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093
* https://github.com/autonomousvision/mip-splatting/issues/18#issuecomment-1929388931
*/
get kernelSize() {
return this._material instanceof GaussianSplattingMaterial ? this._material.kernelSize : 0;
}
/**
* Get the compensation state
*/
get compensation() {
return this._material instanceof GaussianSplattingMaterial ? this._material.compensation : false;
}
/**
* set rendering material
*/
set material(value) {
this._material = value;
this._material.backFaceCulling = false;
this._material.cullBackFaces = false;
value.resetDrawCache();
}
/**
* get rendering material
*/
get material() {
return this._material;
}
static _MakeSplatGeometryForMesh(mesh) {
const vertexData = new VertexData();
const originPositions = [-2, -2, 0, 2, -2, 0, 2, 2, 0, -2, 2, 0];
const originIndices = [0, 1, 2, 0, 2, 3];
const positions = [];
const indices = [];
for (let i = 0; i < GaussianSplattingMeshBase._BatchSize; i++) {
for (let j = 0; j < 12; j++) {
if (j == 2 || j == 5 || j == 8 || j == 11) {
positions.push(i); // local splat index
}
else {
positions.push(originPositions[j]);
}
}
indices.push(originIndices.map((v) => v + i * 4));
}
vertexData.positions = positions;
vertexData.indices = indices.flat();
vertexData.applyToMesh(mesh);
}
/**
* Creates a new gaussian splatting mesh
* @param name defines the name of the mesh
* @param url defines the url to load from (optional)
* @param scene defines the hosting scene (optional)
* @param keepInRam keep datas in ram for editing purpose
*/
constructor(name, url = null, scene = null, keepInRam = false) {
super(name, scene);
/** @internal */
this._vertexCount = 0;
this._worker = null;
this._modelViewProjectionMatrix = Matrix.Identity();
this._canPostToWorker = true;
this._readyToDisplay = false;
this._covariancesATexture = null;
this._covariancesBTexture = null;
this._centersTexture = null;
this._colorsTexture = null;
this._splatPositions = null;
this._splatIndex = null;
this._shTextures = null;
/** @internal */
this._splatsData = null;
/** @internal */
this._shData = null;
this._textureSize = new Vector2(0, 0);
this._keepInRam = false;
this._alwaysRetainSplatsData = false;
this._delayedTextureUpdate = null;
this._useRGBACovariants = false;
this._material = null;
this._tmpCovariances = [0, 0, 0, 0, 0, 0];
this._sortIsDirty = false;
// Cached bounding box for incremental addPart updates (O(1) vs O(N) scan of positions)
this._cachedBoundingMin = null;
this._cachedBoundingMax = null;
/** @internal */
this._shDegree = 0;
this._cameraViewInfos = new Map();
/**
* Cosine value of the angle threshold to update view dependent splat sorting. Default is 0.0001.
*/
this.viewUpdateThreshold = GaussianSplattingMeshBase._DefaultViewUpdateThreshold;
this._disableDepthSort = false;
this._loadingPromise = null;
this._updateTextureFromData = (texture, data, width, lineStart, lineCount) => {
this.getEngine().updateTextureData(texture.getInternalTexture(), data, 0, lineStart, width, lineCount, 0, 0, false);
};
this.subMeshes = [];
new SubMesh(0, 0, 4 * GaussianSplattingMeshBase._BatchSize, 0, 6 * GaussianSplattingMeshBase._BatchSize, this);
this.setEnabled(false);
// webGL2 and webGPU support for RG texture with float16 is fine. not webGL1
this._useRGBACovariants = !this.getEngine().isWebGPU && this.getEngine().version === 1.0;
this._keepInRam = keepInRam;
if (url) {
this._loadingPromise = this.loadFileAsync(url);
}
const gaussianSplattingMaterial = new GaussianSplattingMaterial(this.name + "_material", this._scene);
// Cast is safe: GaussianSplattingMeshBase is @internal; all concrete instances are GaussianSplattingMesh.
gaussianSplattingMaterial.setSourceMesh(this);
this._material = gaussianSplattingMaterial;
// delete meshes created for cameras on camera removal
this._scene.onCameraRemovedObservable.add((camera) => {
const cameraId = camera.uniqueId;
// delete mesh for this camera
if (this._cameraViewInfos.has(cameraId)) {
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
cameraViewInfos?.mesh.dispose();
this._cameraViewInfos.delete(cameraId);
}
});
}
/**
* Get the loading promise when loading the mesh from a URL in the constructor
* @returns constructor loading promise or null if no URL was provided
*/
getLoadingPromise() {
return this._loadingPromise;
}
/**
* Returns the class name
* @returns "GaussianSplattingMeshBase"
*/
getClassName() {
return "GaussianSplattingMeshBase";
}
/**
* Returns the total number of vertices (splats) within the mesh
* @returns the total number of vertices
*/
getTotalVertices() {
return this._vertexCount;
}
/**
* Is this node ready to be used/rendered
* @param completeCheck defines if a complete check (including materials and lights) has to be done (false by default)
* @returns true when ready
*/
isReady(completeCheck = false) {
if (!super.isReady(completeCheck, true)) {
return false;
}
if (!this._readyToDisplay) {
// mesh is ready when worker has done at least 1 sorting
this._postToWorker(true);
return false;
}
return true;
}
_getCameraDirection(camera) {
const cameraViewMatrix = camera.getViewMatrix();
const cameraProjectionMatrix = camera.getProjectionMatrix();
const cameraViewProjectionMatrix = TmpVectors.Matrix[0];
cameraViewMatrix.multiplyToRef(cameraProjectionMatrix, cameraViewProjectionMatrix);
const modelMatrix = this.getWorldMatrix();
const modelViewMatrix = TmpVectors.Matrix[1];
modelMatrix.multiplyToRef(cameraViewMatrix, modelViewMatrix);
modelMatrix.multiplyToRef(cameraViewProjectionMatrix, this._modelViewProjectionMatrix);
// return vector used to compute distance to camera
const localDirection = TmpVectors.Vector3[1];
localDirection.set(modelViewMatrix.m[2], modelViewMatrix.m[6], modelViewMatrix.m[10]);
localDirection.normalize();
return localDirection;
}
/** @internal */
_postToWorker(forced = false) {
const scene = this._scene;
const frameId = scene.getFrameId();
// force update or at least frame update for camera is outdated
let outdated = false;
this._cameraViewInfos.forEach((cameraViewInfos) => {
if (cameraViewInfos.frameIdLastUpdate !== frameId) {
outdated = true;
}
});
// array of cameras used for rendering
const cameras = this._scene.activeCameras?.length ? this._scene.activeCameras : [this._scene.activeCamera];
// list view infos for active cameras
const activeViewInfos = [];
cameras.forEach((camera) => {
if (!camera) {
return;
}
const cameraId = camera.uniqueId;
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
if (cameraViewInfos) {
activeViewInfos.push(cameraViewInfos);
}
else {
// mesh doesn't exist yet for this camera
const cameraMesh = new Mesh(this.name + "_cameraMesh_" + cameraId, this._scene);
// not visible with inspector or the scene graph
cameraMesh.reservedDataStore = { hidden: true };
cameraMesh.setEnabled(false);
cameraMesh.material = this.material;
if (cameraMesh.material && cameraMesh.material instanceof GaussianSplattingMaterial) {
const gsMaterial = cameraMesh.material;
// GaussianSplattingMaterial source mesh may not have been set yet.
// This happens for cloned resources from asset containers for instance,
// where material is cloned before mesh.
if (!gsMaterial.getSourceMesh()) {
// Cast is safe: see constructor comment above.
gsMaterial.setSourceMesh(this);
}
}
GaussianSplattingMeshBase._MakeSplatGeometryForMesh(cameraMesh);
const newViewInfos = {
camera: camera,
cameraDirection: new Vector3(0, 0, 0),
mesh: cameraMesh,
frameIdLastUpdate: frameId,
splatIndexBufferSet: false,
};
activeViewInfos.push(newViewInfos);
this._cameraViewInfos.set(cameraId, newViewInfos);
}
});
// sort view infos by last updated frame id: first item is the least recently updated
activeViewInfos.sort((a, b) => a.frameIdLastUpdate - b.frameIdLastUpdate);
const hasSortFunction = this._worker || Native?.sortSplats || this._disableDepthSort;
if ((forced || outdated) && hasSortFunction && (this._scene.activeCameras?.length || this._scene.activeCamera) && this._canPostToWorker) {
// view infos sorted by least recent updated frame id
activeViewInfos.forEach((cameraViewInfos) => {
const camera = cameraViewInfos.camera;
const cameraDirection = this._getCameraDirection(camera);
const previousCameraDirection = cameraViewInfos.cameraDirection;
const dot = Vector3.Dot(cameraDirection, previousCameraDirection);
if ((forced || Math.abs(dot - 1) >= this.viewUpdateThreshold) && this._canPostToWorker) {
cameraViewInfos.cameraDirection.copyFrom(cameraDirection);
cameraViewInfos.frameIdLastUpdate = frameId;
this._canPostToWorker = false;
if (this._worker) {
const cameraViewMatrix = camera.getViewMatrix();
this._worker.postMessage({
worldMatrix: this.getWorldMatrix().m,
cameraForward: [cameraViewMatrix.m[2], cameraViewMatrix.m[6], cameraViewMatrix.m[10]],
cameraPosition: [camera.globalPosition.x, camera.globalPosition.y, camera.globalPosition.z],
depthMix: this._depthMix,
cameraId: camera.uniqueId,
}, [this._depthMix.buffer]);
}
else if (Native?.sortSplats) {
Native.sortSplats(this._modelViewProjectionMatrix, this._splatPositions, this._splatIndex, this._scene.useRightHandedSystem);
if (cameraViewInfos.splatIndexBufferSet) {
cameraViewInfos.mesh.thinInstanceBufferUpdated("splatIndex");
}
else {
cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
cameraViewInfos.splatIndexBufferSet = true;
}
this._canPostToWorker = true;
this._readyToDisplay = true;
}
}
});
}
else if (this._disableDepthSort) {
activeViewInfos.forEach((cameraViewInfos) => {
if (!cameraViewInfos.splatIndexBufferSet) {
cameraViewInfos.mesh.thinInstanceSetBuffer("splatIndex", this._splatIndex, 16, false);
cameraViewInfos.splatIndexBufferSet = true;
}
});
this._canPostToWorker = true;
this._readyToDisplay = true;
}
}
/**
* Triggers the draw call for the mesh. Usually, you don't need to call this method by your own because the mesh rendering is handled by the scene rendering manager
* @param subMesh defines the subMesh to render
* @param enableAlphaMode defines if alpha mode can be changed
* @param effectiveMeshReplacement defines an optional mesh used to provide info for the rendering
* @returns the current mesh
*/
render(subMesh, enableAlphaMode, effectiveMeshReplacement) {
this._postToWorker();
// geometry used for shadows, bind the first found in the camera view infos
if (!this._geometry && this._cameraViewInfos.size) {
this._geometry = this._cameraViewInfos.values().next().value.mesh.geometry;
}
const cameraId = this._scene.activeCamera.uniqueId;
const cameraViewInfos = this._cameraViewInfos.get(cameraId);
if (!cameraViewInfos || !cameraViewInfos.splatIndexBufferSet) {
return this;
}
if (this.onBeforeRenderObservable) {
this.onBeforeRenderObservable.notifyObservers(this);
}
const mesh = cameraViewInfos.mesh;
mesh.getWorldMatrix().copyFrom(this.getWorldMatrix());
// Propagate render pass material overrides (e.g., GPU picking) to the inner camera mesh.
// When this mesh is rendered into a RenderTargetTexture with a material override (via setMaterialForRendering),
// the override is set on this proxy mesh but needs to be applied to the actual camera mesh that does the rendering.
const engine = this._scene.getEngine();
const renderPassId = engine.currentRenderPassId;
const renderPassMaterial = this.getMaterialForRenderPass(renderPassId);
if (renderPassMaterial) {
mesh.setMaterialForRenderPass(renderPassId, renderPassMaterial);
}
const ret = mesh.render(subMesh, enableAlphaMode, effectiveMeshReplacement);
// Clean up the temporary override to avoid affecting other render passes
if (renderPassMaterial) {
mesh.setMaterialForRenderPass(renderPassId, undefined);
}
if (this.onAfterRenderObservable) {
this.onAfterRenderObservable.notifyObservers(this);
}
return ret;
}
static _TypeNameToEnum(name) {
switch (name) {
case "float":
return 0 /* PLYType.FLOAT */;
case "int":
return 1 /* PLYType.INT */;
case "uint":
return 2 /* PLYType.UINT */;
case "double":
return 3 /* PLYType.DOUBLE */;
case "uchar":
return 4 /* PLYType.UCHAR */;
}
return 5 /* PLYType.UNDEFINED */;
}
static _ValueNameToEnum(name) {
switch (name) {
case "min_x":
return 0 /* PLYValue.MIN_X */;
case "min_y":
return 1 /* PLYValue.MIN_Y */;
case "min_z":
return 2 /* PLYValue.MIN_Z */;
case "max_x":
return 3 /* PLYValue.MAX_X */;
case "max_y":
return 4 /* PLYValue.MAX_Y */;
case "max_z":
return 5 /* PLYValue.MAX_Z */;
case "min_scale_x":
return 6 /* PLYValue.MIN_SCALE_X */;
case "min_scale_y":
return 7 /* PLYValue.MIN_SCALE_Y */;
case "min_scale_z":
return 8 /* PLYValue.MIN_SCALE_Z */;
case "max_scale_x":
return 9 /* PLYValue.MAX_SCALE_X */;
case "max_scale_y":
return 10 /* PLYValue.MAX_SCALE_Y */;
case "max_scale_z":
return 11 /* PLYValue.MAX_SCALE_Z */;
case "packed_position":
return 12 /* PLYValue.PACKED_POSITION */;
case "packed_rotation":
return 13 /* PLYValue.PACKED_ROTATION */;
case "packed_scale":
return 14 /* PLYValue.PACKED_SCALE */;
case "packed_color":
return 15 /* PLYValue.PACKED_COLOR */;
case "x":
return 16 /* PLYValue.X */;
case "y":
return 17 /* PLYValue.Y */;
case "z":
return 18 /* PLYValue.Z */;
case "scale_0":
return 19 /* PLYValue.SCALE_0 */;
case "scale_1":
return 20 /* PLYValue.SCALE_1 */;
case "scale_2":
return 21 /* PLYValue.SCALE_2 */;
case "diffuse_red":
case "red":
return 22 /* PLYValue.DIFFUSE_RED */;
case "diffuse_green":
case "green":
return 23 /* PLYValue.DIFFUSE_GREEN */;
case "diffuse_blue":
case "blue":
return 24 /* PLYValue.DIFFUSE_BLUE */;
case "f_dc_0":
return 26 /* PLYValue.F_DC_0 */;
case "f_dc_1":
return 27 /* PLYValue.F_DC_1 */;
case "f_dc_2":
return 28 /* PLYValue.F_DC_2 */;
case "f_dc_3":
return 29 /* PLYValue.F_DC_3 */;
case "opacity":
return 25 /* PLYValue.OPACITY */;
case "rot_0":
return 30 /* PLYValue.ROT_0 */;
case "rot_1":
return 31 /* PLYValue.ROT_1 */;
case "rot_2":
return 32 /* PLYValue.ROT_2 */;
case "rot_3":
return 33 /* PLYValue.ROT_3 */;
case "min_r":
return 34 /* PLYValue.MIN_COLOR_R */;
case "min_g":
return 35 /* PLYValue.MIN_COLOR_G */;
case "min_b":
return 36 /* PLYValue.MIN_COLOR_B */;
case "max_r":
return 37 /* PLYValue.MAX_COLOR_R */;
case "max_g":
return 38 /* PLYValue.MAX_COLOR_G */;
case "max_b":
return 39 /* PLYValue.MAX_COLOR_B */;
case "f_rest_0":
return 40 /* PLYValue.SH_0 */;
case "f_rest_1":
return 41 /* PLYValue.SH_1 */;
case "f_rest_2":
return 42 /* PLYValue.SH_2 */;
case "f_rest_3":
return 43 /* PLYValue.SH_3 */;
case "f_rest_4":
return 44 /* PLYValue.SH_4 */;
case "f_rest_5":
return 45 /* PLYValue.SH_5 */;
case "f_rest_6":
return 46 /* PLYValue.SH_6 */;
case "f_rest_7":
return 47 /* PLYValue.SH_7 */;
case "f_rest_8":
return 48 /* PLYValue.SH_8 */;
case "f_rest_9":
return 49 /* PLYValue.SH_9 */;
case "f_rest_10":
return 50 /* PLYValue.SH_10 */;
case "f_rest_11":
return 51 /* PLYValue.SH_11 */;
case "f_rest_12":
return 52 /* PLYValue.SH_12 */;
case "f_rest_13":
return 53 /* PLYValue.SH_13 */;
case "f_rest_14":
return 54 /* PLYValue.SH_14 */;
case "f_rest_15":
return 55 /* PLYValue.SH_15 */;
case "f_rest_16":
return 56 /* PLYValue.SH_16 */;
case "f_rest_17":
return 57 /* PLYValue.SH_17 */;
case "f_rest_18":
return 58 /* PLYValue.SH_18 */;
case "f_rest_19":
return 59 /* PLYValue.SH_19 */;
case "f_rest_20":
return 60 /* PLYValue.SH_20 */;
case "f_rest_21":
return 61 /* PLYValue.SH_21 */;
case "f_rest_22":
return 62 /* PLYValue.SH_22 */;
case "f_rest_23":
return 63 /* PLYValue.SH_23 */;
case "f_rest_24":
return 64 /* PLYValue.SH_24 */;
case "f_rest_25":
return 65 /* PLYValue.SH_25 */;
case "f_rest_26":
return 66 /* PLYValue.SH_26 */;
case "f_rest_27":
return 67 /* PLYValue.SH_27 */;
case "f_rest_28":
return 68 /* PLYValue.SH_28 */;
case "f_rest_29":
return 69 /* PLYValue.SH_29 */;
case "f_rest_30":
return 70 /* PLYValue.SH_30 */;
case "f_rest_31":
return 71 /* PLYValue.SH_31 */;
case "f_rest_32":
return 72 /* PLYValue.SH_32 */;
case "f_rest_33":
return 73 /* PLYValue.SH_33 */;
case "f_rest_34":
return 74 /* PLYValue.SH_34 */;
case "f_rest_35":
return 75 /* PLYValue.SH_35 */;
case "f_rest_36":
return 76 /* PLYValue.SH_36 */;
case "f_rest_37":
return 77 /* PLYValue.SH_37 */;
case "f_rest_38":
return 78 /* PLYValue.SH_38 */;
case "f_rest_39":
return 79 /* PLYValue.SH_39 */;
case "f_rest_40":
return 80 /* PLYValue.SH_40 */;
case "f_rest_41":
return 81 /* PLYValue.SH_41 */;
case "f_rest_42":
return 82 /* PLYValue.SH_42 */;
case "f_rest_43":
return 83 /* PLYValue.SH_43 */;
case "f_rest_44":
return 84 /* PLYValue.SH_44 */;
}
return 85 /* PLYValue.UNDEFINED */;
}
/**
* Parse a PLY file header and returns metas infos on splats and chunks
* @param data the loaded buffer
* @returns a PLYHeader
*/
static ParseHeader(data) {
const ubuf = new Uint8Array(data);
const header = new TextDecoder().decode(ubuf.slice(0, 1024 * 10));
const headerEnd = "end_header\n";
const headerEndIndex = header.indexOf(headerEnd);
if (headerEndIndex < 0 || !header) {
// standard splat
return null;
}
const vertexCount = parseInt(/element vertex (\d+)\n/.exec(header)[1]);
const chunkElement = /element chunk (\d+)\n/.exec(header);
let chunkCount = 0;
if (chunkElement) {
chunkCount = parseInt(chunkElement[1]);
}
let rowVertexOffset = 0;
let rowChunkOffset = 0;
const offsets = {
double: 8,
int: 4,
uint: 4,
float: 4,
short: 2,
ushort: 2,
uchar: 1,
list: 0,
};
let ElementMode;
(function (ElementMode) {
ElementMode[ElementMode["Vertex"] = 0] = "Vertex";
ElementMode[ElementMode["Chunk"] = 1] = "Chunk";
ElementMode[ElementMode["SH"] = 2] = "SH";
ElementMode[ElementMode["Unused"] = 3] = "Unused";
})(ElementMode || (ElementMode = {}));
let chunkMode = 1 /* ElementMode.Chunk */;
const vertexProperties = [];
const chunkProperties = [];
const filtered = header.slice(0, headerEndIndex).split("\n");
let shDegree = 0;
for (const prop of filtered) {
if (prop.startsWith("property ")) {
const [, typeName, name] = prop.split(" ");
const value = GaussianSplattingMeshBase._ValueNameToEnum(name);
if (value != 85 /* PLYValue.UNDEFINED */) {
// SH degree 1,2 or 3 for 9, 24 or 45 values
if (value >= 84 /* PLYValue.SH_44 */) {
shDegree = 3;
}
else if (value >= 64 /* PLYValue.SH_24 */) {
shDegree = Math.max(shDegree, 2);
}
else if (value >= 48 /* PLYValue.SH_8 */) {
shDegree = Math.max(shDegree, 1);
}
}
const type = GaussianSplattingMeshBase._TypeNameToEnum(typeName);
if (chunkMode == 1 /* ElementMode.Chunk */) {
chunkProperties.push({ value, type, offset: rowChunkOffset });
rowChunkOffset += offsets[typeName];
}
else if (chunkMode == 0 /* ElementMode.Vertex */) {
vertexProperties.push({ value, type, offset: rowVertexOffset });
rowVertexOffset += offsets[typeName];
}
else if (chunkMode == 2 /* ElementMode.SH */) {
// SH doesn't count for vertex row size but its properties are used to retrieve SH
vertexProperties.push({ value, type, offset: rowVertexOffset });
}
if (!offsets[typeName]) {
Logger.Warn(`Unsupported property type: ${typeName}.`);
}
}
else if (prop.startsWith("element ")) {
const [, type] = prop.split(" ");
if (type == "chunk") {
chunkMode = 1 /* ElementMode.Chunk */;
}
else if (type == "vertex") {
chunkMode = 0 /* ElementMode.Vertex */;
}
else if (type == "sh") {
chunkMode = 2 /* ElementMode.SH */;
}
else {
chunkMode = 3 /* ElementMode.Unused */;
}
}
}
const dataView = new DataView(data, headerEndIndex + headerEnd.length);
const buffer = new ArrayBuffer(GaussianSplattingMeshBase._RowOutputLength * vertexCount);
let shBuffer = null;
let shCoefficientCount = 0;
if (shDegree) {
const shVectorCount = (shDegree + 1) * (shDegree + 1) - 1;
shCoefficientCount = shVectorCount * 3;
shBuffer = new ArrayBuffer(shCoefficientCount * vertexCount);
}
return {
vertexCount: vertexCount,
chunkCount: chunkCount,
rowVertexLength: rowVertexOffset,
rowChunkLength: rowChunkOffset,
vertexProperties: vertexProperties,
chunkProperties: chunkProperties,
dataView: dataView,
buffer: buffer,
shDegree: shDegree,
shCoefficientCount: shCoefficientCount,
shBuffer: shBuffer,
};
}
static _GetCompressedChunks(header, offset) {
if (!header.chunkCount) {
return null;
}
const dataView = header.dataView;
const compressedChunks = new Array(header.chunkCount);
for (let i = 0; i < header.chunkCount; i++) {
const currentChunk = {
min: new Vector3(),
max: new Vector3(),
minScale: new Vector3(),
maxScale: new Vector3(),
minColor: new Vector3(0, 0, 0),
maxColor: new Vector3(1, 1, 1),
};
compressedChunks[i] = currentChunk;
for (let propertyIndex = 0; propertyIndex < header.chunkProperties.length; propertyIndex++) {
const property = header.chunkProperties[propertyIndex];
let value;
switch (property.type) {
case 0 /* PLYType.FLOAT */:
value = dataView.getFloat32(property.offset + offset.value, true);
break;
default:
continue;
}
switch (property.value) {
case 0 /* PLYValue.MIN_X */:
currentChunk.min.x = value;
break;
case 1 /* PLYValue.MIN_Y */:
currentChunk.min.y = value;
break;
case 2 /* PLYValue.MIN_Z */:
currentChunk.min.z = value;
break;
case 3 /* PLYValue.MAX_X */:
currentChunk.max.x = value;
break;
case 4 /* PLYValue.MAX_Y */:
currentChunk.max.y = value;
break;
case 5 /* PLYValue.MAX_Z */:
currentChunk.max.z = value;
break;
case 6 /* PLYValue.MIN_SCALE_X */:
currentChunk.minScale.x = value;
break;
case 7 /* PLYValue.MIN_SCALE_Y */:
currentChunk.minScale.y = value;
break;
case 8 /* PLYValue.MIN_SCALE_Z */:
currentChunk.minScale.z = value;
break;
case 9 /* PLYValue.MAX_SCALE_X */:
currentChunk.maxScale.x = value;
break;
case 10 /* PLYValue.MAX_SCALE_Y */:
currentChunk.maxScale.y = value;
break;
case 11 /* PLYValue.MAX_SCALE_Z */:
currentChunk.maxScale.z = value;
break;
case 34 /* PLYValue.MIN_COLOR_R */:
currentChunk.minColor.x = value;
break;
case 35 /* PLYValue.MIN_COLOR_G */:
currentChunk.minColor.y = value;
break;
case 36 /* PLYValue.MIN_COLOR_B */:
currentChunk.minColor.z = value;
break;
case 37 /* PLYValue.MAX_COLOR_R */:
currentChunk.maxColor.x = value;
break;
case 38 /* PLYValue.MAX_COLOR_G */:
currentChunk.maxColor.y = value;
break;
case 39 /* PLYValue.MAX_COLOR_B */:
currentChunk.maxColor.z = value;
break;
}
}
offset.value += header.rowChunkLength;
}
return compressedChunks;
}
static _GetSplat(header, index, compressedChunks, offset) {
const q = TmpVectors.Quaternion[0];
const temp3 = TmpVectors.Vector3[0];
const rowOutputLength = GaussianSplattingMeshBase._RowOutputLength;
const buffer = header.buffer;
const dataView = header.dataView;
const position = new Float32Array(buffer, index * rowOutputLength, 3);
const scale = new Float32Array(buffer, index * rowOutputLength + 12, 3);
const rgba = new Uint8ClampedArray(buffer, index * rowOutputLength + 24, 4);
const rot = new Uint8ClampedArray(buffer, index * rowOutputLength + 28, 4);
let sh = null;
if (header.shBuffer) {
sh = new Uint8ClampedArray(header.shBuffer, index * header.shCoefficientCount, header.shCoefficientCount);
}
const chunkIndex = index >> 8;
let r0 = 255;
let r1 = 0;
let r2 = 0;
let r3 = 0;
const plySH = [];
for (let propertyIndex = 0; propertyIndex < header.vertexProperties.length; propertyIndex++) {
const property = header.vertexProperties[propertyIndex];
let value;
switch (property.type) {
case 0 /* PLYType.FLOAT */:
value = dataView.getFloat32(offset.value + property.offset, true);
break;
case 1 /* PLYType.INT */:
value = dataView.getInt32(offset.value + property.offset, true);
break;
case 2 /* PLYType.UINT */:
value = dataView.getUint32(offset.value + property.offset, true);
break;
case 3 /* PLYType.DOUBLE */:
value = dataView.getFloat64(offset.value + property.offset, true);
break;
case 4 /* PLYType.UCHAR */:
value = dataView.getUint8(offset.value + property.offset);
break;
default:
continue;
}
switch (property.value) {
case 12 /* PLYValue.PACKED_POSITION */:
{
const compressedChunk = compressedChunks[chunkIndex];
Unpack111011(value, temp3);
position[0] = Scalar.Lerp(compressedChunk.min.x, compressedChunk.max.x, temp3.x);
position[1] = Scalar.Lerp(compressedChunk.min.y, compressedChunk.max.y, temp3.y);
position[2] = Scalar.Lerp(compressedChunk.min.z, compressedChunk.max.z, temp3.z);
}
break;
case 13 /* PLYValue.PACKED_ROTATION */:
{
UnpackRot(value, q);
r0 = q.x;
r1 = q.y;
r2 = q.z;
r3 = q.w;
}
break;
case 14 /* PLYValue.PACKED_SCALE */:
{
const compressedChunk = compressedChunks[chunkIndex];
Unpack111011(value, temp3);
scale[0] = Math.exp(Scalar.Lerp(compressedChunk.minScale.x, compressedChunk.maxScale.x, temp3.x));
scale[1] = Math.exp(Scalar.Lerp(compressedChunk.minScale.y, compressedChunk.maxScale.y, temp3.y));
scale[2] = Math.exp(Scalar.Lerp(compressedChunk.minScale.z, compressedChunk.maxScale.z, temp3.z));
}
break;
case 15 /* PLYValue.PACKED_COLOR */:
{
const compressedChunk = compressedChunks[chunkIndex];
Unpack8888(value, rgba);
rgba[0] = Scalar.Lerp(compressedChunk.minColor.x, compressedChunk.maxColor.x, rgba[0] / 255) * 255;
rgba[1] = Scalar.Lerp(compressedChunk.minColor.y, compressedChunk.maxColor.y, rgba[1] / 255) * 255;
rgba[2] = Scalar.Lerp(compressedChunk.minColor.z, compressedChunk.maxColor.z, rgba[2] / 255) * 255;
}
break;
case 16 /* PLYValue.X */:
position[0] = value;
break;
case 17 /* PLYValue.Y */:
position[1] = value;
break;
case 18 /* PLYValue.Z */:
position[2] = value;
break;
case 19 /* PLYValue.SCALE_0 */:
scale[0] = Math.exp(value);
break;
case 20 /* PLYValue.SCALE_1 */:
scale[1] = Math.exp(value);
break;
case 21 /* PLYValue.SCALE_2 */:
scale[2] = Math.exp(value);
break;
case 22 /* PLYValue.DIFFUSE_RED */:
rgba[0] = value;
break;
case 23 /* PLYValue.DIFFUSE_GREEN */:
rgba[1] = value;
break;
case 24 /* PLYValue.DIFFUSE_BLUE */:
rgba[2] = value;
break;
case 26 /* PLYValue.F_DC_0 */:
rgba[0] = (0.5 + GaussianSplattingMeshBase._SH_C0 * value) * 255;
break;
case 27 /* PLYValue.F_DC_1 */:
rgba[1] = (0.5 + GaussianSplattingMeshBase._SH_C0 * value) * 255;
break;
case 28 /* PLYValue.F_DC_2 */:
rgba[2] = (0.5 + GaussianSplattingMeshBase._SH_C0 * value) * 255;
break;
case 29 /* PLYValue.F_DC_3 */:
rgba[3] = (0.5 + GaussianSplattingMeshBase._SH_C0 * value) * 255;
break;
case 25 /* PLYValue.OPACITY */:
rgba[3] = (1 / (1 + Math.exp(-value))) * 255;
break;
case 30 /* PLYValue.ROT_0 */:
r0 = value;
break;
case 31 /* PLYValue.ROT_1 */:
r1 = value;
break;
case 32 /* PLYValue.ROT_2 */:
r2 = value;
break;
case 33 /* PLYValue.ROT_3 */:
r3 = value;
break;
}
if (sh && property.value >= 40 /* PLYValue.SH_0 */ && property.value <= 84 /* PLYValue.SH_44 */) {
const shIndex = property.value - 40 /* PLYValue.SH_0 */;
if (property.type == 4 /* PLYType.UCHAR */ && header.chunkCount) {
// compressed ply. dataView points to beginning of vertex
// could be improved with a direct copy instead of a per SH index computation + copy
const compressedValue = dataView.getUint8(header.rowChunkLength * header.chunkCount + header.vertexCount * header.rowVertexLength + index * header.shCoefficientCount + shIndex);
// compressed .ply SH import : https://github.com/playcanvas/engine/blob/fda3f0368b45d7381f0b5a1722bd2056128eaebe/src/scene/gsplat/gsplat-compressed-data.js#L88C81-L88C98
plySH[shIndex] = (compressedValue * (8 / 255) - 4) * 127.5 + 127.5;
}
else {
const clampedValue = Scalar.Clamp(value * 127.5 + 127.5, 0, 255);
plySH[shIndex] = clampedValue;
}
}
}
if (sh) {
const shDim = header.shDegree == 1 ? 3 : header.shDegree == 2 ? 8 : 15;
for (let j = 0; j < shDim; j++) {
sh[j * 3 + 0] = plySH[j];
sh[j * 3 + 1] = plySH[j + shDim];
sh[j * 3 + 2] = plySH[j + shDim * 2];
}
}
q.set(r1, r2, r3, r0);
q.normalize();
rot[0] = q.w * 127.5 + 127.5;
rot[1] = q.x * 127.5 + 127.5;
rot[2] = q.y * 127.5 + 127.5;
rot[3] = q.z * 127.5 + 127.5;
offset.value += header.rowVertexLength;
}
/**
* Converts a .ply data with SH coefficients splat
* if data array buffer is not ply, returns the original buffer
* @param data the .ply data to load
* @param useCoroutine use coroutine and yield
* @returns the loaded splat buffer and optional array of sh coefficients
*/
static *ConvertPLYWithSHToSplat(data, useCoroutine = false) {
const header = GaussianSplattingMeshBase.ParseHeader(data);
if (!header) {
return { buffer: data };
}
const offset = { value: 0 };
const compressedChunks = GaussianSplattingMeshBase._GetCompressedChunks(heade