@babylonjs/core
Version:
Getting started? Play directly with the Babylon.js API using our [playground](https://playground.babylonjs.com/). It also contains a lot of samples to learn how to use it.
555 lines (554 loc) • 20.9 kB
TypeScript
import { type Scene } from "../../scene.js";
import { type DeepImmutable, type Nullable } from "../../types.js";
import { type BaseTexture } from "../../Materials/Textures/baseTexture.js";
import { SubMesh } from "../subMesh.js";
import { type AbstractMesh } from "../abstractMesh.js";
import { Mesh } from "../mesh.js";
import { Matrix, Vector2, Vector3 } from "../../Maths/math.vector.js";
import "../thinInstanceMesh.js";
import { type Material } from "../../Materials/material.js";
import { type Effect } from "../../Materials/effect.js";
import { type Camera } from "../../Cameras/camera.js";
interface IUpdateOptions {
flipY?: boolean;
/** @internal When set, skips reprocessing splats [0, previousVertexCount) and copies from cached arrays instead. */
previousVertexCount?: number;
}
/**
* Representation of the types
*/
declare enum PLYType {
FLOAT = 0,
INT = 1,
UINT = 2,
DOUBLE = 3,
UCHAR = 4,
UNDEFINED = 5
}
/**
* Usage types of the PLY values
*/
declare enum PLYValue {
MIN_X = 0,
MIN_Y = 1,
MIN_Z = 2,
MAX_X = 3,
MAX_Y = 4,
MAX_Z = 5,
MIN_SCALE_X = 6,
MIN_SCALE_Y = 7,
MIN_SCALE_Z = 8,
MAX_SCALE_X = 9,
MAX_SCALE_Y = 10,
MAX_SCALE_Z = 11,
PACKED_POSITION = 12,
PACKED_ROTATION = 13,
PACKED_SCALE = 14,
PACKED_COLOR = 15,
X = 16,
Y = 17,
Z = 18,
SCALE_0 = 19,
SCALE_1 = 20,
SCALE_2 = 21,
DIFFUSE_RED = 22,
DIFFUSE_GREEN = 23,
DIFFUSE_BLUE = 24,
OPACITY = 25,
F_DC_0 = 26,
F_DC_1 = 27,
F_DC_2 = 28,
F_DC_3 = 29,
ROT_0 = 30,
ROT_1 = 31,
ROT_2 = 32,
ROT_3 = 33,
MIN_COLOR_R = 34,
MIN_COLOR_G = 35,
MIN_COLOR_B = 36,
MAX_COLOR_R = 37,
MAX_COLOR_G = 38,
MAX_COLOR_B = 39,
SH_0 = 40,
SH_1 = 41,
SH_2 = 42,
SH_3 = 43,
SH_4 = 44,
SH_5 = 45,
SH_6 = 46,
SH_7 = 47,
SH_8 = 48,
SH_9 = 49,
SH_10 = 50,
SH_11 = 51,
SH_12 = 52,
SH_13 = 53,
SH_14 = 54,
SH_15 = 55,
SH_16 = 56,
SH_17 = 57,
SH_18 = 58,
SH_19 = 59,
SH_20 = 60,
SH_21 = 61,
SH_22 = 62,
SH_23 = 63,
SH_24 = 64,
SH_25 = 65,
SH_26 = 66,
SH_27 = 67,
SH_28 = 68,
SH_29 = 69,
SH_30 = 70,
SH_31 = 71,
SH_32 = 72,
SH_33 = 73,
SH_34 = 74,
SH_35 = 75,
SH_36 = 76,
SH_37 = 77,
SH_38 = 78,
SH_39 = 79,
SH_40 = 80,
SH_41 = 81,
SH_42 = 82,
SH_43 = 83,
SH_44 = 84,
UNDEFINED = 85
}
/**
* Property field found in PLY header
*/
export type PlyProperty = {
/**
* Value usage
*/
value: PLYValue;
/**
* Value type
*/
type: PLYType;
/**
* offset in byte from te beginning of the splat
*/
offset: number;
};
/**
* meta info on Splat file
*/
export interface PLYHeader {
/**
* number of splats
*/
vertexCount: number;
/**
* number of spatial chunks for compressed ply
*/
chunkCount: number;
/**
* length in bytes of the vertex info
*/
rowVertexLength: number;
/**
* length in bytes of the chunk
*/
rowChunkLength: number;
/**
* array listing properties per vertex
*/
vertexProperties: PlyProperty[];
/**
* array listing properties per chunk
*/
chunkProperties: PlyProperty[];
/**
* data view for parsing chunks and vertices
*/
dataView: DataView;
/**
* buffer for the data view
*/
buffer: ArrayBuffer;
/**
* degree of SH coefficients
*/
shDegree: number;
/**
* number of coefficient per splat
*/
shCoefficientCount: number;
/**
* buffer for SH coefficients
*/
shBuffer: ArrayBuffer | null;
}
/**
* Base class for Gaussian Splatting meshes. Contains all single-cloud rendering logic.
* @internal Use GaussianSplattingMesh instead; this class is an internal implementation detail.
*/
export declare class GaussianSplattingMeshBase extends Mesh {
/** @internal */
_vertexCount: number;
protected _worker: Nullable<Worker>;
private _modelViewProjectionMatrix;
private _depthMix;
protected _canPostToWorker: boolean;
private _readyToDisplay;
protected _covariancesATexture: Nullable<BaseTexture>;
protected _covariancesBTexture: Nullable<BaseTexture>;
protected _centersTexture: Nullable<BaseTexture>;
protected _colorsTexture: Nullable<BaseTexture>;
protected _splatPositions: Nullable<Float32Array>;
private _splatIndex;
protected _shTextures: Nullable<BaseTexture[]>;
/** @internal */
_splatsData: Nullable<ArrayBuffer>;
/** @internal */
_shData: Nullable<Uint8Array[]>;
private _textureSize;
protected readonly _keepInRam: boolean;
protected _alwaysRetainSplatsData: boolean;
private _delayedTextureUpdate;
protected _useRGBACovariants: boolean;
private _material;
private _tmpCovariances;
private _sortIsDirty;
protected _cachedBoundingMin: Nullable<Vector3>;
protected _cachedBoundingMax: Nullable<Vector3>;
private static _RowOutputLength;
private static _SH_C0;
private static _SplatBatchSize;
private static _PlyConversionBatchSize;
/** @internal */
_shDegree: number;
private static readonly _BatchSize;
private _cameraViewInfos;
private static readonly _DefaultViewUpdateThreshold;
/**
* Cosine value of the angle threshold to update view dependent splat sorting. Default is 0.0001.
*/
viewUpdateThreshold: number;
protected _disableDepthSort: boolean;
/**
* If true, disables depth sorting of the splats (default: false)
*/
get disableDepthSort(): boolean;
set disableDepthSort(value: boolean);
/**
* View direction factor used to compute the SH view direction in the shader.
* @deprecated Not used anymore for SH rendering
*/
get viewDirectionFactor(): import("../../types.js").DeepImmutableObject<Vector3>;
/**
* SH degree. 0 = no sh (default). 1 = 3 parameters. 2 = 8 parameters. 3 = 15 parameters.
* Value is clamped between 0 and the maximum degree available from loaded data.
*/
get shDegree(): number;
set shDegree(value: number);
/**
* Maximum SH degree available from the loaded data.
*/
get maxShDegree(): number;
/**
* Number of splats in the mesh
*/
get splatCount(): number | undefined;
/**
* returns the splats data array buffer that contains in order : postions (3 floats), size (3 floats), color (4 bytes), orientation quaternion (4 bytes)
* Only available if the mesh was created with keepInRam: true
*/
get splatsData(): Nullable<ArrayBuffer>;
/**
* returns the SH data arrays
* Only available if the mesh was created with keepInRam: true
*/
get shData(): Nullable<Uint8Array<ArrayBufferLike>[]>;
/**
* Set the number of batch (a batch is 16384 splats) after which a display update is performed
* A value of 0 (default) means display update will not happens before splat is ready.
*/
static ProgressiveUpdateAmount: number;
/**
* Gets the covariancesA texture
*/
get covariancesATexture(): Nullable<BaseTexture>;
/**
* Gets the covariancesB texture
*/
get covariancesBTexture(): Nullable<BaseTexture>;
/**
* Gets the centers texture
*/
get centersTexture(): Nullable<BaseTexture>;
/**
* Gets the colors texture
*/
get colorsTexture(): Nullable<BaseTexture>;
/**
* Gets the SH textures
*/
get shTextures(): Nullable<BaseTexture[]>;
/**
* Gets the kernel size
* Documentation and mathematical explanations here:
* https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093
* https://github.com/autonomousvision/mip-splatting/issues/18#issuecomment-1929388931
*/
get kernelSize(): number;
/**
* Get the compensation state
*/
get compensation(): boolean;
private _loadingPromise;
/**
* set rendering material
*/
set material(value: Material);
/**
* get rendering material
*/
get material(): Nullable<Material>;
private static _MakeSplatGeometryForMesh;
/**
* Creates a new gaussian splatting mesh
* @param name defines the name of the mesh
* @param url defines the url to load from (optional)
* @param scene defines the hosting scene (optional)
* @param keepInRam keep datas in ram for editing purpose
*/
constructor(name: string, url?: Nullable<string>, scene?: Nullable<Scene>, keepInRam?: boolean);
/**
* Get the loading promise when loading the mesh from a URL in the constructor
* @returns constructor loading promise or null if no URL was provided
*/
getLoadingPromise(): Promise<void> | null;
/**
* Returns the class name
* @returns "GaussianSplattingMeshBase"
*/
getClassName(): string;
/**
* Returns the total number of vertices (splats) within the mesh
* @returns the total number of vertices
*/
getTotalVertices(): number;
/**
* Is this node ready to be used/rendered
* @param completeCheck defines if a complete check (including materials and lights) has to be done (false by default)
* @returns true when ready
*/
isReady(completeCheck?: boolean): boolean;
_getCameraDirection(camera: Camera): Vector3;
/** @internal */
_postToWorker(forced?: boolean): void;
/**
* Triggers the draw call for the mesh. Usually, you don't need to call this method by your own because the mesh rendering is handled by the scene rendering manager
* @param subMesh defines the subMesh to render
* @param enableAlphaMode defines if alpha mode can be changed
* @param effectiveMeshReplacement defines an optional mesh used to provide info for the rendering
* @returns the current mesh
*/
render(subMesh: SubMesh, enableAlphaMode: boolean, effectiveMeshReplacement?: AbstractMesh): Mesh;
private static _TypeNameToEnum;
private static _ValueNameToEnum;
/**
* Parse a PLY file header and returns metas infos on splats and chunks
* @param data the loaded buffer
* @returns a PLYHeader
*/
static ParseHeader(data: ArrayBuffer): PLYHeader | null;
private static _GetCompressedChunks;
private static _GetSplat;
/**
* Converts a .ply data with SH coefficients splat
* if data array buffer is not ply, returns the original buffer
* @param data the .ply data to load
* @param useCoroutine use coroutine and yield
* @returns the loaded splat buffer and optional array of sh coefficients
*/
static ConvertPLYWithSHToSplat(data: ArrayBuffer, useCoroutine?: boolean): Generator<undefined, {
buffer: ArrayBuffer;
sh?: undefined;
} | {
buffer: ArrayBuffer;
sh: Uint8Array<ArrayBuffer>[] | null;
}, unknown>;
/**
* Converts a .ply data array buffer to splat
* if data array buffer is not ply, returns the original buffer
* @param data the .ply data to load
* @param useCoroutine use coroutine and yield
* @returns the loaded splat buffer without SH coefficient, whether ply contains or not SH.
*/
static ConvertPLYToSplat(data: ArrayBuffer, useCoroutine?: boolean): Generator<undefined, ArrayBuffer, unknown>;
/**
* Converts a .ply data array buffer to splat
* if data array buffer is not ply, returns the original buffer
* @param data the .ply data to load
* @returns the loaded splat buffer
*/
static ConvertPLYToSplatAsync(data: ArrayBuffer): Promise<ArrayBuffer>;
/**
* Converts a .ply with SH data array buffer to splat
* if data array buffer is not ply, returns the original buffer
* @param data the .ply data to load
* @returns the loaded splat buffer with SH
*/
static ConvertPLYWithSHToSplatAsync(data: ArrayBuffer): Promise<{
buffer: ArrayBuffer;
sh?: undefined;
} | {
buffer: ArrayBuffer;
sh: Uint8Array<ArrayBuffer>[] | null;
}>;
/**
* Loads a .splat Gaussian Splatting array buffer asynchronously
* @param data arraybuffer containing splat file
* @returns a promise that resolves when the operation is complete
*/
loadDataAsync(data: ArrayBuffer): Promise<void>;
/**
* Loads a Gaussian or Splatting file asynchronously
* @param url path to the splat file to load
* @param scene optional scene it belongs to
* @returns a promise that resolves when the operation is complete
* @deprecated Please use SceneLoader.ImportMeshAsync instead
*/
loadFileAsync(url: string, scene?: Scene): Promise<void>;
/**
* Releases resources associated with this mesh.
* @param doNotRecurse Set to true to not recurse into each children (recurse into each children by default)
*/
dispose(doNotRecurse?: boolean): void;
protected _copyTextures(source: GaussianSplattingMeshBase): void;
/**
* Returns a new Mesh object generated from the current mesh properties.
* @param name is a string, the name given to the new mesh
* @returns a new Gaussian Splatting Mesh
*/
clone(name?: string): GaussianSplattingMeshBase;
private static _CreateWorker;
protected _makeEmptySplat(index: number, covA: Uint16Array, covB: Uint16Array, colorArray: Uint8Array): void;
/**
* Processes a single splat from the source buffer (at srcIndex) and writes the result into
* the destination texture arrays at dstIndex. This decoupling allows addPart to feed multiple
* independent source buffers into a single set of destination arrays without merging them first.
* @param dstIndex - destination splat index (into _splatPositions, covA, covB, colorArray)
* @param fBuffer - float32 view of the source .splat buffer
* @param uBuffer - uint8 view of the source .splat buffer
* @param covA - destination covariancesA array
* @param covB - destination covariancesB array
* @param colorArray - destination color array
* @param minimum - accumulated bounding minimum (updated in-place)
* @param maximum - accumulated bounding maximum (updated in-place)
* @param flipY - whether to negate the Y position
* @param srcIndex - source splat index (defaults to dstIndex when omitted)
*/
protected _makeSplat(dstIndex: number, fBuffer: Float32Array, uBuffer: Uint8Array, covA: Uint16Array, covB: Uint16Array, colorArray: Uint8Array, minimum: Vector3, maximum: Vector3, flipY: boolean, srcIndex?: number): void;
protected _onUpdateTextures(_textureSize: Vector2): void;
/**
* Called when part index data is received during a data load. Override to store and manage
* part index state (e.g. allocating the padded Uint8Array).
* No-op in the base class.
* @param _partIndices - the raw part indices passed in by the caller
* @param _textureLength - the padded texture length (width × height) to allocate into
*/
protected _onIndexDataReceived(_partIndices: Uint8Array, _textureLength: number): void;
/**
* Called at the start of an incremental texture update, before any splats are processed.
* Override to perform incremental-specific setup, such as ensuring the part-index GPU texture
* exists before the sub-texture upload begins.
* No-op in the base class.
* @param _textureSize - current texture dimensions
*/
protected _onIncrementalUpdateStart(_textureSize: Vector2): void;
/**
* Whether this mesh is in compound mode (has at least one part added via addPart).
* Returns `false` in the base class; overridden to return `true` in the compound subclass.
* Consumed by the material and depth renderer to toggle compound-specific shader paths.
* @internal
*/
get isCompound(): boolean;
protected _setDelayedTextureUpdate(covA: Uint16Array, covB: Uint16Array, colorArray: Uint8Array, sh?: Uint8Array[]): void;
protected _updateTextures(covA: Uint16Array, covB: Uint16Array, colorArray: Uint8Array, sh?: Uint8Array[]): void;
/**
* Checks whether the GPU textures can be incrementally updated for a new addPart operation,
* avoiding a full texture re-upload for existing splats.
* Requires that the GPU textures already exist and the texture height won't change.
* @param previousVertexCount - The number of splats previously committed to GPU
* @param vertexCount - The new total number of splats
* @returns true when only the new splat region needs to be uploaded
*/
protected _canReuseCachedData(previousVertexCount: number, vertexCount: number): boolean;
/**
* Posts updated positions to the sort worker and marks the sort as dirty.
* Called after processing new splats so the worker can re-sort with the complete position set.
* Subclasses (e.g. compound) may override to additionally post part-index data.
*/
protected _notifyWorkerNewData(): void;
private _updateData;
/**
* Update asynchronously the buffer
* @param data array buffer containing center, color, orientation and scale of splats
* @param sh optional array of uint8 array for SH data
* @param partIndices optional array of uint8 for rig node indices
* @returns a promise
*/
updateDataAsync(data: ArrayBuffer, sh?: Uint8Array[], partIndices?: Uint8Array): Promise<void>;
/**
* @experimental
* Update data from GS (position, orientation, color, scaling)
* @param data array that contain all the datas
* @param sh optional array of uint8 array for SH data
* @param options optional informations on how to treat data (needs to be 3rd for backward compatibility)
* @param partIndices optional array of uint8 for rig node indices
*/
updateData(data: ArrayBuffer, sh?: Uint8Array[], options?: IUpdateOptions, partIndices?: Uint8Array): void;
/**
* Refreshes the bounding info, taking into account all the thin instances defined
* @returns the current Gaussian Splatting
*/
refreshBoundingInfo(): Mesh;
protected _updateSplatIndexBuffer(vertexCount: number): void;
protected _updateTextureFromData: (texture: BaseTexture, data: ArrayBufferView, width: number, lineStart: number, lineCount: number) => void;
protected _updateSubTextures(centers: Float32Array, covA: Uint16Array, covB: Uint16Array, colors: Uint8Array, lineStart: number, lineCount: number, sh?: Uint8Array[]): void;
protected _instantiateWorker(): void;
protected _getTextureSize(length: number): Vector2;
/**
* Called after the sort worker has been created and the initial positions message has been sent.
* Override in subclasses to post any additional setup messages the worker needs (e.g. group
* indices, per-part matrices, etc.).
* @param _worker the newly created worker
*/
protected _onWorkerCreated(_worker: Worker): void;
/**
* Called by the material to bind any extra shader uniforms that are specific to this mesh type.
* The base implementation is a no-op; override in subclasses to bind additional data.
* @param _effect the shader effect that is being bound
* @internal
*/
bindExtraEffectUniforms(_effect: Effect): void;
/**
* Processes all splats from a source GaussianSplattingMesh directly into the destination
* texture arrays starting at dstOffset. This is the core of the texture-direct compound API:
* no merged CPU buffer is ever created; each source mesh is written straight into its region.
*
* @param source - The source mesh whose splats are appended
* @param dstOffset - The destination splat index at which writing starts
* @param covA - Destination covA array (full texture size)
* @param covB - Destination covB array (full texture size)
* @param colorArray - Destination color array (full texture size)
* @param sh - Destination SH arrays (full texture size), or undefined
* @param minimum - Accumulated bounding min (updated in-place)
* @param maximum - Accumulated bounding max (updated in-place)
* @internal Use GaussianSplattingMesh.addPart instead
*/
protected _appendSourceToArrays(source: GaussianSplattingMeshBase, dstOffset: number, covA: Uint16Array, covB: Uint16Array, colorArray: Uint8Array, sh: Uint8Array[] | undefined, minimum: Vector3, maximum: Vector3): void;
/**
* Modifies the splats according to the passed transformation matrix.
* @param transform defines the transform matrix to use
* @returns the current mesh
*/
bakeTransformIntoVertices(transform: DeepImmutable<Matrix>): Mesh;
}
export {};