From a60052b2e38bc3aa87c914b680c0d0f796374e73 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 20 Nov 2024 11:34:37 +0000 Subject: [PATCH 1/6] first --- src/framework/parsers/ply.js | 107 ++++++------- src/platform/graphics/texture.js | 2 +- src/scene/gsplat/gsplat-compressed-data.js | 34 ++++ .../gsplat/gsplat-compressed-material.js | 151 +++++++++++++++++- src/scene/gsplat/gsplat-compressed.js | 69 +++++++- src/scene/gsplat/gsplat-data.js | 9 ++ src/scene/gsplat/gsplat.js | 2 +- 7 files changed, 313 insertions(+), 61 deletions(-) diff --git a/src/framework/parsers/ply.js b/src/framework/parsers/ply.js index 2171a56ccbf..b12a4715e49 100644 --- a/src/framework/parsers/ply.js +++ b/src/framework/parsers/ply.js @@ -212,11 +212,30 @@ const isCompressedPly = (elements) => { 'packed_position', 'packed_rotation', 'packed_scale', 'packed_color' ]; - return elements.length === 2 && - elements[0].name === 'chunk' && - elements[0].properties.every((p, i) => p.name === chunkProperties[i] && p.type === 'float') && - elements[1].name === 'vertex' && - elements[1].properties.every((p, i) => p.name === vertexProperties[i] && p.type === 'uint'); + const band1Properties = ['0', '1', '2'].map(x => `coeff_${x}`); + const band2Properties = ['3', '4', '5', '6', '7'].map(x => `coeff_${x}`); + const band3Properties = ['8', '9', '10', '11', '12', '13', '14'].map(x => `coeff_${x}`); + const indicesProperties = ['0', '1', '2', '3'].map(x => `packed_sh_${x}`); + + const hasBaseElements = () => { + return elements[0].name === 'chunk' && + elements[0].properties.every((p, i) => p.name === chunkProperties[i] && p.type === 'float') && + elements[1].name === 'vertex' && + elements[1].properties.every((p, i) => p.name === vertexProperties[i] && p.type === 'uint'); + }; + + const hasSHElements = () => { + return elements[2].name === 'sh_band_1' && + elements[2].properties.every((p, i) => p.name === band1Properties[i] && p.type === 'ushort') && + elements[3].name === 'sh_band_2' && + elements[3].properties.every((p, i) => p.name === band2Properties[i] && p.type === 'ushort') && + elements[4].name === 'sh_band_3' && + elements[4].properties.every((p, i) => p.name === band3Properties[i] && p.type === 'ushort') && + elements[5].name === 'vertex_sh' && + elements[5].properties.every((p, i) => p.name === indicesProperties[i] && p.type === 'uint'); + }; + + return (elements.length === 2 && hasBaseElements()) || (elements.length === 6 && hasBaseElements() && hasSHElements()); }; const isFloatPly = (elements) => { @@ -230,10 +249,7 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { const result = new GSplatCompressedData(); const numChunks = elements[0].count; - const chunkSize = 12 * 4; - const numVertices = elements[1].count; - const vertexSize = 4 * 4; // evaluate the storage size for the given count (this must match the // texture size calculation in GSplatCompressed). @@ -244,64 +260,47 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { }; // allocate result - result.numSplats = elements[1].count; + result.numSplats = numVertices; result.chunkData = new Float32Array(evalStorageSize(numChunks) * 12); result.vertexData = new Uint32Array(evalStorageSize(numVertices) * 4); - let uint32StreamData; - const uint32ChunkData = new Uint32Array(result.chunkData.buffer); - const uint32VertexData = result.vertexData; - - // read chunks - let chunks = 0; - while (chunks < numChunks) { - while (streamBuf.remaining < chunkSize) { - /* eslint-disable no-await-in-loop */ - await streamBuf.read(); - } - - // ensure the uint32 view is still valid - if (uint32StreamData?.buffer !== streamBuf.data.buffer) { - uint32StreamData = new Uint32Array(streamBuf.data.buffer, 0, Math.floor(streamBuf.data.buffer.byteLength / 4)); - } + // read length bytes of data into buffer + const read = async (buffer, length) => { + const target = new Uint8Array(buffer); + let cursor = 0; - // read the next chunk of data - const toRead = Math.min(numChunks - chunks, Math.floor(streamBuf.remaining / chunkSize)); + while (cursor < length) { + while (streamBuf.remaining === 0) { + await streamBuf.read(); + } - const dstOffset = chunks * 12; - const srcOffset = streamBuf.head / 4; - for (let i = 0; i < toRead * 12; ++i) { - uint32ChunkData[dstOffset + i] = uint32StreamData[srcOffset + i]; + const toCopy = Math.min(length - cursor, streamBuf.remaining); + const src = streamBuf.data; + for (let i = 0; i < toCopy; ++i) { + target[cursor++] = src[streamBuf.head++]; + } } + }; - streamBuf.head += toRead * chunkSize; - chunks += toRead; - } + // read chunk data + await read(result.chunkData.buffer, numChunks * 12 * 4); - // read vertices - let vertices = 0; - while (vertices < numVertices) { - while (streamBuf.remaining < vertexSize) { - /* eslint-disable no-await-in-loop */ - await streamBuf.read(); - } + // read packed vertices + await read(result.vertexData.buffer, numVertices * 4 * 4); - // ensure the uint32 view is still valid - if (uint32StreamData?.buffer !== streamBuf.data.buffer) { - uint32StreamData = new Uint32Array(streamBuf.data.buffer, 0, Math.floor(streamBuf.data.buffer.byteLength / 4)); - } + // read sh data + if (elements.length === 6) { + result.band1Data = new Uint16Array(elements[2].count * 3); + await read(result.band1Data.buffer, result.band1Data.byteLength); - // read the next chunk of data - const toRead = Math.min(numVertices - vertices, Math.floor(streamBuf.remaining / vertexSize)); + result.band2Data = new Uint16Array(elements[3].count * 5); + await read(result.band2Data.buffer, result.band2Data.byteLength); - const dstOffset = vertices * 4; - const srcOffset = streamBuf.head / 4; - for (let i = 0; i < toRead * 4; ++i) { - uint32VertexData[dstOffset + i] = uint32StreamData[srcOffset + i]; - } + result.band3Data = new Uint16Array(elements[4].count * 7); + await read(result.band3Data.buffer, result.band3Data.byteLength); - streamBuf.head += toRead * vertexSize; - vertices += toRead; + result.packedSHData = new Uint32Array(evalStorageSize(numVertices) * 4); + await read(result.packedSHData.buffer, numVertices * 4 * 4); } return result; diff --git a/src/platform/graphics/texture.js b/src/platform/graphics/texture.js index 00f6bf36171..e98cad52f83 100644 --- a/src/platform/graphics/texture.js +++ b/src/platform/graphics/texture.js @@ -186,7 +186,7 @@ class Texture { * - {@link FUNC_NOTEQUAL} * * Defaults to {@link FUNC_LESS}. - * @param {Uint8Array[]|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|Uint8Array[][]} [options.levels] + * @param {Uint8Array[]|Uint16Array[]|Uint32Array|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|Uint8Array[][]} [options.levels] * - Array of Uint8Array or other supported browser interface; or a two-dimensional array * of Uint8Array if options.arrayLength is defined and greater than zero. * @param {boolean} [options.storage] - Defines if texture can be used as a storage texture by diff --git a/src/scene/gsplat/gsplat-compressed-data.js b/src/scene/gsplat/gsplat-compressed-data.js index aa93704ea7c..fdea590eeda 100644 --- a/src/scene/gsplat/gsplat-compressed-data.js +++ b/src/scene/gsplat/gsplat-compressed-data.js @@ -102,6 +102,36 @@ class GSplatCompressedData { */ vertexData; + // optional spherical harmonics data + + /** + * Contains 3 half values per band 1 palette entry + * @type {Uint16Array} + */ + band1Data; + + /** + * Contains 5 half values per band 2 palette entry + * @type {Uint16Array} + */ + band2Data; + + /** + * Contains 7 half values per band 3 palette entry + * @type {Uint16Array} + */ + band3Data; + + /** + * Contains 4 uint32 per vertex with packed bits referencing the SH palette: + * packed_sh_0 + * packed_sh_1 + * packed_sh_2 + * packed_sh_3 + * @type {Uint32Array} + */ + packedSHData; + /** * Create an iterator for accessing splat data * @@ -209,6 +239,10 @@ class GSplatCompressedData { return true; } + get hasSHData() { + return !!this.packedSHData; + } + // decompress into GSplatData decompress() { const members = [ diff --git a/src/scene/gsplat/gsplat-compressed-material.js b/src/scene/gsplat/gsplat-compressed-material.js index 232587eec18..44e9e576090 100644 --- a/src/scene/gsplat/gsplat-compressed-material.js +++ b/src/scene/gsplat/gsplat-compressed-material.js @@ -200,6 +200,141 @@ const splatCoreVS = /* glsl */ ` return vec4(v1, v2); } + +#if defined(USE_SH) + #define SH_C1 0.4886025119029199f + + #define SH_C2_0 1.0925484305920792f + #define SH_C2_1 -1.0925484305920792f + #define SH_C2_2 0.31539156525252005f + #define SH_C2_3 -1.0925484305920792f + #define SH_C2_4 0.5462742152960396f + + #define SH_C3_0 -0.5900435899266435f + #define SH_C3_1 2.890611442640554f + #define SH_C3_2 -0.4570457994644658f + #define SH_C3_3 0.3731763325901154f + #define SH_C3_4 -0.4570457994644658f + #define SH_C3_5 1.445305721320277f + #define SH_C3_6 -0.5900435899266435f + + uniform highp sampler2D band1Texture; + uniform highp sampler2D band2Texture; + uniform highp sampler2D band3Texture; + uniform highp usampler2D packedSHTexture; + + ivec2 shUV(uint index) { + return ivec2(int(index & 1023u) * 2, int(index / 1024u)); + } + + void readSHData(out vec3 sh[15]) { + // read the splat indices + uvec4 packedSHData = texelFetch(packedSHTexture, packedUV, 0); + + // generate palette uvs from packed bits + + // 11, 11, 10 + ivec2 band1r = ivec2(int(packedSHData.x >> 21u), 0); + ivec2 band1g = ivec2(int((packedSHData.x >> 10u) & 0x7ffu), 0); + ivec2 band1b = ivec2(int(packedSHData.x & 0x3ffu), 0); + + // 15, 15, 15 + ivec2 band2r = shUV((packedSHData.y >> 17u) & 0x7fffu); + ivec2 band2g = shUV((packedSHData.y >> 2u) & 0x7fffu); + ivec2 band2b = shUV(((packedSHData.y << 13u) | (packedSHData.z >> 19u)) & 0x7fffu); + + // 17, 17, 17 + ivec2 band3r = shUV((packedSHData.z >> 2u) & 0x1ffffu); + ivec2 band3g = shUV(((packedSHData.z << 15u) | (packedSHData.w >> 17u)) & 0x1ffffu); + ivec2 band3b = shUV(packedSHData.w & 0x1ffffu); + + // sample palette coefficients + + // band 1 + vec3 a = texelFetch(band1Texture, band1r, 0).xyz; + vec3 b = texelFetch(band1Texture, band1g, 0).xyz; + vec3 c = texelFetch(band1Texture, band1b, 0).xyz; + + // band 2 + vec4 d = texelFetch(band2Texture, band2r, 0); + float e = texelFetch(band2Texture, band2r + ivec2(1, 0), 0).x; + vec4 f = texelFetch(band2Texture, band2g, 0); + float g = texelFetch(band2Texture, band2g + ivec2(1, 0), 0).x; + vec4 h = texelFetch(band2Texture, band2b, 0); + float i = texelFetch(band2Texture, band2b + ivec2(1, 0), 0).x; + + // band 3 + vec4 j = texelFetch(band3Texture, band3r, 0); + vec3 k = texelFetch(band3Texture, band3r + ivec2(1, 0), 0).xyz; + vec4 l = texelFetch(band3Texture, band3g, 0); + vec3 m = texelFetch(band3Texture, band3g + ivec2(1, 0), 0).xyz; + vec4 n = texelFetch(band3Texture, band3b, 0); + vec3 o = texelFetch(band3Texture, band3b + ivec2(1, 0), 0).xyz; + + sh[0] = vec3(a.x, b.x, c.x); + sh[1] = vec3(a.y, b.y, c.y); + sh[2] = vec3(a.z, b.z, c.z); + sh[3] = vec3(d.x, f.x, h.x); + sh[4] = vec3(d.y, f.y, h.y); + sh[5] = vec3(d.z, f.z, h.z); + sh[6] = vec3(d.w, f.w, h.w); + sh[7] = vec3(e, g, i); + sh[8] = vec3(j.x, l.x, n.x); + sh[9] = vec3(j.y, l.y, n.y); + sh[10] = vec3(j.z, l.z, n.z); + sh[11] = vec3(j.w, l.w, n.w); + sh[12] = vec3(k.x, m.x, o.x); + sh[13] = vec3(k.y, m.y, o.y); + sh[14] = vec3(k.z, m.z, o.z); + } + + // see https://github.com/graphdeco-inria/gaussian-splatting/blob/main/utils/sh_utils.py + vec3 evalSH(in vec3 dir) { + + vec3 sh[15]; + readSHData(sh); + + vec3 result = vec3(0.0); + + // 1st degree + float x = dir.x; + float y = dir.y; + float z = dir.z; + + result += SH_C1 * (-sh[0] * y + sh[1] * z - sh[2] * x); + + // 2nd degree + float xx = x * x; + float yy = y * y; + float zz = z * z; + float xy = x * y; + float yz = y * z; + float xz = x * z; + + result += + sh[3] * (SH_C2_0 * xy) * + + sh[4] * (SH_C2_1 * yz) + + sh[5] * (SH_C2_2 * (2.0 * zz - xx - yy)) + + sh[6] * (SH_C2_3 * xz) + + sh[7] * (SH_C2_4 * (xx - yy)); + + // 3rd degree + result += + sh[8] * (SH_C3_0 * y * (3.0 * xx - yy)) + + sh[9] * (SH_C3_1 * xy * z) + + sh[10] * (SH_C3_2 * y * (4.0 * zz - xx - yy)) + + sh[11] * (SH_C3_3 * z * (2.0 * zz - 3.0 * xx - 3.0 * yy)) + + sh[12] * (SH_C3_4 * x * (4.0 * zz - xx - yy)) + + sh[13] * (SH_C3_5 * z * (xx - yy)) + + sh[14] * (SH_C3_6 * x * (xx - 3.0 * yy)); + + return result; + } +#else + vec3 evalSH(in vec3 dir) { + return vec3(0.0); + } +#endif `; const splatCoreFS = /* glsl */ ` @@ -255,8 +390,8 @@ class GSplatCompressedShaderGenerator { const shaderPassDefines = shaderPassInfo.shaderDefines; const defines = - `${shaderPassDefines - }#define DITHER_${options.dither.toUpperCase()}\n` + + `${shaderPassDefines}\n` + + `#define DITHER_${options.dither.toUpperCase()}\n` + `#define TONEMAP_${options.toneMapping === TONEMAP_LINEAR ? 'DISABLED' : 'ENABLED'}\n`; const vs = defines + splatCoreVS + options.vertex; @@ -267,7 +402,9 @@ class GSplatCompressedShaderGenerator { splatCoreFS + options.fragment; const defineMap = new Map(); - options.defines.forEach(value => defineMap.set(value, true)); + options.defines.forEach((value, key) => { + defineMap.set(key, value); + }); return ShaderUtils.createDefinition(device, { name: 'SplatShader', @@ -289,6 +426,8 @@ const splatMainVS = /* glsl */ ` varying mediump vec2 texCoord; varying mediump vec4 color; + uniform vec3 view_position; + mediump vec4 discardVec = vec4(0.0, 0.0, 2.0, 1.0); void main(void) @@ -340,6 +479,12 @@ const splatMainVS = /* glsl */ ` texCoord = vertex_position.xy * scale / 2.0; + #ifdef USE_SH + vec4 worldCenter = matrix_model * vec4(center, 1.0); + vec3 viewDir = normalize((worldCenter.xyz / worldCenter.w - view_position) * mat3(matrix_model)); + color.xyz = max(color.xyz + evalSH(viewDir), 0.0); + #endif + #ifndef DITHER_NONE id = float(splatId); #endif diff --git a/src/scene/gsplat/gsplat-compressed.js b/src/scene/gsplat/gsplat-compressed.js index b0e68fd4fb9..11f1c9de8bb 100644 --- a/src/scene/gsplat/gsplat-compressed.js +++ b/src/scene/gsplat/gsplat-compressed.js @@ -2,7 +2,7 @@ import { Vec2 } from '../../core/math/vec2.js'; import { Texture } from '../../platform/graphics/texture.js'; import { BoundingBox } from '../../core/shape/bounding-box.js'; import { - ADDRESS_CLAMP_TO_EDGE, FILTER_NEAREST, PIXELFORMAT_RGBA32F, PIXELFORMAT_RGBA32U + ADDRESS_CLAMP_TO_EDGE, FILTER_NEAREST, PIXELFORMAT_RGBA16F, PIXELFORMAT_RGBA32F, PIXELFORMAT_RGBA32U } from '../../platform/graphics/constants.js'; import { createGSplatCompressedMaterial } from './gsplat-compressed-material.js'; @@ -31,6 +31,18 @@ class GSplatCompressed { /** @type {Texture} */ chunkTexture; + /** @type {Texture?} */ + band1Texture; + + /** @type {Texture?} */ + band2Texture; + + /** @type {Texture?} */ + band3Texture; + + /** @type {Texture?} */ + packedSHTexture; + /** * @param {GraphicsDevice} device - The graphics device. * @param {GSplatCompressedData} gsplatData - The splat data. @@ -58,6 +70,52 @@ class GSplatCompressed { chunkSize.x *= 3; this.chunkTexture = this.createTexture('chunkData', PIXELFORMAT_RGBA32F, chunkSize, gsplatData.chunkData); + + if (gsplatData.hasSHData) { + const { band1Data, band2Data, band3Data, packedSHData } = gsplatData; + + const calcDim = (numEntries) => { + return new Vec2(2048, Math.ceil(numEntries / 1024)); + }; + + // pad ushort data with zeros + const padUint16 = (target, targetStride, src, srcStride, numEntries) => { + for (let i = 0; i < numEntries; ++i) { + for (let j = 0; j < srcStride; ++j) { + target[i * targetStride + j] = src[i * srcStride + j]; + } + } + }; + + // unpack the palette coefficients to align nicely to RGBA16F boundaries for ease of reading on gpu + + // band1 has max 2k entries, 3 coefficients each, 1 pixel per entry. + const band1Size = band1Data.length / 3; + this.band1Texture = this.createTexture('band1Palette', PIXELFORMAT_RGBA16F, new Vec2(band1Size, 1)); + padUint16(this.band1Texture.lock(), 4, band1Data, 3, band1Size); + + // band2 has max 32k entries, 5 coefficients each, 2 pixels per entry. + const band2Size = band2Data.length / 5; + this.band2Texture = this.createTexture('band2Palette', PIXELFORMAT_RGBA16F, calcDim(band2Size)); + padUint16(this.band2Texture.lock(), 8, band2Data, 5, band2Size); + + // band3 has max 128k entries, 7 coefficients each, 2 pixels per entry. + const band3Size = band3Data.length / 7; + this.band3Texture = this.createTexture('band3Palette', PIXELFORMAT_RGBA16F, calcDim(band3Size)); + padUint16(this.band3Texture.lock(), 8, band3Data, 7, band3Size); + + // packed SH data is loaded directly + this.packedSHTexture = this.createTexture('packedSHData', PIXELFORMAT_RGBA32U, this.evalTextureSize(numSplats), packedSHData); + + this.band1Texture.unlock(); + this.band2Texture.unlock(); + this.band3Texture.unlock(); + } else { + this.band1Texture = null; + this.band2Texture = null; + this.band3Texture = null; + this.packedSHTexture = null; + } } destroy() { @@ -74,6 +132,13 @@ class GSplatCompressed { result.setParameter('packedTexture', this.packedTexture); result.setParameter('chunkTexture', this.chunkTexture); result.setParameter('tex_params', new Float32Array([this.numSplats, this.packedTexture.width, this.chunkTexture.width / 3, 0])); + if (this.packedSHTexture) { + result.setDefine('USE_SH', true); + result.setParameter('band1Texture', this.band1Texture); + result.setParameter('band2Texture', this.band2Texture); + result.setParameter('band3Texture', this.band3Texture); + result.setParameter('packedSHTexture', this.packedSHTexture); + } return result; } @@ -97,7 +162,7 @@ class GSplatCompressed { * @param {string} name - The name of the texture to be created. * @param {number} format - The pixel format of the texture. * @param {Vec2} size - The width and height of the texture. - * @param {Uint8Array} [data] - The initial data to fill the texture with. + * @param {Uint8Array|Uint16Array|Uint32Array} [data] - The initial data to fill the texture with. * @returns {Texture} The created texture instance. */ createTexture(name, format, size, data) { diff --git a/src/scene/gsplat/gsplat-data.js b/src/scene/gsplat/gsplat-data.js index 206a936a61e..ae68d4ca342 100644 --- a/src/scene/gsplat/gsplat-data.js +++ b/src/scene/gsplat/gsplat-data.js @@ -322,6 +322,15 @@ class GSplatData { return false; } + get hasSHData() { + for (let i = 0; i < 45; ++i) { + if (!this.getProp(`f_rest_${i}`)) { + return false; + } + } + return true; + } + calcMortonOrder() { const calcMinMax = (arr) => { let min = arr[0]; diff --git a/src/scene/gsplat/gsplat.js b/src/scene/gsplat/gsplat.js index 0ff2ffe3f59..a7a16fa530b 100644 --- a/src/scene/gsplat/gsplat.js +++ b/src/scene/gsplat/gsplat.js @@ -87,7 +87,7 @@ class GSplat { this.updateTransformData(gsplatData); // initialize SH data - this.hasSH = getSHData(gsplatData).every(x => x); + this.hasSH = gsplatData.hasSHData; if (this.hasSH) { this.sh1to3Texture = this.createTexture('splatSH_1to3', PIXELFORMAT_RGBA32U, size); this.sh4to7Texture = this.createTexture('splatSH_4to7', PIXELFORMAT_RGBA32U, size); From 9c2dba3bb95961e071c9675c67be870330c74298 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 20 Nov 2024 16:24:21 +0000 Subject: [PATCH 2/6] support sh decompress --- src/scene/gsplat/gsplat-compressed-data.js | 97 +++++++++++++++++-- .../gsplat/gsplat-compressed-material.js | 2 +- 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/src/scene/gsplat/gsplat-compressed-data.js b/src/scene/gsplat/gsplat-compressed-data.js index fdea590eeda..8d659ca546e 100644 --- a/src/scene/gsplat/gsplat-compressed-data.js +++ b/src/scene/gsplat/gsplat-compressed-data.js @@ -1,3 +1,4 @@ +import { FloatPacking } from '../../core/math/float-packing.js'; import { Quat } from '../../core/math/quat.js'; import { Vec3 } from '../../core/math/vec3.js'; import { Vec4 } from '../../core/math/vec4.js'; @@ -9,9 +10,34 @@ import { GSplatData } from './gsplat-data.js'; const SH_C0 = 0.28209479177387814; +// https://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ +const halfToFloat = (() => { + const f32 = new Float32Array(1); + const magicF = new Float32Array(1); + const wasInfNanF = new Float32Array(1); + + const u32 = new Uint32Array(f32.buffer); + const magicU = new Uint32Array(magicF.buffer); + const wasInfNanU = new Uint32Array(wasInfNanF.buffer); + + magicU[0] = (254 - 15) << 23; + wasInfNanU[0] = (127 + 16) << 23; + + return (h) => { + u32[0] = (h & 0x7fff) << 13; + f32[0] *= magicF[0]; + + if (f32[0] >= wasInfNanF[0]) { + u32[0] |= 255 << 23; + } + u32[0] |= (h & 0x8000) << 16; // sign bit + return f32[0]; + }; +})(); + // iterator for accessing compressed splat data class SplatCompressedIterator { - constructor(gsplatData, p, r, s, c) { + constructor(gsplatData, p, r, s, c, sh) { const unpackUnorm = (value, bits) => { const t = (1 << bits) - 1; return (value & t) / t; @@ -48,8 +74,7 @@ class SplatCompressedIterator { const lerp = (a, b, t) => a * (1 - t) + b * t; - const chunkData = gsplatData.chunkData; - const vertexData = gsplatData.vertexData; + const { chunkData, vertexData, band1Data, band2Data, band3Data, packedSHData } = gsplatData; this.read = (i) => { const ci = Math.floor(i / 256) * 12; @@ -75,6 +100,49 @@ class SplatCompressedIterator { if (c) { unpack8888(c, vertexData[i * 4 + 3]); } + + if (sh) { + const bits0 = packedSHData[i * 4 + 0]; + const bits1 = packedSHData[i * 4 + 1]; + const bits2 = packedSHData[i * 4 + 2]; + const bits3 = packedSHData[i * 4 + 3]; + + const b1 = [ + bits0 >>> 21, + (bits0 >>> 10) & 0x7ff, + bits0 & 0x3ff + ]; + + const b2 = [ + bits1 >>> 17, + (bits1 >>> 2) & 0x7fff, + ((bits1 << 13) | (bits2 >>> 19)) & 0x7fff + ]; + + const b3 = [ + (bits2 >>> 2) & 0x1ffff, + ((bits2 << 15) | (bits3 >>> 17)) & 0x1ffff, + (bits3 & 0x1ffff) + ]; + + for (let i = 0; i < 3; ++i) { + sh[i * 15 + 0] = halfToFloat(band1Data[b1[i] * 3 + 0]); + sh[i * 15 + 1] = halfToFloat(band1Data[b1[i] * 3 + 1]); + sh[i * 15 + 2] = halfToFloat(band1Data[b1[i] * 3 + 2]); + sh[i * 15 + 3] = halfToFloat(band2Data[b2[i] * 5 + 0]); + sh[i * 15 + 4] = halfToFloat(band2Data[b2[i] * 5 + 1]); + sh[i * 15 + 5] = halfToFloat(band2Data[b2[i] * 5 + 2]); + sh[i * 15 + 6] = halfToFloat(band2Data[b2[i] * 5 + 3]); + sh[i * 15 + 7] = halfToFloat(band2Data[b2[i] * 5 + 4]); + sh[i * 15 + 8] = halfToFloat(band3Data[b3[i] * 7 + 0]); + sh[i * 15 + 9] = halfToFloat(band3Data[b3[i] * 7 + 1]); + sh[i * 15 + 10] = halfToFloat(band3Data[b3[i] * 7 + 2]); + sh[i * 15 + 11] = halfToFloat(band3Data[b3[i] * 7 + 3]); + sh[i * 15 + 12] = halfToFloat(band3Data[b3[i] * 7 + 4]); + sh[i * 15 + 13] = halfToFloat(band3Data[b3[i] * 7 + 5]); + sh[i * 15 + 14] = halfToFloat(band3Data[b3[i] * 7 + 6]); + } + } }; } } @@ -139,10 +207,11 @@ class GSplatCompressedData { * @param {Quat|null} [r] - the quaternion to receive splat rotation * @param {Vec3|null} [s] - the vector to receive splat scale * @param {Vec4|null} [c] - the vector to receive splat color + * @param {Float32Array|null} [sh] - the array to receive spherical harmonics data * @returns {SplatCompressedIterator} - The iterator */ - createIter(p, r, s, c) { - return new SplatCompressedIterator(this, p, r, s, c); + createIter(p, r, s, c, sh) { + return new SplatCompressedIterator(this, p, r, s, c, sh); } /** @@ -252,6 +321,15 @@ class GSplatCompressedData { 'rot_0', 'rot_1', 'rot_2', 'rot_3' ]; + // allocate spherical harmonics data + if (this.hasSHData) { + const shMembers = []; + for (let i = 0; i < 45; ++i) { + shMembers.push(`f_rest_${i}`); + } + members.splice(9, 0, ...shMembers); + } + // allocate uncompressed data const data = {}; members.forEach((name) => { @@ -262,8 +340,9 @@ class GSplatCompressedData { const r = new Quat(); const s = new Vec3(); const c = new Vec4(); + const sh = this.hasSHData ? new Float32Array(45) : null; - const iter = this.createIter(p, r, s, c); + const iter = this.createIter(p, r, s, c, sh); for (let i = 0; i < this.numSplats; ++i) { iter.read(i); @@ -286,6 +365,12 @@ class GSplatCompressedData { data.f_dc_2[i] = (c.z - 0.5) / SH_C0; // convert opacity to log sigmoid taking into account infinities at 0 and 1 data.opacity[i] = (c.w <= 0) ? -40 : (c.w >= 1) ? 40 : -Math.log(1 / c.w - 1); + + if (this.hasSHData) { + for (let c = 0; c < 45; ++c) { + data[`f_rest_${c}`][i] = sh[c]; + } + } } return new GSplatData([{ diff --git a/src/scene/gsplat/gsplat-compressed-material.js b/src/scene/gsplat/gsplat-compressed-material.js index 44e9e576090..73dbd286c67 100644 --- a/src/scene/gsplat/gsplat-compressed-material.js +++ b/src/scene/gsplat/gsplat-compressed-material.js @@ -239,7 +239,7 @@ const splatCoreVS = /* glsl */ ` ivec2 band1b = ivec2(int(packedSHData.x & 0x3ffu), 0); // 15, 15, 15 - ivec2 band2r = shUV((packedSHData.y >> 17u) & 0x7fffu); + ivec2 band2r = shUV(packedSHData.y >> 17u); ivec2 band2g = shUV((packedSHData.y >> 2u) & 0x7fffu); ivec2 band2b = shUV(((packedSHData.y << 13u) | (packedSHData.z >> 19u)) & 0x7fffu); From 5409f3cc6cedf3e7aff4f5edf4a8572da26c5bf4 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 20 Nov 2024 16:39:43 +0000 Subject: [PATCH 3/6] small --- src/scene/gsplat/gsplat-compressed-data.js | 35 ++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/scene/gsplat/gsplat-compressed-data.js b/src/scene/gsplat/gsplat-compressed-data.js index 8d659ca546e..1142f91b5c2 100644 --- a/src/scene/gsplat/gsplat-compressed-data.js +++ b/src/scene/gsplat/gsplat-compressed-data.js @@ -126,21 +126,26 @@ class SplatCompressedIterator { ]; for (let i = 0; i < 3; ++i) { - sh[i * 15 + 0] = halfToFloat(band1Data[b1[i] * 3 + 0]); - sh[i * 15 + 1] = halfToFloat(band1Data[b1[i] * 3 + 1]); - sh[i * 15 + 2] = halfToFloat(band1Data[b1[i] * 3 + 2]); - sh[i * 15 + 3] = halfToFloat(band2Data[b2[i] * 5 + 0]); - sh[i * 15 + 4] = halfToFloat(band2Data[b2[i] * 5 + 1]); - sh[i * 15 + 5] = halfToFloat(band2Data[b2[i] * 5 + 2]); - sh[i * 15 + 6] = halfToFloat(band2Data[b2[i] * 5 + 3]); - sh[i * 15 + 7] = halfToFloat(band2Data[b2[i] * 5 + 4]); - sh[i * 15 + 8] = halfToFloat(band3Data[b3[i] * 7 + 0]); - sh[i * 15 + 9] = halfToFloat(band3Data[b3[i] * 7 + 1]); - sh[i * 15 + 10] = halfToFloat(band3Data[b3[i] * 7 + 2]); - sh[i * 15 + 11] = halfToFloat(band3Data[b3[i] * 7 + 3]); - sh[i * 15 + 12] = halfToFloat(band3Data[b3[i] * 7 + 4]); - sh[i * 15 + 13] = halfToFloat(band3Data[b3[i] * 7 + 5]); - sh[i * 15 + 14] = halfToFloat(band3Data[b3[i] * 7 + 6]); + const i1 = b1[i] * 3; + sh[i * 15 + 0] = halfToFloat(band1Data[i1 + 0]); + sh[i * 15 + 1] = halfToFloat(band1Data[i1 + 1]); + sh[i * 15 + 2] = halfToFloat(band1Data[i1 + 2]); + + const i2 = b2[i] * 5; + sh[i * 15 + 3] = halfToFloat(band2Data[i2 + 0]); + sh[i * 15 + 4] = halfToFloat(band2Data[i2 + 1]); + sh[i * 15 + 5] = halfToFloat(band2Data[i2 + 2]); + sh[i * 15 + 6] = halfToFloat(band2Data[i2 + 3]); + sh[i * 15 + 7] = halfToFloat(band2Data[i2 + 4]); + + const i3 = b3[i] * 7; + sh[i * 15 + 8] = halfToFloat(band3Data[i3 + 0]); + sh[i * 15 + 9] = halfToFloat(band3Data[i3 + 1]); + sh[i * 15 + 10] = halfToFloat(band3Data[i3 + 2]); + sh[i * 15 + 11] = halfToFloat(band3Data[i3 + 3]); + sh[i * 15 + 12] = halfToFloat(band3Data[i3 + 4]); + sh[i * 15 + 13] = halfToFloat(band3Data[i3 + 5]); + sh[i * 15 + 14] = halfToFloat(band3Data[i3 + 6]); } } }; From bd314ae245cfe244dcabb6b0010c05944bb3a87b Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 20 Nov 2024 16:45:08 +0000 Subject: [PATCH 4/6] lint --- src/framework/parsers/ply.js | 1 + src/scene/gsplat/gsplat-compressed-data.js | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/framework/parsers/ply.js b/src/framework/parsers/ply.js index b12a4715e49..2a80cc4f3c6 100644 --- a/src/framework/parsers/ply.js +++ b/src/framework/parsers/ply.js @@ -271,6 +271,7 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { while (cursor < length) { while (streamBuf.remaining === 0) { + /* eslint-disable no-await-in-loop */ await streamBuf.read(); } diff --git a/src/scene/gsplat/gsplat-compressed-data.js b/src/scene/gsplat/gsplat-compressed-data.js index 1142f91b5c2..8829047f3e8 100644 --- a/src/scene/gsplat/gsplat-compressed-data.js +++ b/src/scene/gsplat/gsplat-compressed-data.js @@ -1,4 +1,3 @@ -import { FloatPacking } from '../../core/math/float-packing.js'; import { Quat } from '../../core/math/quat.js'; import { Vec3 } from '../../core/math/vec3.js'; import { Vec4 } from '../../core/math/vec4.js'; From a8baa1d373fdf7e3851c392c2770ce2a4129273c Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 20 Nov 2024 18:15:31 +0000 Subject: [PATCH 5/6] small type fix --- src/platform/graphics/texture.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/graphics/texture.js b/src/platform/graphics/texture.js index e98cad52f83..96feba34c2f 100644 --- a/src/platform/graphics/texture.js +++ b/src/platform/graphics/texture.js @@ -186,7 +186,7 @@ class Texture { * - {@link FUNC_NOTEQUAL} * * Defaults to {@link FUNC_LESS}. - * @param {Uint8Array[]|Uint16Array[]|Uint32Array|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|Uint8Array[][]} [options.levels] + * @param {Uint8Array[]|Uint16Array[]|Uint32Array[]|Float32Array[]|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|Uint8Array[][]} [options.levels] * - Array of Uint8Array or other supported browser interface; or a two-dimensional array * of Uint8Array if options.arrayLength is defined and greater than zero. * @param {boolean} [options.storage] - Defines if texture can be used as a storage texture by From d92c95a9223156a5b6e99e1737e21afa58a50415 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Fri, 22 Nov 2024 14:56:57 +0000 Subject: [PATCH 6/6] changes based on feedback --- src/core/math/float-packing.js | 29 ++++ src/framework/parsers/ply.js | 16 +- src/scene/gsplat/gsplat-compressed-data.js | 154 ++++++++---------- .../gsplat/gsplat-compressed-material.js | 9 +- src/scene/gsplat/gsplat-compressed.js | 53 +++--- 5 files changed, 148 insertions(+), 113 deletions(-) diff --git a/src/core/math/float-packing.js b/src/core/math/float-packing.js index 167824e1a8d..88aff3a0175 100644 --- a/src/core/math/float-packing.js +++ b/src/core/math/float-packing.js @@ -5,6 +5,17 @@ const oneDiv255 = 1 / 255; const floatView = new Float32Array(1); const int32View = new Int32Array(floatView.buffer); +const f32 = new Float32Array(1); +const magicF = new Float32Array(1); +const wasInfNanF = new Float32Array(1); + +const u32 = new Uint32Array(f32.buffer); +const magicU = new Uint32Array(magicF.buffer); +const wasInfNanU = new Uint32Array(wasInfNanF.buffer); + +magicU[0] = (254 - 15) << 23; +wasInfNanU[0] = (127 + 16) << 23; + /** * Utility static class providing functionality to pack float values to various storage * representations. @@ -120,6 +131,24 @@ class FloatPacking { FloatPacking.float2Bytes(value, array, offset, numBytes); } + /** + * Unpacks a 16 bit half floating point value to a float. + * + * Based on https://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ + * + * @param {number} value - The half value to pack as a uint16. + * @returns {number} The packed value. + */ + static half2Float(value) { + u32[0] = (value & 0x7fff) << 13; + f32[0] *= magicF[0]; + if (f32[0] >= wasInfNanF[0]) { + u32[0] |= 255 << 23; + } + u32[0] |= (value & 0x8000) << 16; + return f32[0]; + }; + /** * Packs a float into specified number of bytes, using 1 byte for exponent and the remaining * bytes for the mantissa. diff --git a/src/framework/parsers/ply.js b/src/framework/parsers/ply.js index 2a80cc4f3c6..f82108f8c10 100644 --- a/src/framework/parsers/ply.js +++ b/src/framework/parsers/ply.js @@ -36,6 +36,7 @@ const dataTypeMap = new Map([ ['uchar', Uint8Array], ['short', Int16Array], ['ushort', Uint16Array], + ['half', Uint16Array], ['int', Int32Array], ['uint', Uint32Array], ['float', Float32Array], @@ -205,7 +206,9 @@ const isCompressedPly = (elements) => { 'min_x', 'min_y', 'min_z', 'max_x', 'max_y', 'max_z', 'min_scale_x', 'min_scale_y', 'min_scale_z', - 'max_scale_x', 'max_scale_y', 'max_scale_z' + 'max_scale_x', 'max_scale_y', 'max_scale_z', + 'min_r', 'min_g', 'min_b', + 'max_r', 'max_g', 'max_b' ]; const vertexProperties = [ @@ -226,11 +229,11 @@ const isCompressedPly = (elements) => { const hasSHElements = () => { return elements[2].name === 'sh_band_1' && - elements[2].properties.every((p, i) => p.name === band1Properties[i] && p.type === 'ushort') && + elements[2].properties.every((p, i) => p.name === band1Properties[i] && p.type === 'half') && elements[3].name === 'sh_band_2' && - elements[3].properties.every((p, i) => p.name === band2Properties[i] && p.type === 'ushort') && + elements[3].properties.every((p, i) => p.name === band2Properties[i] && p.type === 'half') && elements[4].name === 'sh_band_3' && - elements[4].properties.every((p, i) => p.name === band3Properties[i] && p.type === 'ushort') && + elements[4].properties.every((p, i) => p.name === band3Properties[i] && p.type === 'half') && elements[5].name === 'vertex_sh' && elements[5].properties.every((p, i) => p.name === indicesProperties[i] && p.type === 'uint'); }; @@ -249,6 +252,7 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { const result = new GSplatCompressedData(); const numChunks = elements[0].count; + const numChunkProperties = elements[0].properties.length; const numVertices = elements[1].count; // evaluate the storage size for the given count (this must match the @@ -261,7 +265,7 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { // allocate result result.numSplats = numVertices; - result.chunkData = new Float32Array(evalStorageSize(numChunks) * 12); + result.chunkData = new Float32Array(numChunks * numChunkProperties); result.vertexData = new Uint32Array(evalStorageSize(numVertices) * 4); // read length bytes of data into buffer @@ -284,7 +288,7 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { }; // read chunk data - await read(result.chunkData.buffer, numChunks * 12 * 4); + await read(result.chunkData.buffer, numChunks * numChunkProperties * 4); // read packed vertices await read(result.vertexData.buffer, numVertices * 4 * 4); diff --git a/src/scene/gsplat/gsplat-compressed-data.js b/src/scene/gsplat/gsplat-compressed-data.js index 8829047f3e8..af5b4938265 100644 --- a/src/scene/gsplat/gsplat-compressed-data.js +++ b/src/scene/gsplat/gsplat-compressed-data.js @@ -2,6 +2,7 @@ import { Quat } from '../../core/math/quat.js'; import { Vec3 } from '../../core/math/vec3.js'; import { Vec4 } from '../../core/math/vec4.js'; import { GSplatData } from './gsplat-data.js'; +import { FloatPacking } from '../../core/math/float-packing.js'; /** * @import { BoundingBox } from '../../core/shape/bounding-box.js' @@ -9,31 +10,6 @@ import { GSplatData } from './gsplat-data.js'; const SH_C0 = 0.28209479177387814; -// https://fgiesen.wordpress.com/2012/03/28/half-to-float-done-quic/ -const halfToFloat = (() => { - const f32 = new Float32Array(1); - const magicF = new Float32Array(1); - const wasInfNanF = new Float32Array(1); - - const u32 = new Uint32Array(f32.buffer); - const magicU = new Uint32Array(magicF.buffer); - const wasInfNanU = new Uint32Array(wasInfNanF.buffer); - - magicU[0] = (254 - 15) << 23; - wasInfNanU[0] = (127 + 16) << 23; - - return (h) => { - u32[0] = (h & 0x7fff) << 13; - f32[0] *= magicF[0]; - - if (f32[0] >= wasInfNanF[0]) { - u32[0] |= 255 << 23; - } - u32[0] |= (h & 0x8000) << 16; // sign bit - return f32[0]; - }; -})(); - // iterator for accessing compressed splat data class SplatCompressedIterator { constructor(gsplatData, p, r, s, c, sh) { @@ -73,10 +49,11 @@ class SplatCompressedIterator { const lerp = (a, b, t) => a * (1 - t) + b * t; - const { chunkData, vertexData, band1Data, band2Data, band3Data, packedSHData } = gsplatData; + const { chunkData, chunkSize, vertexData, band1Data, band2Data, band3Data, packedSHData } = gsplatData; + const { half2Float } = FloatPacking; this.read = (i) => { - const ci = Math.floor(i / 256) * 12; + const ci = Math.floor(i / 256) * chunkSize; if (p) { unpack111011(p, vertexData[i * 4 + 0]); @@ -98,6 +75,11 @@ class SplatCompressedIterator { if (c) { unpack8888(c, vertexData[i * 4 + 3]); + if (chunkSize > 12) { + c.x = lerp(chunkData[ci + 12], chunkData[ci + 15], c.x); + c.y = lerp(chunkData[ci + 13], chunkData[ci + 16], c.y); + c.z = lerp(chunkData[ci + 14], chunkData[ci + 17], c.z); + } } if (sh) { @@ -124,27 +106,27 @@ class SplatCompressedIterator { (bits3 & 0x1ffff) ]; - for (let i = 0; i < 3; ++i) { - const i1 = b1[i] * 3; - sh[i * 15 + 0] = halfToFloat(band1Data[i1 + 0]); - sh[i * 15 + 1] = halfToFloat(band1Data[i1 + 1]); - sh[i * 15 + 2] = halfToFloat(band1Data[i1 + 2]); - - const i2 = b2[i] * 5; - sh[i * 15 + 3] = halfToFloat(band2Data[i2 + 0]); - sh[i * 15 + 4] = halfToFloat(band2Data[i2 + 1]); - sh[i * 15 + 5] = halfToFloat(band2Data[i2 + 2]); - sh[i * 15 + 6] = halfToFloat(band2Data[i2 + 3]); - sh[i * 15 + 7] = halfToFloat(band2Data[i2 + 4]); - - const i3 = b3[i] * 7; - sh[i * 15 + 8] = halfToFloat(band3Data[i3 + 0]); - sh[i * 15 + 9] = halfToFloat(band3Data[i3 + 1]); - sh[i * 15 + 10] = halfToFloat(band3Data[i3 + 2]); - sh[i * 15 + 11] = halfToFloat(band3Data[i3 + 3]); - sh[i * 15 + 12] = halfToFloat(band3Data[i3 + 4]); - sh[i * 15 + 13] = halfToFloat(band3Data[i3 + 5]); - sh[i * 15 + 14] = halfToFloat(band3Data[i3 + 6]); + for (let j = 0; j < 3; ++j) { + const i1 = b1[j] * 3; + sh[j * 15 + 0] = half2Float(band1Data[i1 + 0]); + sh[j * 15 + 1] = half2Float(band1Data[i1 + 1]); + sh[j * 15 + 2] = half2Float(band1Data[i1 + 2]); + + const i2 = b2[j] * 5; + sh[j * 15 + 3] = half2Float(band2Data[i2 + 0]); + sh[j * 15 + 4] = half2Float(band2Data[i2 + 1]); + sh[j * 15 + 5] = half2Float(band2Data[i2 + 2]); + sh[j * 15 + 6] = half2Float(band2Data[i2 + 3]); + sh[j * 15 + 7] = half2Float(band2Data[i2 + 4]); + + const i3 = b3[j] * 7; + sh[j * 15 + 8] = half2Float(band3Data[i3 + 0]); + sh[j * 15 + 9] = half2Float(band3Data[i3 + 1]); + sh[j * 15 + 10] = half2Float(band3Data[i3 + 2]); + sh[j * 15 + 11] = half2Float(band3Data[i3 + 3]); + sh[j * 15 + 12] = half2Float(band3Data[i3 + 4]); + sh[j * 15 + 13] = half2Float(band3Data[i3 + 5]); + sh[j * 15 + 14] = half2Float(band3Data[i3 + 6]); } } }; @@ -155,11 +137,13 @@ class GSplatCompressedData { numSplats; /** - * Contains 12 floats per chunk: + * Contains either 12 or 18 floats per chunk: * min_x, min_y, min_z, * max_x, max_y, max_z, * min_scale_x, min_scale_y, min_scale_z, * max_scale_x, max_scale_y, max_scale_z + * min_r, min_g, min_b, + * max_r, max_g, max_b * @type {Float32Array} */ chunkData; @@ -226,29 +210,25 @@ class GSplatCompressedData { * @returns {boolean} - Whether the calculation was successful. */ calcAabb(result) { - let mx, my, mz, Mx, My, Mz; - - // fast bounds calc using chunk data - const numChunks = Math.ceil(this.numSplats / 256); - - const chunkData = this.chunkData; + const { chunkData, numChunks, chunkSize } = this; let s = Math.exp(Math.max(chunkData[9], chunkData[10], chunkData[11])); - mx = chunkData[0] - s; - my = chunkData[1] - s; - mz = chunkData[2] - s; - Mx = chunkData[3] + s; - My = chunkData[4] + s; - Mz = chunkData[5] + s; + let mx = chunkData[0] - s; + let my = chunkData[1] - s; + let mz = chunkData[2] - s; + let Mx = chunkData[3] + s; + let My = chunkData[4] + s; + let Mz = chunkData[5] + s; for (let i = 1; i < numChunks; ++i) { - s = Math.exp(Math.max(chunkData[i * 12 + 9], chunkData[i * 12 + 10], chunkData[i * 12 + 11])); - mx = Math.min(mx, chunkData[i * 12 + 0] - s); - my = Math.min(my, chunkData[i * 12 + 1] - s); - mz = Math.min(mz, chunkData[i * 12 + 2] - s); - Mx = Math.max(Mx, chunkData[i * 12 + 3] + s); - My = Math.max(My, chunkData[i * 12 + 4] + s); - Mz = Math.max(Mz, chunkData[i * 12 + 5] + s); + const off = i * chunkSize; + s = Math.exp(Math.max(chunkData[off + 9], chunkData[off + 10], chunkData[off + 11])); + mx = Math.min(mx, chunkData[off + 0] - s); + my = Math.min(my, chunkData[off + 1] - s); + mz = Math.min(mz, chunkData[off + 2] - s); + Mx = Math.max(Mx, chunkData[off + 3] + s); + My = Math.max(My, chunkData[off + 4] + s); + Mz = Math.max(Mz, chunkData[off + 5] + s); } result.center.set((mx + Mx) * 0.5, (my + My) * 0.5, (mz + Mz) * 0.5); @@ -261,20 +241,18 @@ class GSplatCompressedData { * @param {Float32Array} result - Array containing the centers. */ getCenters(result) { - const chunkData = this.chunkData; - const vertexData = this.vertexData; - - const numChunks = Math.ceil(this.numSplats / 256); + const { vertexData, chunkData, numChunks, chunkSize } = this; let mx, my, mz, Mx, My, Mz; for (let c = 0; c < numChunks; ++c) { - mx = chunkData[c * 12 + 0]; - my = chunkData[c * 12 + 1]; - mz = chunkData[c * 12 + 2]; - Mx = chunkData[c * 12 + 3]; - My = chunkData[c * 12 + 4]; - Mz = chunkData[c * 12 + 5]; + const off = c * chunkSize; + mx = chunkData[off + 0]; + my = chunkData[off + 1]; + mz = chunkData[off + 2]; + Mx = chunkData[off + 3]; + My = chunkData[off + 4]; + Mz = chunkData[off + 5]; const end = Math.min(this.numSplats, (c + 1) * 256); for (let i = c * 256; i < end; ++i) { @@ -293,17 +271,17 @@ class GSplatCompressedData { * @param {Vec3} result - The result. */ calcFocalPoint(result) { - const chunkData = this.chunkData; - const numChunks = Math.ceil(this.numSplats / 256); + const { chunkData, numChunks, chunkSize } = this; result.x = 0; result.y = 0; result.z = 0; for (let i = 0; i < numChunks; ++i) { - result.x += chunkData[i * 12 + 0] + chunkData[i * 12 + 3]; - result.y += chunkData[i * 12 + 1] + chunkData[i * 12 + 4]; - result.z += chunkData[i * 12 + 2] + chunkData[i * 12 + 5]; + const off = i * chunkSize; + result.x += chunkData[off + 0] + chunkData[off + 3]; + result.y += chunkData[off + 1] + chunkData[off + 4]; + result.z += chunkData[off + 2] + chunkData[off + 5]; } result.mulScalar(0.5 / numChunks); } @@ -312,6 +290,14 @@ class GSplatCompressedData { return true; } + get numChunks() { + return Math.ceil(this.numSplats / 256); + } + + get chunkSize() { + return this.chunkData.length / this.numChunks; + } + get hasSHData() { return !!this.packedSHData; } @@ -331,7 +317,7 @@ class GSplatCompressedData { for (let i = 0; i < 45; ++i) { shMembers.push(`f_rest_${i}`); } - members.splice(9, 0, ...shMembers); + members.splice(members.indexOf('f_dc_0') + 1, 0, ...shMembers); } // allocate uncompressed data diff --git a/src/scene/gsplat/gsplat-compressed-material.js b/src/scene/gsplat/gsplat-compressed-material.js index 73dbd286c67..6bb00969ddc 100644 --- a/src/scene/gsplat/gsplat-compressed-material.js +++ b/src/scene/gsplat/gsplat-compressed-material.js @@ -37,6 +37,8 @@ const splatCoreVS = /* glsl */ ` vec4 chunkDataA; // x: min_x, y: min_y, z: min_z, w: max_x vec4 chunkDataB; // x: max_y, y: max_z, z: scale_min_x, w: scale_min_y vec4 chunkDataC; // x: scale_min_z, y: scale_max_x, z: scale_max_y, w: scale_max_z + vec4 chunkDataD; // x: min_r, y: min_g, z: min_b, w: max_r + vec4 chunkDataE; // x: max_g, y: max_b, z: unused, w: unused uvec4 packedData; // x: position bits, y: rotation bits, z: scale bits, w: color bits // calculate the current splat index and uvs @@ -67,7 +69,7 @@ const splatCoreVS = /* glsl */ ` // calculate chunkUV uint chunkId = splatId / 256u; chunkUV = ivec2( - int((chunkId % chunkWidth) * 3u), + int((chunkId % chunkWidth) * 5u), int(chunkId / chunkWidth) ); @@ -79,6 +81,8 @@ const splatCoreVS = /* glsl */ ` chunkDataA = texelFetch(chunkTexture, chunkUV, 0); chunkDataB = texelFetch(chunkTexture, ivec2(chunkUV.x + 1, chunkUV.y), 0); chunkDataC = texelFetch(chunkTexture, ivec2(chunkUV.x + 2, chunkUV.y), 0); + chunkDataD = texelFetch(chunkTexture, ivec2(chunkUV.x + 3, chunkUV.y), 0); + chunkDataE = texelFetch(chunkTexture, ivec2(chunkUV.x + 4, chunkUV.y), 0); packedData = texelFetch(packedTexture, packedUV, 0); } @@ -127,7 +131,8 @@ const splatCoreVS = /* glsl */ ` } vec4 getColor() { - return unpack8888(packedData.w); + vec4 r = unpack8888(packedData.w); + return vec4(mix(chunkDataD.xyz, vec3(chunkDataD.w, chunkDataE.xy), r.rgb), r.w); } mat3 quatToMat3(vec4 R) { diff --git a/src/scene/gsplat/gsplat-compressed.js b/src/scene/gsplat/gsplat-compressed.js index 11f1c9de8bb..2e394c7eb1f 100644 --- a/src/scene/gsplat/gsplat-compressed.js +++ b/src/scene/gsplat/gsplat-compressed.js @@ -7,13 +7,21 @@ import { import { createGSplatCompressedMaterial } from './gsplat-compressed-material.js'; /** - * @import { BoundingBox } from '../../core/shape/bounding-box.js' * @import { GSplatCompressedData } from './gsplat-compressed-data.js' * @import { GraphicsDevice } from '../../platform/graphics/graphics-device.js' * @import { Material } from '../materials/material.js' * @import { SplatMaterialOptions } from './gsplat-material.js' */ +// copy data with padding +const strideCopy = (target, targetStride, src, srcStride, numEntries) => { + for (let i = 0; i < numEntries; ++i) { + for (let j = 0; j < srcStride; ++j) { + target[i * targetStride + j] = src[i * srcStride + j]; + } + } +}; + class GSplatCompressed { device; @@ -48,8 +56,7 @@ class GSplatCompressed { * @param {GSplatCompressedData} gsplatData - The splat data. */ constructor(device, gsplatData) { - const numSplats = gsplatData.numSplats; - const numChunks = Math.ceil(numSplats / 256); + const { chunkData, chunkSize, numChunks, numSplats, vertexData } = gsplatData; this.device = device; this.numSplats = numSplats; @@ -59,17 +66,30 @@ class GSplatCompressed { gsplatData.calcAabb(this.aabb); // initialize centers - this.centers = new Float32Array(gsplatData.numSplats * 3); + this.centers = new Float32Array(numSplats * 3); gsplatData.getCenters(this.centers); // initialize packed data - this.packedTexture = this.createTexture('packedData', PIXELFORMAT_RGBA32U, this.evalTextureSize(numSplats), gsplatData.vertexData); + this.packedTexture = this.createTexture('packedData', PIXELFORMAT_RGBA32U, this.evalTextureSize(numSplats), vertexData); // initialize chunk data - const chunkSize = this.evalTextureSize(numChunks); - chunkSize.x *= 3; + const chunkTextureSize = this.evalTextureSize(numChunks); + chunkTextureSize.x *= 5; + + this.chunkTexture = this.createTexture('chunkData', PIXELFORMAT_RGBA32F, chunkTextureSize); + const chunkTextureData = this.chunkTexture.lock(); + strideCopy(chunkTextureData, 20, chunkData, chunkSize, numChunks); + + if (chunkSize === 12) { + // if the chunks don't contain color min/max values we must update max to 1 (min is filled with 0's) + for (let i = 0; i < numChunks; ++i) { + chunkTextureData[i * 20 + 15] = 1; + chunkTextureData[i * 20 + 16] = 1; + chunkTextureData[i * 20 + 17] = 1; + } + } - this.chunkTexture = this.createTexture('chunkData', PIXELFORMAT_RGBA32F, chunkSize, gsplatData.chunkData); + this.chunkTexture.unlock(); if (gsplatData.hasSHData) { const { band1Data, band2Data, band3Data, packedSHData } = gsplatData; @@ -78,31 +98,22 @@ class GSplatCompressed { return new Vec2(2048, Math.ceil(numEntries / 1024)); }; - // pad ushort data with zeros - const padUint16 = (target, targetStride, src, srcStride, numEntries) => { - for (let i = 0; i < numEntries; ++i) { - for (let j = 0; j < srcStride; ++j) { - target[i * targetStride + j] = src[i * srcStride + j]; - } - } - }; - // unpack the palette coefficients to align nicely to RGBA16F boundaries for ease of reading on gpu // band1 has max 2k entries, 3 coefficients each, 1 pixel per entry. const band1Size = band1Data.length / 3; this.band1Texture = this.createTexture('band1Palette', PIXELFORMAT_RGBA16F, new Vec2(band1Size, 1)); - padUint16(this.band1Texture.lock(), 4, band1Data, 3, band1Size); + strideCopy(this.band1Texture.lock(), 4, band1Data, 3, band1Size); // band2 has max 32k entries, 5 coefficients each, 2 pixels per entry. const band2Size = band2Data.length / 5; this.band2Texture = this.createTexture('band2Palette', PIXELFORMAT_RGBA16F, calcDim(band2Size)); - padUint16(this.band2Texture.lock(), 8, band2Data, 5, band2Size); + strideCopy(this.band2Texture.lock(), 8, band2Data, 5, band2Size); // band3 has max 128k entries, 7 coefficients each, 2 pixels per entry. const band3Size = band3Data.length / 7; this.band3Texture = this.createTexture('band3Palette', PIXELFORMAT_RGBA16F, calcDim(band3Size)); - padUint16(this.band3Texture.lock(), 8, band3Data, 7, band3Size); + strideCopy(this.band3Texture.lock(), 8, band3Data, 7, band3Size); // packed SH data is loaded directly this.packedSHTexture = this.createTexture('packedSHData', PIXELFORMAT_RGBA32U, this.evalTextureSize(numSplats), packedSHData); @@ -131,7 +142,7 @@ class GSplatCompressed { const result = createGSplatCompressedMaterial(options); result.setParameter('packedTexture', this.packedTexture); result.setParameter('chunkTexture', this.chunkTexture); - result.setParameter('tex_params', new Float32Array([this.numSplats, this.packedTexture.width, this.chunkTexture.width / 3, 0])); + result.setParameter('tex_params', new Float32Array([this.numSplats, this.packedTexture.width, this.chunkTexture.width / 5, 0])); if (this.packedSHTexture) { result.setDefine('USE_SH', true); result.setParameter('band1Texture', this.band1Texture);