diff --git a/src/viser/client/src/Splatting/GaussianSplats.tsx b/src/viser/client/src/Splatting/GaussianSplats.tsx index 94bfd741..f67da3a4 100644 --- a/src/viser/client/src/Splatting/GaussianSplats.tsx +++ b/src/viser/client/src/Splatting/GaussianSplats.tsx @@ -94,6 +94,7 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial( transparent: true, sh_degree: 0, textureBuffer: null, + shTextureBuffer: null, textureT_camera_groups: null, transitionInState: 0.0, }, @@ -180,8 +181,7 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial( vec2 chol45 = unpackHalf2x16(intBufferData.z); // Get spherical harmonic terms from the buffer, there are 48 coeffecients per vertex - // uint shTexStart = sortedIndex * 6u; - uint shTexStart = sortedIndex * 0u; + uint shTexStart = sortedIndex * 6u; ivec2 shTexSize = textureSize(shTextureBuffer, 0); float sh_coeffs_unpacked[48]; for (int i = 0; i < 6; i++) { @@ -280,9 +280,15 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial( vec3 rgb = C0 * sh_coeffs[0]; vec3 pointFive = vec3(0.5, 0.5, 0.5); + vRgba = vec4(rgb + pointFive, float(rgbaUint32 >> uint(24)) / 255.0); - // vRgba = vec4(sh_coeffs[0], 1.0); // im using this to see what the sh_coeffs actually are + //vRgba = vec4(sh_coeffs[0], 1.0); // im using this to see what the sh_coeffs actually are + // if (sh_coeffs[0].x > 1.0 && sh_coeffs[0].x < 1.6) { + // vRgba = vec4(1.0, 0.0, 0.0, 1.0); + // } else { + // vRgba = vec4(1.0, 1.0, 1.0, 1.0); + // } // rgb = rgb - // C1 * y * sh_coeffs[1] + @@ -409,6 +415,7 @@ function SplatRenderer() { if (!initializedBufferTexture) { meshProps.material.uniforms.numGaussians.value = merged.numGaussians; meshProps.textureBuffer.needsUpdate = true; + meshProps.shTextureBuffer.needsUpdate = true; initializedBufferTexture = true; } }; @@ -425,6 +432,7 @@ function SplatRenderer() { React.useEffect(() => { return () => { meshProps.textureBuffer.dispose(); + meshProps.shTextureBuffer.dispose(); meshProps.geometry.dispose(); meshProps.material.dispose(); postToWorker({ close: true }); @@ -554,6 +562,7 @@ function mergeGaussianGroups(groupBufferFromName: { const groupBufferFromNameFiltered = Object.fromEntries( Object.entries(groupBufferFromName).filter(([key]) => !key.startsWith("sh_buffer_")) ); + // groupBufferFromNameFiltered only contains buffers for the gaussians (not sh buffers) for (const buffer of Object.values(groupBufferFromNameFiltered)) { totalBufferLength += buffer.length; } @@ -599,6 +608,7 @@ function mergeGaussianGroups(groupBufferFromName: { combinedSHBuffer.set(sh_buffer, sh_offset) sh_offset += sh_buffer.length; } + console.log(combinedSHBuffer) const numGroups = Object.keys(groupBufferFromNameFiltered).length; return { numGaussians, gaussianBuffer, numGroups, groupIndices, combinedSHBuffer}; @@ -695,6 +705,7 @@ function useGaussianMeshProps(gaussianBuffer: Uint32Array, combinedSHBuffer: Uin geometry, material, textureBuffer, + shTextureBuffer, sortedIndexAttribute, textureT_camera_groups, rowMajorT_camera_groups, diff --git a/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb b/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb index 29c5acf7..d03da0a6 100644 --- a/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb +++ b/src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -128,14 +128,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[-0.18104345 0.54372042 -0.14186212]\n" + "[0.70985258 0.20630107 0.06283242]\n" ] } ], @@ -147,6 +147,182 @@ "print(eval_sh(deg, sh, dirs))\n" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import struct\n", + "import numpy as np\n", + "\n", + "def unpackHalf2x16(value):\n", + " \"\"\"The first float is the least significant 16bits, the second is the most significant 16bits.\"\"\"\n", + " # Convert int32 to its binary representation\n", + " binary = format(value, '032b')\n", + " \n", + " # Split the binary string into two 16-bit parts\n", + " binary1 = binary[:16]\n", + " binary2 = binary[16:]\n", + " \n", + " # Convert each 16-bit binary string to an integer\n", + " int1 = int(binary1, 2)\n", + " int2 = int(binary2, 2)\n", + " \n", + " # Use numpy to convert uint16 to float16\n", + " float1 = np.frombuffer(struct.pack('H', int1), dtype=np.float16)[0]\n", + " float2 = np.frombuffer(struct.pack('H', int2), dtype=np.float16)[0]\n", + " \n", + " return float1, float2\n", + "\n", + "def int32_to_rgba(value):\n", + " # Ensure the input is a 32-bit integer\n", + " value = int(value) & 0xFFFFFFFF\n", + " \n", + " # Extract each 8-bit piece\n", + " r = (value >> 24) & 0xFF\n", + " g = (value >> 16) & 0xFF\n", + " b = (value >> 8) & 0xFF\n", + " a = value & 0xFF\n", + " \n", + " return r / 255.0, g / 255.0, b / 255.0, a / 255.0" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.803, 0.8213)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unpackHalf2x16(980236946)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(-0.00319, 0.786)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unpackHalf2x16(2592619082)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.3686274509803922,\n", + " 0.7215686274509804,\n", + " 0.7254901960784313,\n", + " 0.7294117647058823)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "int32_to_rgba(1589164474)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c3317264494f4ea997d61191fa24a3c5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(FloatSlider(value=0.8212358, description='Red:', max=1.0, step=0.01), FloatSlider(value=0.80300…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import ipywidgets as widgets\n", + "from IPython.display import display, HTML\n", + "import matplotlib.colors as mcolors\n", + "\n", + "def color_picker(r=0.5, g=0.5, b=0.5):\n", + " def update_color(r, g, b):\n", + " color = mcolors.to_hex([r, g, b])\n", + " color_display.value = f'
'\n", + " rgb_display.value = f'RGB: ({r:.2f}, {g:.2f}, {b:.2f})'\n", + " hex_display.value = f'Hex: {color}'\n", + "\n", + " r_slider = widgets.FloatSlider(value=r, min=0, max=1, step=0.01, description='Red:')\n", + " g_slider = widgets.FloatSlider(value=g, min=0, max=1, step=0.01, description='Green:')\n", + " b_slider = widgets.FloatSlider(value=b, min=0, max=1, step=0.01, description='Blue:')\n", + "\n", + " color_display = widgets.HTML()\n", + " rgb_display = widgets.Label()\n", + " hex_display = widgets.Label()\n", + "\n", + " widgets.interactive(update_color, r=r_slider, g=g_slider, b=b_slider)\n", + "\n", + " display(widgets.VBox([r_slider, g_slider, b_slider, color_display, rgb_display, hex_display]))\n", + "\n", + "# Usage\n", + "color_picker(0.8212358, 0.8030037, 0.78623223) # You can change these initial values" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "49ebcdf479814f618fab56eb4f17427b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(FloatSlider(value=0.3686274509803922, description='Red:', max=1.0, step=0.01), FloatSlider(valu…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "color_picker(0.3686274509803922, 0.7215686274509804, 0.7254901960784313)" + ] + }, { "cell_type": "code", "execution_count": null,