From 475cbc86e7bb0a1cf74aee53d04b11dbb07b0109 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Tue, 17 Oct 2023 15:36:38 +0100 Subject: [PATCH 1/6] speed up sort, unify rendering data --- src/sort-worker.ts | 114 ++++++++++++++++-------------- src/splat-resource.ts | 161 ++++++++++++++++++++---------------------- 2 files changed, 140 insertions(+), 135 deletions(-) diff --git a/src/sort-worker.ts b/src/sort-worker.ts index 134a888..ed62003 100644 --- a/src/sort-worker.ts +++ b/src/sort-worker.ts @@ -16,83 +16,93 @@ function SortWorker() { const lastCameraPosition = { x: 0, y: 0, z: 0 }; const lastCameraDirection = { x: 0, y: 0, z: 0 }; - let distanceBuffer: Float32Array; - let orderBuffer: Uint32Array; + let orderBuffer: BigUint64Array; + let orderBuffer32: Uint32Array; let target: Float32Array; const update = () => { if (!data || !stride || !cameraPosition || !cameraDirection) return; + const px = cameraPosition.x; + const py = cameraPosition.y; + const pz = cameraPosition.z; + const dx = cameraDirection.x; + const dy = cameraDirection.y; + const dz = cameraDirection.z; + // early out if camera hasn't moved - if (Math.abs(cameraPosition.x - lastCameraPosition.x) < epsilon && - Math.abs(cameraPosition.y - lastCameraPosition.y) < epsilon && - Math.abs(cameraPosition.z - lastCameraPosition.z) < epsilon && - Math.abs(cameraDirection.x - lastCameraDirection.x) < epsilon && - Math.abs(cameraDirection.y - lastCameraDirection.y) < epsilon && - Math.abs(cameraDirection.z - lastCameraDirection.z) < epsilon) { + if (Math.abs(px - lastCameraPosition.x) < epsilon && + Math.abs(py - lastCameraPosition.y) < epsilon && + Math.abs(pz - lastCameraPosition.z) < epsilon && + Math.abs(dx - lastCameraDirection.x) < epsilon && + Math.abs(dy - lastCameraDirection.y) < epsilon && + Math.abs(dz - lastCameraDirection.z) < epsilon) { return; } + lastCameraPosition.x = px; + lastCameraPosition.y = py; + lastCameraPosition.z = pz; + lastCameraDirection.x = dx; + lastCameraDirection.y = dy; + lastCameraDirection.z = dz; + const numVertices = data.length / stride; // create distance buffer - if (!distanceBuffer || distanceBuffer.length !== numVertices) { - distanceBuffer = new Float32Array(numVertices); - orderBuffer = new Uint32Array(numVertices); + if (orderBuffer?.length !== numVertices) { + orderBuffer = new BigUint64Array(numVertices); + orderBuffer32 = new Uint32Array(orderBuffer.buffer); target = new Float32Array(numVertices * stride); } - // store - lastCameraPosition.x = cameraPosition.x; - lastCameraPosition.y = cameraPosition.y; - lastCameraPosition.z = cameraPosition.z; - lastCameraDirection.x = cameraDirection.x; - lastCameraDirection.y = cameraDirection.y; - lastCameraDirection.z = cameraDirection.z; - - const px = cameraPosition.x; - const py = cameraPosition.y; - const pz = cameraPosition.z; - const dx = cameraDirection.x; - const dy = cameraDirection.y; - const dz = cameraDirection.z; + const strideVertices = numVertices * stride; + + // calc min/max + let minDist = (data[0] - px) * dx + (data[1] - py) * dy + (data[2] - pz) * dz; + let maxDist = minDist; + for (let i = stride; i < strideVertices; i += stride) { + const d = (data[i + 0] - px) * dx + + (data[i + 1] - py) * dy + + (data[i + 2] - pz) * dz; + minDist = Math.min(minDist, d); + maxDist = Math.max(maxDist, d); + } // generate per vertex distance to camera + const range = maxDist - minDist; for (let i = 0; i < numVertices; ++i) { - distanceBuffer[i] = - (data[i * stride + 0] - px) * dx + - (data[i * stride + 1] - py) * dy + - (data[i * stride + 2] - pz) * dz; - orderBuffer[i] = i; + const istride = i * stride; + const d = (data[istride + 0] - px) * dx + + (data[istride + 1] - py) * dy + + (data[istride + 2] - pz) * dz; + orderBuffer32[i * 2 + 1] = Math.floor((d - minDist) / range * 2**32); + orderBuffer32[i * 2] = i; } // sort indices - orderBuffer.sort((a, b) => distanceBuffer[a] - distanceBuffer[b]); - - const orderChanged = orderBuffer.some((v, i) => v !== i); - - if (orderChanged) { - // order the splat data - for (let i = 0; i < numVertices; ++i) { - const ti = i * stride; - const si = orderBuffer[i] * stride; - for (let j = 0; j < stride; ++j) { - target[ti + j] = data[si + j]; - } + orderBuffer.sort(); + + // order the splat data + for (let i = 0; i < numVertices; ++i) { + const ti = i * stride; + const si = orderBuffer32[i * 2] * stride; + for (let j = 0; j < stride; ++j) { + target[ti + j] = data[si + j]; } + } - // swap - const tmp = data; - data = target; - target = tmp; + // swap + const tmp = data; + data = target; + target = tmp; - // send results - self.postMessage({ - data: data.buffer - }, [data.buffer]); + // send results + self.postMessage({ + data: data.buffer + }, [data.buffer]); - data = null; - } + data = null; }; self.onmessage = (message: any) => { diff --git a/src/splat-resource.ts b/src/splat-resource.ts index 9920c75..7f36a77 100644 --- a/src/splat-resource.ts +++ b/src/splat-resource.ts @@ -37,8 +37,8 @@ const splatVS = /* glsl_ */ ` attribute vec2 vertex_position; attribute vec3 splat_center; attribute vec4 splat_color; -attribute vec3 splat_cova; -attribute vec3 splat_covb; +attribute vec3 splat_rotation; +attribute vec3 splat_scale; uniform mat4 matrix_model; uniform mat4 matrix_view; @@ -50,6 +50,55 @@ uniform vec2 focal; varying vec2 texCoord; varying vec4 color; +mat3 quatToMat3(vec3 r) +{ + vec4 R = vec4(r.x, r.y, r.z, sqrt(1.0 - dot(r, r))); + + return mat3( + 1.0 - 2.0 * (R.z * R.z + R.w * R.w), + 2.0 * (R.y * R.z + R.x * R.w), + 2.0 * (R.y * R.w - R.x * R.z), + + 2.0 * (R.y * R.z - R.x * R.w), + 1.0 - 2.0 * (R.y * R.y + R.w * R.w), + 2.0 * (R.z * R.w + R.x * R.y), + + 2.0 * (R.y * R.w + R.x * R.z), + 2.0 * (R.z * R.w - R.x * R.y), + 1.0 - 2.0 * (R.y * R.y + R.z * R.z) + ); +} + +void computeCov3d(in vec3 rot, in vec3 scale, out vec3 covA, out vec3 covB) +{ + mat3 R = quatToMat3(rot); + + // M = S * R + float M[9] = float[9]( + scale.x * R[0][0], + scale.x * R[0][1], + scale.x * R[0][2], + scale.y * R[1][0], + scale.y * R[1][1], + scale.y * R[1][2], + scale.z * R[2][0], + scale.z * R[2][1], + scale.z * R[2][2] + ); + + covA = vec3( + M[0] * M[0] + M[3] * M[3] + M[6] * M[6], + M[0] * M[1] + M[3] * M[4] + M[6] * M[7], + M[0] * M[2] + M[3] * M[5] + M[6] * M[8] + ); + + covB = vec3( + M[1] * M[1] + M[4] * M[4] + M[7] * M[7], + M[1] * M[2] + M[4] * M[5] + M[7] * M[8], + M[2] * M[2] + M[5] * M[5] + M[8] * M[8] + ); +} + void main(void) { vec4 splat_cam = matrix_view * matrix_model * vec4(splat_center, 1.0); @@ -61,6 +110,10 @@ void main(void) return; } + vec3 splat_cova; + vec3 splat_covb; + computeCov3d(splat_rotation, splat_scale, splat_cova, splat_covb); + mat3 Vrk = mat3( splat_cova.x, splat_cova.y, splat_cova.z, splat_cova.y, splat_covb.x, splat_covb.y, @@ -119,7 +172,7 @@ const splatDebugVS = /* glsl_ */ ` attribute vec3 vertex_position; attribute vec3 splat_center; attribute vec4 splat_color; -attribute vec4 splat_rotation; +attribute vec3 splat_rotation; attribute vec3 splat_scale; uniform mat4 matrix_model; @@ -127,12 +180,12 @@ uniform mat4 matrix_viewProjection; varying vec4 color; -mat3 quatToMat3(vec4 quat) +mat3 quatToMat3(vec3 quat) { float x = quat.x; float y = quat.y; float z = quat.z; - float w = quat.w; + float w = sqrt(1.0 - dot(quat, quat)); return mat3( 1.0 - 2.0 * (z * z + w * w), @@ -162,6 +215,7 @@ varying vec4 color; void main(void) { + if (color.a < 0.2) discard; gl_FragColor = color; } `; @@ -200,13 +254,8 @@ class SplatResource extends ContainerResource { vertex_position: SEMANTIC_POSITION, splat_center: SEMANTIC_ATTR11, splat_color: SEMANTIC_COLOR, - ...debugRender ? { - splat_rotation: SEMANTIC_ATTR12, - splat_scale: SEMANTIC_ATTR13 - } : { - splat_cova: SEMANTIC_ATTR12, - splat_covb: SEMANTIC_ATTR13 - } + splat_rotation: SEMANTIC_ATTR12, + splat_scale: SEMANTIC_ATTR13 }); this.quadMaterial.update(); @@ -265,15 +314,13 @@ class SplatResource extends ContainerResource { return null; } - const stride = 4 + (debugRender ? 7 : 6); + const stride = 10; - // position.xyz, color, cova.xyz, covb.xyz, rotation.xyzw, scale.xyz + // position.xyz, color, rotation.xyz, scale.xyz const floatData = new Float32Array(vertexElement.count * stride); const uint8Data = new Uint8ClampedArray(floatData.buffer); const quat = new Quat(); - const r = [0, 0, 0, 0]; - const s = [0, 0, 0]; for (let i = 0; i < vertexElement.count; ++i) { const j = i; @@ -314,65 +361,21 @@ class SplatResource extends ContainerResource { quat.set(rot_0[j], rot_1[j], rot_2[j], rot_3[j]).normalize(); - r[0] = quat.x; - r[1] = quat.y; - r[2] = quat.z; - r[3] = quat.w; - - s[0] = Math.exp(scale_0[j]); - s[1] = Math.exp(scale_1[j]); - s[2] = Math.exp(scale_2[j]); - - if (debugRender) { - // rotation - floatData[i * stride + 4] = r[0]; - floatData[i * stride + 5] = r[1]; - floatData[i * stride + 6] = r[2]; - floatData[i * stride + 7] = r[3]; - - // scale - floatData[i * stride + 8] = s[0]; - floatData[i * stride + 9] = s[1]; - floatData[i * stride + 10] = s[2]; + // rotation + if (quat.w < 0) { + floatData[i * stride + 4] = -quat.x; + floatData[i * stride + 5] = -quat.y; + floatData[i * stride + 6] = -quat.z; } else { - // pre-calculate covariance a & b - const R = [ - 1.0 - 2.0 * (r[2] * r[2] + r[3] * r[3]), - 2.0 * (r[1] * r[2] + r[0] * r[3]), - 2.0 * (r[1] * r[3] - r[0] * r[2]), - - 2.0 * (r[1] * r[2] - r[0] * r[3]), - 1.0 - 2.0 * (r[1] * r[1] + r[3] * r[3]), - 2.0 * (r[2] * r[3] + r[0] * r[1]), - - 2.0 * (r[1] * r[3] + r[0] * r[2]), - 2.0 * (r[2] * r[3] - r[0] * r[1]), - 1.0 - 2.0 * (r[1] * r[1] + r[2] * r[2]) - ]; - - // Compute the matrix product of S and R (M = S * R) - const M = [ - s[0] * R[0], - s[0] * R[1], - s[0] * R[2], - s[1] * R[3], - s[1] * R[4], - s[1] * R[5], - s[2] * R[6], - s[2] * R[7], - s[2] * R[8] - ]; - - // covariance a - floatData[i * stride + 4] = M[0] * M[0] + M[3] * M[3] + M[6] * M[6]; - floatData[i * stride + 5] = M[0] * M[1] + M[3] * M[4] + M[6] * M[7]; - floatData[i * stride + 6] = M[0] * M[2] + M[3] * M[5] + M[6] * M[8]; - - // covariance b - floatData[i * stride + 7] = M[1] * M[1] + M[4] * M[4] + M[7] * M[7]; - floatData[i * stride + 8] = M[1] * M[2] + M[4] * M[5] + M[7] * M[8]; - floatData[i * stride + 9] = M[2] * M[2] + M[5] * M[5] + M[8] * M[8]; + floatData[i * stride + 4] = quat.x; + floatData[i * stride + 5] = quat.y; + floatData[i * stride + 6] = quat.z; } + + // scale + floatData[i * stride + 7] = Math.exp(scale_0[j]); + floatData[i * stride + 8] = Math.exp(scale_1[j]); + floatData[i * stride + 9] = Math.exp(scale_2[j]); } const calcAabb = () => { @@ -393,23 +396,16 @@ class SplatResource extends ContainerResource { const aabb = new BoundingBox(); aabb.setMinMax(new Vec3(xMinMax[0], yMinMax[0], zMinMax[0]), new Vec3(xMinMax[1], yMinMax[1], zMinMax[1])); - console.log(aabb.getMin()); - console.log(aabb.getMax()); - return aabb; }; // create instance data const vertexFormat = new VertexFormat(this.device, [ { semantic: SEMANTIC_ATTR11, components: 3, type: TYPE_FLOAT32 }, - { semantic: SEMANTIC_COLOR, components: 4, type: TYPE_UINT8, normalize: true } - ].concat(debugRender ? [ - { semantic: SEMANTIC_ATTR12, components: 4, type: TYPE_FLOAT32 }, - { semantic: SEMANTIC_ATTR13, components: 3, type: TYPE_FLOAT32 } - ] : [ + { semantic: SEMANTIC_COLOR, components: 4, type: TYPE_UINT8, normalize: true }, { semantic: SEMANTIC_ATTR12, components: 3, type: TYPE_FLOAT32 }, { semantic: SEMANTIC_ATTR13, components: 3, type: TYPE_FLOAT32 } - ])); + ]); const vertexBuffer = new VertexBuffer(this.device, vertexFormat, vertexElement.count, BUFFER_DYNAMIC, floatData.buffer); const meshInstance = new MeshInstance(this.quadMesh, this.quadMaterial); @@ -484,7 +480,6 @@ class SplatResource extends ContainerResource { sum += weight; } result.mulScalar(1 / sum); - console.log(result); }; calcFocalPoint(this.focalPoint); From a1369f67c00f9b65138552c83996642f5ed2a251 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Tue, 17 Oct 2023 16:21:43 +0100 Subject: [PATCH 2/6] start devbug rendering --- src/splat-resource.ts | 108 +++++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 44 deletions(-) diff --git a/src/splat-resource.ts b/src/splat-resource.ts index 7f36a77..a20a8e5 100644 --- a/src/splat-resource.ts +++ b/src/splat-resource.ts @@ -33,6 +33,30 @@ import { SortWorker } from './sort-worker'; // set true to render splats as oriented boxes const debugRender = false; +const quatToMat3 = ` +mat3 quatToMat3(vec3 R) +{ + float x = R.x; + float y = R.y; + float z = R.z; + float w = sqrt(1.0 - dot(R, R)); + + return mat3( + 1.0 - 2.0 * (z * z + w * w), + 2.0 * (y * z + x * w), + 2.0 * (y * w - x * z), + + 2.0 * (y * z - x * w), + 1.0 - 2.0 * (y * y + w * w), + 2.0 * (z * w + x * y), + + 2.0 * (y * w + x * z), + 2.0 * (z * w - x * y), + 1.0 - 2.0 * (y * y + z * z) + ); +} +`; + const splatVS = /* glsl_ */ ` attribute vec2 vertex_position; attribute vec3 splat_center; @@ -50,24 +74,7 @@ uniform vec2 focal; varying vec2 texCoord; varying vec4 color; -mat3 quatToMat3(vec3 r) -{ - vec4 R = vec4(r.x, r.y, r.z, sqrt(1.0 - dot(r, r))); - - return mat3( - 1.0 - 2.0 * (R.z * R.z + R.w * R.w), - 2.0 * (R.y * R.z + R.x * R.w), - 2.0 * (R.y * R.w - R.x * R.z), - - 2.0 * (R.y * R.z - R.x * R.w), - 1.0 - 2.0 * (R.y * R.y + R.w * R.w), - 2.0 * (R.z * R.w + R.x * R.y), - - 2.0 * (R.y * R.w + R.x * R.z), - 2.0 * (R.z * R.w - R.x * R.y), - 1.0 - 2.0 * (R.y * R.y + R.z * R.z) - ); -} +${quatToMat3} void computeCov3d(in vec3 rot, in vec3 scale, out vec3 covA, out vec3 covB) { @@ -146,10 +153,6 @@ void main(void) vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 8.0, 0.0, 0.0) * splat_proj.w; - // gl_Position = vec4(splat_proj.xy / splat_proj.w + - // (vertex_position.x * v1 + vertex_position.y * v2) / viewport * 8.0, - // 0.0, 1.0); - texCoord = vertex_position * 2.0; color = splat_color; } @@ -180,27 +183,7 @@ uniform mat4 matrix_viewProjection; varying vec4 color; -mat3 quatToMat3(vec3 quat) -{ - float x = quat.x; - float y = quat.y; - float z = quat.z; - float w = sqrt(1.0 - dot(quat, quat)); - - return mat3( - 1.0 - 2.0 * (z * z + w * w), - 2.0 * (y * z + x * w), - 2.0 * (y * w - x * z), - - 2.0 * (y * z - x * w), - 1.0 - 2.0 * (y * y + w * w), - 2.0 * (z * w + x * y), - - 2.0 * (y * w + x * z), - 2.0 * (z * w - x * y), - 1.0 - 2.0 * (y * y + z * z) - ); -} +${quatToMat3} void main(void) { @@ -287,6 +270,25 @@ class SplatResource extends ContainerResource { return null; } + // create a debug splat + vertexElement.count = 1; + vertexElement.properties = [ + { type: 'float', name: 'x', storage: [0], byteSize: 4 }, + { type: 'float', name: 'y', storage: [0], byteSize: 4 }, + { type: 'float', name: 'z', storage: [0], byteSize: 4 }, + { type: 'float', name: 'f_dc_0', storage: [1.772453850905516], byteSize: 4 }, + { type: 'float', name: 'f_dc_1', storage: [1.772453850905516], byteSize: 4 }, + { type: 'float', name: 'f_dc_2', storage: [1.772453850905516], byteSize: 4 }, + { type: 'float', name: 'opacity', storage: [1], byteSize: 4 }, + { type: 'float', name: 'scale_0', storage: [1.0], byteSize: 4 }, + { type: 'float', name: 'scale_1', storage: [0.5], byteSize: 4 }, + { type: 'float', name: 'scale_2', storage: [0.2], byteSize: 4 }, + { type: 'float', name: 'rot_0', storage: [0], byteSize: 4 }, + { type: 'float', name: 'rot_1', storage: [0], byteSize: 4 }, + { type: 'float', name: 'rot_2', storage: [0], byteSize: 4 }, + { type: 'float', name: 'rot_3', storage: [1], byteSize: 4 }, + ]; + const find = (name: string) => { return vertexElement.properties.find((property: any) => property.name === name && property.storage)?.storage; }; @@ -394,7 +396,9 @@ class SplatResource extends ContainerResource { const zMinMax = minmax(z); const aabb = new BoundingBox(); - aabb.setMinMax(new Vec3(xMinMax[0], yMinMax[0], zMinMax[0]), new Vec3(xMinMax[1], yMinMax[1], zMinMax[1])); + aabb.setMinMax( + new Vec3(xMinMax[0] - 1, yMinMax[0] - 0.5, zMinMax[0] - 0.2), + new Vec3(xMinMax[1] + 1, yMinMax[1] + 0.5, zMinMax[1] + 0.2)); return aabb; }; @@ -466,6 +470,22 @@ class SplatResource extends ContainerResource { this.quadMaterial.setParameter('viewport', [this.device.width, this.device.height]); this.quadMaterial.setParameter('focal', [focal[0], focal[1]]); // this.quadMaterial.setParameter('focal', [this.device.width / 2, this.device.height / 2]); + + // debug render splat bounds + const immediate = options.app.scene.immediate; + const data = new Float32Array(vertexBuffer.lock()); + for (let i = 0; i < data.length / stride; ++i) { + const x = data[i * stride + 0]; + const y = data[i * stride + 1]; + const z = data[i * stride + 2]; + const rx = data[i * stride + 4]; + const ry = data[i * stride + 5]; + const rz = data[i * stride + 6]; + const sx = data[i * stride + 7]; + const sy = data[i * stride + 8]; + const sz = data[i * stride + 9]; + } + vertexBuffer.unlock(); }); } From 5968a9781aefa873fa7529ff675705b771461763 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 18 Oct 2023 11:09:19 +0100 Subject: [PATCH 3/6] working --- src/splat-resource.ts | 89 +++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/src/splat-resource.ts b/src/splat-resource.ts index a20a8e5..fc1c86b 100644 --- a/src/splat-resource.ts +++ b/src/splat-resource.ts @@ -380,8 +380,27 @@ class SplatResource extends ContainerResource { floatData[i * stride + 9] = Math.exp(scale_2[j]); } + // create instance data + const vertexFormat = new VertexFormat(this.device, [ + { semantic: SEMANTIC_ATTR11, components: 3, type: TYPE_FLOAT32 }, + { semantic: SEMANTIC_COLOR, components: 4, type: TYPE_UINT8, normalize: true }, + { semantic: SEMANTIC_ATTR12, components: 3, type: TYPE_FLOAT32 }, + { semantic: SEMANTIC_ATTR13, components: 3, type: TYPE_FLOAT32 } + ]); + const vertexBuffer = new VertexBuffer(this.device, vertexFormat, vertexElement.count, BUFFER_DYNAMIC, floatData.buffer); + + const meshInstance = new MeshInstance(this.quadMesh, this.quadMaterial); + meshInstance.setInstancing(vertexBuffer); + + const result = new Entity('ply'); + result.addComponent('render', { + type: 'asset', + meshInstances: [meshInstance], + castShadows: false // shadows not supported + }); + + // calculate accurate aabb const calcAabb = () => { - // calc aabb const minmax = (data: Float32Array) => { let min = data[0]; let max = data[0]; @@ -403,24 +422,19 @@ class SplatResource extends ContainerResource { return aabb; }; - // create instance data - const vertexFormat = new VertexFormat(this.device, [ - { semantic: SEMANTIC_ATTR11, components: 3, type: TYPE_FLOAT32 }, - { semantic: SEMANTIC_COLOR, components: 4, type: TYPE_UINT8, normalize: true }, - { semantic: SEMANTIC_ATTR12, components: 3, type: TYPE_FLOAT32 }, - { semantic: SEMANTIC_ATTR13, components: 3, type: TYPE_FLOAT32 } - ]); - const vertexBuffer = new VertexBuffer(this.device, vertexFormat, vertexElement.count, BUFFER_DYNAMIC, floatData.buffer); - - const meshInstance = new MeshInstance(this.quadMesh, this.quadMaterial); - meshInstance.setInstancing(vertexBuffer); - - const result = new Entity('ply'); - result.addComponent('render', { - type: 'asset', - meshInstances: [meshInstance], - castShadows: false // shadows not supported - }); + const calcAabb2 = () => { + for (let i = 0; i < vertexElement.count; ++i) { + const x = floatData[i * stride + 0]; + const y = floatData[i * stride + 1]; + const z = floatData[i * stride + 2]; + const rx = floatData[i * stride + 4]; + const ry = floatData[i * stride + 5]; + const rz = floatData[i * stride + 6]; + const sx = floatData[i * stride + 7]; + const sy = floatData[i * stride + 8]; + const sz = floatData[i * stride + 9]; + } + } // set custom aabb result.render.customAabb = calcAabb(); @@ -434,16 +448,17 @@ class SplatResource extends ContainerResource { sortWorker.onmessage = (message: any) => { const data = message.data.data; - // copy data - const target = new Float32Array(vertexBuffer.lock()); - target.set(new Float32Array(data)); - vertexBuffer.unlock(); + // copy source data + floatData.set(new Float32Array(data)); // send the memory buffer back to worker sortWorker.postMessage({ data: data }, [data]); + // upload new data to GPU + vertexBuffer.unlock(); + // let caller know the view changed options?.onChanged(); }; @@ -472,20 +487,20 @@ class SplatResource extends ContainerResource { // this.quadMaterial.setParameter('focal', [this.device.width / 2, this.device.height / 2]); // debug render splat bounds - const immediate = options.app.scene.immediate; - const data = new Float32Array(vertexBuffer.lock()); - for (let i = 0; i < data.length / stride; ++i) { - const x = data[i * stride + 0]; - const y = data[i * stride + 1]; - const z = data[i * stride + 2]; - const rx = data[i * stride + 4]; - const ry = data[i * stride + 5]; - const rz = data[i * stride + 6]; - const sx = data[i * stride + 7]; - const sy = data[i * stride + 8]; - const sz = data[i * stride + 9]; - } - vertexBuffer.unlock(); + // const immediate = options.app.scene.immediate; + // const data = new Float32Array(vertexBuffer.lock()); + // for (let i = 0; i < data.length / stride; ++i) { + // const x = data[i * stride + 0]; + // const y = data[i * stride + 1]; + // const z = data[i * stride + 2]; + // const rx = data[i * stride + 4]; + // const ry = data[i * stride + 5]; + // const rz = data[i * stride + 6]; + // const sx = data[i * stride + 7]; + // const sy = data[i * stride + 8]; + // const sz = data[i * stride + 9]; + // } + // vertexBuffer.unlock(); }); } From 886ef4dae5473a5142fb19dc862f9a0b86446e09 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 18 Oct 2023 15:07:36 +0100 Subject: [PATCH 4/6] splat size correctly rendered --- src/splat-resource.ts | 158 ++++++++++++++++++++++++++++-------------- 1 file changed, 106 insertions(+), 52 deletions(-) diff --git a/src/splat-resource.ts b/src/splat-resource.ts index fc1c86b..a4f35e6 100644 --- a/src/splat-resource.ts +++ b/src/splat-resource.ts @@ -1,6 +1,8 @@ import { + AppBase, BoundingBox, BUFFER_DYNAMIC, + Color, ContainerResource, createShaderFromCode, createBox, @@ -8,6 +10,7 @@ import { CULLFACE_BACK, Entity, GraphicsDevice, + Mat4, Material, Mesh, MeshInstance, @@ -69,7 +72,6 @@ uniform mat4 matrix_view; uniform mat4 matrix_projection; uniform vec2 viewport; -uniform vec2 focal; varying vec2 texCoord; varying vec4 color; @@ -127,9 +129,11 @@ void main(void) splat_cova.z, splat_covb.y, splat_covb.z ); + float focal = viewport.x * matrix_projection[0][0]; + mat3 J = mat3( - focal.x / splat_cam.z, 0., -(focal.x * splat_cam.x) / (splat_cam.z * splat_cam.z), - 0., focal.y / splat_cam.z, -(focal.y * splat_cam.y) / (splat_cam.z * splat_cam.z), + focal / splat_cam.z, 0., -(focal * splat_cam.x) / (splat_cam.z * splat_cam.z), + 0., focal / splat_cam.z, -(focal * splat_cam.y) / (splat_cam.z * splat_cam.z), 0., 0., 0. ); @@ -150,7 +154,7 @@ void main(void) vec2 v2 = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x); gl_Position = splat_proj + - vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 8.0, + vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 2.0, 0.0, 0.0) * splat_proj.w; texCoord = vertex_position * 2.0; @@ -203,6 +207,75 @@ void main(void) } `; +const vec3 = new Vec3(); +const mat4 = new Mat4(); +const aabb = new BoundingBox(); + +const debugPoints = [new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3()]; +const debugLines = [ + debugPoints[0], debugPoints[1], debugPoints[1], debugPoints[3], debugPoints[3], debugPoints[2], debugPoints[2], debugPoints[0], + debugPoints[4], debugPoints[5], debugPoints[5], debugPoints[7], debugPoints[7], debugPoints[6], debugPoints[6], debugPoints[4], + debugPoints[0], debugPoints[4], debugPoints[1], debugPoints[5], debugPoints[2], debugPoints[6], debugPoints[3], debugPoints[7] +]; +const debugColor = new Color(1, 1, 0, 0.4); + +const getSplatMat = (result: Mat4, data: Float32Array) => { + const px = data[0]; + const py = data[1]; + const pz = data[2]; + const x = data[4]; + const y = data[5]; + const z = data[6]; + const w = Math.sqrt(1 - (x * x + y * y + z * z)); + + // build rotation matrix + result.data.set([ + 1.0 - 2.0 * (z * z + w * w), + 2.0 * (y * z + x * w), + 2.0 * (y * w - x * z), + 0, + + 2.0 * (y * z - x * w), + 1.0 - 2.0 * (y * y + w * w), + 2.0 * (z * w + x * y), + 0, + + 2.0 * (y * w + x * z), + 2.0 * (z * w - x * y), + 1.0 - 2.0 * (y * y + z * z), + 0, + + px, py, pz, 1 + ]); +}; + +const getSplatAabb = (result: BoundingBox, data: Float32Array) => { + getSplatMat(mat4, data); + aabb.center.set(0, 0, 0); + aabb.halfExtents.set(data[7] * 2, data[8] * 2, data[9] * 2); + result.setFromTransformedAabb(aabb, mat4); +}; + +const renderDebugSplat = (app: AppBase, worldMat: Mat4, data: Float32Array) => { + getSplatMat(mat4, data); + mat4.mul2(worldMat, mat4); + + const sx = data[7]; + const sy = data[8]; + const sz = data[9]; + + for (let i = 0; i < 8; ++i) { + vec3.set( + sx * 2 * ((i & 1) ? 1 : -1), + sy * 2 * ((i & 2) ? 1 : -1), + sz * 2 * ((i & 4) ? 1 : -1) + ) + mat4.transformPoint(vec3, debugPoints[i]); + } + + app.drawLines(debugLines, debugColor); +}; + class SplatResource extends ContainerResource { device: GraphicsDevice; elements: PlyElement[]; @@ -270,25 +343,6 @@ class SplatResource extends ContainerResource { return null; } - // create a debug splat - vertexElement.count = 1; - vertexElement.properties = [ - { type: 'float', name: 'x', storage: [0], byteSize: 4 }, - { type: 'float', name: 'y', storage: [0], byteSize: 4 }, - { type: 'float', name: 'z', storage: [0], byteSize: 4 }, - { type: 'float', name: 'f_dc_0', storage: [1.772453850905516], byteSize: 4 }, - { type: 'float', name: 'f_dc_1', storage: [1.772453850905516], byteSize: 4 }, - { type: 'float', name: 'f_dc_2', storage: [1.772453850905516], byteSize: 4 }, - { type: 'float', name: 'opacity', storage: [1], byteSize: 4 }, - { type: 'float', name: 'scale_0', storage: [1.0], byteSize: 4 }, - { type: 'float', name: 'scale_1', storage: [0.5], byteSize: 4 }, - { type: 'float', name: 'scale_2', storage: [0.2], byteSize: 4 }, - { type: 'float', name: 'rot_0', storage: [0], byteSize: 4 }, - { type: 'float', name: 'rot_1', storage: [0], byteSize: 4 }, - { type: 'float', name: 'rot_2', storage: [0], byteSize: 4 }, - { type: 'float', name: 'rot_3', storage: [1], byteSize: 4 }, - ]; - const find = (name: string) => { return vertexElement.properties.find((property: any) => property.name === name && property.storage)?.storage; }; @@ -422,22 +476,28 @@ class SplatResource extends ContainerResource { return aabb; }; - const calcAabb2 = () => { + // calculate scene aabb taking into account splat size + const calcAabb2 = (aabb: BoundingBox) => { + // initialize aabb + aabb.center.set(floatData[0], floatData[1], floatData[2]); + aabb.halfExtents.set(0, 0, 0); + + const splat = new Float32Array(stride); + const tmpAabb = new BoundingBox(); for (let i = 0; i < vertexElement.count; ++i) { - const x = floatData[i * stride + 0]; - const y = floatData[i * stride + 1]; - const z = floatData[i * stride + 2]; - const rx = floatData[i * stride + 4]; - const ry = floatData[i * stride + 5]; - const rz = floatData[i * stride + 6]; - const sx = floatData[i * stride + 7]; - const sy = floatData[i * stride + 8]; - const sz = floatData[i * stride + 9]; + for (let j = 0; j < stride; ++j) { + splat[j] = floatData[i * stride + j]; + } + getSplatAabb(tmpAabb, splat); + aabb.add(tmpAabb); } } // set custom aabb - result.render.customAabb = calcAabb(); + // result.render.customAabb = calcAabb(); + const aabb = new BoundingBox(); + calcAabb2(aabb); + result.render.customAabb = aabb; // create sort worker if (options?.app && options?.camera) { @@ -473,6 +533,8 @@ class SplatResource extends ContainerResource { stride: stride }, [buf]); + const viewport = [ this.device.width, this.device.height ]; + options.app.on('prerender', () => { const t = options.camera.getWorldTransform().data; sortWorker.postMessage({ @@ -480,27 +542,19 @@ class SplatResource extends ContainerResource { cameraDirection: { x: t[8], y: t[9], z: t[10] } }); - const focal = [1164.6601287484507, 1159.5880733038064]; - - this.quadMaterial.setParameter('viewport', [this.device.width, this.device.height]); - this.quadMaterial.setParameter('focal', [focal[0], focal[1]]); - // this.quadMaterial.setParameter('focal', [this.device.width / 2, this.device.height / 2]); + viewport[0] = this.device.width; + viewport[1] = this.device.height; + this.quadMaterial.setParameter('viewport', viewport); // debug render splat bounds - // const immediate = options.app.scene.immediate; - // const data = new Float32Array(vertexBuffer.lock()); - // for (let i = 0; i < data.length / stride; ++i) { - // const x = data[i * stride + 0]; - // const y = data[i * stride + 1]; - // const z = data[i * stride + 2]; - // const rx = data[i * stride + 4]; - // const ry = data[i * stride + 5]; - // const rz = data[i * stride + 6]; - // const sx = data[i * stride + 7]; - // const sy = data[i * stride + 8]; - // const sz = data[i * stride + 9]; + // const modelMat = result.getWorldTransform(); + // const splat = new Float32Array(stride); + // for (let i = 0; i < vertexElement.count; ++i) { + // for (let j = 0; j < stride; ++j) { + // splat[j] = floatData[i * stride + j]; + // } + // renderDebugSplat(options.app, modelMat, splat); // } - // vertexBuffer.unlock(); }); } From 0fff40f5c8493a8ab0b3ea0e607f498690429fc3 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 18 Oct 2023 15:08:52 +0100 Subject: [PATCH 5/6] small --- src/splat-resource.ts | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/src/splat-resource.ts b/src/splat-resource.ts index a4f35e6..20113ec 100644 --- a/src/splat-resource.ts +++ b/src/splat-resource.ts @@ -453,31 +453,8 @@ class SplatResource extends ContainerResource { castShadows: false // shadows not supported }); - // calculate accurate aabb - const calcAabb = () => { - const minmax = (data: Float32Array) => { - let min = data[0]; - let max = data[0]; - for (let i = 1; i < data.length; ++i) { - min = Math.min(min, data[i]); - max = Math.max(max, data[i]); - } - return [min, max]; - }; - const xMinMax = minmax(x); - const yMinMax = minmax(y); - const zMinMax = minmax(z); - - const aabb = new BoundingBox(); - aabb.setMinMax( - new Vec3(xMinMax[0] - 1, yMinMax[0] - 0.5, zMinMax[0] - 0.2), - new Vec3(xMinMax[1] + 1, yMinMax[1] + 0.5, zMinMax[1] + 0.2)); - - return aabb; - }; - // calculate scene aabb taking into account splat size - const calcAabb2 = (aabb: BoundingBox) => { + const calcAabb = (aabb: BoundingBox) => { // initialize aabb aabb.center.set(floatData[0], floatData[1], floatData[2]); aabb.halfExtents.set(0, 0, 0); @@ -494,9 +471,8 @@ class SplatResource extends ContainerResource { } // set custom aabb - // result.render.customAabb = calcAabb(); const aabb = new BoundingBox(); - calcAabb2(aabb); + calcAabb(aabb); result.render.customAabb = aabb; // create sort worker From b48edd3f9d29b91f9b24e634110b628e2e9e8da5 Mon Sep 17 00:00:00 2001 From: Donovan Hutchence Date: Wed, 18 Oct 2023 15:09:53 +0100 Subject: [PATCH 6/6] lint --- src/splat-resource.ts | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/splat-resource.ts b/src/splat-resource.ts index 20113ec..2af6a2c 100644 --- a/src/splat-resource.ts +++ b/src/splat-resource.ts @@ -35,6 +35,7 @@ import { SortWorker } from './sort-worker'; // set true to render splats as oriented boxes const debugRender = false; +const debugRenderBounds = false; const quatToMat3 = ` mat3 quatToMat3(vec3 R) @@ -269,7 +270,7 @@ const renderDebugSplat = (app: AppBase, worldMat: Mat4, data: Float32Array) => { sx * 2 * ((i & 1) ? 1 : -1), sy * 2 * ((i & 2) ? 1 : -1), sz * 2 * ((i & 4) ? 1 : -1) - ) + ); mat4.transformPoint(vec3, debugPoints[i]); } @@ -468,7 +469,7 @@ class SplatResource extends ContainerResource { getSplatAabb(tmpAabb, splat); aabb.add(tmpAabb); } - } + }; // set custom aabb const aabb = new BoundingBox(); @@ -509,7 +510,7 @@ class SplatResource extends ContainerResource { stride: stride }, [buf]); - const viewport = [ this.device.width, this.device.height ]; + const viewport = [this.device.width, this.device.height]; options.app.on('prerender', () => { const t = options.camera.getWorldTransform().data; @@ -523,14 +524,16 @@ class SplatResource extends ContainerResource { this.quadMaterial.setParameter('viewport', viewport); // debug render splat bounds - // const modelMat = result.getWorldTransform(); - // const splat = new Float32Array(stride); - // for (let i = 0; i < vertexElement.count; ++i) { - // for (let j = 0; j < stride; ++j) { - // splat[j] = floatData[i * stride + j]; - // } - // renderDebugSplat(options.app, modelMat, splat); - // } + if (debugRenderBounds) { + const modelMat = result.getWorldTransform(); + const splat = new Float32Array(stride); + for (let i = 0; i < vertexElement.count; ++i) { + for (let j = 0; j < stride; ++j) { + splat[j] = floatData[i * stride + j]; + } + renderDebugSplat(options.app, modelMat, splat); + } + } }); }