Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Splat fix and debug rendering #248

Merged
merged 8 commits into from
Oct 18, 2023
+112 −30
Merged

Splat fix and debug rendering #248

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
475cbc8
speed up sort, unify rendering data
slimbuck Oct 17, 2023
a1369f6
start devbug rendering
slimbuck Oct 17, 2023
cfd3dab
Merge branch 'main' into debug-rendering
slimbuck Oct 18, 2023
5968a97
working
slimbuck Oct 18, 2023
88f96e2
Merge branch 'main' into debug-rendering
slimbuck Oct 18, 2023
886ef4d
splat size correctly rendered
slimbuck Oct 18, 2023
0fff40f
small
slimbuck Oct 18, 2023
b48edd3
lint
slimbuck Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 112 additions & 30 deletions src/splat-resource.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
import {
AppBase,
BoundingBox,
BUFFER_DYNAMIC,
Color,
ContainerResource,
createShaderFromCode,
createBox,
CULLFACE_NONE,
CULLFACE_BACK,
Entity,
GraphicsDevice,
Mat4,
Material,
Mesh,
MeshInstance,
Expand All @@ -32,6 +35,7 @@ import { SortWorker } from './sort-worker';

// set true to render splats as oriented boxes
const debugRender = false;
const debugRenderBounds = false;

const quatToMat3 = `
mat3 quatToMat3(vec3 R)
Expand Down Expand Up @@ -69,7 +73,6 @@ uniform mat4 matrix_view;
uniform mat4 matrix_projection;

uniform vec2 viewport;
uniform vec2 focal;

varying vec2 texCoord;
varying vec4 color;
Expand Down Expand Up @@ -127,9 +130,11 @@ void main(void)
splat_cova.z, splat_covb.y, splat_covb.z
);

float focal = viewport.x * matrix_projection[0][0];

mat3 J = mat3(
focal.x / splat_cam.z, 0., -(focal.x * splat_cam.x) / (splat_cam.z * splat_cam.z),
0., focal.y / splat_cam.z, -(focal.y * splat_cam.y) / (splat_cam.z * splat_cam.z),
focal / splat_cam.z, 0., -(focal * splat_cam.x) / (splat_cam.z * splat_cam.z),
0., focal / splat_cam.z, -(focal * splat_cam.y) / (splat_cam.z * splat_cam.z),
0., 0., 0.
);

Expand All @@ -150,7 +155,7 @@ void main(void)
vec2 v2 = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x);

gl_Position = splat_proj +
vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 8.0,
vec4((vertex_position.x * v1 + vertex_position.y * v2) / viewport * 2.0,
0.0, 0.0) * splat_proj.w;

texCoord = vertex_position * 2.0;
Expand Down Expand Up @@ -203,6 +208,75 @@ void main(void)
}
`;

const vec3 = new Vec3();
const mat4 = new Mat4();
const aabb = new BoundingBox();

const debugPoints = [new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3(), new Vec3()];
const debugLines = [
debugPoints[0], debugPoints[1], debugPoints[1], debugPoints[3], debugPoints[3], debugPoints[2], debugPoints[2], debugPoints[0],
debugPoints[4], debugPoints[5], debugPoints[5], debugPoints[7], debugPoints[7], debugPoints[6], debugPoints[6], debugPoints[4],
debugPoints[0], debugPoints[4], debugPoints[1], debugPoints[5], debugPoints[2], debugPoints[6], debugPoints[3], debugPoints[7]
];
const debugColor = new Color(1, 1, 0, 0.4);

const getSplatMat = (result: Mat4, data: Float32Array) => {
const px = data[0];
const py = data[1];
const pz = data[2];
const x = data[4];
const y = data[5];
const z = data[6];
const w = Math.sqrt(1 - (x * x + y * y + z * z));

// build rotation matrix
result.data.set([
1.0 - 2.0 * (z * z + w * w),
2.0 * (y * z + x * w),
2.0 * (y * w - x * z),
0,

2.0 * (y * z - x * w),
1.0 - 2.0 * (y * y + w * w),
2.0 * (z * w + x * y),
0,

2.0 * (y * w + x * z),
2.0 * (z * w - x * y),
1.0 - 2.0 * (y * y + z * z),
0,

px, py, pz, 1
]);
};

const getSplatAabb = (result: BoundingBox, data: Float32Array) => {
getSplatMat(mat4, data);
aabb.center.set(0, 0, 0);
aabb.halfExtents.set(data[7] * 2, data[8] * 2, data[9] * 2);
result.setFromTransformedAabb(aabb, mat4);
};

const renderDebugSplat = (app: AppBase, worldMat: Mat4, data: Float32Array) => {
getSplatMat(mat4, data);
mat4.mul2(worldMat, mat4);

const sx = data[7];
const sy = data[8];
const sz = data[9];

for (let i = 0; i < 8; ++i) {
vec3.set(
sx * 2 * ((i & 1) ? 1 : -1),
sy * 2 * ((i & 2) ? 1 : -1),
sz * 2 * ((i & 4) ? 1 : -1)
);
mat4.transformPoint(vec3, debugPoints[i]);
}

app.drawLines(debugLines, debugColor);
};

class SplatResource extends ContainerResource {
device: GraphicsDevice;
elements: PlyElement[];
Expand Down Expand Up @@ -380,31 +454,27 @@ class SplatResource extends ContainerResource {
castShadows: false // shadows not supported
});

// calculate accurate aabb
const calcAabb = () => {
const minmax = (data: Float32Array) => {
let min = data[0];
let max = data[0];
for (let i = 1; i < data.length; ++i) {
min = Math.min(min, data[i]);
max = Math.max(max, data[i]);
}
return [min, max];
};
const xMinMax = minmax(x);
const yMinMax = minmax(y);
const zMinMax = minmax(z);
// calculate scene aabb taking into account splat size
const calcAabb = (aabb: BoundingBox) => {
// initialize aabb
aabb.center.set(floatData[0], floatData[1], floatData[2]);
aabb.halfExtents.set(0, 0, 0);

const aabb = new BoundingBox();
aabb.setMinMax(
new Vec3(xMinMax[0] - 1, yMinMax[0] - 0.5, zMinMax[0] - 0.2),
new Vec3(xMinMax[1] + 1, yMinMax[1] + 0.5, zMinMax[1] + 0.2));

return aabb;
const splat = new Float32Array(stride);
const tmpAabb = new BoundingBox();
for (let i = 0; i < vertexElement.count; ++i) {
for (let j = 0; j < stride; ++j) {
splat[j] = floatData[i * stride + j];
}
getSplatAabb(tmpAabb, splat);
aabb.add(tmpAabb);
}
};

// set custom aabb
result.render.customAabb = calcAabb();
const aabb = new BoundingBox();
calcAabb(aabb);
result.render.customAabb = aabb;

// create sort worker
if (options?.app && options?.camera) {
Expand Down Expand Up @@ -440,18 +510,30 @@ class SplatResource extends ContainerResource {
stride: stride
}, [buf]);

const viewport = [this.device.width, this.device.height];

options.app.on('prerender', () => {
const t = options.camera.getWorldTransform().data;
sortWorker.postMessage({
cameraPosition: { x: t[12], y: t[13], z: t[14] },
cameraDirection: { x: t[8], y: t[9], z: t[10] }
});

const focal = [1164.6601287484507, 1159.5880733038064];

this.quadMaterial.setParameter('viewport', [this.device.width, this.device.height]);
this.quadMaterial.setParameter('focal', [focal[0], focal[1]]);
// this.quadMaterial.setParameter('focal', [this.device.width / 2, this.device.height / 2]);
viewport[0] = this.device.width;
viewport[1] = this.device.height;
this.quadMaterial.setParameter('viewport', viewport);

// debug render splat bounds
if (debugRenderBounds) {
const modelMat = result.getWorldTransform();
const splat = new Float32Array(stride);
for (let i = 0; i < vertexElement.count; ++i) {
for (let j = 0; j < stride; ++j) {
splat[j] = floatData[i * stride + j];
}
renderDebugSplat(options.app, modelMat, splat);
}
}
});
}

Expand Down