added files

.gitignore

@@ -0,0 +1,5 @@
+.cache
+build
+*.png
+*.mp4
+*.avi

CMakeLists.txt

@@ -0,0 +1,85 @@
+cmake_minimum_required(VERSION 3.10.2)
+set (CMAKE_CXX_STANDARD 17)
+# add_compile_options(-std=c++17)
+
+set(CMAKE_THREAD_LIBS_INIT "-lpthread")
+set(CMAKE_HAVE_THREADS_LIBRARY 1)
+set(CMAKE_USE_WIN32_THREADS_INIT 0)
+set(CMAKE_USE_PTHREADS_INIT 1)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+
+set(SOURCES main.cpp src/constants.h src/rgbd_utilities.cpp src/rgbd_utilities.h src/image.h 
+    src/surface_mesh.cpp src/surface_mesh.h src/pointcloud_registration.cpp src/pointcloud_registration.h 
+    src/timer.h src/utilities.h src/utilities.cpp src/visualizer.cpp src/visualizer.h 
+    src/uv_unwrapper.h src/uv_unwrapper.cpp src/math_modules.cpp src/math_modules.h src/texture_projector.cpp
+    src/texture_projector.h src/shader_stuff.h src/shader_stuff.cpp 
+    src/s3d_camera.h
+    src/features.h src/features.cpp src/feature_graph.h
+    src/realtime_mesh.h src/realtime_mesh.cpp
+    src/poseg2.h src/poseg2.cpp
+    src/registration_params.h
+    src/pose_predictor.h src/pose_predictor.cpp
+    libraries/xatlas/xatlas.h libraries/xatlas/xatlas.cpp
+    libraries/tinyobj/tiny_obj_loader.h libraries/tinyobj/tiny_obj_loader.cpp
+)
+
+project(Open3DGen)
+
+find_package(Open3D HINTS ${CMAKE_INSTALL_PREFIX}/lib/CMake)
+list(APPEND Open3D_LIBRARIES dl)
+
+find_package(GLEW REQUIRED)
+find_package(glfw3 3.3 REQUIRED)
+find_package(OpenGL REQUIRED)
+
+find_package (Eigen3 3.3 REQUIRED NO_MODULE)
+
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${Open3D_C_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${Open3D_CXX_FLAGS} -fopenmp") 
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${Open3D_EXE_LINKER_FLAGS}")
+
+add_definitions(-DUNIX)
+add_compile_options(-Wno-deprecated-declarations)
+add_compile_options(-Wno-unused-result)
+
+# add_definitions(-g -fopenmp)
+add_definitions(-O3 -fopenmp)
+
+message(STATUS "Found Open3D ${Open3D_VERSION} ${Open3D_LIBRARY_DIRS}")
+
+
+# OpenCV
+find_package(OpenCV REQUIRED)
+message(STATUS "OpenCV library status:")
+message(STATUS "    config: ${OpenCV_DIR}")
+message(STATUS "    version: ${OpenCV_VERSION}")
+message(STATUS "    libraries: ${OpenCV_LIBS}")
+message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")
+
+find_package(GTSAM REQUIRED)
+
+# this is not implemented yet
+# find_package(realsense2 REQUIRED)
+
+link_directories(${Open3D_LIBRARY_DIRS})
+# include_directories(include ${realsense_INCLUDE_DIR})
+
+include_directories(${GLEW_INCLUDE_DIRS} ${EIGEN_INCLUDE_DIR})
+
+add_executable(Open3DGen ${SOURCES})
+
+target_include_directories(Open3DGen PUBLIC ${Open3D_INCLUDE_DIRS} 
+    # ${realsense_INCLUDE_DIR}
+)
+
+target_link_libraries(Open3DGen 
+                        ${Open3D_LIBRARIES} 
+                        ${OpenCV_LIBS}
+                        gtsam
+#                        ${realsense2_LIBRARY}
+                        ${GLEW_LIBRARIES}
+                        glfw
+                        OpenGL::GL
+                        Eigen3::Eigen
+)

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Ultra Video Group
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,157 @@
+# Open3DGen
+
+## Building
+Ubuntu 21.04 -based Linux distros should work. Tested on Pop-os 21.04. 
+
+### Hardware Requirements
+OpenGL 4.6 capable GPU, any decently modern GPU should do. With higher resolution textures (4k, 8k) 64Gb of RAM is recommended. This requiremenet will be fixed in a future release. Multicore CPU is recommended, as Open3DGen is highly parallelized.
+
+</br>
+
+Recommended RAM amounts: 
+- 2k textures: 16Gb
+- 4k textures: 32Gb
+- 8k textures: 64Gb
+
+### Dependencies
+The following dependencies can be found in the repositories:
+```
+libboost1.71-all-dev
+libopencv-dev
+libopencv-contrib-dev
+libopen3d-dev
+libglfw-dev
+libglew-dev
+libopengl-dev
+```
+and can be installed with the command
+```
+sudo apt install -y libboost1.71-all-dev && sudo apt install -y libopencv-dev && sudo apt install -y libopencv-contrib-dev && sudo apt install -y libopen3d-dev && sudo apt install -y libglfw3-dev && sudo apt install -y libglew-dev && sudo apt install -y libopengl-dev
+```
+Installing GTSAM requires building from source: https://github.com/borglab/gtsam.
+
+Make sure clang is also installed. Clang12 was used on the development platform. 
+
+
+### Build
+At this point building the project should be as simple as running `./build.sh`.
+
+### Notes for Developers
+
+If LanguageServerProtocol is used for code analysis and autocompletion, compile_commands can be generated with
+
+```
+cd build
+cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
+cd ..
+ln build/compile_commands.json .
+```
+</br>
+
+A quick way to compile or compile and run is to use the shell scripts `./only_compile.sh` and `./compile_and_run.sh`. Make sure to modify `compile_and_run.sh` to use the correct command line arguments.
+
+## How to Use
+Open3DGen requires at least 3 command line arguments to run:
+```
+in 'path to input rgbd sequence'
+intr 'path to camera intrinsics'
+out 'path to the output directory'
+```
+
+For example, </br>
+
+`./build/Open3DGen in data_files/sequence intr data_files/realsense_intrinsic_1280.matrix out out/ ` </br>
+
+The input RGBD sequence is expected to be separated into 2 folders: rgb/ and depth/. The RGB images should be named in the format of `rgb_number.png` (e.g. `rgb_0.png`) and the depth images should be in the format of `depth_number.png` (e.g. `depth_0.png`). The corresponding rgb and depth image must have the same number, and the numbers are expected to be in ascending order. 
+</br>
+
+The `camera_intrinsics.matrix` -file contains the image dimensions and the calibrated camera intrinsics. The expected format is 
+```
+width;height
+fx;0.0;cx
+0.0;fy;cy;
+0.0;0.0;1.0
+```
+
+</br>
+
+For example, `realsense_intrinsic_1280.matrix` contains 
+```
+1280;720
+912.91984103;0.0;637.74400638
+0.0;912.39513184;358.96757428
+0.0;0.0;1.0
+```
+
+</br>
+
+Do not modify the values set in the texture projection shader, correct values are loaded automatically.
+
+The visualization can be disabled by commenting the line `#define DEBUG_VISUALIZE` in the file `src/constants.h`.
+
+## Demo Scene
+A demo RGB-D sequence is provided [here](http://ultravideo.fi/open3dgen/demo_rock.tar.xz). Extract the `demo_rock` folder into `data_files/`, and execute the following command.
+
+`./build/Open3DGen in data_files/demo_rock/ intr data_files/realsense_intrinsic_1280.matrix out data_files/out/ refine_cameras false reject_blur 1.1 crop false skip_stride 28 unwrap true pipeline full poisson_depth 7 simplify_voxel_size 0.075 out_res 4096 depth_far_clip 3.0 max_feature_count 2000 assets_path data_files/`
+
+The resulting `.obj`, `.mtl` and textures will be in the folder `data_files/out/`. It is recommended the `_dilated.png` texture is used. Use e.g. Blender to visualize the results.
+
+The RGB-D sequence is not particularly high quality and contains an unfortunate amount of blur. Better sequence will be provided when the weather so allows. 
+The result should look like this
+![demo_rock_out](/images/demo_rock_out.png)
+
+</br>
+
+### Command Line Arguments
+The supported command line arguments are as follows
+
+- image sequence input path (`in`)
+- camera input intrinsics (`intr`)
+- output path (`out`)
+- output texture resolution (`out_res`, currently supports only 2048, 4096, and 8192)
+- use only every nth camera view to project textures (`project_every_nth`, 1 by default (= project from every view))
+- mesh poisson depth reconstruction depth (`poisson_depth`, 9 by default)
+- specify the mesh simplification (`simplify_voxel_size`, disabled by default (`0.0`), in meters, e.g. 5cm would be 0.05. Larger values result in lower quality but speed up the texture projection significantly. 5cm-15cm is generally a good range of values to try)
+- export intermediary mesh, before texture projection (`export_mesh_interm`, `true`/`false`, default `false`)
+- use a specific mesh for projection (`mesh_path`)
+- the parts of the pipeline that will be run (`pipeline`, options are `full`, `only_pointcloud`, `only_mesh`, or `only_project`)
+	- `full` runs the entire sequence and the output is a textured mesh
+	- `only_pointcloud` localizes the cameras and creates a pointcloud. Camera positions are serialized into a file in the `out` -folder
+	- `only_mesh` requires the cameras to be already serialized. 
+	- `only_project` only projects the textures, useful for editing the mesh before projection, e.g. re-topology or manually creating uv -coordinates
+- specify camera distortion coefficients (`dist_coeff 'path to file'`, uses zero distortion by default)
+	- loads distortion coefficients from a file, coefficients should be space separated e.g. `0.66386586e-01, -5.26413220e-01, -1.01376611e-03, 1.59777094e-04, 4.65208008e-01`
+- feature candidate chain length (`cchain_len`, 7 by default). Specifies through how many frames should a feature be tracked before it can be made into a landmark.
+- feature landmark match min (`lam_min_count`, 20 by default). Specifies how many features should be matched to landmarks for a frame to be considered succesful.
+- serialized camera and pointcloud (`serialized_cameras_path`, where the serialized cameras and pointcloud should be saved/loaded from)
+- assets path (`assets_path`, where to load assets from, data_files/ -by default)
+- load mesh from path (`load_mesh_path`, none by default. Loads a mesh from given path for projection)
+- unwrap UVs (`unwrap`, options `true`/`false`, `true` by default)
+- refine camera pose after adding cameras (`refine_cameras`, `true`/`false`, `false` by default)
+- texture projection shader work group size (`workgroup_size`, default is 1024)
+- skip first `n` frames after the frist frame (`skip_stride`), after the first frame skips `n` frames, used to help force camera pose quality
+- write intermediary per-frame projected textures (`write_interm_images`, `true`/`false`, `false` by default)
+- set the depth far clipping distance (`depth_far_clip`, units in meters, default is `3.6`)
+- set the maximum number of features detected (`max_feature_count`, default is unlimited (`0`), which is overkill. Note: this can be slow especially in outdoor scenes. To significantly increase performance, limit this one to a more reasonable value, e.g. `1000`)
+- remove the poisson reconstruction "ballooning" artefacts (`crop`, `true`/`false`, default is `true`. May not always work correctly and removes too much detail.)
+- texture projection ray threshold (`ray_threshold`, default is `0.9`, which will project everything except nearly 90d angles, the dot-product between face normal and the projection ray, after which the result is discarded. Larger values project more, but result in more smearing in the textures)
+- texture projection blur threshold (`reject_blur`, default is disabled `0.0`. A value of `1.0` means frames with sharpness less than the average sharpness will be rejected, values greater than `1.0` will thus result in more frames getting rejected)
+
+
+### Known issues and TODO
+- The real-time camera pose estimation gets progressively slower as more frames are added due to the increased amount of landmarks
+	- implement a more efficient landmark search algorithm
+- Add support for the realsense API and real-time capture and reconstruction
+- SIFT is not particularly fast
+	- experiment with SURF and/or ORB
+- The texture projection stage runs out of system RAM
+	- load and save the projected textures individually instead of acting on the entire sequence at once
+- Xatlas produces poor UVs
+	- if we cannot get Xatlas to keep connected triangles connected in the UV -map as well, the plan is to phase out Xatlas at some point in favor of some other solution
+- Also planned in the next release is a better texture projection algorithm based on inverse-matrices. This should not only be faster, but more scalable for very large and high-poly meshes as well. Additionally, this would reduce the issue of UV bleeding.
+- Implement a way of smoothing noisy depth maps, decrease the effect of "edge bleed" on sharp depth transitions. Could view synthesis be used for this? 
+
+
+## Paper
+If you are using Open3DGen in your research, please refer to the following paper: <br/>
+`T. T. Niemirepo, M. Viitanen, and J. Vanne, “Open3DGen: Open-Source software for reconstructing textured 3D models from RGB-D images,” Accepted to ACM Multimedia Syst. Conf., Istanbul, Turkey, Sept.-Oct. 2021. `

build.sh

@@ -0,0 +1,5 @@
+mkdir build
+cd build
+cmake ..
+cmake --build . --parallel 12 --config Debug
+cd ..

compile_and_run.sh

@@ -0,0 +1,3 @@
+cd build
+cmake --build . --parallel 12 --config Debug
+./Open3DGen in ../data_files/capture/demo_rock/ intr ../data_files/realsense_intrinsic_1280.matrix out ../data_files/out/ refine_cameras false reject_blur 1.1 crop false skip_stride 28 unwrap true pipeline full poisson_depth 7 simplify_voxel_size 0.075 out_res 4096 depth_far_clip 3.0 max_feature_count 2000

data_files/bgr_to_rgb.py

@@ -0,0 +1,8 @@
+import cv2
+
+image = "data_files/out/texture_out_gpu_average_dilated.png"
+
+img = cv2.imread(image)
+
+rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+cv2.imwrite(image, rgb)

data_files/debug_camera_mesh.mtl

@@ -0,0 +1,13 @@
+# Blender MTL File: 'None'
+# Material Count: 1
+
+newmtl default
+Ns 500.000001
+Ka 1.000000 1.000000 1.000000
+Kd 0.800000 0.800000 0.800000
+Ks 0.800000 0.800000 0.800000
+Ke 0.000000 0.000000 0.000000
+Ni 1.450000
+d 1.000000
+illum 2
+map_Kd color_grid.png

data_files/debug_camera_mesh.obj

@@ -0,0 +1,70 @@
+# Blender v2.82 (sub 7) OBJ File: ''
+# www.blender.org
+mtllib debug_camera_mesh.mtl
+o Cube_Cube.001
+v 0.049152 0.049152 0.040001
+v 0.049152 0.049152 0.000000
+v 0.049152 -0.049152 -0.000000
+v 0.049152 -0.049152 0.040001
+v -0.049152 0.049152 0.000000
+v -0.049152 -0.049152 -0.000000
+v -0.049152 0.049152 0.040001
+v -0.159677 0.159677 0.240001
+v -0.159677 -0.159677 0.240001
+v -0.049152 -0.049152 0.040001
+v 0.159677 0.159677 0.240001
+v 0.159677 -0.159677 0.240001
+vt 0.384150 0.673807
+vt 0.384150 0.826193
+vt 0.334984 0.826193
+vt 0.334984 0.673807
+vt 0.868171 0.153149
+vt 0.743387 0.153149
+vt 0.743387 0.000762
+vt 0.868171 0.000762
+vt 0.282517 0.673807
+vt 0.282517 0.826193
+vt 0.001650 0.997525
+vt 0.001650 0.502475
+vt 0.002027 0.497525
+vt 0.002027 0.002475
+vt 0.407403 0.002475
+vt 0.407403 0.497525
+vt 0.717483 0.826193
+vt 0.717483 0.673807
+vt 0.998350 0.502475
+vt 0.998350 0.997525
+vt 0.691947 0.173807
+vt 0.691947 0.326193
+vt 0.411080 0.497525
+vt 0.411080 0.002475
+vt 0.741113 0.173807
+vt 0.741113 0.326193
+vt 0.668317 0.826193
+vt 0.668317 0.673807
+vt 0.331683 0.673807
+vt 0.331683 0.826193
+vt 0.665017 0.502475
+vt 0.665017 0.997525
+vn 1.0000 0.0000 0.0000
+vn 0.0000 0.0000 -1.0000
+vn -0.8752 0.0000 -0.4837
+vn 0.0000 0.0000 1.0000
+vn 0.0000 0.8752 -0.4837
+vn 0.0000 -0.8752 -0.4837
+vn 0.0000 -1.0000 0.0000
+vn 0.0000 1.0000 0.0000
+vn -1.0000 0.0000 0.0000
+vn 0.8752 -0.0000 -0.4837
+usemtl default
+s 1
+f 1/1/1 4/2/1 3/3/1 2/4/1
+f 2/5/2 3/6/2 6/7/2 5/8/2
+f 7/9/3 10/10/3 9/11/3 8/12/3
+f 8/13/4 9/14/4 12/15/4 11/16/4
+f 1/17/5 7/18/5 8/19/5 11/20/5
+f 10/21/6 4/22/6 12/23/6 9/24/6
+f 6/25/7 3/26/7 4/22/7 10/21/7
+f 2/27/8 5/28/8 7/18/8 1/17/8
+f 5/29/9 6/30/9 10/10/9 7/9/9
+f 11/31/10 12/32/10 4/2/10 1/1/10