3D voxel reconstruction added using OpenGL

.gitignore

@@ -130,4 +130,3 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Stan Fortoński
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

assignment.py

@@ -0,0 +1,157 @@
+import os
+import sys
+
+import cv2
+import glm
+import random
+import numpy as np
+import backgroundsubtractor
+import xml.etree.ElementTree as ET
+import calibration
+import voxel_construction
+block_size = 1.0
+
+# Parameters for voxel positions function
+# Initialization with loading videos and training background models
+initialized = False
+videos = []
+bg_models = []
+# Background model parameters for every camera
+# figure_threshold, figure_inner_threshold,
+# apply_opening_pre, apply_closing_pre, apply_opening_post, apply_closing_post
+cam_bg_model_params = [
+    [5000, 115, False, False, True, True],
+    [5000, 115, False, False, True, True],
+    [5000, 175, False, True, True, True],
+    [5000, 115, False, False, False, True]
+]
+# Currently loaded frames and their index
+current_frames = []
+frame_count = 0
+previous_masks = []
+# Lookup table for voxels
+lookup_table = None
+voxel_points = None
+
+
+def generate_grid(width, depth):
+    # Generates the floor grid locations
+    # You don't need to edit this function
+    data, colors = [], []
+    for x in range(width):
+        for z in range(depth):
+            data.append([x * block_size - width / 2, -block_size, z * block_size - depth / 2])
+            colors.append([1.0, 1.0, 1.0] if (x + z) % 2 == 0 else [0, 0, 0])
+    return data, colors
+
+
+def set_voxel_positions(width, height, depth):
+    # Generates random voxel locations
+    list_videos = []
+    # list_videos(cv2.VideoCapture(os.path.join("data", "cam" + str(cam + 1))))
+
+    voxel_3d_points = voxel_construction.create_voxel_volume(width, height * 2, depth)
+    lookup_table = voxel_construction.create_lookup_table(voxel_3d_points, 4, "data", "config.xml")
+
+    # Extract foreground mask from video frame for each camera
+    foregrounds = []
+    cam_allframes = []
+    for cam in range(1, 5):
+        background_model = backgroundsubtractor.find_background(os.path.join("data", "cam" + str(cam)),
+                                                                "background.avi", history=500, threshold=300,
+                                                                detect_shadow=False)
+
+        removed=backgroundsubtractor.substract_background(os.path.join("data", "cam" + str(cam)), "video.avi",
+                                                  background_model, 10, True,
+                                                  False)
+        contoured_foreground = backgroundsubtractor.contouring(removed, outer_threshold=1000, inner_threshold=130)
+        foregrounds.append(contoured_foreground)
+        cap = cv2.VideoCapture( os.path.join(os.path.join("data", "cam" + str(cam)), "video.avi"))
+        frames = []
+        while True:
+            ret, frame = cap.read()
+            if not ret:
+                break
+            frames.append(frame)
+            break
+        cam_allframes.append(np.array(frames)[0])
+    #voxel_visible,voxels_visible_colors= voxel_construction.visible_voxels_coloring(lookup_table, foregrounds,cam_allframes )
+    voxel_visible,voxels_visible_colors= voxel_construction.visible_voxels_coloring(lookup_table, foregrounds,cam_allframes )
+
+    allvisible_positions=[]
+    colors=[]
+    scaling_factor=64
+    threshold=4 # if all on for the specific camera view
+    for voxel, camera in voxel_visible.items():
+
+        if sum(camera.values()) >= 4:
+            # Swap y and z and flip sign of y
+            x = voxel[0] / scaling_factor
+            y = - (voxel[2] / scaling_factor)
+            z = voxel[1] / scaling_factor
+            allvisible_positions.append([x, y, z])
+
+
+            # Use color of only 2nd camera (front) and convert to 0-1
+            colors.append(voxels_visible_colors[voxel][2][::-1] / 255.0)
+
+    return allvisible_positions, colors
+
+
+
+def get_cam_positions():
+    """
+        Calculates positions of cameras with rotation and translation vectors. Swaps Y and Z axis to convert OpenCV
+        3D coordinate system to OpenGL and makes the new Y negative to face the viewer.
+
+        :return: returns position for every camera and color vector for every camera
+        """
+    tree = ET.parse('./data/checkerboard.xml')
+    row = 0
+    col = 0
+    chessboard_square_size = 0
+    root = tree.getroot()
+    for child in root:
+        if child.tag == 'CheckerBoardWidth':
+            col = int(child.text)
+        if child.tag == 'CheckerBoardHeight':
+            row = int(child.text)
+        if child.tag == 'CheckerBoardSquareSize':
+            chessboard_square_size = int(child.text)
+
+
+    scale = 1.0 / chessboard_square_size
+
+    # Get all camera positions
+    camera_positions = []
+    for camera in range(4):
+        # Get camera rotation and translation
+        _, _, rvecs, tvecs = voxel_construction.load_config_info(os.path.join("data", "cam" + str(camera + 1)),
+                                                    "config.xml")
+        rmtx, _ = cv2.Rodrigues(rvecs)
+
+        # Get camera position
+        position = -np.matrix(rmtx).T * np.matrix(tvecs) * scale
+
+        # Swap Y and Z axis for OpenGL system and make new Y negative to face the viewer
+        camera_positions.append([position[0][0], -position[2][0], position[1][0]])
+
+    return camera_positions, [[1.0, 0, 0], [0, 1.0, 0], [0, 0, 1.0], [1.0, 1.0, 0]]
+
+
+def get_cam_rotation_matrices():
+    # Generates dummy camera rotation matrices, looking down 45 degrees towards the center of the room
+
+    cam_rotations = []
+    for camera in range(4):
+        # Get camera rotation
+        _, _, rvecs, _ = voxel_construction.load_config_info(os.path.join("data", "cam" + str(camera + 1)),
+                                                "config.xml")
+
+    cam_angles = [[0, 45, -45], [0, 135, -45], [0, 225, -45], [0, 315, -45]]
+    cam_rotations = [glm.mat4(1), glm.mat4(1), glm.mat4(1), glm.mat4(1)]
+    for c in range(len(cam_rotations)):
+        cam_rotations[c] = glm.rotate(cam_rotations[c], cam_angles[c][0] * np.pi / 180, [1, 0, 0])
+        cam_rotations[c] = glm.rotate(cam_rotations[c], cam_angles[c][1] * np.pi / 180, [0, 1, 0])
+        cam_rotations[c] = glm.rotate(cam_rotations[c], cam_angles[c][2] * np.pi / 180, [0, 0, 1])
+    return cam_rotations

calibration.py

@@ -1,7 +1,5 @@
 import numpy as np
 import sys
-path_to_module = "C:/Users/Gholi002/AppData/Local/r-miniconda/envs/myenvironment/Lib/site-packages"
-sys.path.append(path_to_module)
 import cv2
 import os
 import glob

config.json

@@ -0,0 +1,13 @@
+{
+    "app_name": "3D Voxel Visualizer",
+    "debug_mode": false,
+    "fullscreen": false,
+    "world_width": 128,
+    "world_height": 64,
+    "world_depth": 128,
+    "window_width": 1280,
+    "window_height": 720,
+    "near_plane": 0.1,
+    "far_plane": 500,
+    "sampling_level": 4
+}

engine/base/program.py

@@ -0,0 +1,84 @@
+import glm
+from engine.base.shader import Shader
+from OpenGL.GL import *
+from OpenGL.error import NullFunctionError
+
+
+class Program:
+    def __init__(self):
+        self.__programId = 0
+        self.shaders = []
+
+    def attach_shader(self, shader):
+        self.shaders.append(shader)
+
+    def link(self):
+        self.__programId = glCreateProgram()
+        for shader in self.shaders:
+            shader.compile()
+            glAttachShader(self.__programId, shader.getId())
+
+        glLinkProgram(self.__programId)
+
+        for shader in self.shaders:
+            shader.delete()
+        self.shaders.clear()
+
+        if glGetProgramiv(self.__programId, GL_LINK_STATUS) != GL_TRUE:
+            info = glGetProgramInfoLog(self.__programId)
+            self.delete()
+            raise RuntimeError(f'Error in program linking: {info}')
+
+    def __del__(self):
+        self.delete()
+
+    def delete(self):
+        try:
+            glDeleteProgram(self.__programId)
+            self.__programId = 0
+        except NullFunctionError:
+            pass
+
+    def use(self):
+        glUseProgram(self.__programId)
+
+    def getId(self):
+        return self.__programId
+
+    def getAttribLocation(self, name):
+        return glGetAttribLocation(self.__programId, name)
+
+    def getUniformLocation(self, name):
+        return glGetUniformLocation(self.__programId, name)
+
+    def setInt(self, name, value):
+        glUniform1i(self.getUniformLocation(name), value)
+
+    def setFloat(self, name, value):
+        glUniform1f(self.getUniformLocation(name), value)
+
+    def setVec2(self, name, vec):
+        glUniform2fv(self.getUniformLocation(name), 1, glm.value_ptr(vec))
+
+    def setVec3(self, name, vec):
+        glUniform3fv(self.getUniformLocation(name), 1, glm.value_ptr(vec))
+
+    def setVec4(self, name, vec):
+        glUniform4fv(self.getUniformLocation(name), 1, glm.value_ptr(vec))
+
+    def setMat2(self, name, mat):
+        glUniformMatrix2fv(self.getUniformLocation(name), 1, GL_FALSE, glm.value_ptr(mat))
+
+    def setMat3(self, name, mat):
+        glUniformMatrix3fv(self.getUniformLocation(name), 1, GL_FALSE, glm.value_ptr(mat))
+
+    def setMat4(self, name, mat):
+        glUniformMatrix4fv(self.getUniformLocation(name), 1, GL_FALSE, glm.value_ptr(mat))
+
+
+def get_linked_program(vert_path, frag_path):
+    program = Program()
+    program.attach_shader(Shader(vert_path, GL_VERTEX_SHADER))
+    program.attach_shader(Shader(frag_path, GL_FRAGMENT_SHADER))
+    program.link()
+    return program

engine/base/shader.py

@@ -0,0 +1,38 @@
+import os
+from OpenGL.GL import *
+from OpenGL.error import NullFunctionError
+
+
+class Shader:
+    def __init__(self, sourcePath, shaderType):
+        self.__shaderId = 0
+        if not os.path.exists(sourcePath):
+            raise RuntimeError(f'Shader source file {sourcePath} does not exists.')
+        self.__sourcePath = sourcePath
+        self.shaderType = shaderType
+
+    def compile(self):
+        self.__shaderId = glCreateShader(self.shaderType)
+        glShaderSource(self.__shaderId, self.__load_source())
+        glCompileShader(self.__shaderId)
+        if glGetShaderiv(self.__shaderId, GL_COMPILE_STATUS) != GL_TRUE:
+            info = glGetShaderInfoLog(self.__shaderId)
+            raise RuntimeError(f'Shader compilation failed:\n{info}')
+
+    def __load_source(self):
+        with open(self.__sourcePath) as file:
+            source = file.read()
+        return source
+
+    def getId(self):
+        return self.__shaderId
+
+    def __del__(self):
+        self.delete()
+
+    def delete(self):
+        try:
+            glDeleteShader(self.__shaderId)
+            self.__shaderId = 0
+        except NullFunctionError:
+            pass

engine/buffer/blurbuffer.py

@@ -0,0 +1,37 @@
+from OpenGL.GL import *
+from OpenGL.error import NullFunctionError
+from engine.buffer.framebuffer import FrameBuffer
+
+
+class BlurBuffer:
+    def __init__(self):
+        self.colorBuffers = None
+        self.FBOs = None
+
+    def create(self, width, height):
+        self.FBOs = glGenFramebuffers(2)
+        self.colorBuffers = glGenTextures(2)
+        for i in range(2):
+            glBindFramebuffer(GL_FRAMEBUFFER, self.FBOs[i])
+            glBindTexture(GL_TEXTURE_2D, self.colorBuffers[i])
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
+            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB16F, width, height, 0, GL_RGB, GL_FLOAT, None)
+            glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, self.colorBuffers[i], 0)
+
+        if glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE:
+            raise RuntimeError('Error when creating Blur Framebuffers.')
+        glBindFramebuffer(GL_FRAMEBUFFER, 0)
+
+    def __del__(self):
+        self.delete()
+
+    def delete(self):
+        try:
+            glDeleteFramebuffers(2, self.FBOs)
+            glDeleteTextures(2, self.colorBuffers)
+            self.colorBuffers = 0
+        except NullFunctionError:
+            pass

engine/buffer/depthbuffer.py

@@ -0,0 +1,23 @@
+from OpenGL.GL import *
+from OpenGL.error import NullFunctionError
+from engine.buffer.texture import Texture
+
+
+class DepthBuffer(Texture):
+    def __init__(self):
+        super().__init__(GL_TEXTURE_2D)
+
+    def create(self, width, height):
+        self.bind()
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, width, height, 0, GL_DEPTH_COMPONENT, GL_FLOAT, None)
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER)
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER)
+        glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, [1.0, 1.0, 1.0, 1.0])
+        self.unbind()
+
+    def attach(self):
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, self.getId(), 0)
+        glDrawBuffer(GL_NONE)
+        glReadBuffer(GL_NONE)