Import code for optical flow in the browser

.github/workflows/build.yml

@@ -60,3 +60,6 @@ jobs:
       - name: Build depends
         if: steps.restore-depends.outputs.cache-hit != 'true'
         run: tools/build-depends.sh all
+
+      - name: Build libraries
+        run: lib/build-ci.sh

.gitignore

@@ -1,6 +1,9 @@
 # Generated dependencies
 /src/generated
 
+# Generated libraries
+/frontend/public/motion_tracker
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -143,3 +146,8 @@ Temporary
 
 # libzip build files
 /third-party/libzip
+
+# Eclipse project files
+.cproject
+.project
+.settings

lib/CMakeLists.txt

@@ -0,0 +1,174 @@
+################################################################################
+#
+#  Copyright (C) 2020-2023 retro.ai
+#  This file is part of retro-dapp - https://github.com/RetroAI/retro-dapp
+#
+#  SPDX-License-Identifier: Apache-2.0
+#  See the file LICENSE.md for more information.
+#
+################################################################################
+
+################################################################################
+#
+# Build system for C++ libraries
+#
+# Required CMake variables:
+#
+#   CMAKE_FIND_ROOT_PATH - Point this to dependencies compiled with Emscripten
+#   CMAKE_INSTALL_PREFIX - Point this to the "public" folder
+#
+################################################################################
+
+################################################################################
+#
+# Project settings
+#
+################################################################################
+
+project(retroai)
+
+cmake_minimum_required(VERSION 3.0.0)
+
+set(CMAKE_CXX_STANDARD 17)
+
+################################################################################
+#
+# Dependencies
+#
+################################################################################
+
+find_package(Ceres REQUIRED)
+find_package(Eigen3 REQUIRED)
+find_package(Glog REQUIRED)
+find_package(OpenCV REQUIRED)
+
+add_definitions(-DCERES_FOUND=1)
+
+################################################################################
+#
+# Define sources
+#
+################################################################################
+
+#
+# Motion tracker
+#
+
+set(MOTION_TRACKER_SOURCES
+  api/imgproc.cpp
+  api/reconstruction.cpp
+  api/scene.cpp
+  api/video.cpp
+  kernels/cpu/cpu_imgproc.cpp
+  kernels/cpu/cpu_reconstruction.cpp
+  kernels/cpu/cpu_scene.cpp
+  kernels/cpu/cpu_video.cpp
+  motion_tracker/motion_tracker.cpp
+  motion_tracker/motion_tracker_embinder.cpp
+  motion_tracker/vision_graph.cpp
+  utils/emscripten_utils.cpp
+  utils/frame_pool.cpp
+  utils/image_utils.cpp
+  utils/math_utils.cpp
+)
+
+################################################################################
+#
+# Build libraries
+#
+# TODO:
+#
+#   * Build properly instead of shelling out
+#   * Could refactor this into macros
+#
+################################################################################
+
+include_directories(
+  ${CMAKE_SOURCE_DIR}
+)
+
+string(APPEND EMSCRIPTEN_LINK_FLAGS
+  "--bind "
+  # "-o dist/engine.js "
+  # " -std=c++11 "
+  # " -O2 "
+  # " --preload-file textures "
+  # " --preload-file shaders "
+  # " --preload-file fonts "
+  # " --pre-js pre-module.j "
+  # " --post-js post-module.j "
+  "-s ALLOW_MEMORY_GROWTH=1 "
+  "-s ASSERTIONS=1 "
+  # " -s DEMANGLE_SUPPORT=1 "
+  # " -s DISABLE_EXCEPTION_CATCHING=0 "
+  "-s ERROR_ON_UNDEFINED_SYMBOLS=0 "
+  # " -s FULL_ES3=1 "
+  # " -s GL_ASSERTIONS=1 "
+  # " -s GL_UNSAFE_OPTS=0 "
+  # " -s INVOKE_RUN=0 "
+  # " -s LEGACY_GL_EMULATION=0 "
+  #"-s LLD_REPORT_UNDEFINED "
+  # " -s OFFSCREENCANVAS_SUPPORT=1 "
+  # " -s SAFE_HEAP=1 "
+  #"-s TOTAL_MEMORY=67108864 "
+  # " -s USE_FREETYPE=1 "
+  # " -s USE_GLFW=3 "
+  # " -s USE_WEBGL2=1 "
+  "-s USE_ZLIB=1 "
+  # " -s WASM=1 "
+)
+
+#
+# Motion tracker
+#
+
+add_executable(motion_tracker
+  ${MOTION_TRACKER_SOURCES}
+)
+
+target_include_directories(motion_tracker PRIVATE
+  ${OpenCV_INCLUDE_DIRS}
+)
+
+target_link_libraries(motion_tracker PRIVATE
+  ${OpenCV_LIBS}
+)
+
+if (${CMAKE_SYSTEM_NAME} MATCHES "Emscripten")
+  set_target_properties(motion_tracker PROPERTIES
+    COMPILE_FLAGS " \
+      -O0 \
+      -g4 \
+      -s DISABLE_EXCEPTION_CATCHING=0 \
+      -s INITIAL_MEMORY=26214400 \
+    "
+    # 26214400 is 25 MiB
+    LINK_FLAGS " \
+      --bind \
+      --source-map-base https://retro.ai/ \
+      -O0 \
+      -g4 \
+      -s DISABLE_EXCEPTION_CATCHING=0 \
+      -s INITIAL_MEMORY=26214400 \
+    "
+  )
+endif ()
+
+################################################################################
+#
+# Install libraries
+#
+################################################################################
+
+#
+# Motion tracker
+#
+
+INSTALL(
+  FILES
+    "${CMAKE_BINARY_DIR}/motion_tracker.js"
+    "${CMAKE_BINARY_DIR}/motion_tracker.wasm"
+    "${CMAKE_BINARY_DIR}/motion_tracker.wasm.map"
+  DESTINATION
+    motion_tracker
+)

lib/api/imgproc.cpp

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2020 Garrett Brown
+ * This file is part of trajectory-reconstruction - https://github.com/eigendude/trajectory-reconstruction
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * See LICENSE.txt for more information.
+ */
+
+#include "imgproc.hpp"
+
+#include <opencv2/gapi/imgproc.hpp>
+
+cv::GMat imgproc::RGBA2Gray(const cv::GMat& rgbaImage)
+{
+  return cv::gapi::RGBA2Gray(rgbaImage);
+}
+
+cv::GArray<cv::Point2f> imgproc::GoodFeaturesToTrack(const cv::GMat& grayscaleImage,
+                                                     const cv::GScalar& maxFeatures,
+                                                     const cv::GScalar& minDistance,
+                                                     double qualityLevel,
+                                                     const cv::Mat& mask,
+                                                     int blockSize,
+                                                     bool useHarrisDetector,
+                                                     double k)
+{
+  return GGoodFeatures::on(grayscaleImage,
+                           maxFeatures,
+                           qualityLevel,
+                           minDistance,
+                           mask,
+                           blockSize,
+                           useHarrisDetector,
+                           k);
+}

lib/api/imgproc.hpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2020 Garrett Brown
+ * This file is part of trajectory-reconstruction - https: *github.com/eigendude/trajectory-reconstruction
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * See LICENSE.txt for more information.
+ */
+
+#pragma once
+
+#include <opencv2/gapi/garray.hpp>
+#include <opencv2/gapi/gkernel.hpp>
+#include <opencv2/gapi/gmat.hpp>
+#include <opencv2/core/types.hpp>
+
+namespace imgproc
+{
+  G_TYPED_KERNEL(GGoodFeatures,
+                 <cv::GArray<cv::Point2f>(cv::GMat, cv::GScalar, double, cv::GScalar, cv::Mat, int, bool, double)>,
+                 "com.trajectoryReconstruction.imgproc.goodFeaturesToTrack") {
+    static cv::GArrayDesc outMeta(cv::GMatDesc, cv::GScalarDesc, double, cv::GScalarDesc, const cv::Mat&, int, bool, double) {
+      return cv::empty_array_desc();
+    }
+  };
+
+  /*!
+   * \brief Convert RGBA image to grayscale
+   *
+   * \param rgbaImage The 4-channel RGBA image
+   *
+   * \return The single-channel grayscale image
+   */
+  cv::GMat RGBA2Gray(const cv::GMat& rgbaImage);
+
+  /*!
+   * \brief Get some good features to track
+   *
+   * \param grayscaleImage The single-channel grayscale image
+   *
+   * \param maxCorners Maximum number of corners to return.
+   * If there are more corners than are found, the strongest of them is
+   * returned.
+   *
+   * \param qualityLevel Minimal accepted quality of image corners.
+   * This parameter characterizes the minimal accepted quality of
+   * corners.
+   *
+   * The parameter value is multiplied by the best corner quality measure,
+   * which is the minimal eigenvalue or the Harris function response.
+   *
+   * The corners with the quality measure less than the product are rejected.
+   *
+   * For example, if the best corner has the quality measure = 1500, and the
+   * qualityLevel = 0.01, then all the corners with the quality measure less
+   * than 15 are rejected.
+   *
+   * \param minDistance Minimum possible Euclidean distance between the
+   * returned corners
+   *
+   * \return A list of good features to track
+   */
+  cv::GArray<cv::Point2f> GoodFeaturesToTrack(const cv::GMat& grayscaleImage,
+                                              const cv::GScalar& maxFeatures,
+                                              const cv::GScalar& minDistance,
+                                              double qualityLevel = 0.01,
+                                              const cv::Mat& mask = cv::Mat(),
+                                              int blockSize = 3,
+                                              bool useHarrisDetector = false,
+                                              double k = 0.04);
+}

lib/api/reconstruction.cpp

@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2020 Garrett Brown
+ * This file is part of trajectory-reconstruction - https://github.com/eigendude/trajectory-reconstruction
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * See LICENSE.txt for more information.
+ */
+
+#include "reconstruction.hpp"
+
+reconstruction::TplTrajectory reconstruction::ReconstructTrajectory(const cv::GArray<std::vector<cv::Point2f>>& pointHistory, const cv::GMat& cameraMatrix)
+{
+  return GReconstructTrajectory::on(pointHistory, cameraMatrix);
+}

lib/api/reconstruction.hpp

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2020 Garrett Brown
+ * This file is part of trajectory-reconstruction - https://github.com/eigendude/trajectory-reconstruction
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * See LICENSE.txt for more information.
+ */
+
+#pragma once
+
+#include <opencv2/core/types.hpp>
+#include <opencv2/gapi/garray.hpp>
+#include <opencv2/gapi/gkernel.hpp>
+#include <opencv2/gapi/gmat.hpp>
+#include <utility> // std::tuple
+#include <vector>
+
+namespace reconstruction
+{
+  using TplTrajectory = std::tuple<cv::GMat, cv::GMat>;
+  using TplTrajectoryDesc = std::tuple<cv::GMatDesc, cv::GMatDesc>;
+
+  G_TYPED_KERNEL(GReconstructTrajectory, <TplTrajectory(cv::GArray<std::vector<cv::Point2f>>, cv::GMat)>,
+      "com.trajectoryReconstruction.reconstructTrajectory")
+  {
+    static TplTrajectoryDesc outMeta(cv::GArrayDesc pointHistory, cv::GMatDesc cameraDesc)
+    {
+      return std::make_tuple(cv::empty_gmat_desc(), cameraDesc);
+    }
+  };
+
+  /*!
+   * Reconstruct the trajectory using 2d point correspondences
+   *
+   * \param pointHistory Input vector of vectors of 2d points (the inner vector is per image)
+   * \param cameraMatrix Input camera matrix used as initial guess
+   *
+   * \return Tuple consisting of:
+   *    * Output vector with the 3x4 projections matrices of each image
+   *    * Output array with estimated 3d points
+   *    * Output camera matrix
+   */
+  TplTrajectory ReconstructTrajectory(const cv::GArray<std::vector<cv::Point2f>>& pointHistory,
+                                      const cv::GMat& cameraMatrix);
+}

lib/api/scene.cpp

@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2020 Garrett Brown
+ * This file is part of trajectory-reconstruction - https://github.com/eigendude/trajectory-reconstruction
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * See LICENSE.txt for more information.
+ */
+
+#include "scene.hpp"
+
+scene::TplDoubles scene::CalcSceneScore(const cv::GMat& prevImg,
+                          const cv::GMat& nextImg,
+                          const cv::GOpaque<double>& prevMafd,
+                          unsigned int width,
+                          unsigned int height)
+{
+  return GCalcSceneScore::on(prevImg, nextImg, prevMafd, width, height);
+}