Import code for sparse optical flow in the browser

.clang-format

@@ -0,0 +1,92 @@
+---
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: DontAlign
+AlignOperands:   true
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: InlineOnly
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Allman
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     100
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 2
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex:           '<EGL/.*\.h>'
+    Priority:        6
+  - Regex:           '(["/]PlatformDefs|"(system|system_gl|system_egl))\.h"'
+    Priority:        5
+  - Regex:           '"platform/[^/]+/'
+    Priority:        2
+  - Regex:           '^<[a-z0-9_]+>$'
+    Priority:        3
+  - Regex:           '^<(assert|complex|ctype|errno|fenv|float|inttypes|iso646|limits|locale|math|setjmp|signal|stdalign|stdarg|stdatomic|stdbool|stddef|stdint|stdio|stdlib|stdnoreturn|string|tgmath|threads|time|uchar|wchar|wctype)\.h>$'
+    Priority:        3
+  - Regex:           '^<'
+    Priority:        4
+  - Regex:           '.*'
+    Priority:        1
+IncludeIsMainRegex: '$'
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60000
+PointerAlignment: Left
+ReflowComments:  false
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...

.github/workflows/build.yml

@@ -61,3 +61,6 @@ jobs:
       - name: Build depends
         if: steps.restore-depends.outputs.cache-hit != 'true'
         run: tools/build-depends.sh all
+
+      - name: Build libraries
+        run: lib/build-ci.sh

.github/workflows/clang-format-check.yml

@@ -0,0 +1,32 @@
+# .github/workflows/clang-format-check.yml
+name: Clang Format Check
+
+on:
+  push:
+    paths:
+    - 'lib/**.cpp'
+    - 'lib/**.h'
+    - '.clang-format'
+
+  pull_request:
+    paths:
+    - 'lib/**.cpp'
+    - 'lib/**.h'
+    - '.clang-format'
+
+jobs:
+  format:
+    name: Clang Format
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install Clang Format
+      run: sudo apt-get install -y clang-format-15
+
+    - name: Run Clang Format
+      run: |
+        find lib -regex '.*\.\(cpp\|hpp\|h\|cc\|cxx\)' -exec clang-format-15 -style=file -i {} \;
+        git diff --exit-code --color

.gitignore

@@ -1,6 +1,9 @@
 # Generated dependencies
 /frontend/src/generated
 
+# Generated libraries
+/frontend/public/motion_tracker
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -21,7 +24,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/
@@ -143,3 +145,8 @@ Temporary
 
 # libzip build files
 /third-party/libzip
+
+# Eclipse project files
+.cproject
+.project
+.settings

lib/CMakeLists.txt

@@ -0,0 +1,167 @@
+################################################################################
+#
+#  Copyright (C) 2020-2023 retro.ai
+#  This file is part of retro-dapp - https://github.com/RetroAI/retro-dapp
+#
+#  SPDX-License-Identifier: Apache-2.0
+#  See the file LICENSE.md for more information.
+#
+################################################################################
+
+################################################################################
+#
+# Build system for C++ libraries
+#
+# Required CMake variables:
+#
+#   CMAKE_FIND_ROOT_PATH - Point this to dependencies compiled with Emscripten
+#   CMAKE_INSTALL_PREFIX - Point this to the "public" folder
+#
+################################################################################
+
+################################################################################
+#
+# Project settings
+#
+################################################################################
+
+project(retroai)
+
+cmake_minimum_required(VERSION 3.0.0)
+
+set(CMAKE_CXX_STANDARD 17)
+
+################################################################################
+#
+# Dependencies
+#
+################################################################################
+
+find_package(OpenCV REQUIRED)
+
+################################################################################
+#
+# Define sources
+#
+################################################################################
+
+#
+# Motion tracker
+#
+
+set(MOTION_TRACKER_SOURCES
+  api/imgproc.cpp
+  api/scene.cpp
+  api/video.cpp
+  kernels/cpu/cpu_imgproc.cpp
+  kernels/cpu/cpu_scene.cpp
+  kernels/cpu/cpu_video.cpp
+  motion_tracker/motion_tracker.cpp
+  motion_tracker/motion_tracker_embinder.cpp
+  motion_tracker/vision_graph.cpp
+  utils/emscripten_utils.cpp
+  utils/frame_pool.cpp
+  utils/image_utils.cpp
+  utils/math_utils.cpp
+)
+
+################################################################################
+#
+# Build libraries
+#
+# TODO:
+#
+#   * Build properly instead of shelling out
+#   * Could refactor this into macros
+#
+################################################################################
+
+include_directories(
+  ${CMAKE_SOURCE_DIR}
+)
+
+string(APPEND EMSCRIPTEN_LINK_FLAGS
+  "--bind "
+  # "-o dist/engine.js "
+  # " -std=c++11 "
+  # " -O2 "
+  # " --preload-file textures "
+  # " --preload-file shaders "
+  # " --preload-file fonts "
+  # " --pre-js pre-module.j "
+  # " --post-js post-module.j "
+  "-s ALLOW_MEMORY_GROWTH=1 "
+  "-s ASSERTIONS=1 "
+  # " -s DEMANGLE_SUPPORT=1 "
+  # " -s DISABLE_EXCEPTION_CATCHING=0 "
+  "-s ERROR_ON_UNDEFINED_SYMBOLS=0 "
+  # " -s FULL_ES3=1 "
+  # " -s GL_ASSERTIONS=1 "
+  # " -s GL_UNSAFE_OPTS=0 "
+  # " -s INVOKE_RUN=0 "
+  # " -s LEGACY_GL_EMULATION=0 "
+  #"-s LLD_REPORT_UNDEFINED "
+  # " -s OFFSCREENCANVAS_SUPPORT=1 "
+  # " -s SAFE_HEAP=1 "
+  #"-s TOTAL_MEMORY=67108864 "
+  # " -s USE_FREETYPE=1 "
+  # " -s USE_GLFW=3 "
+  # " -s USE_WEBGL2=1 "
+  "-s USE_ZLIB=1 "
+  # " -s WASM=1 "
+)
+
+#
+# Motion tracker
+#
+
+add_executable(motion_tracker
+  ${MOTION_TRACKER_SOURCES}
+)
+
+target_include_directories(motion_tracker PRIVATE
+  ${OpenCV_INCLUDE_DIRS}
+)
+
+target_link_libraries(motion_tracker PRIVATE
+  ${OpenCV_LIBS}
+)
+
+if (${CMAKE_SYSTEM_NAME} MATCHES "Emscripten")
+  set_target_properties(motion_tracker PROPERTIES
+    COMPILE_FLAGS " \
+      -O0 \
+      -g4 \
+      -s DISABLE_EXCEPTION_CATCHING=0 \
+      -s INITIAL_MEMORY=26214400 \
+    "
+    # 26214400 is 25 MiB
+    LINK_FLAGS " \
+      --bind \
+      --source-map-base https://retro.ai/ \
+      -O0 \
+      -g4 \
+      -s DISABLE_EXCEPTION_CATCHING=0 \
+      -s INITIAL_MEMORY=26214400 \
+    "
+  )
+endif ()
+
+################################################################################
+#
+# Install libraries
+#
+################################################################################
+
+#
+# Motion tracker
+#
+
+INSTALL(
+  FILES
+    "${CMAKE_BINARY_DIR}/motion_tracker.js"
+    "${CMAKE_BINARY_DIR}/motion_tracker.wasm"
+    "${CMAKE_BINARY_DIR}/motion_tracker.wasm.map"
+  DESTINATION
+    motion_tracker
+)

lib/api/imgproc.cpp

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2020-2023 retro.ai
+ * This file is part of retro-dapp - https://github.com/RetroAI/retro-dapp
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * See LICENSE.md for more information.
+ */
+
+#include "imgproc.hpp"
+
+#include <opencv2/gapi/imgproc.hpp>
+
+cv::GMat imgproc::RGBA2Gray(const cv::GMat& rgbaImage)
+{
+  return cv::gapi::RGBA2Gray(rgbaImage);
+}
+
+cv::GArray<cv::Point2f> imgproc::GoodFeaturesToTrack(const cv::GMat& grayscaleImage,
+                                                     const cv::GScalar& maxFeatures,
+                                                     const cv::GScalar& minDistance,
+                                                     double qualityLevel,
+                                                     const cv::Mat& mask,
+                                                     int blockSize,
+                                                     bool useHarrisDetector,
+                                                     double k)
+{
+  return GGoodFeatures::on(grayscaleImage, maxFeatures, qualityLevel, minDistance, mask, blockSize,
+                           useHarrisDetector, k);
+}