Computer Vision based Alpha Matting

modules/README.md

@@ -10,6 +10,8 @@ $ cmake -D OPENCV_EXTRA_MODULES_PATH=<opencv_contrib>/modules -D BUILD_opencv_<r
 
 - **aruco**: ArUco and ChArUco Markers -- Augmented reality ArUco marker and "ChARUco" markers where ArUco markers embedded inside the white areas of the checker board.
 
+- **alphamat**: Computer Vision based Alpha Matting -- Given an input image and a trimap, generate an alpha matte.
+
 - **bgsegm**: Background segmentation algorithm combining statistical background image estimation and per-pixel Bayesian segmentation.
 
 - **bioinspired**: Biological Vision -- Biologically inspired vision model: minimize noise and luminance variance, transient event segmentation, high dynamic range tone mapping methods.

modules/alphamat/CMakeLists.txt

@@ -0,0 +1,9 @@
+if(NOT HAVE_EIGEN)
+  message(STATUS "Module opencv_alphamat disabled because the following dependencies are not found: Eigen")
+  ocv_module_disable(alphamat)
+endif()
+
+ocv_define_module(alphamat
+    opencv_core
+    opencv_imgproc
+)

modules/alphamat/README.md

@@ -0,0 +1,23 @@
+# Computer Vision based Alpha Matting
+
+This project was part of the Google Summer of Code 2019.
+
+####Student: Muskaan Kularia
+####Mentor: Sunita Nayak
+***
+Alphamatting is the problem of extracting the foreground from an image. Given the input of an image and its corresponding trimap, we try to extract the foreground from the background.
+
+This project is implementation of "[[Designing Effective Inter-Pixel Information Flow for Natural Image Matting](http://people.inf.ethz.ch/aksoyy/ifm/)]" by Yağız Aksoy, Tunç Ozan Aydın and Marc Pollefeys[1]. It required implementation of parts of other papers [2,3,4].
+
+
+## References
+
+[1] Yagiz Aksoy, Tunc Ozan Aydin, Marc Pollefeys, "[Designing Effective Inter-Pixel Information Flow for Natural Image Matting](http://people.inf.ethz.ch/aksoyy/ifm/)", CVPR, 2017.
+
+[2] Roweis, Sam T., and Lawrence K. Saul. "[Nonlinear dimensionality reduction by locally linear embedding](https://science.sciencemag.org/content/290/5500/2323)" Science 290.5500 (2000): 2323-2326.
+
+[3] Anat Levin, Dani Lischinski, Yair Weiss, "[A Closed Form Solution to Natural Image Matting](https://www.researchgate.net/publication/5764820_A_Closed-Form_Solution_to_Natural_Image_Matting)", IEEE TPAMI, 2008.
+
+[4] Qifeng Chen, Dingzeyu Li, Chi-Keung Tang, "[KNN Matting](http://dingzeyu.li/files/knn-matting-tpami.pdf)", IEEE TPAMI, 2013.
+
+[5] Yagiz Aksoy, "[Affinity Based Matting Toolbox](https://github.com/yaksoy/AffinityBasedMattingToolbox)".

modules/alphamat/doc/alphamat.bib

@@ -0,0 +1,26 @@
+@inproceedings{aksoy2017designing,
+  title={Designing effective inter-pixel information flow for natural image matting},
+  author={Aksoy, Yagiz and Ozan Aydin, Tunc and Pollefeys, Marc},
+  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
+  pages={29--37},
+  year={2017}
+}
+
+@article{roweis2000nonlinear,
+  title={Nonlinear dimensionality reduction by locally linear embedding},
+  author={Roweis, Sam T and Saul, Lawrence K},
+  journal={science},
+  volume={290},
+  number={5500},
+  pages={2323--2326},
+  year={2000},
+  publisher={American Association for the Advancement of Science}
+}
+
+@inproceedings{shahrian2013improving,
+  title={Improving image matting using comprehensive sampling sets},
+  author={Shahrian, Ehsan and Rajan, Deepu and Price, Brian and Cohen, Scott},
+  booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
+  pages={636--643},
+  year={2013}
+}

modules/alphamat/include/opencv2/alphamat.hpp

@@ -0,0 +1,32 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+/** Information Flow algorithm implementaton for alphamatting */
+
+#ifndef _OPENCV_ALPHAMAT_HPP_
+#define _OPENCV_ALPHAMAT_HPP_
+
+/**
+ * @defgroup alphamat Alpha Matting
+ * This module is dedicated to compute alpha matting of images, given the input image and an input trimap.
+ * The samples directory includes easy examples of how to use the module.
+ */
+
+namespace cv { namespace alphamat {
+//! @addtogroup alphamat
+//! @{
+
+/**
+ * The implementation is based on Designing Effective Inter-Pixel Information Flow for Natural Image Matting by Yağız Aksoy, Tunç Ozan Aydın and Marc Pollefeys, CVPR 2019.
+ *
+ * This module has been originally developed by Muskaan Kularia and Sunita Nayak as a project
+ * for Google Summer of Code 2019 (GSoC 19).
+ *
+ */
+CV_EXPORTS_W void infoFlow(InputArray image, InputArray tmap, OutputArray result);
+
+//! @}
+}}  // namespace
+
+#endif

modules/alphamat/samples/information_flow_matting.cpp

@@ -0,0 +1,77 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include <iostream>
+#include "opencv2/highgui.hpp"
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/alphamat.hpp>
+
+using namespace std;
+using namespace cv;
+using namespace cv::alphamat;
+
+const char* keys =
+    "{img || input image name}"
+    "{tri || input trimap image name}"
+    "{out || output image name}"
+    "{help h || print help message}"
+;
+
+int main(int argc, char* argv[])
+{
+    CommandLineParser parser(argc, argv, keys);
+    parser.about("This sample demonstrates Information Flow Alpha Matting");
+
+    if (parser.has("help"))
+    {
+        parser.printMessage();
+        return 0;
+    }
+
+    string img_path = parser.get<std::string>("img");
+    string trimap_path = parser.get<std::string>("tri");
+    string result_path = parser.get<std::string>("out");
+
+    if (!parser.check()
+            || img_path.empty() || trimap_path.empty())
+    {
+        parser.printMessage();
+        parser.printErrors();
+        return 1;
+    }
+
+    Mat image, tmap;
+
+    image = imread(img_path, IMREAD_COLOR);  // Read the input image file
+    if (image.empty())
+    {
+        printf("Cannot read image file: '%s'\n", img_path.c_str());
+        return 1;
+    }
+
+    tmap = imread(trimap_path, IMREAD_GRAYSCALE);
+    if (tmap.empty())
+    {
+        printf("Cannot read trimap file: '%s'\n", trimap_path.c_str());
+        return 1;
+    }
+
+    Mat result;
+    infoFlow(image, tmap, result);
+
+    if (result_path.empty())
+    {
+        namedWindow("result alpha matte", WINDOW_NORMAL);
+        imshow("result alpha matte", result);
+        waitKey(0);
+    }
+    else
+    {
+        imwrite(result_path, result);
+        printf("Result saved: '%s'\n", result_path.c_str());
+    }
+
+    return 0;
+}

modules/alphamat/src/3rdparty/KDTreeVectorOfVectorsAdaptor.h

@@ -0,0 +1,117 @@
+/***********************************************************************
+ * Software License Agreement (BSD License)
+ *
+ * Copyright 2011-16 Jose Luis Blanco ([email protected]).
+ *   All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *************************************************************************/
+
+#pragma once
+
+#include "nanoflann.hpp"
+
+#include <vector>
+
+// ===== This example shows how to use nanoflann with these types of containers: =======
+//typedef std::vector<std::vector<double> > my_vector_of_vectors_t;
+//typedef std::vector<Eigen::VectorXd> my_vector_of_vectors_t;   // This requires #include <Eigen/Dense>
+// =====================================================================================
+
+
+/** A simple vector-of-vectors adaptor for nanoflann, without duplicating the storage.
+  *  The i'th vector represents a point in the state space.
+  *
+  *  \tparam DIM If set to >0, it specifies a compile-time fixed dimensionality for the points in the data set, allowing more compiler optimizations.
+  *  \tparam num_t The type of the point coordinates (typically, double or float).
+  *  \tparam Distance The distance metric to use: nanoflann::metric_L1, nanoflann::metric_L2, nanoflann::metric_L2_Simple, etc.
+  *  \tparam IndexType The type for indices in the KD-tree index (typically, size_t of int)
+  */
+template <class VectorOfVectorsType, typename num_t = double, int DIM = -1, class Distance = nanoflann::metric_L2, typename IndexType = size_t>
+struct KDTreeVectorOfVectorsAdaptor
+{
+  typedef KDTreeVectorOfVectorsAdaptor<VectorOfVectorsType, num_t, DIM,Distance> self_t;
+  typedef typename Distance::template traits<num_t, self_t>::distance_t metric_t;
+  typedef nanoflann::KDTreeSingleIndexAdaptor< metric_t, self_t, DIM, IndexType>  index_t;
+
+  index_t* index; //! The kd-tree index for the user to call its methods as usual with any other FLANN index.
+
+  /// Constructor: takes a const ref to the vector of vectors object with the data points
+  KDTreeVectorOfVectorsAdaptor(const size_t /* dimensionality */, const VectorOfVectorsType &mat, const int leaf_max_size = 10) : m_data(mat)
+  {
+    assert(mat.size() != 0 && mat[0].size() != 0);
+    const size_t dims = mat[0].size();
+    if (DIM>0 && static_cast<int>(dims) != DIM)
+      throw std::runtime_error("Data set dimensionality does not match the 'DIM' template argument");
+    index = new index_t( static_cast<int>(dims), *this /* adaptor */, nanoflann::KDTreeSingleIndexAdaptorParams(leaf_max_size ) );
+    index->buildIndex();
+  }
+
+  ~KDTreeVectorOfVectorsAdaptor() {
+    delete index;
+  }
+
+  const VectorOfVectorsType &m_data;
+
+  /** Query for the \a num_closest closest points to a given point (entered as query_point[0:dim-1]).
+    *  Note that this is a short-cut method for index->findNeighbors().
+    *  The user can also call index->... methods as desired.
+    * \note nChecks_IGNORED is ignored but kept for compatibility with the original FLANN interface.
+    */
+  //inline void query(const num_t *query_point, const size_t num_closest, IndexType *out_indices, num_t *out_distances_sq, const int nChecks_IGNORED = 10) const
+  inline void query(const num_t *query_point, const size_t num_closest, IndexType *out_indices, num_t *out_distances_sq) const
+  {
+    nanoflann::KNNResultSet<num_t, IndexType> resultSet(num_closest);
+    resultSet.init(out_indices, out_distances_sq);
+    index->findNeighbors(resultSet, query_point, nanoflann::SearchParams());
+  }
+
+  /** @name Interface expected by KDTreeSingleIndexAdaptor
+    * @{ */
+
+  const self_t & derived() const {
+    return *this;
+  }
+  self_t & derived()       {
+    return *this;
+  }
+
+  // Must return the number of data points
+  inline size_t kdtree_get_point_count() const {
+    return m_data.size();
+  }
+
+  // Returns the dim'th component of the idx'th point in the class:
+  inline num_t kdtree_get_pt(const size_t idx, const size_t dim) const {
+    return m_data[idx][dim];
+  }
+
+  // Optional bounding-box computation: return false to default to a standard bbox computation loop.
+  // Return true if the BBOX was already computed by the class and returned in "bb" so it can be avoided to redo it again.
+  // Look at bb.size() to find out the expected dimensionality (e.g. 2 or 3 for point clouds)
+  template <class BBOX>
+  bool kdtree_get_bbox(BBOX & /*bb*/) const {
+    return false;
+  }
+
+  /** @} */
+};  // end of KDTreeVectorOfVectorsAdaptor