diff --git a/.gitignore b/.gitignore index 0a9cf455..da7295b0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ results outputs tmp +docs/_build diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..aa209bb2 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/README.md b/README.md index 02cef08a..ade36fab 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # LIMAP + +**The documentations on brief tutorials and APIs are available [here](http://b1ueber2y.me/projects/LIMAP/docs/)**. +

----------------------------------------------------------------- -**Note**: More README and docs will be available soon. ----------------------------------------------------------------- - LIMAP is a toolbox for mapping and localization with line features. The system was initially described in the highlight paper [3D Line Mapping Revisited](https://arxiv.org/abs/2303.17504) at CVPR 2023 in Vancouver, Canada. Contributors to this project are from the [Computer Vision and Geometry Group](https://cvg.ethz.ch/) at [ETH Zurich](https://ethz.ch/en.html). In this project, we provide interfaces for various geometric operations on 2D/3D lines. We support off-the-shelf SfM software including [VisualSfM](http://ccwu.me/vsfm/index.html), [Bundler](https://bundler.io/), and [COLMAP](https://colmap.github.io/) to initialize the camera poses to build 3D line maps on the database. The line detectors, matchers, and vanishing point estimators are abstracted to ensure flexibility to support recent advances and future development. diff --git a/cfgs/examples/fitting_3Dline.yaml b/cfgs/examples/fitting_3Dline.yaml new file mode 100644 index 00000000..93c386e3 --- /dev/null +++ b/cfgs/examples/fitting_3Dline.yaml @@ -0,0 +1,8 @@ + +# fitting config +fitting: + var2d: 4.0 # This is for DeepLSD. This hyperparameter should depend on the detector + ransac_th: 0.75 + min_percentage_inliers: 0.9 + n_jobs: 4 + diff --git a/cfgs/examples/line2d_detect.yaml b/cfgs/examples/line2d_detect.yaml new file mode 100644 index 00000000..66e669c5 --- /dev/null +++ b/cfgs/examples/line2d_detect.yaml @@ -0,0 +1,22 @@ +# global config +dir_save: "tmp" + +# loading config +load_det: False +dir_load: null +skip_exists: False + +# line detection and description config +line2d: + max_num_2d_segs: 3000 + do_merge_lines: False + visualize: False + save_l3dpp: False + compute_descinfo: False + detector: + method: "deeplsd" # ["lsd", "sold2", "hawpv3", "tp_lsd", "deeplsd"] + skip_exists: False + extractor: + method: "wireframe" # ["sold2", "lbd", "l2d2", "linetr", "superpoint_endpoints", "wireframe"] + skip_exists: False + diff --git a/cfgs/examples/line2d_match.yaml b/cfgs/examples/line2d_match.yaml new file mode 100644 index 00000000..ff1d018b --- /dev/null +++ b/cfgs/examples/line2d_match.yaml @@ -0,0 +1,22 @@ +# global config +dir_save: "tmp" + +# loading config +load_match: False +dir_load: null +skip_exists: False + +# line matching config +line2d: + detector: + method: "deeplsd" # ["lsd", "sold2", "hawpv3", "tp_lsd", "deeplsd"] + extractor: + method: "wireframe" # ["sold2", "lbd", "l2d2", "linetr", "superpoint_endpoints", "wireframe"] + matcher: + method: "gluestick" # ["sold2", "lbd", "l2d2", "linetr", "nn_endpoints", "superglue_endpoints", "gluestick"] + n_jobs: 1 + topk: 10 + skip_exists: False + superglue: + weights: "outdoor" # ["indoor", "outdoor"] for selecting superglue models + diff --git a/cfgs/examples/pointsfm.yaml b/cfgs/examples/pointsfm.yaml new file mode 100644 index 00000000..4d27da01 --- /dev/null +++ b/cfgs/examples/pointsfm.yaml @@ -0,0 +1,22 @@ +# global config +dir_save: "tmp" +n_neighbors: 20 + +# loading config +load_meta: False +dir_load: null +skip_exists: False + +# sfm config +sfm: + colmap_output_path: "colmap_outputs" + reuse: False + min_triangulation_angle: 1.0 + neighbor_type: "dice" # ["overlap", "iou", "dice"] + ranges: + range_robust: [0.05, 0.95] + k_stretch: 1.25 + hloc: + descriptor: "superpoint_aachen" + matcher: "NN-superpoint" + diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/css/fix-rtd-property.css b/docs/_static/css/fix-rtd-property.css new file mode 100644 index 00000000..28a675ea --- /dev/null +++ b/docs/_static/css/fix-rtd-property.css @@ -0,0 +1,3 @@ +dl.py.property { + display: block !important; +} \ No newline at end of file diff --git a/docs/api/limap.base.camera.rst b/docs/api/limap.base.camera.rst new file mode 100644 index 00000000..39db9426 --- /dev/null +++ b/docs/api/limap.base.camera.rst @@ -0,0 +1,28 @@ +Cameras, poses, and images +============================== + +.. autoclass:: limap.base.Camera + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise +.. autoclass:: limap.base.CameraImage + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise +.. autoclass:: limap.base.CameraPose + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise +.. autoclass:: limap.base.CameraView + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise +.. autoclass:: limap.base.ImageCollection + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise diff --git a/docs/api/limap.base.line_dists.rst b/docs/api/limap.base.line_dists.rst new file mode 100644 index 00000000..eeb86bb5 --- /dev/null +++ b/docs/api/limap.base.line_dists.rst @@ -0,0 +1,8 @@ +Line distance +================ + +.. autoclass:: limap.base.LineDistType +.. autofunction:: limap.base.compute_distance_2d +.. autofunction:: limap.base.compute_distance_3d +.. autofunction:: limap.base.compute_pairwise_distance_2d +.. autofunction:: limap.base.compute_pairwise_distance_3d diff --git a/docs/api/limap.base.lines.rst b/docs/api/limap.base.lines.rst new file mode 100644 index 00000000..1cdf1c14 --- /dev/null +++ b/docs/api/limap.base.lines.rst @@ -0,0 +1,26 @@ +Lines +=========== + +.. autoclass:: limap.base.Line2d + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise + +.. autoclass:: limap.base.Line3d + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise + +.. autoclass:: limap.base.InfiniteLine2d + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise + +.. autoclass:: limap.base.InfiniteLine3d + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise \ No newline at end of file diff --git a/docs/api/limap.base.linetrack.rst b/docs/api/limap.base.linetrack.rst new file mode 100644 index 00000000..b0f2eff1 --- /dev/null +++ b/docs/api/limap.base.linetrack.rst @@ -0,0 +1,8 @@ +Line track +================ + +.. autoclass:: limap.base.LineTrack + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise diff --git a/docs/api/limap.base.reader.rst b/docs/api/limap.base.reader.rst new file mode 100644 index 00000000..7e0ad648 --- /dev/null +++ b/docs/api/limap.base.reader.rst @@ -0,0 +1,19 @@ +Readers for depths and point clouds +=============================================== + +limap.base.depth\_reader\_base module +------------------------------------- + +.. automodule:: limap.base.depth_reader_base + :members: + :undoc-members: + :show-inheritance: + +limap.base.p3d\_reader\_base module +----------------------------------- + +.. automodule:: limap.base.p3d_reader_base + :members: + :undoc-members: + + diff --git a/docs/api/limap.base.rst b/docs/api/limap.base.rst new file mode 100644 index 00000000..af8da18d --- /dev/null +++ b/docs/api/limap.base.rst @@ -0,0 +1,17 @@ +limap.base package +================== + +.. toctree:: + :maxdepth: 3 + :caption: Module Contents: + + limap.base.camera + limap.base.lines + limap.base.line_dists + limap.base.linetrack + limap.base.reader + +.. toctree:: + :maxdepth: 3 + :caption: Submodules: + diff --git a/docs/api/limap.estimators.absolute_pose.rst b/docs/api/limap.estimators.absolute_pose.rst new file mode 100644 index 00000000..fffd0d6a --- /dev/null +++ b/docs/api/limap.estimators.absolute_pose.rst @@ -0,0 +1,6 @@ +Estimate absolute pose +======================================= + +.. automodule:: limap.estimators.absolute_pose + :members: + :show-inheritance: diff --git a/docs/api/limap.estimators.ransac.rst b/docs/api/limap.estimators.ransac.rst new file mode 100644 index 00000000..305572c9 --- /dev/null +++ b/docs/api/limap.estimators.ransac.rst @@ -0,0 +1,18 @@ +RANSAC options and statistics +============================== + +.. autoclass:: limap.estimators.RansacStatistics + :members: + :undoc-members: +.. autoclass:: limap.estimators.RansacOptions + :members: + :undoc-members: +.. autoclass:: limap.estimators.LORansacOptions + :members: + :undoc-members: +.. autoclass:: limap.estimators.HybridRansacStatistics + :members: + :undoc-members: +.. autoclass:: limap.estimators.HybridLORansacOptions + :members: + :undoc-members: diff --git a/docs/api/limap.estimators.rst b/docs/api/limap.estimators.rst new file mode 100644 index 00000000..a585f451 --- /dev/null +++ b/docs/api/limap.estimators.rst @@ -0,0 +1,20 @@ +limap.estimators package +======================== + +.. toctree:: + :maxdepth: 3 + :caption: Subpackages: + + limap.estimators.absolute_pose + +.. .. automodule:: limap.estimators +.. :members: +.. :undoc-members: +.. :show-inheritance: + +.. toctree:: + :maxdepth: 2 + :caption: Module Contents: + + limap.estimators.ransac + diff --git a/docs/api/limap.evaluation.mesh_evaluator.rst b/docs/api/limap.evaluation.mesh_evaluator.rst new file mode 100644 index 00000000..1ebb5660 --- /dev/null +++ b/docs/api/limap.evaluation.mesh_evaluator.rst @@ -0,0 +1,9 @@ +Evaluate w.r.t. a Mesh +============================== + +.. autoclass:: limap.evaluation.MeshEvaluator + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise + diff --git a/docs/api/limap.evaluation.point_cloud_evaluator.rst b/docs/api/limap.evaluation.point_cloud_evaluator.rst new file mode 100644 index 00000000..8f747fc5 --- /dev/null +++ b/docs/api/limap.evaluation.point_cloud_evaluator.rst @@ -0,0 +1,9 @@ +Evaluate w.r.t. a Point Cloud +============================== + +.. autoclass:: limap.evaluation.PointCloudEvaluator + :members: + :undoc-members: + :special-members: __init__ + :member-order: groupwise + diff --git a/docs/api/limap.evaluation.rst b/docs/api/limap.evaluation.rst new file mode 100644 index 00000000..424f03c8 --- /dev/null +++ b/docs/api/limap.evaluation.rst @@ -0,0 +1,10 @@ +limap.evaluation package +======================== + +.. toctree:: + :maxdepth: 3 + :caption: Module Contents: + + limap.evaluation.mesh_evaluator + limap.evaluation.point_cloud_evaluator + diff --git a/docs/api/limap.line2d.base_detector.rst b/docs/api/limap.line2d.base_detector.rst new file mode 100644 index 00000000..4df831bd --- /dev/null +++ b/docs/api/limap.line2d.base_detector.rst @@ -0,0 +1,9 @@ +Base line detector/descriptor +---------------------------------- + +.. automodule:: limap.line2d.base_detector + :members: + :show-inheritance: + :member-order: groupwise + + diff --git a/docs/api/limap.line2d.base_matcher.rst b/docs/api/limap.line2d.base_matcher.rst new file mode 100644 index 00000000..eaf49f04 --- /dev/null +++ b/docs/api/limap.line2d.base_matcher.rst @@ -0,0 +1,8 @@ +Base line matcher +--------------------------------- + +.. automodule:: limap.line2d.base_matcher + :members: + :show-inheritance: + + diff --git a/docs/api/limap.line2d.register_detector.rst b/docs/api/limap.line2d.register_detector.rst new file mode 100644 index 00000000..1f5efece --- /dev/null +++ b/docs/api/limap.line2d.register_detector.rst @@ -0,0 +1,8 @@ +Instantiate a line detector/descriptor +------------------------------------------------------------- + +.. automodule:: limap.line2d.register_detector + :members: + :show-inheritance: + + diff --git a/docs/api/limap.line2d.register_matcher.rst b/docs/api/limap.line2d.register_matcher.rst new file mode 100644 index 00000000..da392998 --- /dev/null +++ b/docs/api/limap.line2d.register_matcher.rst @@ -0,0 +1,7 @@ +Instantiate a line matcher +------------------------------------- + +.. automodule:: limap.line2d.register_matcher + :members: + :show-inheritance: + diff --git a/docs/api/limap.line2d.rst b/docs/api/limap.line2d.rst new file mode 100644 index 00000000..209ed5af --- /dev/null +++ b/docs/api/limap.line2d.rst @@ -0,0 +1,12 @@ +limap.line2d package +==================== + +.. toctree:: + :maxdepth: 3 + :caption: Module Contents: + + limap.line2d.base_detector + limap.line2d.register_detector + limap.line2d.base_matcher + limap.line2d.register_matcher + diff --git a/docs/api/limap.runners.functions.rst b/docs/api/limap.runners.functions.rst new file mode 100644 index 00000000..8b76f0c4 --- /dev/null +++ b/docs/api/limap.runners.functions.rst @@ -0,0 +1,8 @@ +Main utilities +------------------------------ + +.. automodule:: limap.runners.functions + :members: + :show-inheritance: + + diff --git a/docs/api/limap.runners.line_fitnmerge.rst b/docs/api/limap.runners.line_fitnmerge.rst new file mode 100644 index 00000000..d9f52ba0 --- /dev/null +++ b/docs/api/limap.runners.line_fitnmerge.rst @@ -0,0 +1,9 @@ +Main function: line reconstruction given depths +------------------------------------------------- + +.. automodule:: limap.runners.line_fitnmerge + :members: + :undoc-members: + :show-inheritance: + + diff --git a/docs/api/limap.runners.line_localization.rst b/docs/api/limap.runners.line_localization.rst new file mode 100644 index 00000000..8671f021 --- /dev/null +++ b/docs/api/limap.runners.line_localization.rst @@ -0,0 +1,8 @@ +Main function: hybrid localization +--------------------------------------- + +.. automodule:: limap.runners.line_localization + :members: + :show-inheritance: + + diff --git a/docs/api/limap.runners.line_triangulation.rst b/docs/api/limap.runners.line_triangulation.rst new file mode 100644 index 00000000..281c13f3 --- /dev/null +++ b/docs/api/limap.runners.line_triangulation.rst @@ -0,0 +1,8 @@ +Main function: line triangulation +---------------------------------------- + +.. automodule:: limap.runners.line_triangulation + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/api/limap.runners.rst b/docs/api/limap.runners.rst new file mode 100644 index 00000000..87e43280 --- /dev/null +++ b/docs/api/limap.runners.rst @@ -0,0 +1,12 @@ +limap.runners package +===================== + +.. toctree:: + :maxdepth: 3 + :caption: Module Contents: + + limap.runners.line_triangulation + limap.runners.line_fitnmerge + limap.runners.line_localization + limap.runners.functions + diff --git a/docs/api/limap.triangulation.rst b/docs/api/limap.triangulation.rst new file mode 100644 index 00000000..7a900208 --- /dev/null +++ b/docs/api/limap.triangulation.rst @@ -0,0 +1,10 @@ +limap.triangulation package +=========================== + +.. toctree:: + :maxdepth: 3 + :caption: Module Contents: + + limap.triangulation.triangulation + + diff --git a/docs/api/limap.triangulation.triangulation.rst b/docs/api/limap.triangulation.triangulation.rst new file mode 100644 index 00000000..5bf45c4b --- /dev/null +++ b/docs/api/limap.triangulation.triangulation.rst @@ -0,0 +1,7 @@ +Triangulation +---------------------------------------- + +.. automodule:: limap.triangulation.triangulation + :members: + :show-inheritance: + diff --git a/docs/api/limap.undistortion.rst b/docs/api/limap.undistortion.rst new file mode 100644 index 00000000..f31345a5 --- /dev/null +++ b/docs/api/limap.undistortion.rst @@ -0,0 +1,9 @@ +limap.undistortion package +========================== + +.. toctree:: + :maxdepth: 3 + :caption: Module Contents: + + limap.undistortion.undistortion + diff --git a/docs/api/limap.undistortion.undistortion.rst b/docs/api/limap.undistortion.undistortion.rst new file mode 100644 index 00000000..b4696926 --- /dev/null +++ b/docs/api/limap.undistortion.undistortion.rst @@ -0,0 +1,7 @@ +Undistortion +----------------------------------- + +.. automodule:: limap.undistortion.undistort + :members: + :show-inheritance: + diff --git a/docs/api/limap.visualize.rst b/docs/api/limap.visualize.rst new file mode 100644 index 00000000..bb7d8a2a --- /dev/null +++ b/docs/api/limap.visualize.rst @@ -0,0 +1,10 @@ +limap.visualize package +======================= + +.. toctree:: + :maxdepth: 4 + :caption: Module Contents: + + limap.visualize.vis_lines + + diff --git a/docs/api/limap.visualize.vis_lines.rst b/docs/api/limap.visualize.vis_lines.rst new file mode 100644 index 00000000..d937cd21 --- /dev/null +++ b/docs/api/limap.visualize.vis_lines.rst @@ -0,0 +1,7 @@ +Utilities +----------------------------------- + +.. automodule:: limap.visualize.vis_lines + :members: + :show-inheritance: + diff --git a/docs/api/limap.vplib.JLinkage.rst b/docs/api/limap.vplib.JLinkage.rst new file mode 100644 index 00000000..2ce6b5f8 --- /dev/null +++ b/docs/api/limap.vplib.JLinkage.rst @@ -0,0 +1,11 @@ +limap.vplib.JLinkage package +============================ + +limap.vplib.JLinkage.JLinkage module +------------------------------------ + +.. automodule:: limap.vplib.JLinkage.JLinkage + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/api/limap.vplib.base_vp_detector.rst b/docs/api/limap.vplib.base_vp_detector.rst new file mode 100644 index 00000000..5848484d --- /dev/null +++ b/docs/api/limap.vplib.base_vp_detector.rst @@ -0,0 +1,9 @@ +Base VP detector +------------------------------------- + +.. automodule:: limap.vplib.base_vp_detector + :members: + :undoc-members: + :show-inheritance: + + diff --git a/docs/api/limap.vplib.progressivex.rst b/docs/api/limap.vplib.progressivex.rst new file mode 100644 index 00000000..516fce00 --- /dev/null +++ b/docs/api/limap.vplib.progressivex.rst @@ -0,0 +1,11 @@ +limap.vplib.progressivex package +================================ + +limap.vplib.progressivex.progressivex module +-------------------------------------------- + +.. automodule:: limap.vplib.progressivex.progressivex + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/api/limap.vplib.register_vp_detector.rst b/docs/api/limap.vplib.register_vp_detector.rst new file mode 100644 index 00000000..0e1dd272 --- /dev/null +++ b/docs/api/limap.vplib.register_vp_detector.rst @@ -0,0 +1,8 @@ +Instantiate a VP detector +----------------------------------------- + +.. automodule:: limap.vplib.register_vp_detector + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/api/limap.vplib.rst b/docs/api/limap.vplib.rst new file mode 100644 index 00000000..9b871e27 --- /dev/null +++ b/docs/api/limap.vplib.rst @@ -0,0 +1,11 @@ +limap.vplib package +=================== + +.. toctree:: + :maxdepth: 4 + :caption: Module Contents: + + limap.vplib.base_vp_detector + limap.vplib.register_vp_detector + limap.vplib.subpackages + diff --git a/docs/api/limap.vplib.subpackages.rst b/docs/api/limap.vplib.subpackages.rst new file mode 100644 index 00000000..c18640d6 --- /dev/null +++ b/docs/api/limap.vplib.subpackages.rst @@ -0,0 +1,10 @@ +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + limap.vplib.JLinkage + limap.vplib.progressivex + + diff --git a/docs/build.sh b/docs/build.sh new file mode 100755 index 00000000..053d468b --- /dev/null +++ b/docs/build.sh @@ -0,0 +1,6 @@ +python -m pip install -Ive .. +touch *.rst +touch tutorials/* +touch api/* +# touch api/*.rst +make html diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..d4150083 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,50 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# import limap +# version = limap.__version__ +version = '1.0.0' + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'LIMAP' +copyright = '2023, CVG @ ETH Zurich' +author = 'LIMAP Contributors' +release = version + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx_toolbox.more_autodoc.autonamedtuple'] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = ["_themes"] +html_css_files = [ + 'css/fix-rtd-property.css' # workaround readthedocs/sphinx_rtd_theme#1301 +] + diff --git a/docs/developers.rst b/docs/developers.rst new file mode 100644 index 00000000..8ee13e3d --- /dev/null +++ b/docs/developers.rst @@ -0,0 +1,8 @@ +Main developers +================== + +* Shaohui Liu (`B1ueber2y `_) +* Yifan Yu (`MarkYu98 `_) +* Rémi Pautrat (`rpautrat `_) +* Viktor Larsson (`vlarsson `_) + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..3f84c38b --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,45 @@ +.. LIMAP documentation master file, created by + sphinx-quickstart on Fri Mar 24 12:35:33 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to LIMAP's documentation! +================================= + +.. image:: media/teaser.png + +LIMAP is a toolbox for mapping and localization with line features. The system was initially described in the highlight paper `3D Line Mapping Revisited `_ at CVPR 2023 in Vancouver, Canada. Contributors to this project are from the `Computer Vision and Geometry Group `_ at `ETH Zurich `_. + +In this project, we provide interfaces for various geometric operations on 2D/3D lines. We support off-the-shelf SfM software including `VisualSfM `_, `Bundler `_, and `COLMAP `_ to initialize the camera poses to build 3D line maps on the database. The line detectors, matchers, and vanishing point estimators are abstracted to ensure flexibility to support recent advances and future development. + +.. toctree:: + :maxdepth: 1 + :caption: Tutorials: + + tutorials/installation + tutorials/quickstart + tutorials/line2d + tutorials/triangulation + tutorials/localization + tutorials/visualization + +.. toctree:: + :maxdepth: 2 + :caption: API references: + + api/limap.base + api/limap.line2d + api/limap.estimators + api/limap.evaluation + api/limap.runners + api/limap.triangulation + api/limap.undistortion + api/limap.visualize + api/limap.vplib + +.. toctree:: + :maxdepth: 1 + :caption: Community: + + developers + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/media/teaser.png b/docs/media/teaser.png new file mode 100644 index 00000000..884a2a02 Binary files /dev/null and b/docs/media/teaser.png differ diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..83d917a8 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +sphinx +sphinx_rtd_theme +sphinx-toolbox + diff --git a/docs/tutorials/installation.rst b/docs/tutorials/installation.rst new file mode 100644 index 00000000..3abdf663 --- /dev/null +++ b/docs/tutorials/installation.rst @@ -0,0 +1,75 @@ +Installation +================================= + +------------------ +Dependencies +------------------ + +* CMake >= 3.17 +* COLMAP + + Follow `official document `_ to install COLMAP. + +* PoseLib + + .. code-block:: bash + + git clone --recursive https://github.com/vlarsson/PoseLib.git + cd PoseLib + mkdir build && cd build + cmake .. + sudo make install -j8 + +* HDF5 + + .. code-block:: bash + + sudo apt-get install libhdf5-dev + +* OpenCV (only for installing `pytlbd `_, it's fine to remove it from `requirements.txt` and use LIMAP without OpenCV) + + .. code-block:: bash + + sudo apt-get install libopencv-dev libopencv-contrib-dev libarpack++2-dev libarpack2-dev libsuperlu-dev + +* Python 3.9 + required packages + + Install PyTorch >= 1.12.0 (Note: LIMAP has not been tested with PyTorch 2) + + * CPU version + + .. code-block:: bash + + pip install torch==1.12.0 torchvision==0.13.0 + + + * Or refer to https://pytorch.org/get-started/previous-versions/ to install PyTorch compatible with your CUDA version + + Install other Python packages + + .. code-block:: bash + + git submodule update --init --recursive + pip install -r requirements.txt + +------------------ +Install LIMAP +------------------ + +.. code-block:: bash + + pip install -Ive . + +Alternatively: + +.. code-block:: bash + + mkdir build && cd build + cmake -DPYTHON_EXECUTABLE=`which python` .. + make -j8 + +To test if LIMAP is successfully installed, try ``import limap`` and it should reports no error. + +.. code-block:: bash + + python -c "import limap" diff --git a/docs/tutorials/line2d.rst b/docs/tutorials/line2d.rst new file mode 100644 index 00000000..99d107a2 --- /dev/null +++ b/docs/tutorials/line2d.rst @@ -0,0 +1,55 @@ +Line detection, description and matching +============================================ + +We provide interfaces for running line detection, description and matching with supported modules in LIMAP. + +----------------------------------------------------- +Minimal example on line detection and description +----------------------------------------------------- +To use the interface you need to construct a :class:`limap.base.CameraView` instance for each image. Since the intrinsic and extrinsic parameters are not needed, you can leave it uninitialized. Here shows an minimal example on running line detection and description with `DeepLSD `_ and `SOLD2 `_ on an image `example.png`: + +.. code-block:: python + + import limap.util.config + import limap.base + import limap.line2d + cfg_detector = limap.util.config.load_config("cfgs/examples/line2d_detector.yaml") # example config file + cfg_detector["line2d"]["detector"]["method"] = "deeplsd" + cfg_detector["line2d"]["extractor"]["method"] = "sold2" + view = limap.base.CameraView(limap.base.Camera(0), "example.png") # initiate an limap.base.CameraView instance for detection. You can specify the height and width to resize into in the limap.base.Camera instance at initialization. + detector = limap.line2d.get_detector(cfg_detector) # get a line detector + segs = detector.detect(view) # detection + desc = detector.extract(view, segs) # description + +----------------------------------------------------- +Minimal example on line matching +----------------------------------------------------- +And here shows an minimal example on running line matcher with `SOLD2 `_. Note that the type of matcher should align with the type of extractors in terms of compatibility. + +.. code-block:: python + + global desc1, desc2 # read in some extracted descriptors + import limap.util.config + import limap.base + import limap.line2d + cfg_detector = limap.util.config.load_config("cfgs/examples/line2d_detector.yaml") # example config file + cfg_detector["line2d"]["extractor"]["method"] = "sold2" + extractor = limap.line2d.get_detector(cfg_detector) # get a line extractor + cfg_matcher = limap.util.config.load_config("cfgs/examples/line2d_match.yaml") # example config file + cfg_detector["line2d"]["matcher"]["method"] = "sold2" + matcher = limap.line2d.get_matcher(cfg_detector, extractor) # initiate a line matcher + matches = matcher.match_pair(desc1, desc2) # matching + +---------------------------------------------------- +Multiple images +---------------------------------------------------- +To run line detection, description and matching on multiple images, one can resort to the following API: + +* :py:meth:`limap.runners.functions.compute_2d_segs` +* :py:meth:`limap.runners.functions.compute_exhaustive_matches` +* :py:meth:`limap.runners.functions.compute_matches` + +The outputs detections, descriptions and matches will be saved into the corresponding output folders. + + + diff --git a/docs/tutorials/localization.rst b/docs/tutorials/localization.rst new file mode 100644 index 00000000..6270eb5f --- /dev/null +++ b/docs/tutorials/localization.rst @@ -0,0 +1,74 @@ +Localization with points & lines +================================= + +Currently, runner scripts are provided to run visual localization integrating line along with point features on the following Datasets: + +* `7Scenes Dataset `_ +* `Cambridge Landmarks Dataset `_ +* `InLoc Dataset `_ + +Please follow hloc's guide for downloading and preparing Cambridge and 7Scenes dataset: + +* `7Scenes `_ +* `Cambridge `_ + +Use ``runners//localization.py`` to run localization experiments on these supported datasets, use ``--help`` option and take a look at ``cfgs/localization`` folder for all the possible options and configurations. + +Alternatively, take a look at the :py:meth:`limap.estimators.absolute_pose.pl_estimate_absolute_pose` API or the :py:meth:`limap.runners.line_localization.line_localization` runner to run localization with points and lines, using 2D-3D point and line correspondences directly. + +------------------------------------ +Example on 7Scenes +------------------------------------ + +Here we provide a tutorial to reproduce the visual localization experiment in paper `3D Line Mapping Revisited `_ (in CVPR 2023), specifically on the *Stairs* scene of the `7Scenes `_ dataset. + +This scene best demonstrates the improvement that could be achieved by integrating lines along with point features for visual localization, since traditionally point-based localization struggles in performance. + +Follow `hloc `_, download the images from the project page: + +.. code-block:: bash + + export dataset=datasets/7scenes + for scene in stairs; \ + do wget http://download.microsoft.com/download/2/8/5/28564B23-0828-408F-8631-23B1EFF1DAC8/$scene.zip -P $dataset \ + && unzip $dataset/$scene.zip -d $dataset && unzip $dataset/$scene/'*.zip' -d $dataset/$scene; done + +Download the SIFT SfM models and DenseVLAD image pairs, courtesy of Torsten Sattler: + +.. code-block:: bash + + function download { + wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate "https://docs.google.com/uc?export=download&id=$1" -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=$1" -O $2 && rm -rf /tmp/cookies.txt + unzip $2 -d $dataset && rm $2; + } + download 1cu6KUR7WHO7G4EO49Qi3HEKU6n_yYDjb $dataset/7scenes_sfm_triangulated.zip + download 1IbS2vLmxr1N0f3CEnd_wsYlgclwTyvB1 $dataset/7scenes_densevlad_retrieval_top_10.zip + +Download the rendered depth maps, courtesy of Eric Brachmann for `DSAC* `_: + +.. code-block:: bash + + wget https://heidata.uni-heidelberg.de/api/access/datafile/4037 -O $dataset/7scenes_rendered_depth.tar.gz + mkdir $dataset/depth/ + tar xzf $dataset/7scenes_rendered_depth.tar.gz -C $dataset/depth/ && rm $dataset/7scenes_rendered_depth.tar.gz + +The download could take some time as the compressed data files contain all 7Scenes. You could delete the other scenes since for this example we are only using the Stairs scene. + +Now, to run the localization pipeline with points and lines. As shown above, the configs are passed in as command line arguments. + +.. code-block:: bash + + python runners/7scenes/localization.py --dataset $dataset -s stairs --skip_exists \ + --localization.optimize.loss_func TrivialLoss \ + --localization.optimize.normalize_weight + +It is also possible to use the rendered depth with the ``--use_dense_depth`` flag, in which case the 3D line map will be built using LIMAP's Fit&Merge (enable merging by adding ``--merging.do_merging``) utilities instead of triangulation. + +.. code-block:: bash + + python runners/7scenes/localization.py --dataset $dataset -s stairs --skip_exists \ + --use_dense_depth \ + --localization.optimize.loss_func TrivialLoss \ + --localization.optimize.normalize_weight + +The runner scripts will also run `hloc `_ for extracting and matching the feature points and for comparing the results. The evaluation result will be printed in terminal after localization is finished. You could also evaluate different result ``.txt`` files using the ``--eval`` flag. diff --git a/docs/tutorials/quickstart.rst b/docs/tutorials/quickstart.rst new file mode 100644 index 00000000..112eb079 --- /dev/null +++ b/docs/tutorials/quickstart.rst @@ -0,0 +1,61 @@ +Quickstart +================================= + +Some examples are prepared for users to quickly try out LIMAP for mapping and localization with lines. + +------------------ +Line Mapping +------------------ + +For this example we are using the first scene ``ai_001_001`` from `Hypersim `_ dataset. Download the test scene **(100 images)** with the following command: + +.. code-block:: bash + + bash scripts/quickstart.sh + +To run line mapping using **Line Mapping** (RGB-only) on Hypersim: + +.. code-block:: bash + + python runners/hypersim/triangulation.py --output_dir outputs/quickstart_triangulation + +To run line mapping using **Fitnmerge** (line mapping with available depth maps) on Hypersim: + +.. code-block:: bash + + python runners/hypersim/fitnmerge.py --output_dir outputs/quickstart_fitnmerge + +To run **Visualization** of the 3D line maps after the reconstruction: + +.. code-block:: bash + + python visualize_3d_lines.py --input_dir outputs/quickstart_triangulation/finaltracks \ + # add the camera frustums with "--imagecols outputs/quickstart_triangulation/imagecols.npy" + +[**Tips**] Options are stored in the config folder: ``cfgs``. You can easily change the options with the Python argument parser. Here's an example: + +.. code-block:: bash + + python runners/hypersim/triangulation.py --line2d.detector.method lsd \ + --line2d.visualize --triangulation.IoU_threshold 0.2 \ + --skip_exists --n_visible_views 5 + +In particular, ``skip_exists`` is a very useful option to avoid running point-based SfM and line detection/description repeatedly in each pass. + +Also, the combination ``LSD detector + Endpoints NN matcher`` can be enabled with ``--default_config_file cfgs/triangulation/default_fast.yaml`` for high efficiency (while with non-negligible performance degradation). + +------------------------------------------------- +Hybrid Localization with Points and Lines +------------------------------------------------- + +We provide two query examples for localization from the Stairs scene in the `7Scenes `_ Dataset, where traditional point-based methods normally struggle due to the repeated steps and lack of texture. The examples are provided in ``.npy`` files: ``runners/tests/localization/localization_test_data_stairs_[1|2].npy``, which contains the necessary 2D-3D point and line correspondences along with the necessary configurations. + +To run the examples, for instance the first one: + +.. code-block:: bash + + python runners/tests/localization.py --data runners/tests/localization_test_data_stairs_1.npy + +The script will print the pose error estimated using point-only (hloc), and the pose error estimated by our hybrid point-line localization framework. In addition, two images will be created in the output folder (default to ``outputs/test/localization``) showing the inliers point and line correspondences in hybrid localization projected using the two estimated camera pose (by point-only and point+line) onto the query image with 2D point and line detections marked. + +An improved accuracy of the hybrid point-line method is expected to be observed. diff --git a/docs/tutorials/triangulation.rst b/docs/tutorials/triangulation.rst new file mode 100644 index 00000000..f407af20 --- /dev/null +++ b/docs/tutorials/triangulation.rst @@ -0,0 +1,26 @@ +Line mapping +================================= + +As one of the main features, LIMAP supports line reconstruction on a set of posed images, optionally with assistance of the point-based SfM model or the depth map. We currently support to use the poses from `COLMAP `_, `Bundler `_ and `VisualSfM `_. One can also use customized poses and intrinsics with the main interface :py:meth:`limap.runners.line_triangulation` API by constructing a :class:`limap.base.ImageCollection` instance as the input. + +----------------------------------------- +Line mapping on a set of images +----------------------------------------- + +Specifically, to run the line mapping on a set of images, first pose the images with `COLMAP `_ following the guide `here `_. Then, use the `COLMAP interface `_ in LIMAP to build 3D line maps by: + +.. code-block:: bash + + python runners/colmap_triangulation.py -c ${CONFIG_FILE} -a ${COLMAP_FOLDER} --output_path ${OUTPUT_PATH} + +And the line maps will be stored in the specified output folder. To use point SfM to improve robustness, add additional option ``--triangulation.use_pointsfm.enable --triangulation.use_pointsfm.colmap_folder ${COLMAP_FOLDER}`` in the end of the command. + +The interface for Bundler, VisualSfM and other datasets are all stored in the ``runners`` folder, building on top of the main interface :py:meth:`limap.runners.line_triangulation` API. + +----------------------------------------- +Using auxiliary depth maps +----------------------------------------- + +To use auxiliary depth information, we can run line reconstruction with the :py:meth:`limap.runners.line_fitnmerge` API. One needs to write a customized depth loader inheriting :class:`limap.base.depth_reader_base.BaseDepthReader`. An example on Hypersim dataset is provided `here `_. Each depth loader consists of the file name of the depth image and its width, height and other information, along with the method for loading. + + diff --git a/docs/tutorials/visualization.rst b/docs/tutorials/visualization.rst new file mode 100644 index 00000000..95c70043 --- /dev/null +++ b/docs/tutorials/visualization.rst @@ -0,0 +1,12 @@ +Visualization +================================= + +We provide the visualization interface `here `_ to visualize the reconstructed line maps. Optionally, the camera frustums can also be visualized with an additional :class:`limap.base.ImageCollection` instance as the input. An example visualization command could be: + +.. code-block:: bash + + python visualize_3d_lines.py --input_dir outputs/quickstart_triangulation/finaltracks \ + # add the camera frustums with "--imagecols outputs/quickstart_triangulation/imagecols.npy" + +We use `Open3D `_ as the backend for the visualization. + diff --git a/limap/base/bindings.cc b/limap/base/bindings.cc index da1aee63..496b6f61 100644 --- a/limap/base/bindings.cc +++ b/limap/base/bindings.cc @@ -140,11 +140,19 @@ void bind_transforms(py::module& m) { } void bind_linebase(py::module& m) { - py::class_(m, "Line2d") - .def(py::init<>()) - .def(py::init()) - .def(py::init(), py::arg("start"), py::arg("end")) - .def(py::init(), py::arg("start"), py::arg("end"), py::kw_only(), py::arg("score")) + py::class_(m, "Line2d", "A finite 2D line (segment).") + .def(py::init<>(), R"( + Default constructor + )") + .def(py::init(), R"( + Constructor from :class:`np.array` of shape (2, 2) stacking the two 2D endpoints + )", py::arg("seg2d")) + .def(py::init(), R"( + Constructor from `start` and `end` endpoints, each a :class:`np.array` of shape (2,) + )", py::arg("start"), py::arg("end")) + .def(py::init(), R"( + Constructor from two endpoints and optionally the score + )", py::arg("start"), py::arg("end"), py::arg("score")) .def(py::pickle( [](const Line2d& input) { // dump return input.as_array(); @@ -153,22 +161,57 @@ void bind_linebase(py::module& m) { return Line2d(arr); } )) - .def_readonly("start", &Line2d::start) - .def_readonly("end", &Line2d::end) - .def_readonly("score", &Line2d::score) - .def("length", &Line2d::length) - .def("coords", &Line2d::coords) - .def("as_array", &Line2d::as_array) - .def("midpoint", &Line2d::midpoint) - .def("direction", &Line2d::direction) - .def("point_projection", &Line2d::point_projection) - .def("point_distance", &Line2d::point_distance); - - py::class_(m, "Line3d") - .def(py::init<>()) - .def(py::init()) - .def(py::init(), py::arg("start"), py::arg("end")) - .def(py::init(), py::arg("start"), py::arg("end"), py::kw_only(), py::arg("score"), py::arg("depth_start"), py::arg("depth_end"), py::arg("uncertainty")) + .def_readwrite("start", &Line2d::start, ":class:`np.array` of shape (2,)") + .def_readwrite("end", &Line2d::end, ":class:`np.array` of shape (2,)") + .def_readwrite("score", &Line2d::score, "float") + .def("length", &Line2d::length, R"( + Returns: + float: The length of the 2D line segment + )") + .def("coords", &Line2d::coords, R"( + Returns: + :class:`np.array` of shape (3,): Normalized homogeneous coordinate of the 2D line + )") + .def("as_array", &Line2d::as_array, R"( + Returns: + :class:`np.array` of shape (2, 2): Array stacking `start` and `end` endpoints + )") + .def("midpoint", &Line2d::midpoint, R"( + Returns: + :class:`np.array` of shape (2,): Coordinate of the midpoint of the 2D line segment + )") + .def("direction", &Line2d::direction, R"( + Returns: + :class:`np.array` of shape (2,): Direction vector of the 2D line from `start` to `end` + )") + .def("point_projection", &Line2d::point_projection, R"( + Args: + p (:class:`np.array`): Coordinate of a 2D point, of shape (2,) + + Returns: + :class:`np.array` of shape (2,): Coordinate of the projection of the point `p` on the 2D line segment + )", py::arg("p")) + .def("point_distance", &Line2d::point_distance, R"( + Args: + p (:class:`np.array`): Coordinate of a 2D point, of shape (2,) + + Returns: + float: Distance from the point `p` to the 2D line segment + )", py::arg("p")); + + py::class_(m, "Line3d", "A finite 3D line (segment).") + .def(py::init<>(), R"( + Default constructor + )") + .def(py::init(), R"( + Constructor from :class:`np.array` of shape (2, 3) stacking the two 3D endpoints + )", py::arg("seg3d")) + .def(py::init(), R"( + Constructor from `start` and `end` endpoints, each a :class:`np.array` of shape (3,) + )", py::arg("start"), py::arg("end")) + .def(py::init(), R"( + Constructor from two endpoints, and optionally: the score, the start and/or end depth of the 3D segment, and the uncertainty value + )", py::arg("start"), py::arg("end"), py::arg("score"), py::arg("depth_start"), py::arg("depth_end"), py::arg("uncertainty")) .def(py::pickle( [](const Line3d& input) { // dump return input.as_array(); @@ -177,63 +220,206 @@ void bind_linebase(py::module& m) { return Line3d(arr); } )) - .def_readonly("start", &Line3d::start) - .def_readonly("end", &Line3d::end) - .def_readonly("score", &Line3d::score) - .def_readonly("depths", &Line3d::depths) - .def_readonly("uncertainty", &Line3d::uncertainty) - .def("set_uncertainty", &Line3d::set_uncertainty) - .def("length", &Line3d::length) - .def("as_array", &Line3d::as_array) - .def("projection", &Line3d::projection) - .def("sensitivity", &Line3d::sensitivity) - .def("computeUncertainty", &Line3d::computeUncertainty) - .def("midpoint", &Line3d::midpoint) - .def("direction", &Line3d::direction) - .def("point_projection", &Line3d::point_projection) - .def("point_distance", &Line3d::point_distance); + .def_readonly("start", &Line3d::start, ":class:`np.array` of shape (3,)") + .def_readonly("end", &Line3d::end, ":class:`np.array` of shape (3,)") + .def_readonly("score", &Line3d::score, "float") + .def_readonly("depths", &Line3d::depths, "float") + .def_readonly("uncertainty", &Line3d::uncertainty, "float") + .def("set_uncertainty", &Line3d::set_uncertainty, "Setter for the uncertainty value") + .def("length", &Line3d::length, R"( + Returns: + float: The length of the 3D line segment + )") + .def("as_array", &Line3d::as_array, R"( + Returns: + :class:`np.array` of shape (2, 3): Array stacking `start` and `end` endpoints + )") + .def("projection", &Line3d::projection, R"( + Args: + view (CameraView): :class:`~limap.base.CameraView` instance used to project the 3D line segment to 2D + + Returns: + :class:`~limap.base.Line2d`: The 2D line segment projected from the 3D line segment + )", py::arg("view")) + .def("sensitivity", &Line3d::sensitivity, R"( + Args: + view (CameraView): :class:`~limap.base.CameraView` instance + + Returns: + float: Sensitivity with respect to `view` + )", py::arg("view")) + .def("computeUncertainty", &Line3d::computeUncertainty, R"( + Args: + view (CameraView): :class:`~limap.base.CameraView` instance + var2d (float): Variance in 2D + + Returns: + float: The computed uncertainty value with respect to `view` and `var2d` + )") + .def("midpoint", &Line3d::midpoint, R"( + Returns: + :class:`np.array` of shape (3,): Coordinate of the midpoint of the 3D line segment + )") + .def("direction", &Line3d::direction, R"( + Returns: + :class:`np.array` of shape (3,): Direction vector of the 3D line from `start` to `end` + )") + .def("point_projection", &Line3d::point_projection, R"( + Args: + p (:class:`np.array`): Coordinate of a 3D point, of shape (3,) + + Returns: + :class:`np.array` of shape (3,): Coordinate of the projection of the point `p` on the 3D line segment + )", py::arg("p")) + .def("point_distance", &Line3d::point_distance, R"( + Args: + p (:class:`np.array`): Coordinate of a 3D point, of shape (3,) + + Returns: + float: Distance from the point `p` to the 3D line segment + )", py::arg("p")); m.def("_GetLine2dVectorFromArray", &GetLine2dVectorFromArray); m.def("_GetLine3dVectorFromArray", &GetLine3dVectorFromArray); - py::class_(m, "InfiniteLine2d") - .def(py::init<>()) - .def(py::init()) // coords - .def(py::init()) // point + direction - .def(py::init()) - .def_readonly("coords", &InfiniteLine2d::coords) - .def("point", &InfiniteLine2d::point) - .def("direction", &InfiniteLine2d::direction) - .def("point_projection", &InfiniteLine2d::point_projection) - .def("point_distance", &InfiniteLine2d::point_distance); - - py::class_(m, "InfiniteLine3d") - .def(py::init<>()) - .def(py::init()) - .def(py::init()) - .def_readonly("d", &InfiniteLine3d::d) - .def_readonly("m", &InfiniteLine3d::m) - .def("point", &InfiniteLine3d::point) - .def("direction", &InfiniteLine3d::direction) - .def("matrix", &InfiniteLine3d::matrix) - .def("point_projection", &InfiniteLine3d::point_projection) - .def("point_distance", &InfiniteLine3d::point_distance) - .def("projection", &InfiniteLine3d::projection) - .def("unprojection", &InfiniteLine3d::unprojection) - .def("project_from_infinite_line", &InfiniteLine3d::project_from_infinite_line) - .def("project_to_infinite_line", &InfiniteLine3d::project_to_infinite_line); + py::class_(m, "InfiniteLine2d", "An infinite 2D line.") + .def(py::init<>(), R"( + Default constructor + )") + .def(py::init(), R"( + Constructor from homogeneous coordinate (:class:`np.array` of shape (3,)) + )", py::arg("coords")) // coords + .def(py::init(), R"( + Constructor from a start point and a direction, both :class:`np.array` of shape (2,) + )", py::arg("p"), py::arg("direc")) // point + direction + .def(py::init(), R"( + Constructor from a :class:`~limap.base.Line2d` + )", py::arg("line")) + .def_readonly("coords", &InfiniteLine2d::coords, "Homogeneous coordinate, :class:`np.array` of shape (3,)") + .def("point", &InfiniteLine2d::point, R"( + Returns: + :class:`np.array` of shape (2,): A point on the line (in fact the projection of (0, 0)) + )") + .def("direction", &InfiniteLine2d::direction, R"( + Returns: + :class:`np.array` of shape (2,): The direction of the line + )") + .def("point_projection", &InfiniteLine2d::point_projection, R"( + Args: + p (:class:`np.array`): Coordinate of a 2D point, of shape (2,) + + Returns: + :class:`np.array` of shape (2,): Coordinate of the projection of the point `p` on the 2D infinite line + )", py::arg("p")) + .def("point_distance", &InfiniteLine2d::point_distance, R"( + Args: + p (:class:`np.array`): Coordinate of a 2D point, of shape (2,) + + Returns: + float: Distance from the point `p` to the 2D infinite line + )", py::arg("p")); + + py::class_(m, "InfiniteLine3d", "An infinite 3D line.") + .def(py::init<>(), R"( + Default constructor + )") + .def(py::init(), R"( + | Constructor using normal coordinate (a start point and direction) or Plücker coordinate. + | if `use_normal` is True -> (`a`, `b`) is (`p`, `direc`): normal coordinate with a point and a direction; + | if `use_normal` is False -> (`a`, `b`) is (`direc`, `m`): Plücker coordinate. + )", py::arg("a"), py::arg("b"), py::arg("use_normal")) + .def(py::init(), R"( + Constructor from a :class:`~limap.base.Line3d` + )", py::arg("line")) + .def_readonly("d", &InfiniteLine3d::d, "Direction, :class:`np.array` of shape (3,)") + .def_readonly("m", &InfiniteLine3d::m, "Moment, :class:`np.array` of shape (3,)") + .def("point", &InfiniteLine3d::point, R"( + Returns: + :class:`np.array` of shape (3,): A point on the line (in fact the projection of (0, 0, 0)) + )") + .def("direction", &InfiniteLine3d::direction, R"( + Returns: + :class:`np.array` of shape (3,): The direction of the line (`d`) + )") + .def("matrix", &InfiniteLine3d::matrix, R"( + Returns: + :class:`np.array` of shape (4, 4): The `Plücker matrix `_ + )") + .def("point_projection", &InfiniteLine3d::point_projection, R"( + Args: + p (:class:`np.array`): Coordinate of a 3D point, of shape (3,) + + Returns: + :class:`np.array` of shape (3,): Coordinate of the projection of the point `p` on the 3D infinite line + )", py::arg("p")) + .def("point_distance", &InfiniteLine3d::point_distance, R"( + Args: + p (:class:`np.array`): Coordinate of a 3D point, of shape (3,) + + Returns: + float: Distance from the point `p` to the 3D infinite line + )", py::arg("p")) + .def("projection", &InfiniteLine3d::projection, R"( + Projection from Plücker coordinate to 2D homogeneous line coordinate. + + Args: + view (CameraView): :class:`~limap.base.CameraView` instance used to project the 3D infinite line to 2D + + Returns: + :class:`~limap.base.InfiniteLine2D`: The 2D infinite line projected from the 3D infinite line + )", py::arg("view")) + .def("unprojection", &InfiniteLine3d::unprojection, R"( + Unproject a 2D point by finding the closest point on the 3D line from the camera ray of the 2D point. + + Args: + p2d (:class:`np.array`): The 2D point to unproject, of shape (2,) + view (CameraView): :class:`~limap.base.CameraView` instance to unproject the point + + Returns: + :class:`np.array` of shape (3,): The closest point on the 3D line from the unprojected camera ray of the 2D point + )", py::arg("p2d"), py::arg("view")) + .def("project_from_infinite_line", &InfiniteLine3d::project_from_infinite_line, R"( + Projection from another infinite 3D line by finding the closest point on this 3D line to the other line. + + Args: + line (:class:`~limap.base.InfiniteLine3d`): The other infinite line to project from + + Returns: + :class:`np.array` of shape (3,): The projected point on this 3D line from the other line + )", py::arg("line")) + .def("project_to_infinite_line", &InfiniteLine3d::project_to_infinite_line, R"( + Inverse of the previous function: finding the closest point on the other line to this line. + + Args: + line (:class:`~limap.base.InfiniteLine3d`): The other infinite line to project to + + Returns: + :class:`np.array` of shape (3,): The projected point on the other line from this line + )", py::arg("line")); m.def("_GetLineSegmentFromInfiniteLine3d", py::overload_cast&, const std::vector&, const int>(&GetLineSegmentFromInfiniteLine3d), py::arg("inf_line"), py::arg("camviews"), py::arg("line2ds"), py::arg("num_outliers") = 2); m.def("_GetLineSegmentFromInfiniteLine3d", py::overload_cast&, const int>(&GetLineSegmentFromInfiniteLine3d), py::arg("inf_line"), py::arg("line3ds"), py::arg("num_outliers") = 2); } void bind_linetrack(py::module& m) { - py::class_(m, "LineTrack") - .def(py::init<>()) - .def(py::init()) - .def(py::init&, const std::vector&, const std::vector&>()) - .def(py::init()) - .def("as_dict", &LineTrack::as_dict) + py::class_(m, "LineTrack", "Associated line track across multi-view.") + .def(py::init<>(), R"( + Default constructor + )") + .def(py::init(), R"( + Copy constructor + )", py::arg("track")) + .def(py::init&, const std::vector&, const std::vector&>(), R"( + Constructor from a :class:`~limap.base.Line3d`, a list of associated image IDs, a list of supporting line IDs within each image, + and a list of associated :class:`~limap.base.Line2d` + )", py::arg("line"), py::arg("image_id_list"), py::arg("line_id_list"), py::arg("line2d_list")) + .def(py::init(), R"( + Constructor from a Python dict + )", py::arg("dict")) + .def("as_dict", &LineTrack::as_dict, R"( + Returns: + dict: Python dict representation of this :class:`~limap.base.LineTrack` + )") .def(py::pickle( [](const LineTrack& input) { // dump return input.as_dict(); @@ -242,25 +428,57 @@ void bind_linetrack(py::module& m) { return LineTrack(dict); } )) - .def_readwrite("line", &LineTrack::line) - .def_readonly("node_id_list", &LineTrack::node_id_list) - .def_readonly("image_id_list", &LineTrack::image_id_list) - .def_readonly("line_id_list", &LineTrack::line_id_list) - .def_readonly("line3d_list", &LineTrack::line3d_list) - .def_readonly("line2d_list", &LineTrack::line2d_list) - .def_readonly("score_list", &LineTrack::score_list) - .def_readonly("active", &LineTrack::active) - .def("count_lines", &LineTrack::count_lines) - .def("GetSortedImageIds", &LineTrack::GetSortedImageIds) - .def("count_images", &LineTrack::count_images) - .def("projection", &LineTrack::projection) - .def("HasImage", &LineTrack::HasImage) - .def("Read", &LineTrack::Read) - .def("Write", &LineTrack::Write); + .def_readwrite("line", &LineTrack::line, ":class:`~limap.base.Line3d`, the 3D line") + .def_readonly("image_id_list", &LineTrack::image_id_list, "list[int], the associated image IDs") + .def_readonly("line_id_list", &LineTrack::line_id_list, "list[int], IDs of supporting 2D lines within each image") + .def_readonly("line2d_list", &LineTrack::line2d_list, "list[:class:`~limap.base.Line2d`], the supporting 2D line segments") + .def_readonly("active", &LineTrack::active, "bool, active status for recursive merging") + .def("count_lines", &LineTrack::count_lines, R"( + Returns: + int: The number of supporting 2D lines + )") + .def("GetSortedImageIds", &LineTrack::GetSortedImageIds, R"( + Returns: + list[int]: Sorted (and deduplicated) list of the associated image IDs + )") + .def("count_images", &LineTrack::count_images, R"( + Returns: + int: Number of unique associated images + )") + .def("projection", &LineTrack::projection, R"( + Project the 3D line to 2D using a list of :class:`~limap.base.CameraView`. + + Args: + views (list[:class:`~limap.base.CameraView`]): Camera views to project the 3D line + + Returns: + list[:class:`~limap.base.Line2d`]: The 2D projection segments of the 3D line + )", py::arg("views")) + .def("HasImage", &LineTrack::HasImage, R"( + Check whether the 3D line has a 2D support from a certain image. + + Args: + image_id (int): The image ID + + Returns: + bool: True if there is a supporting 2D line from this image + )", py::arg("image_id")) + .def("Read", &LineTrack::Read, R"( + Read the line track information from a file. + + Args: + filename (str): The file to read from + )", py::arg("filename")) + .def("Write", &LineTrack::Write, R"( + Write the line track information to a file. + + Args: + filename (str): The file to write to + )", py::arg("filename")); } void bind_line_dists(py::module& m) { - py::enum_(m, "LineDistType") + py::enum_(m, "LineDistType", "Enum of supported line distance types.") .value("ANGULAR", LineDistType::ANGULAR) .value("ANGULAR_DIST", LineDistType::ANGULAR_DIST) .value("ENDPOINTS", LineDistType::ENDPOINTS) @@ -280,22 +498,60 @@ void bind_line_dists(py::module& m) { m.def("compute_distance_2d", [](const Line2d& l1, const Line2d& l2, const LineDistType& type) { return compute_distance(l1, l2, type); - } + }, R"( + Compute distance between two :class:`~limap.base.Line2d` using the specified line distance type. + + Args: + l1 (:class:`~limap.base.Line2d`): First 2D line segment + l2 (:class:`~limap.base.Line2d`): Second 2D line segment + type (:class:`~limap.base.LineDistType`): Line distance type + + Returns: + `float`: The computed distance + )", py::arg("l1"), py::arg("l2"), py::arg("type") ); m.def("compute_distance_3d", [](const Line3d& l1, const Line3d& l2, const LineDistType& type) { return compute_distance(l1, l2, type); - } + }, R"( + Compute distance between two :class:`~limap.base.Line3d` using the specified line distance type. + + Args: + l1 (:class:`~limap.base.Line3d`): First 3D line segment + l2 (:class:`~limap.base.Line3d`): Second 3D line segment + type (:class:`~limap.base.LineDistType`): Line distance type + + Returns: + `float`: The computed distance + )", py::arg("l1"), py::arg("l2"), py::arg("type") ); m.def("compute_pairwise_distance_2d", [](const std::vector& lines, const LineDistType& type) { return compute_pairwise_distance(lines, type); - } + }, R"( + Compute pairwise distance among a list of :class:`~limap.base.Line2d` using the specified line distance type. + + Args: + lines (list[:class:`~limap.base.Line2d`]): List of 2D line segments + type (:class:`~limap.base.LineDistType`): Line distance type + + Returns: + :class:`np.array`: The computed pairwise distance matrix + )", py::arg("lines"), py::arg("type") ); m.def("compute_pairwise_distance_3d", [](const std::vector& lines, const LineDistType& type) { return compute_pairwise_distance(lines, type); - } + }, R"( + Compute pairwise distance among a list of :class:`~limap.base.Line3d` using the specified line distance type. + + Args: + lines (list[:class:`~limap.base.Line3d`]): List of 3D line segments + type (:class:`~limap.base.LineDistType`): Line distance type + + Returns: + :class:`np.array`: The computed pairwise distance matrix + )", py::arg("lines"), py::arg("type") ); } @@ -366,15 +622,29 @@ void bind_line_linker(py::module& m) { } void bind_camera(py::module& m) { - py::class_(m, "Camera") + py::class_(m, "Camera", R"( + | Camera model, inherits `COLMAP's camera model `_. + | COLMAP camera models: + | 0, SIMPLE_PINHOLE + | 1, PINHOLE + | 2, SIMPLE_RADIAL + | 3, RADIAL + | 4, OPENCV + | 5, OPENCV_FISHEYE + | 6, FULL_OPENCV + | 7, FOV + | 8, SIMPLE_RADIAL_FISHEYE + | 9, RADIAL_FISHEYE + | 10, THIN_PRISM_FISHEYE + )") .def(py::init<>()) .def(py::init&, int, std::pair>(), py::arg("model_id"), py::arg("params"), py::arg("cam_id")=-1, py::arg("hw")=std::make_pair(-1, -1)) .def(py::init&, int, std::pair>(), py::arg("model_name"), py::arg("params"), py::arg("cam_id")=-1, py::arg("hw")=std::make_pair(-1, -1)) .def(py::init>(), py::arg("K"), py::arg("cam_id")=-1, py::arg("hw")=std::make_pair(-1, -1)) .def(py::init>(), py::arg("model_id"), py::arg("K"), py::arg("cam_id")=-1, py::arg("hw")=std::make_pair(-1, -1)) .def(py::init>(), py::arg("model_name"), py::arg("K"), py::arg("cam_id")=-1, py::arg("hw")=std::make_pair(-1, -1)) - .def(py::init()) - .def(py::init()) + .def(py::init(), py::arg("dict")) + .def(py::init(), py::arg("cam")) .def(py::init>(), py::arg("model_id"), py::arg("cam_id")=-1, py::arg("hw")=std::make_pair(-1, -1)) // empty camera .def(py::init>(), py::arg("model_name"), py::arg("cam_id")=-1, py::arg("hw")=std::make_pair(-1, -1)) // empty camera .def(py::pickle( @@ -385,30 +655,93 @@ void bind_camera(py::module& m) { return Camera(dict); } )) - .def("as_dict", &Camera::as_dict) - .def("h", &Camera::h) - .def("w", &Camera::w) - .def("K", &Camera::K) - .def("K_inv", &Camera::K_inv) - .def("cam_id", &Camera::CameraId) - .def("model_id", &Camera::ModelId) - .def("params", &Camera::params) - .def("num_params", &Camera::NumParams) - .def("resize", &Camera::resize) - .def("set_max_image_dim", &Camera::set_max_image_dim) - .def("set_cam_id", &Camera::SetCameraId) - .def("InitializeParams", &Camera::InitializeParams) - .def("IsUndistorted", &Camera::IsUndistorted) + .def("as_dict", &Camera::as_dict, R"( + Returns: + dict: Python dict representation of this :class:`~limap.base.Camera` + )") + .def("h", &Camera::h, R"( + Returns: + int: Image height in pixels + )") + .def("w", &Camera::w, R"( + Returns: + int: Image width in pixels + )") + .def("K", &Camera::K, R"( + Returns: + :class:`np.array` of shape (3, 3): Camera's intrinsic matrix + )") + .def("K_inv", &Camera::K_inv, R"( + Returns: + :class:`np.array` of shape (3, 3): Inverse of the intrinsic matrix + )") + .def("cam_id", &Camera::CameraId, R"( + Returns: + int: Camera ID + )") + .def("model_id", &Camera::ModelId, R"( + Returns: + int: COLMAP camera model ID + )") + .def("params", &Camera::params, R"( + Returns: + list (float): Minimal representation of intrinsic paramters, length varies according to camera model + )") + .def("num_params", &Camera::NumParams, R"( + Returns: + int: Number of the paramters for minimal representation of intrinsic + )") + .def("resize", &Camera::resize, R"( + Resize camera's width and height. + + Args: + width (int) + height (int) + )", py::arg("width"), py::arg("height")) + .def("set_max_image_dim", &Camera::set_max_image_dim, R"( + Set the maximum image dimension, the camera will be resized if the longer dimension of width or height is larger than this value. + + Args: + val (int) + )", py::arg("val")) + .def("set_cam_id", &Camera::SetCameraId, R"( + Set the camera ID. + )", py::arg("camera_id")) + .def("InitializeParams", &Camera::InitializeParams, R"( + Initialize the intrinsics using focal length, width, and height + + Args: + focal_length (double) + width (int) + height (int) + )", py::arg("focal_length"), py::arg("width"), py::arg("height")) .def("ImageToWorld", &Camera::ImageToWorld) .def("WorldToImage", &Camera::WorldToImage) - .def("IsInitialized", &Camera::IsInitialized); + .def("IsInitialized", &Camera::IsInitialized, R"( + Returns: + bool: True if the camera parameters are initialized + )") + .def("IsUndistorted", &Camera::IsUndistorted, R"( + Returns: + bool: True if the camera model is without distortion + )"); - py::class_(m, "CameraPose") - .def(py::init(), py::arg("initialized")=false) - .def(py::init(), py::arg("qvec"), py::arg("tvec"), py::arg("initialized")=true) - .def(py::init(), py::arg("R"), py::arg("tvec"), py::arg("initialized")=true) - .def(py::init()) - .def(py::init()) + py::class_(m, "CameraPose", "Representing the world-to-cam pose (R, t) with a quaternion and a translation vector. The quaternion convention is `(w, x, y, z)` (real part first).") + .def(py::init(), R"( + Default constructor: identity pose + )", py::arg("initialized")=false) + .def(py::init(), R"( + Copy constructor + )", py::arg("campose")) + .def(py::init(), R"( + Constructor from a quaternion vector and a translation vector + )", py::arg("qvec"), py::arg("tvec"), py::arg("initialized")=true) + .def(py::init(), R"( + Constructor from a rotation matrix and a translation vector + )", py::arg("R"), py::arg("tvec"), py::arg("initialized")=true) + .def(py::init(), R"( + Constructor from a Python dict + )", py::arg("dict")) .def(py::pickle( [](const CameraPose& input) { // dump return input.as_dict(); @@ -417,23 +750,41 @@ void bind_camera(py::module& m) { return CameraPose(dict); } )) - .def("as_dict", &CameraPose::as_dict) - .def_readonly("qvec", &CameraPose::qvec) - .def_readonly("tvec", &CameraPose::tvec) - .def_readwrite("initialized", &CameraPose::initialized) - .def("R", &CameraPose::R) - .def("T", &CameraPose::T) - .def("center", &CameraPose::center) - .def("projdepth", &CameraPose::projdepth); - - py::class_(m, "CameraImage") + .def("as_dict", &CameraPose::as_dict, R"( + Returns: + dict: Python dict representation of this :class:`~limap.base.CameraPose` + )") + .def_readonly("qvec", &CameraPose::qvec, ":class:`np.array` of shape (4,): The quaternion vector `(w, x, y, z)`") + .def_readonly("tvec", &CameraPose::tvec, ":class:`np.array` of shape (3,): The translation vector") + .def_readwrite("initialized", &CameraPose::initialized, "bool: Flag indicating whether the pose has been initialized") + .def("R", &CameraPose::R, R"( + Returns: + :class:`np.array` of shape (3, 3): The rotation matrix + )") + .def("T", &CameraPose::T, R"( + Returns: + :class:`np.array` of shape (3,): The translation vector + )") + .def("center", &CameraPose::center, R"( + Returns: + :class:`np.array` of shape (3,): World-space coordinate of the camera + )") + .def("projdepth", &CameraPose::projdepth, R"( + Args: + p3d (:class:`np.array`): World-space coordinate of a 3D point + + Returns: + float: The projection depth of the 3D point viewed from this camera pose + )", py::arg("p3d")); + + py::class_(m, "CameraImage", "This class associates the ID of a :class:`~limap.base.Camera`, a :class:`~limap.base.CameraPose`, and an image file") .def(py::init<>()) .def(py::init(), py::arg("cam_id"), py::arg("image_name") = "none") // empty image .def(py::init(), py::arg("camera"), py::arg("image_name") = "none") // empty image .def(py::init(), py::arg("cam_id"), py::arg("pose"), py::arg("image_name") = "none") .def(py::init(), py::arg("camera"), py::arg("pose"), py::arg("image_name") = "none") - .def(py::init()) - .def(py::init()) + .def(py::init(), py::arg("dict")) + .def(py::init(), py::arg("camimage")) .def(py::pickle( [](const CameraImage& input) { // dump return input.as_dict(); @@ -442,21 +793,43 @@ void bind_camera(py::module& m) { return CameraImage(dict); } )) - .def("as_dict", &CameraImage::as_dict) - .def_readonly("cam_id", &CameraImage::cam_id) - .def_readonly("pose", &CameraImage::pose) - .def("R", &CameraImage::R) - .def("T", &CameraImage::T) - .def("set_camera_id", &CameraImage::SetCameraId) - .def("image_name", &CameraImage::image_name) - .def("set_image_name", &CameraImage::SetImageName); - - py::class_(m, "CameraView") + .def("as_dict", &CameraImage::as_dict, R"( + Returns: + dict: Python dict representation of this :class:`~limap.base.CameraImage` + )") + .def_readonly("cam_id", &CameraImage::cam_id, "int, the camera ID") + .def_readonly("pose", &CameraImage::pose, ":class:`~limap.base.CameraPose`, the camera pose") + .def("R", &CameraImage::R, R"( + Returns: + :class:`np.array` of shape (3, 3): The rotation matrix of the camera pose + )") + .def("T", &CameraImage::T, R"( + Returns: + :class:`np.array` of shape (3,): The translation vector of the camera pose + )") + .def("set_camera_id", &CameraImage::SetCameraId, R"( + Set the camera ID. + + Args: + cam_id (int) + )", py::arg("cam_id")) + .def("image_name", &CameraImage::image_name, R"( + Returns: + str: The image file name + )") + .def("set_image_name", &CameraImage::SetImageName, R"( + Set the name of the image file. + + Args: + image_name (str) + )", py::arg("image_name")); + + py::class_(m, "CameraView", "Inherits :class:`~limap.base.CameraImage`, incorporating the :class:`~limap.base.Camera` model and its parameters for projection/unprojection between 2D and 3D.") .def(py::init<>()) .def(py::init(), py::arg("camera"), py::arg("image_name") = "none") // empty view .def(py::init(), py::arg("camera"), py::arg("pose"), py::arg("image_name") = "none") - .def(py::init()) - .def(py::init()) + .def(py::init(), py::arg("dict")) + .def(py::init(), py::arg("camview")) .def(py::pickle( [](const CameraView& input) { // dump return input.as_dict(); @@ -465,33 +838,102 @@ void bind_camera(py::module& m) { return CameraView(dict); } )) - .def_readonly("cam", &CameraView::cam) - .def_readonly("pose", &CameraView::pose) - .def("as_dict", &CameraView::as_dict) - .def("read_image", &CameraView::read_image, py::arg("set_gray")=false) - .def("K", &CameraView::K) - .def("K_inv", &CameraView::K_inv) - .def("h", &CameraView::h) - .def("w", &CameraView::w) - .def("R", &CameraView::R) - .def("T", &CameraView::T) - .def("matrix", &CameraView::matrix) - .def("projection", &CameraView::projection) - .def("ray_direction", &CameraView::ray_direction) - .def("get_direction_from_vp", &CameraView::get_direction_from_vp) - .def("image_name", &CameraView::image_name) - .def("set_image_name", &CameraView::SetImageName) - .def("get_initial_focal_length", &CameraView::get_initial_focal_length); - - py::class_(m, "ImageCollection") + .def_readonly("cam", &CameraView::cam, ":class:`~limap.base.Camera`, the camera model") + .def_readonly("pose", &CameraView::pose, ":class:`~limap.base.CameraPose`, the camera pose") + .def("as_dict", &CameraView::as_dict, R"( + Returns: + dict: Python dict representation of this :class:`~limap.base.CameraView` + )") + .def("read_image", &CameraView::read_image, R"( + Read image data from the image file. + + Args: + set_gray (bool): Whether to convert the image to gray. Default False. + + Returns: + :class:`np.array`: The image data matrix + )", py::arg("set_gray")=false) + .def("K", &CameraView::K, R"( + Returns: + :class:`np.array` of shape (3, 3): The intrinsic matrix of the camera + )") + .def("K_inv", &CameraView::K_inv, R"( + Returns: + :class:`np.array` of shape (3, 3): The inverse of the camera's intrinsic matrix + )") + .def("h", &CameraView::h, R"( + Returns: + int: Image height in pixels + )") + .def("w", &CameraView::w, R"( + Returns: + int: Image width in pixels + )") + .def("R", &CameraView::R, R"( + Returns: + :class:`np.array` of shape (3, 3): The rotation matrix of the camera pose + )") + .def("T", &CameraView::T, R"( + Returns: + :class:`np.array` of shape (3,): The translation vector of the camera pose + )") + .def("matrix", &CameraView::matrix, R"( + Returns: + :class:`np.array` of shape (3, 4): The projection matrix `P = K[R|T]` + )") + .def("projection", &CameraView::projection, R"( + Args: + p3d (:class:`np.array`): World-space coordinate of a 3D point + + Returns: + :class:`np.array` of shape (2,): The 2D pixel-space coordinate of the point's projection on image + )", py::arg("p3d")) + .def("ray_direction", &CameraView::ray_direction, R"( + Args: + p2d (:class:`np.array`): Pixel-space coordinate of a 2D point on the image + + Returns: + :class:`np.array` of shape (3,): The world-space direction of the camera ray passing the 2D point + )", py::arg("p2d")) + .def("get_direction_from_vp", &CameraView::get_direction_from_vp, R"( + Args: + vp (:class:`np.array`): The coordinate of a vanishing point + + Returns: + :class:`np.array` of shape (3,): The direction from the vanishing point + )", py::arg("vp")) + .def("image_name", &CameraView::image_name, R"( + Returns: + str: The image file name + )") + .def("set_image_name", &CameraView::SetImageName, R"( + Set the name of the image file. + + Args: + image_name (str) + )", py::arg("image_name")) + .def("get_initial_focal_length", &CameraView::get_initial_focal_length, R"( + Try to get the focal length information from the image's EXIF data. + If not available in image EXIF, the focal length is estimated by the max dimension of the image. + + Returns: + tuple[double, bool]: Initial focal length and a flag indicating if the value is read from image's EXIF data. + )"); + + + py::class_(m, "ImageCollection", R"( + A flexible class that consists of cameras and images in a scene or dataset. In each image stores the corresponding ID of the camera, making it easy to extend to single/multiple sequences or unstructured image collections. + The constructor arguments `input_cameras` and `input_images` can be either list of :class:`~limap.base.Camera` and :class:`~limap.base.CameraImage` + or python dict mapping integer IDs to :class:`~limap.base.Camera` and :class:`~limap.base.CameraImage`. + )") .def(py::init<>()) - .def(py::init&, const std::map&>()) - .def(py::init&, const std::map&>()) - .def(py::init&, const std::vector&>()) - .def(py::init&, const std::vector&>()) - .def(py::init&>()) - .def(py::init()) - .def(py::init()) + .def(py::init&, const std::map&>(), py::arg("input_cameras"), py::arg("input_images")) + .def(py::init&, const std::map&>(), py::arg("input_cameras"), py::arg("input_images")) + .def(py::init&, const std::vector&>(), py::arg("input_cameras"), py::arg("input_images")) + .def(py::init&, const std::vector&>(), py::arg("input_cameras"), py::arg("input_images")) + .def(py::init&>(), py::arg("camviews")) + .def(py::init(), py::arg("dict")) + .def(py::init(), py::arg("imagecols")) .def(py::pickle( [](const ImageCollection& input) { // dump return input.as_dict(); @@ -500,44 +942,225 @@ void bind_camera(py::module& m) { return ImageCollection(dict); } )) - .def("as_dict", &ImageCollection::as_dict) - .def("subset_by_camera_ids", &ImageCollection::subset_by_camera_ids) - .def("subset_by_image_ids", &ImageCollection::subset_by_image_ids) - .def("subset_initialized", &ImageCollection::subset_initialized) - .def("update_neighbors", &ImageCollection::update_neighbors) - .def("get_cameras", &ImageCollection::get_cameras) - .def("get_cam_ids", &ImageCollection::get_cam_ids) - .def("get_images", &ImageCollection::get_images) - .def("get_img_ids", &ImageCollection::get_img_ids) - .def("get_camviews", &ImageCollection::get_camviews) - .def("get_map_camviews", &ImageCollection::get_map_camviews) - .def("get_locations", &ImageCollection::get_locations) - .def("get_map_locations", &ImageCollection::get_map_locations) - .def("exist_cam", &ImageCollection::exist_cam) - .def("exist_image", &ImageCollection::exist_image) - .def("cam", &ImageCollection::cam) - .def("camimage", &ImageCollection::camimage) - .def("campose", &ImageCollection::campose) - .def("camview", &ImageCollection::camview) - .def("image_name", &ImageCollection::image_name) - .def("get_image_name_list", &ImageCollection::get_image_name_list) - .def("get_image_name_dict", &ImageCollection::get_image_name_dict) - .def("NumCameras", &ImageCollection::NumCameras) - .def("NumImages", &ImageCollection::NumImages) - .def("set_max_image_dim", &ImageCollection::set_max_image_dim) - .def("change_camera", &ImageCollection::change_camera) - .def("set_camera_pose", &ImageCollection::set_camera_pose) - .def("get_camera_pose", &ImageCollection::get_camera_pose) - .def("change_image", &ImageCollection::change_image) - .def("change_image_name", &ImageCollection::change_image_name) - .def("IsUndistorted", &ImageCollection::IsUndistorted) - .def("read_image", &ImageCollection::read_image, py::arg("img_id"), py::arg("set_gray")=false) - .def("apply_similarity_transform", &ImageCollection::apply_similarity_transform) - .def("get_first_image_id_by_camera_id", &ImageCollection::get_first_image_id_by_camera_id) - .def("init_uninitialized_cameras", &ImageCollection::init_uninitialized_cameras) - .def("uninitialize_poses", &ImageCollection::uninitialize_poses) - .def("uninitialize_intrinsics", &ImageCollection::uninitialize_intrinsics) - .def("IsUndistortedCameraModel", &ImageCollection::IsUndistortedCameraModel); + .def("as_dict", &ImageCollection::as_dict, R"( + Returns: + dict: Python dict representation of this :class:`~limap.base.ImageCollection` + )") + .def("subset_by_camera_ids", &ImageCollection::subset_by_camera_ids, R"( + Filter the images using camera IDs. + + Args: + valid_camera_ids (list[int]): Images from camera with these IDs are kept in the filtered subset + + Returns: + :class:`~limap.base.ImageCollection`: The filtered subset collection + )", py::arg("valid_camera_ids")) + .def("subset_by_image_ids", &ImageCollection::subset_by_image_ids, R"( + Filter the images using image IDs. + + Args: + valid_image_ids (list[int]): IDs of images to be kept in the filtered subset + + Returns: + :class:`~limap.base.ImageCollection`: The filtered subset collection + )", py::arg("valid_image_ids")) + .def("subset_initialized", &ImageCollection::subset_initialized, R"( + Filter the images to create a subset collection containing only images with initialized camera poses. + + Returns: + :class:`~limap.base.ImageCollection`: The filtered subset collection + )") + .def("update_neighbors", &ImageCollection::update_neighbors, R"( + Update the neighbor information among images (e.g. after filtering). Remove neighboring images that are not in the image collection. + + Args: + neighbors (dict[int -> list[int]]): The input neighbor information + + Returns: + dict[int -> list[int]]: Updated neighbor information + )") + .def("get_cameras", &ImageCollection::get_cameras, R"( + Returns: + list[:class:`~limap.base.Camera`]: All cameras in the collection + )") + .def("get_cam_ids", &ImageCollection::get_cam_ids, R"( + Returns: + list[int]: IDs of all cameras in the collection + )") + .def("get_images", &ImageCollection::get_images, R"( + Returns: + list[:class:`~limap.base.CameraImage`]: All images in the collection + )") + .def("get_img_ids", &ImageCollection::get_img_ids, R"( + Returns: + list[int]: IDs of all images in the collection + )") + .def("get_camviews", &ImageCollection::get_camviews, R"( + Returns: + list[:class:`~limap.base.CameraView`]: The associated :class:`~limap.base.CameraView` from all the images and their cameras in the collection + )") + .def("get_map_camviews", &ImageCollection::get_map_camviews, R"( + Returns: + dict[int -> :class:`~limap.base.CameraView`]: Mapping of image IDs to their associated :class:`~limap.base.CameraView` + )") + .def("get_locations", &ImageCollection::get_locations, R"( + Returns: + list[:class:`np.array`]: The world-space locations of the camera for all images in the collection, each of shape (3, ) + )") + .def("get_map_locations", &ImageCollection::get_map_locations, R"( + Returns: + dict[int -> :class:`np.array`]: Mapping of image IDs to their camera locations in world-space + )") + .def("exist_cam", &ImageCollection::exist_cam, R"( + Args: + cam_id (int) + + Returns: + bool: True if the camera with `cam_id` exists in the collection + )", py::arg("cam_id")) + .def("exist_image", &ImageCollection::exist_image, R"( + Args: + img_id (int) + + Returns: + bool: True if the image with `img_id` exists in the collection + )", py::arg("img_id")) + .def("cam", &ImageCollection::cam, R"( + Args: + cam_id (int) + + Returns: + :class:`~limap.base.Camera`: The camera with `cam_id` + )", py::arg("cam_id")) + .def("camimage", &ImageCollection::camimage, R"( + Args: + img_id (int) + + Returns: + :class:`~limap.base.CameraImage`: The image with `img_id` + )", py::arg("img_id")) + .def("campose", &ImageCollection::campose, R"( + Args: + img_id (int) + + Returns: + :class:`~limap.base.CameraPose`: The camera pose of the image + )", py::arg("img_id")) + .def("camview", &ImageCollection::camview, R"( + Args: + img_id (int) + + Returns: + :class:`~limap.base.CameraView`: The :class:`~limap.base.CameraView` from the image + )", py::arg("img_id")) + .def("image_name", &ImageCollection::image_name, R"( + Args: + img_id (int) + + Returns: + str: The file name of the image + )", py::arg("img_id")) + .def("get_image_name_list", &ImageCollection::get_image_name_list, R"( + Returns: + list[str]: All the image file names + )") + .def("get_image_name_dict", &ImageCollection::get_image_name_dict, R"( + Returns: + dict[int -> str]: Mapping of image IDs to the file names + )") + .def("NumCameras", &ImageCollection::NumCameras, R"( + Returns: + int: The number of cameras in the collection + )") + .def("NumImages", &ImageCollection::NumImages, R"( + Returns: + int: The number of images in the collection + )") + .def("set_max_image_dim", &ImageCollection::set_max_image_dim, R"( + Set the maximum image dimension for all cameras using :py:meth:`~limap.base.Camera.set_max_image_dim`. + + Args: + val (int) + )", py::arg("val")) + .def("change_camera", &ImageCollection::change_camera, R"( + Change the camera model of a specific camera. + + Args: + cam_id (int) + cam (:class:`~limap.base.Camera`) + )", py::arg("cam_id"), py::arg("cam")) + .def("set_camera_pose", &ImageCollection::set_camera_pose, R"( + Set the camera pose for a specific image. + + Args: + img_id (int) + pose (:class:`~limap.base.CameraPose`) + )", py::arg("img_id"), py::arg("pose")) + .def("get_camera_pose", &ImageCollection::get_camera_pose, R"( + Get the camera pose of a specific image. + + Args: + img_id (int) + + Returns: + :class:`~limap.base.CameraPose` + )", py::arg("img_id")) + .def("change_image", &ImageCollection::change_image, R"( + Change an image. + + Args: + img_id (int) + camimage (:class:`~limap.base.CameraImage`) + )", py::arg("img_id"), py::arg("camimage")) + .def("change_image_name", &ImageCollection::change_image_name, R"( + Change the file name of an image. + + Args: + img_id (int) + new_name (str) + )", py::arg("img_id"), py::arg("new_name")) + .def("IsUndistorted", &ImageCollection::IsUndistorted, R"( + Returns: + bool: True if all cameras in the collection are without distortion, see :py:meth:`~limap.base.Camera.IsUndistorted`. + )") + .def("read_image", &ImageCollection::read_image, R"( + Read an image, calls :py:meth:`~limap.base.CameraView.read_image`. + + Args: + img_id (int): The image ID + set_gray (bool): Whether to convert the image to gray. Default False. + + Returns: + :class:`np.array`: The image data matrix + )", py::arg("img_id"), py::arg("set_gray")=false) + .def("apply_similarity_transform", &ImageCollection::apply_similarity_transform, R"( + Apply similarity transform to all image poses. + + Args: + transform (:class:`limap.base.SimilarityTransform3`) + )", py::arg("transform")) + .def("get_first_image_id_by_camera_id", &ImageCollection::get_first_image_id_by_camera_id, R"( + Get the ID of the first image captured with a specific camera. + + Args: + cam_id (int): The camera ID + + Return: + int: The image ID. + )", py::arg("cam_id")) + .def("init_uninitialized_cameras", &ImageCollection::init_uninitialized_cameras, R"( + Initialize all uninitialized cameras by :func:`~limap.base.Camera.InitializeParams`. + )") + .def("uninitialize_poses", &ImageCollection::uninitialize_poses, R"( + Uninitialize camera poses for all images, set them to identity poses and remove the :attr:`~limap.base.CameraPose.initialized` flag. + )") + .def("uninitialize_intrinsics", &ImageCollection::uninitialize_intrinsics, R"( + Uninitialize intrinsics for all cameras. + )") + .def("IsUndistortedCameraModel", &ImageCollection::IsUndistortedCameraModel, R"( + Returns: + bool: True if all camera models are undistorted. + )"); } void bind_pointtrack(py::module& m) { @@ -582,11 +1205,11 @@ void bind_base(py::module& m) { bind_graph(m); bind_transforms(m); bind_pointtrack(m); + bind_camera(m); bind_linebase(m); bind_linetrack(m); bind_line_dists(m); bind_line_linker(m); - bind_camera(m); m.def("get_effective_num_threads", &colmap::GetEffectiveNumThreads); } diff --git a/limap/base/depth_reader_base.py b/limap/base/depth_reader_base.py index 7e83ead5..ba236b7c 100644 --- a/limap/base/depth_reader_base.py +++ b/limap/base/depth_reader_base.py @@ -1,13 +1,32 @@ import cv2 class BaseDepthReader(): + """ + Base class for the depth reader storing the filename and potentially other information + """ def __init__(self, filename): self.filename = filename def read(self, filename): + """ + Virtual method - Read depth from a filename + + Args: + filename (str): The filename of the depth image + Returns: + depth (:class:`np.array` of shape (H, W)): The array for the depth map + """ raise NotImplementedError def read_depth(self, img_hw=None): + """ + Read depth using the read(self, filename) function + + Args: + img_hw (pair of int, optional): The height and width for the read depth. By default we keep the original resolution of the file + Returns: + depth (:class:`np.array` of shape (H, W)): The array for the depth map + """ depth = self.read(self.filename) if img_hw is not None and (depth.shape[0] != img_hw[0] or depth.shape[1] != img_hw[1]): depth = cv2.resize(depth, (img_hw[1], img_hw[0])) diff --git a/limap/base/functions.py b/limap/base/functions.py index 5b19b24c..a6f48f7f 100644 --- a/limap/base/functions.py +++ b/limap/base/functions.py @@ -1,18 +1,46 @@ import _limap._base as _base def get_all_lines_2d(all_2d_segs): + """ + Convert :class:`np.array` representations of 2D line segments to dict of :class:`~limap.base.Line2d`. + + Args: + all_2d_segs (dict[int -> :class:`np.array`]): Map image IDs to :class:`np.array` of shape (N, 4), each row (4 numbers) is concatenated by the start and end of a 2D line segment. + + Returns: + dict[int -> list[:class:`~limap.base.Line2d`]]: Map image IDs to list of :class:`~limap.base.Line2d`. + """ all_lines_2d = {} for img_id in all_2d_segs: all_lines_2d[img_id] = _base._GetLine2dVectorFromArray(all_2d_segs[img_id]) return all_lines_2d -def get_all_lines_3d(seg3d_list): +def get_all_lines_3d(all_3d_segs): + """ + Convert :class:`np.array` representations of 3D line segments to dict of :class:`~limap.base.Line3d`. + + Args: + all_3d_segs (dict[int -> :class:`np.array`]): Map image IDs to :class:`np.array` of shape (N, 2, 3), each 2*3 matrix is stacked from the two endpoints of a 3D line segment. + + Returns: + dict[int -> list[:class:`~limap.base.Line3d`]]: Map image IDs to list of :class:`~limap.base.Line3d`. + """ all_lines_3d = {} - for img_id, segs3d in seg3d_list.items(): + for img_id, segs3d in all_3d_segs.items(): all_lines_3d[img_id] = _base._GetLine3dVectorFromArray(segs3d) return all_lines_3d def get_invert_idmap_from_linetracks(all_lines_2d, linetracks): + """ + Get the mapping from a 2D line segment (identified by an image and its line ID) to the index of its associated linetrack. + + Args: + all_lines_2d (dict[int -> list[:class:`~limap.base.Line2d`]]): Map image IDs to the list of 2D line segments in each image. + linetracks (list[:class:`~limap.base.LineTrack`]): All line tracks. + + Returns: + dict[int -> list[int]]: Map image ID to list of the associated line track indices for each 2D line, -1 if not associated to any track. + """ map = {} for img_id in all_lines_2d: lines_2d = all_lines_2d[img_id] diff --git a/limap/base/p3d_reader_base.py b/limap/base/p3d_reader_base.py index c4f15765..6e91bb01 100644 --- a/limap/base/p3d_reader_base.py +++ b/limap/base/p3d_reader_base.py @@ -1,11 +1,27 @@ + + class BaseP3DReader(): def __init__(self, filename): self.filename = filename def read(self, filename): + """ + Virtual method - Read a point cloud from a filename + + Args: + filename (str): The filename of the depth image + Returns: + point cloud (:class:`np.array` of shape (N, 3)): The array for the 3D points + """ raise NotImplementedError def read_p3ds(self): + """ + Read a point cloud using the read(self, filename) function + + Returns: + point cloud (:class:`np.array` of shape (N, 3)): The array for the 3D points + """ p3ds = self.read(self.filename) return p3ds diff --git a/limap/estimators/absolute_pose/__init__.py b/limap/estimators/absolute_pose/__init__.py index d4b529fc..232eb5d7 100644 --- a/limap/estimators/absolute_pose/__init__.py +++ b/limap/estimators/absolute_pose/__init__.py @@ -1,2 +1,28 @@ -from .pl_estimate_absolute_pose import * +from ._pl_estimate_absolute_pose import _pl_estimate_absolute_pose + +def pl_estimate_absolute_pose(cfg, l3ds, l3d_ids, l2ds, p3ds, p2ds, camera, campose=None, + inliers_line=None, inliers_point=None, jointloc_cfg=None, silent=True, logger=None): + """ + Estimate absolute camera pose of a image from matched 2D-3D line and point correspondences. + + Args: + cfg (dict): Localization config, fields refer to "localization" section in :file:`cfgs/localization/default.yaml` + l3ds (list[:class:`limap.base.Line3d`]): Matched 3D line segments + l3d_ids (list[int]): Indices into `l3ds` for match of each :class:`limap.base.Line3d` in `l2ds`, same length of `l2ds` + l2ds (list[:class:`limap.base.Line2d`]): Matched 2d lines, same length of `l3d_ids` + p3ds (list[:class:`np.array`]): Matched 3D points, same length of `p2ds` + p2ds (list[:class:`np.array`]): Matched 2D points, same length of `p3ds` + camera (:class:`limap.base.Camera`): Camera of the query image + campose (:class:`limap.base.CameraPose`, optional): Initial camera pose, only useful for pose refinement (when ``cfg["ransac"]["method"]`` is :py:obj:`None`) + inliers_line (list[int], optional): Indices of line inliers, only useful for pose refinement + inliers_point (list[int], optional): Indices of point inliers, only useful for pose refinement + jointloc_cfg (dict, optional): Config for joint optimization, fields refer to :class:`limap.optimize.LineLocConfig`, pass :py:obj:`None` for default + silent (bool, optional): Turn off to print logs during Ceres optimization + logger (:class:`logging.Logger`): Logger to print logs for information + + Returns: + tuple[:class:`limap.base.CameraPose`, :class:`limap.estimators.RansacStatistics`]: Estimated pose and ransac statistics. + """ + return _pl_estimate_absolute_pose(cfg, l3ds, l3d_ids, l2ds, p3ds, p2ds, camera, campose=campose, + inliers_line=inliers_line, inliers_point=inliers_point, jointloc_cfg=jointloc_cfg, silent=silent, logger=logger) diff --git a/limap/estimators/absolute_pose/pl_estimate_absolute_pose.py b/limap/estimators/absolute_pose/_pl_estimate_absolute_pose.py similarity index 67% rename from limap/estimators/absolute_pose/pl_estimate_absolute_pose.py rename to limap/estimators/absolute_pose/_pl_estimate_absolute_pose.py index fdf00076..6adce473 100644 --- a/limap/estimators/absolute_pose/pl_estimate_absolute_pose.py +++ b/limap/estimators/absolute_pose/_pl_estimate_absolute_pose.py @@ -4,28 +4,8 @@ import limap.base as _base import numpy as np -def pl_estimate_absolute_pose(cfg, l3ds, l3d_ids, l2ds, p3ds, p2ds, camera, campose=None, - inliers_line=None, inliers_point=None, jointloc_cfg=None, silent=True, logger=None): - """ - Estimate absolute camera pose of a image from matched 2D-3D line and point correspondences. - - :param cfg: dict, fields refer to "localization" section in `cfgs/localization/default.yaml` - :param l3ds: iterable of limap.base.Line3d - :param l3d_ids: iterable of int, indices into l3ds for match of each Line2d in `l2ds`, same length of `l2ds` - :param l2ds: iterable of limap.base.Line2d, matched 2d lines, same length of `l3d_ids` - :param p3ds: iterable of np.array, matched 3D points, same length of `p2ds` - :param p2ds: iterable of np.array, matched 2D points, same length of `p3ds` - :param camera: limap.base.Camera, camera of the query image - :param campose: limap.base.CameraPose (optional), initial camera pose, only useful for pose refinement - (when cfg["ransac"]["method"] is None) - :param inliers_line: iterable of int (optional), indices of line inliers, only useful for pose refinement - :param inliers_point: iterable of int (optional), indices of point inliers, only useful for pose refinement - :param jointloc_cfg: dict (optional), fields corresponding to limap.optimize.LineLocConfig, pass None for default (or set in 'optimize' fields in cfg) - :param silent: boolean (optional), turn off to print logs during Ceres optimization - :param logger: logging.Logger (optional), print logs for information - - :return: tuple, estimated pose and ransac statistics. - """ +def _pl_estimate_absolute_pose(cfg, l3ds, l3d_ids, l2ds, p3ds, p2ds, camera, campose=None, + inliers_line=None, inliers_point=None, jointloc_cfg=None, silent=True, logger=None): if jointloc_cfg is None: jointloc_cfg = {} if cfg.get('optimize'): diff --git a/limap/estimators/bindings.cc b/limap/estimators/bindings.cc index 272ceb2c..40cf5214 100644 --- a/limap/estimators/bindings.cc +++ b/limap/estimators/bindings.cc @@ -19,7 +19,7 @@ namespace limap { void bind_pose(py::module& m); void bind_ransaclib(py::module& m) { using ExtendedHybridLORansacOptions = estimators::ExtendedHybridLORansacOptions; - py::class_(m, "RansacStats") + py::class_(m, "RansacStatistics") .def(py::init<>()) .def_readwrite("num_iterations", &ransac_lib::RansacStatistics::num_iterations) .def_readwrite("best_num_inliers", &ransac_lib::RansacStatistics::best_num_inliers) @@ -36,7 +36,7 @@ void bind_ransaclib(py::module& m) { .def_readwrite("squared_inlier_threshold_", &ransac_lib::RansacOptions::squared_inlier_threshold_) .def_readwrite("random_seed_", &ransac_lib::RansacOptions::random_seed_); - py::class_(m, "LORansacOptions") + py::class_(m, "LORansacOptions", "Inherits :class:`~limap.estimators.RansacOptions`") .def(py::init<>()) .def_readwrite("min_num_iterations_", &ransac_lib::LORansacOptions::min_num_iterations_) .def_readwrite("max_num_iterations_", &ransac_lib::LORansacOptions::max_num_iterations_) @@ -52,7 +52,7 @@ void bind_ransaclib(py::module& m) { .def_readwrite("final_least_squares_", &ransac_lib::LORansacOptions::final_least_squares_); // hybrid ransac - py::class_(m, "HybridRansacStatistics") + py::class_(m, "HybridRansacStatistics", "Inherits :class:`~limap.estimators.RansacStatistics`") .def(py::init<>()) .def_readwrite("num_iterations_total", &ransac_lib::HybridRansacStatistics::num_iterations_total) .def_readwrite("num_iterations_per_solver", &ransac_lib::HybridRansacStatistics::num_iterations_per_solver) @@ -62,24 +62,8 @@ void bind_ransaclib(py::module& m) { .def_readwrite("inlier_ratios", &ransac_lib::HybridRansacStatistics::inlier_ratios) .def_readwrite("inlier_indices", &ransac_lib::HybridRansacStatistics::inlier_indices) .def_readwrite("number_lo_iterations", &ransac_lib::HybridRansacStatistics::number_lo_iterations); - - py::class_(m, "HybridLORansacOptions") - .def(py::init<>()) - .def_readwrite("min_num_iterations_", &ransac_lib::HybridLORansacOptions::min_num_iterations_) - .def_readwrite("max_num_iterations_", &ransac_lib::HybridLORansacOptions::max_num_iterations_) - .def_readwrite("max_num_iterations_per_solver_", &ransac_lib::HybridLORansacOptions::max_num_iterations_per_solver_) - .def_readwrite("success_probability_", &ransac_lib::HybridLORansacOptions::success_probability_) - .def_readwrite("squared_inlier_thresholds_", &ransac_lib::HybridLORansacOptions::squared_inlier_thresholds_) - .def_readwrite("data_type_weights_", &ransac_lib::HybridLORansacOptions::data_type_weights_) - .def_readwrite("random_seed_", &ransac_lib::HybridLORansacOptions::random_seed_) - .def_readwrite("num_lo_steps_", &ransac_lib::HybridLORansacOptions::num_lo_steps_) - .def_readwrite("threshold_multiplier_", &ransac_lib::HybridLORansacOptions::threshold_multiplier_) - .def_readwrite("num_lsq_iterations_", &ransac_lib::HybridLORansacOptions::num_lsq_iterations_) - .def_readwrite("min_sample_multiplicator_", &ransac_lib::HybridLORansacOptions::min_sample_multiplicator_) - .def_readwrite("lo_starting_iterations_", &ransac_lib::HybridLORansacOptions::lo_starting_iterations_) - .def_readwrite("final_least_squares_", &ransac_lib::HybridLORansacOptions::final_least_squares_); - py::class_(m, "ExtendedHybridLORansacOptions") + py::class_(m, "HybridLORansacOptions", "Inherits :class:`~limap.estimators.LORansacOptions`") .def(py::init<>()) .def_readwrite("min_num_iterations_", &ExtendedHybridLORansacOptions::min_num_iterations_) .def_readwrite("max_num_iterations_", &ExtendedHybridLORansacOptions::max_num_iterations_) diff --git a/limap/evaluation/bindings.cc b/limap/evaluation/bindings.cc index 154fa0de..b4e5cbe9 100644 --- a/limap/evaluation/bindings.cc +++ b/limap/evaluation/bindings.cc @@ -19,20 +19,55 @@ namespace limap { void bind_evaluator(py::module& m) { using namespace evaluation; - py::class_(m, "PointCloudEvaluator") - .def(py::init<>()) - .def(py::init&>()) - .def(py::init()) - .def("Build", &PointCloudEvaluator::Build) - .def("Save", &PointCloudEvaluator::Save) - .def("Load", &PointCloudEvaluator::Load) - .def("ComputeDistPoint", &PointCloudEvaluator::ComputeDistPoint) + py::class_(m, "PointCloudEvaluator", "The evaluator for line maps with respect to a GT point cloud (using a K-D Tree).") + .def(py::init<>(), R"( + Default constructor + )") + .def(py::init&>(), R"( + Constructor from list[:class:`np.array`] of shape (3,) + )") + .def(py::init(), R"( + Constructor from :class:`np.array` of shape (N, 3) + )") + .def("Build", &PointCloudEvaluator::Build, R"(Build the indexes of the K-D Tree)") + .def("Save", &PointCloudEvaluator::Save, R"( + Save the built K-D Tree into a file + + Args: + filename (str): The file to write to + )") + .def("Load", &PointCloudEvaluator::Load, R"( + Read the pre-built K-D Tree from a file + + Args: + filename (str): The file to read from + )") + .def("ComputeDistPoint", &PointCloudEvaluator::ComputeDistPoint, R"( + Compute the distance from a query point to the point cloud + + Args: + :class:`np.array` of shape (3,): The query point + + Returns: + float: The distance from the point to the GT point cloud + )") .def("ComputeDistLine", [] (PointCloudEvaluator& self, const Line3d& line, int n_samples) { return self.ComputeDistLine(line, n_samples); }, + R"( + Compute the distance for a set of uniformly sampled points along the line + + Args: + line (Line3d): :class:`~limap.base.Line3d`: instance + n_samples (int, optional): number of samples (default = 1000) + + Returns: + :class:`np.array` of shape (n_samples,): the computed distances + + )", py::arg("line"), py::arg("n_samples") = 1000 ) @@ -43,6 +78,17 @@ void bind_evaluator(py::module& m) { int n_samples) { return self.ComputeInlierRatio(line, threshold, n_samples); }, + R"( + Compute the percentage of the line lying with a certain threshold to the point cloud + + Args: + line (Line3d): :class:`~limap.base.Line3d`: instance + threshold (float): threshold + n_samples (int, optional): number of samples (default = 1000) + + Returns: + float: The computed percentage + )", py::arg("line"), py::arg("threshold"), py::arg("n_samples") = 1000 @@ -54,6 +100,18 @@ void bind_evaluator(py::module& m) { int n_samples) { return self.ComputeInlierSegs(lines, threshold, n_samples); }, + R"( + Compute the inlier parts of the lines that are within a certain threshold to the point cloud, for visualization. + + Args: + lines (list[:class:`limap.base.Line3d`]): Input 3D line segments + threshold (float): threshold + n_samples (int): number of samples (default = 1000) + + Returns: + list[:class:`limap.base.Line3d`]: Inlier parts of all the lines, useful for visualization + + )", py::arg("lines"), py::arg("threshold"), py::arg("n_samples") = 1000 @@ -65,6 +123,18 @@ void bind_evaluator(py::module& m) { int n_samples) { return self.ComputeOutlierSegs(lines, threshold, n_samples); }, + R"( + Compute the outlier parts of the lines that are at least a certain threshold far away from the point cloud, for visualization. + + Args: + lines (list[:class:`limap.base.Line3d`]): Input 3D line segments + threshold (float): threshold + n_samples (int): number of samples (default = 1000) + + Returns: + list[:class:`limap.base.Line3d`]: Outlier parts of all the lines, useful for visualization + + )", py::arg("lines"), py::arg("threshold"), py::arg("n_samples") = 1000 @@ -73,15 +143,38 @@ void bind_evaluator(py::module& m) { .def("ComputeDistsforEachPoint_KDTree", &PointCloudEvaluator::ComputeDistsforEachPoint_KDTree); py::class_(m, "MeshEvaluator") - .def(py::init<>()) - .def(py::init()) - .def("ComputeDistPoint", &MeshEvaluator::ComputeDistPoint) + .def(py::init<>(), R"( + Default constructor + )") + .def(py::init(), R"( + Constructor from a mesh file (str) and a scale (float) + )") + .def("ComputeDistPoint", &MeshEvaluator::ComputeDistPoint, R"( + Compute the distance from a query point to the mesh + + Args: + :class:`np.array` of shape (3,): The query point + + Returns: + float: The distance from the point to the GT mesh + )") .def("ComputeDistLine", [] (MeshEvaluator& self, const Line3d& line, int n_samples) { return self.ComputeDistLine(line, n_samples); }, + R"( + Compute the distance for a set of uniformly sampled points along the line + + Args: + line (Line3d): :class:`~limap.base.Line3d`: instance + n_samples (int, optional): number of samples (default = 1000) + + Returns: + :class:`np.array` of shape (n_samples,): the computed distances + + )", py::arg("line"), py::arg("n_samples") = 1000 ) @@ -92,6 +185,17 @@ void bind_evaluator(py::module& m) { int n_samples) { return self.ComputeInlierRatio(line, threshold, n_samples); }, + R"( + Compute the percentage of the line lying with a certain threshold to the mesh + + Args: + line (Line3d): :class:`~limap.base.Line3d`: instance + threshold (float): threshold + n_samples (int, optional): number of samples (default = 1000) + + Returns: + float: The computed percentage + )", py::arg("line"), py::arg("threshold"), py::arg("n_samples") = 1000 @@ -103,6 +207,19 @@ void bind_evaluator(py::module& m) { int n_samples) { return self.ComputeInlierSegs(lines, threshold, n_samples); }, + R"( + Compute the inlier parts of the lines that are within a certain threshold to the mesh, for visualization. + + Args: + lines (list[:class:`limap.base.Line3d`]): Input 3D line segments + threshold (float): threshold + n_samples (int): number of samples (default = 1000) + + Returns: + list[:class:`limap.base.Line3d`]: Inlier parts of all the lines, useful for visualization + + )", + py::arg("lines"), py::arg("threshold"), py::arg("n_samples") = 1000 @@ -114,6 +231,18 @@ void bind_evaluator(py::module& m) { int n_samples) { return self.ComputeOutlierSegs(lines, threshold, n_samples); }, + R"( + Compute the outlier parts of the lines that are at least a certain threshold far away from the mesh, for visualization. + + Args: + lines (list[:class:`limap.base.Line3d`]): Input 3D line segments + threshold (float): threshold + n_samples (int): number of samples (default = 1000) + + Returns: + list[:class:`limap.base.Line3d`]: Outlier parts of all the lines, useful for visualization + + )", py::arg("lines"), py::arg("threshold"), py::arg("n_samples") = 1000 diff --git a/limap/line2d/base_detector.py b/limap/line2d/base_detector.py index f3c07f6b..948be7a4 100644 --- a/limap/line2d/base_detector.py +++ b/limap/line2d/base_detector.py @@ -6,12 +6,28 @@ import limap.util.io as limapio import limap.visualize as limapvis -from collections import namedtuple -BaseDetectorOptions = namedtuple("BaseDetectorOptions", - ["set_gray", "max_num_2d_segs", "do_merge_lines", "visualize", "weight_path"], - defaults=[True, 3000, False, False, None]) +import collections +from typing import NamedTuple +class BaseDetectorOptions(NamedTuple): + """ + Base options for the line detector + + :param set_gray: whether to set the image to gray scale (sometimes depending on the detector) + :param max_num_2d_segs: maximum number of detected line segments (default = 3000) + :param do_merge_lines: whether to merge close similar lines at post-processing (default = False) + :param visualize: whether to output visualizations into output folder along with the detections (default = False) + :param weight_path: specify path to load weights (at default, weights will be downloaded to ~/.local) + """ + set_gray: bool = True + max_num_2d_segs: int = 3000 + do_merge_lines: bool = False + visualize: bool = False + weight_path: str = None class BaseDetector(): + """ + Virtual class for line detector + """ def __init__(self, options = BaseDetectorOptions()): self.set_gray = options.set_gray self.max_num_2d_segs = options.max_num_2d_segs @@ -21,28 +37,106 @@ def __init__(self, options = BaseDetectorOptions()): # Module name needs to be set def get_module_name(self): + """ + Virtual method (need to be implemented) - return the name of the module + """ raise NotImplementedError # The functions below are required for detectors def detect(self, camview): + """ + Virtual method (for detector) - detect 2D line segments + + Args: + view (:class:`limap.base.CameraView`): The `limap.base.CameraView` instance corresponding to the image + Returns: + :class:`np.array` of shape (N, 5): line detections. Each row corresponds to x1, y1, x2, y2 and score. + """ raise NotImplementedError # The functions below are required for extractors def extract(self, camview, segs): + """ + Virtual method (for extractor) - extract the features for the detected segments + + Args: + view (:class:`limap.base.CameraView`): The `limap.base.CameraView` instance corresponding to the image + segs: :class:`np.array` of shape (N, 5), line detections. Each row corresponds to x1, y1, x2, y2 and score. Computed from the `detect` method. + Returns: + The extracted feature + """ raise NotImplementedError def get_descinfo_fname(self, descinfo_folder, img_id): + """ + Virtual method (for extractor) - Get the target filename of the extracted feature + + Args: + descinfo_folder (str): The output folder + img_id (int): The image id + Returns: + str: target filename + """ raise NotImplementedError def save_descinfo(self, descinfo_folder, img_id, descinfo): + """ + Virtual method (for extractor) - Save the extracted feature to the target folder + + Args: + descinfo_folder (str): The output folder + img_id (int): The image id + descinfo: The features extracted from the function `extract` + """ raise NotImplementedError def read_descinfo(self, descinfo_folder, img_id): + """ + Virtual method (for extractor) - Read in the extracted feature. Dual function for `save_descinfo`. + + Args: + descinfo_folder (str): The output folder + img_id (int): The image id + Returns: + The extracted feature + """ raise NotImplementedError # The functions below are required for double-functioning objects def detect_and_extract(self, camview): + """ + Virtual method (for dual-functional class that can perform both detection and extraction) - Detect and extract on a single image + + Args: + view (:class:`limap.base.CameraView`): The `limap.base.CameraView` instance corresponding to the image + Returns: + segs (:class:`np.array`): of shape (N, 5), line detections. Each row corresponds to x1, y1, x2, y2 and score. Computed from the `detect` method. + descinfo: The features extracted from the function `extract` + """ raise NotImplementedError - def sample_descinfo_by_indexes(descinfo): + def sample_descinfo_by_indexes(self, descinfo, indexes): + """ + Virtual method (for dual-functional class that can perform both detection and extraction) - sample descriptors for a subset of images + + Args: + descinfo: The features extracted from the function `extract`. + indexes (list[int]): List of image ids for the subset. + """ raise NotImplementedError def get_segments_folder(self, output_folder): + """ + Return the folder path to the detected segments + + Args: + output_folder (str): The output folder + Returns: + path_to_segments (str): The path to the saved segments + """ return os.path.join(output_folder, "segments") def get_descinfo_folder(self, output_folder): + """ + Return the folder path to the extracted descriptors + + Args: + output_folder (str): The output folder + Returns: + path_to_descinfos (str): The path to the saved descriptors + """ return os.path.join(output_folder, "descinfos", self.get_module_name()) def merge_lines(self, segs): @@ -51,6 +145,7 @@ def merge_lines(self, segs): segs = merge_lines(segs) segs = segs.reshape(-1, 4) return segs + def take_longest_k(self, segs, max_num_2d_segs=3000): indexes = np.arange(0, segs.shape[0]) if max_num_2d_segs is None or max_num_2d_segs == -1: @@ -75,6 +170,16 @@ def visualize_segs(self, output_folder, imagecols, first_k=10): cv2.imwrite(fname, img) def detect_all_images(self, output_folder, imagecols, skip_exists=False): + """ + Perform line detection on all images and save the line segments + + Args: + output_folder (str): The output folder + imagecols (:class:`limap.base.ImageCollection`): The input image collection + skip_exists (bool): Whether to skip already processed images + Returns: + dict[int -> :class:`np.array`]: The line detection for each image indexed by the image id. Each segment is with shape (N, 5). Each row corresponds to x1, y1, x2, y2 and score. + """ seg_folder = self.get_segments_folder(output_folder) if not skip_exists: limapio.delete_folder(seg_folder) @@ -102,6 +207,17 @@ def detect_all_images(self, output_folder, imagecols, skip_exists=False): return all_2d_segs def extract_all_images(self, output_folder, imagecols, all_2d_segs, skip_exists=False): + """ + Perform line descriptor extraction on all images and save the descriptors. + + Args: + output_folder (str): The output folder. + imagecols (:class:`limap.base.ImageCollection`): The input image collection + all_2d_segs (dict[int -> :class:`np.array`]): The line detection for each image indexed by the image id. Each segment is with shape (N, 5). Each row corresponds to x1, y1, x2, y2 and score. Computed from `detect_all_images` + skip_exists (bool): Whether to skip already processed images. + Returns: + descinfo_folder (str): The path to the saved descriptors. + """ descinfo_folder = self.get_descinfo_folder(output_folder) if not skip_exists: limapio.delete_folder(descinfo_folder) @@ -114,6 +230,17 @@ def extract_all_images(self, output_folder, imagecols, all_2d_segs, skip_exists= return descinfo_folder def detect_and_extract_all_images(self, output_folder, imagecols, skip_exists=False): + """ + Perform line detection and description on all images and save the line segments and descriptors + + Args: + output_folder (str): The output folder + imagecols (:class:`limap.base.ImageCollection`): The input image collection + skip_exists (bool): Whether to skip already processed images + Returns: + all_segs (dict[int -> :class:`np.array`]): The line detection for each image indexed by the image id. Each segment is with shape (N, 5). Each row corresponds to x1, y1, x2, y2 and score. + descinfo_folder (str): Path to the extracted descriptors. + """ assert self.do_merge_lines == False seg_folder = self.get_segments_folder(output_folder) descinfo_folder = self.get_descinfo_folder(output_folder) diff --git a/limap/line2d/base_matcher.py b/limap/line2d/base_matcher.py index 39bbef4e..442fffff 100644 --- a/limap/line2d/base_matcher.py +++ b/limap/line2d/base_matcher.py @@ -4,12 +4,26 @@ import joblib import limap.util.io as limapio -from collections import namedtuple -BaseMatcherOptions = namedtuple("BaseMatcherOptions", - ["topk", "n_neighbors", "n_jobs", "weight_path"], - defaults=[10, 20, 1, None]) +import collections +from typing import NamedTuple +class BaseMatcherOptions(NamedTuple): + """ + Base options for the line matcher + + :param topk: number of top matches for each line (if equal to 0, do mutual nearest neighbor matching) + :param n_neighbors: number of visual neighbors, only for naming the output folder + :param n_jobs: number of jobs at multi-processing (please make sure not to exceed the GPU memory limit with learning methods) + :param weight_path: specify path to load weights (at default, weights will be downloaded to ~/.local) + """ + topk: int = 10 + n_neighbors: int = 20 + n_jobs: int = 1 + weight_path: str = None class BaseMatcher(): + """ + Virtual class for line matcher + """ def __init__(self, extractor, options = BaseMatcherOptions()): self.extractor = extractor self.topk = options.topk @@ -19,25 +33,75 @@ def __init__(self, extractor, options = BaseMatcherOptions()): # The functions below are required for matchers def get_module_name(self): + """ + Virtual method (need to be implemented) - return the name of the module + """ raise NotImplementedError def match_pair(self, descinfo1, descinfo2): + """ + Virtual method (need to be implemented) - match two set of lines based on the descriptors + """ raise NotImplementedError def get_matches_folder(self, output_folder): + """ + Return the folder path to the output matches + + Args: + output_folder (str): The output folder + Returns: + path_to_matches (str): The path to the saved matches + """ return os.path.join(output_folder, "{0}_n{1}_top{2}".format(self.get_module_name(), self.n_neighbors, self.topk)) def read_descinfo(self, descinfo_folder, idx): return self.extractor.read_descinfo(descinfo_folder, idx) def get_match_filename(self, matches_folder, idx): + """ + Return the filename of the matches specified by an image id + + Args: + matches_folder (str): The output matching folder + idx (int): image id + """ fname = os.path.join(matches_folder, "matches_{0}.npy".format(idx)) return fname def save_match(self, matches_folder, idx, matches): + """ + Save the output matches from one image to its neighbors + + Args: + matches_folder (str): The output matching folder + idx (int): image id + matches (dict[int -> :class:`np.array`]): The output matches for each neighboring image, each with shape (N, 2) + """ fname = self.get_match_filename(matches_folder, idx) limapio.save_npy(fname, matches) def read_match(self, matches_folder, idx): + """ + Read the matches for one image with its neighbors + + Args: + matches_folder (str): The output matching folder + idx (int): image id + Returns: + matches (dict[int -> :class:`np.array`]): The output matches for each neighboring image, each with shape (N, 2) + """ fname = self.get_match_filename(matches_folder, idx) - return limapio.read_npy(fname) + return limapio.read_npy(fname).item() def match_all_neighbors(self, output_folder, image_ids, neighbors, descinfo_folder, skip_exists=False): + """ + Match all images with its visual neighbors + + Args: + output_folder (str): The output folder + image_ids (list[int]): list of image ids + neighbors (dict[int -> list[int]]): visual neighbors for each image + descinfo_folder (str): The folder storing all the descriptors + skip_exists (bool): Whether to skip already processed images + Returns: + matches_folder: The output matching folder + """ matches_folder = self.get_matches_folder(output_folder) if not skip_exists: limapio.delete_folder(matches_folder) @@ -59,6 +123,17 @@ def process(self, matches_folder, descinfo_folder, img_id, ng_img_id_list, skip_ return matches_folder def match_all_exhaustive_pairs(self, output_folder, image_ids, descinfo_folder, skip_exists=False): + """ + Match all images exhaustively + + Args: + output_folder (str): The output folder + image_ids (list[int]): list of image ids + descinfo_folder (str): The folder storing all the descriptors + skip_exists (bool): Whether to skip already processed images + Returns: + matches_folder: The output matching folder + """ matches_folder = self.get_matches_folder(output_folder) if not skip_exists: limapio.delete_folder(matches_folder) diff --git a/limap/line2d/register_detector.py b/limap/line2d/register_detector.py index 5920fd16..a76a7670 100644 --- a/limap/line2d/register_detector.py +++ b/limap/line2d/register_detector.py @@ -2,6 +2,12 @@ def get_detector(cfg_detector, max_num_2d_segs=3000, do_merge_lines=False, visualize=False, weight_path=None): + """ + Get a line detector specified by cfg_detector["method"] + + Args: + cfg_detector: config for the line detector + """ options = BaseDetectorOptions() options = options._replace( set_gray=True, max_num_2d_segs=max_num_2d_segs, @@ -27,6 +33,12 @@ def get_detector(cfg_detector, max_num_2d_segs=3000, raise NotImplementedError def get_extractor(cfg_extractor, weight_path=None): + """ + Get a line descriptor speicified by cfg_extractor["method"] + + Args: + cfg_extractor: config for the line extractor + """ options = BaseDetectorOptions() options = options._replace(set_gray=True, weight_path=weight_path) diff --git a/limap/line2d/register_matcher.py b/limap/line2d/register_matcher.py index 9e5910b7..11d53ebb 100644 --- a/limap/line2d/register_matcher.py +++ b/limap/line2d/register_matcher.py @@ -1,6 +1,13 @@ from .base_matcher import BaseMatcherOptions def get_matcher(cfg_matcher, extractor, n_neighbors=20, weight_path=None): + """ + Get a line matcher specified by cfg_matcher["method"] + + Args: + cfg_matcher: config for line matcher + extractor: line extractor inherited from :class:`limap.line2d.base_matcher.BaseMatcher` + """ options = BaseMatcherOptions() options = options._replace( n_neighbors=n_neighbors, topk=cfg_matcher["topk"], diff --git a/limap/pointsfm/colmap_sfm.py b/limap/pointsfm/colmap_sfm.py index e18623fd..70668165 100644 --- a/limap/pointsfm/colmap_sfm.py +++ b/limap/pointsfm/colmap_sfm.py @@ -71,14 +71,9 @@ def run_hloc_matches(cfg, image_path, db_path, keypoints=None, neighbors=None, i feature_conf = extract_features.confs[cfg["descriptor"]] matcher_conf = match_features.confs[cfg["matcher"]] - if keypoints is not None and keypoints != []: - if cfg["descriptor"][:10] != "superpoint": - raise ValueError("Error! Non-superpoint feature extraction is unfortunately not supported in the current implementation.") - # run superpoint - from limap.point2d import run_superpoint - feature_path = run_superpoint(feature_conf, image_path, outputs, keypoints=keypoints) - else: - feature_path = extract_features.main(feature_conf, image_path, outputs) + # run superpoint + from limap.point2d import run_superpoint + feature_path = run_superpoint(feature_conf, image_path, outputs, keypoints=keypoints) if neighbors is None or imagecols is None: # run exhaustive matches sfm_pairs = outputs / "pairs-exhaustive.txt" @@ -92,8 +87,7 @@ def run_hloc_matches(cfg, image_path, db_path, keypoints=None, neighbors=None, i sfm_dir.mkdir(parents=True, exist_ok=True) reconstruction.create_empty_db(db_path) if imagecols is None: - import pycolmap - reconstruction.import_images(image_dir=image_path, database_path=db_path, camera_mode=pycolmap.CameraMode.AUTO) + reconstruction.import_images(image_dir=image_path, database_path=db_path) else: # use the id mapping from imagecols import_images_with_known_cameras(image_path, db_path, imagecols) # use cameras and id mapping @@ -206,5 +200,5 @@ def run_colmap_sfm_with_known_poses(cfg, imagecols, output_path='tmp/tmp_colmap' for img_id in imagecols.get_img_ids(): colmap_images[img_id] = colmap_images[img_id]._replace(name = imagecols.image_name(img_id)) colmap_utils.write_images_binary(colmap_images, fname_images_bin) - return Path(point_triangulation_path) + return Path(point_triangulation_path) diff --git a/limap/runners/functions.py b/limap/runners/functions.py index e0a9cae0..372bef8b 100644 --- a/limap/runners/functions.py +++ b/limap/runners/functions.py @@ -22,12 +22,23 @@ def setup(cfg): print("[LOG] weight dir: {0}".format(cfg["weight_path"])) return cfg -def undistort_images(imagecols, output_dir, fname="image_collection_undistorted.npy", load_undistort=False, n_jobs=-1): +def undistort_images(imagecols, output_dir, fname="image_collection_undistorted.npy", skip_exists=False, n_jobs=-1): + """ + Run undistortion on the images stored in the :class:`limap.base.ImageCollection` instance `imagecols` (only distorted images are undistorted), and store the undistorted images into `output_dir`. The function will return a corresponding `limap.base.ImageCollection` instance for the undistorted images. + + Args: + imagecols (:class:`limap.base.ImageCollection`): Image collection of the images to be undistorted. + output_dir (str): output folder for storing the undistorted images + skip_exists (bool): whether to skip already undistorted images in the output folder. + + Returns: + :class:`limap.base.ImageCollection`: New image collection for the undistorted images + """ import limap.base as _base loaded_ids = [] unload_ids = imagecols.get_img_ids() - if load_undistort: + if skip_exists: print("[LOG] Loading undistorted images (n_images = {0})...".format(imagecols.NumImages())) fname_in = os.path.join(output_dir, fname) if os.path.isfile(fname_in): @@ -96,6 +107,17 @@ def process(imagecols, img_id): return imagecols_undistorted def compute_sfminfos(cfg, imagecols, fname="metainfos.txt"): + """ + Compute visual neighbors and robust 3D ranges from COLMAP point triangulation. + + Args: + cfg (dict): Configuration, fields refer to :file:`cfgs/examples/pointsfm.yaml` as a minimal example + imagecols (:class:`limap.base.ImageCollection`): image collection for the images of interest, storing intrinsics and triangulated poses + Returns: + colmap_output_path (str): path to store the colmap output + neighbors (dict[int -> list[int]]): visual neighbors for each image + ranges (pair of :class:`np.array`, each of shape (3,)): robust 3D ranges for the scene computed from the sfm point cloud. + """ import limap.pointsfm as _psfm if not cfg["load_meta"]: # run colmap sfm and compute neighbors, ranges @@ -118,6 +140,17 @@ def compute_sfminfos(cfg, imagecols, fname="metainfos.txt"): return colmap_output_path, neighbors, ranges def compute_2d_segs(cfg, imagecols, compute_descinfo=True): + """ + Detect and desribe 2D lines for each image in the image collection + + Args: + cfg (dict): Configuration, fields refer to :file:`cfgs/examples/line2d_detect.yaml` as a minimal example + imagecols (:class:`limap.base.ImageCollection`): image collection for the images of interest + compute_descinfo (bool, optional, default=True): whether to extract the line descriptors + Returns: + all_2d_segs (dict[int -> :class:`np.array`], each with shape (N, 4) or (N, 5)): all the line detections for each image + descinfo_folder (str): folder to store the descriptors + """ weight_path = None if "weight_path" not in cfg else cfg["weight_path"] if "extractor" in cfg["line2d"]: print("[LOG] Start 2D line detection and description (detector = {0}, extractor = {1}, n_images = {2})...".format(cfg["line2d"]["detector"]["method"], cfg["line2d"]["extractor"]["method"], imagecols.NumImages())) @@ -154,6 +187,17 @@ def compute_2d_segs(cfg, imagecols, compute_descinfo=True): return all_2d_segs, descinfo_folder def compute_matches(cfg, descinfo_folder, image_ids, neighbors): + """ + Match lines for each image with its visual neighbors + + Args: + cfg (dict): Configuration, fields refeer to :file:`cfgs/examples/line2d_match.yaml` as a minimal example + descinfo_folder (str): path to store the descriptors + image_ids (list[int]): list of image ids + neighbors (dict[int -> list[int]]): visual neighbors for each image + Returns: + matches_folder (str): path to store the computed matches + """ weight_path = None if "weight_path" not in cfg else cfg["weight_path"] print("[LOG] Start matching 2D lines... (extractor = {0}, matcher = {1}, n_images = {2}, n_neighbors = {3})".format(cfg["line2d"]["extractor"]["method"], cfg["line2d"]["matcher"]["method"], len(image_ids), cfg["n_neighbors"])) import limap.line2d @@ -170,6 +214,16 @@ def compute_matches(cfg, descinfo_folder, image_ids, neighbors): return matches_folder def compute_exhausive_matches(cfg, descinfo_folder, image_ids): + """ + Match lines for each image with all the other images exhaustively + + Args: + cfg (dict): Configuration, fields refeer to :file:`cfgs/examples/line2d_match.yaml` as a minimal example + descinfo_folder (str): path to store the descriptors + image_ids (list[int]): list of image ids + Returns: + matches_folder (str): path to store the computed matches + """ print("[LOG] Start exhausive matching 2D lines... (extractor = {0}, matcher = {1}, n_images = {2})".format(cfg["line2d"]["extractor"]["method"], cfg["line2d"]["matcher"]["method"], len(image_ids))) import limap.line2d basedir = os.path.join("line_matchings", cfg["line2d"]["detector"]["method"], "feats_{0}".format(cfg["line2d"]["extractor"]["method"])) diff --git a/limap/runners/line_fitnmerge.py b/limap/runners/line_fitnmerge.py index fd531e19..b3f360e7 100644 --- a/limap/runners/line_fitnmerge.py +++ b/limap/runners/line_fitnmerge.py @@ -13,10 +13,15 @@ def fit_3d_segs(all_2d_segs, imagecols, depths, fitting_config): ''' + Fit 3D line segments over points produced by depth unprojection + Args: - - all_2d_segs: map - - imagecols: limap.base.ImageCollection - - depths: map, where CustomizedDepthReader inherits _base.BaseDepthReader + all_2d_segs (dict[int -> :class:`np.adarray`]): All the 2D line segments for each image + imagecols (:class:`limap.base.ImageCollection`): The image collection of all images of interest + depths (dict[int -> :class:`CustomizedDepthReader`], where :class:`CustomizedDepthReader` inherits :class:`limap.base.depth_reader_base.BaseDepthReader`): The depth map readers for each image + fitting_config (dict): Configuration, fields refer to :file:`cfgs/examples/fitting_3Dline.yaml` + Returns: + output (dict[int -> list[(:class:`np.array`, :class:`np.array`)]]): for each image, output a list of :class:`np.array` pair, representing two endpoints ''' n_images = len(all_2d_segs) seg3d_list = [] @@ -40,10 +45,15 @@ def process(all_2d_segs, imagecols, depths, fitting_config, img_id): def fit_3d_segs_with_points3d(all_2d_segs, imagecols, p3d_reader, fitting_config, inloc_dataset=None): ''' + Fit 3D line segments over a set of 3D points + Args: - - all_2d_segs: map - - imagecols: limap.base.ImageCollection - - p3d_reader: CustomizedP3Dreader inherits _base.BaseP3Dreader + all_2d_segs (dict[int -> :class:`np.adarray`]): All the 2D line segments for each image + imagecols (:class:`limap.base.ImageCollection`): The image collection of all images of interest + p3d_reader (dict[int -> :class:`CustomizedP3DReader`], where :class:`CustomizedP3DReader` inherits :class:`limap.base.p3d_reader_base.BaseP3DReader`): The point cloud readers for each image + fitting_config (dict): Configuration, fields refer to :file:`cfgs/examples/fitting_3Dline.yaml` + Returns: + output (dict[int -> list[(:class:`np.array`, :class:`np.array`)]]): for each image, output a list of :class:`np.array` pair, representing two endpoints ''' seg3d_list = [] def process(all_2d_segs, imagecols, p3d_reader, fitting_config, img_id): @@ -66,9 +76,16 @@ def process(all_2d_segs, imagecols, p3d_reader, fitting_config, img_id): def line_fitnmerge(cfg, imagecols, depths, neighbors=None, ranges=None): ''' + Line reconstruction over multi-view RGB images given depths + Args: - - imagecols: limap.base.ImageCollection - - depths: map, where CustomizedDepthReader inherits _base.BaseDepthReader + cfg (dict): Configuration. Fields refer to :file:`cfgs/fitnmerge/default.yaml` as an example + imagecols (:class:`limap.base.ImageCollection`): The image collection corresponding to all the images of interest + depths (dict[int -> :class:`CustomizedDepthReader`], where :class:`CustomizedDepthReader` inherits :class:`limap.base.depth_reader_base.BaseDepthReader`): The depth map readers for each image + neighbors (dict[int -> list[int]], optional): visual neighbors for each image. By default we compute neighbor information from the covisibility of COLMAP triangulation. + ranges (pair of :class:`np.array` each of shape (3,), optional): robust 3D ranges for the scene. By default we compute range information from the COLMAP triangulation. + Returns: + list[:class:`limap.base.LineTrack`]: list of output 3D line tracks ''' # assertion check assert imagecols.IsUndistorted() == True @@ -165,9 +182,16 @@ def line_fitnmerge(cfg, imagecols, depths, neighbors=None, ranges=None): def line_fitting_with_3Dpoints(cfg, imagecols, p3d_readers, inloc_read_transformations=False): ''' + Line reconstruction over multi-view images with its point cloud + Args: - - imagecols: limap.base.ImageCollection - - p3d_readers: map, where CustomizedP3DReader inherits _base.P3DReader + cfg (dict): Configuration. Fields refer to :file:`cfgs/fitnmerge/default.yaml` as an example + imagecols (:class:`limap.base.ImageCollection`): The image collection corresponding to all the images of interest + p3d_reader (dict[int -> :class:`CustomizedP3DReader`], where :class:`CustomizedP3DReader` inherits :class:`limap.base.p3d_reader_base.BaseP3DReader`): The point cloud readers for each image + neighbors (dict[int -> list[int]], optional): visual neighbors for each image. By default we compute neighbor information from the covisibility of COLMAP triangulation. + ranges (pair of :class:`np.array` each of shape (3,), optional): robust 3D ranges for the scene. By default we compute range information from the COLMAP triangulation. + Returns: + list[:class:`limap.base.LineTrack`]: list of output 3D line tracks ''' # assertion check assert imagecols.IsUndistorted() == True @@ -209,3 +233,4 @@ def line_fitting_with_3Dpoints(cfg, imagecols, p3d_readers, inloc_read_transform track = _base.LineTrack(l3d, [img_id], [line_id], [l2d]) linetracks.append(track) return linetracks + diff --git a/limap/runners/line_localization.py b/limap/runners/line_localization.py index 626fba05..9c7a69cb 100644 --- a/limap/runners/line_localization.py +++ b/limap/runners/line_localization.py @@ -68,29 +68,27 @@ def get_hloc_keypoints_from_log(logs, query_img_name, ref_sfm=None, resize_scale return p2ds, p3ds, inliers -def line_localization(cfg, imagecols_db, imagecols_query, point_corresp, linemap_db, retrieval, results_path, - img_name_dict=None, logger=None): +def line_localization(cfg, imagecols_db, imagecols_query, point_corresp, linemap_db, retrieval, results_path, img_name_dict=None, logger=None): """ Run visual localization on query images with `imagecols`, it takes 2D-3D point correspondences from HLoc; runs line matching using 2D line matcher ("epipolar" for Gao et al. "Pose Refinement with Joint Optimization of Visual Points and Lines"); - calls limap.estimators.pl_estimate_absolute_pose to estimate the absolute camera pose for all query images, + calls :func:`~limap.estimators.absolute_pose.pl_estimate_absolute_pose` to estimate the absolute camera pose for all query images, and writes results in results file in `results_path`. - :param cfg: dict, configurations which fields refer to `cfgs/localization/default.yaml` - :param imagecols_db: limap.base.ImageCollection of database images, with triangulated camera poses - :param imagecols_query: limap.base.ImageCollection of query images, camera poses only used for epipolar matcher/filter as coarse poses, - can be left uninitialized otherwise - :param point_corresp: dict, map query image IDs to extracted point correspondences for the query image, - point correspondences for each image ID stored as a dict with keys 'p2ds', 'p3ds', and optionally 'inliers' - :param linemap_db: iterable of limap.base.LineTrack, LIMAP triangulated/fitted database line tracks - :param retrieval: dict, map query image file path to list of neighbor image file paths, - e.g. returned from hloc.utils.parsers.parse_retrieval - :param results_path: str or Path, file path to write the localization results - :param img_name_dict: dict (optional), map query image IDs to the image file path, by default the image names from `imagecols` - :param logger: logging.Logger (optional), print logs for more information - - :return: Dict, mapping of query image IDs to the localized camera poses for all query images. - """ + Args: + cfg (dict): Configuration, fields refer to :file:`cfgs/localization/default.yaml` + imagecols_db (:class:`limap.base.ImageCollection`): Image collection of database images, with triangulated camera poses + imagecols_query (:class:`limap.base.ImageCollection`): Image collection of query images, camera poses only used for epipolar matcher/filter as coarse poses, can be left uninitialized otherwise + linemap_db (list[:class:`limap.base.LineTrack`]): LIMAP triangulated/fitted line tracks + retrieval (dict): Mapping of query image file path to list of neighbor image file paths, e.g. returned from :func:`hloc.utils.parsers.parse_retrieval` + results_path (str | Path): File path to write the localization results + img_name_dict(dict, optional): Mapping of query image IDs to the image file path, by default the image names from `imagecols` + logger (:class:`logging.Logger`, optional): Logger to print logs for information + + Returns: + Dict[int -> :class:`limap.base.CameraPose`]: Mapping of query image IDs to the localized camera poses for all query images. + """ + if cfg['localization']['2d_matcher'] not in ['epipolar', 'sold2', 'superglue_endpoints', 'gluestick', 'linetr', 'lbd', 'l2d2']: raise ValueError("Unknown 2d line matcher: {}".format(cfg['localization']['2d_matcher'])) diff --git a/limap/runners/line_triangulation.py b/limap/runners/line_triangulation.py index cc0cbd22..8e0b24e5 100644 --- a/limap/runners/line_triangulation.py +++ b/limap/runners/line_triangulation.py @@ -14,8 +14,15 @@ def line_triangulation(cfg, imagecols, neighbors=None, ranges=None): ''' + Main interface of line triangulation over multi-view images. + Args: - - imagecols: limap.base.ImageCollection + cfg (dict): Configuration. Fields refer to :file:`cfgs/triangulation/default.yaml` as an example + imagecols (:class:`limap.base.ImageCollection`): The image collection corresponding to all the images of interest + neighbors (dict[int -> list[int]], optional): visual neighbors for each image. By default we compute neighbor information from the covisibility of COLMAP triangulation. + ranges (pair of :class:`np.array` each of shape (3,), optional): robust 3D ranges for the scene. By default we compute range information from the COLMAP triangulation. + Returns: + list[:class:`limap.base.LineTrack`]: list of output 3D line tracks ''' print("[LOG] Number of images: {0}".format(imagecols.NumImages())) cfg = _runners.setup(cfg) @@ -24,7 +31,7 @@ def line_triangulation(cfg, imagecols, neighbors=None, ranges=None): cfg["triangulation"]["var2d"] = cfg["var2d"][detector_name] # undistort images if not imagecols.IsUndistorted(): - imagecols = _runners.undistort_images(imagecols, os.path.join(cfg["dir_save"], cfg["undistortion_output_dir"]), load_undistort=cfg["load_undistort"] or cfg["skip_exists"], n_jobs=cfg["n_jobs"]) + imagecols = _runners.undistort_images(imagecols, os.path.join(cfg["dir_save"], cfg["undistortion_output_dir"]), skip_exists=cfg["load_undistort"] or cfg["skip_exists"], n_jobs=cfg["n_jobs"]) # resize cameras assert imagecols.IsUndistorted() == True if cfg["max_image_dim"] != -1 and cfg["max_image_dim"] is not None: diff --git a/limap/triangulation/triangulation.py b/limap/triangulation/triangulation.py index bec5a128..c9e79f17 100644 --- a/limap/triangulation/triangulation.py +++ b/limap/triangulation/triangulation.py @@ -6,29 +6,124 @@ def get_normal_direction(l, view): return _tri.get_normal_direction(l, view) def get_direction_from_VP(vp, view): + """ + Get the 3d direction from a 2D vanishing point + + Args: + vp (:class:`np.array` of shape (3,)) + view (:class:`limap.base.CameraView`) + Returns: + direction (:class:`np.array` of shape (3,)) + """ return _tri.get_direction_from_VP(vp, view) def compute_essential_matrix(view1, view2): + """ + Get the essential matrix between two views + + Args: + view1 (:class:`limap.base.CameraView`) + view2 (:class:`limap.base.CameraView`) + Returns: + essential_matrix (:class:`np.array` of shape (3, 3)) + """ return _tri.compute_essential_matrix(view1, view2) def compute_fundamental_matrix(view1, view2): + """ + Get the essential matrix between two views + + Args: + view1 (:class:`limap.base.CameraView`) + view2 (:class:`limap.base.CameraView`) + Returns: + fundamental_matrix (:class:`np.array` of shape (3, 3)) + """ return _tri.compute_fundamental_matrix(view1, view2) def compute_epipolar_IoU(l1, view1, l2, view2): + """ + Get the IoU between two lines from different views by intersecting the epipolar lines + + Args: + l1 (:class:`limap.base.Line2d`) + view1 (:class:`limap.base.CameraView`) + l2 (:class:`limap.base.Line2d`) + view2 (:class:`limap.base.CameraView`) + Returns: + IoU (float): The calculated epipolar IoU + """ return _tri.compute_epipolar_IoU(l1, view1, l2, view2) def point_triangulation(p1, view1, p2, view2): + """ + Two-view point triangulation (mid-point) + + Args: + p1 (:class:`np.array` of shape (2,)) + view1 (:class:`limap.base.CameraView`) + p2 (:class:`np.array` of shape (2,)) + view2 (:class:`limap.base.CameraView`) + Returns: + point3d (:class:`np.array` of shape (3,)) + """ return _tri.point_triangulation(p1, view1, p2, view2) def triangulate_endpoints(l1, view1, l2, view2): + """ + Two-view triangulation of lines with point triangulation on both endpoints (assuming correspondences) + + Args: + l1 (:class:`limap.base.Line2d`) + view1 (:class:`limap.base.CameraView`) + l2 (:class:`limap.base.Line2d`) + view2 (:class:`limap.base.CameraView`) + Returns: + line3d (:class:`limap.base.Line3d`) + """ return _tri.triangulate_endpoints(l1, view1, l2, view2) def triangulate(l1, view1, l2, view2): + """ + Two-view triangulation of lines by ray-plane intersection + + Args: + l1 (:class:`limap.base.Line2d`) + view1 (:class:`limap.base.CameraView`) + l2 (:class:`limap.base.Line2d`) + view2 (:class:`limap.base.CameraView`) + Returns: + line3d (:class:`limap.base.Line3d`) + """ return _tri.triangulate(l1, view1, l2, view2) def triangulate_with_one_point(l1, view1, l2, view2, p): + """ + Two-view triangulation of lines with a known 3D point on the line + + Args: + l1 (:class:`limap.base.Line2d`) + view1 (:class:`limap.base.CameraView`) + l2 (:class:`limap.base.Line2d`) + view2 (:class:`limap.base.CameraView`) + point (:class:`np.array` of shape (3,)) + Returns: + line3d (:class:`limap.base.Line3d`) + """ return _tri.triangulate_with_one_point(l1, view1, l2, view2, p) def triangulate_with_direction(l1, view1, l2, view2, direc): + """ + Two-view triangulation of lines with known 3D line direction + + Args: + l1 (:class:`limap.base.Line2d`) + view1 (:class:`limap.base.CameraView`) + l2 (:class:`limap.base.Line2d`) + view2 (:class:`limap.base.CameraView`) + direction (:class:`np.array` of shape (3,)) + Returns: + line3d (:class:`limap.base.Line3d`) + """ return _tri.triangulate_with_direction(l1, view1, l2, view2, direc) diff --git a/limap/undistortion/undistort.py b/limap/undistortion/undistort.py index 2cd6fd37..c396fe34 100644 --- a/limap/undistortion/undistort.py +++ b/limap/undistortion/undistort.py @@ -4,6 +4,17 @@ import copy def UndistortImageCamera(camera, imname_in, imname_out): + """ + Run COLMAP undistortion on one single image with an input camera. The undistortion is only applied if the camera model is neither "SIMPLE_PINHOLE" nor "PINHOLE". + + Args: + camera (:class:`limap.base.Camera`): The camera (type + parameters) for the image. + imname_in (str): filename for the input image + imname_out (str): filename for the output undistorted image + + Returns: + :class:`limap.base.Camera`: The undistorted camera + """ if camera.IsUndistorted(): # no distortion img = cv2.imread(imname_in) cv2.imwrite(imname_out, img) @@ -22,5 +33,16 @@ def UndistortImageCamera(camera, imname_in, imname_out): return camera_undistorted def UndistortPoints(points, distorted_camera, undistorted_camera): + """ + Run COLMAP undistortion on the keypoints. + + Args: + points (list[:class:`np.array`]): List of 2D keypoints on the distorted image + distorted_camera (:class:`limap.base.Camera`): The camera before undistortion + undistorted_camera (:class:`limap.base.Camera`): The camera after undistortion + + Returns: + list[:class:`np.array`]: List of the corresponding 2D keypoints on the undistorted image + """ return _undistortion._UndistortPoints(points, distorted_camera, undistorted_camera) diff --git a/limap/visualize/vis_lines.py b/limap/visualize/vis_lines.py index fed82ce2..e5ca875f 100644 --- a/limap/visualize/vis_lines.py +++ b/limap/visualize/vis_lines.py @@ -2,10 +2,13 @@ from .vis_utils import test_point_inside_ranges, test_line_inside_ranges def pyvista_vis_3d_lines(lines, img_hw=(600, 800), width=2, ranges=None, scale=1.0): - ''' - Input: - - lines: list of _base.Line3d - ''' + """ + Visualize a 3D line map with `PyVista `_ + + Args: + lines (list[:class:`limap.base.Line3d`]): The 3D line map + width (float, optional): width of the line + """ import pyvista as pv plotter = pv.Plotter(window_size=[img_hw[1], img_hw[0]]) for line in lines: @@ -122,6 +125,13 @@ def open3d_add_cameras(w, imagecols, color=[1.0, 0.0, 0.0], ranges=None, scale_c return w def open3d_vis_3d_lines(lines, width=2, ranges=None, scale=1.0): + """ + Visualize a 3D line map with `Open3D `_ + + Args: + lines (list[:class:`limap.base.Line3d`]): The 3D line map + width (float, optional): width of the line + """ import open3d as o3d vis = o3d.visualization.Visualizer() vis.create_window(height=1080, width=1920) diff --git a/limap/vplib/base_vp_detector.py b/limap/vplib/base_vp_detector.py index b9c4dcb6..3b8dff78 100644 --- a/limap/vplib/base_vp_detector.py +++ b/limap/vplib/base_vp_detector.py @@ -2,10 +2,15 @@ import joblib from tqdm import tqdm -from collections import namedtuple -BaseVPDetectorOptions = namedtuple("BaseVPDetectorOptions", - ["n_jobs"], - defaults = [1]) +import collections +from typing import NamedTuple +class BaseVPDetectorOptions(NamedTuple): + """ + Base options for the vanishing point detector + + :param n_jobs: number of jobs at multi-processing (please make sure not to exceed the GPU memory limit with learning methods) + """ + n_jobs: int = 1 class BaseVPDetector(): def __init__(self, options = BaseVPDetectorOptions()): @@ -13,18 +18,31 @@ def __init__(self, options = BaseVPDetectorOptions()): # Module name needs to be set def get_module_name(self): + """ + Virtual method (need to be implemented) - return the name of the module + """ raise NotImplementedError # The functions below are required for VP detectors def detect_vp(self, lines, camview=None): ''' - Input: - - lines type: std::vector - Output: - - vpresult type: limap.vplib.VPResult + Virtual method - detect vanishing points + + Args: + lines (list[:class:`limap.base.Line2d`]): list of input 2D lines. + camview (:class:`limap.base.CameraView`): optional, the `limap.base.CameraView` instance corresponding to the image. + Returns: + vpresult type: list[:class:`limap.vplib.VPResult`] ''' raise NotImplementedError def detect_vp_all_images(self, all_lines, camviews=None): + ''' + Detect vanishing points on multiple images with multiple processes + + Args: + all_lines (dict[int, list[:class:`limap.base.Line2d`]]): map storing all the lines for each image + camviews (dict[int, :class:`limap.base.CameraView`]): optional, the `limap.base.CameraView` instances, each corresponding to one image + ''' def process(self, lines): return self.detect_vp(lines) def process_camview(self, lines, camview): diff --git a/limap/vplib/register_vp_detector.py b/limap/vplib/register_vp_detector.py index b9b0242e..b37d798a 100644 --- a/limap/vplib/register_vp_detector.py +++ b/limap/vplib/register_vp_detector.py @@ -1,6 +1,12 @@ from .base_vp_detector import BaseVPDetectorOptions def get_vp_detector(cfg_vp_detector, n_jobs=1): + ''' + Get a vanishing point detector specified by cfg_vp_detector["method"] + + Args: + cfg_vp_detector: config for the vanishing point detector + ''' options = BaseVPDetectorOptions() options = options._replace(n_jobs = n_jobs) diff --git a/runners/cambridge/utils.py b/runners/cambridge/utils.py index 1e7724ff..e27e8ea4 100644 --- a/runners/cambridge/utils.py +++ b/runners/cambridge/utils.py @@ -63,7 +63,7 @@ def undistort_and_resize(cfg, imagecols, logger=None): # undistort images logger.info('Performing undistortion...') if not imagecols.IsUndistorted(): - imagecols = _runners.undistort_images(imagecols, os.path.join(cfg['output_dir'], cfg['undistortion_output_dir']), load_undistort=cfg['load_undistort'] or cfg['skip_exists'], n_jobs=cfg['n_jobs']) + imagecols = _runners.undistort_images(imagecols, os.path.join(cfg['output_dir'], cfg['undistortion_output_dir']), skip_exists=cfg['load_undistort'] or cfg['skip_exists'], n_jobs=cfg['n_jobs']) image_dir = cfg['undistortion_output_dir'] if cfg['max_image_dim'] != -1 and cfg['max_image_dim'] is not None: image_dir = cfg['resized_output_dir'] @@ -169,7 +169,7 @@ def eval(filename, poses_gt, query_ids, id_to_name, logger): out += f'\n\t{th_t*100:.0f}cm, {th_R:.0f}deg : {ratio*100:.2f}%' logger.info(out) -def run_hloc_cambridge(cfg, image_dir, imagecols, neighbors, train_ids, query_ids, id_to_origin_name, +def run_hloc_cambridge(cfg, image_dir, imagecols, neighbors, train_ids, query_ids, id_to_origin_name, results_file, num_loc=10, logger=None): feature_conf = { 'output': 'feats-superpoint-n4096-r1024', @@ -219,7 +219,7 @@ def run_hloc_cambridge(cfg, image_dir, imagecols, neighbors, train_ids, query_id neighbors_train = imagecols_train.update_neighbors(neighbors) ref_sfm_path = _psfm.run_colmap_sfm_with_known_poses( - cfg['sfm'], imagecols_train, os.path.join(cfg['output_dir'], 'tmp_colmap'), neighbors=neighbors_train, + cfg['sfm'], imagecols_train, os.path.join(cfg['output_dir'], 'tmp_colmap'), neighbors=neighbors_train, map_to_original_image_names=False, skip_exists=cfg['skip_exists'] ) ref_sfm = pycolmap.Reconstruction(ref_sfm_path) @@ -261,4 +261,4 @@ def run_hloc_cambridge(cfg, image_dir, imagecols, neighbors, train_ids, query_id if logger: logger.info(f'Coarse pose read from {results_file}') hloc_log_file = f'{results_file}_logs.pkl' - return ref_sfm, poses, hloc_log_file \ No newline at end of file + return ref_sfm, poses, hloc_log_file diff --git a/runners/tests/localization.py b/runners/tests/localization.py index 7a783b3f..66fc7e5f 100644 --- a/runners/tests/localization.py +++ b/runners/tests/localization.py @@ -27,17 +27,17 @@ def parse_args(): arg_parser = argparse.ArgumentParser(description='minimal test for visual localization with points and lines') - arg_parser.add_argument('--data', type=Path, default='runners/tests/localization_test_data_stairs_1.npy', + arg_parser.add_argument('--data', type=Path, default='runners/tests/data/localization/localization_test_data_stairs_1.npy', help='Path to test data file, default: %(default)s') arg_parser.add_argument('--outputs', type=Path, default='outputs/test/localization', help='Path to the output directory, default: %(default)s') - arg_parser.add_argument('--ransac_method', choices=['ransac', 'solver', 'hybrid'], default='hybrid', + arg_parser.add_argument('--ransac_method', choices=['ransac', 'solver', 'hybrid'], default='hybrid', help='RANSAC method') - arg_parser.add_argument('--thres', type=float, default=5.0, + arg_parser.add_argument('--thres', type=float, default=5.0, help='Threshold for RANSAC/Solver first RANSAC, default: %(default)s') - arg_parser.add_argument('--thres_point', type=float, default=5.0, + arg_parser.add_argument('--thres_point', type=float, default=5.0, help='Threshold for points in hybrid RANSAC, default: %(default)s') - arg_parser.add_argument('--thres_line', type=float, default=5.0, + arg_parser.add_argument('--thres_line', type=float, default=5.0, help='Threshold for lines in hybrid RANSAC, default: %(default)s') arg_parser.add_argument('--line2d_matcher', type=str, default='sold2', help='2D matcher for lines, default: %(default)s') @@ -69,7 +69,7 @@ def main(): final_pose, ransac_stats = _estimators.pl_estimate_absolute_pose( cfg, l3ds, l3d_ids, l2ds, p3ds, p2ds, cam, silent=True, logger=logger) - + # Let's Check some RANSAC status log = "RANSAC stats: \n" log += f"num_iterations_total: {ransac_stats.num_iterations_total}\n" @@ -79,7 +79,7 @@ def main(): logger.info(log) R_gt, t_gt = data['pose_gt'].R(), data['pose_gt'].tvec - + log = "Results: \n" log += f"Result(P+L) Pose (qvec, tvec): {final_pose.qvec}, {final_pose.tvec}\n" log += f"HLoc(Point) Pose (qvec, tvec): {data['pose_point'].qvec}, {data['pose_point'].tvec}\n" @@ -96,7 +96,7 @@ def main(): cos = np.clip((np.trace(np.dot(R_gt.T, R)) - 1) / 2, -1., 1.) e_R = np.rad2deg(np.abs(np.arccos(cos))) log += f'HLoc(Point) Pose errors: {e_t:.3f}m, {e_R:.3f}deg' - + logger.info(log) inlier_indices = ransac_stats.inlier_indices @@ -133,4 +133,5 @@ def main(): cv2.imwrite((args.outputs / "pose_p+l.png").as_posix(), img) if __name__ == '__main__': - main() \ No newline at end of file + main() + diff --git a/runners/tests/localization_test_data_stairs_1.npy b/runners/tests/localization_test_data_stairs_1.npy deleted file mode 100644 index 667db8b2..00000000 Binary files a/runners/tests/localization_test_data_stairs_1.npy and /dev/null differ diff --git a/runners/tests/localization_test_data_stairs_2.npy b/runners/tests/localization_test_data_stairs_2.npy deleted file mode 100644 index a2b3ed83..00000000 Binary files a/runners/tests/localization_test_data_stairs_2.npy and /dev/null differ