Add tanks and temples dataset loader

.github/scripts/download_single_benchmark.sh

@@ -62,6 +62,10 @@ function download_and_unzip_dataset_files {
     WGET_URL1=https://www.dropbox.com/s/q02mgq1unbw068t/2011205_rc3.zip
     WGET_URL2=https://github.com/johnwlambert/gtsfm-cache/releases/download/2011205_rc3_deep_front_end_cache/cache_rc3_deep.tar.gz
     ZIP_FNAME=2011205_rc3.zip
+  elif [ "$DATASET_NAME" == "tanks-and-temples-barn-410" ]; then 
+    # Tanks and Temples Dataset, "Barn" scene.
+    WGET_URL1=https://github.com/johnwlambert/gtsfm-datasets-mirror/releases/download/tanks-and-temples-barn/Tanks_and_Temples_Barn_410.zip
+    ZIP_FNAME=Tanks_and_Temples_Barn_410.zip
   fi
 
   # Download the data.
@@ -125,6 +129,10 @@ function download_and_unzip_dataset_files {
   elif [ "$DATASET_NAME" == "2011205_rc3" ]; then
     unzip -qq 2011205_rc3.zip
     tar -xvf cache_rc3_deep.tar.gz
+  elif [ "$DATASET_NAME" == "tanks-and-temples-barn-410" ]; then
+    unzip -qq Tanks_and_Temples_Barn_410.zip
+    pwd
+    pwd .
   fi
 }
 

.github/scripts/execute_single_benchmark.sh

@@ -27,6 +27,9 @@ elif [ "$DATASET_NAME" == "skydio-501" ]; then
 elif [ "$DATASET_NAME" == "notre-dame-20" ]; then
   IMAGES_DIR=notre-dame-20/images
   COLMAP_FILES_DIRPATH=notre-dame-20/notre-dame-20-colmap
+elif [ "$DATASET_NAME" == "tanks-and-temples-barn-410" ]; then
+  DATASET_ROOT="Tanks_and_Temples_Barn_410"
+  SCENE_NAME="Barn"
 fi
 
 echo "Config: ${CONFIG_NAME}, Loader: ${LOADER_NAME}"
@@ -66,4 +69,9 @@ elif [ "$LOADER_NAME" == "astrovision" ]; then
     --max_frame_lookahead $MAX_FRAME_LOOKAHEAD \
     --max_resolution ${MAX_RESOLUTION} \
     ${SHARE_INTRINSICS_ARG}
+elif [ "$LOADER_NAME" == "tanks-and-temples" ]; then
+  python gtsfm/runner/run_scene_optimizer_synthetic_tanks_and_temples.py \
+    --config_name ${CONFIG_NAME}.yaml \
+    --dataset_root $DATASET_ROOT \
+    --scene_name $SCENE_NAME
 fi

.github/workflows/benchmark.yml

@@ -23,6 +23,7 @@ jobs:
             [deep_front_end,             notre-dame-20,         20,         jpg,           gdrive,     colmap-loader,  760,     false],
             [sift_front_end_astrovision, 2011205_rc3,           65,         png,           wget,       astrovision,    1024,    true],
             [deep_front_end_astrovision, 2011205_rc3,           65,         png,           wget,       astrovision,    1024,    true],
+            [synthetic_front_end,   tanks-and-temples-barn-410, 3,          jpg,           wget,    tanks-and-temples, 1080,    true],
           ]
     defaults:
       run:

gtsfm/configs/synthetic_front_end.yaml

@@ -0,0 +1,82 @@
+# Default deep front-end configuration.
+
+SceneOptimizer:
+  _target_: gtsfm.scene_optimizer.SceneOptimizer
+  save_gtsfm_data: True
+  save_two_view_correspondences_viz: False
+  save_3d_viz: True
+  pose_angular_error_thresh: 5 # degrees
+
+  retriever:
+    _target_: gtsfm.retriever.sequential_retriever.SequentialRetriever
+    max_frame_lookahead: 4
+
+  correspondence_generator:
+    _target_: gtsfm.frontend.correspondence_generator.synthetic_correspondence_generator.SyntheticCorrespondenceGenerator
+    #dataset_root: /Users/johnlambert/Downloads/Tanks_and_Temples_Barn_410
+    dataset_root: /home/runner/work/gtsfm/gtsfm/Tanks_and_Temples_Barn_410
+    scene_name: Barn
+
+  two_view_estimator:
+    _target_: gtsfm.two_view_estimator.TwoViewEstimator
+    bundle_adjust_2view: True
+    eval_threshold_px: 4 # in px
+    ba_reproj_error_thresholds: [0.5]
+    bundle_adjust_2view_maxiters: 100
+
+    verifier:
+      _target_: gtsfm.frontend.verifier.ransac.Ransac
+      use_intrinsics_in_verification: True
+      estimation_threshold_px: 4 # for H/E/F estimators
+
+    triangulation_options:
+      _target_: gtsfm.data_association.point3d_initializer.TriangulationOptions
+      mode:
+        _target_: gtsfm.data_association.point3d_initializer.TriangulationSamplingMode
+        value: NO_RANSAC
+
+    inlier_support_processor:
+      _target_: gtsfm.two_view_estimator.InlierSupportProcessor
+      min_num_inliers_est_model: 15
+      min_inlier_ratio_est_model: 0.1
+
+  multiview_optimizer:
+    _target_: gtsfm.multi_view_optimizer.MultiViewOptimizer
+
+    # comment out to not run
+    view_graph_estimator:
+      _target_: gtsfm.view_graph_estimator.cycle_consistent_rotation_estimator.CycleConsistentRotationViewGraphEstimator
+      edge_error_aggregation_criterion: MEDIAN_EDGE_ERROR
+
+    rot_avg_module:
+      _target_: gtsfm.averaging.rotation.shonan.ShonanRotationAveraging
+
+    trans_avg_module:
+      _target_: gtsfm.averaging.translation.averaging_1dsfm.TranslationAveraging1DSFM
+      robust_measurement_noise: True
+      projection_sampling_method: SAMPLE_INPUT_MEASUREMENTS
+
+    data_association_module:
+      _target_: gtsfm.data_association.data_assoc.DataAssociation
+      min_track_len: 2
+      triangulation_options:
+        _target_: gtsfm.data_association.point3d_initializer.TriangulationOptions
+        reproj_error_threshold: 10
+        mode:
+          _target_: gtsfm.data_association.point3d_initializer.TriangulationSamplingMode
+          value: RANSAC_SAMPLE_UNIFORM
+        max_num_hypotheses: 100
+      save_track_patches_viz: False
+
+    bundle_adjustment_module:
+      _target_: gtsfm.bundle.bundle_adjustment.BundleAdjustmentOptimizer
+      reproj_error_thresholds: [10, 5, 3] # for (multistage) post-optimization filtering
+      robust_measurement_noise: True
+      shared_calib: False
+      cam_pose3_prior_noise_sigma: 0.1
+      calibration_prior_noise_sigma: 1e-5
+      measurement_noise_sigma: 1.0
+
+  # # comment out to not run
+  # dense_multiview_optimizer:
+  #   _target_: gtsfm.densify.mvs_patchmatchnet.MVSPatchmatchNet

gtsfm/frontend/correspondence_generator/synthetic_correspondence_generator.py

@@ -0,0 +1,105 @@
+"""Correspondence generator that creates synthetic keypoint correspondences using a 3d mesh.
+
+Authors: John Lambert
+"""
+import tempfile
+from typing import Any, Dict, List, Tuple
+
+from dask.distributed import Client, Future
+import numpy as np
+import open3d
+
+from gtsfm.common.image import Image
+from gtsfm.common.keypoints import Keypoints
+from gtsfm.common.types import CALIBRATION_TYPE, CAMERA_TYPE
+from gtsfm.frontend.correspondence_generator.correspondence_generator_base import CorrespondenceGeneratorBase
+from gtsfm.frontend.correspondence_generator.keypoint_aggregator.keypoint_aggregator_base import KeypointAggregatorBase
+from gtsfm.frontend.correspondence_generator.keypoint_aggregator.keypoint_aggregator_dedup import (
+    KeypointAggregatorDedup,
+)
+from gtsfm.frontend.correspondence_generator.keypoint_aggregator.keypoint_aggregator_unique import (
+    KeypointAggregatorUnique,
+)
+from gtsfm.loader.loader_base import LoaderBase
+from gtsfm.loader.tanks_and_temples_loader import TanksAndTemplesLoader
+
+
+class SyntheticCorrespondenceGenerator(CorrespondenceGeneratorBase):
+    """Pair-wise synthetic keypoint correspondence generator."""
+
+    def __init__(self, dataset_root: str, scene_name: str, deduplicate: bool = True) -> None:
+        """
+        Args:
+            dataset_root: str
+            scene_name: Name of scene from Tanks & Temples dataset.
+            deduplicate: Whether to de-duplicate with a single image the detections received from each image pair.
+        """
+        self._dataset_root = dataset_root
+        self._scene_name = scene_name
+        self._aggregator: KeypointAggregatorBase = (
+            KeypointAggregatorDedup() if deduplicate else KeypointAggregatorUnique()
+        )
+
+    def generate_correspondences(
+        self,
+        client: Client,
+        images: List[Future],
+        image_pairs: List[Tuple[int, int]],
+    ) -> Tuple[List[Keypoints], Dict[Tuple[int, int], np.ndarray]]:
+        """Apply the correspondence generator to generate putative correspondences.
+
+        Args:
+            client: Dask client, used to execute the front-end as futures.
+            images: List of all images, as futures.
+            image_pairs: Indices of the pairs of images to estimate two-view pose and correspondences.
+
+        Returns:
+            List of keypoints, one entry for each input images.
+            Putative correspondence as indices of keypoints, for pairs of images.
+        """
+        dataset_root = self._dataset_root
+        scene_name = self._scene_name
+
+        img_dir = f'{dataset_root}/{scene_name}'
+        poses_fpath = f'{dataset_root}/{scene_name}_COLMAP_SfM.log'
+        lidar_ply_fpath = f'{dataset_root}/{scene_name}.ply'
+        colmap_ply_fpath = f'{dataset_root}/{scene_name}_COLMAP.ply'
+        ply_alignment_fpath = f'{dataset_root}/{scene_name}_trans.txt'
+        bounding_polyhedron_json_fpath = f'{dataset_root}/{scene_name}.json'
+        loader = TanksAndTemplesLoader(
+            img_dir=img_dir,
+            poses_fpath=poses_fpath,
+            lidar_ply_fpath=lidar_ply_fpath,
+            ply_alignment_fpath=ply_alignment_fpath,
+            bounding_polyhedron_json_fpath=bounding_polyhedron_json_fpath,
+            colmap_ply_fpath=colmap_ply_fpath,
+        )
+
+
+        mesh = loader.reconstruct_mesh()
+        # TODO(jolambert): File Open3d bug to add pickle support for TriangleMesh.
+        open3d_mesh_path = tempfile.NamedTemporaryFile(suffix='.obj').name
+        open3d.io.write_triangle_mesh(filename=open3d_mesh_path, mesh=mesh)
+
+        loader_future = client.scatter(loader, broadcast=False)
+        def apply_synthetic_corr_generator(
+            loader_: LoaderBase, camera_i1, camera_i2, open3d_mesh_fpath: str
+        ) -> Tuple[Keypoints, Keypoints]:
+            return loader_.generate_synthetic_correspondences_for_image_pair(camera_i1, camera_i2, open3d_mesh_fpath)
+
+        pairwise_correspondence_futures = {
+            (i1, i2): client.submit(apply_synthetic_corr_generator,
+                loader_future,
+                loader.get_camera(index=i1),
+                loader.get_camera(index=i2),
+                open3d_mesh_path
+            )
+            for i1, i2 in image_pairs
+        }
+
+        pairwise_correspondences: Dict[Tuple[int, int], Tuple[Keypoints, Keypoints]] = client.gather(
+            pairwise_correspondence_futures
+        )
+
+        keypoints_list, putative_corr_idxs_dict = self._aggregator.aggregate(keypoints_dict=pairwise_correspondences)
+        return keypoints_list, putative_corr_idxs_dict