Support GLOMAP as a better SfM backend (#33)

* fix compilation of optimization on point trajectory

* support glomap

* update README

* fix table

* Update README.md

* Update README.md

* Update README.md

* Update README.md

* Update README.md

README.md

@@ -4,7 +4,7 @@
   <div align="center"></div>
 </p>
 
-Code release for our ECCV 2022 paper "ParticleSfM: Exploiting Dense Point Trajectories for Localizing Moving Cameras in the Wild." by [Wang Zhao](https://github.com/thuzhaowang), [Shaohui Liu](http://www.b1ueber2y.me/), [Hengkai Guo](https://github.com/guohengkai), [Wenping Wang](https://engineering.tamu.edu/cse/profiles/Wang-Wenping.html) and [Yong-Jin Liu](https://cg.cs.tsinghua.edu.cn/people/~Yongjin/Yongjin.htm).
+Code release for our ECCV 2022 paper "ParticleSfM: Exploiting Dense Point Trajectories for Localizing Moving Cameras in the Wild." by [Wang Zhao](https://thuzhaowang.github.io/), [Shaohui Liu](http://www.b1ueber2y.me/), [Hengkai Guo](https://guohengkai.github.io/), [Wenping Wang](https://engineering.tamu.edu/cse/profiles/Wang-Wenping.html) and [Yong-Jin Liu](https://cg.cs.tsinghua.edu.cn/people/~Yongjin/Yongjin.htm).
 
 **[Introduction]** ParticleSfM is an offline structure-from-motion system for videos (image sequences). Inspired by [Particle video](http://rvsn.csail.mit.edu/pv/), our method connects pairwise optical flows and optimizes dense point trajectories as long-range video correpondences, which are used in a customized global structure-from-motion framework with similarity averaging and global bundle adjustment. In particular, for dynamic scenes, the acquired dense point trajectories can be fed into a specially designed trajectory-based motion segmentation module to select static point tracks, enabling the system to produce reliable camera trajectories on in-the-wild sequences with complex foreground motion. 
 
@@ -14,12 +14,33 @@ Contact Wang Zhao ([email protected]), Shaohui Liu ([email protected]) and H
 
 If you are interested in potential collaboration or internship at ByteDance, please feel free to contact Hengkai Guo ([email protected]).
 
+## Update by 2025.02.05
+We support [GLOMAP](https://github.com/colmap/glomap) in our pipeline, which achieves more accurate results on 13 sequences of the [Sintel dataset](http://sintel.is.tue.mpg.de/):
+
+| Method | ATE (m) | RPE trans (m) | RPE rot (deg) | SfM runtime (min) | #Frames |
+|:-:|:-:|:-:|:-:|:-:|:-:|
+| Global SfM - Ours w/ gcolmap(Theia) | 0.104 | 0.054 | 0.414 | **3.35** | 45.6 | 
+| Global SfM - Ours w/ GLOMAP | **0.057** | **0.031** | **0.201** | 6.97 | 45.6 | 
+
+Test it by simply changing the `sfm_type` to `global_glomap`:
+```
+python run_particlesfm.py --image_dir /path/to/the/image/folder/ \
+                          --output_dir /path/to/output/workspace/ \
+                          --sfm_type global_glomap  # "global_theia" for the paper version
+```
+
 ## Installation
 1. Install dependencies:
+
+For using gcolmap (Theia) as in the original ParticleSfM paper:
 * Ceres 2.0.0 [[Guide](./misc/doc/ceres.md)]
 * COLMAP <= 3.8 [[Guide](./misc/doc/colmap.md)]
 * Theia SfM (customized version) [[Guide](./misc/doc/theia.md)]
 
+Alternatively, if you want to use our latest [GLOMAP](https://github.com/colmap/glomap) support:
+* Ceres with lastest version
+* GLOMAP
+
 2. Set up Python environment with Conda:
 ```
 conda env create -f particlesfm_env.yaml
@@ -148,6 +169,11 @@ python train_seq.py ./configs/your-config-file
 cd ..
 ```
 
+## Applications
+* Motion Control for Video Generation: [MotionCtrl](https://wzhouxiff.github.io/projects/MotionCtrl/), [CamCo](https://ir1d.github.io/CamCo/)
+* Motion Evaluation for Video Generation: [AnimateAnything](https://yu-shaonian.github.io/Animate_Anything/), [AC3D](https://snap-research.github.io/ac3d/)
+* Kinematic Control Annotation: [EgoVid-5M](https://egovid.github.io/)
+
 ## Citation
 ```
 @inproceedings{zhao2022particlesfm,

point_trajectory/optimize/CMakeLists.txt

@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.4)
 project(traj_ceres)
 
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+set(CMAKE_CXX_STANDARD 14)
 
 find_package(Ceres REQUIRED)
 include_directories(${CERES_INCLUDE_DIRS})

run_particlesfm.py

@@ -75,13 +75,19 @@ def sfm_reconstruction(args, image_dir, output_dir, traj_dir, skip_exists=False,
     sfm_dir = os.path.join(output_dir, "sfm")
 
     # sfm reconstruction
-    from sfm import main_global_sfm, main_incremental_sfm, write_depth_pose_from_colmap_format
-    if not args.incremental_sfm:
-        print("[ParticleSfM] Running global structure-from-motion........")
+    from sfm import main_global_sfm, main_incremental_sfm, main_global_sfm_glomap, write_depth_pose_from_colmap_format
+    if args.sfm_type == 'global_theia':
+        print("[ParticleSfM] Running global structure-from-motion with Theia........")
         main_global_sfm(sfm_dir, image_dir, traj_dir, remove_dynamic=(not args.assume_static), skip_exists=skip_exists)
-    else:
+    elif args.sfm_type == 'incremental_colmap':
         print("[ParticleSfM] Running incremental structure-from-motion with COLMAP........")
         main_incremental_sfm(sfm_dir, image_dir, traj_dir, remove_dynamic=(not args.assume_static), skip_exists=skip_exists)
+    elif args.sfm_type == 'global_glomap':
+        print("[ParticleSfM] Running global structure-from-motion with GLOMAP........")
+        main_global_sfm_glomap(sfm_dir, image_dir, traj_dir, remove_dynamic=(not args.assume_static), skip_exists=skip_exists)
+    else:
+        print('error sfm type: ', args.sfm_type)
+        assert False
 
     # # write depth and pose files from COLMAP format
     write_depth_pose_from_colmap_format(sfm_dir, os.path.join(output_dir, "colmap_outputs_converted"))
@@ -122,7 +128,7 @@ def parse_args():
     parser.add_argument("--window_size", type=int, default=10, help='the window size for trajectory motion segmentation')
     parser.add_argument("--traj_max_num", type=int, default=100000, help='the maximum number of trajs inside a window')
     # sfm
-    parser.add_argument("--incremental_sfm", action='store_true', help='whether to use incremental sfm or not')
+    parser.add_argument("--sfm_type", default='global_theia', choices=['incremental_colmap', 'global_theia', 'global_glomap'], help='sfm type')
     # pipeline control
     parser.add_argument("--skip_path_consistency", action='store_true', help='whether to skip the path consistency optimization or not')
     parser.add_argument("--assume_static", action='store_true', help='whether to skip the motion segmentation or not')

sfm/__init__.py

@@ -1,2 +1,2 @@
-from .main_sfm import main_global_sfm, main_incremental_sfm
+from .main_sfm import main_global_sfm, main_incremental_sfm, main_global_sfm_glomap
 from .convert import write_depth_pose_from_colmap_format

sfm/main_sfm.py

@@ -21,6 +21,7 @@
 from pathlib import Path
 import shutil
 import pprint
+import time
 
 import os, sys
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
@@ -113,7 +114,10 @@ def main_incremental_sfm(sfm_dir, image_dir, traj_dir, colmap_path="colmap", sin
         '--Mapper.ba_refine_principal_point', str(0),
         '--Mapper.ba_refine_extra_params', str(0)]
     print(' '.join(cmd))
+    start = time.time()
     subprocess.run(cmd, check=True)
+    end = time.time()
+    print('incremental sfm: {}s'.format(end - start))
 
     # analyze model
     stats = compute_model_stats(model_path)
@@ -144,7 +148,35 @@ def main_global_sfm(sfm_dir, image_dir, traj_dir, gcolmap_path=None, colmap_path
     cmd += ['--GlobalMapper.ba_refine_principal_point', str(0),
         '--GlobalMapper.ba_refine_extra_params', str(0)]
     print(' '.join(cmd))
+    start = time.time()
     subprocess.run(cmd, check=True)
+    end = time.time()
+    print('global sfm: {}s'.format(end - start))
+
+    # analyze model
+    stats = compute_model_stats(model_path)
+    if stats is not None:
+        print(stats)
+
+def main_global_sfm_glomap(sfm_dir, image_dir, traj_dir, gcolmap_path=None, glomap_path="glomap", colmap_path="colmap", single_camera=True, remove_dynamic=True, skip_geometric_verification=False, min_num_matches=None, skip_exists=False):
+    """
+    Global structure-from-motion for videos using GLOMAP
+    """
+    database_path, pair_txt_path = build_database(sfm_dir, image_dir, traj_dir, colmap_path=colmap_path, single_camera=single_camera, remove_dynamic=remove_dynamic, skip_geometric_verification=skip_geometric_verification, skip_exists=skip_exists)
+    model_path = Path(sfm_dir) / 'model'
+    model_path.mkdir(exist_ok=True, parents=True)
+
+    # run global structure-from-motion
+    cmd = [
+        str(glomap_path), 'mapper',
+        '--database_path', str(database_path),
+        '--image_path', str(image_dir),
+        '--output_path', str(model_path)]
+    print(' '.join(cmd))
+    start = time.time()
+    subprocess.run(cmd, check=True)
+    end = time.time()
+    print('global sfm with GLOMAP: {}s'.format(end - start))
 
     # analyze model
     stats = compute_model_stats(model_path)