Add Deep Blending results, full evaluation script, and per-scene results

examples/benchmarks/compression/eval_all_datasets.py

@@ -0,0 +1,108 @@
+# skript for running evaluations on all datasets and writing per scene csv files for 3dgs.zip survey
+
+import os
+import json
+import csv
+import subprocess
+import shutil
+
+RESULT_DIR_STRING="results/benchmark_{dataset}_mcmc_{num_g}_png_compression"
+
+datasets = ["tt", "db", "mip"]
+
+num_gaussians= [
+    (360000,  "0_36M"),
+    (490000,  "0_49M"),
+    (1000000, "1M"),
+    (4000000, "4M")
+    ]
+
+named_methods = {
+    1000000: "-1.00M"
+}
+
+def run_all_evals():
+    for dataset in datasets:
+        for num_g in num_gaussians:
+
+            RESULT_DIR = RESULT_DIR_STRING.format(dataset=dataset, num_g=num_g[1])
+            CAP_MAX = num_g[0]
+
+            if dataset == "mip":
+                filename = "mcmc.sh"
+            else:
+                filename = f"mcmc_{dataset}.sh"
+
+            print(f"Running {dataset} with {CAP_MAX} Gaussians, writing results to {RESULT_DIR}")
+            os.system(f"bash benchmarks/compression/{filename} {RESULT_DIR} {CAP_MAX}")
+
+
+def write_results():   
+    # fetch per-scene results and write into csvs
+    for dataset in datasets:
+
+        if dataset == "tt":
+                dataset_name = "TanksAndTemples"
+        elif dataset == "db":
+            dataset_name = "DeepBlending"
+        elif dataset == "mip":
+            dataset_name = "MipNeRF360"
+
+        csv_dir = os.path.join("benchmarks", "compression", "results", dataset_name)
+
+        if os.path.exists(csv_dir):
+            shutil.rmtree(csv_dir)
+        os.makedirs(csv_dir)
+
+        for num_g in num_gaussians:
+            RESULT_DIR = RESULT_DIR_STRING.format(dataset=dataset, num_g=num_g[1])
+            try:
+                scenes = os.listdir(RESULT_DIR)
+            except:
+                print("no results for", RESULT_DIR)
+
+            for scene in scenes:
+                scene_dir = os.path.join(RESULT_DIR, scene)
+                csv_file_path = os.path.join(csv_dir, scene+".csv")
+                metrics_path = os.path.join(RESULT_DIR, scene, "stats", "compress_step29999.json")
+                if not os.path.exists(scene_dir):
+                    continue
+
+                #extract size
+                zip_path = f"{scene_dir}/compression.zip"
+                if os.path.exists(zip_path):
+                    subprocess.run(f"rm {zip_path}", shell=True)
+                subprocess.run(f"zip -r {zip_path} {scene_dir}/compression/", shell=True)
+                out = subprocess.run(
+                    f"stat -c%s {zip_path}", shell=True, capture_output=True
+                )
+                size = int(out.stdout)
+
+                #extract metrics
+                with open(metrics_path, 'r') as file:
+                    data = json.load(file)
+
+                if num_g[0] in named_methods:
+                    name = named_methods[num_g[0]]
+                else:
+                    name = ""
+
+                new_row = [name, data["psnr"], data["ssim"], data["lpips"], size, num_g[0]]
+
+                with open(csv_file_path, 'a', newline='') as csv_file:
+                    writer = csv.writer(csv_file)
+
+                    # Check if the file is empty before writing the header
+                    if os.path.getsize(csv_file_path) == 0:
+                        header = ["Submethod", "PSNR", "SSIM", "LPIPS", "Size [Bytes]", "#Gaussians"]
+                        writer.writerow(header)
+
+                    writer.writerow(new_row)
+
+if __name__ == "__main__":
+    run_all_evals()
+    write_results()
+
+
+
+

examples/benchmarks/compression/mcmc.sh

@@ -18,6 +18,10 @@ CAP_MAX=1000000
 # RESULT_DIR="results/benchmark_mcmc_4M_png_compression"
 # CAP_MAX=4000000
 
+# Override default values if provided as arguments
+[ ! -z "$1" ] && RESULT_DIR="$1"
+[ ! -z "$2" ] && CAP_MAX="$2"
+
 
 for SCENE in $SCENE_LIST;
 do

examples/benchmarks/compression/mcmc_db.sh

@@ -0,0 +1,52 @@
+SCENE_DIR="data/DeepBlending"
+
+SCENE_LIST="drjohnson playroom"
+
+# # 0.36M GSs
+# RESULT_DIR="results/benchmark_db_mcmc_0_36M_png_compression"
+# CAP_MAX=360000
+
+# # 0.49M GSs
+# RESULT_DIR="results/benchmark_db_mcmc_0_49M_png_compression"
+# CAP_MAX=490000
+
+# 1M GSs
+RESULT_DIR="results/benchmark_db_mcmc_1M_png_compression"
+CAP_MAX=1000000
+
+# # 4M GSs
+# RESULT_DIR="results/benchmark_db_mcmc_4M_png_compression"
+# CAP_MAX=4000000
+
+# Override default values if provided as arguments
+[ ! -z "$1" ] && RESULT_DIR="$1"
+[ ! -z "$2" ] && CAP_MAX="$2"
+
+for SCENE in $SCENE_LIST;
+do
+    echo "Running $SCENE"
+
+    # train without eval
+    CUDA_VISIBLE_DEVICES=0 python simple_trainer.py mcmc --eval_steps -1 --disable_viewer --data_factor 1 \
+        --strategy.cap-max $CAP_MAX \
+        --data_dir $SCENE_DIR/$SCENE/ \
+        --result_dir $RESULT_DIR/$SCENE/
+
+    # eval: use vgg for lpips to align with other benchmarks
+    CUDA_VISIBLE_DEVICES=0 python simple_trainer.py mcmc --disable_viewer --data_factor 1 \
+        --strategy.cap-max $CAP_MAX \
+        --data_dir $SCENE_DIR/$SCENE/ \
+        --result_dir $RESULT_DIR/$SCENE/ \
+        --lpips_net vgg \
+        --compression png \
+        --ckpt $RESULT_DIR/$SCENE/ckpts/ckpt_29999_rank0.pt
+done
+
+# Zip the compressed files and summarize the stats
+if command -v zip &> /dev/null
+then
+    echo "Zipping results"
+    python benchmarks/compression/summarize_stats.py --results_dir $RESULT_DIR --scenes $SCENE_LIST
+else
+    echo "zip command not found, skipping zipping"
+fi

examples/benchmarks/compression/mcmc_tt.sh

@@ -1,5 +1,5 @@
 SCENE_DIR="data/tandt"
-# eval all 9 scenes for benchmarking
+
 SCENE_LIST="train truck"
 
 # # 0.36M GSs
@@ -18,6 +18,10 @@ CAP_MAX=1000000
 # RESULT_DIR="results/benchmark_tt_mcmc_4M_png_compression"
 # CAP_MAX=4000000
 
+# Override default values if provided as arguments
+[ ! -z "$1" ] && RESULT_DIR="$1"
+[ ! -z "$2" ] && CAP_MAX="$2"
+
 for SCENE in $SCENE_LIST;
 do
     echo "Running $SCENE"

examples/benchmarks/compression/results/DeepBlending/drjohnson.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,29.201324462890625,0.9001765251159668,0.2728179395198822,6696497,360000
+,29.313091278076172,0.9025779962539673,0.263129860162735,8469791,490000
+-1.00M,29.479978561401367,0.9068419933319092,0.2507980763912201,15664460,1000000
+,29.039730072021484,0.9078600406646729,0.24312639236450195,56163073,4000000

examples/benchmarks/compression/results/DeepBlending/playroom.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,30.262693405151367,0.9101507067680359,0.2688414454460144,6867113,360000
+,30.337688446044922,0.9118587970733643,0.26298797130584717,8706740,490000
+-1.00M,29.99859619140625,0.9106613397598267,0.252819299697876,15990548,1000000
+,29.75582504272461,0.9131292104721069,0.23756566643714905,57211413,4000000

examples/benchmarks/compression/results/MipNeRF360/bicycle.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,24.314237594604492,0.7095159888267517,0.3013736307621002,7055770,360000
+,24.534364700317383,0.7251359224319458,0.2785482406616211,8813338,490000
+-1.00M,24.911056518554688,0.7516974210739136,0.23711256682872772,15926996,1000000
+,25.325092315673828,0.7788066864013672,0.1876075118780136,58617041,4000000

examples/benchmarks/compression/results/MipNeRF360/bonsai.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,31.02326011657715,0.9329836368560791,0.22257398068904877,6830786,360000
+,31.281402587890625,0.9321386218070984,0.215091735124588,8648204,490000
+-1.00M,31.807077407836914,0.9414350390434265,0.20146769285202026,15826537,1000000
+,32.30672836303711,0.9459878206253052,0.1877104789018631,56735351,4000000

examples/benchmarks/compression/results/MipNeRF360/counter.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,28.405691146850586,0.8973114490509033,0.2164955884218216,6878603,360000
+,28.529481887817383,0.9007075428962708,0.2095101773738861,8730556,490000
+-1.00M,28.866003036499023,0.9071822762489319,0.19578693807125092,15947230,1000000
+,29.177431106567383,0.9131742119789124,0.17931070923805237,57486357,4000000

examples/benchmarks/compression/results/MipNeRF360/flowers.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,21.042749404907227,0.5605924725532532,0.40565335750579834,7161082,360000
+,21.23011589050293,0.574547290802002,0.38778236508369446,9020882,490000
+-1.00M,21.494901657104492,0.6000087261199951,0.3538331985473633,16371725,1000000
+,21.674463272094727,0.6261774301528931,0.30014270544052124,59397940,4000000

examples/benchmarks/compression/results/MipNeRF360/garden.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,25.743114471435547,0.7966399192810059,0.2254912108182907,7095776,360000
+,26.028152465820312,0.8122509717941284,0.20099054276943207,9005181,490000
+-1.00M,26.60516357421875,0.8374983668327332,0.15973007678985596,16546320,1000000
+,27.302112579345703,0.8619086742401123,0.1209760457277298,59521047,4000000

examples/benchmarks/compression/results/MipNeRF360/kitchen.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,29.573772430419922,0.9109970331192017,0.1568634957075119,6886401,360000
+,29.951499938964844,0.9146948456764221,0.14950363337993622,8583446,490000
+-1.00M,30.56005859375,0.9218958020210266,0.13678385317325592,15597179,1000000
+,31.493282318115234,0.9290380477905273,0.1236552819609642,56478553,4000000

examples/benchmarks/compression/results/MipNeRF360/room.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,31.20783233642578,0.9134507179260254,0.2248891294002533,6680829,360000
+,31.438037872314453,0.919510006904602,0.21646593511104584,8558287,490000
+-1.00M,31.8409366607666,0.9252573847770691,0.20395036041736603,15785810,1000000
+,32.10432434082031,0.927442729473114,0.19184806942939758,57016712,4000000

examples/benchmarks/compression/results/MipNeRF360/stump.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,25.656526565551758,0.732469379901886,0.2942078113555908,6919767,360000
+,25.865047454833984,0.7448430061340332,0.2767064571380615,8828910,490000
+-1.00M,26.30782699584961,0.7667227983474731,0.24127236008644104,15854509,1000000
+,26.782581329345703,0.7874260544776917,0.20132017135620117,57520011,4000000

examples/benchmarks/compression/results/MipNeRF360/treehill.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,22.8242130279541,0.6230133175849915,0.3928559720516205,6875408,360000
+,22.85292625427246,0.6296589970588684,0.3755051791667938,8739408,490000
+-1.00M,22.878782272338867,0.6389139890670776,0.3419164717197418,16072601,1000000
+,22.965383529663086,0.6463788747787476,0.28769150376319885,58275545,4000000

examples/benchmarks/compression/results/TanksAndTemples/train.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,21.811771392822266,0.8045595288276672,0.23488155007362366,6802125,360000
+,21.922565460205078,0.8111316561698914,0.2212243378162384,8693415,490000
+-1.00M,22.31643295288086,0.8283930420875549,0.19330424070358276,16086616,1000000
+,22.71592903137207,0.8432173728942871,0.16197781264781952,57268346,4000000

examples/benchmarks/compression/results/TanksAndTemples/truck.csv

@@ -0,0 +1,5 @@
+Submethod,PSNR,SSIM,LPIPS,Size [Bytes],#Gaussians
+,25.094594955444336,0.871437668800354,0.1670973002910614,6831513,360000
+,25.42046356201172,0.876853883266449,0.1554252803325653,8691262,490000
+-1.00M,25.811182022094727,0.8866869211196899,0.1334751546382904,16115589,1000000
+,26.319414138793945,0.8979613184928894,0.10413430631160736,59028839,4000000