[release] pre-trained ckpt

GNFactor/agents/gnfactor_bc/models_embed.py

@@ -47,8 +47,7 @@ def __init__(self, conf, stop_encoder_grad=False):
         )
         self.use_code = conf.use_code  # Positional encoding
         self.use_code_viewdirs = conf.use_code_viewdirs
-        # regluarization on positional encoding
-        self.use_freenerf = conf.use_freenerf
+
 
         # Enable view directions
         self.use_viewdirs = conf.use_viewdirs
@@ -314,10 +313,6 @@ def forward(self, xyz, coarse=True, viewdirs=None, far=False, ret_last_feat=Fals
                 if self.use_code and not self.use_code_viewdirs:
                     # Positional encoding (no viewdirs)
                     z_feature = self.code(z_feature)
-                    if self.use_freenerf:
-                        import ipdb; ipdb.set_trace()
-                        z_feature = z_feature[int(t/T * L +3):] = 0
-
 
                 if self.use_viewdirs:
                     # * Encode the view directions

GNFactor/conf/eval.yaml

@@ -1,9 +1,10 @@
 method:
-    name: "PERACT_BC"
+    name: "GNFACTOR_BC"
 
 rlbench:
     task_name: "multi"
-    tasks: [open_drawer,slide_block_to_color_target]
+    # change the task as you want. in our paper, we select these 10 tasks.
+    tasks: [close_jar,open_drawer,sweep_to_dustpan_of_size,meat_off_grill,turn_tap,slide_block_to_color_target,put_item_in_drawer,reach_and_drag,push_buttons,stack_blocks]
     demo_path: /my/demo/path
     episode_length: 25
     cameras: [front]

GNFactor/conf/method/GNFACTOR_BC.yaml

@@ -111,7 +111,6 @@ neural_renderer:
   ret_last_feat: False
   use_code: True
   use_code_viewdirs: False
-  use_freenerf: False
   use_xyz: True
 
   # sampling

GNFactor/eval.py

@@ -146,7 +146,7 @@ def eval_seed(train_cfg,
     # evaluate several checkpoints in parallel
     # NOTE: in multi-task settings, each task is evaluated serially, which makes everything slow!
     split_n = utils.split_list(num_weights_to_eval, eval_cfg.framework.eval_envs)
-
+    
     for split in split_n:
         processes = []
         for e_idx, weight_idx in enumerate(split):
@@ -162,7 +162,7 @@ def eval_seed(train_cfg,
                               eval_cfg.cinematic_recorder,
                               None))
             p.start()
-            processes.append(p)            
+            processes.append(p)        
 
         for p in processes:
             p.join()

GNFactor/helpers/custom_rlbench_env.py

@@ -374,14 +374,7 @@ def reset_to_demo(self, i, variation_number=-1, novel_command=None):
             self._set_new_task()
             self._episodes_this_task = 0
         self._episodes_this_task += 1
-
         self._i = 0
-        # super(CustomMultiTaskRLBenchEnv, self).reset()
-
-        # if variation_number == -1:
-        #     self._task.sample_variation()
-        # else:
-        #     self._task.set_variation(variation_number)
 
         self._task.set_variation(-1)
         d = self._task.get_demos(
@@ -390,7 +383,6 @@ def reset_to_demo(self, i, variation_number=-1, novel_command=None):
         self._task.set_variation(d.variation_number)
         _, obs = self._task.reset_to_demo(d)
         self._lang_goal = self._task.get_task_descriptions()[0] if novel_command is None else novel_command
-
         self._previous_obs_dict = self.extract_obs(obs)
         self._record_current_episode = (
                 self.eval and self._episode_index % self._record_every_n == 0)

README.md

@@ -37,11 +37,10 @@ Multi-Task Real Robot Learning with <a>G</a>eneralizable <a>N</a>eural Feature <
   <img src="./docs/gnfactor_method.png" alt="Logo" width="100%">
 </div>
 
-
-
 https://github.com/YanjieZe/GNFactor/assets/59699800/0ad86126-ad24-4291-a4ba-6bb591d9934a
 
 
+
 # 💻 Installation
 
 See [INSTALL.md](docs/INSTALL.md) for installation instructions. 
@@ -72,7 +71,7 @@ We provide scripts to generate demonstrations, train and evaluate policies, and
 We also provide the results (*.csv files, 3 seeds) of our paper in `GNFactor_results/`. Researchers could directly compare their own algorithm with these results under the similar setting.
 
 
-## Generate Demonstrations
+## 🦉 Generate Demonstrations
 In this work, we select 10 challenging tasks from RLBench:
 ```bash
 tasks=[close_jar,open_drawer,sweep_to_dustpan_of_size,meat_off_grill,turn_tap,slide_block_to_color_target,put_item_in_drawer,reach_and_drag,push_buttons,stack_blocks]
@@ -92,7 +91,7 @@ For example, running `bash scripts/gen_demonstrations.sh close_jar` will generat
 To generate demonstrations for all 10 tasks, just simply run this command for different tasks.
 
 
-## Training and Evaluation
+## 📈 Training and Evaluation
 We use wandb to log some curves and visualizations. Login to wandb before running the scripts.
 ```bash
 wandb login
@@ -115,13 +114,11 @@ bash scripts/train_and_eval_with_one_gpu.sh PERACT_BC
 ```
 But this means the batch size is 1 and the results would be worse.
 
-To save the evaluation videos, set `cinematic_recorder.enabled=True` in `conf/eval.yaml`.
 
+Configurations for training and evaluation are in `GNFactor/conf/`. You could change the parameters in these files for your own setting.
 
-All configurations for training and evaluation are in `GNFactor/conf/`. You could change the parameters in these files for your own setting.
 
-
-## Analyze Evaluation Results
+## 📊 Analyze Evaluation Results
 For the users' convenience, we provide the results (`*.csv` files, 3 seeds) of our paper in `GNFactor_results/`.
 
 We also provide a python script `scripts/compute_results.py` to compute the average success rates from the `*.csv` files (generated during the evaluation). An example to compute success rates given our provided csv files:
@@ -134,6 +131,19 @@ In our main paper, we only report the results of the **last** checkpoint for fai
 python scripts/compute_results.py --file_paths GNFactor_results/0.csv GNFactor_results/1.csv GNFactor_results/2.csv --method best
 ```
 
+## 🆕 Pre-Trained Checkpoints
+We provide the pre-trained checkpoints for users' convenience. You could download from the google drive [here](https://drive.google.com/file/d/1UOP9Zftk7zsfQTBVQvbm7GKz_V3-DaSx/view?usp=sharing). Then, unzip this file into `REPO_DIR/GNFactor/logs/`. The overall file structure should be like this:
+```
+docs/
+GNFactor/
+    logs/
+        GNFACTOR_BC_released/ # our provided files
+GNFactor_results/
+```
+After this, use our script to evaluate the checkpoints (remember to generate the evaluation demonstrations first):
+```bash
+bash scripts/eval.sh GNFACTOR_BC released
+```
 
 # 🏷️ License
 This repository is released under the MIT license. See [LICENSE](LICENSE) for additional details.
@@ -152,4 +162,4 @@ If you find our work useful, please consider citing:
   journal={CoRL}, 
   year={2023},
 }
-```
+```

docs/ERROR_CATCH.md

@@ -1,7 +1,8 @@
 
 # Error Catching
 
-# PyRender error
+- PyRender error.
+
 add following to bashrc:
 ```
 export DISPLAY=:0
@@ -10,63 +11,63 @@ export PYOPENGL_PLATFORM=egl
 ```
 
 
-# `qt.qpa.plugin: Could not find the Qt platform plugin "xcb" in "xxxx/cv2/qt/plugins`
+- `qt.qpa.plugin: Could not find the Qt platform plugin "xcb" in "xxxx/cv2/qt/plugins`
 ```
 pip uninstall opencv-python (this is very important!)
 pip install opencv-python-headless
 ```
 
-# xcb errors during generating data
+- xcb errors during generating data
 ```
 sudo apt-get install qt5-default
 ```
 
-# import transformers error: `packaging.version.InvalidVersion: Invalid version: '0.10.1,<0.11'`
+- import transformers error: `packaging.version.InvalidVersion: Invalid version: '0.10.1,<0.11'`
 ```
 pip install packaging==21.3
 ```
 
-# `AssertionError: Invalid type <class 'NoneType'> for key WEIGHT_DECAY_BIAS; valid types = {<class 'tuple'>, <class 'int'>, <class 'float'>, <class 'bool'>, <class 'list'>, <class 'str'>}`
+- `AssertionError: Invalid type <class 'NoneType'> for key WEIGHT_DECAY_BIAS; valid types = {<class 'tuple'>, <class 'int'>, <class 'float'>, <class 'bool'>, <class 'list'>, <class 'str'>}`
 ```
 pip uninstall yacs
 pip install yacs --upgrade
 ```
 
-# `ImportError: cannot import name 'SCMode' from 'omegaconf' (/home/yanjieze/miniconda3/envs/vision/lib/python3.8/site-packages/omegaconf/__init__.py)`
+- `ImportError: cannot import name 'SCMode' from 'omegaconf' (/home/yanjieze/miniconda3/envs/vision/lib/python3.8/site-packages/omegaconf/__init__.py)`
 ```
 pip uninstall omegaconf
 pip install omegaconf==2.1.1
 ```
 
-# `ImportError: cannot import name 'VisionEncoderDecoderModel' from 'transformers' (unknown location)`
+- `ImportError: cannot import name 'VisionEncoderDecoderModel' from 'transformers' (unknown location)`
 
 ```
 pip uninstall transformers
 pip install transformers==4.20.1
 ```
 
-# `AttributeError: partially initialized module 'cv2' has no attribute 'gapi_wip_gst_GStreamerPipeline' (most likely due to a circular import)`
+- `AttributeError: partially initialized module 'cv2' has no attribute 'gapi_wip_gst_GStreamerPipeline' (most likely due to a circular import)`
 ```
 pip install opencv-python==4.5.5.64
 ```
 
-# `ImportError: cannot import name 'get_num_classes' from 'torchmetrics.utilities.data' `
+- `ImportError: cannot import name 'get_num_classes' from 'torchmetrics.utilities.data' `
 ```
 pip install torchmetrics==0.5.1
 ```
 
-# `assert mdl is not None`
+- `assert mdl is not None`
 ```
 pip install hydra-core==1.1
 ```
 
-# `MultiScaleDeformableAttention....` error
+- `MultiScaleDeformableAttention....` error
 ```
 cd ~/miniconda3/envs/nerfact/lib/python3.9/site-packages
 rm -rf MultiScaleDeformableAttention.cpython-39-x86_64-linux-gnu.so 
 ```
 
-# `qt.qpa.plugin: Could not find the Qt platform plugin "xcb" in "/data/yanjieze/miniconda3/envs/nerfact/lib/python3.9/site-packages/cv2/qt/plugins`
+- `qt.qpa.plugin: Could not find the Qt platform plugin "xcb" in "/data/yanjieze/miniconda3/envs/nerfact/lib/python3.9/site-packages/cv2/qt/plugins`
 ```
 pip uninstall opencv-python
 pip install opencv-python-headless

scripts/compute_results.py

@@ -43,6 +43,9 @@ def main(file_paths, method):
         # Select the checkpoint based on the method
         if method == 'best':
             selected_return = avg_return.max()
+            # get the index of the best checkpoint
+            best_checkpoint = avg_return.idxmax()
+            print(f"Best checkpoint for {file_path}: {best_checkpoint}, with average return: {selected_return:.2f}")
         elif method == 'last':
             selected_return = avg_return.iloc[-1]
         else:

scripts/eval.sh

@@ -0,0 +1,30 @@
+# this script is for evaluating a given checkpoint.
+# example to evaluate our GNFactor:
+#       bash scripts/eval.sh GNFACTOR_BC released
+
+
+# some params specified by user
+method_name=$1
+addition_info=$2
+exp_name="${method_name}_${addition_info}"
+
+# set the seed number
+seed="0"
+# set the gpu id for evaluation. we use one gpu for parallel evaluation.
+eval_gpu="0"
+
+
+# cur_dir=$(pwd)
+train_demo_path="${cur_dir}/data/train_data"
+test_demo_path="${cur_dir}/data/test_data"
+
+
+cd GNFactor
+
+# eval stage
+CUDA_VISIBLE_DEVICES=${eval_gpu} xvfb-run -a python eval.py \
+    method.name=$method \
+    rlbench.task_name=${exp_name} \
+    rlbench.demo_path=${test_demo_path} \
+    framework.start_seed=${seed} \
+