first commit

.gitignore

@@ -0,0 +1,9 @@
+.vscode/
+__pycache__/
+results/
+checkpoints/
+runs/
+outputs/
+models/custom_op*/
+config/*.yaml
+models/gtad_lib/

README.md

@@ -0,0 +1,82 @@
+# AEI: Actors-Environment Interaction with Adaptive Attention for Temporal Action Proposals Generation
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/aei-actors-environment-interaction-with/temporal-action-proposal-generation-on)](https://paperswithcode.com/sota/temporal-action-proposal-generation-on?p=aei-actors-environment-interaction-with)
+
+A pytorch-version implementation codes of paper:
+ "AEI: Actors-Environment Interaction with Adaptive Attention for Temporal Action Proposals Generation",
+  which is accepted in BMVC 2021.
+
+## Installation Guide
+
+1. conda create -n aei python=3.8
+2. conda activate aei
+3. conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch
+4. pip install tqdm pandas tensorboard matplotlib fvcore scipy
+5. cd into tapg-aei (root)
+6. git clone https://github.com/frostinassiky/align1d
+7. cd into tapg-aei/align1d
+8. pip install -e .
+9. cd ..
+
+## Download Features
+3D Resnet-50 features extracted from rescaled videos of ActivityNet-1.3 can be downloaded below:
+* Environment features are [here](https://drive.google.com/file/d/1hPhcQ7EzyCh0A3SyZfgZScFVFZMEvVhe/view?usp=sharing) (~80GB uncompressed).
+* Actor features are [here](https://drive.google.com/file/d/1lOQG1FgDseRKDs3RNgpKd000OOZiag1s/view?usp=sharing) (~215GB uncompressed).
+* Annotations of [Activitynet-1.3](http://ec2-52-25-205-214.us-west-2.compute.amazonaws.com/files/activity_net.v1-3.min.json) can be downloaded from the [official website](http://activity-net.org/download.html).
+
+## Training and Testing  of AEI
+Default configurations of AEI are stored in config/defaults.py.
+The modified configurations are stored in config/*.yaml for training and testing of AEI on different datasets (ActivityNet-1.3 and THUMOS-14).
+We can also modify configurations through commandline arguments.
+
+1. To train AEI on TAPG task of ActivityNet-1.3 with 1 GPU:
+```
+python main.py --cfg-file config/anet_proposals.yaml MODE 'training' GPU_IDS [0]
+```
+
+2. To evaluate AEI on validation set of ActivityNet-1.3 with 1 GPU:
+```
+python main.py --cfg-file config/anet_proposals.yaml MODE 'validation' GPU_IDS [0]
+```
+
+## Reference
+
+This implementation is partly based on this [pytorch-implementation of BMN](https://github.com/JJBOY/BMN-Boundary-Matching-Network.git) for the boundary matching module.
+
+paper: [AEI: Actors-Environment Interaction with Adaptive Attention for Temporal Action Proposals Generation](https://arxiv.org/abs/2110.11474)
+
+
+## Q&A
+1q. "UserWarning: This DataLoader will create 12 worker processes in total. Our suggested max number of worker in current system is #, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary."
+
+1a. Change num_workers to # in line 171 of root>main.py>inference function
+
+## Citation
+If you find AEI useful for your research, please consider citing:
+```
+@article{khoavoAEI2021,
+  author    = {Khoa Vo and
+               Hyekang Joo and
+               Kashu Yamazaki and
+               Sang Truong and
+               Kris Kitani and
+               Minh{-}Triet Tran and
+               Ngan Le},
+  title     = {{AEI:} Actors-Environment Interaction with Adaptive Attention for
+               Temporal Action Proposals Generation},
+  journal   = {CoRR},
+  volume    = {abs/2110.11474},
+  year      = {2021},
+  url       = {https://arxiv.org/abs/2110.11474},
+  eprinttype = {arXiv},
+  eprint    = {2110.11474},
+}
+```
+
+
+## Contact
+Khoa Vo:
+```
+[email protected]
+```
+

config/defaults.py

@@ -0,0 +1,86 @@
+from fvcore.common.config import CfgNode
+
+
+_C = CfgNode()
+
+_C.GPU_IDS = [0, 1, 2, 3]
+_C.MODE = 'training'
+_C.EVAL_TYPE = 'proposal'
+_C.DATASET = 'anet'
+_C.USE_ENV = True
+_C.USE_AGENT = True
+_C.EVAL_SCORE = 'AUC'
+
+_C.TRAIN = CfgNode()
+_C.TRAIN.SPLIT = 'training'
+_C.TRAIN.NUM_EPOCHS = 10
+_C.TRAIN.BATCH_SIZE = 16
+_C.TRAIN.STEP_PERIOD = 1
+_C.TRAIN.ATTENTION_STEPS = 1
+_C.TRAIN.LR = 0.001
+_C.TRAIN.WEIGHT_DECAY = 0.0001
+_C.TRAIN.CHECKPOINT_FILE_PATH = ''
+_C.TRAIN.LOG_DIR = 'runs/c3d_runs/'
+
+_C.VAL = CfgNode()
+_C.VAL.SPLIT = 'validation'
+_C.VAL.BATCH_SIZE = 32
+
+_C.TEST = CfgNode()
+_C.TEST.SPLIT = 'testing'
+_C.TEST.BATCH_SIZE = 32
+_C.TEST.CHECKPOINT_PATH = 'checkpoints/c3d_checkpoints/checkpoint_6/best_auc.pth'
+
+_C.DATA = CfgNode()
+_C.DATA.ANNOTATION_FILE = '../datasets/activitynet/annotations/activity_net.v1-3.min.json'
+_C.DATA.DETECTION_GT_FILE = None
+_C.DATA.ENV_FEATURE_DIR = '../datasets/activitynet/c3d_env_features/'
+_C.DATA.AGENT_FEATURE_DIR = '../datasets/activitynet/c3d_agent_features/'
+_C.DATA.CLASSIFICATION_PATH = 'results/classification_results.json'
+_C.DATA.RESULT_PATH = 'results/results.json'
+_C.DATA.FIGURE_PATH = 'results/result_figure.jpg'
+_C.DATA.TEMPORAL_DIM = 100
+_C.DATA.MAX_DURATION = 100
+
+_C.MODEL = CfgNode()
+_C.MODEL.BOUNDARY_MATCHING_MODULE = 'bmn'
+_C.MODEL.SCORE_PATH = 'checkpoints/c3d_checkpoints/scores.json'
+_C.MODEL.CHECKPOINT_DIR = 'checkpoints/c3d_checkpoints/'
+_C.MODEL.ATTENTION_HEADS = 4
+_C.MODEL.ATTENTION_LAYERS = 1
+_C.MODEL.AGENT_DIM = 2048
+_C.MODEL.ENV_DIM = 2048
+_C.MODEL.FEAT_DIM = 512
+_C.MODEL.TRANSFORMER_DIM = 1024
+_C.MODEL.ENV_HIDDEN_DIM = None
+_C.MODEL.AGENT_HIDDEN_DIM = None
+_C.MODEL.HIDDEN_DIM_1D = 256  # 256
+_C.MODEL.HIDDEN_DIM_2D = 128  # 128
+_C.MODEL.HIDDEN_DIM_3D = 512  # 512
+_C.MODEL.TOPK_AGENTS = 4
+
+_C.BMN = CfgNode()
+_C.BMN.NUM_SAMPLES = 32
+_C.BMN.NUM_SAMPLES_PER_BIN = 3
+_C.BMN.PROP_BOUNDARY_RATIO = 0.5
+
+_C.BMN.POST_PROCESS = CfgNode()
+_C.BMN.POST_PROCESS.USE_HARD_NMS = False
+_C.BMN.POST_PROCESS.SOFT_NMS_ALPHA = 0.4
+_C.BMN.POST_PROCESS.SOFT_NMS_LOW_THRESHOLD = 0.5
+_C.BMN.POST_PROCESS.SOFT_NMS_HIGH_THRESHOLD = 0.9
+_C.BMN.POST_PROCESS.HARD_NMS_THRESHOLD = 0.65
+_C.BMN.POST_PROCESS.NUM_THREADS = 12
+_C.BMN.POST_PROCESS.MAX_PROPOSALS = 100
+
+
+def _assert_and_infer_cfg(cfg):
+    assert cfg.TRAIN.BATCH_SIZE % len(cfg.GPU_IDS) == 0
+    return cfg
+
+
+def get_cfg():
+    """
+    Get a copy of the default config.
+    """
+    return _assert_and_infer_cfg(_C.clone())