From b397ebb1c240ed74097af8814198a72c4481a4c7 Mon Sep 17 00:00:00 2001 From: dythu Date: Wed, 23 Mar 2022 13:54:20 +0800 Subject: [PATCH] update --- README.md | 42 ++++++------------------------------------ detpro.sh | 12 ++++++++++++ prepare.sh | 4 ++++ transfer.sh | 16 ++++++++++------ vild_detpro.sh | 5 +++++ 5 files changed, 37 insertions(+), 42 deletions(-) create mode 100644 detpro.sh create mode 100644 prepare.sh create mode 100644 vild_detpro.sh diff --git a/README.md b/README.md index a968bdf..fc12c7b 100644 --- a/README.md +++ b/README.md @@ -48,42 +48,12 @@ pip install git+https://github.com/lvis-dataset/lvis-api.git pip install mmcv-full==1.2.5 -f https://download.openmmlab.com/mmcv/dist/cu110/torch1.7.0/index.html ``` # Get Started -## Prepare proposal embedding, label, iou for DetPro training -For training the DetPro, we need to save the embeddings of proposals embedded by the clip image encoder, correponding labels and ious with gt box. -```shell -./tools/dist_train.sh configs/lvis/prompt_save_train.py 8 --work-dir workdirs/prompt_save_train -./tools/dist_train.sh configs/lvis/prompt_save_val.py 8 --work-dir workdirs/prompt_save_val -``` +## Prepare data for DetPro training +see prepare.sh ## Train DetPro -```shell -python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_5_6_end soft 0.5 0.5 0.6 8 end -python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_6_7_end soft 0.5 0.6 0.7 8 end -python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_7_8_end soft 0.5 0.7 0.8 8 end -python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_8_9_end soft 0.5 0.8 0.9 8 end -python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_9_10_end soft 0.5 0.9 1.1 8 end -python prompt/run.py test data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_10_end soft 0.0 0.5 0.5 checkpoints/exp/fg_bg_5_5_6_endepoch6.pth checkpoints/exp/fg_bg_5_6_7_endepoch6.pth checkpoints/exp/fg_bg_5_7_8_endepoch6.pth checkpoints/exp/fg_bg_5_8_9_endepoch6.pth checkpoints/exp/fg_bg_5_9_10_endepoch6.pth -``` +see detpro.sh ## Train ViLD with DetPro (Mask R-CNN) -``` -#save the embeddings of precomputed proposals encoded by the clip image encoder for reuse，this will take a long time. -./tools/dist_train.sh configs/lvis/detpro_ens_20e.py 8 --work-dir workdirs/vild_ens_20e_fg_bg_5_10_end --cfg-options model.roi_head.prompt_path=checkpoints/exp/fg_bg_5_10_end_ens.pth model.roi_head.load_feature=False totol_epochs=1 -#compress the embeddings into a zip file -zip -r lvis_clip_image_embedding.zip data/lvis_clip_image_embedding/* -./tools/dist_train.sh configs/lvis/detpro_ens_20e.py 8 --work-dir workdirs/vild_ens_20e_fg_bg_5_10_end --cfg-options model.roi_head.prompt_path=checkpoints/exp/fg_bg_5_10_end_ens.pth model.roi_head.load_feature=True -``` -## Generate class embeddings for tranfer datasets (take Objects365 as example) -``` -python gen_cls_embedding.py checkpoints/exp/fg_bg_5_5_6_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 -python gen_cls_embedding.py checkpoints/exp/fg_bg_5_6_7_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 -python gen_cls_embedding.py checkpoints/exp/fg_bg_5_7_8_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 -python gen_cls_embedding.py checkpoints/exp/fg_bg_5_8_9_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 -python gen_cls_embedding.py checkpoints/exp/fg_bg_5_9_10_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 -python prompt/run.py test data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/obj365 fg_bg_5_10_obj365 soft 0.0 0.5 0.5 checkpoints/obj365/fg_bg_5_5_6_obj365.pth checkpoints/obj365/fg_bg_5_6_7_obj365.pth checkpoints/obj365/fg_bg_5_7_8_obj365.pth checkpoints/obj365/fg_bg_5_8_9_obj365.pth checkpoints/obj365/fg_bg_5_9_10_obj365.pth -``` -## Transfer to Pascal VOC, COCO and Objects365 -``` -./tools/dist_test.sh configs/transfer/transfer_voc.py workdirs/vild_ens_20e_fg_bg_5_10_end/epoch_20.pth 8 --eval mAP --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/voc/fg_bg_5_10_voc_ens.pth model.roi_head.fixed_lambda=0.6 -./tools/dist_test.sh configs/transfer/transfer_coco.py workdirs/vild_ens_20e_fg_bg_5_10_end/epoch_20.pth 8 --eval bbox --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/coco/fg_bg_5_10_voc_ens.pth model.roi_head.fixed_lambda=0.6 -./tools/dist_test.sh configs/transfer/transfer_objects365.py workdirs/vild_ens_20e_fg_bg_5_10_end/epoch_20.pth 8 --eval bbox --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/obj365/fg_bg_5_10_obj365_ens.pth model.roi_head.fixed_lambda=0.6 -``` +see vild_detpro.sh +## Tranfer Experiments +see transer.sh # Citation diff --git a/detpro.sh b/detpro.sh new file mode 100644 index 0000000..d36786a --- /dev/null +++ b/detpro.sh @@ -0,0 +1,12 @@ +#train prompt using positive with ious in [0,5,0.6] and negative proposals +python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_5_6_end soft 0.5 0.5 0.6 8 end +#train prompt using positive with ious in [0,6,0.7] and negative proposals +python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_6_7_end soft 0.5 0.6 0.7 8 end +#train prompt using positive with ious in [0,7,0.8] and negative proposals +python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_7_8_end soft 0.5 0.7 0.8 8 end +#train prompt using positive with ious in [0,8,0.9] and negative proposals +python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_8_9_end soft 0.5 0.8 0.9 8 end +#train prompt using positive with ious in [0,9,1.] and negative proposals +python prompt/run.py train data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_9_10_end soft 0.5 0.9 1.1 8 end +#ensemble +python prompt/run.py test data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/exp fg_bg_5_10_end soft 0.0 0.5 0.5 checkpoints/exp/fg_bg_5_5_6_endepoch6.pth checkpoints/exp/fg_bg_5_6_7_endepoch6.pth checkpoints/exp/fg_bg_5_7_8_endepoch6.pth checkpoints/exp/fg_bg_5_8_9_endepoch6.pth checkpoints/exp/fg_bg_5_9_10_endepoch6.pth \ No newline at end of file diff --git a/prepare.sh b/prepare.sh new file mode 100644 index 0000000..7cfae6d --- /dev/null +++ b/prepare.sh @@ -0,0 +1,4 @@ + +#For training the DetPro, we need to save the embeddings of proposals embedded by the clip image encoder, correponding labels and ious with gt boxes. +./tools/dist_train.sh configs/lvis/prompt_save_train.py 8 --work-dir workdirs/prompt_save_train +./tools/dist_train.sh configs/lvis/prompt_save_val.py 8 --work-dir workdirs/prompt_save_val \ No newline at end of file diff --git a/transfer.sh b/transfer.sh index e683cab..4fc89ac 100644 --- a/transfer.sh +++ b/transfer.sh @@ -1,11 +1,15 @@ -#Generate class embedding for tranfer Datasets(take objects365 as example) +# Generate class embeddings for tranfer datasets (take Objects365 as example) +``` python gen_cls_embedding.py checkpoints/exp/fg_bg_5_5_6_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 python gen_cls_embedding.py checkpoints/exp/fg_bg_5_6_7_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 python gen_cls_embedding.py checkpoints/exp/fg_bg_5_7_8_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 python gen_cls_embedding.py checkpoints/exp/fg_bg_5_8_9_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 python gen_cls_embedding.py checkpoints/exp/fg_bg_5_9_10_endepoch6_prompt.pth checkpoints/obj365/fg_bg_5_5_6_obj365 obj365 -python run.py test data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/obj365 fg_bg_5_10_obj365 soft 0.0 0.5 0.5 checkpoints/obj365/fg_bg_5_5_6_obj365.pth checkpoints/obj365/fg_bg_5_6_7_obj365.pth checkpoints/obj365/fg_bg_5_7_8_obj365.pth checkpoints/obj365/fg_bg_5_8_9_obj365.pth checkpoints/obj365/fg_bg_5_9_10_obj365.pth -# Transfer to other datasets -./tools/dist_test.sh configs/transfer/transfer_voc.py workdirs/vild_ens_2x_neg5_ens/epoch_24.pth 8 --eval mAP --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/voc/fg_bg_6_10_voc_neg30_ens.pth model.roi_head.fixed_lambda=0.6 -./tools/dist_test.sh configs/transfer/transfer_coco.py workdirs/vild_ens_2x_neg5_ens/epoch_24.pth 8 --eval bbox --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/coco/fg_bg_6_10_voc_neg30_ens.pth model.roi_head.fixed_lambda=0.6 -./tools/dist_test.sh configs/transfer/transfer_objects365.py workdirs/vild_ens_2x_neg5_ens/epoch_20.pth 8 --eval bbox --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/obj365/fg_bg_6_10_obj365_neg30_ens.pth model.roi_head.fixed_lambda=0.6 \ No newline at end of file +python prompt/run.py test data/lvis_clip_image_proposal_embedding/train data/lvis_clip_image_proposal_embedding/val checkpoints/obj365 fg_bg_5_10_obj365 soft 0.0 0.5 0.5 checkpoints/obj365/fg_bg_5_5_6_obj365.pth checkpoints/obj365/fg_bg_5_6_7_obj365.pth checkpoints/obj365/fg_bg_5_7_8_obj365.pth checkpoints/obj365/fg_bg_5_8_9_obj365.pth checkpoints/obj365/fg_bg_5_9_10_obj365.pth +``` +# Transfer to Pascal VOC, COCO and Objects365 +``` +./tools/dist_test.sh configs/transfer/transfer_voc.py workdirs/vild_ens_20e_fg_bg_5_10_end/epoch_20.pth 8 --eval mAP --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/voc/fg_bg_5_10_voc_ens.pth model.roi_head.fixed_lambda=0.6 +./tools/dist_test.sh configs/transfer/transfer_coco.py workdirs/vild_ens_20e_fg_bg_5_10_end/epoch_20.pth 8 --eval bbox --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/coco/fg_bg_5_10_voc_ens.pth model.roi_head.fixed_lambda=0.6 +./tools/dist_test.sh configs/transfer/transfer_objects365.py workdirs/vild_ens_20e_fg_bg_5_10_end/epoch_20.pth 8 --eval bbox --cfg-options model.roi_head.load_feature=False model.roi_head.prompt_path=checkpoints/obj365/fg_bg_5_10_obj365_ens.pth model.roi_head.fixed_lambda=0.6 +``` \ No newline at end of file diff --git a/vild_detpro.sh b/vild_detpro.sh new file mode 100644 index 0000000..b1f97a4 --- /dev/null +++ b/vild_detpro.sh @@ -0,0 +1,5 @@ +#save the embeddings of precomputed proposals encoded by the clip image encoder for reuse，this will take a long time. +./tools/dist_train.sh configs/lvis/detpro_ens_20e.py 8 --work-dir workdirs/vild_ens_20e_fg_bg_5_10_end --cfg-options model.roi_head.prompt_path=checkpoints/exp/fg_bg_5_10_end_ens.pth model.roi_head.load_feature=False totol_epochs=1 +#compress the embeddings into a zip file +zip -r lvis_clip_image_embedding.zip data/lvis_clip_image_embedding/* +./tools/dist_train.sh configs/lvis/detpro_ens_20e.py 8 --work-dir workdirs/vild_ens_20e_fg_bg_5_10_end --cfg-options model.roi_head.prompt_path=checkpoints/exp/fg_bg_5_10_end_ens.pth model.roi_head.load_feature=True \ No newline at end of file