diff --git a/.github/workflows/test_onnxruntime.yml b/.github/workflows/test_onnxruntime.yml index e4463bb0a3..bf7f15e263 100644 --- a/.github/workflows/test_onnxruntime.yml +++ b/.github/workflows/test_onnxruntime.yml @@ -58,10 +58,10 @@ jobs: - name: Test with pytest (in series) run: | - pytest tests/onnxruntime -m "run_in_series" --durations=0 -vvvv -s + pytest tests/onnxruntime -m "run_in_series" --durations=0 -vvvv - name: Test with pytest (in parallel) run: | - pytest tests/onnxruntime -m "not run_in_series" --durations=0 -vvvv -s -n auto + pytest tests/onnxruntime -m "not run_in_series" --durations=0 -vvvv -n auto env: HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} diff --git a/.github/workflows/test_onnxruntime_gpu.yml b/.github/workflows/test_onnxruntime_gpu.yml index 70770a1aa5..45c9bb89b7 100644 --- a/.github/workflows/test_onnxruntime_gpu.yml +++ b/.github/workflows/test_onnxruntime_gpu.yml @@ -51,4 +51,4 @@ jobs: - name: Test with pytest run: | - pytest tests/onnxruntime -m "cuda_ep_test or trt_ep_test" --durations=0 -vvvv -s -n auto + pytest tests/onnxruntime -m "cuda_ep_test or trt_ep_test" --durations=0 -vvvv -n auto diff --git a/.github/workflows/test_onnxruntime_slow.yml b/.github/workflows/test_onnxruntime_slow.yml index f18881c1bb..603b44c4fe 100644 --- a/.github/workflows/test_onnxruntime_slow.yml +++ b/.github/workflows/test_onnxruntime_slow.yml @@ -47,4 +47,4 @@ jobs: - name: Test with pytest run: | - RUN_SLOW=1 pytest tests/onnxruntime -m "run_slow" --durations=0 -s -vvvv -n auto + RUN_SLOW=1 pytest tests/onnxruntime -m "run_slow" --durations=0 -vvvv diff --git a/.github/workflows/test_onnxruntime_training.yml b/.github/workflows/test_onnxruntime_training.yml index eceae9b2e3..ac3124e95c 100644 --- a/.github/workflows/test_onnxruntime_training.yml +++ b/.github/workflows/test_onnxruntime_training.yml @@ -55,12 +55,12 @@ jobs: - name: Test with pytest (trainer) run: | - RUN_SLOW=1 pytest tests/onnxruntime-training/test_trainer.py --durations=0 -s -vvvv + RUN_SLOW=1 pytest tests/onnxruntime-training/test_trainer.py --durations=0 -vvvv env: HF_DATASETS_TRUST_REMOTE_CODE: 1 - name: Test with pytest (examples) run: | - RUN_SLOW=1 pytest tests/onnxruntime-training/test_examples.py --durations=0 -s -vvvv + RUN_SLOW=1 pytest tests/onnxruntime-training/test_examples.py --durations=0 -vvvv env: HF_DATASETS_TRUST_REMOTE_CODE: 1 diff --git a/tests/onnxruntime/test_modeling.py b/tests/onnxruntime/test_modeling.py index 61c78bb378..f6efa447c8 100644 --- a/tests/onnxruntime/test_modeling.py +++ b/tests/onnxruntime/test_modeling.py @@ -2867,8 +2867,8 @@ class ORTModelForImageClassificationIntegrationTest(ORTModelTestMixin): ORTMODEL_CLASS = ORTModelForImageClassification TASK = "image-classification" - ATOL = 2e-3 - RTOL = 2e-3 + ATOL = 2e-3 # 0.02 difference in logits + RTOL = 1e-2 # 1% difference in logits def _get_model_ids(self, model_arch): model_ids = MODEL_NAMES[model_arch] diff --git a/tests/onnxruntime/test_timm.py b/tests/onnxruntime/test_timm.py index c51bcc01a0..2c175a37c2 100644 --- a/tests/onnxruntime/test_timm.py +++ b/tests/onnxruntime/test_timm.py @@ -17,38 +17,39 @@ class ORTModelForImageClassificationIntegrationTest(ORTModelTestMixin): TIMM_SUPPORTED_MODELS = [ "timm/inception_v3.tf_adv_in1k", - "timm/tf_efficientnet_b0.in1k", - "timm/cspdarknet53.ra_in1k", - "timm/cspresnet50.ra_in1k", - "timm/cspresnext50.ra_in1k", - "timm/densenet121.ra_in1k", - "timm/dla102.in1k", - "timm/dpn107.mx_in1k", - "timm/ecaresnet101d.miil_in1k", - "timm/efficientnet_b1_pruned.in1k", - "timm/inception_resnet_v2.tf_ens_adv_in1k", - "timm/fbnetc_100.rmsp_in1k", - "timm/xception41.tf_in1k", - "timm/senet154.gluon_in1k", - "timm/seresnext26d_32x4d.bt_in1k", - "timm/hrnet_w18.ms_aug_in1k", - "timm/inception_v3.gluon_in1k", - "timm/inception_v4.tf_in1k", - "timm/mixnet_s.ft_in1k", - "timm/mnasnet_100.rmsp_in1k", - "timm/mobilenetv2_100.ra_in1k", - "timm/mobilenetv3_small_050.lamb_in1k", - "timm/nasnetalarge.tf_in1k", - "timm/tf_efficientnet_b0.ns_jft_in1k", - "timm/pnasnet5large.tf_in1k", - "timm/regnetx_002.pycls_in1k", - "timm/regnety_002.pycls_in1k", - "timm/res2net101_26w_4s.in1k", - "timm/res2next50.in1k", - "timm/resnest101e.in1k", - "timm/spnasnet_100.rmsp_in1k", - "timm/resnet18.fb_swsl_ig1b_ft_in1k", - "timm/tresnet_l.miil_in1k", + # This is too much for the CI + # "timm/tf_efficientnet_b0.in1k", + # "timm/cspdarknet53.ra_in1k", + # "timm/cspresnet50.ra_in1k", + # "timm/cspresnext50.ra_in1k", + # "timm/densenet121.ra_in1k", + # "timm/dla102.in1k", + # "timm/dpn107.mx_in1k", + # "timm/ecaresnet101d.miil_in1k", + # "timm/efficientnet_b1_pruned.in1k", + # "timm/inception_resnet_v2.tf_ens_adv_in1k", + # "timm/fbnetc_100.rmsp_in1k", + # "timm/xception41.tf_in1k", + # "timm/senet154.gluon_in1k", + # "timm/seresnext26d_32x4d.bt_in1k", + # "timm/hrnet_w18.ms_aug_in1k", + # "timm/inception_v3.gluon_in1k", + # "timm/inception_v4.tf_in1k", + # "timm/mixnet_s.ft_in1k", + # "timm/mnasnet_100.rmsp_in1k", + # "timm/mobilenetv2_100.ra_in1k", + # "timm/mobilenetv3_small_050.lamb_in1k", + # "timm/nasnetalarge.tf_in1k", + # "timm/tf_efficientnet_b0.ns_jft_in1k", + # "timm/pnasnet5large.tf_in1k", + # "timm/regnetx_002.pycls_in1k", + # "timm/regnety_002.pycls_in1k", + # "timm/res2net101_26w_4s.in1k", + # "timm/res2next50.in1k", + # "timm/resnest101e.in1k", + # "timm/spnasnet_100.rmsp_in1k", + # "timm/resnet18.fb_swsl_ig1b_ft_in1k", + # "timm/tresnet_l.miil_in1k", ] @parameterized.expand(TIMM_SUPPORTED_MODELS)