Skip to content

Commit

Permalink
Merge pytest parser options (#3269)
Browse files Browse the repository at this point in the history 
* Merge pytest parser options

Signed-off-by: Kim, Vinnam <[email protected]>

* Fix pre-commit ruff

Signed-off-by: Kim, Vinnam <[email protected]>

---------

Signed-off-by: Kim, Vinnam <[email protected]>
  • Loading branch information
@vinnamkim
vinnamkim authored Apr 8, 2024
1 parent 61196fa commit 0a18171
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 149 deletions.
101 changes: 101 additions & 0 deletions tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,107 @@
from torchvision.tv_tensors import Image, Mask


def pytest_addoption(parser: pytest.Parser):
"""Add custom options for perf tests."""
parser.addoption(
"--model-category",
action="store",
default="all",
choices=("speed", "balance", "accuracy", "default", "other", "all"),
help="Choose speed|balcence|accuracy|default|other|all. Defaults to all.",
)
parser.addoption(
"--data-group",
action="store",
default="all",
choices=("small", "medium", "large", "all"),
help="Choose small|medium|large|all. Defaults to all.",
)
parser.addoption(
"--num-repeat",
action="store",
default=0,
help="Overrides default per-data-group number of repeat setting. "
"Random seeds are set to 0 ~ num_repeat-1 for the trials. "
"Defaults to 0 (small=3, medium=3, large=1).",
)
parser.addoption(
"--num-epoch",
action="store",
default=0,
help="Overrides default per-model number of epoch setting. "
"Defaults to 0 (per-model epoch & early-stopping).",
)
parser.addoption(
"--eval-upto",
action="store",
default="train",
choices=("train", "export", "optimize"),
help="Choose train|export|optimize. Defaults to train.",
)
parser.addoption(
"--data-root",
action="store",
default="data",
help="Dataset root directory.",
)
parser.addoption(
"--output-root",
action="store",
help="Output root directory. Defaults to temp directory.",
)
parser.addoption(
"--summary-csv",
action="store",
help="Path to output summary cvs file. Defaults to {output-root}/benchmark-summary.csv",
)
parser.addoption(
"--dry-run",
action="store_true",
default=False,
help="Print OTX commands without execution.",
)
parser.addoption(
"--deterministic",
action="store_true",
default=False,
help="Turn on deterministic training.",
)
parser.addoption(
"--user-name",
type=str,
default="anonymous",
help='Sign-off the user name who launched the regression tests this time, e.g., `--user-name "John Doe"`.',
)
parser.addoption(
"--mlflow-tracking-uri",
type=str,
help="URI for MLFlow Tracking server to store the regression test results.",
)
parser.addoption(
"--otx-ref",
type=str,
default="__CURRENT_BRANCH_COMMIT__",
help="Target OTX ref (tag / branch name / commit hash) on main repo to test. Defaults to the current branch. "
"`pip install otx[full]@https://github.com/openvinotoolkit/training_extensions.git@{otx_ref}` will be executed before run, "
"and reverted after run. Works only for v2.x assuming CLI compatibility.",
)
parser.addoption(
"--open-subprocess",
action="store_true",
help="Open subprocess for each CLI test case. "
"This option can be used for easy memory management "
"while running consecutive multiple tests (default: false).",
)
parser.addoption(
"--task",
action="store",
default="all",
type=str,
help="Task type of OTX to use test.",
)


@pytest.fixture(scope="session")
def fxt_seg_data_entity() -> tuple[tuple, SegDataEntity, SegBatchDataEntity]:
img_size = (32, 32)
Expand Down
17 changes: 0 additions & 17 deletions tests/e2e/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,6 @@
from tests.test_helpers import find_folder


def pytest_addoption(parser: pytest.Parser) -> None:
parser.addoption(
"--open-subprocess",
action="store_true",
help="Open subprocess for each CLI test case. "
"This option can be used for easy memory management "
"while running consecutive multiple tests (default: false).",
)
parser.addoption(
"--task",
action="store",
default="all",
type=str,
help="Task type of OTX to use test.",
)


@pytest.fixture(scope="session")
def fxt_ci_data_root() -> Path:
data_root = Path(os.environ.get("CI_DATA_ROOT", "/home/validation/data/v2"))
Expand Down
17 changes: 0 additions & 17 deletions tests/integration/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,6 @@
from otx.core.types.task import OTXTaskType


def pytest_addoption(parser: pytest.Parser) -> None:
parser.addoption(
"--open-subprocess",
action="store_true",
help="Open subprocess for each CLI integration test case. "
"This option can be used for easy memory management "
"while running consecutive multiple tests (default: false).",
)
parser.addoption(
"--task",
action="store",
default="all",
type=str,
help="Task type of OTX to use integration test.",
)


@pytest.fixture(scope="module", autouse=True)
def fxt_open_subprocess(request: pytest.FixtureRequest) -> bool:
"""Open subprocess for each CLI integration test case.
Expand Down
87 changes: 0 additions & 87 deletions tests/perf/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,93 +21,6 @@
log = logging.getLogger(__name__)


def pytest_addoption(parser):
"""Add custom options for perf tests."""
parser.addoption(
"--model-category",
action="store",
default="all",
choices=("speed", "balance", "accuracy", "default", "other", "all"),
help="Choose speed|balcence|accuracy|default|other|all. Defaults to all.",
)
parser.addoption(
"--data-group",
action="store",
default="all",
choices=("small", "medium", "large", "all"),
help="Choose small|medium|large|all. Defaults to all.",
)
parser.addoption(
"--num-repeat",
action="store",
default=0,
help="Overrides default per-data-group number of repeat setting. "
"Random seeds are set to 0 ~ num_repeat-1 for the trials. "
"Defaults to 0 (small=3, medium=3, large=1).",
)
parser.addoption(
"--num-epoch",
action="store",
default=0,
help="Overrides default per-model number of epoch setting. "
"Defaults to 0 (per-model epoch & early-stopping).",
)
parser.addoption(
"--eval-upto",
action="store",
default="train",
choices=("train", "export", "optimize"),
help="Choose train|export|optimize. Defaults to train.",
)
parser.addoption(
"--data-root",
action="store",
default="data",
help="Dataset root directory.",
)
parser.addoption(
"--output-root",
action="store",
help="Output root directory. Defaults to temp directory.",
)
parser.addoption(
"--summary-csv",
action="store",
help="Path to output summary cvs file. Defaults to {output-root}/benchmark-summary.csv",
)
parser.addoption(
"--dry-run",
action="store_true",
default=False,
help="Print OTX commands without execution.",
)
parser.addoption(
"--deterministic",
action="store_true",
default=False,
help="Turn on deterministic training.",
)
parser.addoption(
"--user-name",
type=str,
default="anonymous",
help='Sign-off the user name who launched the regression tests this time, e.g., `--user-name "John Doe"`.',
)
parser.addoption(
"--mlflow-tracking-uri",
type=str,
help="URI for MLFlow Tracking server to store the regression test results.",
)
parser.addoption(
"--otx-ref",
type=str,
default="__CURRENT_BRANCH_COMMIT__",
help="Target OTX ref (tag / branch name / commit hash) on main repo to test. Defaults to the current branch. "
"`pip install otx[full]@https://github.com/openvinotoolkit/training_extensions.git@{otx_ref}` will be executed before run, "
"and reverted after run. Works only for v2.x assuming CLI compatibility.",
)


@pytest.fixture(scope="session")
def fxt_model_category(request: pytest.FixtureRequest) -> str:
"""Model category to run the benchmark."""
Expand Down
29 changes: 1 addition & 28 deletions tests/regression/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,33 +19,6 @@
log = logging.getLogger(__name__)


def pytest_addoption(parser: pytest.Parser) -> None:
parser.addoption(
"--user-name",
type=str,
required=True,
help="Sign-off the user name who launched the regression tests this time, " 'e.g., `--user-name "John Doe"`.',
)
parser.addoption(
"--dataset-root-dir",
type=Path,
required=True,
help="Dataset root directory path for the regression tests",
)
parser.addoption(
"--mlflow-tracking-uri",
type=str,
required=True,
help="URI for MLFlow Tracking server to store the regression test results.",
)
parser.addoption(
"--num-repeat",
type=int,
default=1,
help="The number of repetitions for each test case with different seed (default=1).",
)


@pytest.fixture(scope="module", autouse=True)
def fxt_user_name(request: pytest.FixtureRequest) -> str:
"""User name to sign off the regression test execution.
Expand All @@ -64,7 +37,7 @@ def fxt_dataset_root_dir(request: pytest.FixtureRequest) -> Path:
This should be given by the PyTest CLI option.
"""
dataset_root_dir = request.config.getoption("--dataset-root-dir")
dataset_root_dir = request.config.getoption("--data-root")
msg = f"dataset_root_dir: {dataset_root_dir}"
log.info(msg)
return dataset_root_dir
Expand Down

0 comments on commit 0a18171

Please sign in to comment.