From 4b31ac2a62b2662ff826eaa625f584c83a1df5f6 Mon Sep 17 00:00:00 2001 From: c-bata Date: Tue, 5 Nov 2024 18:03:35 +0900 Subject: [PATCH 1/4] Update e2e test to reproduce bug reported by @ZWL-S --- e2e_tests/test_dashboard/visual_regression_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e_tests/test_dashboard/visual_regression_test.py b/e2e_tests/test_dashboard/visual_regression_test.py index 3d38d6de0..424593307 100644 --- a/e2e_tests/test_dashboard/visual_regression_test.py +++ b/e2e_tests/test_dashboard/visual_regression_test.py @@ -38,11 +38,11 @@ def run_single_trial_objective_study(storage: optuna.storages.InMemoryStorage) - study = optuna.create_study(study_name="single-trial", storage=storage, sampler=sampler) def objective(trial: optuna.Trial) -> float: - x1 = trial.suggest_float("x1", 0, 10) - x2 = trial.suggest_float("x2", 0, 10) - return (x1 - 2) ** 2 + (x2 - 5) ** 2 + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y - study.optimize(objective, n_trials=1) + study.optimize(objective, n_trials=20) return study From 068ad1590ad44ef32d270eb3f845c0344dbcaee7 Mon Sep 17 00:00:00 2001 From: porink0424 Date: Wed, 6 Nov 2024 17:50:49 +0900 Subject: [PATCH 2/4] Fix clear_inmemory_cache() function --- e2e_tests/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/e2e_tests/utils.py b/e2e_tests/utils.py index 2520fc65c..0d6033e47 100644 --- a/e2e_tests/utils.py +++ b/e2e_tests/utils.py @@ -1,5 +1,7 @@ from optuna_dashboard._storage import trials_cache from optuna_dashboard._storage import trials_cache_lock +from optuna_dashboard._cached_extra_study_property import cached_extra_study_property_cache +from optuna_dashboard._cached_extra_study_property import cached_extra_study_property_cache_lock from optuna_dashboard._storage import trials_last_fetched_at from playwright.sync_api import Page @@ -8,6 +10,8 @@ def clear_inmemory_cache() -> None: with trials_cache_lock: trials_cache.clear() trials_last_fetched_at.clear() + with cached_extra_study_property_cache_lock: + cached_extra_study_property_cache.clear() def count_components(page: Page, component_name: str): From 4475bff5150ea2899e38059b4a288afc089cf6e6 Mon Sep 17 00:00:00 2001 From: porink0424 Date: Wed, 6 Nov 2024 17:51:24 +0900 Subject: [PATCH 3/4] Correct e2e tests --- .../test_dashboard/visual_regression_test.py | 92 ++++++++++--------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/e2e_tests/test_dashboard/visual_regression_test.py b/e2e_tests/test_dashboard/visual_regression_test.py index 424593307..8ea7dd864 100644 --- a/e2e_tests/test_dashboard/visual_regression_test.py +++ b/e2e_tests/test_dashboard/visual_regression_test.py @@ -169,20 +169,22 @@ def test_study_list( run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: study = run_study(storage) - study_id = study._study_id - study_name = study.study_name - + errors = [] + page.on( + "console", lambda message: errors.append(message.text) if message.type == "error" else None + ) page.goto(server_url) page.click(f"a[href='/dashboard/studies/{study_id}']") - page.wait_for_selector(".MuiTypography-body1") - element = page.query_selector(".MuiTypography-body1") - assert element is not None + # TODO(porink0424): Temporary sleep to wait for the page to load. + # This should be replaced with a more robust solution, + # such as capturing logs when rendering graphs. + import time - title = element.text_content() - assert title is not None - assert study_name in title + time.sleep(10) + + assert len(errors) == 0 @parameterize_studies @@ -194,20 +196,20 @@ def test_study_analytics( ) -> None: study = run_study(storage) study_id = study._study_id - study_name = study.study_name url = f"{server_url}/studies/{study_id}" - - page.on("console", lambda msg: print(f"error: {msg.text}") if msg.type == "error" else None) + errors = [] + page.on( + "console", lambda message: errors.append(message.text) if message.type == "error" else None + ) page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/analytics']") - page.wait_for_selector(".MuiTypography-body1", timeout=60 * 1000) - element = page.query_selector(".MuiTypography-body1") - assert element is not None + # TODO(porink0424): Same as above. + import time + + time.sleep(10) - title = element.text_content() - assert title is not None - assert study_name in title + assert len(errors) == 0 @parameterize_studies @@ -218,21 +220,21 @@ def test_trial_list( run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: study = run_study(storage) - study_id = study._study_id - study_name = study.study_name url = f"{server_url}/studies/{study_id}" - + errors = [] + page.on( + "console", lambda message: errors.append(message.text) if message.type == "error" else None + ) page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/trials']") - page.wait_for_selector(".MuiTypography-body1") - element = page.query_selector(".MuiTypography-body1") - assert element is not None + # TODO(porink0424): Same as above. + import time - title = element.text_content() - assert title is not None - assert study_name in title + time.sleep(10) + + assert len(errors) == 0 @parameterize_studies @@ -243,21 +245,21 @@ def test_trial_table( run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: study = run_study(storage) - study_id = study._study_id - study_name = study.study_name url = f"{server_url}/studies/{study_id}" - + errors = [] + page.on( + "console", lambda message: errors.append(message.text) if message.type == "error" else None + ) page.goto(url) page.click(f"a[href='/dashboard/studies/{study_id}/trialTable']") - page.wait_for_selector(".MuiTypography-body1") - element = page.query_selector(".MuiTypography-body1") - assert element is not None + # TODO(porink0424): Same as above. + import time + + time.sleep(10) - title = element.text_content() - assert title is not None - assert study_name in title + assert len(errors) == 0 @parameterize_studies @@ -268,18 +270,18 @@ def test_trial_note( run_study: Callable[[optuna.storages.InMemoryStorage], optuna.Study], ) -> None: study = run_study(storage) - study_id = study._study_id - study_name = study.study_name url = f"{server_url}/studies/{study_id}" - + errors = [] + page.on( + "console", lambda message: errors.append(message.text) if message.type == "error" else None + ) page.goto(url) - page.wait_for_selector(".MuiTypography-body1") page.click(f"a[href='/dashboard/studies/{study_id}/note']") - element = page.query_selector(".MuiTypography-body1") - assert element is not None + # TODO(porink0424): Same as above. + import time + + time.sleep(10) - title = element.text_content() - assert title is not None - assert study_name in title + assert len(errors) == 0 From 43fcf55501c5284442bc671a0fb5cc9e39905e99 Mon Sep 17 00:00:00 2001 From: porink0424 Date: Wed, 6 Nov 2024 18:12:19 +0900 Subject: [PATCH 4/4] Apply formatter --- e2e_tests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e_tests/utils.py b/e2e_tests/utils.py index 0d6033e47..ecd7d2cc1 100644 --- a/e2e_tests/utils.py +++ b/e2e_tests/utils.py @@ -1,7 +1,7 @@ -from optuna_dashboard._storage import trials_cache -from optuna_dashboard._storage import trials_cache_lock from optuna_dashboard._cached_extra_study_property import cached_extra_study_property_cache from optuna_dashboard._cached_extra_study_property import cached_extra_study_property_cache_lock +from optuna_dashboard._storage import trials_cache +from optuna_dashboard._storage import trials_cache_lock from optuna_dashboard._storage import trials_last_fetched_at from playwright.sync_api import Page