From f1e2b74f0937ed171a02fbf4e4bcc02b7661cecc Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 14:34:16 -0400 Subject: [PATCH 1/9] Add browser fixture in function scope --- pyproject.toml | 5 ----- tests/README.md | 23 +++++++++++++++++++++++ tests/conftest.py | 9 ++++++++- tests/test_placeholder.py | 15 +++++++++++++-- tests/test_process.py | 10 ++++++++-- 5 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 tests/README.md diff --git a/pyproject.toml b/pyproject.toml index c823f6e9..01cdd267 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,8 +30,3 @@ enabled = true [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" -# addopts = "-v -n auto --timeout=4" -## Verbosity: -# -vv -rA -## Parallelism: -# -n auto diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..61e3ac13 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,23 @@ +# Testing + +## Running: + +### Process Control Tests + +- Verbose: `pytest -W error -n auto -vvv -rA --capture=tee-sys tests/test_process.py` +- Quiet:`pytest -W error -n auto -v -rFe --capture=fd tests/test_process.py` + +### Browser Interaction Tests + +- Verbose: `pytest --debug -n auto -W error -vvv -rA --capture=tee-sys --ignore=tests/test_process.py` +- Quiet :`pytest -W error -n auto -v -rFe --capture=fd --ignore=tests/test_process.py` + +You can also add "--no-headless" to these if you want to see the browser pop up + +## Writing Tests: + +Put async and sync tests in different files. Add `_sync.py` to synchronous tests. + +If doing process tests, maybe use the same decorators and fixtures in the `test_process.py` file. + +If doing browser interaction tests, use `test_placeholder.py` as the minimum template. diff --git a/tests/conftest.py b/tests/conftest.py index ca437bed..da747546 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ import pytest - +import pytest_asyncio +import devtools @pytest.fixture(params=[True, False], ids=["headless", ""]) def headless(request): @@ -14,3 +15,9 @@ def debug(request): @pytest.fixture(params=[True, False], ids=["debug_browser", ""]) def debug_browser(request): return request.param + +@pytest_asyncio.fixture(scope="function", loop_scope="function") +async def browser(): + browser = await devtools.Browser() + yield browser + await browser.close() diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py index 201975fc..d626402d 100644 --- a/tests/test_placeholder.py +++ b/tests/test_placeholder.py @@ -1,2 +1,13 @@ -def test_placeholder(): - pass +import pytest +import asyncio + +# allows to create a browser pool for tests +pytestmark = pytest.mark.asyncio(loop_scope="function") + +async def test_placeholder(capsys, browser): + print("") + assert capsys.readouterr().out == "\n", "stdout should be silent!" + + assert "result" in await browser.send_command("Target.getTargets") + + await asyncio.sleep(0) diff --git a/tests/test_process.py b/tests/test_process.py index c432bca6..7a43721e 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -6,7 +6,7 @@ from async_timeout import timeout -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") async def test_context(capsys, headless, debug, debug_browser): async with devtools.Browser( headless=headless, @@ -16,13 +16,16 @@ async def test_context(capsys, headless, debug, debug_browser): response = await browser.send_command(command="Target.getTargets") assert "result" in response and "targetInfos" in response["result"] assert (len(response["result"]["targetInfos"]) != 0) != headless + if not headless: + assert isinstance(browser.get_tab(), devtools.tab.Tab) + assert len(browser.get_tab().sessions) == 1 print("") # this makes sure that capturing is working # stdout should be empty, but not because capsys is broken, because nothing was print assert capsys.readouterr().out == "\n", "stdout should be silent!" # let asyncio do some cleaning up if it wants, may prevent warnings await asyncio.sleep(0) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") async def test_no_context(capsys, headless, debug, debug_browser): browser = await devtools.Browser( headless=headless, @@ -34,6 +37,9 @@ async def test_no_context(capsys, headless, debug, debug_browser): response = await browser.send_command(command="Target.getTargets") assert "result" in response and "targetInfos" in response["result"] assert (len(response["result"]["targetInfos"]) != 0) != headless + if not headless: + assert isinstance(browser.get_tab(), devtools.tab.Tab) + assert len(browser.get_tab().sessions) == 1 finally: await browser.close() print("") # this make sure that capturing is working From 7b5f87da151f89a653a834cbc54f99e74103fc82 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 15:15:14 -0400 Subject: [PATCH 2/9] Create basic timeout for all browser consumers --- tests/conftest.py | 38 ++++++++++++++++++++++++++++++++++++++ tests/test_process.py | 4 ++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index da747546..06cae116 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import asyncio import pytest import pytest_asyncio import devtools @@ -18,6 +19,43 @@ def debug_browser(request): @pytest_asyncio.fixture(scope="function", loop_scope="function") async def browser(): + # this needs also to be set by command line TODO browser = await devtools.Browser() yield browser await browser.close() + +@pytest_asyncio.fixture(scope="function", loop_scope="function") +async def browser_verbose(): + browser = await devtools.Browser(debug=True, debug_browser=True) + yield browser + await browser.close() + +# add a 2 second timeout if there is a browser fixture +# we do timeouts manually in test_process because it +# does/tests its own browser.close() +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_setup(item: pytest.Item): + yield + + if "browser" in item.funcargs or "browser_verbose" not in item.funcargs: + raw_test_fn = item.obj + timeouts = [k for k in item.funcargs if k.startswith("timeout")] + timeout = item.funcargs[timeouts[-1]] if len(timeouts) else pytest.default_timeout + if item.get_closest_marker("asyncio") and timeout: + async def wrapped_test_fn(*args, **kwargs): + try: + return await asyncio.wait_for( + raw_test_fn(*args, **kwargs), timeout=timeout + ) + except TimeoutError: + pytest.fail(f"Test {item.name} failed a timeout. This can be extended, but shouldn't be. See conftest.py.") + item.obj = wrapped_test_fn + +def pytest_configure(): + # change this by command line TODO, throw warning if timeout longer than global + pytest.default_timeout = 4 + +# add this fixture to extend timeout +@pytest.fixture(scope="session") +def timeout_long(): + return 10 diff --git a/tests/test_process.py b/tests/test_process.py index 7a43721e..5a4f6db6 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -12,7 +12,7 @@ async def test_context(capsys, headless, debug, debug_browser): headless=headless, debug=debug, debug_browser=debug_browser, - ) as browser, timeout(2): + ) as browser, timeout(pytest.default_timeout): response = await browser.send_command(command="Target.getTargets") assert "result" in response and "targetInfos" in response["result"] assert (len(response["result"]["targetInfos"]) != 0) != headless @@ -33,7 +33,7 @@ async def test_no_context(capsys, headless, debug, debug_browser): debug_browser=debug_browser, ) try: - async with timeout(2): + async with timeout(pytest.default_timeout): response = await browser.send_command(command="Target.getTargets") assert "result" in response and "targetInfos" in response["result"] assert (len(response["result"]["targetInfos"]) != 0) != headless From abc3511de4bf5ddf4f4260e2ff305102392c6750 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 15:36:05 -0400 Subject: [PATCH 3/9] Add comments, fix numbers --- pyproject.toml | 1 + tests/conftest.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 01cdd267..d263bb64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,3 +30,4 @@ enabled = true [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" + diff --git a/tests/conftest.py b/tests/conftest.py index 06cae116..b10c9222 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,10 +52,12 @@ async def wrapped_test_fn(*args, **kwargs): item.obj = wrapped_test_fn def pytest_configure(): - # change this by command line TODO, throw warning if timeout longer than global - pytest.default_timeout = 4 + # change this by command line TODO + pytest.default_timeout = 3 # add this fixture to extend timeout +# there is 6 second max test length for all +# which kills all tests @pytest.fixture(scope="session") def timeout_long(): - return 10 + return 6 From 7ea825c5509ec0cc97607101526a903e004aab2e Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 16:13:59 -0400 Subject: [PATCH 4/9] Fix bad conditional --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index b10c9222..1867639c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,7 +37,7 @@ async def browser_verbose(): def pytest_runtest_setup(item: pytest.Item): yield - if "browser" in item.funcargs or "browser_verbose" not in item.funcargs: + if "browser" in item.funcargs or "browser_verbose" in item.funcargs: raw_test_fn = item.obj timeouts = [k for k in item.funcargs if k.startswith("timeout")] timeout = item.funcargs[timeouts[-1]] if len(timeouts) else pytest.default_timeout From ef7fb0c937f6823cc0969f84e35c9a805784a042 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 16:26:04 -0400 Subject: [PATCH 5/9] Increase timeouts for GHA --- .github/workflows/test.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2fd36ed8..d3cb7e01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,14 +18,14 @@ jobs: - name: Test Process Control DEBUG if: runner.debug run: xvfb-run pytest -W error -n auto -vvv -rA --capture=tee-sys tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 - name: Test Process Control if: ${{ ! runner.debug }} run: xvfb-run pytest -W error -n auto -v -rfE --capture=fd tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 - name: Test the Rest run: pytest -vv -rfE --ignore=tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 test-windows: runs-on: windows-latest steps: @@ -42,14 +42,14 @@ jobs: - name: Test Process Control DEBUG if: runner.debug run: pytest -W error -n auto -vvv -rA --capture=tee-sys tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 - name: Test Process Control if: ${{ ! runner.debug }} run: pytest -W error -n auto -v -rFe --capture=fd tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 - name: Test the Rest run: pytest -vv -rfE --ignore=tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 test-mac: runs-on: macos-latest steps: @@ -66,11 +66,11 @@ jobs: - name: Test Process Control DEBUG if: runner.debug run: pytest -W error -n auto -vvv -rA --capture=tee-sys tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 - name: Test Process Control if: ${{ ! runner.debug }} run: pytest -W error -n auto -v -rFe --capture=fd tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 - name: Test the Rest run: pytest -vv -rfE --ignore=tests/test_process.py - timeout-minutes: 1 + timeout-minutes: 2 From fae81a2a61f531b1cc3f9763c104ee288e5f01bb Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 16:35:43 -0400 Subject: [PATCH 6/9] Change to no capture on DEBUG --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3cb7e01..a50fda1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: xvfb-run pytest -W error -n auto -vvv -rA --capture=tee-sys tests/test_process.py + run: xvfb-run pytest -W error -n auto -vvv -rA -s tests/test_process.py timeout-minutes: 2 - name: Test Process Control if: ${{ ! runner.debug }} @@ -41,7 +41,7 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: pytest -W error -n auto -vvv -rA --capture=tee-sys tests/test_process.py + run: pytest -W error -n auto -vvv -rA -s tests/test_process.py timeout-minutes: 2 - name: Test Process Control if: ${{ ! runner.debug }} @@ -65,7 +65,7 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: pytest -W error -n auto -vvv -rA --capture=tee-sys tests/test_process.py + run: pytest -W error -n auto -vvv -rA -s tests/test_process.py timeout-minutes: 2 - name: Test Process Control if: ${{ ! runner.debug }} From cc3f9ce9b6bff79f8c4395183b981df1846237b3 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 16:38:05 -0400 Subject: [PATCH 7/9] Dont do debug in GHA en parallel --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a50fda1e..fd0d2271 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,8 +17,8 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: xvfb-run pytest -W error -n auto -vvv -rA -s tests/test_process.py - timeout-minutes: 2 + run: xvfb-run pytest -W error -vvv -rA -s tests/test_process.py + timeout-minutes: 5 - name: Test Process Control if: ${{ ! runner.debug }} run: xvfb-run pytest -W error -n auto -v -rfE --capture=fd tests/test_process.py @@ -41,8 +41,8 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: pytest -W error -n auto -vvv -rA -s tests/test_process.py - timeout-minutes: 2 + run: pytest -W error -vvv -rA -s tests/test_process.py + timeout-minutes: 5 - name: Test Process Control if: ${{ ! runner.debug }} run: pytest -W error -n auto -v -rFe --capture=fd tests/test_process.py @@ -65,8 +65,8 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: pytest -W error -n auto -vvv -rA -s tests/test_process.py - timeout-minutes: 2 + run: pytest -W error -vvv -rA -s tests/test_process.py + timeout-minutes: 5 - name: Test Process Control if: ${{ ! runner.debug }} run: pytest -W error -n auto -v -rFe --capture=fd tests/test_process.py From b891922187861efbc4ad9df5658e36f1f438f6f4 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 17:07:53 -0400 Subject: [PATCH 8/9] Bring back tee-sys and try w/o capsys --- .github/workflows/test.yml | 6 +++--- tests/test_process.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd0d2271..be200e45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: xvfb-run pytest -W error -vvv -rA -s tests/test_process.py + run: xvfb-run pytest -W error -vvv -rA --capture=tee-sys tests/test_process.py timeout-minutes: 5 - name: Test Process Control if: ${{ ! runner.debug }} @@ -41,7 +41,7 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: pytest -W error -vvv -rA -s tests/test_process.py + run: pytest -W error -vvv -rA --capture=tee-sys tests/test_process.py timeout-minutes: 5 - name: Test Process Control if: ${{ ! runner.debug }} @@ -65,7 +65,7 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: pytest -W error -vvv -rA -s tests/test_process.py + run: pytest -W error -vvv -rA --capute=tee-sys tests/test_process.py timeout-minutes: 5 - name: Test Process Control if: ${{ ! runner.debug }} diff --git a/tests/test_process.py b/tests/test_process.py index 5a4f6db6..d20a875a 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -7,7 +7,7 @@ @pytest.mark.asyncio(loop_scope="function") -async def test_context(capsys, headless, debug, debug_browser): +async def test_context(headless, debug, debug_browser): async with devtools.Browser( headless=headless, debug=debug, @@ -21,7 +21,7 @@ async def test_context(capsys, headless, debug, debug_browser): assert len(browser.get_tab().sessions) == 1 print("") # this makes sure that capturing is working # stdout should be empty, but not because capsys is broken, because nothing was print - assert capsys.readouterr().out == "\n", "stdout should be silent!" + #assert capsys.readouterr().out == "\n", "stdout should be silent!" # let asyncio do some cleaning up if it wants, may prevent warnings await asyncio.sleep(0) From 5b17f459953efeda6fbf593652dd4cc5b727d16f Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Fri, 4 Oct 2024 17:09:02 -0400 Subject: [PATCH 9/9] Fix typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be200e45..495948d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,7 +65,7 @@ jobs: timeout-minutes: 1 - name: Test Process Control DEBUG if: runner.debug - run: pytest -W error -vvv -rA --capute=tee-sys tests/test_process.py + run: pytest -W error -vvv -rA --capture=tee-sys tests/test_process.py timeout-minutes: 5 - name: Test Process Control if: ${{ ! runner.debug }}