diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 1a706026..0a97bdb2 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -57,6 +57,19 @@ jobs: use-llms: "" secrets: inherit # pragma: allowlist secret + test-playwright-without-llms: + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + fail-fast: false + uses: ./.github/workflows/test-playwright.yaml + with: + python-version: ${{ matrix.python-version }} + environment: null + use-llms: "" + secrets: inherit # pragma: allowlist secret + + test-with-anthropic: uses: ./.github/workflows/test.yaml with: @@ -137,6 +150,7 @@ jobs: needs: - test-without-llms - test-with-llm + - test-playwright-without-llms - test-with-anthropic - test-with-azure_oai - test-with-openai @@ -165,8 +179,8 @@ jobs: - run: ls -la coverage - run: coverage combine coverage - - run: coverage report - - run: coverage html --show-contexts --title "FastAgency coverage for ${{ github.sha }}" + - run: coverage report -i + - run: coverage html -i --show-contexts --title "FastAgency coverage for ${{ github.sha }}" - name: Store coverage html uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test-playwright.yaml b/.github/workflows/test-playwright.yaml new file mode 100644 index 00000000..11fce552 --- /dev/null +++ b/.github/workflows/test-playwright.yaml @@ -0,0 +1,143 @@ +name: Internal test runner + +on: + workflow_call: + inputs: + environment: + description: 'Environment to run the tests in' + required: false + default: null + type: string + python-version: + description: 'Python version to run the tests in' + required: true + type: string + use-llms: + description: 'Use LLM in the tests' + required: false + type: string + default: "" + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: ${{ inputs.environment }} + services: + nats: + image: diementros/nats:js + ports: + - 4222:4222 + env: + NATS_URL: nats://localhost:4222 + steps: + - name: Set up environment variables + run: | + # check if an environment var or secret is defined and set env var to its value + + # vars + + if [ -n "${{ vars.AZURE_API_VERSION }}" ]; then + echo "AZURE_API_VERSION=${{ vars.AZURE_API_VERSION }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_API_ENDPOINT }}" ]; then + echo "AZURE_API_ENDPOINT=${{ vars.AZURE_API_ENDPOINT }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_GPT35_MODEL }}" ]; then + echo "AZURE_GPT35_MODEL=${{ vars.AZURE_GPT35_MODEL }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_GPT4_MODEL }}" ]; then + echo "AZURE_GPT4_MODEL=${{ vars.AZURE_GPT4_MODEL }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_GPT4o_MODEL }}" ]; then + echo "AZURE_GPT4o_MODEL=${{ vars.AZURE_GPT4o_MODEL }}" >> $GITHUB_ENV + fi + + # secrets + + if [ -n "${{ secrets.AZURE_OPENAI_API_KEY }}" ]; then + echo "AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.TOGETHER_API_KEY }}" ]; then + echo "TOGETHER_API_KEY=${{ secrets.TOGETHER_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then + echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.ANTHROPIC_API_KEY }}" ]; then + echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.BING_API_KEY }}" ]; then + echo "BING_API_KEY=${{ secrets.BING_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.POSTMAN_API_KEY }}" ]; then + echo "POSTMAN_API_KEY=${{ secrets.POSTMAN_API_KEY }}" >> $GITHUB_ENV + fi + + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: "pip" + cache-dependency-path: pyproject.toml + - uses: actions/cache@v4 + id: cache + with: + path: ${{ env.pythonLocation }} + key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-test-v03 + - name: Install Dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: pip install .[docs,testing] + - name: Install Pydantic v2 + run: pip install --pre "pydantic>=2,<3" + - run: mkdir coverage + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: npm ci + - name: Install Playwright Browsers + run: npx playwright install --with-deps + - name: Start fastagency + run: | + # Start fastagency and grab its pid + nohup fastagency run e2e/llm-sans/main.py & + # Get the process ID (PID) + FAST_PID=$! + echo "Started fastagency with PID: $FAST_PID" + env: + COVERAGE_PROCESS_START: e2e/playwright.coverage.cfg + - name: Run Playwright tests without LLMs + if: ${{ inputs.python-version != '3.9' }} + run: npx playwright test -c "playwright.llm-sans.config.ts" + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 + + - name: Kill the program + run: | + # Send SIGTERM to the program (graceful shutdown) + echo "killing the fastagency:", $FAST_PID + kill $FAST_PID + # Wait for the program to exit (timeout after 10 seconds) + wait -f $PID || timeout 50 kill -9 $PID + echo "Fastagncy killed and exited." + + - run: ls -al .coverage + - name: Move coverage file + run: mv .coverage .coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} + - name: Check coverage file + run: ls -al coverage + - name: Store coverage files + if: ${{ inputs.python-version != '3.9' }} + uses: actions/upload-artifact@v4 + with: + name: .coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} + path: coverage/.coverage.playwright.* + if-no-files-found: error + overwrite: true + include-hidden-files: true diff --git a/e2e/playwright.base.config.ts b/e2e/playwright.base.config.ts index 203619dc..64048b0a 100644 --- a/e2e/playwright.base.config.ts +++ b/e2e/playwright.base.config.ts @@ -17,10 +17,13 @@ import { devices, defineConfig as originalDefineConfig, PlaywrightTestConfig } f * See https://playwright.dev/docs/test-configuration. */ +//const testEnv = { ...process.env, COVERAGE_PROCESS_START: 'playwright.coverage.cfg' } as { [key: string]: string } + + const baseConfig: PlaywrightTestConfig = { testDir: './e2e', /* Run tests in files in parallel */ - fullyParallel: true, + fullyParallel: false, /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ @@ -80,7 +83,10 @@ const baseConfig: PlaywrightTestConfig = { webServer: { command: 'fastagency run e2e/llm-sans/main.py', url: 'http://127.0.0.1:32123', - reuseExistingServer: !process.env.CI, + // env: testEnv, + reuseExistingServer: true, + //reuseExistingServer: !process.env.CI, + //reuseExistingServer: false, }, } diff --git a/fastagency/app.py b/fastagency/app.py index 3f00eddf..6e44770e 100644 --- a/fastagency/app.py +++ b/fastagency/app.py @@ -1,5 +1,6 @@ __all__ = ["FastAgency"] +import os from collections.abc import Awaitable, Generator from contextlib import contextmanager from typing import Any, Callable, Optional, Union @@ -37,6 +38,13 @@ def __init__( title (Optional[str], optional): The title of the FastAgency. If None, the default string will be used. Defaults to None. description (Optional[str], optional): The description of the FastAgency. If None, the default string will be used. Defaults to None. """ + # check if we need to start coverage + coverage_process_start = os.environ.get("COVERAGE_PROCESS_START") + if coverage_process_start: + logger.info(f"Coverage process start detected: {coverage_process_start}") + import coverage + + coverage.process_startup() _self: Runnable = self self._title = title or "FastAgency application" self._description = description or "FastAgency application" diff --git a/playwright.gunicorn.config.ts b/playwright.gunicorn.config.ts index 9081e8b0..3ba8c91a 100644 --- a/playwright.gunicorn.config.ts +++ b/playwright.gunicorn.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "./e2e/playwright.base.config.ts"; export default defineConfig( { - testDir: './e2e/llm', + testDir: './e2e/llm-sans', use: { /* Base URL to use in actions like `await page.goto('/')`. */ baseURL: 'http://127.0.0.1:8000',