diff --git a/.github/workflows/testing-from-build.yml b/.github/workflows/testing-from-build.yml index 647b3897ff..b098ec1fca 100644 --- a/.github/workflows/testing-from-build.yml +++ b/.github/workflows/testing-from-build.yml @@ -54,29 +54,61 @@ jobs: coverageFile: ./coverage.xml token: ${{ secrets.GITHUB_TOKEN }} - # a11y-testing: - # runs-on: ubuntu-latest - # env: - # ENV: TESTING - # SAM_API_KEY: ${{ secrets.SAM_API_KEY }} - # DJANGO_BASE_URL: "http://localhost:8000" - # DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }} - # LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }} - # SECRET_KEY: ${{ secrets.SECRET_KEY }} - # ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost" - # DISABLE_AUTH: True - # steps: - # - uses: actions/checkout@v4 - # - uses: actions/setup-node@v4 - # with: - # node-version: 18 - # - name: Start services - # working-directory: ./backend - # run: | - # touch .env - # docker compose -f docker-compose.yml up -d - # - name: Run A11y tests - # working-directory: ./backend - # run: | - # sudo npm ci - # npx cypress run --spec "cypress/e2e/accessibility.cy.js" + a11y-testing: + runs-on: ubuntu-latest + env: + ENV: TESTING + SAM_API_KEY: ${{ secrets.SAM_API_KEY }} + DJANGO_BASE_URL: "http://localhost:8000" + DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }} + LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost" + DISABLE_AUTH: True + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install Node dependencies + working-directory: ./backend + run: npm ci + + - name: Create .env file + working-directory: ./backend + run: touch .env + + - name: Start services + working-directory: ./backend + run: docker compose -f docker-compose.yml up -d + + - name: Perform first run steps + working-directory: ./backend + run: make docker-first-run + + - name: Load fixtures + working-directory: ./backend + run: docker compose run web python manage.py load_fixtures + + - name: Create materialized view + working-directory: ./backend + run: docker compose run web python manage.py materialized_views --create + + - name: Run A11y tests + uses: cypress-io/github-action@v6 + with: + browser: chrome + working-directory: ./backend + spec: cypress/e2e/accessibility.cy.js + + - uses: actions/upload-artifact@v4 + # add the line below to store screenshots only on failures + if: failure() + with: + name: cypress-screenshots + path: backend/cypress/screenshots + if-no-files-found: warn # 'warn' or 'error' are also available, defaults to `warn` diff --git a/.github/workflows/testing-from-ghcr.yml b/.github/workflows/testing-from-ghcr.yml index 009a390b85..bca91c2757 100644 --- a/.github/workflows/testing-from-ghcr.yml +++ b/.github/workflows/testing-from-ghcr.yml @@ -1,8 +1,8 @@ --- name: Run Django and A11y Tests on: - workflow_dispatch: workflow_call: + workflow_dispatch: jobs: django-test: @@ -56,29 +56,61 @@ jobs: coverageFile: ./coverage.xml token: ${{ secrets.GITHUB_TOKEN }} - # a11y-testing: - # runs-on: ubuntu-latest - # env: - # ENV: TESTING - # SAM_API_KEY: ${{ secrets.SAM_API_KEY }} - # DJANGO_BASE_URL: "http://localhost:8000" - # DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }} - # LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }} - # SECRET_KEY: ${{ secrets.SECRET_KEY }} - # ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost" - # DISABLE_AUTH: True - # steps: - # - uses: actions/checkout@v4 - # - uses: actions/setup-node@v4 - # with: - # node-version: 16 - # - name: Start services - # working-directory: ./backend - # run: | - # touch .env - # docker compose -f docker-compose.yml up -d - # - name: Run A11y tests - # working-directory: ./backend - # run: | - # sudo npm ci - # npx cypress run --spec "cypress/e2e/accessibility.cy.js" + a11y-testing: + runs-on: ubuntu-latest + env: + ENV: TESTING + SAM_API_KEY: ${{ secrets.SAM_API_KEY }} + DJANGO_BASE_URL: "http://localhost:8000" + DJANGO_SECRET_LOGIN_KEY: ${{ secrets.DJANGO_SECRET_LOGIN_KEY }} + LOGIN_CLIENT_ID: ${{ secrets.LOGIN_CLIENT_ID }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + ALLOWED_HOSTS: "0.0.0.0 127.0.0.1 localhost" + DISABLE_AUTH: True + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install Node dependencies + working-directory: ./backend + run: npm ci + + - name: Create .env file + working-directory: ./backend + run: touch .env + + - name: Start services + working-directory: ./backend + run: docker compose -f docker-compose-web.yml up -d + + - name: Perform first run steps + working-directory: ./backend + run: make docker-first-run + + - name: Load fixtures + working-directory: ./backend + run: docker compose run web python manage.py load_fixtures + + - name: Create materialized view + working-directory: ./backend + run: docker compose run web python manage.py materialized_views --create + + - name: Run A11y tests + uses: cypress-io/github-action@v6 + with: + browser: chrome + working-directory: ./backend + spec: cypress/e2e/accessibility.cy.js + + - uses: actions/upload-artifact@v4 + # add the line below to store screenshots only on failures + if: failure() + with: + name: cypress-screenshots + path: backend/cypress/screenshots + if-no-files-found: warn # 'warn' or 'error' are also available, defaults to `warn` diff --git a/backend/audit/templates/audit/upload-report.html b/backend/audit/templates/audit/upload-report.html index f772c2606f..2b32ee85f3 100644 --- a/backend/audit/templates/audit/upload-report.html +++ b/backend/audit/templates/audit/upload-report.html @@ -97,7 +97,7 @@

{% if already_submitted %}Re-upload{% else - Cancel + Cancel diff --git a/backend/audit/views/views.py b/backend/audit/views/views.py index 6627272912..6e504e035f 100644 --- a/backend/audit/views/views.py +++ b/backend/audit/views/views.py @@ -58,6 +58,7 @@ validate_secondary_auditors_json, ) +from dissemination.remove_workbook_artifacts import remove_workbook_artifacts from dissemination.file_downloads import get_download_url, get_filename from dissemination.models import General @@ -741,6 +742,8 @@ def post(self, request, *args, **kwargs): event_user=request.user, event_type=SubmissionEvent.EventType.DISSEMINATED, ) + # Remove workbook artifacts after the report has been disseminated. + remove_workbook_artifacts(sac) else: pass # FIXME: We should now provide a reasonable error to the user. diff --git a/backend/dev-requirements.txt b/backend/dev-requirements.txt index 1615fb5aac..8f596b3af4 100644 --- a/backend/dev-requirements.txt +++ b/backend/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --allow-unsafe --generate-hashes --output-file=dev-requirements.txt ./requirements/dev-requirements.in @@ -115,9 +115,9 @@ coverage==7.4.0 \ cssbeautifier==1.14.11 \ --hash=sha256:40544c2b62bbcb64caa5e7f37a02df95654e5ce1bcacadac4ca1f3dc89c31513 # via djlint -django==5.0.4 \ - --hash=sha256:4bd01a8c830bb77a8a3b0e7d8b25b887e536ad17a81ba2dce5476135c73312bd \ - --hash=sha256:916423499d75d62da7aa038d19aef23d23498d8df229775eb0a6309ee1013775 +django==5.1 \ + --hash=sha256:848a5980e8efb76eea70872fb0e4bc5e371619c70fffbe48e3e1b50b2c09455d \ + --hash=sha256:d3b811bf5371a26def053d7ee42a9df1267ef7622323fe70a601936725aa4557 # via # -c ./requirements\../requirements.txt # django-debug-toolbar @@ -317,9 +317,9 @@ pbr==6.0.0 \ --hash=sha256:4a7317d5e3b17a3dccb6a8cfe67dab65b20551404c52c8ed41279fa4f0cb4cda \ --hash=sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9 # via stevedore -pip-tools==7.3.0 \ - --hash=sha256:8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e \ - --hash=sha256:8e9c99127fe024c025b46a0b2d15c7bd47f18f33226cf7330d35493663fc1d1d +pip-tools==7.4.1 \ + --hash=sha256:4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9 \ + --hash=sha256:864826f5073864450e24dbeeb85ce3920cdfb09848a3d69ebf537b521f14bcc9 # via -r ./requirements/dev-requirements.in platformdirs==4.1.0 \ --hash=sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380 \ @@ -340,7 +340,9 @@ pygments==2.17.2 \ pyproject-hooks==1.0.0 \ --hash=sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8 \ --hash=sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5 - # via build + # via + # build + # pip-tools python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 @@ -592,9 +594,9 @@ pip==23.3.2 \ --hash=sha256:5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76 \ --hash=sha256:7fd9972f96db22c8077a1ee2691b172c8089b17a5652a44494a9ecb0d78f9149 # via pip-tools -setuptools==69.4.0 \ - --hash=sha256:659e902e587e77fab8212358f5b03977b5f0d18d4724310d4a093929fee4ca1a \ - --hash=sha256:b6df12d754b505e4ca283c61582d5578db83ae2f56a979b3bc9a8754705ae3bf +setuptools==71.0.2 \ + --hash=sha256:ca359bea0cd5c8ce267d7463239107e87f312f2e2a11b6ca6357565d82b6c0d7 \ + --hash=sha256:f6640114f96be808024fbd1f721161215543796d3a68da4524349de700604ce8 # via # -c ./requirements\../requirements.txt # pip-tools diff --git a/backend/dissemination/remove_workbook_artifacts.py b/backend/dissemination/remove_workbook_artifacts.py new file mode 100644 index 0000000000..33d74c0495 --- /dev/null +++ b/backend/dissemination/remove_workbook_artifacts.py @@ -0,0 +1,74 @@ +import logging + +from django.conf import settings +from audit.models.models import ExcelFile +from boto3 import client as boto3_client +from botocore.client import ClientError, Config + +logger = logging.getLogger(__name__) + + +def remove_workbook_artifacts(sac): + """ + Remove all workbook artifacts associated with the given sac. + """ + try: + excel_files = ExcelFile.objects.filter(sac=sac) + files = [f"excel/{excel_file.filename}" for excel_file in excel_files] + + if files: + # Delete the records from the database + count = excel_files.count() + excel_files.delete() + logger.info( + f"Deleted {count} excelfile records from fac-db for report: {sac.report_id}" + ) + + # Delete the files from S3 in bulk + delete_files_in_bulk(files, sac) + + except ExcelFile.DoesNotExist: + logger.info(f"No files found to delete for report: {sac.report_id}") + except Exception as e: + logger.error( + f"Failed to delete files from fac-db and S3 for report: {sac.report_id}. Error: {e}" + ) + + +def delete_files_in_bulk(filenames, sac): + """Delete files from S3 in bulk.""" + # This client uses the internal endpoint URL because we're making a request to S3 from within the app + s3_client = boto3_client( + service_name="s3", + region_name=settings.AWS_S3_PRIVATE_REGION_NAME, + aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY, + endpoint_url=settings.AWS_S3_PRIVATE_INTERNAL_ENDPOINT, + config=Config(signature_version="s3v4"), + ) + + try: + delete_objects = [{"Key": filename} for filename in filenames] + + response = s3_client.delete_objects( + Bucket=settings.AWS_PRIVATE_STORAGE_BUCKET_NAME, + Delete={"Objects": delete_objects}, + ) + + deleted_files = response.get("Deleted", []) + for deleted in deleted_files: + logger.info( + f"Successfully deleted {deleted['Key']} from S3 for report: {sac.report_id}" + ) + + errors = response.get("Errors", []) + if errors: + for error in errors: + logger.error( + f"Failed to delete {error['Key']} from S3 for report: {sac.report_id}. Error: {error['Message']}" # nosec B608 + ) + + except ClientError as e: + logger.error( + f"Failed to delete files from S3 for report: {sac.report_id}. Error: {e}" + ) diff --git a/backend/dissemination/templates/summary.html b/backend/dissemination/templates/summary.html index dfe6cc34a9..3cdc781d75 100644 --- a/backend/dissemination/templates/summary.html +++ b/backend/dissemination/templates/summary.html @@ -53,7 +53,8 @@

Single audit summary

+ rel="noopener noreferrer" + aria-label="SF-SACs are typically available for download a day after submission. Click to learn more">

Upload completed worksheet - Cancel + Cancel diff --git a/backend/requirements.txt b/backend/requirements.txt index 4f09e62587..30c4848328 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt ./requirements/requirements.in @@ -240,9 +240,9 @@ dj-email-url==1.0.6 \ --hash=sha256:55ffe3329e48f54f8a75aa36ece08f365e09d61f8a209773ef09a1d4760e699a \ --hash=sha256:cbd08327fbb08b104eac160fb4703f375532e4c0243eb230f5b960daee7a96db # via environs -django==5.0.7 \ - --hash=sha256:bd4505cae0b9bd642313e8fb71810893df5dc2ffcacaa67a33af2d5cd61888f2 \ - --hash=sha256:f216510ace3de5de01329463a315a629f33480e893a9024fc93d8c32c22913da +django==5.1 \ + --hash=sha256:848a5980e8efb76eea70872fb0e4bc5e371619c70fffbe48e3e1b50b2c09455d \ + --hash=sha256:d3b811bf5371a26def053d7ee42a9df1267ef7622323fe70a601936725aa4557 # via # -r ./requirements/requirements.in # dj-database-url diff --git a/backend/requirements/dev-requirements.in b/backend/requirements/dev-requirements.in index b263a9a612..9936b05181 100644 --- a/backend/requirements/dev-requirements.in +++ b/backend/requirements/dev-requirements.in @@ -12,7 +12,7 @@ jsonnet model_bakery mypy pandas -pip-tools +pip-tools>=7.4.1 tblib toml types-python-slugify diff --git a/backend/requirements/requirements.in b/backend/requirements/requirements.in index 62d89a5411..26d5533450 100644 --- a/backend/requirements/requirements.in +++ b/backend/requirements/requirements.in @@ -5,7 +5,7 @@ django-cors-headers django-csp django-dbbackup django-storages[boto3] -Django>=5.0.7 +Django>=5.0.8 djangorestframework>=3.15.2 djangorestframework-simplejwt django-fsm